Add: Incremental Books

Remove: code to remove secuties
Add: code to connect to Live databento
2024-10-04 17:05:02 +00:00 · 2024-10-03 18:03:10 +05:00 · 2024-10-03 18:03:08 +05:00 · 2024-10-03 18:01:49 +05:00 · 2024-10-03 18:01:49 +05:00 · 2024-10-03 18:01:49 +05:00
2488 changed files with 43721 additions and 76700 deletions
@@ -9,7 +9,7 @@
      ]
    },
    "fable": {
-      "version": "4.24.0",
+      "version": "4.0.6",
      "commands": [
        "fable"
      ]
@@ -27,4 +27,4 @@
      ]
    }
  }
-}
+}
@@ -65,7 +65,7 @@ RUN \
   && apt-get install -y cmake

 # Dotnet
-RUN apt-get update && apt-get install -y dotnet8
+RUN apt-get update && apt-get install -y dotnet7

 # Node
 ARG node_major=18
@@ -90,12 +90,6 @@ RUN apt-get update && apt-get install -y libpcap0.8-dev
 # Databento (deps)
 RUN apt-get update && apt-get -y install libzstd-dev

-# Docker in Docker
-RUN apt-get update && apt-get install -y \
-    docker.io \
-    docker-compose \
-    docker-compose-v2
-
 # Set up default executables (update-alternatives)
 RUN \
   update-alternatives --install /usr/bin/clang clang /usr/bin/clang-$clang_major 100; \
@@ -1,10 +1,15 @@
 {
  "dockerFile": "Dockerfile",
+  // Creates local dir to mount to HOME in docker 
+  "initializeCommand": "mkdir -p ${localEnv:HOME}/.bts/devcontainer-mounts/${devcontainerId}",
  "hostRequirements": {
-    "memory": "8gb",
-    "storage": "10gb"
+    "memory": "4gb",
+    "storage": "6gb"
  },
+  "postCreateCommand": "/home/bts/spark/docker/sparkd/dev/devcontainer_post_create.sh",
  "containerUser": "bts",
+  "workspaceMount": "source=${localWorkspaceFolder},target=/home/bts/spark,type=bind",
+  "workspaceFolder": "/home/bts/spark",
  "updateRemoteUserUID": true,
  "shutdownAction": "none",
  // Note: If running multiple containers on the same host, each must use a
@@ -12,20 +17,34 @@
  // example, if 8000-8003 is taken, go with 9000-9003, then 10000-10003, etc.
  "runArgs": [
    "--shm-size",
-    "8gb"
+    "2gb",
+    "-p",
+    "8000:8000",
+    "-p",
+    "8001:8001",
+    "-p",
+    "8002:8002",
+    "-p",
+    "8003:8003",
+    "-p",
+    "12345:12345"
  ],
  "mounts": [
-    "source=/var/run/docker.sock,target=/var/run/docker.sock,type=bind"
+    "source=${localEnv:HOME}/.bts/devcontainer-mounts/${devcontainerId},target=/home/bts,type=bind"
  ],
  "customizations": {
    "vscode": {
+      "settings": {
+        "clangd.arguments": [
+          "--compile-commands-dir=${workspaceFolder}/build/clang/debug"
+        ]
+      },
      "extensions": [
        "eamodio.gitlens",
        "llvm-vs-code-extensions.vscode-clangd",
        "ms-vscode.cmake-tools",
-        "ms-vscode.cpptools",
-        "xaver.clang-format"
+        "ms-vscode.cpptools"
      ]
    }
  }
-}
+}
@@ -1,48 +0,0 @@
-# This file defines the global default configuration for docker deployments.
-# It is tracked in source control. For temporary modifications, please create
-# a separate file called ".env.local" (it's in .gitignore, so won't be tracked).
-# If a variable is defined in both files, the value in .env.local will be used.
-#
-# About the deployment procedure:
-#   We use Docker Compose to build images and launch containers for various services
-#   that make up our software suite:
-#     - sparkd
-#     - TODO: pubsub
-#     - TODO: md-publisher
-#     - TODO: webapp (DSQA on the web)
-#     - TODO: api-server (websocket based)
-#   From inside a devcontainer, you can launch all services in separate containers
-#   on your machine by running:
-#
-#     tools/deploy.sh
-#
-#   This script builds release versions for each service, packaged (generally) in
-#   self-contained directories. Next, it runs a few "docker compose" commands to
-#   build docker images loaded with the self-contained binaries from the previous
-#   step, and start containers with those images to run each service. To view the
-#   commands that deploy.sh runs internally (without actually running them), run:
-#
-#     tools/deploy.sh --dry-run
-#
-#   For additional information/help, run:
-#
-#     tools/deploy.sh --help
-#
-
-### Sparkd ###
-# The internal image for sparkd to use. For more info, view:
-#   src/server/sparkd/Dockerfile
-SPARKD__IMAGE=sparkd
-# The config dir for this service to use
-SPARKD__CONFIG_DIR=src/server/sparkd/configurations/dev
-# Mount a specific volume or dir to "~/sparkdata/data" inside the docker.
-# Defaults to a volume named "sparkd_<config_port>" (e.g., sparkd_8000).
-# SPARKD__DATA_DIR=/home/bts/sparkdata/data
-SPARKD__SOURCE_DIR=install/sparkd-release-gcc
-SPARKD__RUNTIME_CONF_DIR=/home/ubuntu/8000/sparkdata/conf/
-SPARKD__CONFIG_PORT=8000
-SPARKD__QUICKBUS_PORT=8001
-SPARKD__SIM_FIX_PORT=8002
-SPARKD__SHM_SIZE=4gb
-
-# TODO: add config for other services (pubsub, md-publisher, webapp, api-server, etc.)
@@ -59,5 +59,3 @@ trace_data/
 .devcontainer/
 *.Generated.cs
 *.lst
-.env.local
-.secrets
@@ -1,6 +1,6 @@
 [submodule "externals"]
 	path = externals
-	url = https://bitbucket.org/bluetradesys/externals.git
+	url = git@bitbucket.org:bluetradesys/externals.git
 [submodule "vcpkg"]
 	path = vcpkg
 	url = https://github.com/microsoft/vcpkg.git
@@ -1,14 +1,6 @@
 {
  "version": "0.2.0",
  "configurations": [
-    {
-      "name": "Python Debugger: Current File with Arguments",
-      "type": "debugpy",
-      "request": "launch",
-      "program": "${file}",
-      "console": "integratedTerminal",
-      "args": "${command:pickArgs}"
-    },
    {
      "name": ".NET Core Attach",
      "type": "coreclr",
@@ -30,7 +22,7 @@
        "--config",
        "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json"
      ],
-      "cwd": "build/Debug/bin/net8.0",
+      "cwd": "build/Debug/bin/net7.0",
      "stopAtEntry": false,
      "console": "integratedTerminal"
    },
@@ -39,39 +31,7 @@
      "type": "cppdbg",
      "request": "launch",
      "program": "/home/bts/spark/build/sparkd-debug-clang/server/sparkd",
-      "args": [
-        "run"
-      ],
-      "stopAtEntry": false,
-      "cwd": "/home/bts/spark/build/sparkd-debug-clang/server",
-      "environment": [],
-      "externalConsole": false,
-      "MIMode": "gdb",
-    },
-    {
-      "name": "spreader tests2",
-      "type": "cppdbg",
-      "request": "launch",
-      "program": "/home/bts/spark/build/sparkd-debug-clang/server/sparkd",
-      "args": [
-        "test",
-        "[spreader2]"
-      ],
-      "stopAtEntry": false,
-      "cwd": "/home/bts/spark/build/sparkd-debug-clang/server",
-      "environment": [],
-      "externalConsole": false,
-      "MIMode": "gdb",
-    },
-    {
-      "name": "spreader tests",
-      "type": "cppdbg",
-      "request": "launch",
-      "program": "/home/bts/spark/build/sparkd-debug-clang/server/sparkd",
-      "args": [
-        "test",
-        "[spreader]"
-      ],
+      "args": ["run"],
      "stopAtEntry": false,
      "cwd": "/home/bts/spark/build/sparkd-debug-clang/server",
      "environment": [],
@@ -90,7 +50,7 @@
          "--config",
          "${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json"
        ],
-        "cwd": "build/Debug/bin/net8.0-windows",
+        "cwd": "build/Debug/bin/net7.0-windows",
        "env": {
          "PYTHON_EXE": "${workspaceFolder}/.venv/Scripts/python.exe"
        },
@@ -114,7 +74,7 @@
        "--config",
        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json"
      ],
-      "cwd": "build/Debug/bin/net8.0",
+      "cwd": "build/Debug/bin/net7.0",
      "stopAtEntry": false,
      "console": "integratedTerminal"
    },
@@ -126,19 +86,7 @@
      "windows": {
        "preLaunchTask": "Build Spark (Debug)",
        "program": "Spark.exe",
-        "cwd": "build/Debug/bin/net8.0-windows",
-      },
-      "stopAtEntry": false,
-      "console": "integratedTerminal"
-    },
-    {
-      "name": "Spark (Skip Build)",
-      "type": "coreclr",
-      "request": "launch",
-      "program": "",
-      "windows": {
-        "program": "Spark.exe",
-        "cwd": "build/Debug/bin/net8.0-windows",
+        "cwd": "build/Debug/bin/net7.0-windows",
      },
      "stopAtEntry": false,
      "console": "integratedTerminal"
@@ -155,100 +103,27 @@
          "--sparkd",
          "${input:sparkConfigServer}"
        ],
-        "cwd": "build/Debug/bin/net8.0-windows",
-      },
-      "stopAtEntry": false,
-      "console": "integratedTerminal"
-    },
-    {
-      "name": "Spark Test",
-      "type": "coreclr",
-      "request": "launch",
-      "program": "",
-      "windows": {
-        "preLaunchTask": "Build Spark (Debug)",
-        "program": "Spark.Console.exe",
-        "cwd": "build/Debug/bin/net8.0",
-        "args": [
-          "test",
-          "${input:sparkTest}"
-        ]
-      },
-      "stopAtEntry": false,
-      "console": "integratedTerminal"
-    },
-    {
-      "name": "Spark Test (Skip Build)",
-      "type": "coreclr",
-      "request": "launch",
-      "program": "",
-      "windows": {
-        "program": "Spark.Console.exe",
-        "cwd": "build/Debug/bin/net8.0",
-        "args": [
-          "test",
-          "${input:sparkTest}"
-        ]
+        "cwd": "build/Debug/bin/net7.0-windows",
      },
      "stopAtEntry": false,
      "console": "integratedTerminal"
    },
    {
      "name": "Sparkd",
-      "type": "cppdbg",
+      "type": "coreclr",
      "request": "launch",
+      "preLaunchTask": "Build Sparkd (Debug | Clang)",
      "program": "",
      "linux": {
-        "program": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd/sparkd",
+        "program": "sparkd",
      },
      "args": [
        "run"
      ],
-      "cwd": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd",
+      "cwd": "build/sparkd-debug-clang/server",
      "stopAtEntry": false,
-      "externalConsole": false,
-      "MIMode": "gdb",
-      "setupCommands": [
-        {
-          "description": "Enable pretty-printing for gdb",
-          "text": "-enable-pretty-printing",
-          "ignoreFailures": true
-        },
-        {
-          "description": "Set Disassembly Flavor to Intel",
-          "text": "-gdb-set disassembly-flavor intel",
-          "ignoreFailures": true
-        }
-      ]
-    },
-    {
-      "name": "Sparkd (Skip Build)",
-      "type": "cppdbg",
-      "request": "launch",
-      "program": "",
-      "linux": {
-        "program": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd/sparkd",
-      },
-      "args": [
-        "run"
-      ],
-      "cwd": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd",
-      "stopAtEntry": false,
-      "externalConsole": false,
-      "MIMode": "gdb",
-      "setupCommands": [
-        {
-          "description": "Enable pretty-printing for gdb",
-          "text": "-enable-pretty-printing",
-          "ignoreFailures": true
-        },
-        {
-          "description": "Set Disassembly Flavor to Intel",
-          "text": "-gdb-set disassembly-flavor intel",
-          "ignoreFailures": true
-        }
-      ]
-    },
+      "console": "integratedTerminal"
+    }
  ],
  "inputs": [
    {
@@ -279,15 +154,5 @@
      "description": "Config Server URL",
      "default": "http://localhost:8000/config",
    },
-    {
-      "id": "sparkTest",
-      "type": "promptString",
-      "description": "Test to run",
-    },
-    {
-      "id": "sparkdTest",
-      "type": "promptString",
-      "description": "Test to run",
-    },
  ]
-}
+}
@@ -641,21 +641,6 @@
        "color": "terminal.ansiGreen",
      }
    },
-    {
-      "label": "Run Spark (Skip Build)",
-      "type": "process",
-      "command": "",
-      "windows": {
-        "command": "Spark.exe",
-        "options": {
-          "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0-windows"
-        },
-      },
-      "icon": {
-        "id": "run",
-        "color": "terminal.ansiGreen",
-      }
-    },
    {
      "label": "Run Spark Config Query",
      "dependsOn": "Build Spark",
@@ -699,24 +684,6 @@
        "color": "terminal.ansiGreen",
      },
    },
-    {
-      "label": "Run Sparkd (Skip Build)",
-      "type": "process",
-      "command": "",
-      "linux": {
-        "command": "sparkd",
-      },
-      "args": [
-        "run"
-      ],
-      "options": {
-        "cwd": "${workspaceFolder}/build/${input:sparkdCmakePreset}/server"
-      },
-      "icon": {
-        "id": "run",
-        "color": "terminal.ansiGreen",
-      },
-    },
    {
      "label": "Run Sparkd Config Query",
      "dependsOn": "Build Sparkd",
@@ -738,65 +705,6 @@
        "color": "terminal.ansiGreen",
      },
    },
-    {
-      "label": "Test Spark (Debug)",
-      "dependsOn": "Build Spark (Debug)",
-      "type": "process",
-      "icon": {
-        "id": "coverage",
-        "color": "terminal.ansiYellow",
-      },
-      "command": "",
-      "windows": {
-        "command": "Spark.Console.exe",
-        "args": [
-          "test",
-          "${input:sparkTest}"
-        ],
-        "options": {
-          "cwd": "${workspaceFolder}/build/Debug/bin/net7.0",
-        }
-      },
-    },
-    {
-      "label": "Test Sparkd",
-      "dependsOn": "Build Sparkd",
-      "type": "process",
-      "command": "",
-      "linux": {
-        "command": "sparkd",
-      },
-      "args": [
-        "test",
-        "${input:sparkdTest}",
-      ],
-      "options": {
-        "cwd": "${workspaceFolder}/build/${input:sparkdCmakePreset}/server"
-      },
-      "icon": {
-        "id": "coverage",
-        "color": "terminal.ansiYellow",
-      },
-    },
-    {
-      "label": "Test Sparkd (Skip Build)",
-      "type": "process",
-      "command": "",
-      "linux": {
-        "command": "sparkd",
-      },
-      "args": [
-        "test",
-        "${input:sparkdTest}",
-      ],
-      "options": {
-        "cwd": "${workspaceFolder}/build/${input:sparkdCmakePreset}/server"
-      },
-      "icon": {
-        "id": "coverage",
-        "color": "terminal.ansiYellow",
-      },
-    },
    {
      "label": "View DataStreamQueryAnalyzer Logs (Last Run)",
      "type": "process",
@@ -915,16 +823,6 @@
      "id": "sparkdConfigQuery",
      "type": "promptString",
      "description": "Config Type(s)"
-    },
-    {
-      "id": "sparkTest",
-      "type": "promptString",
-      "description": "Test to run",
-    },
-    {
-      "id": "sparkdTest",
-      "type": "promptString",
-      "description": "Test to run",
-    },
+    }
  ]
 }
@@ -1,10 +1,7 @@
 cmake_minimum_required(VERSION 3.12)
 project(spark)
-include(GNUInstallDirs)
-include(${CMAKE_SOURCE_DIR}/cmake/GitVersion.cmake)
 set(CMAKE_CXX_STANDARD 20)

-get_version_from_git()
 if(WIN32)
  set(platform "windows")
 else()
@@ -22,9 +19,9 @@ set(
 # can run simultaneously without fighting over the same shm files (e.g., sparkd,
 # api server, etc.).
 set(SHARED_MEMORY_PREFIX "bts" CACHE STRING "Prefix to use when creating shared memory files")
-set(EXTERNALS_BIN "${EXTERNALS_ROOT}/bin")
-set(EXTERNALS_INCLUDE "${EXTERNALS_ROOT}/include")
-set(EXTERNALS_LIB "${EXTERNALS_ROOT}/lib/$<CONFIG>")
+set(SPARK_BIN "${EXTERNALS_ROOT}/bin")
+set(SPARK_INCLUDE "${EXTERNALS_ROOT}/include")
+set(SPARK_LIB "${EXTERNALS_ROOT}/lib/$<CONFIG>")
 set(BUILD_DOCS OFF CACHE BOOL "Build documentation")
 set(BUILD_SDK OFF CACHE BOOL "Build the sdk")
 set(EXPORT_MD_CONSUMER OFF CACHE BOOL "Export (extern) C functions for MD consumer interop")
@@ -36,8 +33,8 @@ else()
  set(BUILD_ADMIN_WEB_APP ON CACHE BOOL "Build the Admin WebApp")
 endif()

-include_directories(BEFORE SYSTEM ${EXTERNALS_INCLUDE} ${EXTERNALS_ROOT}/include/zlib)
-link_directories(${EXTERNALS_LIB})
+include_directories(BEFORE SYSTEM ${SPARK_INCLUDE} ${EXTERNALS_ROOT}/include/zlib)
+link_directories(${SPARK_LIB})

 add_compile_definitions(SHARED_MEMORY_PREFIX="${SHARED_MEMORY_PREFIX}")
 add_compile_definitions(APP_DATA_DIR="${APP_DATA_DIR}")
@@ -47,8 +44,14 @@ endif()
 add_compile_definitions(EXPORT_MD_CONSUMER="${EXPORT_MD_CONSUMER}")
 if(WIN32)
  # subdirs
-  add_subdirectory(src/lib_spark)
+  add_subdirectory(src/lib)
 else()
+  # make sparkd look for libspark.so in its directory
+  set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE)
+  set(CMAKE_INSTALL_RPATH ".")
+  set(CMAKE_INSTALL_PREFIX "${CMAKE_SOURCE_DIR}/install")
+  set(SPARK_INSTALL_PREFIX "${CMAKE_INSTALL_PREFIX}/$<CONFIG>")
+
  if(BUILD_SPARKD)
    # Note: All macros that begin w/ 'CCAPI' are ccapi's macros (defined and/or
    # read by their code). We override some below.
@@ -86,15 +89,9 @@ else()
    add_compile_definitions(BINANCE_US_PRICE_STATS_PATH="/api/v3/ticker/24hr")

    # ** Binance USDS Futures **
-    add_compile_definitions(CCAPI_BINANCE_USDS_FUTURES_URL_REST_BASE="https://fapi.binance.com")
-    add_compile_definitions(BINANCE_USDS_FUTURES_INSTRUMENTS_PATH="/fapi/v1/exchangeInfo")
-    add_compile_definitions(BINANCE_USDS_FUTURES_PRICE_STATS_PATH="/fapi/v1/ticker/24hr")
-    add_compile_definitions(BINANCE_USDS_FUTURES_USER_DATA_PATH="/api/v3/userDataStream")
-
-    # ** Binance Coin Futures **
-    add_compile_definitions(CCAPI_BINANCE_COIN_FUTURES_URL_REST_BASE="https://dapi.binance.com")
-    add_compile_definitions(BINANCE_COIN_FUTURES_INSTRUMENTS_PATH="/dapi/v1/exchangeInfo")
-    add_compile_definitions(BINANCE_COIN_FUTURES_PRICE_STATS_PATH="/dapi/v1/ticker/24hr")
+    add_compile_definitions(CCAPI_BINANCE_USDS_FUTURES_URL_REST_BASE="https://dapi.binance.com")
+    add_compile_definitions(BINANCE_USDS_FUTURES_INSTRUMENTS_PATH="/dapi/v1/exchangeInfo")
+    add_compile_definitions(BINANCE_USDS_FUTURES_PRICE_STATS_PATH="/dapi/v1/ticker/24hr")

    # ** Deribit **
    add_compile_definitions(DERIBIT_INSTRUMENT_PATH="/api/v2/public/get_instrument")
@@ -114,14 +111,10 @@ else()

  # subdirs
  if (BUILD_SPARKD)
-    add_subdirectory(src/lib_onixs_fix)
-    add_subdirectory(src/lib_binance_execution)
-    add_subdirectory(src/lib_okx_execution)
-    add_subdirectory(src/lib_python_md)
-    add_subdirectory(src/app_sparkd)
+    add_subdirectory(src/server/sparkd)
    add_subdirectory(src/data_files)
  endif()
-  add_subdirectory(src/lib_spark)
+  add_subdirectory(src/lib)
  if (BUILD_ADMIN_WEB_APP)
    add_subdirectory(src/Spark.Admin.WebApp)
  endif()
@@ -131,4 +124,34 @@ else()
  if(BUILD_SDK)
    add_subdirectory(src/sdk)
  endif()
+
+  if(BUILD_SPARKD)
+    add_custom_command(
+      OUTPUT ${CMAKE_BINARY_DIR}/server
+      COMMAND mkdir -p ${CMAKE_BINARY_DIR}/server
+      COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/src/server/sparkd/sparkd ${CMAKE_BINARY_DIR}/server
+      COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/data_files ${CMAKE_BINARY_DIR}/server
+      COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/src/lib/libspark.so ${CMAKE_BINARY_DIR}/server
+      COMMAND ln ARGS -sf ${EXTERNALS_ROOT}/lib/${CMAKE_BUILD_TYPE}/libonixs-fix-engine.so.4 ${CMAKE_BINARY_DIR}/server
+      
+      COMMAND
+        ln ARGS -sf
+        ${EXTERNALS_ROOT}/lib/Release/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.16.0.0
+        ${CMAKE_BINARY_DIR}/server
+      COMMAND ln ARGS -sf ${EXTERNALS_ROOT}/lib/Release/libACE-5.7.5.so ${CMAKE_BINARY_DIR}/server
+      COMMAND mkdir ARGS -p ${CMAKE_BINARY_DIR}/server/plugins
+      COMMENT "Creating server runtime layout"
+      DEPENDS sparkd data_files_dir)
+
+    add_custom_target(server_layout ALL DEPENDS ${CMAKE_BINARY_DIR}/server)
+
+    install(
+      DIRECTORY "${CMAKE_BINARY_DIR}/server/plugins"
+      DESTINATION "${SPARK_INSTALL_PREFIX}/bin")
+    install(
+      FILES
+      "${SPARK_LIB}/libACE-5.7.5.so"
+      "${SPARK_LIB}/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.16.0.0"
+      DESTINATION "${SPARK_INSTALL_PREFIX}/bin")
+  endif()
 endif()
@@ -26,13 +26,13 @@
        "CMAKE_CXX_FLAGS_DEBUG": "$env{SPARK_CMAKE_CXX_FLAGS} /MTd",
        "CMAKE_CXX_FLAGS_RELEASE": "$env{SPARK_CMAKE_CXX_FLAGS} /MT",
        "CMAKE_CXX_STANDARD": "20",
-        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
        "CMAKE_SHARED_LINKER_FLAGS_DEBUG": "/DEBUG",
        "CMAKE_SHARED_LINKER_FLAGS_RELEASE": "/OPT:REF /OPT:ICF /LTCG",
        "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
        "EXTERNALS_ROOT": "${sourceDir}/externals/windows",
-        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0;${sourceDir}/build/$<CONFIG>/bin/net8.0-windows",
+        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
+        "DOTNET_WINDOWS_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0-windows",
        "EXPORT_MD_CONSUMER": {
          "type": "BOOL",
          "value": "ON"
@@ -57,18 +57,12 @@
        "APP_DATA_DIR_OVERRIDE": "${dollar}{SPARKD_DATA_DIR}",
        "CMAKE_BUILD_WITH_INSTALL_RPATH": {
          "type": "BOOL",
-          "value": "OFF"
+          "value": "ON"
        },
        "CMAKE_CXX_STANDARD": "20",
        "CMAKE_INSTALL_RPATH": ".",
-        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
-        "CMAKE_PREFIX_PATH": "${sourceDir}/cmake/;${sourceDir}/.venv/lib/python3.10/site-packages/pybind11/share/cmake/pybind11",
+        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}/sparkd",
        "CMAKE_EXE_LINKER_FLAGS": "-Wno-undef",
-        "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
-        "DATABENTO_USE_EXTERNAL_JSON": {
-          "type": "BOOL",
-          "value": "ON"
-        },
        "EXTERNALS_ROOT": "${sourceDir}/externals/linux",
        "BUILD_SDK": {
          "type": "BOOL",
@@ -93,7 +87,7 @@
          "value": "ON"
        },
        "CCAPI_LOG_LEVEL": "Info",
-        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/md_dev/*.conf;spark/config/profiles/default/*.conf"
+        "CONFIG_DEFAULTS_PATH": "config/profiles/md_dev/*.conf;config/profiles/default/*.conf"
      }
    },

@@ -104,7 +98,7 @@
      "cacheVariables": {
        "CMAKE_BUILD_TYPE": "Release",
        "CCAPI_LOG_LEVEL": "Warn",
-        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/md_prod/*.conf;spark/config/profiles/default/*.conf"
+        "CONFIG_DEFAULTS_PATH": "config/profiles/md_prod/*.conf;config/profiles/default/*.conf"
      }
    },

@@ -186,12 +180,8 @@
        "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
-        "CMAKE_EXPORT_COMPILE_COMMANDS": {
-          "type": "BOOL",
-          "value": "ON"
-        },
-        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0",
+        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
        "EXPORT_MD_CONSUMER": {
          "type": "BOOL",
          "value": "ON"
@@ -231,7 +221,7 @@
      "cacheVariables": {
        "APP_DATA_DIR": "${dollar}{APPDATA}/bts/DataStream",
        "APP_DATA_DIR_OVERRIDE": "${dollar}{DATA_STREAM_DATA_DIR}",
-        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf;spark/config/profiles/data_stream/*.conf",
+        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf",
        "SHARED_MEMORY_PREFIX": "data_stream"
      }
    },
@@ -266,7 +256,8 @@
        "CMAKE_CXX_COMPILER": "clang++",
        "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
-        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf;spark/config/profiles/data_stream/*.conf",
+        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
+        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf",
        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
        "EXPORT_MD_CONSUMER": {
          "type": "BOOL",
@@ -311,7 +302,16 @@
        "lhs": "${hostSystemName}",
        "rhs": "Linux"
      },
-      "hidden": true
+      "hidden": true,
+      "targets": [
+        "data_files_dir",
+        "admin_web_app_bundle",
+        "install",
+        "libspark",
+        "rapidjson_project",
+        "server_layout",
+        "sparkd"
+      ]
    },

    {
@@ -14,34 +14,6 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "spark_test", "src\client\sp
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Spark.Console", "src\client\Spark.Console\Spark.Console.csproj", "{89D666E6-B500-498F-B026-4D167F68ADFC}"
 EndProject
-Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{267F81EF-A8AB-49FD-9F56-D7EC3C1F7C7C}"
-EndProject
-Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "DataStreamSuite", "DataStreamSuite", "{C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamCore", "src\DataStreamSuite\DataStreamCore\DataStreamCore.csproj", "{EF26A475-6101-4C9C-A717-D60BB92A2302}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SharedStreams", "src\DataStreamSuite\SharedStreams\SharedStreams.csproj", "{911B08A3-515A-441D-84C8-8A43740B5FC1}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PubSub", "src\DataStreamSuite\PubSub\PubSub.csproj", "{14140310-2233-430A-821B-F3058A25234B}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BaseUtilities", "src\DataStreamSuite\BaseUtilities\BaseUtilities.csproj", "{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ExpressionCore", "src\DataStreamSuite\ExpressionCore\ExpressionCore.csproj", "{6F22DB96-6EAB-4034-99D9-287021C36F01}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ArithmeticExpressions", "src\DataStreamSuite\ArithmeticExpressions\ArithmeticExpressions.csproj", "{A5C33956-A41F-49B3-A41D-1082CBA59D02}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamExpressions", "src\DataStreamSuite\DataStreamExpressions\DataStreamExpressions.csproj", "{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnSortExpressions", "src\DataStreamSuite\Bts.ColumnSortExpressions\Bts.ColumnSortExpressions.csproj", "{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ColumnMapExpressions", "src\DataStreamSuite\ColumnMapExpressions\ColumnMapExpressions.csproj", "{B945A183-2174-4ACB-B289-8A5C546656FE}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BooleanExpressions", "src\DataStreamSuite\BooleanExpressions\BooleanExpressions.csproj", "{9F6D7DE8-185B-4C1D-B809-3314D092DF28}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnExpressions", "src\DataStreamSuite\Bts.ColumnExpressions\Bts.ColumnExpressions.csproj", "{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStream", "src\DataStreamSuite\DataStream\DataStream.csproj", "{39AEA075-6457-46C0-8821-1158BE6EFA57}"
-EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|x64 = Debug|x64
@@ -98,102 +70,6 @@ Global
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.Release|x64.Build.0 = Release|x64
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|x64.ActiveCfg = Debug|x64
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|x64.Build.0 = Debug|x64
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Debug|x64.Build.0 = Debug|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Release|x64.ActiveCfg = Release|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Release|x64.Build.0 = Release|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Debug|x64.Build.0 = Debug|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Release|x64.ActiveCfg = Release|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Release|x64.Build.0 = Release|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.Debug|x64.Build.0 = Debug|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.Release|x64.ActiveCfg = Release|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.Release|x64.Build.0 = Release|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Debug|x64.Build.0 = Debug|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Release|x64.ActiveCfg = Release|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Release|x64.Build.0 = Release|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Debug|x64.Build.0 = Debug|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Release|x64.ActiveCfg = Release|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Release|x64.Build.0 = Release|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Debug|x64.Build.0 = Debug|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Release|x64.ActiveCfg = Release|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Release|x64.Build.0 = Release|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Debug|x64.Build.0 = Debug|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Release|x64.ActiveCfg = Release|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Release|x64.Build.0 = Release|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Debug|x64.Build.0 = Debug|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Release|x64.ActiveCfg = Release|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Release|x64.Build.0 = Release|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.Debug|x64.Build.0 = Debug|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.Release|x64.ActiveCfg = Release|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.Release|x64.Build.0 = Release|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Debug|x64.Build.0 = Debug|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Release|x64.ActiveCfg = Release|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Release|x64.Build.0 = Release|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Debug|x64.Build.0 = Debug|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Release|x64.ActiveCfg = Release|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Release|x64.Build.0 = Release|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Debug|x64.Build.0 = Debug|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Release|x64.ActiveCfg = Release|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Release|x64.Build.0 = Release|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -204,19 +80,4 @@ Global
 	GlobalSection(Performance) = preSolution
 		HasPerformanceSessions = true
 	EndGlobalSection
-	GlobalSection(NestedProjects) = preSolution
-		{C09ED521-D2DF-40BC-A76F-EB3CB870B2F6} = {267F81EF-A8AB-49FD-9F56-D7EC3C1F7C7C}
-		{EF26A475-6101-4C9C-A717-D60BB92A2302} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-		{911B08A3-515A-441D-84C8-8A43740B5FC1} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-		{14140310-2233-430A-821B-F3058A25234B} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-		{6F22DB96-6EAB-4034-99D9-287021C36F01} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-		{B945A183-2174-4ACB-B289-8A5C546656FE} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-		{39AEA075-6457-46C0-8821-1158BE6EFA57} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-	EndGlobalSection
 EndGlobal
@@ -1,7 +0,0 @@
-include_guard(GLOBAL)
-
-macro(export_to_parent)
-  foreach(var IN ITEMS ${ARGN})
-    set(${var} ${${var}} PARENT_SCOPE)
-  endforeach()
-endmacro()
@@ -1,48 +0,0 @@
-include_guard(GLOBAL)
-
-function(get_version_from_git)
-  find_package(Git QUIET)
-  if(NOT Git_FOUND)
-    message(WARNING "Git not found")
-    return()
-  endif()
-
-  execute_process(
-    COMMAND ${GIT_EXECUTABLE} describe --tags --always
-    WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
-    OUTPUT_VARIABLE GIT_TAG
-    OUTPUT_STRIP_TRAILING_WHITESPACE
-    RESULT_VARIABLE GIT_RESULT
-  )
-
-  if(NOT GIT_RESULT EQUAL 0)
-    message(WARNING "Failed to get git tag")
-    return()
-  endif()
-
-  execute_process(
-    COMMAND ${GIT_EXECUTABLE} rev-parse --short=7 HEAD
-    WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
-    OUTPUT_VARIABLE GIT_COMMIT_SHORT_HASH
-    OUTPUT_STRIP_TRAILING_WHITESPACE
-  )
-  set(PROJECT_VERSION_BUILD_METADATA ${GIT_COMMIT_SHORT_HASH})
-  set(PROJECT_VERSION_BUILD_METADATA ${GIT_COMMIT_SHORT_HASH} PARENT_SCOPE)
-
-  string(REGEX REPLACE "^v" "" CLEAN_TAG "${GIT_TAG}")
-  if(CLEAN_TAG MATCHES "^([0-9]+)\\.([0-9]+)\\.([0-9]+)(-.*)?$")
-    set(PROJECT_VERSION_MAJOR ${CMAKE_MATCH_1})
-    set(PROJECT_VERSION_MAJOR ${CMAKE_MATCH_1} PARENT_SCOPE)
-    set(PROJECT_VERSION_MINOR ${CMAKE_MATCH_2})
-    set(PROJECT_VERSION_MINOR ${CMAKE_MATCH_2} PARENT_SCOPE)
-    set(PROJECT_VERSION_PATCH ${CMAKE_MATCH_3})
-    set(PROJECT_VERSION_PATCH ${CMAKE_MATCH_3} PARENT_SCOPE)
-
-    set(PROJECT_VERSION "${CMAKE_MATCH_1}.${CMAKE_MATCH_2}.${CMAKE_MATCH_3}") 
-    set(PROJECT_VERSION "${CMAKE_MATCH_1}.${CMAKE_MATCH_2}.${CMAKE_MATCH_3}" PARENT_SCOPE)
-    set(PROJECT_VERSION_FULL "${CMAKE_MATCH_1}.${CMAKE_MATCH_2}.${CMAKE_MATCH_3}+${GIT_COMMIT_SHORT_HASH}")
-    set(PROJECT_VERSION_FULL "${CMAKE_MATCH_1}.${CMAKE_MATCH_2}.${CMAKE_MATCH_3}+${GIT_COMMIT_SHORT_HASH}" PARENT_SCOPE)
-  else()
-    message(WARNING "Tag '${CLEAN_TAG}' does not match semver format")
-  endif()
-endfunction()
@@ -1,142 +0,0 @@
-include_guard(DIRECTORY)
-include(${CMAKE_SOURCE_DIR}/cmake/ExportToParent.cmake)
-include(${CMAKE_SOURCE_DIR}/cmake/InstallBasic.cmake)
-include(${CMAKE_SOURCE_DIR}/cmake/ProtobufGenerateCpp.cmake)
-include(${CMAKE_SOURCE_DIR}/cmake/SetFirstDefined.cmake)
-include(${CMAKE_SOURCE_DIR}/cmake/safe_dep_glob.cmake)
-
-function(init_library)
-  set(_options SHARED STATIC)
-  set(_one_value_args
-    NAME
-    SOURCE_DIR
-    INCLUDE_DIR
-  )
-  set(_multi_value_args)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-  if (_ARGS_SHARED AND _ARGS_STATIC)
-    message(FATAL_ERROR "Invalid args passed to init_library: cannot specify STATIC and SHARED")
-  endif()
-  set_first_defined(_name _ARGS_NAME PROJECT_NAME)
-  set_first_defined(_source_dir _ARGS_SOURCE_DIR _name)
-  set_first_defined(_include_dir _ARGS_INCLUDE_DIR _source_dir)
-
-  # Cpp sources
-  file(
-    GLOB_RECURSE ${_name}_SOURCES
-    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
-    ${_source_dir}/*.cpp
-  )
-  # Note: cpp files inside 'tests' dir(s) are considered tests.
-  # Remove tests from main sources, and add them to test sources
-  foreach(item ${${_name}_SOURCES})
-    if(${item} MATCHES "tests/.*\.cpp$")
-      list(APPEND ${_name}_TEST_SOURCES ${item})
-    endif()
-  endforeach()
-  foreach(item ${${_name}_TEST_SOURCES})
-    list(REMOVE_ITEM ${_name}_SOURCES ${item})
-  endforeach()
-
-  # Proto sources 
-  file(
-    GLOB_RECURSE ${_name}_PROTO_SOURCES
-    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
-    ${_source_dir}/protobufs/*.proto
-  )
-  update_deps_file("${${_name}_PROTO_SOURCES};")
-  protobuf_generate_cpp(
-    CXX_SOURCES ${_name}_CXX_PROTO_SOURCES
-    CXX_HEADERS ${_name}_CXX_PROTO_HEADERS
-    CXX_OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}
-    PROTO_INCLUDE_DIRS
-      ${EXTERNALS_INCLUDE}
-      ${CMAKE_CURRENT_SOURCE_DIR}
-    PROTO_SOURCES
-      ${${_name}_PROTO_SOURCES}
-  )
-
-  export_to_parent(
-    ${_name}_SOURCES
-    ${_name}_TEST_SOURCES
-    ${_name}_PROTO_SOURCES
-    ${_name}_CXX_PROTO_SOURCES
-    ${_name}_CXX_PROTO_HEADERS
-  )
-
-  # Add main lib
-  if (_ARGS_STATIC)
-    add_library(${_name} STATIC ${${_name}_SOURCES} ${${_name}_CXX_PROTO_SOURCES})
-  elseif(_ARGS_SHARED)
-    add_library(${_name} SHARED ${${_name}_SOURCES} ${${_name}_CXX_PROTO_SOURCES})
-  else()
-    add_library(${_name} ${${_name}_SOURCES} ${${_name}_CXX_PROTO_SOURCES})
-  endif()
-  target_include_directories(
-    ${_name}
-    SYSTEM
-    BEFORE
-    PUBLIC
-      $<BUILD_INTERFACE:${EXTERNALS_ROOT}/include>
-      $<BUILD_INTERFACE:${EXTERNALS_ROOT}/include/zlib>
-  )
-  target_include_directories(
-    ${_name}
-    PUBLIC
-      $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
-      $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>
-  )
-  # Include protos (if any)
-  if (${_name}_PROTO_SOURCES)
-    target_include_directories(
-      ${_name}
-      PUBLIC
-        $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
-    )
-  endif()
-  target_link_directories(
-    ${_name}
-    PUBLIC
-      $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
-      $<BUILD_INTERFACE:${EXTERNALS_ROOT}/lib/$<CONFIG>>
-      $<INSTALL_INTERFACE:${CMAKE_INSTALL_LIBDIR}>
-  )
-  # Install main lib
-  foreach(_hdr ${${_name}_CXX_PROTO_HEADERS})
-    file(RELATIVE_PATH _relative_path ${CMAKE_CURRENT_BINARY_DIR} ${_hdr})
-    install(FILES ${_hdr} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/${_relative_path})
-  endforeach()
-  install_basic(
-    NAME ${_name}
-    INCLUDE_DIR ${_include_dir}
-  )
-
-  # Add tests executable (if there are any)
-  if (${_name}_TEST_SOURCES AND NOT WIN32)
-    add_executable(${_name}-tests ${${_name}_TEST_SOURCES})
-
-    get_target_property(_lib_type ${_name} TYPE)
-    if (_lib_type STREQUAL "SHARED_LIBRARY")
-      # Set RPATH so the tests exe knows where to find the shared lib.
-      set_target_properties(
-        ${_name}-tests
-        PROPERTIES
-          BUILD_RPATH "$ORIGIN"
-          INSTALL_RPATH "$ORIGIN/../${CMAKE_INSTALL_LIBDIR}"
-      )
-    endif()
-
-    # Link main lib
-    target_link_directories(
-      ${_name}-tests
-      PRIVATE
-        $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
-        $<INSTALL_INTERFACE:${CMAKE_INSTALL_LIBDIR}>
-    )
-    target_link_libraries(${_name}-tests PRIVATE ${_name})
-
-    # Catch2
-    find_package(Catch2 CONFIG REQUIRED COMPONENTS Catch2WithMain)
-    target_link_libraries(${_name}-tests PRIVATE Catch2::Catch2WithMain)
-  endif()
-endfunction()
@@ -1,39 +0,0 @@
-include_guard(DIRECTORY)
-include(GNUInstallDirs)
-include(${CMAKE_SOURCE_DIR}/cmake/PackBasic.cmake)
-include(${CMAKE_SOURCE_DIR}/cmake/SetFirstDefined.cmake)
-
-function(install_basic)
-  set(_options)
-  set(_one_value_args NAME INCLUDE_DIR)
-  set(_multi_value_args)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-  set_first_defined(_name _ARGS_NAME PROJECT_NAME)
-  set_first_defined(_include_dir _ARGS_INCLUDE_DIR _name)
-
-  file(
-    GLOB_RECURSE ${_name}_HEADERS
-    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
-    ${_include_dir}/*.h)
-
-  target_sources(${_name}
-    PUBLIC
-      FILE_SET HEADERS FILES ${${_name}_HEADERS})
-
-  install(
-    TARGETS ${_name}
-    EXPORT ${_name}Targets
-    RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
-    LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
-    ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
-    FILE_SET HEADERS DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}
-  )
-
-  install(
-    EXPORT ${_name}Targets
-    FILE ${_name}Targets.cmake
-    NAMESPACE ${_name}::
-    DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name})
-
-  pack_basic()
-endfunction()
@@ -1,44 +0,0 @@
-include_guard(DIRECTORY)
-include(CMakePackageConfigHelpers)
-include(${CMAKE_SOURCE_DIR}/cmake/SetFirstDefined.cmake)
-
-function(pack_basic)
-  set(_options)
-  set(_one_value_args NAME)
-  set(_multi_value_args)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-  set_first_defined(_name _ARGS_NAME PROJECT_NAME)
-
-  find_package(Git QUIET)
-
-  if(GIT_FOUND)
-    execute_process(
-      COMMAND ${GIT_EXECUTABLE} describe --tags --always --dirty
-      WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
-      OUTPUT_VARIABLE PROJECT_VERSION
-      OUTPUT_STRIP_TRAILING_WHITESPACE
-    )
-  endif()
-
-  if(NOT GIT_FOUND OR PROJECT_VERSION STREQUAL "")
-    set(PROJECT_VERSION "0.0.0")
-  endif()
-
-  write_basic_package_version_file(
-    "${CMAKE_CURRENT_BINARY_DIR}/${_name}ConfigVersion.cmake"
-    VERSION ${PROJECT_VERSION}
-    COMPATIBILITY AnyNewerVersion
-  )
-
-  configure_package_config_file(
-    "${CMAKE_CURRENT_SOURCE_DIR}/Config.cmake.in"
-    "${CMAKE_CURRENT_BINARY_DIR}/${_name}Config.cmake"
-    INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name}
-  )
-
-  install(
-    FILES
-      "${CMAKE_CURRENT_BINARY_DIR}/${_name}Config.cmake"
-      "${CMAKE_CURRENT_BINARY_DIR}/${_name}ConfigVersion.cmake"
-    DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name})
-endfunction()
@@ -1,23 +1,12 @@
-set(Protobuf_PROTOC_EXECUTABLE "${EXTERNALS_BIN}/protoc")
-set(PROTOBUF_INCLUDE_DIRS "${EXTERNALS_INCLUDE}")
+set(Protobuf_PROTOC_EXECUTABLE "${SPARK_BIN}/protoc")
+set(PROTOBUF_INCLUDE_DIRS "${SPARK_INCLUDE}")
 set(PROTOBUF_STATIC_LIB "libprotobuf")
-set(_protobuf_src_dir ${CMAKE_SOURCE_DIR}/src/lib_spark)
-set(_protobuf_bin_dir ${CMAKE_BINARY_DIR}/src/lib_spark)

-function(Protobuf_Generate_Cpp)
-  set(_options)
-  set(_one_value_args CXX_SOURCES CXX_HEADERS CXX_OUT_DIR)
-  set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-  set(${_ARGS_CXX_SOURCES})
-  set(${_ARGS_CXX_HEADERS})
+function(Protobuf_Generate_Cpp SRCS HDRS)
+  set(${SRCS})
+  set(${HDRS})

-  set(_proto_include_cli_args)
-  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
-    list(APPEND _proto_include_cli_args -I ${_dir})
-  endforeach()
-
-  foreach(FIL ${_ARGS_PROTO_SOURCES})
+  foreach(FIL ${ARGN})
    get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
    get_filename_component(FIL_WE ${FIL} NAME_WE)
    get_filename_component(FIL_DIR ${FIL} DIRECTORY)
@@ -25,56 +14,48 @@ function(Protobuf_Generate_Cpp)
      set(FIL_WE "${FIL_DIR}/${FIL_WE}")
    endif()

-    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc")
-    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h")
+    list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc")
+    list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h")

    add_custom_command(
-      OUTPUT "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
-             "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
+      OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc"
+             "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h"
      COMMAND
-        ${Protobuf_PROTOC_EXECUTABLE} ARGS --cpp_out=${_ARGS_CXX_OUT_DIR} ${_proto_include_cli_args} ${ABS_FIL}
+        ${Protobuf_PROTOC_EXECUTABLE} ARGS --cpp_out=${CMAKE_BINARY_DIR}/src -I
+        ${PROTOBUF_INCLUDE_DIRS} -I ${CMAKE_SOURCE_DIR}/src ${ABS_FIL}
      DEPENDS ${ABS_FIL}
      COMMENT "Running C++ protocol buffer compiler on ${FIL}"
      VERBATIM)
  endforeach()

-  set_source_files_properties(${${_ARGS_CXX_SOURCES}} ${${_ARGS_CXX_HEADERS}} PROPERTIES GENERATED TRUE)
+  set_source_files_properties(${${SRCS}} ${${HDRS}} PROPERTIES GENERATED TRUE)

  # remove some warnings from protobuf generated files
  if(WIN32)
-    set_source_files_properties(${${_ARGS_CXX_SOURCES}} PROPERTIES COMPILE_FLAGS
+    set_source_files_properties(${${SRCS}} PROPERTIES COMPILE_FLAGS
                                                      "/wd4309 /wd4838")
  else()
    set_source_files_properties(
-      ${${_ARGS_CXX_SOURCES}}
+      ${${SRCS}}
      PROPERTIES
        COMPILE_FLAGS
        "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
    )
  endif()

-  set(${_ARGS_CXX_SOURCES}
-      ${${_ARGS_CXX_SOURCES}}
+  set(${SRCS}
+      ${${SRCS}}
      PARENT_SCOPE)
-  set(${_ARGS_CXX_HEADERS}
-      ${${_ARGS_CXX_HEADERS}}
+  set(${HDRS}
+      ${${HDRS}}
      PARENT_SCOPE)
 endfunction()

-function(Service_Compiler_Generate_Cpp)
-  set(_options)
-  set(_one_value_args CXX_SOURCES CXX_HEADERS CXX_OUT_DIR)
-  set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-  set(${_ARGS_CXX_SOURCES})
-  set(${_ARGS_CXX_HEADERS})
+function(Service_Compiler_Generate_Cpp SRCS HDRS)
+  set(${SRCS})
+  set(${HDRS})

-  set(_proto_include_cli_args)
-  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
-    list(APPEND _proto_include_cli_args -I ${_dir})
-  endforeach()
-
-  foreach(FIL ${_ARGS_PROTO_SOURCES})
+  foreach(FIL ${ARGN})
    get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
    get_filename_component(FIL_WE ${FIL} NAME_WE)
    get_filename_component(FIL_DIR ${FIL} DIRECTORY)
@@ -82,58 +63,58 @@ function(Service_Compiler_Generate_Cpp)
      set(FIL_WE "${FIL_DIR}/${FIL_WE}")
    endif()

-    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc")
-    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h")
+    list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc")
+    list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h")

-    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc")
-    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h")
+    list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.cc")
+    list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.h")

-    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc")
-    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h")
+    list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.cc")
+    list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.h")

    set(TEMPLATES
-        ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/client.pb.cc
-        ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/client.pb.h
-        ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/server.pb.cc
-        ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/server.pb.h
+        ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/templates/cpp/client.pb.cc
+        ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/templates/cpp/client.pb.h
+        ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/templates/cpp/server.pb.cc
+        ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/templates/cpp/server.pb.h
    )

-    set(_plugin_py ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/plugin.py)
    if(WIN32)
-      set(_plugin_cmd ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/plugin.cmd)
      add_custom_command(
-        OUTPUT "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
-               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
-               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc"
-               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h"
-               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc"
-               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h"
+        OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc"
+               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h"
+               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.cc"
+               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.h"
+               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.cc"
+               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.h"
        COMMAND SET PYTHONPATH=${SPARK_PYTHON}
        COMMAND ${CMAKE_SOURCE_DIR}/.venv/Scripts/activate.bat
        COMMAND
-          ${Protobuf_PROTOC_EXECUTABLE} ARGS --cpp_out=${_ARGS_CXX_OUT_DIR}
-          --plugin=protoc-gen-service=${_plugin_cmd}
-          --service_out=lang=cpp:${_ARGS_CXX_OUT_DIR} -I
-          ${PROTOBUF_INCLUDE_DIRS} -I ${_protobuf_src_dir} ${ABS_FIL}
-        DEPENDS ${ABS_FIL} ${TEMPLATES} ${_plugin_cmd} ${_plugin_py}
+          ${Protobuf_PROTOC_EXECUTABLE} ARGS --cpp_out=${CMAKE_BINARY_DIR}/src
+          --plugin=protoc-gen-service=${CMAKE_SOURCE_DIR}/src/lib/api/service_compiler/plugin.cmd
+          --service_out=lang=cpp:${CMAKE_BINARY_DIR}/src -I
+          ${PROTOBUF_INCLUDE_DIRS} -I ${CMAKE_SOURCE_DIR}/src ${ABS_FIL}
+        DEPENDS ${ABS_FIL} ${TEMPLATES}
+                ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/plugin.py
        COMMENT "Running C++ service compiler on ${FIL}"
        VERBATIM)
    else()
      add_custom_command(
-        OUTPUT "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
-               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
-               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc"
-               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h"
-               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc"
-               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h"
+        OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc"
+               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h"
+               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.cc"
+               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.h"
+               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.cc"
+               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.h"
        COMMAND ${CMAKE_SOURCE_DIR}/tools/venv_exec.sh
        ARGS
          ${Protobuf_PROTOC_EXECUTABLE}
-          --cpp_out=${_ARGS_CXX_OUT_DIR}
-          --plugin=protoc-gen-service=${_plugin_py}
-          --service_out=lang=cpp,make_file_name=${ABS_FIL}:${_ARGS_CXX_OUT_DIR}
-          ${_proto_include_cli_args} ${ABS_FIL}
-        DEPENDS ${ABS_FIL} ${TEMPLATES} ${_plugin_py}
+          --cpp_out=${CMAKE_BINARY_DIR}/src
+          --plugin=protoc-gen-service=${CMAKE_SOURCE_DIR}/src/lib/api/service_compiler/plugin.py
+          --service_out=lang=cpp,make_file_name=${ABS_FIL}:${CMAKE_BINARY_DIR}/src
+          -I ${PROTOBUF_INCLUDE_DIRS} -I ${CMAKE_SOURCE_DIR}/src ${ABS_FIL}
+        DEPENDS ${ABS_FIL} ${TEMPLATES}
+                ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/plugin.py
        COMMENT "Running C++ service compiler on ${FIL}"
        VERBATIM)
    endif()
@@ -142,22 +123,22 @@ function(Service_Compiler_Generate_Cpp)

  # remove some warnings from protobuf generated files
  if(WIN32)
-    set_source_files_properties(${${_ARGS_CXX_SOURCES}} PROPERTIES COMPILE_FLAGS
+    set_source_files_properties(${${SRCS}} PROPERTIES COMPILE_FLAGS
                                                      "/wd4309 /wd4838")
  else()
    set_source_files_properties(
-      ${${_ARGS_CXX_SOURCES}}
+      ${${SRCS}}
      PROPERTIES
        COMPILE_FLAGS
        "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
    )
  endif()

-  set(${_ARGS_CXX_SOURCES}
-      ${${_ARGS_CXX_SOURCES}}
+  set(${SRCS}
+      ${${SRCS}}
      PARENT_SCOPE)
-  set(${_ARGS_CXX_HEADERS}
-      ${${_ARGS_CXX_HEADERS}}
+  set(${HDRS}
+      ${${HDRS}}
      PARENT_SCOPE)

 endfunction()
@@ -1,83 +0,0 @@
-include_guard(GLOBAL)
-
-function(protobuf_generate_cpp)
-  set(_options)
-  set(_one_value_args
-    CXX_SOURCES
-    CXX_HEADERS
-    CXX_OUT_DIR
-    PROTOC
-  )
-  set(_multi_value_args
-    PROTO_INCLUDE_DIRS
-    PROTO_SOURCES
-  )
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-
-  if (NOT DEFINED _ARGS_PROTOC)
-    if (WIN32)
-      set(_ARGS_PROTOC "${EXTERNALS_BIN}/protoc.exe")
-    else()
-      set(_ARGS_PROTOC "${EXTERNALS_BIN}/protoc")
-    endif()
-  endif()
-  set(${_ARGS_CXX_SOURCES})
-  set(${_ARGS_CXX_HEADERS})
-
-  set(_proto_include_cli_args)
-  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
-    list(APPEND _proto_include_cli_args -I ${_dir})
-  endforeach()
-
-  foreach(_fil ${_ARGS_PROTO_SOURCES})
-    get_filename_component(_abs_fil ${_fil} ABSOLUTE)
-    get_filename_component(_fil_we ${_fil} NAME_WE)
-    get_filename_component(_fil_dir ${_fil} DIRECTORY)
-    if(_fil_dir)
-      set(_fil_we "${_fil_dir}/${_fil_we}")
-    endif()
-    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.cc")
-    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.h")
-    message(STATUS "add_custom_command for ${_fil} -> ${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.h")
-    # TODO: Pass proto sources by relative path (so this fn can be called from
-    # arbitrary subdirs).
-    add_custom_command(
-      OUTPUT
-        "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.cc"
-        "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.h"
-      COMMAND
-        ${_ARGS_PROTOC}
-      ARGS
-        --cpp_out=${_ARGS_CXX_OUT_DIR} ${_proto_include_cli_args} ${_abs_fil}
-      DEPENDS ${_abs_fil}
-      COMMENT "Running C++ protocol buffer compiler on ${_fil}"
-      VERBATIM
-    )
-  endforeach()
-
-  set_source_files_properties(
-    ${${_ARGS_CXX_SOURCES}}
-    ${${_ARGS_CXX_HEADERS}}
-    PROPERTIES
-      GENERATED TRUE
-  )
-
-  # remove some warnings from protobuf generated files
-  if(WIN32)
-    set_source_files_properties(
-      ${${_ARGS_CXX_SOURCES}}
-      PROPERTIES
-        COMPILE_FLAGS "/wd4309 /wd4838"
-    )
-  else()
-    set_source_files_properties(
-      ${${_ARGS_CXX_SOURCES}}
-      PROPERTIES
-        COMPILE_FLAGS "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
-    )
-  endif()
-
-  set(${_ARGS_CXX_SOURCES} ${${_ARGS_CXX_SOURCES}} PARENT_SCOPE)
-  set(${_ARGS_CXX_HEADERS} ${${_ARGS_CXX_HEADERS}} PARENT_SCOPE)
-endfunction()
-
@@ -1,61 +0,0 @@
-include_guard(GLOBAL)
-
-function(protobuf_generate_cpp_config)
-  set(_options)
-  set(_one_value_args
-    PROTO_SOURCES_HEADER
-    PROTO_DEFAULTS_HEADER
-  )
-  set(_multi_value_args
-    PROTO_SOURCES
-    PROTO_DEFAULTS
-  )
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-
-  # Configure the 'config_proto_files' header. It provides an array of strings populated with the
-  # files in PROTO_SOURCES. At runtime, we use this array to generate a special 'config' proto
-  # message with fields for each message defined in said files. A global instance of this 'config'
-  # proto is then constructed, and populated with [compile-time] defaults, and [runtime] overrides
-  # (i.e., in spark.conf). Calls to `bts::config::get<T>()` retrieve the value from the field
-  # associated with type `T` in this global instance.
-  message(STATUS "PROTO_SOURCES: ${_ARGS_PROTO_SOURCES}")
-  message(STATUS "PROTO_SOURCES_HEADER: ${_ARGS_PROTO_SOURCES_HEADER}")
-  message(STATUS "PROTO_DEFAULTS_HEADER: ${_ARGS_PROTO_DEFAULTS_HEADER}")
-  if (_ARGS_PROTO_SOURCES_HEADER)
-    set(PROTO_SOURCES)
-    set(PROTO_SOURCES_CXX_ARRAY_ITEMS "")
-    foreach(_proto_path ${_ARGS_PROTO_SOURCES})
-      list(APPEND PROTO_SOURCES ${_proto_path})
-      string(APPEND PROTO_SOURCES_CXX_ARRAY_ITEMS "\"${_proto_path}\",\n")
-    endforeach()
-    list(LENGTH PROTO_SOURCES PROTO_SOURCES_LENGTH)
-    message(STATUS "CONFIGURING PROTO_SOURCES_HEADER): ${_ARGS_PROTO_SOURCES_HEADER}")
-    configure_file(${CMAKE_SOURCE_DIR}/cmake/config_proto_files.h.in ${_ARGS_PROTO_SOURCES_HEADER})
-  endif()
-
-  # Generate static defaults. Ideally, this would be a single string, but we use an array because
-  # the compiler complains if the string is longer than 65k.
-  set(_cfg_defaults_str "inline const char * config_static_defaults[] = { ")
-
-  foreach(_cfg_file_path ${_ARGS_PROTO_DEFAULTS})
-    file(READ ${_cfg_file_path} _cfg_file_content)
-    string(REPLACE "\"" "\\\"" _cfg_file_content_escaped ${_cfg_file_content})
-    string(REPLACE "\n" ";" _cfg_lines ${_cfg_file_content_escaped})
-
-    foreach(_line ${_cfg_lines})
-      string(APPEND _cfg_defaults_str "\n  \"${_line}\",")
-    endforeach(_line)
-  endforeach(_cfg_file_path)
-
-  string(APPEND _cfg_defaults_str "};")
-  set(_old_cfg_defaults_str "")
-
-  if(EXISTS ${_ARGS_PROTO_DEFAULTS_HEADER})
-    file(READ ${_ARGS_PROTO_DEFAULTS_HEADER} _old_cfg_defaults_str)
-  endif()
-
-  if(NOT _cfg_defaults_str STREQUAL _old_cfg_defaults_str)
-    file(WRITE "${_ARGS_PROTO_DEFAULTS_HEADER}" "${_cfg_defaults_str}")
-  endif()
-endfunction()
-
@@ -1,49 +0,0 @@
-include_guard(DIRECTORY)
-
-function(protobuf_generate_fs)
-  set(_options)
-  set(_one_value_args TARGET_NAME OUT_DIR PROTOC)
-  set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-
-  if (NOT DEFINED _ARGS_TARGET_NAME)
-    set(_ARGS_TARGET_NAME "${PROJECT_NAME}_generate_fs_protos")
-  endif()
-  if (NOT DEFINED _ARGS_PROTOC)
-    set(_ARGS_PROTOC "${EXTERNALS_BIN}/protoc")
-  endif()
-
-  set(_proto_include_cli_args)
-  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
-    list(APPEND _proto_include_cli_args -I ${_dir})
-  endforeach()
-
-  set(_outputs)
-  foreach(_proto_source ${_ARGS_PROTO_SOURCES})
-    # get_filename_component(_abs_proto_source ${_proto_source} ABSOLUTE)
-    get_filename_component(_proto_source_we ${_proto_source} NAME_WE)
-    # get_filename_component(_proto_source_dir ${_proto_source} DIRECTORY)
-    list(APPEND _outputs ${_ARGS_OUT_DIR}/${_proto_source_we}.fable.fs)
-  endforeach()
-
-  set(_venv_exec ${CMAKE_SOURCE_DIR}/tools/venv_exec.sh)
-  set(_plugin_py ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/plugin.py)
-  set(_template_fs ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/fs/fable.fs)
-  add_custom_command(
-    OUTPUT ${_outputs}
-    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
-    COMMAND ${_venv_exec}
-      ${_ARGS_PROTOC}
-      --plugin=protoc-gen-service=${_plugin_py}
-      --service_out=lang=fs,strip_out_path=1:${_ARGS_OUT_DIR}
-      ${_proto_include_cli_args}
-      ${_ARGS_PROTO_SOURCES}
-    DEPENDS
-      ${_venv_exec}
-      ${_plugin_py}
-      ${_template_fs}
-      ${CMAKE_CURRENT_SOURCE_DIR}/package-lock.json
-      ${_ARGS_PROTO_SOURCES}
-  )
-  add_custom_target(${_ARGS_TARGET_NAME} ALL DEPENDS "${_outputs}")
-endfunction()
@@ -1,33 +0,0 @@
-include_guard(DIRECTORY)
-
-function(protobuf_generate_js)
-  set(_options)
-  set(_one_value_args TARGET_NAME OUTPUT)
-  set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-
-  if (NOT DEFINED _ARGS_TARGET_NAME)
-    set(_ARGS_TARGET_NAME "${PROJECT_NAME}_generate_js_protos")
-  endif()
-
-  set(_proto_include_cli_args)
-  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
-    list(APPEND _proto_include_cli_args -p ${_dir})
-  endforeach()
-
-  add_custom_command(
-    OUTPUT ${_ARGS_OUTPUT}
-    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
-    COMMAND npx
-      --no-install pbjs
-      -t json-module
-      -w commonjs
-      -o ${_ARGS_OUTPUT}
-      ${_proto_include_cli_args}
-      ${_ARGS_PROTO_SOURCES}
-    DEPENDS
-      ${CMAKE_CURRENT_SOURCE_DIR}/package-lock.json
-      ${_ARGS_PROTO_SOURCES}
-  )
-  add_custom_target(${_ARGS_TARGET_NAME} ALL DEPENDS ${_ARGS_OUTPUT})
-endfunction()
@@ -1,10 +0,0 @@
-include_guard(GLOBAL)
-
-macro(set_first_defined RESULT)
-  foreach(_arg ${ARGN})
-    if (DEFINED ${_arg})
-      set(${RESULT} ${${_arg}})
-      break()
-    endif()
-  endforeach()
-endmacro()
@@ -1,49 +0,0 @@
-include_guard(DIRECTORY)
-include(${CMAKE_SOURCE_DIR}/cmake/TernaryOperator.cmake)
-
-function(sym_link_dir)
-  set(_options)
-  set(_one_value_args TARGET_NAME SOURCE DEST COMMENT)
-  set(_multi_value_args)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-  add_custom_command(
-    OUTPUT ${_ARGS_DEST}
-    COMMAND ln ARGS -sf ${_ARGS_SOURCE} ${_ARGS_DEST}
-  )
-  set_ternary(_comment _ARGS_COMMENT "${_ARGS_COMMENT}" "linking dir ${_ARGS_SOURCE} -> ${_ARGS_DEST}")
-  add_custom_target(
-    ${_ARGS_TARGET_NAME} ALL
-    DEPENDS ${_ARGS_DEST}
-    COMMENT "${_comment}"
-  )
-endfunction()
-
-function(sym_link_file)
-  set(_options)
-  set(_one_value_args SOURCE DEST INSTALL_DEST)
-  set(_multi_value_args)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-  file(
-    CREATE_LINK
-      ${_ARGS_SOURCE}
-      ${_ARGS_DEST}
-    SYMBOLIC
-  )
-  if (DEFINED _ARGS_INSTALL_DEST)
-    install(
-      FILES
-      ${_ARGS_SOURCE}
-      DESTINATION ${_ARGS_INSTALL_DEST}
-    )
-  endif()
-endfunction()
-
-function(sym_link_local_lib LIB_NAME)
-  set(_lib_file_name "${CMAKE_SHARED_LIBRARY_PREFIX}${LIB_NAME}.so")
-  file(
-    CREATE_LINK
-      "${CMAKE_BINARY_DIR}/src/lib_${LIB_NAME}/${_lib_file_name}"
-      "${CMAKE_CURRENT_BINARY_DIR}/${_lib_file_name}"
-    SYMBOLIC
-  )
-endfunction()
@@ -1,9 +0,0 @@
-include_guard(GLOBAL)
-
-macro(set_ternary RESULT CONDITION TRUE_VALUE FALSE_VALUE)
-  if(${CONDITION})
-    set(${RESULT} ${TRUE_VALUE})
-  else()
-    set(${RESULT} ${FALSE_VALUE})
-  endif()
-endmacro()
@@ -1,11 +0,0 @@
-#pragma once
-
-#include <array>
-
-namespace bts::@PROJECT_NAME@::protobufs {
-
-inline std::array<const char *, @PROTO_SOURCES_LENGTH@> config_proto_files = {
-@PROTO_SOURCES_CXX_ARRAY_ITEMS@
-};
-
-} // namespace bts::@PROJECT_NAME@::protobufs
@@ -1,11 +0,0 @@
-#ifndef BTS_SPARK_VERSION_H
-#define BTS_SPARK_VERSION_H
-
-#define BTS_SPARK_VERSION_MAJOR @PROJECT_VERSION_MAJOR@
-#define BTS_SPARK_VERSION_MINOR @PROJECT_VERSION_MINOR@
-#define BTS_SPARK_VERSION_PATCH @PROJECT_VERSION_PATCH@
-#define BTS_SPARK_VERSION_BUILD_METADATA @PROJECT_VERSION_BUILD_METADATA@
-#define BTS_SPARK_VERSION "@PROJECT_VERSION@"
-#define BTS_SPARK_VERSION_FULL "@PROJECT_VERSION_FULL@"
-
-#endif
@@ -0,0 +1,24 @@
+cmake_minimum_required(VERSION 3.5)
+
+# Explicitly set policy to convert relative paths to absolute (although this is a default policy,
+# setting it explicitly prevents a compile-warning)
+cmake_policy(SET CMP0076 NEW)
+
+# add protos to the library to facilitate source_group
+target_sources(libspark PUBLIC ${Config_Protos} ${NonConfig_Protos} ${Service_Protos})
+
+function(assign_source_group)
+  foreach(_source IN ITEMS ${ARGN})
+    if(IS_ABSOLUTE "${_source}")
+      file(RELATIVE_PATH _source_rel "${CMAKE_CURRENT_SOURCE_DIR}" "${_source}")
+    else()
+      set(_source_rel "${_source}")
+    endif()
+    get_filename_component(_source_path "${_source_rel}" PATH)
+    string(REPLACE "/" "\\" _source_path_msvc "${_source_path}")
+    source_group("${_source_path_msvc}" FILES "${_source}")
+    set(SOURCE_FILES "${_source}")
+  endforeach()
+endfunction(assign_source_group)
+
+assign_source_group(${SparkLib_Sources} ${Config_Protos} ${NonConfig_Protos} ${Service_Protos})
@@ -1,104 +0,0 @@
-services:
-  sparkd:
-    volumes:
-      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/sparkdata/data
-    build:
-      dockerfile: src/app_sparkd/Dockerfile
-      args:
-        - IMAGE=${SPARKD__IMAGE}
-        - SOURCE_DIR=${SPARKD__SOURCE_DIR}
-        - CONFIG_DIR=${SPARKD__CONFIG_DIR}
-    shm_size: "${SPARKD__SHM_SIZE:-512mb}"
-    command: >
-      sh -c "
-        echo bts | sudo -S chown -R bts:bts /home/bts/sparkdata/data
-        if [ -e "/home/bts/sparkdata/data/logging.sparkd/log" ]; then
-          mv "/home/bts/sparkdata/data/logging.sparkd/log" "/home/bts/sparkdata/data/logging.sparkd/log_$(date '+%Y%m%d-%H%M')"
-        fi
-        if [ -e "/home/bts/sparkdata/conf/.env.local" ]; then
-          echo 'Sourcing /home/bts/sparkdata/conf/.env.local'
-          set -a
-          . /home/bts/sparkdata/conf/.env.local
-          set +a
-        fi
-        ./sparkd run"
-    env_file:
-      - .env
-      - ${SPARKD__CONFIG_DIR}/.env
-    ports:
-      - "${SPARKD__CONFIG_PORT}:${SPARKD__CONFIG_PORT}"
-      - "${SPARKD__QUICKBUS_PORT}:${SPARKD__QUICKBUS_PORT}"
-      - "${SPARKD__SIM_FIX_PORT}:${SPARKD__SIM_FIX_PORT}"
-    deploy:
-      resources:
-        limits:
-          memory: "${SPARKD__MEMORY:-4gb}"
-
-  sparkd-symbols:
-    volumes:
-      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/sparkdata/data
-      - ${SPARKD__RUNTIME_CONF_DIR:-sparkd_conf}:/home/bts/sparkdata/conf
-    build:
-      dockerfile: src/server/sparkd/Dockerfile
-      args:
-        - IMAGE=${SPARKD__IMAGE}
-        - SOURCE_DIR=${SPARKD__SOURCE_DIR}
-        - CONFIG_DIR=${SPARKD__CONFIG_DIR}
-    shm_size: "${SPARKD__SHM_SIZE:-512mb}"
-    command: >
-      sh -c "
-        echo bts | sudo -S chown -R bts:bts /home/bts/sparkdata
-        if [ -e "/home/bts/sparkdata/data/logging.sparkd/log" ]; then
-          mv "/home/bts/sparkdata/data/logging.sparkd/log" "/home/bts/sparkdata/data/logging.sparkd/log_$(date '+%Y%m%d-%H%M')"
-        fi
-        ls -alh /home/bts/sparkdata/conf/
-        cat /home/bts/sparkdata/conf/.env.local
-        if [ -e "/home/bts/sparkdata/conf/.env.local" ]; then
-          echo 'Sourcing /home/bts/sparkdata/conf/.env.local'
-          set -a
-          . /home/bts/sparkdata/conf/.env.local
-          set +a
-        fi
-        ./sparkd run"
-    env_file:
-      - .env
-      - ${SPARKD__CONFIG_DIR}/.env
-    ports:
-      - "${SPARKD__CONFIG_PORT}:${SPARKD__CONFIG_PORT}"
-      - "${SPARKD__QUICKBUS_PORT}:${SPARKD__QUICKBUS_PORT}"
-      - "${SPARKD__SIM_FIX_PORT}:${SPARKD__SIM_FIX_PORT}"
-    deploy:
-      restart_policy:
-        condition: any
-        delay: 3s
-      resources:
-        limits:
-          memory: "${SPARKD__MEMORY:-4gb}"
-
-  sparkd-tail:
-    image: bluetradingsystems/linux_dotnet7
-    volumes:
-      - type: volume
-        read_only: true
-        source: sparkd_data
-        target: /home/bts/sparkdata/data/logging.sparkd
-        volume:
-          subpath: logging.sparkd
-    command: ["tail", "-f", "/home/bts/sparkdata/data/logging.sparkd/log"]
-
-  sparkd-ping-test:
-    depends_on:
-      - sparkd
-    profiles:
-      - ping-test
-    build:
-      context: .
-      dockerfile_inline: |
-        FROM bluetradingsystems/linux_dotnet7
-        USER "root"
-        RUN apt-get update && apt-get -y install iputils-ping
-    command: [ "ping", "sparkd" ]
-
-volumes:
-  sparkd_data:
-    name: "sparkd_${SPARKD__CONFIG_PORT:-8000}"
@@ -124,8 +124,8 @@ service OrderService {
 ```
 ### C++ Example
 ```cpp
-#include "spark/api/public/ServiceContext.h"
-#include "spark/api/services/OrderService.client.pb.h"
+#include "lib/api/public/ServiceContext.h"
+#include "lib/api/services/OrderService.client.pb.h"
 using namespace bts::api;
 ServiceContext ctx;
 auto orders = ctx.locate<orderman::OrderService>();
@@ -29,9 +29,9 @@ namespace sample {
 ```

 ```cpp
-#include "spark/api/public/ServiceContext.h"
-#include "spark/domain_model/Security.h"
-#include "spark/api/services/MarketDataService.client.pb.h"
+#include "lib/api/public/ServiceContext.h"
+#include "spark/lib/domain_model/Security.h"
+#include "lib/api/services/MarketDataService.client.pb.h"
 #include <thread>

 using namespace bts::api;
@@ -1,38 +0,0 @@
-FROM ubuntu:22.04
-
-ARG DEBIAN_FRONTEND=noninteractive
-
-RUN apt-get update && apt-get install -y pkg-config
-RUN apt-get update && apt-get install -y locales
-RUN apt-get update && apt-get install -y ca-certificates
-RUN apt-get update && apt-get install -y libssh2-1-dev
-RUN apt-get update && apt-get install -y software-properties-common
-RUN apt-get update && apt-get install -y sudo
-
-# GCC
-ARG gcc_major=13
-RUN add-apt-repository ppa:ubuntu-toolchain-r/test \
-  && apt-get update \
-  && apt-get install -y g++-$gcc_major
-
-RUN apt-get update \
-  && apt-get install -y dotnet7
-
-RUN locale-gen en_US.UTF-8
-
-ARG docker_user="bts"
-ARG docker_uid="1000"
-ARG docker_pass="bts"
-
-RUN \
-  groupadd -g "$docker_uid" "$docker_user" \
-  && useradd \
-  -m \
-  -u "$docker_uid" \
-  -g "$docker_uid" \
-  -G sudo \
-  --password "$(openssl passwd -1 "$docker_pass")" \
-  "$docker_user"
-
-USER bts
-WORKDIR /home/bts
@@ -1,20 +1,11 @@
 astroid==2.7.3
 autopep8==1.5.7
-blinker==1.9.0
-boto3==1.37.9
-botocore==1.37.9
 certifi==2023.5.7
 charset-normalizer==3.1.0
-click==8.1.8
 colorama==0.4.4
-Flask==3.1.0
 idna==3.4
 isort==5.9.3
-itsdangerous==2.2.0
-Jinja2==3.1.6
-jmespath==1.0.1
 lazy-object-proxy==1.6.0
-MarkupSafe==3.0.2
 mccabe==0.6.1
 platformdirs==2.3.0
 protobuf==4.21.12
@@ -23,14 +14,9 @@ pycodestyle==2.7.0
 pycparser==2.20
 pylint==2.10.2
 PyMeta3==0.5.1
-python-dateutil==2.9.0.post0
-python-dotenv==1.0.1
 requests==2.31.0
-s3transfer==0.11.4
-six==1.17.0
 termcolor==1.1.0
 toml==0.10.2
 urllib3==2.0.3
-Werkzeug==3.1.3
 wrapt==1.12.1
 yapf==0.31.0
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
    <ImplicitUsings>enable</ImplicitUsings>
    <Nullable>enable</Nullable>
  </PropertyGroup>
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
    <ImplicitUsings>enable</ImplicitUsings>
    <Nullable>enable</Nullable>
  </PropertyGroup>
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
    <ImplicitUsings>enable</ImplicitUsings>
    <Nullable>enable</Nullable>
  </PropertyGroup>
@@ -1,14 +1,14 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
    <ImplicitUsings>enable</ImplicitUsings>
    <Nullable>enable</Nullable>
  </PropertyGroup>

  <ItemGroup>
    <PackageReference Include="Google.Protobuf" Version="3.24.3" />
-    <PackageReference Include="YaccLexTools" Version="1.2.3" />
+    <PackageReference Include="YaccLexTools" Version="1.2.0" />
  </ItemGroup>

  <ItemGroup>
@@ -24,6 +24,8 @@ namespace QUT.GplexBuffers
        public BufferException(string message) : base(message) { }
        public BufferException(string message, Exception innerException)
            : base(message, innerException) { }
+        protected BufferException(SerializationInfo info, StreamingContext context)
+            : base(info, context) { }
    }

    internal abstract class ScanBuff
@@ -58,10 +60,12 @@ namespace QUT.GplexBuffers
            return new BuildBuffer(source);
        }

+#if (!BYTEMODE)
        public static ScanBuff GetBuffer(Stream source, int fallbackCodePage)
        {
            return new BuildBuffer(source, fallbackCodePage);
        }
+#endif // !BYTEMODE
 #endif // !NOFILES
    }

@@ -349,15 +353,17 @@ namespace QUT.GplexBuffers
        {
            FileStream fStrm = (stream as FileStream);
            if (fStrm != null) FileName = fStrm.Name;
-            NextBlk = BlockReaderFactory.Get(stream);
+            NextBlk = BlockReaderFactory.Raw(stream);
        }

+#if (!BYTEMODE)
        public BuildBuffer(Stream stream, int fallbackCodePage)
        {
            FileStream fStrm = (stream as FileStream);
            if (fStrm != null) FileName = fStrm.Name;
            NextBlk = BlockReaderFactory.Get(stream, fallbackCodePage);
        }
+#endif

        /// <summary>
        /// Marks a conservative lower bound for the buffer,
@@ -428,19 +434,43 @@ namespace QUT.GplexBuffers
    //
    internal static class BlockReaderFactory
    {
+        public static BlockReader Raw(Stream stream)
+        {
+            return delegate(char[] block, int index, int number)
+            {
+                byte[] b = new byte[number];
+                int count = stream.Read(b, 0, number);
+                int i = 0;
+                int j = index;
+                for (; i < count; i++, j++)
+                    block[j] = (char)b[i];
+                return count;
+            };
+        }

-        public static BlockReader Get(Stream stream, int fallbackCodePage = -2)
+#if (!BYTEMODE)
+        public static BlockReader Get(Stream stream, int fallbackCodePage)
        {
            Encoding encoding;
            int preamble = Preamble(stream);

            if (preamble != 0)  // There is a valid BOM here!
                encoding = Encoding.GetEncoding(preamble);
-            else if (fallbackCodePage >= 0) // Anything but "guess"
+            else if (fallbackCodePage == -1) // Fallback is "raw" bytes
+                return Raw(stream);
+            else if (fallbackCodePage != -2) // Anything but "guess"
                encoding = Encoding.GetEncoding(fallbackCodePage);
            else // This is the "guess" option
-                encoding = GuessEncoding(stream);
-
+            {
+                int guess = new Guesser(stream).GuessCodePage();
+                stream.Seek(0, SeekOrigin.Begin);
+                if (guess == -1) // ==> this is a 7-bit file
+                    encoding = Encoding.ASCII;
+                else if (guess == 65001)
+                    encoding = Encoding.UTF8;
+                else             // ==> use the machine default
+                    encoding = Encoding.Default;
+            }
            StreamReader reader = new StreamReader(stream, encoding);
            return reader.Read;
        }
@@ -458,14 +488,6 @@ namespace QUT.GplexBuffers
            int b2 = stream.ReadByte();
            if (b0 == 0xef && b1 == 0xbb && b2 == 0xbf)
                return 65001; // UTF8
-
-            int b3 = stream.ReadByte();
-            // Check for UTF-32 BOM (Byte Order Mark)
-            if (b0 == 0x00 && b1 == 0x00 && b2 == 0xFE && b3 == 0xFF)
-                return 12001; // UTF-32BE
-            if (b0 == 0xFF && b1 == 0xFE && b2 == 0x00 && b3 == 0x00)
-                return 12000; // UTF-32LE
-
            //
            // There is no unicode preamble, so we
            // return denoter for the machine default.
@@ -473,150 +495,7 @@ namespace QUT.GplexBuffers
            stream.Seek(0, SeekOrigin.Begin);
            return 0;
        }
-
-        private static Encoding GuessEncoding(Stream stream)
-        {
-            stream.Seek(0, SeekOrigin.Begin);
-            if (IsUtf16(stream, out Encoding encoding))
-            {
-                stream.Seek(0, SeekOrigin.Begin);
-                return encoding;
-            }
-
-            stream.Seek(0, SeekOrigin.Begin);
-            if (IsUtf8(stream))
-            {
-                stream.Seek(0, SeekOrigin.Begin);
-                return Encoding.UTF8;
-            }
-
-            stream.Seek(0, SeekOrigin.Begin);
-            if (IsAscii(stream))
-            {
-                stream.Seek(0, SeekOrigin.Begin);
-                return Encoding.ASCII;
-            }
-
-#if !NET462_OR_GREATER
-            stream.Seek(0, SeekOrigin.Begin);
-            if (IsLatin1(stream))
-            {
-                stream.Seek(0, SeekOrigin.Begin);
-                return Encoding.Latin1;
-            }
-#endif
-            stream.Seek(0, SeekOrigin.Begin);
-            return Encoding.Default;
-        }
-
-        private static bool IsUtf8(Stream stream)
-        {
-            int b;
-            while ((b = stream.ReadByte()) != -1)
-            {
-                if ((b & 0x80) == 0) // 0xxxxxxx
-                {
-                    continue;
-                }
-                else if ((b & 0xE0) == 0xC0) // 110xxxxx 10xxxxxx
-                {
-                    if (!IsValidUtf8ContinuationByte(stream)) return false;
-                }
-                else if ((b & 0xF0) == 0xE0) // 1110xxxx 10xxxxxx 10xxxxxx
-                {
-                    if (!IsValidUtf8ContinuationByte(stream) || !IsValidUtf8ContinuationByte(stream)) return false;
-                }
-                else if ((b & 0xF8) == 0xF0) // 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
-                {
-                    if (!IsValidUtf8ContinuationByte(stream) || !IsValidUtf8ContinuationByte(stream) || !IsValidUtf8ContinuationByte(stream)) return false;
-                }
-                else
-                {
-                    return false;
-                }
-            }
-            return true;
-        }
-
-        private static bool IsValidUtf8ContinuationByte(Stream stream)
-        {
-            int b = stream.ReadByte();
-            return b != -1 && (b & 0xC0) == 0x80; // 10xxxxxx
-        }
-
-        private static bool IsUtf16(Stream stream, out Encoding encoding)
-        {
-            encoding = Encoding.Default;
-
-            if (stream.Length >= 2)
-            {
-                // Read the first two bytes
-                int b0 = stream.ReadByte();
-                int b1 = stream.ReadByte();
-
-                // Check for UTF-16 BOM (Byte Order Mark)
-                if (b0 == 0xFF && b1 == 0xFE)
-                {
-                    encoding = Encoding.Unicode;
-                    return true; // UTF-16LE
-                }
-                if (b0 == 0xFE && b1 == 0xFF)
-                {
-                    encoding = Encoding.BigEndianUnicode;
-                    return true; // UTF-16BE
-                }
-
-                // Reset the stream position
-                stream.Seek(0, SeekOrigin.Begin);
-
-                // Check for common UTF-16 patterns
-                byte[] buffer = new byte[2];
-                while (stream.Read(buffer, 0, 2) == 2)
-                {
-                    if (buffer[0] == 0x00 && buffer[1] != 0x00)
-                    {
-                        encoding = Encoding.BigEndianUnicode;
-                        return true; // Likely UTF-16BE
-                    }
-                    if (buffer[0] != 0x00 && buffer[1] == 0x00)
-                    {
-                        encoding = Encoding.Unicode;
-                        return true; // Likely UTF-16LE
-                    }
-                }
-            }
-
-            return false;
-        }
-
-        private static bool IsAscii(Stream stream)
-        {
-            int b;
-            while ((b = stream.ReadByte()) != -1)
-            {
-                // Latin1 includes byte values from 0x00 to 0xFF
-                if (b > 0x7F)
-                {
-                    return false;
-                }
-            }
-            return true;
-        }
-
-        private static bool IsLatin1(Stream stream)
-        {
-            int b;
-            while ((b = stream.ReadByte()) != -1)
-            {
-                // Latin1 includes byte values from 0x00 to 0xFF
-                if (b > 0xFF)
-                {
-                    return false;
-                }
-            }
-            return true;
-        }
-
+#endif // !BYTEMODE
    }
 #endif // !NOFILES
    #endregion Buffer classes
@@ -661,6 +540,180 @@ namespace QUT.GplexBuffers
            return 0;
        }
    }
+#region guesser
+#if (!BYTEMODE)
+    // ==============================================================
+    // ============          Encoding Guesser           =============
+    // ==============================================================
+
+    /// <summary>
+    /// This class provides a simple finite state automaton that
+    /// scans the file looking for (1) valid UTF-8 byte patterns,
+    /// (2) bytes >= 0x80 which are not part of a UTF-8 sequence.
+    /// The method then guesses whether it is UTF-8 or maybe some 
+    /// local machine default encoding.  This works well for the
+    /// various Latin encodings.
+    /// </summary>
+    internal class Guesser
+    {
+        ScanBuff buffer;
+
+        public int GuessCodePage() { return Scan(); }
+
+        const int maxAccept = 10;
+        const int initial = 0;
+        const int eofNum = 0;
+        const int goStart = -1;
+        const int INITIAL = 0;
+        const int EndToken = 0;
+
+        #region user code
+        /* 
+         *  Reads the bytes of a file to determine if it is 
+         *  UTF-8 or a single-byte code page file.
+         */
+        public long utfX;
+        public long uppr;
+        #endregion user code
+
+        int state;
+        int currentStart = startState[0];
+        int code;
+
+        #region ScannerTables
+        static int[] startState = new int[] { 11, 0 };
+
+        #region CharacterMap
+        static sbyte[] map = new sbyte[256] {
+/*     '\0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*   '\x10' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*   '\x20' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*      '0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*      '@' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*      'P' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*      '`' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*      'p' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*   '\x80' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
+/*   '\x90' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
+/*   '\xA0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
+/*   '\xB0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
+/*   '\xC0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 
+/*   '\xD0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 
+/*   '\xE0' */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 
+/*   '\xF0' */ 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5 };
+        #endregion
+
+        static sbyte[][] nextState = new sbyte[][] {
+            new sbyte[] {0, 0, 0, 0, 0, 0},
+            new sbyte[] {-1, -1, 10, -1, -1, -1},
+            new sbyte[] {-1, -1, -1, -1, -1, -1},
+            new sbyte[] {-1, -1, 8, -1, -1, -1},
+            new sbyte[] {-1, -1, 5, -1, -1, -1},
+            new sbyte[] {-1, -1, 6, -1, -1, -1},
+            new sbyte[] {-1, -1, 7, -1, -1, -1},
+            null,
+            new sbyte[] {-1, -1, 9, -1, -1, -1},
+            null,
+            null,
+            new sbyte[] {-1, 1, 2, 3, 4, 2}
+        };
+
+
+        [SuppressMessage("Microsoft.Performance", "CA1810:InitializeReferenceTypeStaticFieldsInline")]
+        // Reason for suppression: cannot have self-reference in array initializer.
+        static Guesser()
+        {
+            nextState[7] = nextState[2];
+            nextState[9] = nextState[2];
+            nextState[10] = nextState[2];
+        }
+
+        int NextState()
+        {
+            if (code == ScanBuff.EndOfFile)
+                return eofNum;
+            else
+                return nextState[state][map[code]];
+        }
+        #endregion
+
+        public Guesser(System.IO.Stream file) { SetSource(file); }
+
+        public void SetSource(System.IO.Stream source)
+        {
+            this.buffer = new BuildBuffer(source);
+            code = buffer.Read();
+        }
+
+        int Scan()
+        {
+            for (; ; )
+            {
+                int next;
+                state = currentStart;
+                while ((next = NextState()) == goStart)
+                    code = buffer.Read();
+
+                state = next;
+                code = buffer.Read();
+
+                while ((next = NextState()) > eofNum)
+                {
+                    state = next;
+                    code = buffer.Read();
+                }
+                if (state <= maxAccept)
+                {
+                    #region ActionSwitch
+#pragma warning disable 162
+                    switch (state)
+                    {
+                        case eofNum:
+                            switch (currentStart)
+                            {
+                                case 11:
+                                    if (utfX == 0 && uppr == 0) return -1; /* raw ascii */
+                                    else if (uppr * 10 > utfX) return 0;   /* default code page */
+                                    else return 65001;                     /* UTF-8 encoding */
+                                    break;
+                            }
+                            return EndToken;
+                        case 1: // Recognized '{Upper128}',	Shortest string "\xC0"
+                        case 2: // Recognized '{Upper128}',	Shortest string "\x80"
+                        case 3: // Recognized '{Upper128}',	Shortest string "\xE0"
+                        case 4: // Recognized '{Upper128}',	Shortest string "\xF0"
+                            uppr++;
+                            break;
+                        case 5: // Recognized '{Utf8pfx4}{Utf8cont}',	Shortest string "\xF0\x80"
+                            uppr += 2;
+                            break;
+                        case 6: // Recognized '{Utf8pfx4}{Utf8cont}{2}',	Shortest string "\xF0\x80\x80"
+                            uppr += 3;
+                            break;
+                        case 7: // Recognized '{Utf8pfx4}{Utf8cont}{3}',	Shortest string "\xF0\x80\x80\x80"
+                            utfX += 3;
+                            break;
+                        case 8: // Recognized '{Utf8pfx3}{Utf8cont}',	Shortest string "\xE0\x80"
+                            uppr += 2;
+                            break;
+                        case 9: // Recognized '{Utf8pfx3}{Utf8cont}{2}',	Shortest string "\xE0\x80\x80"
+                            utfX += 2;
+                            break;
+                        case 10: // Recognized '{Utf8pfx2}{Utf8cont}',	Shortest string "\xC0\x80"
+                            utfX++;
+                            break;
+                        default:
+                            break;
+                    }
+#pragma warning restore 162
+                    #endregion
+                }
+            }
+        }
+    } // end class Guesser
+    
+#endif // !BYTEMODE
+#endregion
 #endif // !NOFILES

 // End of code copied from embedded resource
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
    <ImplicitUsings>enable</ImplicitUsings>
    <Nullable>enable</Nullable>
  </PropertyGroup>
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
    <ImplicitUsings>enable</ImplicitUsings>
    <Nullable>enable</Nullable>
  </PropertyGroup>
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
    <ImplicitUsings>enable</ImplicitUsings>
    <Nullable>enable</Nullable>
  </PropertyGroup>
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
    <Nullable>enable</Nullable>
  </PropertyGroup>

@@ -18,12 +18,8 @@

  <ItemGroup>
    <PackageReference Include="HtmlAgilityPack" Version="1.11.57" />
-    <PackageReference Include="Microsoft.Data.Analysis" Version="0.21.1" />
    <PackageReference Include="MongoDB.Bson" Version="2.23.1" />
    <PackageReference Include="MongoDB.Driver" Version="2.23.1" />
-    <PackageReference Include="Parquet.Net" Version="5.0.1" />
-    <PackageReference Include="System.Data.SQLite" Version="1.0.118" />
-    <PackageReference Include="MathNet.Numerics" Version="5.0.0" />
  </ItemGroup>

 </Project>
@@ -1,32 +0,0 @@
-namespace bts.SilverPine {
-   using System;
-   using BCG;
-   using System.Collections.Generic;
-   using Bts.DataStreamExpressions;
-
-   // All this does is take in a dsl expression, parse it and fire the data that come out of running it.
-   // So for example, if an expresison is passed 
-   [DataStreamUserRespresentation("dsl")]
-   public class DSLStream : ViewStream {
-      private readonly string _dsl;
-      public DSLStream(string dsl) {
-         Log.Gen.info($"DSLStream.cons dsl: {dsl}");
-         _dsl = dsl;
-         ViewStream rs = build_stream(dsl);
-         rs = rs ?? new LiteralStream("error", $"msg:invalid dsl \"{dsl}\"");
-         mySetRealSource(rs);
-      }
-
-      static ViewStream? build_stream(string stream_expression) {
-         try {
-            var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression);
-            var rv = exp.getView(new Dictionary<string, StatementExpression>(0));
-            return rv;
-
-         } catch (Exception ex) {
-            Log.Gen.error($"Could not build stream: {stream_expression} ex: {ex}");
-            return null;
-         }
-      }
-   }
-}
@@ -1,24 +0,0 @@
-namespace bts.SilverPine {
-   using BCG;
-
-   using System;
-
-   public class NullFilterStream : ViewStream {
-      public NullFilterStream(ViewStream parent_str, string col_name)
-      : this(parent_str, col_name, true) { }
-      public NullFilterStream(ViewStream parent_str, string col_name, bool filter_out_doublenans = false) {
-         ViewStream rs = new FilterStream(parent_str,
-         r => {
-            bool is_dbnull = r.is_null(col_name);
-            if (is_dbnull) {
-               return false;
-            }
-            if (r.schema[col_name].type_code() == TypeCode.Double) {
-               return !Double.IsNaN(r.get<double>(col_name));
-            }
-            return true;
-         });
-         mySetRealSource(rs);
-      }
-   }
-}
@@ -1,102 +1,102 @@
-// using System;
-// using System.Collections.Generic;
-// using System.Linq;
-// using System.Text;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;

-// namespace BCG
-// {
+namespace BCG
+{
   
-//     public class AverageGroupsStream : ViewStream
-//     {
-//         public AverageGroupsStream(ViewStream input, string colToAverage, string keyColumn)
-//         {
+    public class AverageGroupsStream : ViewStream
+    {
+        public AverageGroupsStream(ViewStream input, string colToAverage, string keyColumn)
+        {
            
-//             myColToAverage = colToAverage;
-//             myInput = input;
-//             myKeyColumn = keyColumn;
-//         }
+            myColToAverage = colToAverage;
+            myInput = input;
+            myKeyColumn = keyColumn;
+        }

-//         private ViewStream myInput;
-//         private string myColToAverage;
-//         private BlueTableSchema mySchema;
-//         private string myKeyColumn;
+        private ViewStream myInput;
+        private string myColToAverage;
+        private BlueTableSchema mySchema;
+        private string myKeyColumn;

-//         protected override void myOnFirstSubscriber()
-//         {
-//             myInput.onChange += new ChangeDelegate(myInput_onChange);
-//         }
-//         protected override void myOnLastSubscriber()
-//         {
-//             myInput.onChange -= myInput_onChange;
-//         }
-//         private void myInput_onChange(object newValue)
-//         {
-//             if (mySchema == null)
-//                 myBuildSchema(((DataPacket)newValue).schema);
+        protected override void myOnFirstSubscriber()
+        {
+            myInput.onChange += new ChangeDelegate(myInput_onChange);
+        }
+        protected override void myOnLastSubscriber()
+        {
+            myInput.onChange -= myInput_onChange;
+        }
+        private void myInput_onChange(object newValue)
+        {
+            if (mySchema == null)
+                myBuildSchema(((DataPacket)newValue).schema);

-//             IBlueRowCollection newRows = myEvaluate((DataPacket)newValue);
+            IBlueRowCollection newRows = myEvaluate((DataPacket)newValue);

-//             if (myLastValue == null)
-//                 myFireEvent(new DataPacket(newRows, newRows, new BlueTable(mySchema)));
-//             else
-//             {
-//                 IBlueRowCollection added, deleted;
-//                 BlueDataUtility.computeDifferences(newRows, ((DataPacket) myLastValue).newRows, out deleted, out added);
-//                 myFireEvent(new DataPacket(newRows, added, deleted));
-//             }
-//         }
+            if (myLastValue == null)
+                myFireEvent(new DataPacket(newRows, newRows, new BlueTable(mySchema)));
+            else
+            {
+                IBlueRowCollection added, deleted;
+                BlueDataUtility.computeDifferences(newRows, ((DataPacket) myLastValue).newRows, out deleted, out added);
+                myFireEvent(new DataPacket(newRows, added, deleted));
+            }
+        }

-//         private void myBuildSchema(IBlueTableSchema schema)
-//         {
-//             List<ColumnInfo> columns = new List<ColumnInfo>();
+        private void myBuildSchema(IBlueTableSchema schema)
+        {
+            List<ColumnInfo> columns = new List<ColumnInfo>();

-//             foreach (ColumnInfo ci in schema.columnInfo)
-//                 columns.Add(ci);
+            foreach (ColumnInfo ci in schema.columnInfo)
+                columns.Add(ci);

-//             columns.Add(new ColumnInfo("average_"+myColToAverage, columns.Count, typeof(double)));
-//             mySchema = new BlueTableSchema("average_of_" + schema.tableName, columns.ToArray());
-//         }
+            columns.Add(new ColumnInfo("average_"+myColToAverage, columns.Count, typeof(double)));
+            mySchema = new BlueTableSchema("average_of_" + schema.tableName, columns.ToArray());
+        }



-//         private IBlueRowCollection myEvaluate(DataPacket packet)
-//         {
+        private IBlueRowCollection myEvaluate(DataPacket packet)
+        {
         
-//             IDictionary<string, double> colToRunningTotal = new Dictionary<string, double>();
-//             IDictionary<string, int> colToCount = new Dictionary<string, int>();
-//             IDictionary<string, IBlueRow> colToLastRow = new Dictionary<string, IBlueRow>();
+            IDictionary<string, double> colToRunningTotal = new Dictionary<string, double>();
+            IDictionary<string, int> colToCount = new Dictionary<string, int>();
+            IDictionary<string, IBlueRow> colToLastRow = new Dictionary<string, IBlueRow>();

-//             foreach (IBlueRow r in packet.newRows)
-//             {
-//                 double thisVal = Convert.ToDouble(r[myColToAverage]);
+            foreach (IBlueRow r in packet.newRows)
+            {
+                double thisVal = Convert.ToDouble(r[myColToAverage]);

-//                 string keyVal = r[myKeyColumn].ToString();
-//                 if (!colToCount.ContainsKey(keyVal))
-//                 {
-//                     colToRunningTotal[keyVal] = thisVal;
-//                     colToCount[keyVal] = 1;
-//                 }
-//                 else
-//                 {
-//                     colToRunningTotal[keyVal] += thisVal;
-//                     colToCount[keyVal] += 1;
-//                 }
-//                 colToLastRow[keyVal] = r;
-//             }
+                string keyVal = r[myKeyColumn].ToString();
+                if (!colToCount.ContainsKey(keyVal))
+                {
+                    colToRunningTotal[keyVal] = thisVal;
+                    colToCount[keyVal] = 1;
+                }
+                else
+                {
+                    colToRunningTotal[keyVal] += thisVal;
+                    colToCount[keyVal] += 1;
+                }
+                colToLastRow[keyVal] = r;
+            }

-//             BlueTable result = new BlueTable(mySchema);
-//             foreach (string key in colToCount.Keys)
-//             {
-//                 object[] values = new object[mySchema.columnInfo.Length];
+            BlueTable result = new BlueTable(mySchema);
+            foreach (string key in colToCount.Keys)
+            {
+                object[] values = new object[mySchema.columnInfo.Length];

-//                 colToLastRow[key].CopyTo(values, 0);
-//                 values[values.Length - 1] = packet.newRows.Count > 0 ? colToRunningTotal[key] / packet.newRows.Count : 0.0;
-//                 result.addRow(new BlueRow(mySchema, values));
-//             }
+                colToLastRow[key].CopyTo(values, 0);
+                values[values.Length - 1] = packet.newRows.Count > 0 ? colToRunningTotal[key] / packet.newRows.Count : 0.0;
+                result.addRow(new BlueRow(mySchema, values));
+            }

-//             return result;
-//         }
+            return result;
+        }

      
-//     }
-// }
+    }
+}
@@ -1,31 +0,0 @@
-namespace bts.SilverPine {
-   using BCG;
-
-   using System;
-   using System.Collections.Generic;
-   using System.Linq;
-
-   [DataStreamUserRespresentation("Average")]
-   public class AverageNonIncGroupsStream : ViewStream {
-      public AverageNonIncGroupsStream(ViewStream parent_stream, string inp_col_name, params string[] group_by_cols) {
-         NonIncGroupByStream rs = new NonIncGroupByStream(parent_stream, () => group_by_cols,
-                                                         (g_rows) => _find_mean(inp_col_name, g_rows));
-         mySetRealSource(rs);
-      }
-
-      private List<IBlueRow> _find_mean(string inp_col_name, List<IBlueRow> g_rows) {
-         IBlueRow inp_row = g_rows.ElementAt(0);
-         IEnumerable<double> inp_vals = BlueDataUtility.get_colvalues<double>(g_rows, inp_col_name);
-         double sum = 0.0;
-         inp_vals.ToList().ForEach(d => sum += d);
-         string out_col_name = $"average_{inp_col_name}";
-         double average = sum / inp_vals.Count();
-         IBlueTableSchema schema_out = inp_row.schema.add_column(TypeCode.Double, out_col_name);
-         MutableBlueRow out_row = new MutableBlueRow(schema_out, inp_row);
-         out_row.setValue(out_col_name, average);
-         List<IBlueRow> rv = new List<IBlueRow>();
-         rv.Add(out_row);
-         return rv;
-      }
-   }
-}
@@ -1,126 +0,0 @@
-namespace bts.SilverPine {
-   using BCG;
-
-   using System;
-   using System.Collections.Generic;
-   using System.Linq;
-   //using MathNet.Numerics.Statistics;
-
-
-   public enum MultiRowOp {
-      Mean,
-      Median,
-      Mode,
-      Range,
-      StdDev,
-      Max,
-      Min,
-      First,
-      Last,
-      Sum,
-      EdgeDiff,
-      EdgeDiffPercentage,
-      //IsEdgeDiff,
-   }
-   public class MultiRowOpHelper {
-      public readonly List<Tuple<string, MultiRowOp>> operations;
-
-      public static MultiRowOpHelper parse(string text) {
-         // text is inp_col: op, inp_col: op, inp_col:op
-         List<Tuple<string, MultiRowOp>> rv = new List<Tuple<string, MultiRowOp>>();
-         string[] calcs = text.Split(',');
-         foreach (string c in calcs) {
-            string[] toks = c.Split(':');
-            string inp_col = toks[0].Trim();
-            string op_txt = toks[1].Trim();
-            MultiRowOp op = Enum.Parse<MultiRowOp>(op_txt, true);
-            rv.Add(Tuple.Create(inp_col, op));
-         }
-         return new MultiRowOpHelper(rv);
-      }
-
-      public MultiRowOpHelper(List<Tuple<string, MultiRowOp>> operations_) {
-         operations = operations_;
-      }
-
-      public static IBlueRow group_by(List<IBlueRow> rows, MultiRowOpHelper parameters) {
-         if (rows.Count() == 0) {
-            return null;
-         }
-
-         IBlueRow ir = rows.ElementAt(0);
-         IBlueTableSchema in_schema = ir.schema;
-         IBlueTableSchema out_schema = in_schema;
-         IDictionary<string, Tuple<string, MultiRowOp>> out_col_map =
-            new Dictionary<string, Tuple<string, MultiRowOp>>();
-         foreach (var t in parameters.operations) {
-            string out_col_name = $"{t.Item1}_{t.Item2}".ToLower();
-            out_schema = out_schema.add_column(TypeCode.Double, out_col_name);
-            out_col_map[out_col_name] = t;
-         }
-         MutableBlueRow or = new MutableBlueRow(out_schema, ir);
-         foreach (var kv in out_col_map) {
-            string out_col_name = kv.Key;
-            string in_col_name = kv.Value.Item1;
-            MultiRowOp op = kv.Value.Item2;
-            IEnumerable<double> col_vals = BlueDataUtility.get_colvalues<double>(rows, in_col_name);
-            double comp_res = compute_op(col_vals, op);
-            or.setValue(out_col_name, comp_res);
-         }
-         return or;
-      }
-
-      // rolling_rows are the rolling window of rows. the 0th row is the one thats gettin
-      // calculated. rows 1 - n-1 are its window rows. the schema is based on the operations.
-
-      public IBlueRow rolling_calc(List<IBlueRow> rolling_rows) {
-         if (rolling_rows.Count() == 0) {
-            return null;
-         }
-         IBlueRow ir = rolling_rows.ElementAt(0);
-         IBlueTableSchema in_schema = ir.schema;
-         IBlueTableSchema out_schema = in_schema;
-         IDictionary<string, Tuple<string, MultiRowOp>> out_col_map =
-            new Dictionary<string, Tuple<string, MultiRowOp>>();
-         foreach (var t in operations) {
-            string out_col_name = $"{t.Item1}_{t.Item2}".ToLower();
-            out_schema = out_schema.add_column(TypeCode.Double, out_col_name);
-            out_col_map[out_col_name] = t;
-         }
-         MutableBlueRow or = new MutableBlueRow(out_schema, ir);
-         foreach (var kv in out_col_map) {
-            string out_col_name = kv.Key;
-            string in_col_name = kv.Value.Item1;
-            MultiRowOp op = kv.Value.Item2;
-            IEnumerable<double> col_vals = BlueDataUtility.get_colvalues<double>(rolling_rows, in_col_name);
-            double comp_res = compute_op(col_vals, op);
-            or.setValue(out_col_name, comp_res);
-         }
-         return or;
-      }
-
-      public static double compute_op(IEnumerable<double> col_vals, MultiRowOp op) {
-         switch (op) {
-            case MultiRowOp.Max:
-               return col_vals.Max<double>();
-            case MultiRowOp.Min:
-               return col_vals.Min<double>();
-            case MultiRowOp.Mean:
-               return col_vals.Sum() / col_vals.Count();
-            case MultiRowOp.Sum:
-               return col_vals.Sum();
-            case MultiRowOp.First:
-               return col_vals.First();
-            case MultiRowOp.Last:
-               return col_vals.Last();
-            case MultiRowOp.StdDev:
-               return MathNet.Numerics.Statistics.Statistics.StandardDeviation(col_vals);
-            default:
-               // TODO
-               Log.assert(false);
-               return Double.NaN;
-         }
-      }
-
-   }
-}
@@ -10,6 +10,6 @@ namespace BCG {
        }

        public MaxStream(ViewStream stream, string outputCol)
-           : this(new AddColumnsStream(stream, "fake_col: 0"), outputCol, new string[] { "fake_col" }) { }
+           : this(new AddColumnsStream(stream, "fakeCol=0"), outputCol, new string[] { "fakeCol" }) { }
    }
 }
@@ -1,20 +0,0 @@
-namespace bts.SilverPine {
-   using BCG;
-
-   using System;
-
-   [DataStreamUserRespresentation("AggregationStream")]
-   public class AgregationStream : ViewStream {
-
-      public AgregationStream(ViewStream parent_str, string operation_txt) {
-         ViewStream rs = new NonIncGroupByStream(parent_str, String.Empty, operation_txt);
-         mySetRealSource(rs);
-      }
-
-      public AgregationStream(ViewStream parent_str, string groupby_col, string operation_txt) {
-         ViewStream rs = new NonIncGroupByStream(parent_str, groupby_col, operation_txt);
-         mySetRealSource(rs);
-      }
-   }
-
-}
@@ -1,40 +1,13 @@
 using System;
 using System.Collections.Generic;
 using System.Diagnostics;
-using System.Linq;
-using System.Text.Json;

 using bts;
-using bts.SilverPine;

-namespace BCG {
-   [DataStreamUserRespresentation("AggregateStream")]
-   public class NonIncGroupByStream : ViewStream {
-      private class DelNonIncGroupByEvaluator : INonIncGroupByEvaluator {
-         Func<string[]> _groupby_cols_func;
-         Func<List<IBlueRow>, List<IBlueRow>> _evaluate_func;
-         public DelNonIncGroupByEvaluator(Func<string[]> groupby_cols_func,
-                                          Func<List<IBlueRow>, List<IBlueRow>> evaluate_func) {
-            _groupby_cols_func = groupby_cols_func;
-            _evaluate_func = evaluate_func;
-         }
-         public IBlueRowCollection evaluate(IDictionary<int, List<IBlueRow>> grouped_rows) {
-            BlueTable rv = null;
-            foreach (var kv in grouped_rows) {
-               List<IBlueRow> g_rows = kv.Value;
-               List<IBlueRow> out_rows = _evaluate_func.Invoke(g_rows);
-               if (out_rows != null && out_rows.Count > 0) {
-                  rv = rv ?? new BlueTable(out_rows.ElementAt(0).schema);
-               }
-               rv.addRows(out_rows);
-            }
-            return rv;
-         }
-         public string[] get_group_by_cols() {
-            return _groupby_cols_func();
-         }
-
-      }
+namespace BCG
+{
+   public class NonIncGroupByStream : ViewStream
+   {
      private ViewStream _main_stream;
      private INonIncGroupByEvaluator _groupby_func;
      private BlueRowHasher _row_hasher;
@@ -42,50 +15,24 @@ namespace BCG {
      private Stopwatch _timer = new();
      private int _n_total_fires = 0, _n_expensive_fires = 0;

-      public NonIncGroupByStream(ViewStream main_stream, INonIncGroupByEvaluator groupby_func) {
+      public NonIncGroupByStream(ViewStream main_stream, INonIncGroupByEvaluator groupby_func)
+      {
         _main_stream = main_stream;
         _groupby_func = groupby_func;
         _groupby_cols = _groupby_func.get_group_by_cols();
-         _row_hasher = new BlueRowHasher(_groupby_cols == null ? 0 : _groupby_cols.Length);
+         _row_hasher = new BlueRowHasher(_groupby_cols.Length);
      }

-      public NonIncGroupByStream(ViewStream main_stream, Func<string[]> groupby_cols_func,
-                                 Func<List<IBlueRow>, List<IBlueRow>> evaluate_func)
-                                 : this(main_stream, new DelNonIncGroupByEvaluator(groupby_cols_func, evaluate_func)) { }
-
-      public NonIncGroupByStream(ViewStream parent_str, string groupby_col, string standard_group_by_text)
-      : this(parent_str, groupby_col, standard_group_by_text, String.Empty) { }
-
-      public NonIncGroupByStream(ViewStream parent_str, string groupby_col, string standard_group_by_text, string optional_sort_col) {
-         MultiRowOpHelper std_group_by = MultiRowOpHelper.parse(standard_group_by_text);
-         Func<string[]> groupby_cols_func = () => {
-            return String.IsNullOrEmpty(groupby_col) ? null : new string[] { groupby_col };
-         };
-         Func<List<IBlueRow>, List<IBlueRow>> evaluate_func = (group_rows) => {
-            if (!String.IsNullOrEmpty(optional_sort_col))
-               group_rows.Sort((r1, r2) => BlueDataUtility.compare_rows(r2, r1, optional_sort_col));
-            IBlueRow rv = MultiRowOpHelper.group_by(group_rows, std_group_by);
-            return new IBlueRow[] { rv }.ToList();
-         };
-         ViewStream rs = new NonIncGroupByStream(parent_str, groupby_cols_func, evaluate_func);
-         mySetRealSource(rs);
-      }
-      public NonIncGroupByStream(ViewStream parent_str, string standard_group_by_text) :
-      this(parent_str, String.Empty, standard_group_by_text) {
-      }
-
-
      protected override void myOnFirstSubscriber() { _main_stream.onChange += _main_stream_onchange; }

      protected override void myOnLastSubscriber() { _main_stream.onChange -= _main_stream_onchange; }

-      private void _main_stream_onchange(object nv) {
+      private void _main_stream_onchange(object nv)
+      {
         DataPacket dp = nv as DataPacket;
         ///grouping together
         _timer.Restart();
-         IDictionary<int, List<IBlueRow>> grouped_rows = (_groupby_cols == null || _groupby_cols.Length == 0) ?
-         new Dictionary<int, List<IBlueRow>>() { { 0, dp.newRows.to_list() } } :
-         BlueDataUtility.group_together(dp.newRows, _groupby_cols, ref _row_hasher);
+         IDictionary<int, List<IBlueRow>> grouped_rows = BlueDataUtility.group_together (dp.newRows, _groupby_cols, ref _row_hasher);
         _timer.Stop();
         var groupElapsedMs = _timer.ElapsedMilliseconds;
         if (groupElapsedMs >= 50) {
@@ -97,15 +44,14 @@ namespace BCG {
         IBlueRowCollection total_state = _groupby_func.evaluate(grouped_rows);
         _timer.Stop();
         var applyElapsedMs = _timer.ElapsedMilliseconds;
-         _n_total_fires++;
-         if (applyElapsedMs >= 50) {
+         _n_total_fires++;         
+         if (applyElapsedMs >= 50)
+         {
            _n_expensive_fires++;
            Log.DataFlow.info(
               $"Time to evaluate {grouped_rows.Count} groups {_timer.ElapsedMilliseconds}ms, ratio {_n_expensive_fires}/{_n_total_fires}");
-         }
-         // TODO,
-         if (total_state != null)
-            myFireEvent(new DataPacket(total_state, total_state, myLastValue == null ? new BlueTable(total_state.schema) : (myLastValue as DataPacket).newRows));
+         }         
+         myFireEvent(new DataPacket(total_state, total_state, myLastValue == null ? new BlueTable(total_state.schema) : (myLastValue as DataPacket).newRows));             
      }
   }
 }
@@ -1,16 +1,15 @@
 using bts;
-using bts.collections;

 using mbts;

 using System;
 using System.Collections.Generic;
 using System.Diagnostics;
-using System.Linq;
-using System.Security.Cryptography;

-namespace BCG {
-   public class NonIncJoinBookkeeper : IDisposable {
+namespace BCG
+{
+   public class NonIncJoinBookkeeper : IDisposable
+   {
      public Action on_new_data;
      private readonly NonIncJoinSpec _join_spec;
      private readonly int _join_position;
@@ -27,71 +26,41 @@ namespace BCG {

      private int[] _join_col_indices = null,
                     _data_col_indices = null;
-
-      public NonIncJoinBookkeeper(NonIncJoinSpec the_join_spec, int join_position, int start_index_in_output, BlueRowHasher row_hasher) {
+      
+      public NonIncJoinBookkeeper(NonIncJoinSpec the_join_spec, int join_position, int start_index_in_output, BlueRowHasher row_hasher)
+      {
         _join_spec = the_join_spec;
         _join_position = join_position;
         _start_index_in_output = start_index_in_output;
         _row_hasher = row_hasher;
         _is_left_most = _join_position == 0;
         _is_tag_along = _join_spec.join_col_names.Length == 0;
-      }
-
-      private bool _filled_missing_data = false;
-      private void _fill_missing_data(DataPacket dp) {
-         if (_filled_missing_data)
-            return;
-         Func<string[], bool> null_or_empty = (str_arr) => {
-            if (str_arr == null || str_arr.Length == 0)
-               return true;
-            foreach (string s in str_arr) {
-               if (!String.IsNullOrEmpty(s)) {
-                  return false;
-               }
-            }
-            return true;
-         };
-         Func<string[], string[]> remove_bad_cols = (col_names) => {
-            HashSet<string> schema_col_map = new HashSet<string>(dp.schema.columnNames);
-            List<string> cols_list = new List<string>(col_names);
-            cols_list.RemoveWhere(c => !schema_col_map.Contains(c));
-            return cols_list.ToArray();
-         };
-         if (null_or_empty(_join_spec.data_col_names)) {
-            _join_spec.data_col_names = dp.schema.columnNames;
-         }
-         _join_spec.data_col_names = remove_bad_cols(_join_spec.data_col_names);
-         if (null_or_empty(_join_spec.data_col_names)) {
-            _join_spec.data_col_names = dp.schema.columnNames;
-         }
-         if (null_or_empty(_join_spec.projected_col_names)) {
-            _join_spec.projected_col_names = _join_spec.data_col_names;
-         }
-      }
-
-      public void init() {
         _join_spec.stream.onChange += _on_stream_cb;
      }

-      public void _on_stream_cb(object dp_obj) {
-         //Log.Gen.info($"NonIncJoinBookkeeper._on_stream_cb dp_obj: {dp_obj}");
+      public void _on_stream_cb(object dp_obj)
+      {
         var timer = Stopwatch.StartNew();
         DataPacket dp = (DataPacket)dp_obj;
-         _fill_missing_data(dp);
-         if (_schema == null) {
+         if (_schema == null)
+         {
            _schema = dp.schema;
            _join_col_indices = BlueDataUtility.get_column_indices(_schema, _join_spec.join_col_names);
            _data_col_indices = BlueDataUtility.get_column_indices(_schema, _join_spec.data_col_names);
         }

-         if (_is_left_most) {
+         if (_is_left_most)
+         {
            _row_list.Clear();
            foreach (IBlueRow r in dp.newRows)
-               _row_list.Add(new Pair<int, IBlueRow>(_row_hasher.get_hash(r, _join_col_indices), r));
-         } else {
+               _row_list.Add(new Pair<int, IBlueRow>(_row_hasher.get_hash(r, _join_col_indices), r));            
+         }
+         else
+         {
            if (_is_tag_along)
               _solitary_row = BlueDataUtility.extractRow(dp.newRows, 0);
-            else {
+            else
+            {
               _row_map.Clear();
               foreach (IBlueRow r in dp.newRows)
                  _row_map[_row_hasher.get_hash(r, _join_col_indices)] = r;
@@ -110,25 +79,31 @@ namespace BCG {

      public bool is_ready { get { return _schema != null; } }

-      public void constribute_cols_to_schema(List<ColumnInfo> cis) {
+      public void constribute_cols_to_schema(List<ColumnInfo> cis)
+      {
         for (int i = 0; i < _join_spec.data_col_names.Length; i++)
            cis.Add(new ColumnInfo(_join_spec.projected_col_names[i], cis.Count, _schema[_join_spec.data_col_names[i]].colType));
      }

      public List<Pair<int, IBlueRow>> get_list() { return _row_list; }

-      public void copy_master_row_to_output_row(int row_index, object[] output_row) {
+      public void copy_master_row_to_output_row(int row_index, object[] output_row)
+      {
         IBlueRow this_row = _row_list[row_index].second;
         BlueDataUtility.copy_to_row(this_row, _data_col_indices, output_row, _start_index_in_output);
      }

-      public void copy_matching_row_to_output_row(int row_hash, object[] output_row) {
-         if (_is_tag_along) {
+      public void copy_matching_row_to_output_row(int row_hash, object[] output_row)
+      {
+         if (_is_tag_along)
+         {
            if (_solitary_row != null)
               BlueDataUtility.copy_to_row(_solitary_row, _data_col_indices, output_row, _start_index_in_output);
            else
               copy_dbnull_to_output_row(output_row);
-         } else {
+         }
+         else
+         {
            IBlueRow this_row = null;
            if (_row_map.TryGetValue(row_hash, out this_row))
               BlueDataUtility.copy_to_row(this_row, _data_col_indices, output_row, _start_index_in_output);
@@ -137,7 +112,8 @@ namespace BCG {
         }
      }

-      private void copy_dbnull_to_output_row(object[] output_row) {
+      private void copy_dbnull_to_output_row(object[] output_row)
+      {
         for (int i = 0; i < _data_col_indices.Length; i++)
            output_row[i + _start_index_in_output] = DBNull.Value;
      }
@@ -3,63 +3,63 @@ using System.Collections.Generic;
 using System.Linq;
 using System.Text;
 using System.Threading.Tasks;
-
 using mbts;

-namespace BCG {
-   // this a collection of data, theres no joining logic in here
-   public class NonIncJoinSpec {
+namespace BCG
+{
+   public class NonIncJoinSpec
+   {
      public ViewStream stream;
-      // join col names are the names on which the streams are joined (for equality),
-      // data col names are the names in the original stream
-      // projected col names are their new names, ie if its x:l_x, then x is in the 
-      // data_col_names and l_x in projected cols
      public string[] join_col_names, data_col_names, projected_col_names;

      public NonIncJoinSpec() { }

      public NonIncJoinSpec(ViewStream the_stream, string[] the_join_col_names, string[] the_data_col_names)
-         : this(the_stream, the_join_col_names, the_data_col_names, the_data_col_names) { }
+         : this (the_stream, the_join_col_names, the_data_col_names, the_data_col_names)
+      { }

-      public NonIncJoinSpec(ViewStream the_stream, string[] the_join_col_names, string[] the_data_col_names, string[] the_projected_col_names) {
+      public NonIncJoinSpec(ViewStream the_stream, string[] the_join_col_names, string[] the_data_col_names, string[] the_projected_col_names)
+      {
         stream = the_stream;
         join_col_names = the_join_col_names;
         data_col_names = the_data_col_names;
         projected_col_names = the_projected_col_names;
      }

-      public static NonIncJoinSpec[] from_triplet(params Triplet<ViewStream, string, string>[] join_specs) {
+      public static NonIncJoinSpec[] from_triplet(params Triplet<ViewStream, string, string>[] join_specs)
+      {
         List<NonIncJoinSpec> rv = new List<NonIncJoinSpec>();
-         foreach (Triplet<ViewStream, string, string> r in join_specs) {
-            string[] keys = String.IsNullOrEmpty(r.second) ? new string[] { } : r.second.Split(' ');
+         foreach (Triplet<ViewStream, string, string> r in join_specs)
+         {
+            string[] keys = String.IsNullOrEmpty (r.second) ? new string[] {} : r.second.Split(' ');
            string[] visible_cols = r.third.Split(' ');
            List<string> data_col_names = new List<string>(), mapped_col_names = new List<string>();
-            foreach (string s in visible_cols) {
+            foreach (string s in visible_cols)
+            {
               string[] toks = s.Split(':');
-               if (toks.Length == 1) {
+               if (toks.Length == 1)
+               {
                  data_col_names.Add(toks[0]);
                  mapped_col_names.Add(toks[0]);
-               } else if (toks.Length == 2) {
+               }
+               else if (toks.Length == 2)
+               {
                  data_col_names.Add(toks[0]);
                  mapped_col_names.Add(toks[1]);
               }
            }
-            rv.Add(new NonIncJoinSpec(r.first, keys, data_col_names.ToArray(), mapped_col_names.ToArray()));
+            rv.Add (new NonIncJoinSpec (r.first, keys, data_col_names.ToArray (), mapped_col_names.ToArray()));
         }
-         return rv.ToArray();
-      }
-
-      public override string ToString() {
-         Func<string[], string> arr_txt = (a) => {
-            string rv = "";
-            foreach (string s in a) {
-               rv += "{s} ";
-            }
-            rv = rv.Trim();
-            return rv;
-         };
-         string rv = $"stream: {stream} join_col_names: {arr_txt(join_col_names)} data_col_names: {arr_txt(data_col_names)} projected_col_names: {arr_txt(projected_col_names)}";
-         return rv;
+         return rv.ToArray ();
      }
   }
+
+   //[DataStreamUserRespresentation("ninctest")]
+   //public class NonIncJoinTestStream : ViewStream
+   //{
+   //   public NonIncJoinTestStream (string key_cols, string data_cols)
+   //   {
+   //      NonIncJoinSpec.from_triplet(new Triplet<ViewStream, string, string>(null, key_cols, data_cols));
+   //   }
+   //}
 }
@@ -6,9 +6,11 @@ using System;
 using System.Collections.Generic;
 using System.Diagnostics;

-namespace BCG {
+namespace BCG
+{
   [DataStreamUserRespresentation("fastJoin")]
-   public class NonIncJoinStream : ViewStream, BCG.Utilities.General.IWorker {
+   public class NonIncJoinStream : ViewStream, BCG.Utilities.General.IWorker
+   {
      private readonly NonIncJoinSpec[] _join_specs;
      private readonly INonIncJoinInlineMapper _output_mapper;
      private readonly BlueRowHasher _row_hasher;
@@ -26,19 +28,24 @@ namespace BCG {
               {
                  new Triplet<ViewStream, string, string> (left, left_join_cols, left_proj),
                  new Triplet<ViewStream, string, string> (right, right_join_cols, right_proj)
-               }) { }
+               })
+      { }

      public NonIncJoinStream(params Triplet<ViewStream, string, string>[] join_specs)
-         : this(NonIncJoinSpec.from_triplet(join_specs), null, 0) { }
+         : this(NonIncJoinSpec.from_triplet(join_specs), null, 0)
+      { }

      public NonIncJoinStream(ViewStreamDatabase vsd, INonIncJoinInlineMapper output_mapper, int throttles_ms,
                              params Triplet<ViewStream, string, string>[] join_specs)
-         : this(vsd, NonIncJoinSpec.from_triplet(join_specs), output_mapper, throttles_ms) { }
+         : this(vsd, NonIncJoinSpec.from_triplet(join_specs), output_mapper, throttles_ms)
+      { }

      public NonIncJoinStream(NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper output_mapper, int throttles_ms = 0)
-         : this(ViewStreamDatabase.instance, join_specs, output_mapper, throttles_ms) { }
+         : this(ViewStreamDatabase.instance, join_specs, output_mapper, throttles_ms)
+      { }

-      public NonIncJoinStream(ViewStreamDatabase vsd, NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper output_mapper, int throttles_ms = 0) {
+      public NonIncJoinStream(ViewStreamDatabase vsd, NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper output_mapper, int throttles_ms = 0)
+      {
         _join_specs = join_specs;
         _output_mapper = output_mapper;
         _row_hasher = new BlueRowHasher(_join_specs[0].join_col_names.Length);
@@ -46,11 +53,12 @@ namespace BCG {
         _vsd = vsd;
      }

-      protected override void myOnFirstSubscriber() {
+      protected override void myOnFirstSubscriber()
+      {
         int data_col_no = 1;
-         for (int jp = 0; jp < _join_specs.Length; jp++) {
+         for (int jp = 0; jp < _join_specs.Length; jp++)
+         {
            NonIncJoinBookkeeper s = new NonIncJoinBookkeeper(_join_specs[jp], jp, data_col_no, _row_hasher);
-            //Log.Gen.info($"NonIncJoinStream.myOnFirstSubscriber jp: {jp} spec: {_join_specs[jp]}");
            data_col_no += _join_specs[jp].data_col_names.Length;
            s.on_new_data += _stream_callback;
            _book_keeping_data.Add(s);
@@ -58,11 +66,10 @@ namespace BCG {
         if (_output_mapper != null)
            _output_mapper.onMapperChange += _on_mapper_change;
         _vsd.addWorker(this);
-         foreach (var bkd in _book_keeping_data)
-            bkd.init();
      }

-      protected override void myOnLastSubscriber() {
+      protected override void myOnLastSubscriber()
+      {
         foreach (NonIncJoinBookkeeper d in _book_keeping_data)
            d.Dispose();
         if (_output_mapper != null)
@@ -70,26 +77,30 @@ namespace BCG {
         _vsd.removeWorker(this);
      }

-      private void _stream_callback() {
-         //Log.Gen.info("NonIncJoinStream._stream_callback received callback");
+      private void _stream_callback()
+      {
         _is_dirty = true;
         doWork();
      }

-      private void _on_mapper_change() {
+      private void _on_mapper_change()
+      {
         _is_dirty = true;
         doWork();
      }

-      private bool _is_ready() {
+      private bool _is_ready()
+      {
         bool rv = true;
         foreach (NonIncJoinBookkeeper d in _book_keeping_data)
            rv &= d.is_ready;
         return rv;
      }

-      private void _construct_schema() {
-         if (_output_schema == null) {
+      private void _construct_schema()
+      {
+         if (_output_schema == null)
+         {
            List<ColumnInfo> cis = new List<ColumnInfo>(new ColumnInfo[] { new ColumnInfo("id", 0, typeof(int)) });
            foreach (NonIncJoinBookkeeper d in _book_keeping_data)
               d.constribute_cols_to_schema(cis);
@@ -98,23 +109,27 @@ namespace BCG {
      }

      private int _n_total_fires = 0, _n_expensive_fires = 0;
-      private void _form_and_fire() {
-         //Log.Gen.info($"NonIncJoinStream._form_and_fire _is_ready(): {_is_ready()}");
-         if (_is_ready()) {
+      private void _form_and_fire()
+      {
+         if (_is_ready())
+         {
            _construct_schema();
            var timer = Stopwatch.StartNew();
            IList<Pair<int, IBlueRow>> left_row_list = _book_keeping_data[0].get_list();
            List<object[]> output_rows = new List<object[]>();

-            for (int i = 0; i < left_row_list.Count; i++) {
+            for (int i = 0; i < left_row_list.Count; i++)
+            {
               object[] or = new object[_output_schema.columnInfo.Length];
               or[0] = i;
               output_rows.Add(or);
            }

-            for (int ji = 0; ji < _book_keeping_data.Count; ji++) {
+            for (int ji = 0; ji < _book_keeping_data.Count; ji++)
+            {
               NonIncJoinBookkeeper bk_data = _book_keeping_data[ji];
-               for (int i = 0; i < left_row_list.Count; i++) {
+               for (int i = 0; i < left_row_list.Count; i++)
+               {
                  if (ji == 0)
                     bk_data.copy_master_row_to_output_row(i, output_rows[i]);
                  else
@@ -124,20 +139,19 @@ namespace BCG {
            IBlueRowCollection dp_output_rows = _apply_map(new SequentialIDBlueTable(_output_schema, output_rows));
            timer.Stop();
            _n_total_fires++;
-            //if (timer.ElapsedMilliseconds >= 250) {
-            _n_expensive_fires++;
-            //string log_msg = $"Join and map time for {_book_keeping_data.Count} "
-            //   + $"tables {timer.Elapsed.TotalMilliseconds} msecs, ratio {_n_expensive_fires}/{_n_total_fires}";
-            //Log.Gen.info(log_msg);
-            //}
-            var dp_out = BlueDataUtility.build_dp_inc(dp_output_rows, myLastValue as DataPacket, true);
-            myFireEvent(dp_out);
-            //myFireEvent(new DataPacket(dp_output_rows, dp_output_rows, myLastValue == null ? new BlueTable(_output_schema) : (myLastValue as DataPacket).newRows));
+            if (timer.ElapsedMilliseconds >= 250) {
+               _n_expensive_fires++;
+               string log_msg = $"Join and map time for {_book_keeping_data.Count} "
+                  + $"tables {timer.Elapsed} ratio {_n_expensive_fires}/{_n_total_fires}";
+            }
+            myFireEvent(new DataPacket(dp_output_rows, dp_output_rows, myLastValue == null ? new BlueTable(_output_schema) : (myLastValue as DataPacket).newRows));
         }
      }

-      private IBlueRowCollection _apply_map(IBlueRowCollection unmapped_rows) {
-         if (_output_mapper != null) {
+      private IBlueRowCollection _apply_map(IBlueRowCollection unmapped_rows)
+      {
+         if (_output_mapper != null)
+         {
            IBlueRowCollection output_rows;
            var timer = Stopwatch.StartNew();
            output_rows = _output_mapper.map_rows(unmapped_rows);
@@ -146,14 +160,17 @@ namespace BCG {
               Log.Gen.info($"Time to map {unmapped_rows.Count} rows into {output_rows.Count} rows {timer.Elapsed}");
            }
            return output_rows;
-         } else {
+         }
+         else
+         {
            return unmapped_rows;
         }
      }

-      public void doWork() {
-         //Log.Gen.info($"NonIncJoinStream.doWork _last_fire_time: {_last_fire_time} _is_dirty: {_is_dirty}");
-         if (DateTime.Now - _last_fire_time >= _throttle_period && _is_dirty) {
+      public void doWork()
+      {
+         if (DateTime.Now - _last_fire_time >= _throttle_period && _is_dirty)
+         {
            _is_dirty = false;
            _last_fire_time = DateTime.Now;
            _form_and_fire();
@@ -1,68 +0,0 @@
-namespace bts {
-   using BCG;
-   using BCG.Utilities.General;
-
-   using System;
-   using System.Diagnostics;
-
-   using ZstdSharp.Unsafe;
-
-   // functon that joins the 2 rows and returns the outputput row. The function
-   // can return a null which means the 2 rows dont need to be joined
-   using JoinFunc = System.Func<BCG.IBlueRow, BCG.IBlueRow, BCG.IBlueRow>;
-   public class ProximityJoinStream : ViewStream, IWorker {
-      private ViewStream _lvs, _rvs;
-      private JoinFunc _join_func;
-      private bool _dirty = false;
-      private Stopwatch _stopwatch = new Stopwatch();
-      private ViewStreamDatabase _vsd = ViewStreamDatabase.instance;
-      public ProximityJoinStream(ViewStream lvs, ViewStream rvs, JoinFunc join_func) {
-         _lvs = lvs;
-         _rvs = rvs;
-         _join_func = join_func;
-      }
-
-      protected override void myOnFirstSubscriber() {
-         _lvs.onChange += _handle_data;
-         _rvs.onChange += _handle_data;
-         _vsd.addWorker(this);
-      }
-
-      protected override void myOnLastSubscriber() {
-         _lvs.onChange -= _handle_data;
-         _rvs.onChange -= _handle_data;
-         _vsd.removeWorker(this);
-      }
-
-      private void _handle_data(object dp_obj) {
-         _dirty = true;
-         doWork();
-      }
-
-      public void doWork() {
-         Func<bool> all_data_revcd = () => {
-            return _lvs.rows != null && _rvs.rows != null;
-         };
-         _stopwatch.Start();
-         if (_dirty && all_data_revcd()) {
-            _dirty = false;
-            BlueTable rv = null;
-            foreach (IBlueRow lr in _lvs.rows) {
-               foreach (IBlueRow rr in _rvs.rows) {
-                  IBlueRow resr = _join_func(lr, rr);
-                  if (resr == null)
-                     continue;
-                  rv = rv ?? new BlueTable(resr.schema);
-                  rv.addRow(resr);
-               }
-            }
-            _stopwatch.Stop();
-            Log.Gen.info($"ProximityJoinStream left_count: {_lvs.rows.Count} right_count: {_rvs.rows.Count} result_count: {rv.Count} time: {_stopwatch.ElapsedMilliseconds} msecs");
-            var dp = BlueDataUtility.build_dp_inc(rv, myLastValue as DataPacket);
-            myFireEvent(dp);
-         }
-
-      }
-
-   }
-}
@@ -1,47 +0,0 @@
-namespace bts.SilverPine {
-   using BCG;
-
-   using System;
-   using System.CodeDom;
-   using System.Configuration;
-   using System.Reactive.Disposables;
-
-   public class TimeProximityJoinStream : ViewStream {
-      private int _id = 0;
-
-      public TimeProximityJoinStream(ViewStream ls, string l_time_col, ViewStream rs, string r_time_col,
-                                       string pre_join_period, string post_join_period) :
-            this(ls, l_time_col, rs, r_time_col, TimeBasedFilter.parse_duration(pre_join_period),
-            TimeBasedFilter.parse_duration(post_join_period)) { }
-
-
-      public TimeProximityJoinStream(ViewStream ls, string l_time_col, ViewStream rs, string r_time_col,
-                                       TimeSpan pre_join_period, TimeSpan post_join_period) {
-         Func<IBlueRow, IBlueRow, IBlueRow> create_row = (lr, rr) => {
-            var id = 0;
-            lock (this) {
-               id = _id++;
-            }
-            IBlueRow rv = BlueDataUtility.create_composite_row(lr, rr, id);
-            return rv;
-         };
-         Func<IBlueRow, IBlueRow, IBlueRow> join_func = (lr, rr) => {
-            DateTime ldt = lr.get_datetime(l_time_col);
-            DateTime rdt = rr.get_datetime(r_time_col);
-            TimeSpan diff = ldt - rdt;
-            if (ldt < rdt) {
-               if (diff.Duration() > pre_join_period) {
-                  return null;
-               }
-            } else {
-               if (diff > post_join_period)
-                  return null;
-            }
-            return create_row(lr, rr);
-         };
-
-         ViewStream rstr = new ProximityJoinStream(ls, rs, join_func);
-         mySetRealSource(rstr);
-      }
-   }
-}
@@ -2,13 +2,9 @@ using System;
 using System.Collections;
 using System.Collections.Generic;

-using Amazon.Runtime.Internal.Util;
-
 using BCG;
 using BCG.Utilities;

-using bts;
-
 namespace BCG {
   public class LiteralStream : ViewStream {

@@ -26,34 +22,6 @@ namespace BCG {
         return rv;
      }

-      //literal("table1", "x:123, y:1.001, z:abc;x:123, y:1.001, z:abc")
-      public LiteralStream(string table_name, string rows) {
-         Func<string, int, IBlueRow> get_row = (json_row, id) => {
-            Dictionary<string, object> map = new Dictionary<string, object>();
-            string[] toks = json_row.Split(',', StringSplitOptions.TrimEntries);
-            //Logger.assert(col_vals.Length == 2);
-            foreach (string t in toks) {
-               var col_vals = t.Split(':', StringSplitOptions.TrimEntries);
-               string col = col_vals[0];
-               string val = col_vals[1];
-               map.Add(col, val);
-            }
-            map["id"] = id;
-            return new BlueRow(map);
-         };
-         string[] row_arr = rows.Split(';', StringSplitOptions.TrimEntries);
-         BlueTable table = null;
-         foreach (string json_string in row_arr) {
-            IBlueRow r = //BlueDataUtility.convert_json_to_row(json_string, table == null ? 0 : table.Count);
-            get_row(json_string, table == null ? 0 : table.Count);
-            table = table ?? new BlueTable(r.schema);
-            table.addRow(r);
-         }
-         ViewStream str = new BlueTableStream(table);
-         str = new SchemaGuesserStream(str);
-         mySetRealSource(str);
-      }
-
      public LiteralStream(string tableName, string schema, string value) {
         string[] columns = schema.Split(' ');

@@ -87,7 +55,7 @@ namespace BCG {
         mySetRealSource(new BlueRowCollectionStaticStream(t));
      }

-      // public LiteralStream(string schema, string value) : this("literal", schema, value) { }
+      public LiteralStream(string schema, string value) : this("literal", schema, value) { }

      // public LiteralStream(string value) {

@@ -1,74 +0,0 @@
-using BCG;
-
-using MongoDB.Driver.Core.WireProtocol.Messages;
-
-using System;
-using System.Collections.Generic;
-using System.Linq;
-
-// the first arg is all the rows, the second arg is the index of the row that needs mapping, and 
-// the func returns the mapped row (ie mapped version of the index row)
-using GlobalMap = System.Func<System.Collections.Generic.List<BCG.IBlueRow>, uint, BCG.IBlueRow>;
-
-namespace bts {
-   // In GlobalMap, each row is mapped with the full view of all rows, ie the map functions for a row
-   // can use the state fo all rows.
-   public class GlobalMapStream : ViewStream {
-      private ViewStream _base_stream;
-      private readonly string _sort_col_name, _group_by_col;
-      private GlobalMap _map_func;
-      public GlobalMapStream(ViewStream base_stream, string sort_col_name, string group_by_col, GlobalMap map_func) {
-         _base_stream = base_stream;
-         _sort_col_name = sort_col_name;
-         _group_by_col = group_by_col;
-         _map_func = map_func;
-      }
-
-      public GlobalMapStream(ViewStream base_stream, string sort_col_name, GlobalMap map_func) :
-      this(base_stream, sort_col_name, null, map_func) {
-      }
-
-      protected override void myOnFirstSubscriber() {
-         _base_stream.onChange += _handle_stream_update;
-      }
-
-      protected override void myOnLastSubscriber() {
-         _base_stream.onChange -= _handle_stream_update;
-      }
-
-      void _handle_stream_update(object dp_obj) {
-         DataPacket dp = dp_obj as DataPacket;
-         IDictionary<int, List<IBlueRow>> groups = !String.IsNullOrEmpty(_group_by_col) ?
-         BlueDataUtility.group_together(dp.newRows, new string[] { _group_by_col })
-         :
-         new Dictionary<int, List<IBlueRow>>() { { 0, dp.newRows.to_list() } };
-         BlueTable out_table = null;
-         foreach (var kv in groups) {
-            List<IBlueRow> group_rows = kv.Value;
-            group_rows.Sort((r1, r2) => BlueDataUtility.compare_rows(r2, r1, _sort_col_name));
-            uint i = 0;
-            foreach (IBlueRow r in group_rows) {
-               IBlueRow nr = _map_func(group_rows, i++);
-               out_table = out_table ?? new BlueTable(nr.schema);
-               out_table.addRow(nr);
-            }
-         }
-
-
-         // List<IBlueRow> rows = dp.newRows.to_ienum().ToList();
-         // rows.Sort((r1, r2) => BlueDataUtility.compare_rows(r1, r2, _sort_col_name));
-         // uint i = 0;
-
-         // BlueTable out_table = null;
-         // foreach (IBlueRow r in rows) {
-         //    IBlueRow nr = _map_func(rows, i++);
-         //    out_table = out_table ?? new BlueTable(nr.schema);
-         //    out_table.addRow(nr);
-         // }
-         DataPacket dp_out = BlueDataUtility.build_dp_inc(out_table, myLastValue as DataPacket);
-         myFireEvent(dp_out);
-      }
-   }
-
-
-}
@@ -13,8 +13,9 @@ namespace BCG {
   /// desired row ie newRow = f(row), where f is the mapping fucntion. Note that the id 
   /// is preserved across the rows, ie row["id"] = f(row)["id"]
   /// </summary>
-   [DataStreamUserRespresentation(false)]
-   public class MapStream : MethodDrivenStream {
+   [DataStreamUserRespresentation (false)]
+   public class MapStream : MethodDrivenStream
+   {
      protected IStreamMapper myMapper;
      protected IChangeableStreamMapper myChangeableMapper = null;
      private bool myUseNewMap = Configuration.get<DataStreamConfig>().UseNewMap;
@@ -1,14 +0,0 @@
-namespace bts.SilverPine
-{
-   using BCG;
-
-   [DataStreamUserRespresentation("RollingComputationStream")]
-   public class RollingComputationStream : ViewStream
-   {
-      private ViewStream _parent_str;
-      public RollingComputationStream(ViewStream parent_str, string operation_txt, string roll_column, uint window_sz)
-      {
-
-      }
-   }
-}
@@ -1,10 +0,0 @@
-namespace bts.SilverPine {
-   using BCG;
-
-
-   [DataStreamUserRespresentation("RollingComputationStream")]
-   public class RollingComputationStream : ViewStream {
-      public RollingComputationStream(ViewStream parent_str, string operation_txt, string roll_column, uint window_sz) {
-      }
-   }
-}
@@ -158,6 +158,10 @@ namespace bts
         myFireEvent(dp_out);
      }

+      private string _get_next_id() {
+         return _table.Count.ToString();
+      }
+
      private string _get_next_id()
      {
         return _table.Count.ToString();
@@ -1,8 +1,9 @@
+using Google.Protobuf.WellKnownTypes;
+
 using System;
+using BCG.Utilities.General;

-using BCG;
-
-namespace bts.SilverPine {
+namespace BCG {
   // Keeps adding changing rows to the stream, hence preserving the history
   // and creating a time series, for example this can be used to make bbo updates
   // into a time series of prices/qty 
@@ -40,6 +41,7 @@ namespace bts.SilverPine {
         }
         _table.addRows(added_rows);
         myFireEvent(new DataPacket(_table, added_rows, new BlueTable(_table.schema)));
+
      }
   }

@@ -108,7 +108,6 @@ namespace bts {

   }

-   [DataStreamUserRespresentation("Project")]
   public class NewProjectStream : ViewStream, IStreamMapper {

      private ViewStream _stream;
@@ -138,24 +137,10 @@ namespace bts {
         HashSet<string> removed_cols = new HashSet<string>();
         Dictionary<string, string> mapped_to_orig_map = new Dictionary<string, string>();
         Dictionary<string, string> orig_to_mapped = new Dictionary<string, string>();
-         // by default every col will show up, so if we have the projection as (a:a1)
-         // it will convert a to a1, 
-         // examples: if a schema has cols a b c
-         // project(parent_str, "a:aa ") -> aa b c
-         // project(parent_str, "a:aa !c") -> aa b
-         // project(parent_str, "a:aa !") -> aa
-         // project(parent_str, "a:aa b = (aa*aa)!") -> aa b
-         bool hide_all = false;
         foreach (string p_tok in project_toks) {
            if (p_tok.StartsWith("!")) {
-               if (p_tok.Length == 1) {
-                  // if it is a "blanket "!"=> hide all cols except the one that
-                  // are stated explicitly
-                  hide_all = true;
-               } else {
-                  string removed_col = p_tok.Substring(1);
-                  removed_cols.Add(removed_col);
-               }
+               string removed_col = p_tok.Substring(1);
+               removed_cols.Add(removed_col);
            } else if (p_tok.Contains(":")) {
               string[] map_toks = p_tok.Split(":");
               string orig_col = map_toks[0];
@@ -170,11 +155,12 @@ namespace bts {
               }
            }
         }
-
         // make the new schema
         List<ColumnInfo> new_cols = new List<ColumnInfo>();
         foreach (ColumnInfo ci in input_schema.columnInfo) {
            if (ci.colName == "id") {
+               // if there's a hidden col, other columns will show by default, but if cols are being
+               // projected epxlicitely, than others wil be hidden
               ColumnInfo nc = new ColumnInfo(ci.colName, new_cols.Count, ci.colType);
               new_cols.Add(nc);
               continue;
@@ -186,13 +172,12 @@ namespace bts {
               ColumnInfo nc = new ColumnInfo(mapped_name, new_cols.Count, ci.colType);
               new_cols.Add(nc);
            } else {
-               if (hide_all)
-                  continue;
-               //if (ci.colName == "id" || removed_cols.Count > 0) {
-               // unless columns are  explicitely hidden, all columns show up
-               ColumnInfo nc = new ColumnInfo(ci.colName, new_cols.Count, ci.colType);
-               new_cols.Add(nc);
-               //}
+               if (ci.colName == "id" || removed_cols.Count > 0) {
+                  // if there's a hidden col, other columns will show by default, but if cols are being
+                  // projected epxlicitely, than others wil be hidden
+                  ColumnInfo nc = new ColumnInfo(ci.colName, new_cols.Count, ci.colType);
+                  new_cols.Add(nc);
+               }
            }
         }
         IBlueTableSchema new_schema = new BlueTableSchema(input_schema.tableName, new_cols.ToArray());
@@ -1,122 +0,0 @@
-using System;
-using System.Data;
-using System.Data.SQLite;
-using System.Diagnostics;
-using System.Collections;
-
-
-using BCG;
-using System.Linq;
-using System.Data.Entity;
-using System.Collections.Generic;
-using System.Data.SQLite;
-
-
-namespace bts.SilverPine {
-
-   public static class SQLiteHelper {
-      public static string location = "C:\\Users\\m_tah\\development\\data\\";
-      public static string db_name = "stream_data.db";
-
-      public static DataTable execute_select_query(string connection_string, string query) {
-         // Create a new DataTable to store the results
-         DataTable dataTable = new DataTable();
-
-         // Create and open a SQLite connection using the provided connection string
-         using (SQLiteConnection connection = new SQLiteConnection(connection_string)) {
-            try {
-               connection.Open();
-
-               // Create a SQLite command with the provided query
-               using (SQLiteCommand command = new SQLiteCommand(query, connection)) {
-                  // Use SQLiteDataAdapter to fill the DataTable
-                  using (SQLiteDataAdapter adapter = new SQLiteDataAdapter(command)) {
-                     adapter.Fill(dataTable);
-                  }
-               }
-            } catch (Exception ex) {
-               Console.WriteLine("An error occurred: " + ex.Message);
-            }
-         }
-         // Return the populated DataTable
-         return dataTable;
-      }
-
-      public static List<Dictionary<string, object>> ExecuteBulkRead(string connectionString, string query) {
-         var result = new List<Dictionary<string, object>>();
-
-         using (var connection = new SQLiteConnection(connectionString)) {
-            connection.Open();
-
-            // Begin a transaction to optimize read speed
-            using (var transaction = connection.BeginTransaction())
-            using (var command = new SQLiteCommand(query, connection, transaction))
-            using (var reader = command.ExecuteReader()) {
-               while (reader.Read()) {
-                  var row = new Dictionary<string, object>();
-
-                  for (int i = 0; i < reader.FieldCount; i++) {
-                     string columnName = reader.GetName(i);
-                     object columnValue = reader.IsDBNull(i) ? null : reader.GetValue(i);
-                     row[columnName] = columnValue;
-                  }
-
-                  result.Add(row);
-               }
-
-               // Commit the transaction
-               transaction.Commit();
-            }
-         }
-
-         return result;
-      }
-
-      public static List<Dictionary<string, object>> get_table(string connectionString, string query) {
-         var result = new List<Dictionary<string, object>>();
-
-         using (var connection = new SQLiteConnection(connectionString)) {
-            try {
-               connection.Open();
-
-               using (var command = new SQLiteCommand(query, connection))
-               using (var reader = command.ExecuteReader()) {
-                  while (reader.Read()) {
-                     var row = new Dictionary<string, object>();
-
-                     for (int i = 0; i < reader.FieldCount; i++) {
-                        string columnName = reader.GetName(i);
-                        object columnValue = reader.IsDBNull(i) ? null : reader.GetValue(i);
-                        row[columnName] = columnValue;
-                     }
-
-                     result.Add(row);
-                  }
-               }
-            } catch (Exception ex) {
-               Console.WriteLine("An error occurred: " + ex.Message);
-            }
-         }
-         return result;
-      }
-
-      public static IEnumerable<Dictionary<string, object>> get_table_2(string connectionString, string query) {
-         using (var connection = new SQLiteConnection(connectionString)) {
-            connection.Open();
-            using (var command = new SQLiteCommand(query, connection))
-            using (var reader = command.ExecuteReader()) {
-               while (reader.Read()) {
-                  var row = new Dictionary<string, object>();
-                  for (int i = 0; i < reader.FieldCount; i++) {
-                     string columnName = reader.GetName(i);
-                     object columnValue = reader.IsDBNull(i) ? null : reader.GetValue(i);
-                     row[columnName] = columnValue;
-                  }
-                  yield return row;
-               }
-            }
-         }
-      }
-   }
-}
-
@@ -1,73 +0,0 @@
-namespace bts.SilverPine {
-   using System;
-   using System.Data;
-   using System.Data.SQLite;
-   using System.Diagnostics;
-   using System.Collections;
-
-
-   using BCG;
-
-   [DataStreamUserRespresentation("SQLite")]
-   public class SQLiteStream : ViewStream {
-      private readonly string _connection_string, _query;
-      private Stopwatch _stopwatch = new Stopwatch();
-
-      private SQLiteStream(string connection_string, string query, bool noop) {
-         _connection_string = connection_string;
-         _query = query;
-      }
-
-      public SQLiteStream(string db_name, string query)
-      :
-      this($"Data Source={SQLiteHelper.location}{db_name};Version=3", query, true) { }
-
-      public SQLiteStream(string query)
-      :
-      this(SQLiteHelper.db_name, query) {
-      }
-
-
-      private void _read_from_sqlite_and_fire() {
-         _stopwatch.Start();
-         DataTable dt = SQLiteHelper.execute_select_query(_connection_string, _query);
-         _stopwatch.Stop();
-         Log.Gen.info($"SQLiteStream.ExecuteSelectQuery time to read db: {_stopwatch.ElapsedMilliseconds} msecs");
-         _stopwatch.Restart();
-         #region perf measures
-         // SQLiteHelper.get_table(_connection_string, _query);
-         // _stopwatch.Stop();
-         // Log.Gen.info($"SQLiteStream.get_table time to read db: {_stopwatch.ElapsedMilliseconds} msecs");
-         // _stopwatch.Restart();
-         // SQLiteHelper.ExecuteBulkRead(_connection_string, _query);
-         // _stopwatch.Stop();
-         // Log.Gen.info($"SQLiteStream.ExecuteBulkRead time to read db: {_stopwatch.ElapsedMilliseconds} msecs");
-         // _stopwatch.Restart();
-         // SQLiteHelper.get_table_2(_connection_string, _query);
-         // _stopwatch.Stop();
-         // Log.Gen.info($"SQLiteStream.get_table_2 time to read db: {_stopwatch.ElapsedMilliseconds} msecs");
-         // _stopwatch.Restart();
-         #endregion
-         IBlueRowCollection rows = BCG.BlueDataUtility.toIBlueRowCollection(dt);
-         //IBlueRowCollection rows = new DataTableWrapperTable(dt);
-         _stopwatch.Stop();
-         Log.Gen.info($"SQLiteStream._read_from_sqlite_and_fire time to load into IBlueRowCollection: {_stopwatch.ElapsedMilliseconds} msecs");
-         DataPacket dp = BlueDataUtility.build_dp_inc(rows, myLastValue as DataPacket);
-         myFireEvent(dp);
-      }
-
-      protected override void myOnFirstSubscriber() {
-         _read_from_sqlite_and_fire();
-      }
-
-      protected override void myOnLastSubscriber() {
-         // this value will be cleared by the base class
-         // but in this particular stream this behaviour
-         // is not usefull since any new subscribers 
-         // would force the query to be recreated
-         //myLastDP = myLastValue;
-      }
-
-   }
-
-}
@@ -3,15 +3,12 @@ using System.Collections.Generic;
 using System.Text;
 using System.IO;
 using BCG.Utilities;
-using System.Diagnostics;
-using Microsoft.Data.Analysis;
-using Parquet;
-using Parquet.Data;

-
-namespace BCG {
+namespace BCG
+{
   [DataStreamUserRespresentation("CSVSaver")]
-   public class CSVSaverStream : ViewStream, ISideEffectStream {
+   public class CSVSaverStream : ViewStream, ISideEffectStream
+   {
      private IBlueTableSchema myResultSchema = new BlueTableSchema("CSVResult", new ColumnInfo("id", 0, typeof(int)),
         new ColumnInfo("fileName", 1, typeof(string)),
         new ColumnInfo("nWrites", 2, typeof(int)),
@@ -23,9 +20,11 @@ namespace BCG {
      private int myNWrites = 0, myNMaxWrites = Int32.MaxValue;

      public CSVSaverStream(ViewStream vs, string fullyQualifiedFileName)
-         : this(vs, fullyQualifiedFileName, true, Int32.MaxValue, false) { }
+         : this(vs, fullyQualifiedFileName, true, Int32.MaxValue, false)
+      { }

-      public CSVSaverStream(ViewStream vs, string fullyQualifiedFileName, bool overwrite, int nMaxWrites, bool fireResultSnapshot) {
+      public CSVSaverStream(ViewStream vs, string fullyQualifiedFileName, bool overwrite, int nMaxWrites, bool fireResultSnapshot)
+      {
         myStream = vs;
         myFullyQualifiedFileName = fullyQualifiedFileName;
         myOverwrite = overwrite;
@@ -37,14 +36,18 @@ namespace BCG {

      protected override void myOnLastSubscriber() { myStream.onChange -= new ChangeDelegate(myHandleChanges); }

-      private void myHandleChanges(object newValue) {
+      private void myHandleChanges(object newValue)
+      {
         IBlueRowCollection totalState = (newValue as DataPacket).newRows;
-         if (totalState.Count > 0 && myNWrites < myNMaxWrites) {
+         if (totalState.Count > 0 && myNWrites < myNMaxWrites)
+         {
            myNWrites++;
            int i = 0;
            StreamWriter writer = new StreamWriter(new FileStream(myGetNewFileName(), FileMode.Create, FileAccess.Write));
-            foreach (IBlueRow r in totalState) {
-               if (i++ == 0) {
+            foreach (IBlueRow r in totalState)
+            {
+               if (i++ == 0)
+               {
                  string header = String.Empty;
                  foreach (string col in r.schema.columnNames)
                     if (col != "id")
@@ -56,17 +59,20 @@ namespace BCG {
               writer.Flush();
            }
            writer.Close();
-            if (myFireResultSnapshot) {
+            if (myFireResultSnapshot)
+            {
               IBlueRow nr = new BlueRow(myResultSchema, 0, myFullyQualifiedFileName, myNWrites, totalState.Count);
               BlueTable nRows = new BlueTable(myResultSchema);
               nRows.addRow(nr);
               myFireEvent(new DataPacket(nRows, nRows, new BlueTable(myLastValue == null ? new BlueTable(myResultSchema) : (myLastValue as DataPacket).newRows)));
-            } else
+            }
+            else
               myFireEvent(newValue);
         }
      }

-      private string myGetNewFileName() {
+      private string myGetNewFileName()
+      {
         string[] tags = myFullyQualifiedFileName.Split('.');
         if (myOverwrite)
            return tags[0] + ".csv";
@@ -74,11 +80,14 @@ namespace BCG {
            return tags[0] + "_" + myFormateDateTime(DateTime.Now) + ".csv";
      }

-      private static string myGetCSVRow(IBlueRow r) {
+      private static string myGetCSVRow(IBlueRow r)
+      {
         string rv = String.Empty;
         int i = 0;
-         foreach (string col in r.schema.columnNames) {
-            if (col != "id") {
+         foreach (string col in r.schema.columnNames)
+         {
+            if (col != "id")
+            {
               string colText = String.Empty;
               if (r[col] != null && r[col] != DBNull.Value)
                  colText = r[col].ToString();
@@ -90,7 +99,8 @@ namespace BCG {
         return rv;
      }

-      private string myFormateDateTime(DateTime dt) {
+      private string myFormateDateTime(DateTime dt)
+      {
         return
         string.Format("{0}-{1}-{2}-{3}-{4}-{5}-{6}",
                        dt.Year.ToString("0000"),
@@ -104,49 +114,52 @@ namespace BCG {
   }

   [DataStreamUserRespresentation("CSVReader")]
-   public class CSVReaderStream : ViewStream {
-      private readonly string _file_address;
-      Stopwatch _stopwatch = new Stopwatch();
-      public CSVReaderStream(string fileAddress) {
-         _file_address = fileAddress;
+   public class CSVReaderStream : ViewStream
+   {
+      private IBlueRowCollection myRows;
+
+      public CSVReaderStream(string fileAddress)
+      {
+         myRows = BlueDataUtility.createRowsFromCSVFileWithoutTypes(fileAddress);
      }

-      protected override void myOnFirstSubscriber() {
-         _stopwatch.Start();
-         DataFrame df = DataFrame.LoadCsv(_file_address);
-         _stopwatch.Stop();
-         bts.Log.Gen.info($"CSVReaderStream.myOnFirstSubscriber stream csv load col_count: {df.Columns.Count} rows, time: {_stopwatch.ElapsedMilliseconds} msecs");
-         _stopwatch.Start();
-         IBlueRowCollection rows = BlueDataUtility.createRowsFromCSVFileWithoutTypes(_file_address);
-         _stopwatch.Stop();
-         bts.Log.Gen.info($"CSVReaderStream.myOnFirstSubscriber stream csv load count: {rows.Count} rows, time: {_stopwatch.ElapsedMilliseconds} msecs");
-         DataPacket dp_out = BlueDataUtility.build_dp_inc(rows, myLastValue as DataPacket);
-         myFireEvent(dp_out);
+      protected override void myOnFirstSubscriber()
+      {
+         myFireEvent(new DataPacket(myRows, myRows, new BlueTable(myRows.schema)));
      }
   }

-   [DataStreamUserRespresentation("CSVReaderStreamMapper")]
-   public class CSVReaderStreamMapperStream : ViewStream {
-      public CSVReaderStreamMapperStream(ViewStream stream, string output_schema) {
+   [DataStreamUserRespresentation ("CSVReaderStreamMapper")]
+   public class CSVReaderStreamMapperStream : ViewStream
+   {
+      public CSVReaderStreamMapperStream(ViewStream stream, string output_schema)
+      {
         mySetRealSource(new MapStream(stream, new CSVReaderStreamMapper(output_schema)));
      }
   }

-   public class CSVReaderStreamMapper : IStreamMapper {
+   public class CSVReaderStreamMapper : IStreamMapper
+   {
      private IBlueTableSchema mySchema;
      private readonly string[] myOutputSchemaStringArray;

-      public CSVReaderStreamMapper(string outputSchema) {
+      public CSVReaderStreamMapper(string outputSchema)
+      {
         myOutputSchemaStringArray = outputSchema.Split(' ');
      }

-      public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema) {
-         if (mySchema == null) {
+      public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema)
+      {
+         if (mySchema == null)
+         {
            ColumnInfo[] ci = inputSchema.columnInfo;
-            foreach (string col in myOutputSchemaStringArray) {
+            foreach (string col in myOutputSchemaStringArray)
+            {
               string colName = col.Substring(0, col.IndexOf('\\')).ToLower();
-               for (int i = 0; i < ci.Length; i++) {
-                  if (ci[i].colName.ToLower().Equals(colName)) {
+               for (int i = 0; i < ci.Length; i++)
+               {
+                  if (ci[i].colName.ToLower().Equals(colName))
+                  {
                     Type t = myGetType(col.Substring(col.IndexOf('\\') + 1));
                     ci[i] = new ColumnInfo(ci[i].colName, i, t);
                     break;
@@ -158,52 +171,74 @@ namespace BCG {
         return mySchema;
      }

-      public IBlueRow map(IBlueRow row) {
+      public IBlueRow map(IBlueRow row)
+      {
         object[] vals = new object[row.schema.columnInfo.Length];
         row.Values.CopyTo(vals, 0);
-         for (int i = 0; i < mySchema.columnInfo.Length; i++) {
+         for (int i = 0; i < mySchema.columnInfo.Length; i++)
+         {
            Type t = mySchema.columnInfo[i].colType;
-            try {
+            try
+            {
               vals[i] = System.Convert.ChangeType(vals[i], mySchema.columnInfo[i].colType, System.Globalization.CultureInfo.InvariantCulture);
-            } catch {
-               if (t.Equals(typeof(Int16))) {
+            }
+            catch
+            {
+               if (t.Equals(typeof(Int16)))
+               {
                  Int16 result;
-                  if (Int16.TryParse((string)vals[i], out result)) {
+                  if (Int16.TryParse((string)vals[i], out result))
+                  {
                     vals[i] = result;
                     continue;
                  }
                  double r2;
-                  if (Double.TryParse((string)vals[i], out r2)) {
-                     try {
+                  if (Double.TryParse((string)vals[i], out r2))
+                  {
+                     try
+                     {
                        vals[i] = (Int16)r2;
                        continue;
-                     } catch { }
+                     }
+                     catch { }
                  }
-               } else if (t.Equals(typeof(Int32))) {
+               }
+               else if (t.Equals(typeof(Int32)))
+               {
                  Int32 result;
-                  if (Int32.TryParse((string)vals[i], out result)) {
+                  if (Int32.TryParse((string)vals[i], out result))
+                  {
                     vals[i] = result;
                     continue;
                  }
                  double r2;
-                  if (Double.TryParse((string)vals[i], out r2)) {
-                     try {
+                  if (Double.TryParse((string)vals[i], out r2))
+                  {
+                     try
+                     {
                        vals[i] = (Int32)r2;
                        continue;
-                     } catch { }
+                     }
+                     catch { }
                  }
-               } else if (t.Equals(typeof(Int64))) {
+               }
+               else if (t.Equals(typeof(Int64)))
+               {
                  Int64 result;
-                  if (Int64.TryParse((string)vals[i], out result)) {
+                  if (Int64.TryParse((string)vals[i], out result))
+                  {
                     vals[i] = result;
                     continue;
                  }
                  double r2;
-                  if (Double.TryParse((string)vals[i], out r2)) {
-                     try {
+                  if (Double.TryParse((string)vals[i], out r2))
+                  {
+                     try
+                     {
                        vals[i] = (Int64)r2;
                        continue;
-                     } catch { }
+                     }
+                     catch { }
                  }
               }
               vals[i] = DBNull.Value;
@@ -212,7 +247,8 @@ namespace BCG {
         return new BlueRow(mySchema, vals);
      }

-      private Type myGetType(string type) {
+      private Type myGetType(string type)
+      {
         type = type.ToLower();
         if (type.Contains("int"))
            return typeof(int);
@@ -1,41 +1,31 @@
 using System;
 using System.Collections;
 using System.Reflection;
-
 using BCG.Utilities;

-namespace BCG {
-   // if a table needs to be fired, it can be wrapped around this
-   public class BlueTableStream : ViewStream {
-      private IBlueRowCollection _rows;
-      public BlueTableStream(IBlueRowCollection rows) {
-         _rows = rows;
+namespace BCG
+{
+   [DataStreamUserRespresentation (false)]
+   public class BlueRowCollectionStaticStream : ViewStream
+   {
+      public BlueRowCollectionStaticStream () {}
+
+      public BlueRowCollectionStaticStream (IBlueRowCollection rows) {loadCollection (rows);}
+
+      public void loadCollection (IBlueRowCollection rows)
+      {
+         myFireEvent (new DataPacket (rows, rows, myLastValue != null ?
+                                                   new BlueTable((myLastValue as DataPacket).newRows) : new BlueTable (rows.schema)));
      }

-      protected override void myOnFirstSubscriber() {
-         var dp_out = BlueDataUtility.build_dp_inc(_rows, myLastValue as DataPacket);
-         myFireEvent(dp_out);
-      }
-   }
-
-   [DataStreamUserRespresentation(false)]
-   public class BlueRowCollectionStaticStream : ViewStream {
-      public BlueRowCollectionStaticStream() { }
-
-      public BlueRowCollectionStaticStream(IBlueRowCollection rows) { loadCollection(rows); }
-
-      public void loadCollection(IBlueRowCollection rows) {
-         myFireEvent(new DataPacket(rows, rows, myLastValue != null ?
-                                                   new BlueTable((myLastValue as DataPacket).newRows) : new BlueTable(rows.schema)));
-      }
-
-      public void crush_with_single_row(IBlueRow r) {
+      public void crush_with_single_row(IBlueRow r)
+      {
         BlueTable t = new BlueTable(r.schema);
         t.addRow(r);
         loadCollection(t);
      }

-      public void loadPacket(DataPacket dp) { myFireEvent(dp); }
+      public void loadPacket (DataPacket dp) {myFireEvent (dp);}

      /// <summary>
      /// Will work with streams that take ViewStream as first argument ex project, filter, distinct etc
@@ -45,16 +35,17 @@ namespace BCG {
      /// <param name="streamType"></param>
      /// <param name="args"></param>
      /// <returns></returns>
-      public static IBlueRowCollection statelessOp(IBlueRowCollection rows, Type streamType, params object[] args) {
-         ArrayList totArgs = new ArrayList(args);
-         BlueRowCollectionStaticStream v1 = new BlueRowCollectionStaticStream();
-         totArgs.Insert(0, v1);
-         ViewStream retVal = Activator.CreateInstance(streamType, totArgs.ToArray()) as ViewStream;
-         v1.loadCollection(rows);
+      public static IBlueRowCollection statelessOp (IBlueRowCollection rows, Type streamType, params object[] args)
+      {
+         ArrayList totArgs = new ArrayList (args);
+         BlueRowCollectionStaticStream v1 = new BlueRowCollectionStaticStream ();
+         totArgs.Insert (0, v1);
+         ViewStream retVal = Activator.CreateInstance (streamType, totArgs.ToArray()) as ViewStream;
+         v1.loadCollection (rows);
         return retVal.rows;
      }

   }

-
+   
 }
@@ -80,7 +80,7 @@ namespace BCG {

      public IBlueRow map(IBlueRow row) {
         MutableBlueRow rv = new MutableBlueRow(getOutputSchema(row.schema), row);
-         rv.setValue(myConversionColName, mapDateTime(row.get_datetime(myConversionColName), myConversion));
+         rv.setValue(myConversionColName, mapDateTime(row[myConversionColName], myConversion));
         return rv;
         //return BlueDataUtility.getNewReplacedRow(row, mapDateTime(row[myConversionColName], myConversion), myConversionColName);
      }
@@ -1,38 +0,0 @@
-namespace bts.SilverPine {
-   using bts;
-
-   using BCG;
-
-   using System;
-   [DataStreamUserRespresentation("DateTimeExpanderStream")]
-   public class DateTimeExpanstionStream : ViewStream {
-      public DateTimeExpanstionStream(ViewStream parent_stream, string inp_col) {
-         Func<IBlueTableSchema, IBlueTableSchema> get_schema = (s) => {
-            IBlueTableSchema rv = s.add_column(System.TypeCode.DateTime, $"{inp_col}_date");
-            rv = rv.add_column(System.TypeCode.String, $"{inp_col}_day");
-            rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_day_of_month");
-            rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_month");
-            rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_hour");
-            rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_min");
-            rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_seconds_of_day");
-            return rv;
-         };
-         ViewStream rs = new MapStream(parent_stream,
-         s => get_schema(s),
-         r => {
-            DateTime dt = r.get_datetime(inp_col);
-            IBlueTableSchema s = get_schema(r.schema);
-            MutableBlueRow rv = new MutableBlueRow(s, r);
-            rv.setValue($"{inp_col}_date", dt.Date);
-            rv.setValue($"{inp_col}_day_of_week", dt.DayOfWeek.ToString());
-            rv.setValue($"{inp_col}_day_of_month", dt.Day);
-            rv.setValue($"{inp_col}_month", dt.Date.Month);
-            rv.setValue($"{inp_col}_hour", dt.Hour);
-            rv.setValue($"{inp_col}_minute", dt.Minute);
-            rv.setValue($"{inp_col}_seconds_of_day", (dt - dt.Date).TotalSeconds);
-            return rv;
-         });
-         mySetRealSource(rs);
-      }
-   }
-}
@@ -37,42 +37,4 @@ namespace BCG
      }
         
   }
-
-   [DataStreamUserRespresentation("throttle")]
-   public class ThrottleStream : ViewStream{
-      private readonly ViewStream _main_stream;
-      private readonly TimeSpan _ts;
-      private readonly ViewStreamDatabase _vsd;
-      private DateTime _last_fire_time = DateTime.MinValue;
-
-      public ThrottleStream (ViewStream main_stream, int throttle_seconds)
-      :
-      this(main_stream, TimeSpan.FromSeconds(throttle_seconds), ViewStreamDatabase.instance)
-      {
-      }
-
-      public ThrottleStream (ViewStream main_stream, TimeSpan ts, ViewStreamDatabase vsd){
-         _main_stream = main_stream;
-         _ts = ts;
-         _vsd = vsd;
-      }
-
-       protected override void myOnFirstSubscriber() {
-         _main_stream.onChange += _handle_base_stream_update;
-      }
-
-      protected override void myOnLastSubscriber() {
-         _main_stream.onChange -= _handle_base_stream_update;
-      }
-
-      private void _handle_base_stream_update(object dp_obj){
-         if (DateTime.Now - _last_fire_time >= _ts){
-            DataPacket dp_in = dp_obj as DataPacket;
-            _last_fire_time = DateTime.Now;
-            DataPacket dp = BlueDataUtility.build_dp_inc(dp_in.newRows, myLastValue as DataPacket);
-            myFireEvent(dp);
-         }
-      }
- }
-
 }
@@ -1,40 +0,0 @@
-namespace bts.SilverPine{
-   using BGC;
-   using System;
-   [DataStreamUserRespresentation("throttle")]
-   public class ThrottleStream : ViewStream{
-      private readonly ViewStream _main_stream;
-      private readonly TimeSpan _ts;
-      private readonly ViewStreamDatabase _vsd;
-      private DateTime _last_fire_time = DateTime.MinValue;
-
-      public ThrottleStream (ViewStream main_stream, int throttle_seconds)
-      :
-      this(main_stream, TimeSpan.FromSeconds(throttle_seconds), ViewStreamDatabase.instance)
-      {
-      }
-
-      public ThrottleStream (ViewStream main_stream, TimeSpan ts, ViewStreamDatabase vsd){
-         _main_stream = main_stream;
-         _ts = ts;
-         _vsd;
-      }
-
-       protected override void myOnFirstSubscriber() {
-         _main_stream.onChange += _handle_base_stream_update;
-      }
-
-      protected override void myOnLastSubscriber() {
-         _main_stream.onChange -= _handle_base_stream_update;
-      }
-
-      private void _handle_base_stream_update(object dp_obj){
-         if (DateTime.Now - _last_fire_time >= _ts){
-            DataPacket dp_in = dp_obj;
-            _last_fire_time = DateTime.Now;
-            DataPacket dp = BlueDataUtility.build_dp_inc(dp_in.newRow, myLastValue as DataPacket);
-            myFireEvent(dp);
-         }
-      }
- }
-}
@@ -1,124 +0,0 @@
-namespace bts.SilverPine {
-   using BCG;
-
-   using bts;
-   using bts.collections;
-
-   using System;
-   using System.CodeDom;
-   using System.Collections.Generic;
-   using System.Linq;
-
-   // Given 2 filters, and 2 columns, finds the different in the value of the 2 columns from the 2 rows
-   // so:
-   // f1(base_stream.rows)[col1] - f1(base_stream.rows)[col2]
-   [DataStreamUserRespresentation("ColDiff")]
-   public class ColumnDifferenceStream : ViewStream {
-      private readonly ViewStream _base_stream;
-      private readonly FilterStream.OnFilter _f1, _f2;
-      private readonly string _c1, _c2;
-
-      public ColumnDifferenceStream(ViewStream base_stream, FilterStream.OnFilter f1, FilterStream.OnFilter f2, string col1, string col2) {
-         _base_stream = base_stream;
-         _f1 = f1;
-         _f2 = f2;
-         _c1 = col1;
-         _c2 = col2;
-      }
-
-      public ColumnDifferenceStream(ViewStream base_stream, string f1, string f2, string col1, string col2) {
-         FilterStream.OnFilter f1_del = r => {
-            BooleanExpressionFilter rv = new BooleanExpressionFilter(f1);
-            return rv.filter(r);
-         };
-         FilterStream.OnFilter f2_del = r => {
-            BooleanExpressionFilter rv = new BooleanExpressionFilter(f2);
-            return rv.filter(r);
-         };
-         ViewStream rs = new ColumnDifferenceStream(base_stream, f1_del, f2_del, col1, col2);
-         mySetRealSource(rs);
-      }
-
-      // for each row, does a row[new_col_name] = row[col1] - rows[row_ffset][col2]
-      // we should generlize this for any math operation
-      public ColumnDifferenceStream(ViewStream parent_stream, string sort_col_name, int row_offset, string col1, string col2, string diff_col_name, double optional_perc_diff_filter = 0.0) {
-         // use GlobalMapStream
-         string diff_col_perc_name = $"{diff_col_name}_percentage";
-         System.Func<System.Collections.Generic.List<BCG.IBlueRow>, uint, BCG.IBlueRow> g_map_func = (rows, idx) => {
-            int minuend_idx = Convert.ToInt32(idx);
-            var inp_schema = rows[0].schema;
-            var out_schema = inp_schema.add_column(TypeCode.Double, diff_col_name);
-            out_schema = out_schema.add_column(TypeCode.Double, diff_col_perc_name);
-            // minuend - subtrahend
-            IBlueRow minuend_row = rows.ElementAt(minuend_idx);
-            MutableBlueRow rv = new MutableBlueRow(out_schema, minuend_row);
-            int subtrahend_index = minuend_idx + row_offset;
-            double diff_val = Double.NaN, diff_val_perc = Double.NaN;
-            if (subtrahend_index >= 0) {
-               try {
-                  IBlueRow subtrahend_row = rows.ElementAt(subtrahend_index);
-                  diff_val = minuend_row.get<double>(col1) - subtrahend_row.get<double>(col2);
-                  diff_val_perc = 100.0 * diff_val / subtrahend_row.get<double>(col1);
-               } catch (Exception ex) {
-                  Log.assert(false, ex.Message);
-               }
-            }
-            rv.setValue(diff_col_name, diff_val);
-            rv.setValue(diff_col_perc_name, diff_val_perc);
-            return rv;
-         };
-         ViewStream rs = new GlobalMapStream(parent_stream, sort_col_name, g_map_func);
-         if (optional_perc_diff_filter != 0) {
-            rs = new FilterStream(rs, r => {
-               double diff_val_perc = r.get<double>(diff_col_perc_name);
-               double abs_diff_val_perc = Math.Abs(diff_val_perc);
-               return abs_diff_val_perc >= optional_perc_diff_filter;
-            });
-         }
-         mySetRealSource(rs);
-      }
-
-      protected override void myOnFirstSubscriber() {
-         _base_stream.onChange += _handle_base_stream_update;
-      }
-
-      protected override void myOnLastSubscriber() {
-         _base_stream.onChange -= _handle_base_stream_update;
-      }
-
-      private void _handle_base_stream_update(object dp_obj) {
-         // could have been done with filter and cartesian prod, and then math
-         DataPacket dp = dp_obj as DataPacket;
-         List<IBlueRow> rows = dp.newRows.to_list();
-         var f1_rows = rows.Where(r => _f1(r));
-         //Log.assert(f1_rows.Count() == 1);
-         var f2_rows = rows.Where(r => _f2(r));
-         //Log.assert(f2_rows.Count() == 1);
-         if (f1_rows != null && f1_rows.Count() > 0 && f2_rows != null && f2_rows.Count() > 0) {
-            IBlueRow f1_r = f1_rows.ElementAt(0);
-            IBlueRow f2_r = f2_rows.ElementAt(0);
-            double diff = f1_r.get<double>(_c1) - f2_r.get<double>(_c2);
-            IBlueTableSchema out_schema = f1_r.schema.add_columns(Tuple.Create(typeof(double), "diff"));
-            BlueTable out_table = new BlueTable(out_schema);
-            MutableBlueRow out_row = new MutableBlueRow(out_schema, f1_r);
-            out_row.setValue("diff", diff);
-            out_table.addRow(out_row);
-            DataPacket dp_out = BlueDataUtility.build_dp(out_table, myLastValue as DataPacket);
-            myFireEvent(dp_out);
-         } else {
-            IBlueRow f1_r = rows.ElementAt(0);
-            IBlueRow f2_r = rows.ElementAt(0);
-            double diff = Double.NaN;
-            IBlueTableSchema out_schema = f1_r.schema.add_columns(Tuple.Create(typeof(double), "diff"));
-            BlueTable out_table = new BlueTable(out_schema);
-            MutableBlueRow out_row = new MutableBlueRow(out_schema, f1_r);
-            out_row.setValue("diff", diff);
-            out_table.addRow(out_row);
-            DataPacket dp_out = BlueDataUtility.build_dp(out_table, myLastValue as DataPacket);
-            myFireEvent(dp_out);
-         }
-
-      }
-   }
-
-}
@@ -1,51 +0,0 @@
-namespace bts.SilverPine {
-   using BCG;
-
-   using bts;
-
-   using System.Collections.Generic;
-   using System.Linq;
-
-   // Given a sort order, an optional group by col, it observes the values of a column 
-   // within that group (or the whole table, if grouping not specified) and returns all rows
-   // where the value changes, ie row[current][col] != row[current-1][col]
-   // TODO: a more general way to express a filter where row[current][col] != row[previous][col],
-   // or even expand it to row[current][col_a] - row[previous][col_b] < 0.5
-   [DataStreamUserRespresentation("ColEvol")]
-   public class ColumnEvolutionStream : ViewStream {
-      private readonly ViewStream _base_stream;
-      private readonly string _sort_col;
-      private readonly string _evol_col;
-      public ColumnEvolutionStream(ViewStream base_stream, string sort_col, string evolution_col) {
-         _base_stream = base_stream;
-         _sort_col = sort_col;
-         _evol_col = evolution_col;
-      }
-
-      protected override void myOnFirstSubscriber() {
-         _base_stream.onChange += _handle_base_stream_update;
-      }
-
-      protected override void myOnLastSubscriber() {
-         base.myOnLastSubscriber();
-      }
-
-      private void _handle_base_stream_update(object dp_obj) {
-         DataPacket dp = dp_obj as DataPacket;
-         List<IBlueRow> rows = dp.newRows.to_list();
-         rows.Sort((r1, r2) => BlueDataUtility.compare_rows(r2, r1, _sort_col));
-         BlueTable rv = new BlueTable(dp.schema);
-         IBlueRow prev_row = rows.ElementAt(0);
-         foreach (IBlueRow r in rows) {
-            var cr = BlueDataUtility.compare_rows(r, prev_row, _evol_col);
-            if (cr != 0) {
-               rv.addRow(r);
-            }
-            prev_row = r;
-         }
-         DataPacket dp_out = BlueDataUtility.build_dp(rv, myLastValue as DataPacket);
-         myFireEvent(dp_out);
-      }
-   }
-
-}
@@ -1,15 +1,18 @@
 using System;
 using System.Collections.Generic;
-
 using mbts;

-namespace BCG {
-   internal class MutableAppendTable : IBlueRowCollection {
-      private class MyRow : IBlueRow {
+namespace BCG
+{
+   internal class MutableAppendTable : IBlueRowCollection
+   {
+      private class MyRow : IBlueRow
+      {
         private IBlueRow myRow;
         private int myID;

-         public MyRow(IBlueRow r, int id) {
+         public MyRow(IBlueRow r, int id)
+         {
            myRow = r;
            myID = id;
         }
@@ -18,8 +21,10 @@ namespace BCG {

         public IBlueTableSchema schema { get { return myRow.schema; } }

-         public object this[int index] {
-            get {
+         public object this[int index]
+         {
+            get
+            {
               if (myRow.schema[index].colName == "id")
                  return myID;
               else
@@ -47,17 +52,20 @@ namespace BCG {

         public System.Collections.ICollection Keys { get { return myRow.Keys; } }

-         public void Remove(object key) { throw new Exception("The method or operation is not implemented."); }
+         public void Remove(object key) {throw new Exception("The method or operation is not implemented.");}

-         public System.Collections.ICollection Values {
-            get {
+         public System.Collections.ICollection Values
+         {
+            get
+            {
               object[] vals = new object[myRow.Count];
               CopyTo(vals, 0);
               return vals;
            }
         }

-         public object this[object key] {
+         public object this[object key]
+         {
            get { return key is string ? this[key as string] : this[(int)key]; }
            set { throw new Exception("The method or operation is not implemented."); }
         }
@@ -66,7 +74,8 @@ namespace BCG {

         #region ICollection Members

-         public void CopyTo(Array array, int index) {
+         public void CopyTo(Array array, int index)
+         {
            myRow.CopyTo(array, index);
            array.SetValue(myID, index + myRow.schema["id"].colNo);
         }
@@ -81,7 +90,8 @@ namespace BCG {

         #region IEnumerable Members

-         System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() { return new BlueRowEnumerator(this); }
+         System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator()
+         { return new BlueRowEnumerator(this); }

         #endregion
      }
@@ -96,7 +106,8 @@ namespace BCG {

      public MutableAppendTable(IBlueTableSchema schema) { myRows = new BlueTable(schema); }

-      public IBlueRow addRow(IBlueRow r, int tableNo) {
+      public IBlueRow addRow(IBlueRow r, int tableNo)
+      {
         if (!myIDMap.ContainsKey(tableNo))
            myIDMap.Add(tableNo, new Dictionary<int, IBlueRow>());
         IBlueRow nr = new MyRow(r, myID++); // BlueDataUtility.getNewReplacedRow(r, myID++, "id");
@@ -105,17 +116,20 @@ namespace BCG {
         return nr;
      }

-      public IBlueRow removeRow(IBlueRow r, int tableNo) {
+      public IBlueRow removeRow(IBlueRow r, int tableNo)
+      {
         IBlueRow dr = myIDMap[tableNo][(int)r["id"]];
         myRows.deleteRow(dr);
         myIDMap[tableNo].Remove((int)r["id"]);
         return dr;
      }

-      public IBlueRowCollection removeTable(int tableNo) {
+      public IBlueRowCollection removeTable(int tableNo)
+      {
         if (!myIDMap.ContainsKey(tableNo))
            return null;
-         else {
+         else
+         {
            BlueTable rv = new BlueTable(myRows.schema);
            foreach (IBlueRow r in myIDMap[tableNo].Values)
               rv.addRow(r);
@@ -125,7 +139,8 @@ namespace BCG {
         }
      }

-      public void clear() {
+      public void clear()
+      {
         myRows.clear();
         myIDMap.Clear();
      }
@@ -136,7 +151,7 @@ namespace BCG {

      public IBlueTableSchema schema { get { return myRows.schema; } }

-      public bool containsID(int id) { return myRows.containsID(id); }
+      public bool containsID(int id) {return myRows.containsID(id);}

      #endregion

@@ -154,7 +169,7 @@ namespace BCG {

      #region IEnumerable Members

-      public System.Collections.IEnumerator GetEnumerator() { return myRows.GetEnumerator(); }
+      public System.Collections.IEnumerator GetEnumerator() {return myRows.GetEnumerator();}

      #endregion
   }
@@ -164,8 +179,8 @@ namespace BCG {
   /// to append (or remove) more streams at
   /// runtime
   /// </summary>
-   [DataStreamUserRespresentation("UnionStream")]
-   public class MutableAppendStream : ViewStream {
+   public class MutableAppendStream : ViewStream
+   {
      private bool mySubscriptionActive = false;
      private MutableAppendTable myTable;
      private int myMovingIndex;
@@ -173,17 +188,22 @@ namespace BCG {
      private bool _allign_col_orders = true;

      public MutableAppendStream(params ViewStream[] streams)
-         : this(new List<ViewStream>(streams)) { }
+         : this(new List<ViewStream>(streams))
+      { }

      public MutableAppendStream(bool allign_col_order, params ViewStream[] streams)
-         : this(allign_col_order, new List<ViewStream>(streams)) { }
+         : this(allign_col_order, new List<ViewStream>(streams))
+      { }

      public MutableAppendStream(ICollection<ViewStream> streams)
-         : this(true, streams) { }
+         : this(true, streams)
+      { }

-      public MutableAppendStream(bool allign_col_orders, ICollection<ViewStream> streams) {
+      public MutableAppendStream(bool allign_col_orders, ICollection<ViewStream> streams)
+      {
         _allign_col_orders = allign_col_orders;
-         foreach (ViewStream vs in streams) {
+         foreach (ViewStream vs in streams)
+         {
            myMovingIndex++;
            ViewStream pj = new ChangeTableNameStream(vs, myMovingIndex.ToString());
            if (_allign_col_orders)
@@ -192,7 +212,8 @@ namespace BCG {
         }
      }

-      public void addStream(ViewStream vs) {
+      public void addStream(ViewStream vs)
+      {
         myMovingIndex++;
         ///the change name is done to identify the callbacks, and the project to
         ///make all the schemas col orders the same
@@ -204,14 +225,19 @@ namespace BCG {
            pj.onChange += new ChangeDelegate(myHandleStreamChanges);
      }

-      public void removeStream(ViewStream vs) {
+      public void removeStream(ViewStream vs)
+      {
         int index = -1;
-         foreach (KeyValuePair<int, Pair<ViewStream, ViewStream>> p in myStreamIndex) {
-            if (p.Value.first == vs) {
+         foreach (KeyValuePair<int, Pair<ViewStream, ViewStream>> p in myStreamIndex)
+         {
+            if (p.Value.first == vs)
+            {
               index = p.Key;
-               if (mySubscriptionActive) {
+               if (mySubscriptionActive)
+               {
                  p.Value.second.onChange -= new ChangeDelegate(myHandleStreamChanges);
-                  if (myTable != null) {
+                  if (myTable != null)
+                  {
                     IBlueRowCollection delRows = myTable.removeTable(p.Key);
                     if (delRows != null)
                        myFireEvent(new DataPacket(myTable, new BlueTable(myTable.schema), delRows));
@@ -224,13 +250,15 @@ namespace BCG {
         }
      }

-      protected override void myOnFirstSubscriber() {
+      protected override void myOnFirstSubscriber()
+      {
         mySubscriptionActive = true;
         foreach (KeyValuePair<int, Pair<ViewStream, ViewStream>> p in myStreamIndex)
            p.Value.second.onChange += new ChangeDelegate(myHandleStreamChanges);
      }

-      protected override void myOnLastSubscriber() {
+      protected override void myOnLastSubscriber()
+      {
         mySubscriptionActive = false;
         foreach (KeyValuePair<int, Pair<ViewStream, ViewStream>> p in myStreamIndex)
            p.Value.second.onChange -= new ChangeDelegate(myHandleStreamChanges);
@@ -238,17 +266,20 @@ namespace BCG {
            myTable.clear();
      }

-      private void myHandleStreamChanges(object newValue) {
+      private void myHandleStreamChanges(object newValue)
+      {         
         DataPacket dp = newValue as DataPacket;
         myTable = myTable ?? new MutableAppendTable(new BlueTableSchema("MutableAppend", dp.schema.columnInfo));
         BlueTable adds = new BlueTable(myTable.schema),
                     dels = new BlueTable(myTable.schema);
         int index = Int32.Parse(dp.schema.tableName);
-         foreach (IBlueRow r in dp.deletedRows) {
+         foreach (IBlueRow r in dp.deletedRows)
+         {
            IBlueRow dr = myTable.removeRow(r, index);
            dels.addRow(dr);
         }
-         foreach (IBlueRow r in dp.addedRows) {
+         foreach (IBlueRow r in dp.addedRows)
+         {
            IBlueRow nr = myTable.addRow(r, index);
            adds.addRow(nr);
         }
@@ -15,75 +15,10 @@ using mbts;
 using BCG.Extensions;
 using System.Text.Json;
 using System.Text.Json.Nodes;
-using Newtonsoft.Json;
-using BCG;
-using System.Runtime.CompilerServices;
-using bts;
-
-namespace bts {
-
-   // public class CompositeRow : IBlueRow {
-   //    private IBlueRow _lrow, _rrow;
-   //    private BlueRow _out_row;
-   //    public CompositeRow(IBlueRow lrow, IBlueRow rrow) {
-   //       _lrow = lrow;
-   //       _rrow = rrow;
-   //       IBlueTableSchema schema = _lrow.schema;
-   //       List<object> vals = new List<object>();
-   //       foreach (var o in _lrow) {
-   //          vals.Add(o);
-   //       }
-   //       foreach (ColumnInfo ci in _rrow.schema.columnInfo) {
-   //          if (ci.colName == "id") {
-   //             continue;
-   //          }
-   //          vals.Add(_rrow[ci.colNo]);
-   //          schema = schema.add_column(ci.type_code(), ci.colName);
-   //       }
-   //       _out_row = new BlueRow(schema, vals.ToArray());
-   //    }
-   //    public IBlueTableSchema schema {
-   //       get { return _out_row.schema; }
-   //    }
-   //    public object? this[int index] {
-   //       get { return _out_row[index]; }
-   //    }
-   //    public object? this[string colName] {
-   //       get {
-   //          return _out_row[colName];
-   //       }
-   //    }
-
-   // }
-}

 namespace BCG {
   public static class BlueDataUtility {
-      public static IBlueRow create_composite_row(IBlueRow lr, IBlueRow rr, int id) {
-         IBlueTableSchema schema = lr.schema;
-         List<object> vals = new List<object>();
-         foreach (ColumnInfo ci in lr.schema.columnInfo) {
-            if (ci.colName == "id") {
-               vals.Add(id);
-            }
-            vals.Add(lr[ci.colName]);
-         }
-         foreach (ColumnInfo ci in rr.schema.columnInfo) {
-            if (ci.colName == "id") {
-               continue;
-            }
-            schema = schema.add_column(ci.type_code(), ci.colName);
-            vals.Add(rr[ci.colName]);
-         }
-         IBlueRow r = new BlueRow(schema, vals.ToArray());
-         return r;
-      }
      #region Extensions
-      public static bool is_null(this IBlueRow r, string col_name) {
-         object col_val = r[col_name];
-         return col_val == null || DBNull.Value.Equals(col_val);
-      }
-
      public static DateTime get_datetime(this IBlueRow r, string col_name, DateTime err_value = default(DateTime)) {
         return BlueDataUtility.get_row_datetime_value(r, col_name, err_value);
      }
@@ -114,17 +49,6 @@ namespace BCG {
            rv.Add(r.get<T>(col));
         return rv;
      }
-      public static IEnumerable<T> get_colvalues<T>(IEnumerable<IBlueRow> rows, string col, int start_index, int count) {
-         List<T> rv = new List<T>();
-         int i = 0;
-         int n_copied = 0;
-         foreach (IBlueRow r in rows)
-            if (i++ >= start_index && n_copied < count) {
-               rv.Add(r.get<T>(col));
-               n_copied++;
-            }
-         return rv;
-      }

      public static IDictionary<T, IEnumerable<IBlueRow>> group_by<T>(IBlueRowCollection rows, string gp_col) {
         Dictionary<T, List<IBlueRow>> rv = new Dictionary<T, List<IBlueRow>>();
@@ -188,17 +112,13 @@ namespace BCG {
         }
      }

-      public static List<IBlueRow> to_list(this IBlueRowCollection rows) {
+      public static IEnumerable<IBlueRow> to_ienum(this IBlueRowCollection rows) {
         List<IBlueRow> rv = new List<IBlueRow>();
         foreach (IBlueRow r in rows)
            rv.Add(r);
         return rv;
      }

-      public static IEnumerable<IBlueRow> to_ienum(this IBlueRowCollection rows) {
-         return rows.to_list();
-      }
-
      public static IDictionary<int, IBlueRow> to_map(this IBlueRowCollection rows) {
         Dictionary<int, IBlueRow> rv = new Dictionary<int, IBlueRow>();
         foreach (IBlueRow r in rows) {
@@ -208,10 +128,6 @@ namespace BCG {
         return rv;
      }

-      public static bool contains_column(this IBlueTableSchema input_schema, string col) {
-         return input_schema.columnNames.Contains(col);
-      }
-
      public static IBlueTableSchema add_columns(this IBlueTableSchema input_schema, params Tuple<Type, string>[] cols) {
         List<ColumnInfo> cis = new List<ColumnInfo>(input_schema.columnInfo);
         foreach (Tuple<Type, string> c in cols)
@@ -219,11 +135,6 @@ namespace BCG {
         return new BlueTableSchema(input_schema.tableName, cis.ToArray());
      }

-      public static IBlueTableSchema add_column(this IBlueTableSchema input_schema, TypeCode type_arg, string col_name) {
-         Tuple<Type, string> tup = new Tuple<Type, string>(_typecode_to_type[type_arg], col_name);
-         return add_columns(input_schema, tup);
-      }
-
      public static IBlueRowCollection to_collection(this IBlueRow r) {
         if (r == null)
            return null;
@@ -280,18 +191,6 @@ namespace BCG {

      #region json

-      public static Dictionary<string, string> convert_json_to_dictionary(string json_string) {
-         Dictionary<string, string> dictionary = JsonConvert.DeserializeObject<Dictionary<string, string>>(json_string);
-         return dictionary;
-      }
-      public static IBlueRow convert_json_to_row(string json_string, int id = 0) {
-         Dictionary<string, object> dictionary = JsonConvert.DeserializeObject<Dictionary<string, object>>(json_string);
-         if (!dictionary.ContainsKey("id")) {
-            dictionary.Add("id", id);
-         }
-         return new BlueRow(dictionary);
-      }
-
      public static List<JsonNode> json_to_json_node_list(string jsonString) {
         var jsonNodeList = new List<JsonNode>();
         bool noe = String.IsNullOrEmpty(jsonString);
@@ -310,7 +209,7 @@ namespace BCG {
            foreach (JsonNode jsonNode in jsonArray) {
               jsonNodeList.Add(jsonNode);
            }
-         } catch (System.Text.Json.JsonException ex) {
+         } catch (JsonException ex) {
            Console.WriteLine($"Invalid JSON: {ex.Message}");
         }

@@ -587,16 +486,7 @@ namespace BCG {
         ColumnInfo[] cis = new ColumnInfo[table.Columns.Count];
         foreach (DataColumn col in table.Columns)
            cis[col.Ordinal] = new ColumnInfo(col.ColumnName, col.Ordinal, col.DataType);
-         IBlueTableSchema rv = new BlueTableSchema(cis, table.TableName);
-         rv = rv.add_id_col_if_missing();
-         return rv;
-      }
-      public static IBlueTableSchema add_id_col_if_missing(this IBlueTableSchema schema) {
-         IBlueTableSchema rv = schema;
-         if (!schema.columnNames.Contains("id")) {
-            rv = schema.add_columns(Tuple.Create(typeof(int), "id"));
-         }
-         return rv;
+         return new BlueTableSchema(cis, table.TableName);
      }

      /// <summary>
@@ -854,16 +744,10 @@ namespace BCG {
      }

      public static IBlueRowCollection toIBlueRowCollection(DataTable dt) {
-         IBlueTableSchema schema = getSchema(dt);
-         BlueTable rv = new BlueTable(schema);
-         foreach (DataRow dr in dt.Rows) {
-            MutableBlueRow nr = new MutableBlueRow(schema);
-            foreach (DataColumn col in dr.Table.Columns)
-               nr.setValue(col.ColumnName, dr[col.ColumnName]);
-            nr.setValue("id", rv.Count);
-            rv.addRow(nr);
-         }
-         return rv;
+         BlueTable retVal = new BlueTable(getSchema(dt));
+         foreach (DataRow dr in dt.Rows)
+            retVal.addRow(dr);
+         return retVal;
      }

      public static IBlueRowCollection toIBlueRowCollection(DataRow[] rows) {
@@ -935,20 +819,20 @@ namespace BCG {
      /// </summary>
      /// <param name="r1"></param>
      /// <param name="r2"></param>
-      /// <param name="col_names"></param>
+      /// <param name="colNames"></param>
      /// <returns></returns>
      public static int compare_rows(IBlueRow r1,
         IBlueRow r2,
-         params string[] col_names) {
-         int comp_res = 0;
-         foreach (string cn in col_names) {
-            object val1 = r1[cn];
-            object val2 = r2[cn];
-            comp_res = _compare_row_cols(val1, val2);
-            if (comp_res != 0)
+         params string[] colNames) {
+         int retVal = 0;
+         foreach (string colName in colNames) {
+            object val1 = r1[colName];
+            object val2 = r2[colName];
+            retVal = _compare_row_cols(val1, val2);
+            if (retVal != 0)
               break;
         }
-         return comp_res;
+         return retVal;
      }

      private static int _compare_row_cols(object v1, object v2) {
@@ -1724,10 +1608,8 @@ namespace BCG {

      public static int[] get_column_indices(IBlueTableSchema schema, string[] col_names) {
         List<int> rv = new List<int>();
-         foreach (string c in col_names) {
-            Log.assert(schema.contains_column(c), $"Could not find col: {c} in table: {schema.tableName}");
+         foreach (string c in col_names)
            rv.Add(schema[c].colNo);
-         }
         return rv.ToArray();
      }

@@ -2214,53 +2096,23 @@ namespace BCG {
         try {
            if (v == null || v == DBNull.Value)
               return error_val;
-
-            if (v is DateTime) {
-               return (DateTime)v;
-            } else if (v is string) {
-               DateTime rv_dt;
-
-               // Try parsing the string as a DateTime
-               if (DateTime.TryParse((string)v,
-                                     System.Globalization.CultureInfo.InvariantCulture,
-                                     System.Globalization.DateTimeStyles.RoundtripKind,
-                                     out rv_dt)) {
+            else {
+               if (v is DateTime)
+                  return (DateTime)v;
+               else if (v is string) {
+                  DateTime rv_dt = error_val;
+                  DateTime.TryParse((string)v, out rv_dt);
                  return rv_dt;
               } else {
-                  return error_val; // Return the error value if parsing fails
+                  double oa_date = (double)Convert.ChangeType(v, typeof(Double));
+                  return DateTime.FromOADate(oa_date);
               }
-            } else {
-               // Assume the input can be converted to a double and use FromOADate
-               double oa_date = (double)Convert.ChangeType(v, typeof(Double));
-               return DateTime.FromOADate(oa_date);
            }
         } catch {
            return error_val;
         }
      }

-
-      // private static DateTime _get_datetime_value(object v, DateTime error_val) {
-      //    try {
-      //       if (v == null || v == DBNull.Value)
-      //          return error_val;
-      //       else {
-      //          if (v is DateTime)
-      //             return (DateTime)v;
-      //          else if (v is string) {
-      //             DateTime rv_dt = error_val;
-      //             DateTime.TryParse((string)v, out rv_dt);
-      //             return rv_dt;
-      //          } else {
-      //             double oa_date = (double)Convert.ChangeType(v, typeof(Double));
-      //             return DateTime.FromOADate(oa_date);
-      //          }
-      //       }
-      //    } catch {
-      //       return error_val;
-      //    }
-      // }
-
      #endregion

      #region CSV methods
@@ -2,15 +2,12 @@

  <PropertyGroup>
    <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
    <Nullable>disable</Nullable>
  </PropertyGroup>

  <ItemGroup>
    <PackageReference Include="Google.Protobuf" Version="3.24.3" />
-    <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
-    <PackageReference Include="OpenAI" Version="2.0.0" />
-    <PackageReference Include="SimMetrics.Net" Version="1.0.5" />
  </ItemGroup>

  <ItemGroup>
@@ -19,8 +16,8 @@
  </ItemGroup>

  <ItemGroup>
-    <Proto Include="../../lib_spark/spark/config/protos/data_stream_config.proto" CSName="DataStreamConfig">
-      <OutDir>$(ProtoOutputPath)/lib_spark/spark/config/protos</OutDir>
+    <Proto Include="../../lib/config/protos/data_stream_config.proto" CSName="DataStreamConfig">
+      <OutDir>$(ProtoOutputPath)/lib/config/protos</OutDir>
    </Proto>
  </ItemGroup>

@@ -47,11 +47,6 @@ namespace BCG {
      /// </summary>
      /// <param name="assemblyName"></param>
      public void loadAssembly(string assemblyName) {
-         Func<Type, bool> is_silverpine_only = (t) => {
-            if (!assemblyName.Contains("SharedStreams.dll"))
-               return true;
-            return t.Namespace == "bts.SilverPine";
-         };
         bts.Log.Gen.info($"PrimitiveManager.loadAssembly assemblyName: {assemblyName}");
         if (myAssemblyNames.Contains(assemblyName))
            return;
@@ -61,7 +56,7 @@ namespace BCG {
            Assembly.LoadFrom(assemblyName);

         foreach (Type type in loadedAssembly.GetTypes())
-            if (!type.IsInterface && is_silverpine_only(type)
+            if (!type.IsInterface
               && !type.IsAbstract
               && type.IsSubclassOf(typeof(ViewStream))
               && type.IsPublic) {
@@ -9,15 +9,7 @@
 Letter                   [a-zA-Z]
 BooleanConstant          'true'|'false'
 CSStringConstant         \"([^\"\\]*|\\.)*\"
-YearInt                  [0-9]{4}
-MonthInt                 [0-9]{1,2}
-DayInt                   [0-9]{1,2}
-/**
- * DateTimeConstant supports American (mm/dd/yyyy) and European (yyyy/mm/dd)
- * formats. Month and day can be 1 or 2 digits. It does not validate that the
- * numbers represent an actual date.
- */
-DateTimeConstant         ({MonthInt}\/{DayInt}\/{YearInt}|{YearInt}\/{MonthInt}\/{DayInt})
+DateTimeConstant         ([1-9]|'10'|'11'|'12')\/([1-9]|([1-2][0-9])|'30'|'31')\/[0-9][0-9][0-9]*
 IntegerConstant          -?[0-9]+
 NullConstant             'null'
 RealConstant             -?[0-9]+\.[0-9]+
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
    <ImplicitUsings>enable</ImplicitUsings>
    <Nullable>enable</Nullable>
  </PropertyGroup>
@@ -2,7 +2,7 @@

  <PropertyGroup>
    <OutputType>Exe</OutputType>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
    <Nullable>enable</Nullable>
    <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
  </PropertyGroup>
@@ -1,62 +1,60 @@
-{
-  "user": "bts",
-  "sparkd": "http://192.168.1.80:8000/config",
-  "pubsub": {
-    "milliseconds_between_sub_info_updates": 1000,
-    "milliseconds_between_time_slice_updates": 5000,
-    "milliseconds_between_no_buffer_space_retries": 100,
-    "retries_on_no_buffer_space": 1,
-    "do_it_all_seconds_between_logs": 0,
-    "heartbeat_interval": 15000,
-    "missing_heartbeats_before_gc": 50,
-    "use_managed_sockets": true,
-    "state_update_period": 5000,
-    "statistics_update_period": 5000,
-    "disposed_connection_check_period": 1000,
-    "message_processing_warning_threshold": 1000,
-    "server_port": 12345,
-    "publisher_routes": [
-      {
-        "name": "default",
-        "server_address": "bts-cloud://192.168.1.80:12345",
-        "tag": ".*"
-      }
-    ],
-    "subscriber_routes": [
-      {
-        "name": "default",
-        "server_address": "bts-cloud://192.168.1.80:12345",
-        "tag": ".*"
-      }
-    ],
-    "max_packet_size": 500000000,
-    "blue_comm_must_die": true,
-    "crushable_tags": [],
-    "tag_type_to_assembly_map": [
-      {
-        "handler_class": "BCG.PubSubServerTagHandler",
-        "assembly": "DataStreamCore.dll"
-      }
-    ],
-    "server_queue": "PubSubQueue",
-    "port_range": {
-      "first": 30400,
-      "last": 31700
-    },
-    "bind_to_random_port": false,
-    "debug_tags": "^NOTAGS$",
-    "latency_report_threshold_ms": 1500,
-    "use_substations": false,
-    "use_dynamic_clients": false,
-    "is_substation": false,
-    "internally_publish_sub_info": true,
-    "use_managed_clients_in_native_code": false,
-    "compressed_tags": "^NOTAGS$",
-    "tags_to_report_compression_ratio": "^NOTAGS$",
-    "disable_msmq": false,
-    "use_prepublished_server_state": false,
-    "ip_translation": null,
-    "gc_activation_threshold": 0,
-    "wait_period_after_gc": 1000
-  }
+"user": "bts",
+"pubsub": {
+  "milliseconds_between_sub_info_updates": 1000,
+  "milliseconds_between_time_slice_updates": 5000,
+  "milliseconds_between_no_buffer_space_retries": 100,
+  "retries_on_no_buffer_space": 1,
+  "do_it_all_seconds_between_logs": 0,
+  "heartbeat_interval": 15000,
+  "missing_heartbeats_before_gc": 50,
+  "use_managed_sockets": true,
+  "state_update_period": 5000,
+  "statistics_update_period": 5000,
+  "disposed_connection_check_period": 1000,
+  "message_processing_warning_threshold": 1000,
+  "server_port": 12345,
+  "publisher_routes": [
+    {
+      "name": "default",
+      "server_address": "bts-cloud://192.168.1.80:12345",
+      "tag": ".*"
+    }
+  ],
+  "subscriber_routes": [
+    {
+      "name": "default",
+      "server_address": "bts-cloud://192.168.1.80:12345",
+      "tag": ".*"
+    }
+  ],
+  "max_packet_size": 500000000,
+  "blue_comm_must_die": true,
+  "crushable_tags": [],
+  "tag_type_to_assembly_map": [
+    {
+      "handler_class": "BCG.PubSubServerTagHandler",
+      "assembly": "DataStreamCore.dll"
+    }
+  ],
+  "server_queue": "PubSubQueue",
+  "port_range": {
+    "first": 30400,
+    "last": 31700
+  },
+  "bind_to_random_port": false,
+  "debug_tags": "^NOTAGS$",
+  "latency_report_threshold_ms": 1500,
+  "use_substations": false,
+  "use_dynamic_clients": false,
+  "is_substation": false,
+  "internally_publish_sub_info": true,
+  "use_managed_clients_in_native_code": false,
+  "compressed_tags": "^NOTAGS$",
+  "tags_to_report_compression_ratio": "^NOTAGS$",
+  "disable_msmq": false,
+  "use_prepublished_server_state": false,
+  "ip_translation": null,
+  "gc_activation_threshold": 0,
+  "wait_period_after_gc": 1000
+}
 }
@@ -2,7 +2,7 @@

  <PropertyGroup>
    <OutputType>Exe</OutputType>
-    <TargetFramework>net8.0-windows</TargetFramework>
+    <TargetFramework>net7.0-windows</TargetFramework>
    <Nullable>enable</Nullable>
    <UseWindowsForms>true</UseWindowsForms>
  </PropertyGroup>
@@ -169,10 +169,8 @@ namespace BCG
      }
      #endregion

-      private Stopwatch _stopwatch = new Stopwatch();
      private void myTryExpression(string expStr)
      {
-         _stopwatch.Restart();
         myFreezeView(false); // force unfreeze the view
         DataStreamExpression oldExp = myExp;
         try
@@ -346,7 +344,6 @@ namespace BCG
      // this is the stream callback
      private void myCallback(object o)
      {
-         _stopwatch.Stop();
         var timer = Stopwatch.StartNew();
         DataPacket dp = o as DataPacket;
         myCrono.Stop();
@@ -362,8 +359,7 @@ namespace BCG
         myToolStripStatusLabelRows.BackColor = Color.Black;
         myToolStripStatusLabelTime.ForeColor = Color.White;
         myToolStripStatusLabelRows.ForeColor = Color.White;
-         string time_text = "Last update: " + DateTime.Now.ToString("hh:mm:ss:fff tt") + $" time: {_stopwatch.ElapsedMilliseconds} msecs\t\t";
-         myToolStripStatusLabelTime.Text = time_text;
+         myToolStripStatusLabelTime.Text = "Last update: " + DateTime.Now.ToString("hh:mm:ss:fff tt");
         myToolStripStatusLabelRows.Text = "New = " + dp.newRows.Count + "  Added = " + dp.addedRows.Count + "  Deleted = " + dp.deletedRows.Count;
         Log.Ui.diag(Text);
         int hscrollval = myChangeStyle ? 0 : myDataGrid.HorizScrollBarValue;
@@ -36,8 +36,6 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnExpressions", "Bt
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnSortExpressions", "Bts.ColumnSortExpressions\Bts.ColumnSortExpressions.csproj", "{B038CB72-4E81-4AF1-976F-ED8655C2C3D6}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamQueryAnalyzer", "DataStreamQueryAnalyzer\DataStreamQueryAnalyzer.csproj", "{BFA5BA5C-927A-41A9-A696-1E2F4B5FFD15}"
-EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -115,9 +113,5 @@ Global
 		{B038CB72-4E81-4AF1-976F-ED8655C2C3D6}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{B038CB72-4E81-4AF1-976F-ED8655C2C3D6}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{B038CB72-4E81-4AF1-976F-ED8655C2C3D6}.Release|Any CPU.Build.0 = Release|Any CPU
-		{BFA5BA5C-927A-41A9-A696-1E2F4B5FFD15}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{BFA5BA5C-927A-41A9-A696-1E2F4B5FFD15}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{BFA5BA5C-927A-41A9-A696-1E2F4B5FFD15}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{BFA5BA5C-927A-41A9-A696-1E2F4B5FFD15}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 EndGlobal
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
    <ImplicitUsings>enable</ImplicitUsings>
    <Nullable>enable</Nullable>

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
    <ImplicitUsings>enable</ImplicitUsings>
    <Nullable>enable</Nullable>
  </PropertyGroup>
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net8.0-windows</TargetFramework>
+    <TargetFramework>net7.0-windows</TargetFramework>
    <ImplicitUsings>enable</ImplicitUsings>
    <Nullable>enable</Nullable>
    <UseWindowsForms>true</UseWindowsForms>
@@ -6,18 +6,12 @@ namespace BCG
 	public interface IPublisher
 	{
      void publish( IPubItem pi );
-#pragma warning disable SYSLIB0011 // Type or member is obsolete
      void publish( IPubItem pi, IFormatter f );
-#pragma warning restore SYSLIB0011 // Type or member is obsolete
      void publish( IPubItem[] pi );
-#pragma warning disable SYSLIB0011 // Type or member is obsolete
      void publish( IPubItem[] pi, IFormatter f );
-#pragma warning restore SYSLIB0011 // Type or member is obsolete
   }

-#pragma warning disable SYSLIB0011 // Type or member is obsolete
   public interface IPublisher<T> where T : IFormatter
-#pragma warning restore SYSLIB0011 // Type or member is obsolete
   {
      void publish (IPubItem pi);
      void publish (IPubItem[] pi);
@@ -117,10 +117,10 @@ public class PubSubConfig {
         WaitPeriodAfterGc = 1000,
         SimpleEchoServer = false,
         PublisherRoutes = new() {
-            new("bluto://127.0.0.1:12345", ".*")
+            new("bts-cloud://192.168.1.80:12345", ".*")
         },
         SubscriberRoutes = new() {
-            new("bluto://127.0.0.1:12345", ".*")
+            new("bts-cloud://192.168.1.80:12345", ".*")
         }
      };
      return rv;
@@ -5,21 +5,23 @@ using System.Runtime.Serialization.Formatters.Binary;

 namespace BCG
 {
-#pragma warning disable SYSLIB0011 // Type or member is obsolete
   public class GenericFormatter<F> : IGenericFormatter where F : IFormatter, new ()
   {
      IFormatter myFormatter = new F ();

      public T Deserialize<T> (Stream serializationStream)
      {
+         #pragma warning disable 
         return (T) myFormatter.Deserialize (serializationStream);
+         #pragma warning restore SYSLIB0011
      }
      public void Serialize<T> (Stream serializationStream, T graph)
      {
+         #pragma warning disable 
         myFormatter.Serialize (serializationStream, graph);
+         #pragma warning restore SYSLIB0011
      }
   }

   public class GenericBinaryFormatter : GenericFormatter<BinaryFormatter> { }
-#pragma warning restore SYSLIB0011 // Type or member is obsolete
 }
@@ -6,7 +6,7 @@

  <PropertyGroup>
      <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
    <Nullable>enable</Nullable>
  </PropertyGroup>

@@ -2,7 +2,7 @@

  <PropertyGroup>
    <OutputType>Exe</OutputType>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
    <Nullable>enable</Nullable>
  </PropertyGroup>

@@ -1,252 +0,0 @@
-using bts.SilverPine;
-
-namespace bts.SilverPine {
-   using bts.api;
-   using bts.spark;
-
-   using System;
-
-   using ZstdSharp.Unsafe;
-
-   public static class ContractUtils {
-      public class FutureTradableContract {
-         public string symbol;
-         public DateTime expiration;
-         public Exchange exchange;
-         public string contract_text;
-      }
-      public class FutureSpreadContract {
-         public List<Tuple<double, bts.SilverPine.ContractUtils.FutureTradableContract>> legs;
-         public string symbol {
-            get {
-               return legs == null || legs.Count == 0 ? String.Empty : legs.ElementAt(0).Item2.symbol;
-            }
-         }
-         public string contract_text;
-      }
-      public class FutureContinousContract {
-         public string symbol;
-         public Exchange exchange;
-      }
-      // this is for all future conytracts belonging to a symbol
-      public class FutureWildcardContract {
-         public string symbol;
-         public Exchange exchange;
-      }
-      public class FutureContract {
-         public FutureWildcardContract wildcard_contract = null;
-         public FutureTradableContract future_tradable_contract = null;
-         public FutureSpreadContract spread_tradable_contract = null;
-         public FutureContinousContract continous_constract = null;
-
-         public FutureContract(string txt) {
-            wildcard_contract = parse_wildcard_contract(txt);
-            if (wildcard_contract != null)
-               return;
-            FutureTradableContract fc = parse_future_contract(txt);
-            if (fc != null) {
-               future_tradable_contract = fc;
-               return;
-            }
-            FutureSpreadContract sc = parse_future_spread_contract(txt);
-            if (sc != null) {
-               spread_tradable_contract = sc;
-               return;
-            }
-            // this is not totally true, since ParseFutureSpreadContract does not
-            // parse ICS spreads
-            continous_constract = new FutureContinousContract() { symbol = txt };
-         }
-         public bool is_continous_contract => continous_constract != null;
-         public string symbol() {
-            if (wildcard_contract != null)
-               return wildcard_contract.symbol;
-            if (continous_constract != null) {
-               return continous_constract.symbol;
-            } else if (future_tradable_contract != null) {
-               return future_tradable_contract.symbol;
-            } else if (spread_tradable_contract != null) {
-               return spread_tradable_contract.symbol;
-            }
-            Log.assert(false);
-            return String.Empty;
-         }
-         public string contract_text {
-            get {
-               if (future_tradable_contract != null)
-                  return future_tradable_contract.contract_text;
-               if (spread_tradable_contract != null)
-                  return spread_tradable_contract.contract_text;
-               return null;
-            }
-         }
-      }
-      #region Parsing
-      public static FutureWildcardContract parse_wildcard_contract(string contract_txt, Exchange exchange = Exchange.CME) {
-         if (contract_txt.EndsWith("*")) {
-            contract_txt = contract_txt.Replace("*", String.Empty);
-            return new FutureWildcardContract() {
-               symbol = contract_txt,
-               exchange = exchange
-            };
-         }
-         return null;
-      }
-      public static FutureTradableContract parse_future_contract(string contract_txt, Exchange exchange = Exchange.CME) {
-         // Validate the future string length
-         if (!contract_txt.All(c => Char.IsLetterOrDigit(c)))
-            return null;
-         if (contract_txt.Length < 4) {
-            return null;
-         }
-
-         // Extract the symbol and the expiration month/year codes
-         string symbol = contract_txt.Substring(0, contract_txt.Length - 2);
-         char monthCode = contract_txt[contract_txt.Length - 2];
-         char yearCode = contract_txt[contract_txt.Length - 1];
-
-         // Map month code to month, return null if the month code is invalid
-         int month = monthCode switch {
-            'F' => 1,  // January
-            'G' => 2,  // February
-            'H' => 3,  // March
-            'J' => 4,  // April
-            'K' => 5,  // May
-            'M' => 6,  // June
-            'N' => 7,  // July
-            'Q' => 8,  // August
-            'U' => 9,  // September
-            'V' => 10, // October
-            'X' => 11, // November
-            'Z' => 12, // December
-            _ => 0 // Return 0 for invalid month code
-         };
-
-         if (month == 0) {
-            return null; // Invalid month code
-         }
-
-         // Calculate the year based on the current decade
-         int currentYear = DateTime.Now.Year;
-         int decadeStart = currentYear / 10 * 10;
-         int year = decadeStart + (yearCode - '0');
-
-         // If the year is in the past, assume the next decade
-         if (year < currentYear) {
-            year += 10;
-         }
-
-         // Create the FutureContract instance and set its properties
-         DateTime expiration = new DateTime(year, month, 1);
-         var futureContract = new FutureTradableContract {
-            symbol = symbol,
-            expiration = expiration,
-            exchange = exchange,
-            contract_text = contract_txt
-         };
-
-         return futureContract;
-      }
-      // Method to parse a futures spread, e.g. "CLX4-CLZ4"
-      public static FutureSpreadContract parse_future_spread_contract(string contract_txt, Exchange exchange = Exchange.CME) {
-         // Split the string by '-' to handle each leg
-         var spreadParts = contract_txt.Split('-');
-         if (spreadParts.Length < 2) {
-            return null; // Not a valid spread format
-         }
-
-         var spreadContract = new FutureSpreadContract { legs = new List<Tuple<double, FutureTradableContract>>() };
-
-         foreach (var part in spreadParts) {
-            // Split each leg by '*', assuming the format is weight*future or just future
-            string[] legParts = part.Split('*');
-            double weight = 1.0;
-            string futurePart;
-
-            if (legParts.Length == 2) {
-               if (!double.TryParse(legParts[0], out weight)) {
-                  return null; // Invalid weight, parsing failed
-               }
-               futurePart = legParts[1];
-            } else if (legParts.Length == 1) {
-               futurePart = legParts[0];
-            } else {
-               return null; // Invalid format, too many '*' parts
-            }
-
-            // Parse the future contract
-            var futureContract = parse_future_contract(futurePart, exchange);
-            if (futureContract == null) {
-               return null; // Failed to parse the future part
-            }
-
-            // Add the leg to the spread contract
-            spreadContract.legs.Add(new Tuple<double, FutureTradableContract>(weight, futureContract));
-         }
-         return spreadContract;
-      }
-
-      // public static ContractType DetermineFutureOrSpread(string contract) {
-      //    if (contract.Contains("-")) {
-      //       return ContractType.FutureSpread;
-      //    } else {
-      //       return ContractType.Future;
-      //    }
-      // }
-      #endregion
-
-      public static ContractType get_type(string contract_txt) {
-         FutureTradableContract fc = parse_future_contract(contract_txt);
-         if (fc != null) {
-            return ContractType.Future;
-         }
-         FutureSpreadContract sc = parse_future_spread_contract(contract_txt);
-         if (sc != null) {
-            return ContractType.FutureSpread;
-         }
-         // tehre are some non calendar spreads we cant parse
-         //Log.assert(false);
-         return bts.SilverPine.ContractUtils.ContractType.FutureSpread;
-      }
-
-      public static ContractCategory get_category(string symbol) {
-         var tp = get_symbol_category(symbol);
-         return tp.Item1;
-      }
-
-      public static Tuple<ContractCategory, string> get_symbol_category(string symbol) {
-         ContractUtils.FutureTradableContract fc = parse_future_contract(symbol);
-         if (fc != null) {
-            return Tuple.Create(ContractCategory.FutureString, fc.symbol);
-         }
-         var sc = parse_future_spread_contract(symbol);
-         if (sc != null) {
-            return Tuple.Create(ContractCategory.FutureString, sc.symbol);
-         }
-         return Tuple.Create(ContractCategory.Symbol, symbol);
-      }
-
-
-
-      public enum ContractType {
-         Spot,
-         Option,
-         Future,
-         FutureSpread,
-         OptionSpread
-      }
-      public enum ContractCategory {
-         Symbol = 0,
-         FutureString,
-         SpreadString
-      }
-      public enum CallPut {
-         Call,
-         Put
-      }
-      public enum Exchange {
-         CME
-      }
-
-   }
-}
@@ -1,23 +0,0 @@
-namespace bts.SilverPine {
-   using BCG;
-
-   // Finds the regions around specified events 
-   [DataStreamUserRespresentation("EventsPriceAction")]
-   public class EventsPriceActionStream : ViewStream {
-      public EventsPriceActionStream(string symbol, string event_type, string pre_event_period,
-            string post_event_period, int n_events) :
-         this(symbol, event_type, TimeBasedFilter.parse_duration(pre_event_period), TimeBasedFilter.parse_duration(post_event_period),
-         n_events, ViewStreamDatabase.instance) { }
-
-      public EventsPriceActionStream(string symbol, string event_type, TimeSpan pre_event_period,
-      TimeSpan post_event_period, int n_events, ViewStreamDatabase vsd) {
-         // TODO: figure out which bars to use, 1 min or 1 day or live, based on pre_event_period / post_event_period
-         FuturesActiveContractStream olhcv_str = new FuturesActiveContractStream(symbol);
-         ViewStream events_str = new EventsStream(vsd, event_type);
-         // todo
-         //events_str = new TopStream(events_str, n_events, "-ts");
-         ViewStream rs = new TimeProximityJoinStream(olhcv_str, "ts_event", events_str, "ts", pre_event_period, post_event_period);
-         mySetRealSource(rs);
-      }
-   }
-}
@@ -1,20 +0,0 @@
-namespace bts.SilverPine {
-   using BCG;
-
-   public class IndicatorStream : ViewStream {
-      enum Indicator {
-         StandardDeviation = 0
-      }
-      public class IndicatorInfo {
-         Indicator indicator;
-         IDictionary<string, object> parameters;
-      }
-      public IndicatorStream(string symbol_name, string indicator_name, string indicator_params_json) {
-
-      }
-      public IndicatorStream(string symbol_name, IndicatorInfo info) {
-
-      }
-
-   }
-}
@@ -1,57 +0,0 @@
-namespace bts.SilverPine {
-   using BCG;
-
-   using ZstdSharp.Unsafe;
-
-   [DataStreamUserRespresentation("AbsVolume")]
-   // Calculates the volume within a time window (this does not do rates, like highest daily volume etc,
-   // but with some thought can perhaps be extended)
-   public class AbsoluteVolumeStream : ViewStream {
-      // TODO, abstract this out
-      // adds contract_type to OHLCV stream
-      private static ViewStream _get_ohlcv_steam(string symbol, DateTime d1, DateTime d2) {
-         var ohlcv_params = OHLCVStream.get_params(d1, d2);
-         ViewStream rs = new OHLCVStream(symbol, ohlcv_params);
-         Func<IBlueTableSchema, IBlueTableSchema> create_schema =
-         s => BlueDataUtility.add_columns(s, Tuple.Create(typeof(string), "contract_type"));
-         rs = new SuperMapStream(rs,
-         s => create_schema(s),
-         r => {
-            IBlueTableSchema out_schema = create_schema(r.schema);
-            string contract_txt = r.get<string>("contract");
-            var contr_type = ContractUtils.get_type(contract_txt);
-            MutableBlueRow out_row = new MutableBlueRow(out_schema, r);
-            out_row.setValue("contract_type", contr_type.ToString());
-            return new IBlueRow[] { out_row };
-         });
-         return rs;
-      }
-
-      public AbsoluteVolumeStream(string symbol, string ctype, DateTime d1)
-      : this(symbol, ctype, d1, d1) { }
-
-      public AbsoluteVolumeStream(string symbol, string ctype, DateTime d1, DateTime d2)
-      : this(symbol, Enum.Parse<ContractUtils.ContractType>(ctype), d1, d2) {
-      }
-
-      static void swap_if_necessary(ref DateTime dt1, ref DateTime dt2) {
-         if (dt1 < dt2) {
-            DateTime aux = dt1;
-            dt1 = dt2;
-            dt2 = aux;
-         }
-      }
-
-      private AbsoluteVolumeStream(string symbol, ContractUtils.ContractType ctype, DateTime d1, DateTime d2) {
-         swap_if_necessary(ref d1, ref d2);
-         ViewStream ohlcv_str = _get_ohlcv_steam(symbol, d1, d2);
-         ohlcv_str = new FilterStream(ohlcv_str,
-            r => r.get_datetime("ts_event") <= d1 && r.get_datetime("ts_event") >= d2);
-         ohlcv_str = new FilterStream(ohlcv_str, r => String.Compare(ctype.ToString(), r.get<string>("contract_type"), true) == 0);
-         ViewStream rs = new MaxStream(ohlcv_str, "volume");
-         rs = new NewProjectStream(rs, "volume:total_volume contract !");
-         mySetRealSource(rs);
-      }
-
-   }
-}
@@ -1,152 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-
-using bts;
-
-namespace bts.SilverPine {
-   public static class IndicatorCalc {
-      public static double calculate_pearson_correlation(List<double> prices1, List<double> prices2) {
-         if (prices1 == null)
-            throw new ArgumentNullException(nameof(prices1), "First price list cannot be null.");
-         if (prices2 == null)
-            throw new ArgumentNullException(nameof(prices2), "Second price list cannot be null.");
-         if (prices1.Count != prices2.Count)
-            throw new ArgumentException("Price lists must have the same number of elements.");
-         if (prices1.Count < 2)
-            throw new ArgumentException("At least two data points are required to calculate correlation.");
-
-         int n = prices1.Count;
-         double sumX = 0;
-         double sumY = 0;
-         double sumX2 = 0;
-         double sumY2 = 0;
-         double sumXY = 0;
-
-         for (int i = 0; i < n; i++) {
-            double x = prices1[i];
-            double y = prices2[i];
-
-            sumX += x;
-            sumY += y;
-            sumX2 += x * x;
-            sumY2 += y * y;
-            sumXY += x * y;
-         }
-
-         double numerator = (n * sumXY) - (sumX * sumY);
-         double denominatorPart1 = (n * sumX2) - (sumX * sumX);
-         double denominatorPart2 = (n * sumY2) - (sumY * sumY);
-         double denominator = Math.Sqrt(denominatorPart1 * denominatorPart2);
-
-         if (denominator == 0)
-            throw new InvalidOperationException("Denominator is zero, correlation is undefined.");
-
-         return numerator / denominator;
-      }
-      public static double calculate_VWAP(List<Tuple<int, double>> volumePriceData) {
-         Log.Gen.info($"ChartingHelper.calculate_VWAP n_vals: {volumePriceData.Count}");
-         double totalVolume = 0;
-         double totalPriceVolume = 0;
-
-         foreach (var data in volumePriceData) {
-            int volume = data.Item1; // Volume
-            double price = data.Item2; // Closing price
-
-            // Accumulate total price-volume (price * volume) and total volume
-            totalPriceVolume += price * volume;
-            totalVolume += volume;
-         }
-
-         // VWAP is totalPriceVolume divided by totalVolume
-         return totalVolume > 0 ? totalPriceVolume / totalVolume : 0;
-      }
-
-      // Calculate Volatility (Standard Deviation of returns) over a given period
-      public static double calculate_volatility(IList<double> prices, int period = 252) {
-         if (prices == null || prices.Count < period + 1) {
-            throw new ArgumentException("Insufficient data to calculate volatility.");
-         }
-
-         // Calculate daily returns
-         var returns = new List<double>();
-         for (int i = 1; i < prices.Count; i++) {
-            double dailyReturn = (prices[i] - prices[i - 1]) / prices[i - 1];
-            returns.Add(dailyReturn);
-         }
-
-         // Get the returns over the desired period
-         var recentReturns = returns.Skip(returns.Count - period).Take(period).ToList();
-
-         // Calculate the mean of returns
-         double meanReturn = recentReturns.Average();
-
-         // Calculate the variance (sum of squared differences from the mean)
-         double variance = recentReturns.Sum(r => Math.Pow(r - meanReturn, 2)) / period;
-
-         // Volatility is the square root of variance (standard deviation)
-         double volatility = Math.Sqrt(variance);
-
-         // Annualize volatility by multiplying by the square root of 252 (number of trading days in a year)
-         return volatility * Math.Sqrt(252);
-      }
-
-      // Calculate SMA (Simple Moving Average) for a given period (e.g., 50, 100, or 200)
-      public static double calculate_SMA(IList<double> prices, int period = 50) {
-         if (prices == null || prices.Count < period) {
-            throw new ArgumentException("Insufficient data to calculate SMA.");
-         }
-
-         // Sum up the prices over the given period
-         double sum = prices.Skip(prices.Count - period).Take(period).Sum();
-
-         // Return the average
-         return sum / period;
-      }
-
-      public static double calculate_RSI(IList<double> prices, int period = 14) {
-         if (prices == null || prices.Count < period + 1) {
-            return Double.NaN;
-            //throw new ArgumentException("Insufficient data to calculate RSI.");
-         }
-
-         double gainSum = 0;
-         double lossSum = 0;
-
-         // Calculate the initial average gain and loss over the first 'period' days
-         for (int i = 1; i <= period; i++) {
-            double change = prices[i] - prices[i - 1];
-            if (change > 0) {
-               gainSum += change;
-            } else {
-               lossSum -= change; // Subtract since loss is negative
-            }
-         }
-
-         double avgGain = gainSum / period;
-         double avgLoss = lossSum / period;
-
-         // Calculate RSI using smoothed values
-         for (int i = period + 1; i < prices.Count; i++) {
-            double change = prices[i] - prices[i - 1];
-            if (change > 0) {
-               avgGain = (avgGain * (period - 1) + change) / period;
-               avgLoss = (avgLoss * (period - 1)) / period;
-            } else {
-               avgGain = (avgGain * (period - 1)) / period;
-               avgLoss = (avgLoss * (period - 1) - change) / period;
-            }
-         }
-
-         // Prevent division by zero in case of no losses
-         if (avgLoss == 0) {
-            return 100;
-         }
-
-         double rs = avgGain / avgLoss;
-         double rsi = 100 - (100 / (1 + rs));
-
-         return rsi;
-      }
-   }
-}
@@ -1,31 +0,0 @@
-namespace bts.SilverPine {
-   using BCG;
-
-   using bts;
-
-   [DataStreamUserRespresentation("DailyChange")]
-   // takes in the ohlcv stream and computes the closing vs closing difference
-   public class DailyChange : ViewStream {
-      public DailyChange(ViewStream ohlc_str) {
-         GlobalMapStream rs = new GlobalMapStream(ohlc_str, "ts_event", "contract",
-         (rows, idx) => {
-            double daily_change = Double.NaN;
-            double daily_change_perc = Double.NaN;
-            IBlueRow ir = rows.ElementAt((int)idx);
-            if (idx < rows.Count - 1) {
-               IBlueRow pr = rows.ElementAt((int)idx + 1);
-               double close = ir.get<double>("close"), prev_close = pr.get<double>("close");
-               daily_change = close - prev_close;
-               daily_change_perc = 100.0 * daily_change / prev_close;
-            }
-            IBlueTableSchema schema = ir.schema.add_columns(Tuple.Create(typeof(double), "daily_change"));
-            schema = schema.add_columns(Tuple.Create(typeof(double), "daily_change_perc"));
-            MutableBlueRow nr = new MutableBlueRow(schema, ir);
-            nr.setValue("daily_change", daily_change);
-            nr.setValue("daily_change_perc", daily_change_perc);
-            return nr;
-         });
-         mySetRealSource(rs);
-      }
-   }
-}
Author	SHA1	Message	Date
Adeel Khan	11e6d25001	Add: Incremental Books	2024-10-04 17:05:02 +00:00
Mohammad Arham	83d66ae9d7	Remove: code to remove secuties	2024-10-03 18:03:10 +05:00
Mohammad Arham	e5e74b7b42	Add: code to connect to Live databento	2024-10-03 18:03:08 +05:00
Mohammad Arham	6ce2fb2cc5	Add: databento repllay filename to cofig	2024-10-03 18:01:49 +05:00
Mohammad Arham	9d63a004ec	Add: added onix fix to lib to build dir	2024-10-03 18:01:49 +05:00
Mohammad Arham	ccd53d71ea	Add: entry for onix fix	2024-10-03 18:01:49 +05:00
Mohammad Arham	74a0235227	Add databento to external	2024-10-03 18:01:49 +05:00
Taha Afzal	42bb101b43	Adding comments	2024-10-03 18:01:49 +05:00
Taha Afzal	c53561c6ef	Creating scaffolding for incremental book updates in GeneralMarketDataListener	2024-10-03 18:01:49 +05:00
Taha	3701a70bd9	Fixing build relted to BCG vs bts namespace	2024-10-03 18:01:49 +05:00
Taha	df56de3dd2	Cleaning up FileStreams	2024-10-03 18:01:49 +05:00
Taha Afzal	c9a418ce4b	spaces, newlines?	2024-10-03 18:01:49 +05:00
Taha	aaad404507	Added benzinga streamd to MDPub's test config	2024-10-03 18:01:49 +05:00
Taha Afzal	45acaa20b7	PubSubSingleTagState is not needed any more	2024-10-03 18:01:49 +05:00
Taha Afzal	96a08500fb	Fixing problems where PSS's client socket's IP was different than what the client was sending in PubSubClientSubscription	2024-10-03 18:01:49 +05:00
Taha Afzal	510e7b0050	Fixing PubsubEchoTestStream	2024-10-03 18:01:47 +05:00
Taha Afzal	2ab76c753e	Merging	2024-10-03 17:57:34 +05:00
Taha Afzal	a02c6b77d4	src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnection_AsyncWrapper.cs	2024-10-03 17:57:32 +05:00
Taha	6a289a5561	Icomplete SharedStreamsConfig was breaking build, fixing	2024-10-03 17:55:30 +05:00
Taha	fddd310a03	Added new benzinga streams	2024-10-03 17:55:30 +05:00
Taha	d2a2d5fb37	Adding TableMetaDataStream	2024-10-03 17:55:30 +05:00
Taha Afzal	4b76442a8f	Cleaning Config 1.0 (not completely done)	2024-10-03 17:55:30 +05:00
Taha	09b863f2ab	Added pubsub routes to the Title of DSQA	2024-10-03 17:55:30 +05:00
Taha	dd221aba10	Fixing the bug where messages were being sent by the pubsubserver out of order	2024-10-03 17:55:28 +05:00
Taha	5ef862b026	some buf gixes + cleanup	2024-10-03 17:54:28 +05:00
Taha	cc43e9d3a4	PubSuMessageReactor empties the message queue (instead oif one elem ata a time) in the main loop	2024-10-03 17:52:47 +05:00
Taha	b6eb727bc9	PubSubServer's linux issues fixed	2024-10-03 17:52:38 +05:00
John Colvin	8ace69ab74	Add CBOE to spark admin webapp	2024-10-03 17:46:51 +05:00
John Colvin	dc88b4ee10	Add TAIFEX to spark admin webapp	2024-10-03 17:46:51 +05:00
Taha	dee32dead4	Fixing implied vols for SPX (market data was getting divide dby the fwd)	2024-10-03 17:46:51 +05:00
John Colvin	b2e022505f	fix build: Add ".list" to SymbolLists config For new exchanges, TAIFEX and CBOE	2024-10-03 17:46:51 +05:00
John Colvin	ac9b3ec365	fix (build): Add break statement to new venue switch cases Namely, TAIFEX and CBOE	2024-10-03 17:46:51 +05:00
John Colvin	ed89da7f38	fix: Preserve vs exp sec type (Option) when computing implied vol Because the exp keys in the data map have sec type Option.	2024-10-03 17:46:51 +05:00
John Colvin	a334d9be88	ui: Highlight launcher buttons when selected Namely, the expandable menu buttons in Analytics shell module	2024-10-03 17:46:51 +05:00
John Colvin	aee56569d1	Replace trivial refs to ThreadedBlackboard w/ IBus	2024-10-03 17:46:51 +05:00
John Colvin	7fdd3f6b1b	fix: Move accounts subscription to dispatcher thread We update a collection that is bound to the view in the callback (mutation ops throw if performed on a different thread).	2024-10-03 17:46:51 +05:00
John Colvin	3e1949d94f	fix: Check 'HasTimestamp' before accessing prop in man fill row Pending adds (i.e., unpersisted new rows) don't have timestamps. In such cases, the 'Timestamp' getter throws an exception.	2024-10-03 17:46:51 +05:00
John Colvin	4cb3dbc313	Cache/index options for faster lookups in SystemContractData	2024-10-03 17:46:51 +05:00
John Colvin	0268bd476f	fix: Include venue w/ symbol when publishing interp greeks	2024-10-03 17:46:51 +05:00
John Colvin	2511b0d966	Fix typo in comment	2024-10-03 17:46:50 +05:00
John Colvin	92456649df	fix: SelectorList.RemoveAt doesn't need 'convert_back' delegate So don't throw an exception if it is null.	2024-10-03 17:46:50 +05:00
John Colvin	ec32078dd5	fix: Don't assume venue is in spread def string Pass it as an argument to the parse fn. This should be fine for now, but obviously a singular venue param won't work for cross exchange spreads.	2024-10-03 17:46:50 +05:00
John Colvin	6bcce832ac	improvement: Move system contracts sub off of dispatcher thread Also add 200ms throttle. Contracts shouldn't change that often, so we don't need lightening speed here. Furthermore, such changes may trigger expensive cpu ops. Note: this change also replaces refs to ThreadedBlackboard w/ IBus (the former is somewhat tethered to the dispatcher thread, particularly by its timer_callback).	2024-10-03 17:46:50 +05:00
John Colvin	85529b2232	fix: Pass symbol/venue from vs usage cfg to AnalyticsService proto	2024-10-03 17:46:50 +05:00
John Colvin	c3cb8995bf	cleanup: Remove unused code	2024-10-03 17:46:50 +05:00
John Colvin	ab2f0f975e	refactor: Index vol and theos data by symbol/venue tuple Rather than just symbol, to avoid conflicts when the same symbol exists at multiple exchanges	2024-10-03 17:46:50 +05:00
John Colvin	2e133ea182	cleanup: Remove redundant type VolSurfaceSymbolStructure There is an identical type (in terms of prop types and corresponding proto indices) called VolSurfaceSymbolRelationship that is used in more places.	2024-10-03 17:46:50 +05:00
John Colvin	30615f3429	Move nested vm classes into own files VolSurfaceSettingsEditor has too much code (still). It is hard to navigate.	2024-10-03 17:46:50 +05:00
John Colvin	61371804a2	Simplify driver symbol structure in vs settings vm The code was confusing and difficult to follow.	2024-10-03 17:46:50 +05:00
John Colvin	02d82e39bc	fix: Raise prop changed event when is_selected changes	2024-10-03 17:46:50 +05:00
John Colvin	50188e6688	Add venue to vs calc fns	2024-10-03 17:46:50 +05:00
John Colvin	5137785f7f	fix: Provide venue param w/ symbol when computing fut slide	2024-10-03 17:46:50 +05:00
John Colvin	def021612c	diag: Log added/removed exps when syncing vs input w/ system exps This shouldn't happen often, but if it does, we should know about it. PS: We recently fixed a bug caused by inequalities between VolSurfaceExpirations in vs input, and in SystemContractData. User inputs to the vs grid would be immediately reverted, as the exps associated with the inputs were not found in the SystemContractData. This bug would have been easier to track down with this logging in place.	2024-10-03 17:46:50 +05:00
John Colvin	217c5be131	Add venue to VolSurface(DisplayGrid)Expiration So we can handle symbol/exp date conflicts for securities that differ only by venue (exchange)	2024-10-03 17:46:50 +05:00
John Colvin	a75441cd07	Clean up VolSurfaceExpiration * Enable nullable lang feat * Use file-scoped namespace * Sort members * Include sec type in Compare impl	2024-10-03 17:46:50 +05:00
John Colvin	78eb096d6f	Add null check in SelectorList ctr	2024-10-03 17:46:50 +05:00
John Colvin	2b66f46db1	cleanup: Remove unused code	2024-10-03 17:46:50 +05:00
John Colvin	ae3029d9ed	refactor: Replace ThreadedBlackboard w/ IBus In SingleVolSurfaceCommittedOutputRetriever	2024-10-03 17:46:50 +05:00
John Colvin	d4a0603933	Clean up VolSurfaceTimeParameters * Enable nullable lang feat * Use file-scoped namespace * Implement ICloneable	2024-10-03 17:46:50 +05:00
John Colvin	db6e96cf00	cleanup: Remove unnecessary ternary op	2024-10-03 17:46:50 +05:00
John Colvin	371db4afd2	fix: Actually use (sec) type arg in to_security_expiration	2024-10-03 17:46:50 +05:00
John Colvin	08484a0954	fix: Check security_type in vs exp Equals impl	2024-10-03 17:46:50 +05:00
John Colvin	39db44ed63	fix: Maintain sec type when converting SecurityExpiration to vs exp Avoids data loss	2024-10-03 17:46:50 +05:00
John Colvin	b4dca8e23f	refactor: Get rid of sec type 'Option' default (magic const) Better to be explict	2024-10-03 17:46:50 +05:00
John Colvin	b4bf06d17a	Fix selection binding issues in vs settings editor This specifically addresses issues in the driver symbol grids. Unfortunately, we can't bind directly to ComboCheckBox.SelectedItems because the view is contained in TabControl. When the selected tab changes, the previously selected tab clears its ItemsSource (and consequently the SelectedItems) as a side effect of being unloaded.	2024-10-03 17:46:50 +05:00
John Colvin	8c17705f02	refactor: Enable nullable (ref types) in ImmutableListExtensions	2024-10-03 17:46:50 +05:00
John Colvin	551f26e5f3	Fix typo in namespace bts.collection -> bts.collections	2024-10-03 17:46:50 +05:00
John Colvin	f4adf35e49	fix: Use ComboCheckBoxItem.IsSelectedProperty Instead of Selector.IsSelectedProperty. Unlike ComboBoxItem, ComboCheckItem's IsSelectedProperty isn't linked to Selector.IsSelectedProperty.	2024-10-03 17:46:50 +05:00
John Colvin	90128863d9	Add currency column to mview	2024-10-03 17:46:50 +05:00
John Colvin	300831c97a	fix: Use SetCurrentValue instead of SetValue So we propagate values to/from the binding data source, rather than overwrite the data source itself.	2024-10-03 17:46:50 +05:00
John Colvin	f3f6a82d9a	Add venue, sec type dropdown filters to exp params ui	2024-10-03 17:46:50 +05:00
John Colvin	7c8a41b2b7	fix: Add null check in ComboCheckBox.SelectedItemsCollectionChanged	2024-10-03 17:46:50 +05:00
John Colvin	408114c1f6	fix: Disconnect old INotifyCollectionChanged source when dp changes	2024-10-03 17:46:50 +05:00
John Colvin	5c9be0bfb9	fix: Index opt/fut exps by venue in vs analytics	2024-10-03 17:46:50 +05:00
John Colvin	c0f970ab89	fix: Support multiple exchanges in exp params view/vm	2024-10-03 17:46:50 +05:00
John Colvin	25e1804286	cleanup: Remove unused 'option_symbol' param from compute_atm_data	2024-10-03 17:46:50 +05:00
John Colvin	3c84dc48f6	fix: Filter by exch when getting atm data for sheets This is necessary now that we support exchanges with overlapping securities	2024-10-03 17:46:50 +05:00
John Colvin	9ae20f0c4d	fix: Remove sec id prop from SecurityExpiration This class isn't designed to represent a particular contract; it's basically a key to get option theos.	2024-10-03 17:46:50 +05:00
John Colvin	e8a72e7bd9	refactor: Add 'onDispatcher' param to 'ThreadedBlackboard.subscribe' This will allow us to gradually move subscriptions off of the dispatcher (render) thread. When the render thread is pegged, the ui is laggy and unresponsive. The new param defaults to true, so it won't change anything until we explicitly set it to false in 'subscribe' calls.	2024-10-03 17:46:50 +05:00
John Colvin	e99e2d817a	fix: Remove redundant SparkTimer threads This addresses some slowness on the front end. Prior to this commit, each ThreadedBlackboard would start its own timer thread. Since multiple ThreadedBlackboard run concurrently, this multiplied the frequency of the SparkTimer callbacks by the number of active ThreadedBlackboards. The fix is to start this thread in an IShellDaemon rather than the ThreadedBlackboard. This avoids duplicate threads because the former is guarenteed to be instantiated only once, via dependency injection.	2024-10-03 17:46:50 +05:00
John Colvin	7f4468c812	refactor: Simplify FakeMDDaemon code that delays post Use Task.Delay/ContinueWith instead of subscribing to the SparkTimer tag.	2024-10-03 17:46:50 +05:00
John Colvin	6206cbc425	refactor: Use IBus instead of ThreadedBlackboard This should address some slowness/mouse lag in the front end (recently reported by Brandon at Immutable Cap)	2024-10-03 17:46:50 +05:00
John Colvin	21e50745e2	fix: Detect and log ccapi depth msg parse errors This fixes a crash that would happen if a depth item's price or quantity were an empty string (now we log an error and keep going).	2024-10-03 17:46:50 +05:00
John Colvin	990e21e387	fix: Actually store currency in shared mem This field has always been in the Security proto, though it until now it had been omitted from the shared memory representation (FixedSecurity).	2024-10-03 17:46:50 +05:00
Mohammad Arham	357be39db3	Merge branch 'development.new_exchanges' of bitbucket.org:bluetradesys/spark into development.new_exchanges	2024-09-24 07:34:09 -04:00
Mohammad Arham	68e04cb3f5	FIX: CHnaged databanto symbol source to asset	2024-09-24 11:31:49 +00:00
Mohammad Arham	b4a5d5c832	Added cpp to security common	2024-09-24 10:53:57 +00:00
Adeel Khan	242fd758f4	Add: Taifex Trades	2024-09-16 01:20:44 +05:00