refactor: Use generic collections in PrimitiveManager

The class that instantiates ViewStreams via reflection
Fix typo in NLStream's namespace
2024-11-02 16:48:42 +00:00 · 2024-11-02 02:57:19 +00:00 · 2024-10-30 20:42:14 +00:00 · 2024-10-30 20:13:40 +00:00 · 2024-10-30 19:37:48 +00:00 · 2024-10-30 19:37:48 +00:00
951 changed files with 26506 additions and 30894 deletions
@@ -87,6 +87,10 @@ RUN dpkg -i /emumba/emumba-deribit-connector-3.3.1-Linux.LICENSE_DISABLED.deb \
 # Pcap
 RUN apt-get update && apt-get install -y libpcap0.8-dev

+# Databento (deps)
+RUN apt-get update && apt-get -y install libzstd-dev
+
+# Docker in Docker
 RUN apt-get update && apt-get install -y \
    docker.io \
    docker-compose \
@@ -16,11 +16,20 @@
        "program": "spark_api_server.exe",
        "args": [
          "run",
-          "--sparkd",
-          "http://127.0.0.1:8000/config"
        ],
        "cwd": "build/Debug/bin/net7.0",
      },
+      "linux": {
+        "preLaunchTask": "Build Api Server (Debug)",
+        "program": "spark_api_server",
+        "args": [
+          "run",
+        ],
+        "cwd": "build/Debug/bin/net7.0",
+      },
+      "env": {
+        "ASPNETCORE_ENVIRONMENT": "Development",
+      },
      "stopAtEntry": false,
      "console": "integratedTerminal"
    },
@@ -33,11 +42,19 @@
        "program": "spark_api_server.exe",
        "args": [
          "run",
-          "--sparkd",
-          "http://127.0.0.1:8000/config"
        ],
        "cwd": "build/Debug/bin/net7.0",
      },
+      "linux": {
+        "program": "spark_api_server",
+        "args": [
+          "run",
+        ],
+        "cwd": "build/Debug/bin/net7.0",
+      },
+      "env": {
+        "ASPNETCORE_ENVIRONMENT": "Development",
+      },
      "stopAtEntry": false,
      "console": "integratedTerminal"
    },
@@ -105,6 +122,18 @@
      "stopAtEntry": false,
      "console": "integratedTerminal"
    },
+    {
+      "name": "(gdb) Launch",
+      "type": "cppdbg",
+      "request": "launch",
+      "program": "/home/bts/spark/build/sparkd-debug-clang/server/sparkd",
+      "args": ["run"],
+      "stopAtEntry": false,
+      "cwd": "/home/bts/spark/build/sparkd-debug-clang/server",
+      "environment": [],
+      "externalConsole": false,
+      "MIMode": "gdb",
+    },
    {
      "name": "DataStreamQueryAnalyzer",
      "type": "coreclr",
@@ -234,7 +263,35 @@
        "program": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd/sparkd",
      },
      "args": [
-        "test-py-md-provider"
+        "run"
+      ],
+      "cwd": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd",
+      "stopAtEntry": false,
+      "externalConsole": false,
+      "MIMode": "gdb",
+      "setupCommands": [
+        {
+          "description": "Enable pretty-printing for gdb",
+          "text": "-enable-pretty-printing",
+          "ignoreFailures": true
+        },
+        {
+          "description": "Set Disassembly Flavor to Intel",
+          "text": "-gdb-set disassembly-flavor intel",
+          "ignoreFailures": true
+        }
+      ]
+    },
+    {
+      "name": "Sparkd Web (Skip Build)",
+      "type": "cppdbg",
+      "request": "launch",
+      "program": "",
+      "linux": {
+        "program": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd/sparkd",
+      },
+      "args": [
+        "web"
      ],
      "cwd": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd",
      "stopAtEntry": false,
@@ -1,5 +1,5 @@
 cmake_minimum_required(VERSION 3.12)
-project(spark_root)
+project(spark)
 set(CMAKE_CXX_STANDARD 20)

 if(WIN32)
@@ -33,91 +33,85 @@ else()
  set(BUILD_ADMIN_WEB_APP ON CACHE BOOL "Build the Admin WebApp")
 endif()

-include_directories(BEFORE SYSTEM ${EXTERNALS_INCLUDE} ${EXTERNALS_ROOT}/include/zlib)
-link_directories(${EXTERNALS_LIB})
-
 add_compile_definitions(SHARED_MEMORY_PREFIX="${SHARED_MEMORY_PREFIX}")
 if(DEFINED APP_DATA_DIR)
  add_compile_definitions(APP_DATA_DIR="${APP_DATA_DIR}")
 endif()
 add_compile_definitions(EXPORT_MD_CONSUMER="${EXPORT_MD_CONSUMER}")
-if(WIN32)
-  # subdirs
-  add_subdirectory(src/lib_spark)
-else()
-  if(BUILD_SPARKD)
-    # Note: All macros that begin w/ 'CCAPI' are ccapi's macros (defined and/or
-    # read by their code). We override some below.
-    add_compile_definitions(SPARK_ENABLE_CCAPI)

-    # This allows us to override hardcoded exchange-urls with ones read from
-    # config (i.e., resolved at runtime rather than compile-time)
-    add_compile_definitions(CCAPI_EXPOSE_INTERNAL)
+if(BUILD_SPARKD)
+  # Note: All macros that begin w/ 'CCAPI' are ccapi's macros (defined and/or
+  # read by their code). We override some below.
+  add_compile_definitions(SPARK_ENABLE_CCAPI)

-    # Enable ccapi services
-    add_compile_definitions(CCAPI_ENABLE_SERVICE_MARKET_DATA)
-    add_compile_definitions(CCAPI_ENABLE_SERVICE_EXECUTION_MANAGEMENT)
-    # Enable ccapi exchanges
-    add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE)
-    add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE_COIN_FUTURES)
-    add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE_US)
-    add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE_USDS_FUTURES)
-    add_compile_definitions(CCAPI_ENABLE_EXCHANGE_COINBASE)
-    add_compile_definitions(CCAPI_ENABLE_EXCHANGE_DERIBIT)
-    add_compile_definitions(CCAPI_ENABLE_EXCHANGE_OKX)
+  # This allows us to override hardcoded exchange-urls with ones read from
+  # config (i.e., resolved at runtime rather than compile-time)
+  add_compile_definitions(CCAPI_EXPOSE_INTERNAL)

-    set(CCAPI_LOG_LEVEL
-      "Warn"
-      CACHE STRING "<Fatal|Error|Warn|Info|Debug|Trace> Log level for ccapi")
-    string(TOUPPER "${CCAPI_LOG_LEVEL}" _ccapi_log_level_upper)
-    add_compile_definitions("CCAPI_ENABLE_LOG_${_ccapi_log_level_upper}")
+  # Enable ccapi services
+  add_compile_definitions(CCAPI_ENABLE_SERVICE_MARKET_DATA)
+  add_compile_definitions(CCAPI_ENABLE_SERVICE_EXECUTION_MANAGEMENT)
+  # Enable ccapi exchanges
+  add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE)
+  add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE_COIN_FUTURES)
+  add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE_US)
+  add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE_USDS_FUTURES)
+  add_compile_definitions(CCAPI_ENABLE_EXCHANGE_COINBASE)
+  add_compile_definitions(CCAPI_ENABLE_EXCHANGE_DERIBIT)
+  add_compile_definitions(CCAPI_ENABLE_EXCHANGE_OKX)

-    # ** Binance **
-    add_compile_definitions(CCAPI_BINANCE_URL_REST_BASE="https://api.binance.com")
-    add_compile_definitions(BINANCE_INSTRUMENTS_PATH="/api/v3/exchangeInfo")
-    add_compile_definitions(BINANCE_PRICE_STATS_PATH="/api/v3/ticker/24hr")
+  set(CCAPI_LOG_LEVEL
+    "Warn"
+    CACHE STRING "<Fatal|Error|Warn|Info|Debug|Trace> Log level for ccapi")
+  string(TOUPPER "${CCAPI_LOG_LEVEL}" _ccapi_log_level_upper)
+  add_compile_definitions("CCAPI_ENABLE_LOG_${_ccapi_log_level_upper}")

-    # ** Binance US **
-    add_compile_definitions(BINANCE_US_INSTRUMENTS_PATH="/api/v3/exchangeInfo")
-    add_compile_definitions(BINANCE_US_PRICE_STATS_PATH="/api/v3/ticker/24hr")
+  # ** Binance **
+  add_compile_definitions(CCAPI_BINANCE_URL_REST_BASE="https://api.binance.com")
+  add_compile_definitions(BINANCE_INSTRUMENTS_PATH="/api/v3/exchangeInfo")
+  add_compile_definitions(BINANCE_PRICE_STATS_PATH="/api/v3/ticker/24hr")

-    # ** Binance USDS Futures **
-    add_compile_definitions(CCAPI_BINANCE_USDS_FUTURES_URL_REST_BASE="https://dapi.binance.com")
-    add_compile_definitions(BINANCE_USDS_FUTURES_INSTRUMENTS_PATH="/dapi/v1/exchangeInfo")
-    add_compile_definitions(BINANCE_USDS_FUTURES_PRICE_STATS_PATH="/dapi/v1/ticker/24hr")
+  # ** Binance US **
+  add_compile_definitions(BINANCE_US_INSTRUMENTS_PATH="/api/v3/exchangeInfo")
+  add_compile_definitions(BINANCE_US_PRICE_STATS_PATH="/api/v3/ticker/24hr")

-    # ** Deribit **
-    add_compile_definitions(DERIBIT_INSTRUMENT_PATH="/api/v2/public/get_instrument")
-    add_compile_definitions(DERIBIT_INSTRUMENTS_PATH="/api/v2/public/get_instruments")
-    add_compile_definitions(DERIBIT_COMBOS_PATH="/api/v2/public/get_combos")
-    add_compile_definitions(DERIBIT_TICKER_PATH="/api/v2/public/ticker")
-    add_compile_definitions(DERIBIT_CURRENCIES_METHOD="public/get_currencies")
-    add_compile_definitions(DERIBIT_INSTRUMENT_METHOD="public/get_instrument")
-    add_compile_definitions(DERIBIT_INSTRUMENTS_METHOD="public/get_instruments")
-    add_compile_definitions(DERIBIT_COMBO_DETAILS_METHOD="public/get_combo_details")
-    add_compile_definitions(DERIBIT_COMBOS_METHOD="public/get_combos")
+  # ** Binance USDS Futures **
+  add_compile_definitions(CCAPI_BINANCE_USDS_FUTURES_URL_REST_BASE="https://dapi.binance.com")
+  add_compile_definitions(BINANCE_USDS_FUTURES_INSTRUMENTS_PATH="/dapi/v1/exchangeInfo")
+  add_compile_definitions(BINANCE_USDS_FUTURES_PRICE_STATS_PATH="/dapi/v1/ticker/24hr")

-    # ** Okx **
-    add_compile_definitions(OKX_INSTRUMENTS_PATH="/api/v5/public/instruments")
-    add_compile_definitions(OKX_UNDERLYING_PATH="/api/v5/public/underlying")
-  endif()
+  # ** Deribit **
+  add_compile_definitions(DERIBIT_INSTRUMENT_PATH="/api/v2/public/get_instrument")
+  add_compile_definitions(DERIBIT_INSTRUMENTS_PATH="/api/v2/public/get_instruments")
+  add_compile_definitions(DERIBIT_COMBOS_PATH="/api/v2/public/get_combos")
+  add_compile_definitions(DERIBIT_TICKER_PATH="/api/v2/public/ticker")
+  add_compile_definitions(DERIBIT_CURRENCIES_METHOD="public/get_currencies")
+  add_compile_definitions(DERIBIT_INSTRUMENT_METHOD="public/get_instrument")
+  add_compile_definitions(DERIBIT_INSTRUMENTS_METHOD="public/get_instruments")
+  add_compile_definitions(DERIBIT_COMBO_DETAILS_METHOD="public/get_combo_details")
+  add_compile_definitions(DERIBIT_COMBOS_METHOD="public/get_combos")

-  # subdirs
-  if (BUILD_SPARKD)
-    add_subdirectory(src/lib_system)
-    add_subdirectory(src/lib_domain)
-    add_subdirectory(src/lib_python_md)
-    add_subdirectory(src/app_sparkd)
-    add_subdirectory(src/data_files)
-  endif()
-  add_subdirectory(src/lib_spark)
-  if (BUILD_ADMIN_WEB_APP)
-    add_subdirectory(src/Spark.Admin.WebApp)
-  endif()
-  if(BUILD_DOCS)
-    add_subdirectory(doc/user)
-  endif()
-  if(BUILD_SDK)
-    add_subdirectory(src/sdk)
-  endif()
+  # ** Okx **
+  add_compile_definitions(OKX_INSTRUMENTS_PATH="/api/v5/public/instruments")
+  add_compile_definitions(OKX_UNDERLYING_PATH="/api/v5/public/underlying")
+endif()
+
+# subdirs
+add_subdirectory(src/lib_system)
+add_subdirectory(src/lib_domain)
+add_subdirectory(src/lib_http)
+if (BUILD_SPARKD)
+  add_subdirectory(src/lib_python_md)
+  add_subdirectory(src/app_sparkd)
+  add_subdirectory(src/data_files)
+endif()
+add_subdirectory(src/lib_spark)
+if (BUILD_ADMIN_WEB_APP)
+  add_subdirectory(src/Spark.Admin.WebApp)
+endif()
+if(BUILD_DOCS)
+  add_subdirectory(doc/user)
+endif()
+if(BUILD_SDK)
+  add_subdirectory(src/sdk)
 endif()
@@ -28,7 +28,7 @@
        "CMAKE_SHARED_LINKER_FLAGS_RELEASE": "/OPT:REF /OPT:ICF /LTCG",
        "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
        "EXTERNALS_ROOT": "${sourceDir}/externals/windows",
-        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
+        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf",
        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0;${sourceDir}/build/$<CONFIG>/bin/net7.0-windows",
        "EXPORT_MD_CONSUMER": {
          "type": "BOOL",
@@ -53,6 +53,7 @@
        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
        "CMAKE_PREFIX_PATH": "${sourceDir}/cmake/;${sourceDir}/.venv/lib/python3.10/site-packages/pybind11/share/cmake/pybind11",
        "CMAKE_EXE_LINKER_FLAGS": "-Wno-undef",
+        "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
        "EXTERNALS_ROOT": "${sourceDir}/externals/linux",
        "BUILD_SDK": {
          "type": "BOOL",
@@ -76,7 +77,7 @@
          "value": "ON"
        },
        "CCAPI_LOG_LEVEL": "Info",
-        "CONFIG_DEFAULTS_PATH": "config/profiles/md_dev/*.conf;config/profiles/default/*.conf"
+        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/md_dev/*.conf;spark/config/profiles/default/*.conf"
      }
    },

@@ -87,7 +88,7 @@
      "cacheVariables": {
        "CMAKE_BUILD_TYPE": "Release",
        "CCAPI_LOG_LEVEL": "Warn",
-        "CONFIG_DEFAULTS_PATH": "config/profiles/md_prod/*.conf;config/profiles/default/*.conf"
+        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/md_prod/*.conf;spark/config/profiles/default/*.conf"
      }
    },

@@ -167,8 +168,8 @@
          "type": "BOOL",
          "value": "ON"
        },
-        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/Spark.Api.Server/$<CONFIG>/bin/net7.0",
+        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf",
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
        "EXPORT_MD_CONSUMER": {
          "type": "BOOL",
          "value": "ON"
@@ -206,7 +207,7 @@
      "description": "Data Stream utils ported over from Edge",
      "binaryDir": "${sourceDir}/build/${presetName}",
      "cacheVariables": {
-        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf",
+        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf;spark/config/profiles/data_stream/*.conf",
        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0;${sourceDir}/build/$<CONFIG>/bin/net7.0-windows"
      }
    },
@@ -239,8 +240,7 @@
        "CMAKE_CXX_COMPILER": "clang++",
        "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
-        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf",
-        "CONFIG_PROFILES": "default;data_stream",
+        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf;spark/config/profiles/data_stream/*.conf",
        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0;${sourceDir}/build/$<CONFIG>/bin/net8.0",
        "EXPORT_MD_CONSUMER": {
          "type": "BOOL",
@@ -14,6 +14,16 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "spark_test", "src\client\sp
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Spark.Console", "src\client\Spark.Console\Spark.Console.csproj", "{89D666E6-B500-498F-B026-4D167F68ADFC}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Net", "src\DataStreamSuite\Bts.Net\Bts.Net.csproj", "{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamCore", "src\DataStreamSuite\DataStreamCore\DataStreamCore.csproj", "{CB62D5D2-3495-4488-BFBC-7C7A3725A1EE}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStream", "src\DataStreamSuite\DataStream\DataStream.csproj", "{05995ED5-650A-4955-9DFC-3242B737E77A}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SharedStreams", "src\DataStreamSuite\SharedStreams\SharedStreams.csproj", "{A8B861DE-E098-4291-98EE-76AA0F5F5A7B}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PubSub", "src\DataStreamSuite\PubSub\PubSub.csproj", "{EFA9F78D-083E-457A-BABB-9386C88C32C0}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|x64 = Debug|x64
@@ -70,6 +80,46 @@ Global
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.Release|x64.Build.0 = Release|x64
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|x64.ActiveCfg = Debug|x64
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|x64.Build.0 = Debug|x64
+		{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.Debug|x64.Build.0 = Debug|Any CPU
+		{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.Release|x64.ActiveCfg = Release|Any CPU
+		{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.Release|x64.Build.0 = Release|Any CPU
+		{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{CB62D5D2-3495-4488-BFBC-7C7A3725A1EE}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{CB62D5D2-3495-4488-BFBC-7C7A3725A1EE}.Debug|x64.Build.0 = Debug|Any CPU
+		{CB62D5D2-3495-4488-BFBC-7C7A3725A1EE}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{CB62D5D2-3495-4488-BFBC-7C7A3725A1EE}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{CB62D5D2-3495-4488-BFBC-7C7A3725A1EE}.Release|x64.ActiveCfg = Release|Any CPU
+		{CB62D5D2-3495-4488-BFBC-7C7A3725A1EE}.Release|x64.Build.0 = Release|Any CPU
+		{CB62D5D2-3495-4488-BFBC-7C7A3725A1EE}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{CB62D5D2-3495-4488-BFBC-7C7A3725A1EE}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{05995ED5-650A-4955-9DFC-3242B737E77A}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{05995ED5-650A-4955-9DFC-3242B737E77A}.Debug|x64.Build.0 = Debug|Any CPU
+		{05995ED5-650A-4955-9DFC-3242B737E77A}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{05995ED5-650A-4955-9DFC-3242B737E77A}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{05995ED5-650A-4955-9DFC-3242B737E77A}.Release|x64.ActiveCfg = Release|Any CPU
+		{05995ED5-650A-4955-9DFC-3242B737E77A}.Release|x64.Build.0 = Release|Any CPU
+		{05995ED5-650A-4955-9DFC-3242B737E77A}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{05995ED5-650A-4955-9DFC-3242B737E77A}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{A8B861DE-E098-4291-98EE-76AA0F5F5A7B}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{A8B861DE-E098-4291-98EE-76AA0F5F5A7B}.Debug|x64.Build.0 = Debug|Any CPU
+		{A8B861DE-E098-4291-98EE-76AA0F5F5A7B}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{A8B861DE-E098-4291-98EE-76AA0F5F5A7B}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{A8B861DE-E098-4291-98EE-76AA0F5F5A7B}.Release|x64.ActiveCfg = Release|Any CPU
+		{A8B861DE-E098-4291-98EE-76AA0F5F5A7B}.Release|x64.Build.0 = Release|Any CPU
+		{A8B861DE-E098-4291-98EE-76AA0F5F5A7B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{A8B861DE-E098-4291-98EE-76AA0F5F5A7B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{EFA9F78D-083E-457A-BABB-9386C88C32C0}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{EFA9F78D-083E-457A-BABB-9386C88C32C0}.Debug|x64.Build.0 = Debug|Any CPU
+		{EFA9F78D-083E-457A-BABB-9386C88C32C0}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{EFA9F78D-083E-457A-BABB-9386C88C32C0}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{EFA9F78D-083E-457A-BABB-9386C88C32C0}.Release|x64.ActiveCfg = Release|Any CPU
+		{EFA9F78D-083E-457A-BABB-9386C88C32C0}.Release|x64.Build.0 = Release|Any CPU
+		{EFA9F78D-083E-457A-BABB-9386C88C32C0}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{EFA9F78D-083E-457A-BABB-9386C88C32C0}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -1,3 +1,5 @@
+include_guard(GLOBAL)
+
 macro(export_to_parent)
  foreach(var IN ITEMS ${ARGN})
    set(${var} ${${var}} PARENT_SCOPE)
@@ -1,66 +1,142 @@
 include_guard(DIRECTORY)
 include(${CMAKE_SOURCE_DIR}/cmake/ExportToParent.cmake)
-include(${CMAKE_SOURCE_DIR}/cmake/TernaryOperator.cmake)
+include(${CMAKE_SOURCE_DIR}/cmake/InstallBasic.cmake)
+include(${CMAKE_SOURCE_DIR}/cmake/ProtobufGenerateCpp.cmake)
+include(${CMAKE_SOURCE_DIR}/cmake/SetFirstDefined.cmake)
+include(${CMAKE_SOURCE_DIR}/cmake/safe_dep_glob.cmake)

 function(init_library)
  set(_options SHARED STATIC)
-  set(_one_value_args SOURCE_DIR)
+  set(_one_value_args
+    NAME
+    SOURCE_DIR
+    INCLUDE_DIR
+  )
  set(_multi_value_args)
  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-
  if (_ARGS_SHARED AND _ARGS_STATIC)
    message(FATAL_ERROR "Invalid args passed to init_library: cannot specify STATIC and SHARED")
  endif()
+  set_first_defined(_name _ARGS_NAME PROJECT_NAME)
+  set_first_defined(_source_dir _ARGS_SOURCE_DIR _name)
+  set_first_defined(_include_dir _ARGS_INCLUDE_DIR _source_dir)

-  set_ternary(_source_dir _ARGS_SOURCE_DIR _ARGS_SOURCE_DIR ${PROJECT_NAME})
-  # Glob cpp sources
+  # Cpp sources
  file(
-    GLOB_RECURSE ${PROJECT_NAME}_SOURCES
+    GLOB_RECURSE ${_name}_SOURCES
    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
    ${_source_dir}/*.cpp
  )
  # Note: cpp files inside 'tests' dir(s) are considered tests.
  # Remove tests from main sources, and add them to test sources
-  foreach(item ${${PROJECT_NAME}_SOURCES})
-    if(${item} MATCHES "tests/*.cpp")
-      list(REMOVE_ITEM ${PROJECT_NAME}_SOURCES ${item})
-      list(APPEND ${PROJECT_NAME}_TEST_SOURCES ${item})
+  foreach(item ${${_name}_SOURCES})
+    if(${item} MATCHES "tests/.*\.cpp$")
+      list(APPEND ${_name}_TEST_SOURCES ${item})
    endif()
  endforeach()
+  foreach(item ${${_name}_TEST_SOURCES})
+    list(REMOVE_ITEM ${_name}_SOURCES ${item})
+  endforeach()

-  export_to_parent(
-    ${PROJECT_NAME}_SOURCES
-    ${PROJECT_NAME}_TEST_SOURCES
+  # Proto sources 
+  file(
+    GLOB_RECURSE ${_name}_PROTO_SOURCES
+    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
+    ${_source_dir}/protobufs/*.proto
+  )
+  update_deps_file("${${_name}_PROTO_SOURCES};")
+  protobuf_generate_cpp(
+    CXX_SOURCES ${_name}_CXX_PROTO_SOURCES
+    CXX_HEADERS ${_name}_CXX_PROTO_HEADERS
+    CXX_OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}
+    PROTO_INCLUDE_DIRS
+      ${EXTERNALS_INCLUDE}
+      ${CMAKE_CURRENT_SOURCE_DIR}
+    PROTO_SOURCES
+      ${${_name}_PROTO_SOURCES}
  )

-  # Add main library
-  if (_ARGS_STATIC)
-    add_library(${PROJECT_NAME} STATIC ${${PROJECT_NAME}_SOURCES})
-  elseif(_ARGS_SHARED)
-    add_library(${PROJECT_NAME} SHARED ${${PROJECT_NAME}_SOURCES})
-  else()
-    add_library(${PROJECT_NAME} ${${PROJECT_NAME}_SOURCES})
-  endif()
+  export_to_parent(
+    ${_name}_SOURCES
+    ${_name}_TEST_SOURCES
+    ${_name}_PROTO_SOURCES
+    ${_name}_CXX_PROTO_SOURCES
+    ${_name}_CXX_PROTO_HEADERS
+  )

+  # Add main lib
+  if (_ARGS_STATIC)
+    add_library(${_name} STATIC ${${_name}_SOURCES} ${${_name}_CXX_PROTO_SOURCES})
+  elseif(_ARGS_SHARED)
+    add_library(${_name} SHARED ${${_name}_SOURCES} ${${_name}_CXX_PROTO_SOURCES})
+  else()
+    add_library(${_name} ${${_name}_SOURCES} ${${_name}_CXX_PROTO_SOURCES})
+  endif()
  target_include_directories(
-    ${PROJECT_NAME}
+    ${_name}
+    SYSTEM
+    BEFORE
+    PUBLIC
+      $<BUILD_INTERFACE:${EXTERNALS_ROOT}/include>
+      $<BUILD_INTERFACE:${EXTERNALS_ROOT}/include/zlib>
+  )
+  target_include_directories(
+    ${_name}
    PUBLIC
      $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
      $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>
  )
+  # Include protos (if any)
+  if (${_name}_PROTO_SOURCES)
+    target_include_directories(
+      ${_name}
+      PUBLIC
+        $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
+    )
+  endif()
  target_link_directories(
-    ${PROJECT_NAME}
+    ${_name}
    PUBLIC
      $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
+      $<BUILD_INTERFACE:${EXTERNALS_ROOT}/lib/$<CONFIG>>
      $<INSTALL_INTERFACE:${CMAKE_INSTALL_LIBDIR}>
  )
+  # Install main lib
+  foreach(_hdr ${${_name}_CXX_PROTO_HEADERS})
+    file(RELATIVE_PATH _relative_path ${CMAKE_CURRENT_BINARY_DIR} ${_hdr})
+    install(FILES ${_hdr} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/${_relative_path})
+  endforeach()
+  install_basic(
+    NAME ${_name}
+    INCLUDE_DIR ${_include_dir}
+  )

  # Add tests executable (if there are any)
-  if (${PROJECT_NAME}_TEST_SOURCES)
-    add_executable(${PROJECT_NAME}-tests ${${PROJECT_NAME}_TEST_SOURCES})
-    target_link_libraries(${PROJECT_NAME}-tests PRIVATE ${PROJECT_NAME})
+  if (${_name}_TEST_SOURCES)
+    add_executable(${_name}-tests ${${_name}_TEST_SOURCES})
+
+    get_target_property(_lib_type ${_name} TYPE)
+    if (_lib_type STREQUAL "SHARED_LIBRARY")
+      # Set RPATH so the tests exe knows where to find the shared lib.
+      set_target_properties(
+        ${_name}-tests
+        PROPERTIES
+          BUILD_RPATH "$ORIGIN"
+          INSTALL_RPATH "$ORIGIN/../${CMAKE_INSTALL_LIBDIR}"
+      )
+    endif()
+
+    # Link main lib
+    target_link_directories(
+      ${_name}-tests
+      PRIVATE
+        $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
+        $<INSTALL_INTERFACE:${CMAKE_INSTALL_LIBDIR}>
+    )
+    target_link_libraries(${_name}-tests PRIVATE ${_name})
+
    # Catch2
    find_package(Catch2 CONFIG REQUIRED COMPONENTS Catch2WithMain)
-    target_link_libraries(${PROJECT_NAME}-tests PRIVATE Catch2::Catch2WithMain)
+    target_link_libraries(${_name}-tests PRIVATE Catch2::Catch2WithMain)
  endif()
 endfunction()
@@ -1,55 +0,0 @@
-function(init_shared_library)
-  include(${CMAKE_SOURCE_DIR}/cmake/ExportToParent.cmake)
-  set(source_dir)
-  if (${ARGC} EQUAL 0)
-    set(source_dir ${PROJECT_NAME})
-  elseif(${ARGC} GREATER 1)
-    message(FATAL_ERROR "init_shared_library received invalid arguments. Usage: init_shared_library(<OPTIONAL:source_dir>)")
-  else()
-    set(source_dir ${ARGV})
-  endif()
-
-  # Glob cpp sources
-  file(
-    GLOB_RECURSE ${PROJECT_NAME}_SOURCES
-    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
-    ${source_dir}/*.cpp
-  )
-  # Note: cpp files inside 'tests' dir(s) are considered tests.
-  # Remove tests from main sources, and add them to test sources
-  foreach(item ${${PROJECT_NAME}_SOURCES})
-    if(${item} MATCHES "tests/*.cpp")
-      list(REMOVE_ITEM ${PROJECT_NAME}_SOURCES ${item})
-      list(APPEND ${PROJECT_NAME}_TEST_SOURCES ${item})
-    endif()
-  endforeach()
-
-  export_to_parent(
-    ${PROJECT_NAME}_SOURCES
-    ${PROJECT_NAME}_TEST_SOURCES
-  )
-
-  # Add main library
-  add_library(${PROJECT_NAME} SHARED ${${PROJECT_NAME}_SOURCES})
-  target_include_directories(
-    ${PROJECT_NAME}
-    PUBLIC
-      $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
-      $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>
-  )
-  target_link_directories(
-    ${PROJECT_NAME}
-    PUBLIC
-      $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
-      $<INSTALL_INTERFACE:${CMAKE_INSTALL_LIBDIR}>
-  )
-
-  # Add tests executable (if there are any)
-  if (${PROJECT_NAME}_TEST_SOURCES)
-    add_executable(${PROJECT_NAME}-tests ${${PROJECT_NAME}_TEST_SOURCES})
-    target_link_libraries(${PROJECT_NAME}-tests PRIVATE ${PROJECT_NAME})
-    # Catch2
-    find_package(Catch2 CONFIG REQUIRED COMPONENTS Catch2WithMain)
-    target_link_libraries(${PROJECT_NAME}-tests PRIVATE Catch2::Catch2WithMain)
-  endif()
-endfunction(init_shared_library)
@@ -0,0 +1,39 @@
+include_guard(DIRECTORY)
+include(GNUInstallDirs)
+include(${CMAKE_SOURCE_DIR}/cmake/PackBasic.cmake)
+include(${CMAKE_SOURCE_DIR}/cmake/SetFirstDefined.cmake)
+
+function(install_basic)
+  set(_options)
+  set(_one_value_args NAME INCLUDE_DIR)
+  set(_multi_value_args)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+  set_first_defined(_name _ARGS_NAME PROJECT_NAME)
+  set_first_defined(_include_dir _ARGS_INCLUDE_DIR _name)
+
+  file(
+    GLOB_RECURSE ${_name}_HEADERS
+    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
+    ${_include_dir}/*.h)
+
+  target_sources(${_name}
+    PUBLIC
+      FILE_SET HEADERS FILES ${${_name}_HEADERS})
+
+  install(
+    TARGETS ${_name}
+    EXPORT ${_name}Targets
+    RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
+    LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
+    ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
+    FILE_SET HEADERS DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}
+  )
+
+  install(
+    EXPORT ${_name}Targets
+    FILE ${_name}Targets.cmake
+    NAMESPACE ${_name}::
+    DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name})
+
+  pack_basic()
+endfunction()
@@ -1,30 +0,0 @@
-include(${CMAKE_SOURCE_DIR}/cmake/PackBasic.cmake)
-include(GNUInstallDirs)
-
-function(install_basic_config)
-  file(
-    GLOB_RECURSE ${PROJECT_NAME}_HEADERS
-    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
-    *.h)
-
-  target_sources(${PROJECT_NAME}
-    PUBLIC
-      FILE_SET HEADERS FILES ${${PROJECT_NAME}_HEADERS})
-
-  install(
-    TARGETS ${PROJECT_NAME}
-    EXPORT ${PROJECT_NAME}Targets
-    RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
-    LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
-    ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
-    FILE_SET HEADERS DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}
-  )
-
-  install(
-    EXPORT ${PROJECT_NAME}Targets
-    FILE ${PROJECT_NAME}Targets.cmake
-    NAMESPACE ${PROJECT_NAME}::
-    DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME})
-
-  pack_basic()
-endfunction()
@@ -1,6 +1,14 @@
+include_guard(DIRECTORY)
 include(CMakePackageConfigHelpers)
+include(${CMAKE_SOURCE_DIR}/cmake/SetFirstDefined.cmake)

 function(pack_basic)
+  set(_options)
+  set(_one_value_args NAME)
+  set(_multi_value_args)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+  set_first_defined(_name _ARGS_NAME PROJECT_NAME)
+
  find_package(Git QUIET)

  if(GIT_FOUND)
@@ -17,20 +25,20 @@ function(pack_basic)
  endif()

  write_basic_package_version_file(
-    "${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}ConfigVersion.cmake"
+    "${CMAKE_CURRENT_BINARY_DIR}/${_name}ConfigVersion.cmake"
    VERSION ${PROJECT_VERSION}
    COMPATIBILITY AnyNewerVersion
  )

  configure_package_config_file(
    "${CMAKE_CURRENT_SOURCE_DIR}/Config.cmake.in"
-    "${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}Config.cmake"
-    INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}
+    "${CMAKE_CURRENT_BINARY_DIR}/${_name}Config.cmake"
+    INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name}
  )

  install(
    FILES
-      "${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}Config.cmake"
-      "${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}ConfigVersion.cmake"
-    DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME})
+      "${CMAKE_CURRENT_BINARY_DIR}/${_name}Config.cmake"
+      "${CMAKE_CURRENT_BINARY_DIR}/${_name}ConfigVersion.cmake"
+    DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name})
 endfunction()
@@ -2,8 +2,16 @@ include_guard(GLOBAL)

 function(protobuf_generate_cpp)
  set(_options)
-  set(_one_value_args CXX_SOURCES CXX_HEADERS CXX_OUT_DIR PROTOC)
-  set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
+  set(_one_value_args
+    CXX_SOURCES
+    CXX_HEADERS
+    CXX_OUT_DIR
+    PROTOC
+  )
+  set(_multi_value_args
+    PROTO_INCLUDE_DIRS
+    PROTO_SOURCES
+  )
  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})

  if (NOT DEFINED _ARGS_PROTOC)
@@ -26,6 +34,7 @@ function(protobuf_generate_cpp)
    endif()
    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.cc")
    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.h")
+    message(STATUS "add_custom_command for ${_fil} -> ${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.h")
    add_custom_command(
      OUTPUT
        "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.cc"
@@ -33,7 +42,7 @@ function(protobuf_generate_cpp)
      COMMAND
        ${_ARGS_PROTOC}
      ARGS
-        --cpp_out=${_ARGS_CXX_OUT_DIR} ${_proto_include_cli_args} ${_abs_fil}
+        --cpp_out=${_ARGS_CXX_OUT_DIR} ${_proto_include_cli_args} ${_fil}
      DEPENDS ${_abs_fil}
      COMMENT "Running C++ protocol buffer compiler on ${_fil}"
      VERBATIM
@@ -0,0 +1,58 @@
+function(protobuf_generate_cpp_config)
+  set(_options)
+  set(_one_value_args
+    PROTO_SOURCES_HEADER
+    PROTO_DEFAULTS_HEADER
+  )
+  set(_multi_value_args
+    PROTO_SOURCES
+    PROTO_DEFAULTS
+  )
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+
+  # Configure the 'config_proto_files' header. It provides an array of strings populated with the
+  # files in PROTO_SOURCES. At runtime, we use this array to generate a special 'config' proto
+  # message with fields for each message defined in said files. A global instance of this 'config'
+  # proto is then constructed, and populated with [compile-time] defaults, and [runtime] overrides
+  # (i.e., in spark.conf). Calls to `bts::config::get<T>()` retrieve the value from the field
+  # associated with type `T` in this global instance.
+  message(STATUS "PROTO_SOURCES: ${_ARGS_PROTO_SOURCES}")
+  message(STATUS "PROTO_SOURCES_HEADER: ${_ARGS_PROTO_SOURCES_HEADER}")
+  message(STATUS "PROTO_DEFAULTS_HEADER: ${_ARGS_PROTO_DEFAULTS_HEADER}")
+  if (_ARGS_PROTO_SOURCES_HEADER)
+    set(PROTO_SOURCES)
+    set(PROTO_SOURCES_CXX_ARRAY_ITEMS "")
+    foreach(_proto_path ${_ARGS_PROTO_SOURCES})
+      list(APPEND PROTO_SOURCES ${_proto_path})
+      string(APPEND PROTO_SOURCES_CXX_ARRAY_ITEMS "\"${_proto_path}\",\n")
+    endforeach()
+    list(LENGTH PROTO_SOURCES PROTO_SOURCES_LENGTH)
+    message(STATUS "CONFIGURING PROTO_SOURCES_HEADER): ${_ARGS_PROTO_SOURCES_HEADER}")
+    configure_file(${CMAKE_SOURCE_DIR}/cmake/config_proto_files.h.in ${_ARGS_PROTO_SOURCES_HEADER})
+  endif()
+
+  # Generate static defaults. Ideally, this would be a single string, but we use an array because
+  # the compiler complains if the string is longer than 65k.
+  set(_cfg_defaults_str "inline const char * config_static_defaults[] = { ")
+
+  foreach(_cfg_file_path ${_ARGS_PROTO_DEFAULTS})
+    file(READ ${_cfg_file_path} _cfg_file_content)
+    string(REPLACE "\"" "\\\"" _cfg_file_content_escaped ${_cfg_file_content})
+    string(REPLACE "\n" ";" _cfg_lines ${_cfg_file_content_escaped})
+
+    foreach(_line ${_cfg_lines})
+      string(APPEND _cfg_defaults_str "\n  \"${_line}\",")
+    endforeach(_line)
+  endforeach(_cfg_file_path)
+
+  string(APPEND _cfg_defaults_str "};")
+  set(_old_cfg_defaults_str "")
+
+  if(EXISTS ${_ARGS_PROTO_DEFAULTS_HEADER})
+    file(READ ${_ARGS_PROTO_DEFAULTS_HEADER} _old_cfg_defaults_str)
+  endif()
+
+  if(NOT _cfg_defaults_str STREQUAL _old_cfg_defaults_str)
+    file(WRITE "${_ARGS_PROTO_DEFAULTS_HEADER}" "${_cfg_defaults_str}")
+  endif()
+endfunction()
@@ -0,0 +1,10 @@
+include_guard(GLOBAL)
+
+macro(set_first_defined RESULT)
+  foreach(_arg ${ARGN})
+    if (DEFINED ${_arg})
+      set(${RESULT} ${${_arg}})
+      break()
+    endif()
+  endforeach()
+endmacro()
@@ -20,16 +20,14 @@ endfunction()

 function(sym_link_file)
  set(_options)
-  set(_one_value_args TARGET_NAME SOURCE DEST INSTALL_DEST)
+  set(_one_value_args SOURCE DEST INSTALL_DEST)
  set(_multi_value_args)
  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-  add_custom_command(
-    OUTPUT ${_ARGS_DEST}
-    COMMAND ln ARGS -s ${_ARGS_SOURCE} ${_ARGS_DEST}
-  )
-  add_custom_target(
-    ${_ARGS_TARGET_NAME} ALL
-    DEPENDS ${_ARGS_DEST}
+  file(
+    CREATE_LINK
+      ${_ARGS_SOURCE}
+      ${_ARGS_DEST}
+    SYMBOLIC
  )
  if (DEFINED _ARGS_INSTALL_DEST)
    install(
@@ -0,0 +1,11 @@
+#pragma once
+
+#include <array>
+
+namespace bts::@PROJECT_NAME@::protobufs {
+
+inline std::array<const char *, @PROTO_SOURCES_LENGTH@> config_proto_files = {
+@PROTO_SOURCES_CXX_ARRAY_ITEMS@
+};
+
+} // namespace bts::@PROJECT_NAME@::protobufs
@@ -1,15 +1,24 @@
 services:
  sparkd:
    build:
-      dockerfile: src/server/sparkd/Dockerfile
+      dockerfile: src/app_sparkd/Dockerfile
+      args:
+        - SOURCE_DIR=install/sparkd-release-gcc
+        - CONF_FILE=src/app_sparkd/spark.conf
    command: [ "./sparkd", "run" ]
-    ports: [ "8000:8000", "8001:8001", "8002:8002" ]
+    ports:
+      - 8000:8000
+      - 8001:8001
+      - 8002:8002

  pubsub:
    environment:
      - DOTNET_ENVIRONMENT=Staging
    build:
      dockerfile: src/DataStreamSuite/PubSubServer/Dockerfile
+      args:
+        - SOURCE_DIR=build/Release/bin/net7.0/linux-x64/publish
+        - CONF_FILE=src/DataStreamSuite/PubSubServer/spark.conf
    ports:
      - 22019:22019
    command: [ "./PubSubServer" ]
@@ -17,20 +26,13 @@ services:
  md-publisher:
    build:
      dockerfile: src/DataStreamSuite/DataStreamPublisher/Dockerfile
+      args:
+        - SOURCE_DIR=build/Release/bin/net7.0/linux-x64/publish
+        - CONF_FILE=src/DataStreamSuite/DataStreamPublisher/spark.conf
    depends_on:
      - pubsub
      - sparkd
-    command:
-      [
-        "dockerize",
-        "-wait",
-        "tcp://pubsub:22019",
-        "-wait",
-        "tcp://sparkd:8000",
-        "-timeout",
-        "10s",
-        "./DataStreamPublisher"
-      ]
+    command: [ "dockerize", "-wait", "tcp://pubsub:22019", "-wait", "tcp://sparkd:8000", "-timeout", "10s", "./DataStreamPublisher" ]
    environment:
      - DOTNET_ENVIRONMENT=MarketData

@@ -40,13 +42,15 @@ services:
      - 8443:8443
    build:
      dockerfile: src/DataStreamSuite/Bts.DataStreamWebApp.Server/Dockerfile
+      args:
+        - SOURCE_DIR=build/Release/bin/net7.0/linux-x64/publish
+        - WEB_ROOT=build/Bts.DataStreamWebApp.Client/production/dist
+        - CONF_FILE=src/DataStreamSuite/Bts.DataStreamWebApp.Server/spark.conf
    command: [ "./Bts.DataStreamWebApp.Server" ]
    environment:
      - ASPNETCORE_ENVIRONMENT=Staging

  api-server:
-    profiles:
-      - api-server
    depends_on:
      - sparkd
    ports:
@@ -54,7 +58,7 @@ services:
      - 9443:9443
    build:
      dockerfile: src/client/spark_api_server/Dockerfile
-    command: [ "./spark_api_server", "run" ]
+    command: [ "dockerize", "-wait", "tcp://sparkd:8000", "-timeout", "10s", "./spark_api_server", "run" ]
    environment:
      - ASPNETCORE_ENVIRONMENT=Staging

@@ -24,7 +24,6 @@ RUN locale-gen en_US.UTF-8
 ARG docker_user="bts"
 ARG docker_uid="1000"
 ARG docker_pass="bts"
-# ARG entrypoint

 RUN \
  groupadd -g "$docker_uid" "$docker_user" \
@@ -1 +1 @@
-VITE_DATASTREAM_WEBSOCKET_URL=wss://localhost:443
+VITE_DATASTREAM_WEBSOCKET_URL=wss://localhost:8443/ws
@@ -1 +1 @@
-VITE_DATASTREAM_WEBSOCKET_URL=wss://localhost:443
+VITE_DATASTREAM_WEBSOCKET_URL=wss://localhost:8443/ws
@@ -1,32 +0,0 @@
-module.exports = {
-  root: true,
-  env: { browser: true, es2020: true },
-  extends: [
-    'eslint:recommended',
-    'plugin:react/recommended',
-    'plugin:react/jsx-runtime',
-    'plugin:react-hooks/recommended',
-  ],
-  ignorePatterns: ['dist', '.eslintrc.cjs'],
-  parserOptions: { ecmaVersion: 'latest', sourceType: 'module' },
-  settings: { react: { version: '18.3.1' } },
-  plugins: ['react-refresh'],
-  rules: {
-    "import/order": [
-      "error",
-      {
-        "groups": [
-          "builtin",
-          "external",
-          "internal",
-          ["parent", "sibling", "index"]
-        ],
-        "newlines-between": "always"
-      }
-    ],
-    'react-refresh/only-export-components': [
-      'warn',
-      { allowConstantExport: true },
-    ],
-  },
-}
@@ -0,0 +1,36 @@
+import importPlugin from "eslint-plugin-import";
+import reactPkg from "eslint-plugin-react";
+const { configs: reactConfigs } = reactPkg;
+import js from "@eslint/js";
+import defineConfig from "./vite.config.js";
+
+export default [
+  js.configs.recommended,
+  importPlugin.flatConfigs.recommended,
+  reactConfigs.flat.recommended,
+  reactConfigs.flat["jsx-runtime"],
+  {
+    files: ["**/*.{js,jsx}"],
+    languageOptions: {
+      ecmaVersion: "latest",
+      sourceType: "module",
+      globals: {
+        console: "readonly"
+      }
+    },
+    settings: {
+      "import/resolver": {
+        vite: {
+          viteConfig: defineConfig({ mode: "development" })
+        }
+      }
+    },
+    rules: {
+      semi: "error",
+      "prefer-const": "error",
+      "no-unused-vars": "warn",
+      "import/no-dynamic-require": "warn",
+      "import/no-nodejs-modules": "warn",
+    }
+  }
+];
@@ -10,35 +10,37 @@
    "preview": "vite preview"
  },
  "dependencies": {
-    "@angular/material": "^18.0.4",
-    "@emotion/react": "^11.11.4",
-    "@emotion/styled": "^11.11.5",
-    "@fontsource/noto-sans": "^5.0.22",
-    "@material/data-table": "^15.0.0-canary.7f224ddd4.0",
-    "@mui/icons-material": "^5.15.20",
-    "@mui/material": "^5.15.20",
-    "@mui/x-data-grid": "^7.7.1",
-    "bulma": "^1.0.1",
+    "@angular/material": "^18.2.8",
+    "@emotion/react": "^11.13.3",
+    "@emotion/styled": "^11.13.0",
+    "@fontsource/noto-sans": "^5.1.0",
+    "@material/data-table": "^15.0.0-canary.423edc3dc.0",
+    "@mui/icons-material": "^6.1.4",
+    "@mui/material": "^6.1.4",
+    "@mui/x-data-grid": "^7.20.0",
+    "bulma": "^1.0.2",
    "classnames": "^2.5.1",
    "react": "^18.3.1",
-    "react-data-grid": "^7.0.0-beta.44",
+    "react-data-grid": "^7.0.0-beta.47",
    "react-data-table-component": "^7.6.2",
    "react-dom": "^18.3.1",
-    "react-json-view-lite": "^1.4.0",
+    "react-json-view-lite": "^1.5.0",
    "react-pro-sidebar": "^1.1.0",
-    "react-use-websocket": "^4.8.1",
-    "rpc-websockets": "^7.11.2"
+    "react-use-websocket": "^4.9.0",
+    "rpc-websockets": "^9.0.4"
  },
  "devDependencies": {
-    "@types/react": "^18.3.3",
-    "@types/react-dom": "^18.3.0",
-    "@vitejs/plugin-react": "^4.3.1",
-    "eslint": "^8.57.0",
-    "eslint-plugin-import": "^2.29.1",
-    "eslint-plugin-react": "^7.34.2",
-    "eslint-plugin-react-hooks": "^4.6.2",
-    "eslint-plugin-react-refresh": "^0.4.7",
-    "react-mui-sidebar": "^1.3.5",
-    "vite": "^5.2.13"
+    "@types/react": "^18.3.11",
+    "@types/react-dom": "^18.3.1",
+    "@vitejs/plugin-react": "^4.3.2",
+    "eslint": "^9.12.0",
+    "@eslint/js": "^9.12.0",
+    "eslint-plugin-import": "^2.31.0",
+    "eslint-plugin-react": "^7.37.1",
+    "eslint-plugin-react-hooks": "^5.0.0",
+    "eslint-plugin-react-refresh": "^0.4.12",
+    "eslint-import-resolver-vite": "^2.0.1",
+    "react-mui-sidebar": "^1.3.8",
+    "vite": "^5.4.9"
  }
 }
@@ -1,5 +1,5 @@
 import { fileURLToPath, URL } from 'node:url';
-import { defineConfig, loadEnv } from 'vite';
+import { defineConfig } from 'vite';
 import plugin from '@vitejs/plugin-react';
 import fs from 'fs';
 import path from 'path';
@@ -1,7 +1,13 @@
 FROM bluetradingsystems/linux_dotnet7

+ARG CS_SOURCE_DIR="build/Release/bin/net7.0/linux-x64/publish"
+ARG CPP_SOURCE_DIR="install/data-stream-suite-linux"
+ARG WEB_ROOT="build/Bts.DataStreamWebApp.Client/production/dist"
+ARG CONF_FILE="src/DataStreamSuite/Bts.DataStreamWebApp.Server/spark.conf"
+
 RUN dotnet dev-certs https --trust
-COPY --chown=bts:bts "build/Release/bin/net7.0/linux-x64/publish" "bin"
-COPY --chown=bts:bts "build/Bts.DataStreamWebApp.Client/production/dist" "bin/wwwroot"
-COPY --chown=bts:bts "src/DataStreamSuite/Bts.DataStreamWebApp.Server/spark.conf" ".local/share/bts/Bts.DataStreamWebApp.Server/conf/spark.conf"
+COPY --chown=bts:bts "${CPP_SOURCE_DIR}/lib" "bin"
+COPY --chown=bts:bts "${CS_SOURCE_DIR}" "bin"
+COPY --chown=bts:bts "${WEB_ROOT}" "bin/wwwroot"
+COPY --chown=bts:bts "${CONF_FILE}" ".local/share/bts/Bts.DataStreamWebApp.Server/conf/spark.conf"
 WORKDIR "/home/bts/bin"
@@ -1,14 +1,4 @@
 {
-  "Kestrel": {
-    "Endpoints": {
-      "Http": {
-        "Url": "http://localhost:8080"
-      },
-      "Https": {
-        "Url": "https://localhost:8443"
-      }
-    }
-  },
  "PubSub": {
    "Routing": {
      "PublisherRoutes": [
@@ -30,5 +30,15 @@
        "benzinga_news"
      ]
    }
-  ]
+  ],
+  "Spark": {
+    "ConfigServer": "http://sparkd:8000/config"
+  },
+  "SparkApi": {
+    "IsEnabled": true,
+    "Credentials": {
+      "User": "bts",
+      "Password": ""
+    }
+  }
 }
@@ -1,6 +0,0 @@
-FROM bluetradingsystems/linux_dotnet7
-
-RUN dotnet dev-certs https --trust
-COPY --chown=bts:bts "bin" "bin"
-COPY --chown=bts:bts "webapp.spark.conf" ".local/share/DataStream/conf/spark.conf"
-WORKDIR "/home/bts/bin"
@@ -18,8 +18,11 @@

  <ItemGroup>
    <PackageReference Include="HtmlAgilityPack" Version="1.11.57" />
+    <PackageReference Include="Microsoft.Data.Analysis" Version="0.21.1" />
    <PackageReference Include="MongoDB.Bson" Version="2.23.1" />
    <PackageReference Include="MongoDB.Driver" Version="2.23.1" />
+    <PackageReference Include="Parquet.Net" Version="5.0.1" />
+    <PackageReference Include="System.Data.SQLite" Version="1.0.118" />
  </ItemGroup>

 </Project>
@@ -11,6 +11,9 @@ public static class DataStreamExtensions {
   public static ViewStream AsViewStream(this IObservable<DataPacket> source) =>
      source as ViewStream ?? new ObservableViewStream(source);

+   public static ViewStream AsViewStream<T>(this IObservable<IDataPacket<T>> source) where T : IBlueRow =>
+      source as ViewStream ?? new ObservableViewStream(source.Select(dp => new DataPacket(dp.newRows, dp.addedRows, dp.deletedRows)));
+
   public static IObservable<DataPacket> AsObservable(this ViewStream source) {
      return source as IObservable<DataPacket> ?? Observable.Create<DataPacket>(Subscribe);

@@ -0,0 +1,32 @@
+namespace bts.SilverPine {
+   using System;
+   using BCG;
+   using System.Collections.Generic;
+   using Bts.DataStreamExpressions;
+
+   // All this does is take in a dsl expression, parse it and fire the data that come out of running it.
+   // So for example, if an expresison is passed 
+   [DataStreamUserRespresentation("dsl")]
+   public class DSLStream : ViewStream {
+      private readonly string _dsl;
+      public DSLStream(string dsl) {
+         Log.Gen.info($"DSLStream.cons dsl: {dsl}");
+         _dsl = dsl;
+         ViewStream rs = build_stream(dsl);
+         rs = rs ?? new LiteralStream("error", $"msg:invalid dsl \"{dsl}\"");
+         mySetRealSource(rs);
+      }
+
+      static ViewStream? build_stream(string stream_expression) {
+         try {
+            var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression);
+            var rv = exp.getView(new Dictionary<string, StatementExpression>(0));
+            return rv;
+
+         } catch (Exception ex) {
+            Log.Gen.error($"Could not build stream: {stream_expression} ex: {ex}");
+            return null;
+         }
+      }
+   }
+}
@@ -25,12 +25,12 @@ namespace BCG {
         byte[] rem_req_bytes = rem_req.serialize();
         _publisher_core.publish("System", "RemoteRequest", rem_req_bytes);
         _source_str = new TableStream(_name);
-         _source_str.onChange += new ChangeDelegate(_handle_data);
+         _source_str.onChange += _handle_data;
      }

      protected override void myOnLastSubscriber() {
         //maybe there should be a way to turn remote requests off ?
-         _source_str.onChange -= new ChangeDelegate(_handle_data);
+         _source_str.onChange -= _handle_data;
      }

      static string get_expression(params string[] lines) {
@@ -21,13 +21,11 @@ namespace BCG
         : this (new TableStream (tableName))
      {}

-      protected override void myOnFirstSubscriber () {
-         myVS.onChange += new ChangeDelegate(myHandleChanges);
-      }
-      protected override void myOnLastSubscriber () {
-         myVS.onChange -= new ChangeDelegate(myHandleChanges);
-      }
+      protected override void myOnFirstSubscriber () =>
+         myVS.onChange += myHandleChanges;

+      protected override void myOnLastSubscriber () =>
+         myVS.onChange -= myHandleChanges;

      private void myHandleChanges(object newValue)
      {
@@ -30,7 +30,7 @@ namespace BCG

        protected override void myOnFirstSubscriber()
        {
-            myToCrush.onChange += new ChangeDelegate(myToCrush_onChange);
+            myToCrush.onChange += myToCrush_onChange;
        }

        protected override void myOnLastSubscriber()
@@ -15,11 +15,11 @@ namespace BCG
            
            //set up an anonymous delegate for each of the streams
            //we do this so that we can unsubscribe
-            myDelegates = new ChangeDelegate[myToCrush.Length];
+            myDelegates = new DataStreamChangeDelegate[myToCrush.Length];
            for (int i = 0; i < myToCrush.Length; ++i)
            {
                int tempI = i;
-                myDelegates[tempI] = new ChangeDelegate((o) => { myHandleChange(tempI, ((DataPacket)o).newRows); });
+                myDelegates[tempI] = new((o) => { myHandleChange(tempI, ((DataPacket)o).newRows); });
            }
        }

@@ -27,7 +27,7 @@ namespace BCG
        private ViewStream[] myToCrush;
        private ITableCrusher myCrusher;

-        private ChangeDelegate[] myDelegates;
+        private DataStreamChangeDelegate[] myDelegates;

        protected override void myOnFirstSubscriber()
        {
@@ -28,7 +28,7 @@ namespace BCG

      protected override void myOnFirstSubscriber()
      {
-         myDataSource.onChange += new ChangeDelegate(myDataSource_onChange);
+         myDataSource.onChange += myDataSource_onChange;
      }

      protected override void myOnLastSubscriber()
@@ -1,49 +0,0 @@
-// using System;
-// using System.Collections.Generic;
-// using System.Text;
-// using System.IO;
-
-// using System.Runtime.Serialization.Formatters.Binary;
-
-// namespace BCG
-// {
-//    public class DumpLoaderStream : ViewStream, ISideEffectStream
-//    {
-//       //wtf
-//       //public class Exception : System.Exception
-//       //{public Exception(string msg) : base(msg) { }}
-
-//       public DumpLoaderStream(string fileName, int secondsBetweenPolls)
-//       {
-//          mySetRealSource(
-//             new PollStream(() => new LoaderStream(fileName).rows, secondsBetweenPolls, 
-//                ViewStreamDatabase.instance));
-//       }
-
-//       public DumpLoaderStream(string fileName)
-//       {
-//         if (fileName == null)
-//            throw new LoaderStream.Exception("File name cannot be null");
-
-//         for (int i = 0; i < 5; i++)
-//            try
-//            {
-//               using (FileStream fs = new FileStream(fileName, FileMode.Open, FileAccess.Read))
-//               {
-//                  IClientDeserializer deser = new ClientUniversalDeserializer();
-//                  byte[] the_ops;
-//                  Array the_rows;
-//                  deser.deserialize(new Stream[] { fs }, out the_ops, out the_rows);
-
-//                  mySetRealSource(new BlueRowCollectionStaticStream((IBlueRowCollection)the_rows.GetValue(0)));
-//               }
-//               break;
-//            }
-//            catch (IOException)
-//            {
-//               System.Threading.Thread.Sleep(50);
-//            }
-
-//       }
-//    }
-// }
@@ -1,6 +1,7 @@
 using BCG;

 namespace bts {
+   // This is the other side of the SaverStream, its "loads" the stream and fires once
   [DataStreamUserRespresentation("loader")]
   public class LoaderStream : ViewStream {

@@ -5,10 +5,11 @@ using System.Text;
 using System.IO;
 using Google.Protobuf;
 using System.Globalization;
-using bts;
+using BCG;

-namespace BCG {
-   // Writes all the udpates to a stream in a file.
+namespace bts {
+   // Writes all the udpates to a stream in a file or multiples files dependong on the rollover size (as opposed to Saver
+   // which just writes one update, as a create)
   // File naming:
   // token_epoch_yyyyMMdd_HHmmss
   // File location: directory(passed)\token\
@@ -94,10 +95,10 @@ namespace BCG {
      void _on_change(object new_value) {
         DataPacket dp = new_value as DataPacket;
         byte[] serialized_update = null;
-         bool _write_create = _serializer == null || _n_updates_in_current_file == 0;
+         bool first_write = _serializer == null || _n_updates_in_current_file == 0;
         _serializer = _serializer ?? new UniversalStreamIOSerializer();
-         if (_write_create) {
-            // new files always starts with create
+         if (first_write) {
+            // first file always starts with create
            serialized_update = _serializer.serialize(new bts.StreamIOMessage(new bts.StreamIOMessage.CreateMessage(dp.newRows)));
         } else
            serialized_update = _serializer.serialize(new bts.StreamIOMessage(new bts.StreamIOMessage.MultipleRemoveAddMessage(dp.deletedRows, dp.addedRows)));
@@ -105,8 +106,8 @@ namespace BCG {
         long utc_epoch = epoch_now();

         //time, length of data, data
-         bw.Write(utc_epoch);
-         bw.Write(serialized_update.Length);
+         //bw.Write(utc_epoch);
+         //bw.Write(serialized_update.Length);
         bw.Write(serialized_update);
         _out_fs.Flush();
         myFireEvent(dp);
@@ -1,190 +0,0 @@
-// using System;
-// using System.Collections.Generic;
-// using System.Linq;
-// using System.Text;
-// using System.IO;
-// using System.Runtime.Serialization.Formatters.Binary;
-// using BCG.Utilities.General;
-// using bts.collections;
-// using System.Reflection.Metadata;
-
-
-// namespace BCG {
-
-//    public class PlaybackStream : ViewStream, IWorker {
-//       private readonly PlaybackType _type;
-//       private BlueTable _current_table;
-//       private IClientDeserializer _deserilializer = new ClientUniversalDeserializer();
-//       // these are the packets already downloaded from the current file
-//       private List<PlaybackPacket> _packets = new List<PlaybackPacket>();
-//       //private IEnumerable<FileStream> _file_streams = new List<FileStream>();
-//       private IEnumerable<string> _file_names;
-//       private Tuple<int, FileStream> _current_fs = new Tuple<int, FileStream>(-1, null);
-//       private DateTime _playback_t0, _playback_t1;
-//       private bool _all_files_read = false;
-//       SnapshotData _snapshot_data = null; // only initialized when in snapshot mode
-
-//       public PlaybackStream(string datetime_start, string datetime_end, string directory_name, string token) {
-//          _type = PlaybackType.Streaming;
-//          _playback_t0 = PersistorStream.parse_datetime(datetime_start);
-//          _playback_t1 = PersistorStream.parse_datetime(datetime_end);
-//          string token_directory_name = $"{directory_name}\\{token}";
-//          bts.Log.Gen.info($"PlaybackStream t0: {_playback_t0} t1: {_playback_t1} dir: {token_directory_name}");
-//          List<string> files_in_range = new List<string>();
-//          if (Directory.Exists(token_directory_name)) {
-//             _file_names =
-//          Directory.GetFiles(token_directory_name).Where(f => {
-//             DateTime file_time = PersistorStream.parse_datetime_from_filename(f);
-//             return file_time > _playback_t0 && file_time < _playback_t1;
-//          });
-//          }
-//       }
-
-//       protected override void myOnFirstSubscriber() {
-//          FileStream fs = new FileStream(_snapshot_data.snapshot_file, FileMode.Open, FileAccess.Read);
-//          _snapshot_data.packets = StreamPersistorUtils.load_packets_from_file(fs);
-//          PlaybackPacket snapshot_packet = StreamPersistorUtils.find_packet_at_time(_snapshot_data.packets, _snapshot_data.snapshot_time);
-//          if (snapshot_packet != null) {
-//             foreach (PlaybackPacket p in _snapshot_data.packets) {
-//                byte[] ops;
-//                Array row_array;
-//                _deserilializer.deserialize(new Stream[] { new MemoryStream(p.data) }, out ops, out row_array);
-//                for (int i = 0; i < ops.Length; i++) {
-//                   IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
-//                   byte op = ops[i];
-//                   if (op == StreamOpCode.create) {
-//                      IBlueRowCollection full_table = rows_args[0];
-//                      _current_table = new BlueTable(full_table);
-//                   } else if (op == StreamOpCode.multipleRemoveAdd) {
-//                      //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
-//                      IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
-//                      IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
-//                      BlueDataUtility.absorbChanges(_current_table, dels, adds);
-//                   } else {
-//                      bts.Log.assert(false, $"Oprator: {op} is not expected");
-//                   }
-//                   if (p.idx == snapshot_packet.idx) {
-//                      myFireEvent(new DataPacket(_current_table, _current_table, new BlueTable(_current_table.schema)));
-//                      break;
-//                   }
-//                }
-//             }
-//          }
-//       }
-
-
-//       private void _load_packets_from_current_file() {
-//          int curr_idx = _current_fs.Item1;
-//          FileStream curr_fs = _current_fs.Item2;
-//          _packets = new List<PlaybackPacket>();
-//          _packets.AddRange(StreamPersistorUtils.load_packets_from_file(curr_fs));
-//       }
-
-//       private bool current_file_all_replayed {
-//          get {
-//             bool rv = true;
-//             _packets.ForEach(p => rv = rv && p.replayed);
-//             return rv;
-//          }
-//       }
-
-//       protected override void myOnLastSubscriber() {
-//          var fs = _current_fs.Item2;
-//          if (fs != null) {
-//             fs.Close();
-//          }
-//          ViewStreamDatabase.instance.removeWorker(this);
-//       }
-
-//       // public event PlaybackStreamProgress onProgress;
-//       // public event Action onFileExhausted;
-//       private PlaybackPacket _locate_packet_to_replay() {
-//          foreach (var p in _packets) {
-//             // returns the first non replayed packet
-//             if (!p.replayed)
-//                return p;
-//          }
-//          //else
-//          {
-//             double secondsElapsed = new BinaryReader(myFileReader).ReadDouble();
-//             myNextPacketTime = myPlaybackStart + new TimeSpan(0, 0, 0, 0, (int)(1000 * secondsElapsed / myPlaybackSpeed));
-//             if (onProgress != null)
-//                onProgress(myFileReader.Position, myFileReader.Length);
-//          }
-//       }
-
-//       #region IWorker Members
-
-//       public void doWork() {
-//          if (_all_files_read)
-//             return;
-//          if (current_file_all_replayed) {
-//             _load_next_file();
-//          }
-//          PlaybackPacket curr_packet = _locate_packet_to_replay();
-//          if (curr_packet.idx == 0) { // first packet in the file, fire it rightaway 
-//             byte[] ops;
-//             Array row_array;
-//             // what a stupid interface!
-//             _deserilializer.deserialize(new Stream[] { new MemoryStream(curr_packet.data) }, out ops, out row_array);
-//             _fire_packet(ops, row_array);
-//             curr_packet.replay_timestamp = PersistorStream.epoch_now();
-//          } else {
-//             PlaybackPacket prev_packet = _get_previous_packet(curr_packet);
-//             // this is the time we need to fire (replay) this packet
-//             long time_to_fire = prev_packet.replay_timestamp + curr_packet.offset_from_previous;
-//             long now_epoch = PersistorStream.epoch_now();
-//             // fire when we are past the 
-//             if (now_epoch >= time_to_fire) {
-//                byte[] ops;
-//                Array row_array;
-//                _deserilializer.deserialize(new Stream[] { new MemoryStream(curr_packet.data) }, out ops, out row_array);
-//                _fire_packet(ops, row_array);
-//                curr_packet.replay_timestamp = PersistorStream.epoch_now();
-//             }
-//          }
-//       }
-
-//       private void _fire_packet(byte[] ops, Array row_array) {
-//          for (int i = 0; i < ops.Length; i++) {
-//             IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
-//             byte op = ops[i];
-//             if (op == StreamOpCode.create) {
-//                IBlueRowCollection full_table = rows_args[0];
-//                _current_table = new BlueTable(full_table);
-//                //_current_table.addRows(full_table);
-//                myFireEvent(new DataPacket(full_table, full_table, myLastValue == null ? new BlueTable(full_table.schema) :
-//                                           (myLastValue as DataPacket).newRows));
-//             } else if (op == StreamOpCode.multipleRemoveAdd) {
-//                //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
-//                IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
-//                IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
-//                BlueDataUtility.absorbChanges(_current_table, dels, adds);
-//                myFireEvent(new DataPacket(_current_table, adds, dels));
-//             } else {
-//                bts.Log.assert(false, $"Oprator: {op} is not expected");
-//             }
-
-//          }
-//       }
-
-//       private void _load_next_file() {
-//          var new_idx = _current_fs.Item1 + 1;
-//          if (new_idx >= _file_names.Count()) {
-//             _all_files_read = true;
-//             bts.Log.Gen.info("All files read");
-//             return;
-//          }
-//          var prev_fs = _current_fs.Item2;
-//          if (prev_fs != null)
-//             prev_fs.Close();
-//          var new_filename = _file_names.ElementAt(new_idx);
-//          bts.Log.Gen.info("PlaybackStream_load_next_file new_filename=: {new_filename}");
-//          var new_fs = new FileStream(new_filename, FileMode.Open, FileAccess.Read);
-//          _current_fs = new Tuple<int, FileStream>(new_idx, new_fs);
-//          _load_packets_from_current_file();
-//       }
-
-//       #endregion
-//    }
-// }
@@ -1,68 +0,0 @@
-// using System;
-// using System.Collections.Generic;
-// using System.Linq;
-// using System.Text;
-// using System.IO;
-
-// namespace BCG
-// {
-//    public class RecorderStream : ViewStream
-//    {
-//       private ViewStream mySource;
-//       private DateTime myStartTime;
-//       private FileStream myOutFile;
-//       private string myFileName;
-//       private IBlueDataSerializer mySerializer;
-
-//       public RecorderStream(ViewStream src, string recordFile)
-//       {
-//          mySource = src;
-//          myFileName = recordFile;
-
-//          myFileName = myFileName.Replace("%DATE%", DateTime.Now.ToString("yyyyMMdd_HHmmss")); 
-//       }
-
-//       protected override void myOnFirstSubscriber()
-//       {
-//          myStartTime = DateTime.Now;         
-//          mySource.onChange += new ChangeDelegate(mySource_onChange);
-//          myOutFile = new FileStream(myFileName, FileMode.OpenOrCreate);
-//       }
-
-//       protected override void myOnLastSubscriber()
-//       {
-//          mySource.onChange -= mySource_onChange;
-//          myOutFile.Close();
-//       }
-
-//       void mySource_onChange(object newValue)
-//       {
-//          DataPacket dp = (DataPacket)newValue;         
-
-//          BlueTable toWrite = new BlueTable(dp.schema);
-
-//          if (mySerializer == null)
-//          {
-//             mySerializer = new UniversalSerializer(dp.schema);
-//          }
-//          foreach (IBlueRow r in dp.newRows)
-//             toWrite.addRow(r);
-
-//          byte[] result;
-
-//          if (myLastValue == null)
-//             mySerializer.serialize(StreamOpCode.create, new IBlueRowCollection[] { dp.newRows }, out result);
-//          else
-//             mySerializer.serialize(StreamOpCode.multipleRemoveAdd, new IBlueRowCollection[] { dp.deletedRows, dp.addedRows }, out result);
-
-//          BinaryWriter bw = new BinaryWriter(myOutFile);
-//          //write down what time this happened
-//          bw.Write((DateTime.Now - myStartTime).TotalSeconds);
-//          bw.Write(result.Length);
-//          bw.Write(result);         
-//          myOutFile.Flush();
-
-//          myFireEvent(dp);
-//       }
-//    }
-// }
@@ -5,7 +5,8 @@ using System.IO;
 using BCG;

 namespace bts {
-   // this saves the total state in the filr provided in the cons
+   // this saves the updates as a Create on the same file, each update calls -save, which
+   // reqrites the whole data to the packet. The other side of this stream is the Loader
   [DataStreamUserRespresentation("saver")]
   public class SaverStream : ViewStream {
      private ViewStream _stream;
@@ -17,7 +18,8 @@ namespace bts {
         _filename = filename;
      }

-      protected override void myOnFirstSubscriber() { _stream.onChange += new ChangeDelegate(_handle_changes); }
+      protected override void myOnFirstSubscriber() =>
+         _stream.onChange += _handle_changes;

      void _handle_changes(object dp_obj) {
         DataPacket dp = dp_obj as DataPacket;
@@ -25,13 +27,13 @@ namespace bts {
         myFireEvent(dp_obj);
      }

-      protected override void myOnLastSubscriber() { _stream.onChange -= new ChangeDelegate(_handle_changes); }
+      protected override void myOnLastSubscriber() =>
+         _stream.onChange -= _handle_changes;

      protected void _save(DataPacket dp) {
         StreamIOMessage.CreateMessage cr_msg = new StreamIOMessage.CreateMessage(dp.newRows);
         byte[] str_bytes = _serializer.serialize(new StreamIOMessage(cr_msg));
         StreamSerializationUtils.write_to_file(str_bytes, _filename);
-
      }
   }
 }
@@ -3,6 +3,7 @@ using System.Collections.Generic;
 using System.Linq;
 using System.Globalization;
 using System.IO;
+using bts;


 namespace BCG {
@@ -37,12 +37,12 @@ namespace BCG.StreamWorld.Addition

      protected override void myOnFirstSubscriber()
      {
-         myBaseStream.onChange += new ChangeDelegate(myHandleChanges);
+         myBaseStream.onChange += myHandleChanges;
      }

      protected override void myOnLastSubscriber()
      {
-         myBaseStream.onChange -= new ChangeDelegate(myHandleChanges);
+         myBaseStream.onChange -= myHandleChanges;
         myBaseTable.clear();
         myStateTable.clear();
      }
@@ -228,13 +228,13 @@ namespace BCG
      #endregion

      #region overridden methods
-      protected override void myOnFirstSubscriber()
-      { myParentStream.onChange += new ChangeDelegate(myHandleCallbacks); }
+      protected override void myOnFirstSubscriber() =>
+         myParentStream.onChange += myHandleCallbacks;

      protected override void myOnLastSubscriber()
      {
         myLastFiredRows = null;
-         myParentStream.onChange -= new ChangeDelegate(myHandleCallbacks);
+         myParentStream.onChange -= myHandleCallbacks;
      }

      protected override void myFireEvent(object o)
@@ -0,0 +1,22 @@
+namespace bts.SilverPine {
+   using BCG;
+
+   using System;
+
+   public class NullFilterStream : ViewStream {
+      public NullFilterStream(ViewStream parent_str, string col_name, bool filter_out_doublenans = false) {
+         ViewStream rs = new FilterStream(parent_str,
+         r => {
+            bool is_dbnull = r.is_null(col_name);
+            if (is_dbnull) {
+               return false;
+            }
+            if (r.schema[col_name].type_code() == TypeCode.Double) {
+               return !Double.IsNaN(r.get<double>(col_name));
+            }
+            return true;
+         });
+         mySetRealSource(rs);
+      }
+   }
+}
@@ -36,11 +36,11 @@ namespace BCG
         get{return myIsFrozen;}
      }

-      protected override void myOnFirstSubscriber ()
-      {myStreamToFreeze.onChange += new DataStream.ChangeDelegate (myCallBack);}
+      protected override void myOnFirstSubscriber () =>
+         myStreamToFreeze.onChange += myCallBack;

-      protected override void myOnLastSubscriber ()
-      {myStreamToFreeze.onChange -= new DataStream.ChangeDelegate (myCallBack);}
+      protected override void myOnLastSubscriber () =>
+         myStreamToFreeze.onChange -= myCallBack;

      internal void myCallBack(object o)
      {
@@ -96,15 +96,15 @@ namespace BCG

      protected override void myOnFirstSubscriber()
      {
-         this.myLeftSource.onChange += new ChangeDelegate(myLeft_onChange);
-         this.myRightSource.onChange += new ChangeDelegate(myRight_onChange);
+         myLeftSource.onChange += myLeft_onChange;
+         myRightSource.onChange += myRight_onChange;
         this.myVsd.addWorker(this);
      }

      protected override void myOnLastSubscriber()
      {
-         this.myLeftSource.onChange -= new ChangeDelegate(myLeft_onChange);
-         this.myRightSource.onChange -= new ChangeDelegate(myRight_onChange);
+         myLeftSource.onChange -= myLeft_onChange;
+         myRightSource.onChange -= myRight_onChange;
         this.myVsd.removeWorker(this);
      }

@@ -1,102 +1,102 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
+// using System;
+// using System.Collections.Generic;
+// using System.Linq;
+// using System.Text;

-namespace BCG
-{
+// namespace BCG
+// {
   
-    public class AverageGroupsStream : ViewStream
-    {
-        public AverageGroupsStream(ViewStream input, string colToAverage, string keyColumn)
-        {
+//     public class AverageGroupsStream : ViewStream
+//     {
+//         public AverageGroupsStream(ViewStream input, string colToAverage, string keyColumn)
+//         {
            
-            myColToAverage = colToAverage;
-            myInput = input;
-            myKeyColumn = keyColumn;
-        }
+//             myColToAverage = colToAverage;
+//             myInput = input;
+//             myKeyColumn = keyColumn;
+//         }

-        private ViewStream myInput;
-        private string myColToAverage;
-        private BlueTableSchema mySchema;
-        private string myKeyColumn;
+//         private ViewStream myInput;
+//         private string myColToAverage;
+//         private BlueTableSchema mySchema;
+//         private string myKeyColumn;

-        protected override void myOnFirstSubscriber()
-        {
-            myInput.onChange += new ChangeDelegate(myInput_onChange);
-        }
-        protected override void myOnLastSubscriber()
-        {
-            myInput.onChange -= myInput_onChange;
-        }
-        private void myInput_onChange(object newValue)
-        {
-            if (mySchema == null)
-                myBuildSchema(((DataPacket)newValue).schema);
+//         protected override void myOnFirstSubscriber()
+//         {
+//             myInput.onChange += new ChangeDelegate(myInput_onChange);
+//         }
+//         protected override void myOnLastSubscriber()
+//         {
+//             myInput.onChange -= myInput_onChange;
+//         }
+//         private void myInput_onChange(object newValue)
+//         {
+//             if (mySchema == null)
+//                 myBuildSchema(((DataPacket)newValue).schema);

-            IBlueRowCollection newRows = myEvaluate((DataPacket)newValue);
+//             IBlueRowCollection newRows = myEvaluate((DataPacket)newValue);

-            if (myLastValue == null)
-                myFireEvent(new DataPacket(newRows, newRows, new BlueTable(mySchema)));
-            else
-            {
-                IBlueRowCollection added, deleted;
-                BlueDataUtility.computeDifferences(newRows, ((DataPacket) myLastValue).newRows, out deleted, out added);
-                myFireEvent(new DataPacket(newRows, added, deleted));
-            }
-        }
+//             if (myLastValue == null)
+//                 myFireEvent(new DataPacket(newRows, newRows, new BlueTable(mySchema)));
+//             else
+//             {
+//                 IBlueRowCollection added, deleted;
+//                 BlueDataUtility.computeDifferences(newRows, ((DataPacket) myLastValue).newRows, out deleted, out added);
+//                 myFireEvent(new DataPacket(newRows, added, deleted));
+//             }
+//         }

-        private void myBuildSchema(IBlueTableSchema schema)
-        {
-            List<ColumnInfo> columns = new List<ColumnInfo>();
+//         private void myBuildSchema(IBlueTableSchema schema)
+//         {
+//             List<ColumnInfo> columns = new List<ColumnInfo>();

-            foreach (ColumnInfo ci in schema.columnInfo)
-                columns.Add(ci);
+//             foreach (ColumnInfo ci in schema.columnInfo)
+//                 columns.Add(ci);

-            columns.Add(new ColumnInfo("average_"+myColToAverage, columns.Count, typeof(double)));
-            mySchema = new BlueTableSchema("average_of_" + schema.tableName, columns.ToArray());
-        }
+//             columns.Add(new ColumnInfo("average_"+myColToAverage, columns.Count, typeof(double)));
+//             mySchema = new BlueTableSchema("average_of_" + schema.tableName, columns.ToArray());
+//         }



-        private IBlueRowCollection myEvaluate(DataPacket packet)
-        {
+//         private IBlueRowCollection myEvaluate(DataPacket packet)
+//         {
         
-            IDictionary<string, double> colToRunningTotal = new Dictionary<string, double>();
-            IDictionary<string, int> colToCount = new Dictionary<string, int>();
-            IDictionary<string, IBlueRow> colToLastRow = new Dictionary<string, IBlueRow>();
+//             IDictionary<string, double> colToRunningTotal = new Dictionary<string, double>();
+//             IDictionary<string, int> colToCount = new Dictionary<string, int>();
+//             IDictionary<string, IBlueRow> colToLastRow = new Dictionary<string, IBlueRow>();

-            foreach (IBlueRow r in packet.newRows)
-            {
-                double thisVal = Convert.ToDouble(r[myColToAverage]);
+//             foreach (IBlueRow r in packet.newRows)
+//             {
+//                 double thisVal = Convert.ToDouble(r[myColToAverage]);

-                string keyVal = r[myKeyColumn].ToString();
-                if (!colToCount.ContainsKey(keyVal))
-                {
-                    colToRunningTotal[keyVal] = thisVal;
-                    colToCount[keyVal] = 1;
-                }
-                else
-                {
-                    colToRunningTotal[keyVal] += thisVal;
-                    colToCount[keyVal] += 1;
-                }
-                colToLastRow[keyVal] = r;
-            }
+//                 string keyVal = r[myKeyColumn].ToString();
+//                 if (!colToCount.ContainsKey(keyVal))
+//                 {
+//                     colToRunningTotal[keyVal] = thisVal;
+//                     colToCount[keyVal] = 1;
+//                 }
+//                 else
+//                 {
+//                     colToRunningTotal[keyVal] += thisVal;
+//                     colToCount[keyVal] += 1;
+//                 }
+//                 colToLastRow[keyVal] = r;
+//             }

-            BlueTable result = new BlueTable(mySchema);
-            foreach (string key in colToCount.Keys)
-            {
-                object[] values = new object[mySchema.columnInfo.Length];
+//             BlueTable result = new BlueTable(mySchema);
+//             foreach (string key in colToCount.Keys)
+//             {
+//                 object[] values = new object[mySchema.columnInfo.Length];

-                colToLastRow[key].CopyTo(values, 0);
-                values[values.Length - 1] = packet.newRows.Count > 0 ? colToRunningTotal[key] / packet.newRows.Count : 0.0;
-                result.addRow(new BlueRow(mySchema, values));
-            }
+//                 colToLastRow[key].CopyTo(values, 0);
+//                 values[values.Length - 1] = packet.newRows.Count > 0 ? colToRunningTotal[key] / packet.newRows.Count : 0.0;
+//                 result.addRow(new BlueRow(mySchema, values));
+//             }

-            return result;
-        }
+//             return result;
+//         }

      
-    }
-}
+//     }
+// }
@@ -0,0 +1,31 @@
+namespace bts.SilverPine {
+   using BCG;
+
+   using System;
+   using System.Collections.Generic;
+   using System.Linq;
+
+   [DataStreamUserRespresentation("Average")]
+   public class AverageNonIncGroupsStream : ViewStream {
+      public AverageNonIncGroupsStream(ViewStream parent_stream, string inp_col_name, params string[] group_by_cols) {
+         NonIncGroupByStream rs = new NonIncGroupByStream(parent_stream, () => group_by_cols,
+                                                         (g_rows) => _find_mean(inp_col_name, g_rows));
+         mySetRealSource(rs);
+      }
+
+      private List<IBlueRow> _find_mean(string inp_col_name, List<IBlueRow> g_rows) {
+         IBlueRow inp_row = g_rows.ElementAt(0);
+         IEnumerable<double> inp_vals = BlueDataUtility.get_colvalues<double>(g_rows, inp_col_name);
+         double sum = 0.0;
+         inp_vals.ToList().ForEach(d => sum += d);
+         string out_col_name = $"average_{inp_col_name}";
+         double average = sum / inp_vals.Count();
+         IBlueTableSchema schema_out = inp_row.schema.add_column(TypeCode.Double, out_col_name);
+         MutableBlueRow out_row = new MutableBlueRow(schema_out, inp_row);
+         out_row.setValue(out_col_name, average);
+         List<IBlueRow> rv = new List<IBlueRow>();
+         rv.Add(out_row);
+         return rv;
+      }
+   }
+}
@@ -175,7 +175,7 @@ namespace BCG {
         //gets resubscribed, we start the process of building data again, and treat the callback as the first 
         //callback.
         myInit();
-         myStream.onChange -= new DataStream.ChangeDelegate(myChangeHandler);
+         myStream.onChange -= myChangeHandler;
      }

      protected override void myInit() {
@@ -10,6 +10,6 @@ namespace BCG {
        }

        public MaxStream(ViewStream stream, string outputCol)
-           : this(new AddColumnsStream(stream, "fakeCol=0"), outputCol, new string[] { "fakeCol" }) { }
+           : this(new AddColumnsStream(stream, "fake_col: 0"), outputCol, new string[] { "fake_col" }) { }
    }
 }
@@ -82,12 +82,12 @@ namespace BCG

      protected override void myOnFirstSubscriber() 
      {
-         myStream.onChange += new DataStream.ChangeDelegate (myChangeHandler);
+         myStream.onChange += myChangeHandler;
      }

      protected override void myOnLastSubscriber () 
      {
-         myStream.onChange -= new DataStream.ChangeDelegate (myChangeHandler);
+         myStream.onChange -= myChangeHandler;
      }
   }
 }
@@ -1,13 +1,37 @@
 using System;
 using System.Collections.Generic;
 using System.Diagnostics;
+using System.Linq;

 using bts;

-namespace BCG
-{
-   public class NonIncGroupByStream : ViewStream
-   {
+namespace BCG {
+   public class NonIncGroupByStream : ViewStream {
+      private class DelNonIncGroupByEvaluator : INonIncGroupByEvaluator {
+         Func<string[]> _groupby_cols_func;
+         Func<List<IBlueRow>, List<IBlueRow>> _evaluate_func;
+         public DelNonIncGroupByEvaluator(Func<string[]> groupby_cols_func,
+                                          Func<List<IBlueRow>, List<IBlueRow>> evaluate_func) {
+            _groupby_cols_func = groupby_cols_func;
+            _evaluate_func = evaluate_func;
+         }
+         public IBlueRowCollection evaluate(IDictionary<int, List<IBlueRow>> grouped_rows) {
+            BlueTable rv = null;
+            foreach (var kv in grouped_rows) {
+               List<IBlueRow> g_rows = kv.Value;
+               List<IBlueRow> out_rows = _evaluate_func.Invoke(g_rows);
+               if (out_rows != null && out_rows.Count > 0) {
+                  rv = rv ?? new BlueTable(out_rows.ElementAt(0).schema);
+               }
+               rv.addRows(out_rows);
+            }
+            return rv;
+         }
+         public string[] get_group_by_cols() {
+            return _groupby_cols_func();
+         }
+
+      }
      private ViewStream _main_stream;
      private INonIncGroupByEvaluator _groupby_func;
      private BlueRowHasher _row_hasher;
@@ -15,24 +39,29 @@ namespace BCG
      private Stopwatch _timer = new();
      private int _n_total_fires = 0, _n_expensive_fires = 0;

-      public NonIncGroupByStream(ViewStream main_stream, INonIncGroupByEvaluator groupby_func)
-      {
+      public NonIncGroupByStream(ViewStream main_stream, INonIncGroupByEvaluator groupby_func) {
         _main_stream = main_stream;
         _groupby_func = groupby_func;
         _groupby_cols = _groupby_func.get_group_by_cols();
         _row_hasher = new BlueRowHasher(_groupby_cols.Length);
      }

+      public NonIncGroupByStream(ViewStream main_stream, Func<string[]> groupby_cols_func,
+                                 Func<List<IBlueRow>, List<IBlueRow>> evaluate_func)
+                                 : this(main_stream, new DelNonIncGroupByEvaluator(groupby_cols_func, evaluate_func)) { }
+
+
      protected override void myOnFirstSubscriber() { _main_stream.onChange += _main_stream_onchange; }

      protected override void myOnLastSubscriber() { _main_stream.onChange -= _main_stream_onchange; }

-      private void _main_stream_onchange(object nv)
-      {
+      private void _main_stream_onchange(object nv) {
         DataPacket dp = nv as DataPacket;
         ///grouping together
         _timer.Restart();
-         IDictionary<int, List<IBlueRow>> grouped_rows = BlueDataUtility.group_together (dp.newRows, _groupby_cols, ref _row_hasher);
+         IDictionary<int, List<IBlueRow>> grouped_rows = (_groupby_cols == null || _groupby_cols.Length == 0) ?
+         new Dictionary<int, List<IBlueRow>>() { { 0, dp.newRows.to_list() } } :
+         BlueDataUtility.group_together(dp.newRows, _groupby_cols, ref _row_hasher);
         _timer.Stop();
         var groupElapsedMs = _timer.ElapsedMilliseconds;
         if (groupElapsedMs >= 50) {
@@ -48,14 +77,13 @@ namespace BCG
            return;
         }
         var applyElapsedMs = _timer.ElapsedMilliseconds;
-         _n_total_fires++;         
-         if (applyElapsedMs >= 50)
-         {
+         _n_total_fires++;
+         if (applyElapsedMs >= 50) {
            _n_expensive_fires++;
            Log.DataFlow.info(
               $"Time to evaluate {grouped_rows.Count} groups {_timer.ElapsedMilliseconds}ms, ratio {_n_expensive_fires}/{_n_total_fires}");
-         }         
-         myFireEvent(new DataPacket(total_state, total_state, myLastValue == null ? new BlueTable(total_state.schema) : (myLastValue as DataPacket).newRows));             
+         }
+         myFireEvent(new DataPacket(total_state, total_state, myLastValue == null ? new BlueTable(total_state.schema) : (myLastValue as DataPacket).newRows));
      }
   }
 }
@@ -31,15 +31,15 @@ namespace BCG
      {
         myLeftOperand.master = this;
         myRightOperand.master = this;
-         myLeftStream.onChange += new DataStream.ChangeDelegate (myLeftOperand.changeHandler);
-         myRightStream.onChange += new DataStream.ChangeDelegate (myRightOperand.changeHandler);
+         myLeftStream.onChange += myLeftOperand.changeHandler;
+         myRightStream.onChange += myRightOperand.changeHandler;
      }
      
      protected override void myOnLastSubscriber () 
      {
         myInit ();
-         myLeftStream.onChange -= new DataStream.ChangeDelegate (myLeftOperand.changeHandler);
-         myRightStream.onChange -= new DataStream.ChangeDelegate (myRightOperand.changeHandler);
+         myLeftStream.onChange -= myLeftOperand.changeHandler;
+         myRightStream.onChange -= myRightOperand.changeHandler;
      }

      public JoinOperand getOtherOperand (JoinOperand jo)
@@ -116,7 +116,7 @@ namespace BCG
               IDictionary  map = resolver.getMap ();
               map["id"] = "id";
               myPs = new ProjectStream (myIjs, map);
-               myPs.onChange += new DataStream.ChangeDelegate (myProjectChangeHandler);
+               myPs.onChange += myProjectChangeHandler;
               return;
            }
            else
@@ -129,26 +129,26 @@ namespace BCG
      private void myProjectChangeHandler (object o)
      {
         if (myIjs.myOnChangeDelegate.GetInvocationList().Length > 1)
-            myIjs.onChange -= new DataStream.ChangeDelegate (myInternalJoinChangeHandler);
+            myIjs.onChange -= myInternalJoinChangeHandler;
         myFireEvent ((o as ICloneable).Clone());
         
      }

      protected override void myOnFirstSubscriber ()
      {
-         myIjs.onChange += new DataStream.ChangeDelegate (myInternalJoinChangeHandler);
+         myIjs.onChange += myInternalJoinChangeHandler;
      }

      protected override void myOnLastSubscriber ()
      {
         // how does this relate to the -= in myProjectChangeHandler?
-         myIjs.onChange -= new DataStream.ChangeDelegate (myInternalJoinChangeHandler);
+         myIjs.onChange -= myInternalJoinChangeHandler;

         // if there is a  project after the ijs we need to 
         // unsubscribe to it otherwise references will be kept and 
         // a memory leak will occur
         if (myPs != null)
-            myPs.onChange -= new DataStream.ChangeDelegate (myProjectChangeHandler);
+            myPs.onChange -= myProjectChangeHandler;
         myPs = null;
      }

@@ -5,11 +5,10 @@ using mbts;
 using System;
 using System.Collections.Generic;
 using System.Diagnostics;
+using System.Security.Cryptography;

-namespace BCG
-{
-   public class NonIncJoinBookkeeper : IDisposable
-   {
+namespace BCG {
+   public class NonIncJoinBookkeeper : IDisposable {
      public Action on_new_data;
      private readonly NonIncJoinSpec _join_spec;
      private readonly int _join_position;
@@ -26,41 +25,38 @@ namespace BCG

      private int[] _join_col_indices = null,
                     _data_col_indices = null;
-      
-      public NonIncJoinBookkeeper(NonIncJoinSpec the_join_spec, int join_position, int start_index_in_output, BlueRowHasher row_hasher)
-      {
+
+      public NonIncJoinBookkeeper(NonIncJoinSpec the_join_spec, int join_position, int start_index_in_output, BlueRowHasher row_hasher) {
         _join_spec = the_join_spec;
         _join_position = join_position;
         _start_index_in_output = start_index_in_output;
         _row_hasher = row_hasher;
         _is_left_most = _join_position == 0;
         _is_tag_along = _join_spec.join_col_names.Length == 0;
+      }
+
+      public void init() {
         _join_spec.stream.onChange += _on_stream_cb;
      }

-      public void _on_stream_cb(object dp_obj)
-      {
+      public void _on_stream_cb(object dp_obj) {
+         Log.Gen.info($"NonIncJoinBookkeeper._on_stream_cb dp_obj: {dp_obj}");
         var timer = Stopwatch.StartNew();
         DataPacket dp = (DataPacket)dp_obj;
-         if (_schema == null)
-         {
+         if (_schema == null) {
            _schema = dp.schema;
            _join_col_indices = BlueDataUtility.get_column_indices(_schema, _join_spec.join_col_names);
            _data_col_indices = BlueDataUtility.get_column_indices(_schema, _join_spec.data_col_names);
         }

-         if (_is_left_most)
-         {
+         if (_is_left_most) {
            _row_list.Clear();
            foreach (IBlueRow r in dp.newRows)
-               _row_list.Add(new Pair<int, IBlueRow>(_row_hasher.get_hash(r, _join_col_indices), r));            
-         }
-         else
-         {
+               _row_list.Add(new Pair<int, IBlueRow>(_row_hasher.get_hash(r, _join_col_indices), r));
+         } else {
            if (_is_tag_along)
               _solitary_row = BlueDataUtility.extractRow(dp.newRows, 0);
-            else
-            {
+            else {
               _row_map.Clear();
               foreach (IBlueRow r in dp.newRows)
                  _row_map[_row_hasher.get_hash(r, _join_col_indices)] = r;
@@ -79,31 +75,25 @@ namespace BCG

      public bool is_ready { get { return _schema != null; } }

-      public void constribute_cols_to_schema(List<ColumnInfo> cis)
-      {
+      public void constribute_cols_to_schema(List<ColumnInfo> cis) {
         for (int i = 0; i < _join_spec.data_col_names.Length; i++)
            cis.Add(new ColumnInfo(_join_spec.projected_col_names[i], cis.Count, _schema[_join_spec.data_col_names[i]].colType));
      }

      public List<Pair<int, IBlueRow>> get_list() { return _row_list; }

-      public void copy_master_row_to_output_row(int row_index, object[] output_row)
-      {
+      public void copy_master_row_to_output_row(int row_index, object[] output_row) {
         IBlueRow this_row = _row_list[row_index].second;
         BlueDataUtility.copy_to_row(this_row, _data_col_indices, output_row, _start_index_in_output);
      }

-      public void copy_matching_row_to_output_row(int row_hash, object[] output_row)
-      {
-         if (_is_tag_along)
-         {
+      public void copy_matching_row_to_output_row(int row_hash, object[] output_row) {
+         if (_is_tag_along) {
            if (_solitary_row != null)
               BlueDataUtility.copy_to_row(_solitary_row, _data_col_indices, output_row, _start_index_in_output);
            else
               copy_dbnull_to_output_row(output_row);
-         }
-         else
-         {
+         } else {
            IBlueRow this_row = null;
            if (_row_map.TryGetValue(row_hash, out this_row))
               BlueDataUtility.copy_to_row(this_row, _data_col_indices, output_row, _start_index_in_output);
@@ -112,8 +102,7 @@ namespace BCG
         }
      }

-      private void copy_dbnull_to_output_row(object[] output_row)
-      {
+      private void copy_dbnull_to_output_row(object[] output_row) {
         for (int i = 0; i < _data_col_indices.Length; i++)
            output_row[i + _start_index_in_output] = DBNull.Value;
      }
@@ -3,63 +3,62 @@ using System.Collections.Generic;
 using System.Linq;
 using System.Text;
 using System.Threading.Tasks;
+
 using mbts;

-namespace BCG
-{
-   public class NonIncJoinSpec
-   {
+namespace BCG {
+   public class NonIncJoinSpec {
      public ViewStream stream;
+      // join col names are the names on which the streams are joined (for equality),
+      // data col names are the names in the original stream
+      // projected col naes are their new names, ie if its x:l_x, then x is in the 
+      // data_col_names and l_x in projected cols
      public string[] join_col_names, data_col_names, projected_col_names;

      public NonIncJoinSpec() { }

      public NonIncJoinSpec(ViewStream the_stream, string[] the_join_col_names, string[] the_data_col_names)
-         : this (the_stream, the_join_col_names, the_data_col_names, the_data_col_names)
-      { }
+         : this(the_stream, the_join_col_names, the_data_col_names, the_data_col_names) { }

-      public NonIncJoinSpec(ViewStream the_stream, string[] the_join_col_names, string[] the_data_col_names, string[] the_projected_col_names)
-      {
+      public NonIncJoinSpec(ViewStream the_stream, string[] the_join_col_names, string[] the_data_col_names, string[] the_projected_col_names) {
         stream = the_stream;
         join_col_names = the_join_col_names;
         data_col_names = the_data_col_names;
         projected_col_names = the_projected_col_names;
      }

-      public static NonIncJoinSpec[] from_triplet(params Triplet<ViewStream, string, string>[] join_specs)
-      {
+      public static NonIncJoinSpec[] from_triplet(params Triplet<ViewStream, string, string>[] join_specs) {
         List<NonIncJoinSpec> rv = new List<NonIncJoinSpec>();
-         foreach (Triplet<ViewStream, string, string> r in join_specs)
-         {
-            string[] keys = String.IsNullOrEmpty (r.second) ? new string[] {} : r.second.Split(' ');
+         foreach (Triplet<ViewStream, string, string> r in join_specs) {
+            string[] keys = String.IsNullOrEmpty(r.second) ? new string[] { } : r.second.Split(' ');
            string[] visible_cols = r.third.Split(' ');
            List<string> data_col_names = new List<string>(), mapped_col_names = new List<string>();
-            foreach (string s in visible_cols)
-            {
+            foreach (string s in visible_cols) {
               string[] toks = s.Split(':');
-               if (toks.Length == 1)
-               {
+               if (toks.Length == 1) {
                  data_col_names.Add(toks[0]);
                  mapped_col_names.Add(toks[0]);
-               }
-               else if (toks.Length == 2)
-               {
+               } else if (toks.Length == 2) {
                  data_col_names.Add(toks[0]);
                  mapped_col_names.Add(toks[1]);
               }
            }
-            rv.Add (new NonIncJoinSpec (r.first, keys, data_col_names.ToArray (), mapped_col_names.ToArray()));
+            rv.Add(new NonIncJoinSpec(r.first, keys, data_col_names.ToArray(), mapped_col_names.ToArray()));
         }
-         return rv.ToArray ();
+         return rv.ToArray();
+      }
+
+      public override string ToString() {
+         Func<string[], string> arr_txt = (a) => {
+            string rv = "";
+            foreach (string s in a) {
+               rv += "{s} ";
+            }
+            rv = rv.Trim();
+            return rv;
+         };
+         string rv = $"stream: {stream} join_col_names: {arr_txt(join_col_names)} data_col_names: {arr_txt(data_col_names)} projected_col_names: {arr_txt(projected_col_names)}";
+         return rv;
      }
   }
-
-   //[DataStreamUserRespresentation("ninctest")]
-   //public class NonIncJoinTestStream : ViewStream
-   //{
-   //   public NonIncJoinTestStream (string key_cols, string data_cols)
-   //   {
-   //      NonIncJoinSpec.from_triplet(new Triplet<ViewStream, string, string>(null, key_cols, data_cols));
-   //   }
-   //}
 }
@@ -6,11 +6,9 @@ using System;
 using System.Collections.Generic;
 using System.Diagnostics;

-namespace BCG
-{
+namespace BCG {
   [DataStreamUserRespresentation("fastJoin")]
-   public class NonIncJoinStream : ViewStream, BCG.Utilities.General.IWorker
-   {
+   public class NonIncJoinStream : ViewStream, BCG.Utilities.General.IWorker {
      private readonly NonIncJoinSpec[] _join_specs;
      private readonly INonIncJoinInlineMapper _output_mapper;
      private readonly BlueRowHasher _row_hasher;
@@ -28,24 +26,19 @@ namespace BCG
               {
                  new Triplet<ViewStream, string, string> (left, left_join_cols, left_proj),
                  new Triplet<ViewStream, string, string> (right, right_join_cols, right_proj)
-               })
-      { }
+               }) { }

      public NonIncJoinStream(params Triplet<ViewStream, string, string>[] join_specs)
-         : this(NonIncJoinSpec.from_triplet(join_specs), null, 0)
-      { }
+         : this(NonIncJoinSpec.from_triplet(join_specs), null, 0) { }

      public NonIncJoinStream(ViewStreamDatabase vsd, INonIncJoinInlineMapper output_mapper, int throttles_ms,
                              params Triplet<ViewStream, string, string>[] join_specs)
-         : this(vsd, NonIncJoinSpec.from_triplet(join_specs), output_mapper, throttles_ms)
-      { }
+         : this(vsd, NonIncJoinSpec.from_triplet(join_specs), output_mapper, throttles_ms) { }

      public NonIncJoinStream(NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper output_mapper, int throttles_ms = 0)
-         : this(ViewStreamDatabase.instance, join_specs, output_mapper, throttles_ms)
-      { }
+         : this(ViewStreamDatabase.instance, join_specs, output_mapper, throttles_ms) { }

-      public NonIncJoinStream(ViewStreamDatabase vsd, NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper output_mapper, int throttles_ms = 0)
-      {
+      public NonIncJoinStream(ViewStreamDatabase vsd, NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper output_mapper, int throttles_ms = 0) {
         _join_specs = join_specs;
         _output_mapper = output_mapper;
         _row_hasher = new BlueRowHasher(_join_specs[0].join_col_names.Length);
@@ -53,12 +46,11 @@ namespace BCG
         _vsd = vsd;
      }

-      protected override void myOnFirstSubscriber()
-      {
+      protected override void myOnFirstSubscriber() {
         int data_col_no = 1;
-         for (int jp = 0; jp < _join_specs.Length; jp++)
-         {
+         for (int jp = 0; jp < _join_specs.Length; jp++) {
            NonIncJoinBookkeeper s = new NonIncJoinBookkeeper(_join_specs[jp], jp, data_col_no, _row_hasher);
+            Log.Gen.info($"NonIncJoinStream.myOnFirstSubscriber jp: {jp} spec: {_join_specs[jp]}");
            data_col_no += _join_specs[jp].data_col_names.Length;
            s.on_new_data += _stream_callback;
            _book_keeping_data.Add(s);
@@ -66,10 +58,11 @@ namespace BCG
         if (_output_mapper != null)
            _output_mapper.onMapperChange += _on_mapper_change;
         _vsd.addWorker(this);
+         foreach (var bkd in _book_keeping_data)
+            bkd.init();
      }

-      protected override void myOnLastSubscriber()
-      {
+      protected override void myOnLastSubscriber() {
         foreach (NonIncJoinBookkeeper d in _book_keeping_data)
            d.Dispose();
         if (_output_mapper != null)
@@ -77,30 +70,26 @@ namespace BCG
         _vsd.removeWorker(this);
      }

-      private void _stream_callback()
-      {
+      private void _stream_callback() {
+         Log.Gen.info("NonIncJoinStream._stream_callback received callback");
         _is_dirty = true;
         doWork();
      }

-      private void _on_mapper_change()
-      {
+      private void _on_mapper_change() {
         _is_dirty = true;
         doWork();
      }

-      private bool _is_ready()
-      {
+      private bool _is_ready() {
         bool rv = true;
         foreach (NonIncJoinBookkeeper d in _book_keeping_data)
            rv &= d.is_ready;
         return rv;
      }

-      private void _construct_schema()
-      {
-         if (_output_schema == null)
-         {
+      private void _construct_schema() {
+         if (_output_schema == null) {
            List<ColumnInfo> cis = new List<ColumnInfo>(new ColumnInfo[] { new ColumnInfo("id", 0, typeof(int)) });
            foreach (NonIncJoinBookkeeper d in _book_keeping_data)
               d.constribute_cols_to_schema(cis);
@@ -109,27 +98,23 @@ namespace BCG
      }

      private int _n_total_fires = 0, _n_expensive_fires = 0;
-      private void _form_and_fire()
-      {
-         if (_is_ready())
-         {
+      private void _form_and_fire() {
+         Log.Gen.info($"NonIncJoinStream._form_and_fire _is_ready(): {_is_ready()}");
+         if (_is_ready()) {
            _construct_schema();
            var timer = Stopwatch.StartNew();
            IList<Pair<int, IBlueRow>> left_row_list = _book_keeping_data[0].get_list();
            List<object[]> output_rows = new List<object[]>();

-            for (int i = 0; i < left_row_list.Count; i++)
-            {
+            for (int i = 0; i < left_row_list.Count; i++) {
               object[] or = new object[_output_schema.columnInfo.Length];
               or[0] = i;
               output_rows.Add(or);
            }

-            for (int ji = 0; ji < _book_keeping_data.Count; ji++)
-            {
+            for (int ji = 0; ji < _book_keeping_data.Count; ji++) {
               NonIncJoinBookkeeper bk_data = _book_keeping_data[ji];
-               for (int i = 0; i < left_row_list.Count; i++)
-               {
+               for (int i = 0; i < left_row_list.Count; i++) {
                  if (ji == 0)
                     bk_data.copy_master_row_to_output_row(i, output_rows[i]);
                  else
@@ -139,19 +124,20 @@ namespace BCG
            IBlueRowCollection dp_output_rows = _apply_map(new SequentialIDBlueTable(_output_schema, output_rows));
            timer.Stop();
            _n_total_fires++;
-            if (timer.ElapsedMilliseconds >= 250) {
-               _n_expensive_fires++;
-               string log_msg = $"Join and map time for {_book_keeping_data.Count} "
-                  + $"tables {timer.Elapsed} ratio {_n_expensive_fires}/{_n_total_fires}";
-            }
-            myFireEvent(new DataPacket(dp_output_rows, dp_output_rows, myLastValue == null ? new BlueTable(_output_schema) : (myLastValue as DataPacket).newRows));
+            //if (timer.ElapsedMilliseconds >= 250) {
+            _n_expensive_fires++;
+            string log_msg = $"Join and map time for {_book_keeping_data.Count} "
+               + $"tables {timer.Elapsed.TotalMilliseconds} msecs, ratio {_n_expensive_fires}/{_n_total_fires}";
+            Log.Gen.info(log_msg);
+            //}
+            var dp_out = BlueDataUtility.build_dp_inc(dp_output_rows, myLastValue as DataPacket);
+            myFireEvent(dp_out);
+            //myFireEvent(new DataPacket(dp_output_rows, dp_output_rows, myLastValue == null ? new BlueTable(_output_schema) : (myLastValue as DataPacket).newRows));
         }
      }

-      private IBlueRowCollection _apply_map(IBlueRowCollection unmapped_rows)
-      {
-         if (_output_mapper != null)
-         {
+      private IBlueRowCollection _apply_map(IBlueRowCollection unmapped_rows) {
+         if (_output_mapper != null) {
            IBlueRowCollection output_rows;
            var timer = Stopwatch.StartNew();
            output_rows = _output_mapper.map_rows(unmapped_rows);
@@ -160,17 +146,14 @@ namespace BCG
               Log.Gen.info($"Time to map {unmapped_rows.Count} rows into {output_rows.Count} rows {timer.Elapsed}");
            }
            return output_rows;
-         }
-         else
-         {
+         } else {
            return unmapped_rows;
         }
      }

-      public void doWork()
-      {
-         if (DateTime.Now - _last_fire_time >= _throttle_period && _is_dirty)
-         {
+      public void doWork() {
+         Log.Gen.info($"NonIncJoinStream.doWork _last_fire_time: {_last_fire_time} _is_dirty: {_is_dirty}");
+         if (DateTime.Now - _last_fire_time >= _throttle_period && _is_dirty) {
            _is_dirty = false;
            _last_fire_time = DateTime.Now;
            _form_and_fire();
@@ -181,8 +181,8 @@ namespace BCG

       protected override void myOnFirstSubscriber()
       {
-           myVLeft.onChange += new ChangeDelegate(myVLeft_onChange);
-           myVRight.onChange += new ChangeDelegate(myVRight_onChange);
+           myVLeft.onChange += myVLeft_onChange;
+           myVRight.onChange += myVRight_onChange;
       }

       void myVRight_onChange(object newValue)
@@ -0,0 +1,68 @@
+namespace bts {
+   using BCG;
+   using BCG.Utilities.General;
+
+   using System;
+   using System.Diagnostics;
+
+   using ZstdSharp.Unsafe;
+
+   // functon that joins the 2 rows and returns the outputput row. The function
+   // can return a null which means the 2 rows dont need to be joined
+   using JoinFunc = System.Func<BCG.IBlueRow, BCG.IBlueRow, BCG.IBlueRow>;
+   public class ProximityJoinStream : ViewStream, IWorker {
+      private ViewStream _lvs, _rvs;
+      private JoinFunc _join_func;
+      private bool _dirty = false;
+      private Stopwatch _stopwatch = new Stopwatch();
+      private ViewStreamDatabase _vsd = ViewStreamDatabase.instance;
+      public ProximityJoinStream(ViewStream lvs, ViewStream rvs, JoinFunc join_func) {
+         _lvs = lvs;
+         _rvs = rvs;
+         _join_func = join_func;
+      }
+
+      protected override void myOnFirstSubscriber() {
+         _lvs.onChange += _handle_data;
+         _rvs.onChange += _handle_data;
+         _vsd.addWorker(this);
+      }
+
+      protected override void myOnLastSubscriber() {
+         _lvs.onChange -= _handle_data;
+         _rvs.onChange -= _handle_data;
+         _vsd.removeWorker(this);
+      }
+
+      private void _handle_data(object dp_obj) {
+         _dirty = true;
+         doWork();
+      }
+
+      public void doWork() {
+         Func<bool> all_data_revcd = () => {
+            return _lvs.rows != null && _rvs.rows != null;
+         };
+         _stopwatch.Start();
+         if (_dirty && all_data_revcd()) {
+            _dirty = false;
+            BlueTable rv = null;
+            foreach (IBlueRow lr in _lvs.rows) {
+               foreach (IBlueRow rr in _rvs.rows) {
+                  IBlueRow resr = _join_func(lr, rr);
+                  if (resr == null)
+                     continue;
+                  rv = rv ?? new BlueTable(resr.schema);
+                  rv.addRow(resr);
+               }
+            }
+            _stopwatch.Stop();
+            Log.Gen.info($"ProximityJoinStream left_count: {_lvs.rows.Count} right_count: {_rvs.rows.Count} result_count: {rv.Count} time: {_stopwatch.ElapsedMilliseconds} msecs");
+            var dp = BlueDataUtility.build_dp_inc(rv, myLastValue as DataPacket);
+            myFireEvent(dp);
+         }
+
+      }
+
+   }
+}
@@ -0,0 +1,47 @@
+namespace bts.SilverPine {
+   using BCG;
+
+   using System;
+   using System.CodeDom;
+   using System.Configuration;
+   using System.Reactive.Disposables;
+
+   public class TimeProximityJoinStream : ViewStream {
+      private int _id = 0;
+
+      public TimeProximityJoinStream(ViewStream ls, string l_time_col, ViewStream rs, string r_time_col,
+                                       string pre_join_period, string post_join_period) :
+            this(ls, l_time_col, rs, r_time_col, TimeBasedFilter.parse_duration(pre_join_period),
+            TimeBasedFilter.parse_duration(post_join_period)) { }
+
+
+      public TimeProximityJoinStream(ViewStream ls, string l_time_col, ViewStream rs, string r_time_col,
+                                       TimeSpan pre_join_period, TimeSpan post_join_period) {
+         Func<IBlueRow, IBlueRow, IBlueRow> create_row = (lr, rr) => {
+            var id = 0;
+            lock (this) {
+               id = _id++;
+            }
+            IBlueRow rv = BlueDataUtility.create_composite_row(lr, rr, id);
+            return rv;
+         };
+         Func<IBlueRow, IBlueRow, IBlueRow> join_func = (lr, rr) => {
+            DateTime ldt = lr.get_datetime(l_time_col);
+            DateTime rdt = rr.get_datetime(r_time_col);
+            TimeSpan diff = ldt - rdt;
+            if (ldt < rdt) {
+               if (diff.Duration() > pre_join_period) {
+                  return null;
+               }
+            } else {
+               if (diff > post_join_period)
+                  return null;
+            }
+            return create_row(lr, rr);
+         };
+
+         ViewStream rstr = new ProximityJoinStream(ls, rs, join_func);
+         mySetRealSource(rstr);
+      }
+   }
+}
@@ -327,14 +327,14 @@ namespace BCG

      protected override void myOnFirstSubscriber()
      {
-         myLeftStream.onChange += new ChangeDelegate(myHandleLeftStreamChanges);
-         myRightStream.onChange += new ChangeDelegate(myHandleRightStreamChanges);
+         myLeftStream.onChange += myHandleLeftStreamChanges;
+         myRightStream.onChange += myHandleRightStreamChanges;
      }

      protected override void myOnLastSubscriber()
      {
-         myLeftStream.onChange -= new ChangeDelegate(myHandleLeftStreamChanges);
-         myRightStream.onChange -= new ChangeDelegate(myHandleRightStreamChanges);
+         myLeftStream.onChange -= myHandleLeftStreamChanges;
+         myRightStream.onChange -= myHandleRightStreamChanges;
         myLeftKeyToLeftRows.Clear();
         myRightKeyToRightRows.Clear();
         myLeftIDToResultRows.Clear();
@@ -23,15 +23,15 @@ namespace BCG
         myRightOperand.master = this;
         //the events are not hooked onto the operands delegate, but rather a level of indirection
         //added in the form on myLeftChangeHandler
-         myLeftStream.onChange += new DataStream.ChangeDelegate (myLeftChangeHandler);
-         myRightStream.onChange += new DataStream.ChangeDelegate (myRightChangeHandler);
+         myLeftStream.onChange += myLeftChangeHandler;
+         myRightStream.onChange += myRightChangeHandler;
      }
       
      protected override void myOnLastSubscriber () 
      {
         myInit ();
-         myLeftStream.onChange -= new DataStream.ChangeDelegate (myLeftChangeHandler);
-         myRightStream.onChange -= new DataStream.ChangeDelegate (myRightChangeHandler);
+         myLeftStream.onChange -= myLeftChangeHandler;
+         myRightStream.onChange -= myRightChangeHandler;
      }

      protected override void myInit()
@@ -2,9 +2,13 @@ using System;
 using System.Collections;
 using System.Collections.Generic;

+using Amazon.Runtime.Internal.Util;
+
 using BCG;
 using BCG.Utilities;

+using bts;
+
 namespace BCG {
   public class LiteralStream : ViewStream {

@@ -22,6 +26,34 @@ namespace BCG {
         return rv;
      }

+      //literal("table1", "x:123, y:1.001, z:abc;x:123, y:1.001, z:abc")
+      public LiteralStream(string table_name, string rows) {
+         Func<string, int, IBlueRow> get_row = (json_row, id) => {
+            Dictionary<string, object> map = new Dictionary<string, object>();
+            string[] toks = json_row.Split(',', StringSplitOptions.TrimEntries);
+            //Logger.assert(col_vals.Length == 2);
+            foreach (string t in toks) {
+               var col_vals = t.Split(':', StringSplitOptions.TrimEntries);
+               string col = col_vals[0];
+               string val = col_vals[1];
+               map.Add(col, val);
+            }
+            map["id"] = id;
+            return new BlueRow(map);
+         };
+         string[] row_arr = rows.Split(';', StringSplitOptions.TrimEntries);
+         BlueTable table = null;
+         foreach (string json_string in row_arr) {
+            IBlueRow r = //BlueDataUtility.convert_json_to_row(json_string, table == null ? 0 : table.Count);
+            get_row(json_string, table == null ? 0 : table.Count);
+            table = table ?? new BlueTable(r.schema);
+            table.addRow(r);
+         }
+         ViewStream str = new BlueTableStream(table);
+         str = new SchemaGuesserStream(str);
+         mySetRealSource(str);
+      }
+
      public LiteralStream(string tableName, string schema, string value) {
         string[] columns = schema.Split(' ');

@@ -55,7 +87,7 @@ namespace BCG {
         mySetRealSource(new BlueRowCollectionStaticStream(t));
      }

-      public LiteralStream(string schema, string value) : this("literal", schema, value) { }
+      // public LiteralStream(string schema, string value) : this("literal", schema, value) { }

      // public LiteralStream(string value) {

@@ -0,0 +1,72 @@
+using BCG;
+
+using MongoDB.Driver.Core.WireProtocol.Messages;
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+
+// the first arg is all the rows, the second arg is the index of the row that needs mapping, and 
+// the func returns the mapped row (ie mapped version of the index row)
+using GlobalMap = System.Func<System.Collections.Generic.List<BCG.IBlueRow>, uint, BCG.IBlueRow>;
+
+namespace bts {
+   // In GlobalMap, each row is mapped with the full view of all rows, ie the map functions for a row
+   // can use the state fo all rows.
+   public class GlobalMapStream : ViewStream {
+      private ViewStream _base_stream;
+      private readonly string _sort_col_name, _group_by_col;
+      private GlobalMap _map_func;
+      public GlobalMapStream(ViewStream base_stream, string sort_col_name, string group_by_col, GlobalMap map_func) {
+         _base_stream = base_stream;
+         _sort_col_name = sort_col_name;
+         _group_by_col = group_by_col;
+         _map_func = map_func;
+      }
+
+      public GlobalMapStream(ViewStream base_stream, string sort_col_name, GlobalMap map_func) :
+      this(base_stream, sort_col_name, null, map_func) {
+      }
+
+      protected override void myOnFirstSubscriber() {
+         _base_stream.onChange += _handle_stream_update;
+      }
+
+      protected override void myOnLastSubscriber() {
+         _base_stream.onChange -= _handle_stream_update;
+      }
+
+      void _handle_stream_update(object dp_obj) {
+         DataPacket dp = dp_obj as DataPacket;
+         IDictionary<int, List<IBlueRow>> groups = !String.IsNullOrEmpty(_group_by_col) ?
+         BlueDataUtility.group_together(dp.newRows, new string[] { _group_by_col })
+         :
+         new Dictionary<int, List<IBlueRow>>() { { 0, dp.newRows.to_list() } };
+         BlueTable out_table = null;
+         foreach (var kv in groups) {
+            List<IBlueRow> group_rows = kv.Value;
+            group_rows.Sort((r1, r2) => BlueDataUtility.compare_rows(r2, r1, _sort_col_name));
+            uint i = 0;
+            foreach (IBlueRow r in group_rows) {
+               IBlueRow nr = _map_func(group_rows, i++);
+               out_table = out_table ?? new BlueTable(nr.schema);
+               out_table.addRow(nr);
+            }
+         }
+
+
+         // List<IBlueRow> rows = dp.newRows.to_ienum().ToList();
+         // rows.Sort((r1, r2) => BlueDataUtility.compare_rows(r1, r2, _sort_col_name));
+         // uint i = 0;
+
+         // BlueTable out_table = null;
+         // foreach (IBlueRow r in rows) {
+         //    IBlueRow nr = _map_func(rows, i++);
+         //    out_table = out_table ?? new BlueTable(nr.schema);
+         //    out_table.addRow(nr);
+         // }
+         DataPacket dp_out = BlueDataUtility.build_dp_inc(out_table, myLastValue as DataPacket);
+         myFireEvent(dp_out);
+      }
+   }
+}
@@ -119,13 +119,13 @@ namespace BCG {


      protected override void myOnFirstSubscriber() {
-         myStream.onChange += new DataStream.ChangeDelegate(myChangeHandler);
+         myStream.onChange += myChangeHandler;
      }

      protected override void myOnLastSubscriber() {
         myInit();
         if (myStream != null)
-            myStream.onChange -= new DataStream.ChangeDelegate(myChangeHandler);
+            myStream.onChange -= myChangeHandler;
      }

      protected override void myOnTableClear() {
@@ -83,10 +83,11 @@ namespace BCG {
         myMapper.onMapperChange += new MapperChangeDelegate(myHandleMapChange);
      }

-      protected override void myOnFirstSubscriber() { myStream.onChange += new ChangeDelegate(myHandleData); }
+      protected override void myOnFirstSubscriber() =>
+         myStream.onChange += myHandleData;

      protected override void myOnLastSubscriber() {
-         myStream.onChange -= new ChangeDelegate(myHandleData);
+         myStream.onChange -= myHandleData;
         myRows = null;
         myBaseRows = null;
         myMap.Clear();
@@ -16,10 +16,10 @@ namespace BCG {
      }

      protected override void myOnFirstSubscriber() {
-         _stream.onChange += new ChangeDelegate(_onstream_change);
+         _stream.onChange += _onstream_change;
      }
      protected override void myOnLastSubscriber() {
-         _stream.onChange -= new ChangeDelegate(_onstream_change);
+         _stream.onChange -= _onstream_change;
      }

      private void _onstream_change(object o) {
@@ -45,4 +45,4 @@ namespace BCG {

      }
   }
-}
+}
@@ -17,10 +17,10 @@ namespace BCG {
      }

      protected override void myOnFirstSubscriber() {
-         _stream.onChange += new ChangeDelegate(_onstream_change);
+         _stream.onChange += _onstream_change;
      }
      protected override void myOnLastSubscriber() {
-         _stream.onChange -= new ChangeDelegate(_onstream_change);
+         _stream.onChange -= _onstream_change;
      }


@@ -41,7 +41,6 @@ namespace BCG {
         }
         _table.addRows(added_rows);
         myFireEvent(new DataPacket(_table, added_rows, new BlueTable(_table.schema)));
-
      }
   }

@@ -137,10 +137,24 @@ namespace bts {
         HashSet<string> removed_cols = new HashSet<string>();
         Dictionary<string, string> mapped_to_orig_map = new Dictionary<string, string>();
         Dictionary<string, string> orig_to_mapped = new Dictionary<string, string>();
+         // by default every col will show up, so if we have the projection as (a:a1)
+         // it will convert a to a1, 
+         // examples: if a schema has cols a b c
+         // project(parent_str, "a:aa ") -> aa b c
+         // project(parent_str, "a:aa !c") -> aa b
+         // project(parent_str, "a:aa !") -> aa
+         // project(parent_str, "a:aa b = (aa*aa)!") -> aa b
+         bool hide_all = false;
         foreach (string p_tok in project_toks) {
            if (p_tok.StartsWith("!")) {
-               string removed_col = p_tok.Substring(1);
-               removed_cols.Add(removed_col);
+               if (p_tok.Length == 1) {
+                  // if it is a "blanket "!"=> hide all cols except the one that
+                  // are stated explicitly
+                  hide_all = true;
+               } else {
+                  string removed_col = p_tok.Substring(1);
+                  removed_cols.Add(removed_col);
+               }
            } else if (p_tok.Contains(":")) {
               string[] map_toks = p_tok.Split(":");
               string orig_col = map_toks[0];
@@ -155,12 +169,11 @@ namespace bts {
               }
            }
         }
+
         // make the new schema
         List<ColumnInfo> new_cols = new List<ColumnInfo>();
         foreach (ColumnInfo ci in input_schema.columnInfo) {
            if (ci.colName == "id") {
-               // if there's a hidden col, other columns will show by default, but if cols are being
-               // projected epxlicitely, than others wil be hidden
               ColumnInfo nc = new ColumnInfo(ci.colName, new_cols.Count, ci.colType);
               new_cols.Add(nc);
               continue;
@@ -172,12 +185,13 @@ namespace bts {
               ColumnInfo nc = new ColumnInfo(mapped_name, new_cols.Count, ci.colType);
               new_cols.Add(nc);
            } else {
-               if (ci.colName == "id" || removed_cols.Count > 0) {
-                  // if there's a hidden col, other columns will show by default, but if cols are being
-                  // projected epxlicitely, than others wil be hidden
-                  ColumnInfo nc = new ColumnInfo(ci.colName, new_cols.Count, ci.colType);
-                  new_cols.Add(nc);
-               }
+               if (hide_all)
+                  continue;
+               //if (ci.colName == "id" || removed_cols.Count > 0) {
+               // unless columns are  explicitely hidden, all columns show up
+               ColumnInfo nc = new ColumnInfo(ci.colName, new_cols.Count, ci.colType);
+               new_cols.Add(nc);
+               //}
            }
         }
         IBlueTableSchema new_schema = new BlueTableSchema(input_schema.tableName, new_cols.ToArray());
@@ -54,12 +54,12 @@ namespace BCG {

      protected override void myOnFirstSubscriber()
      {
-         myVs.onChange += new BCG.DataStream.ChangeDelegate (myHandleChange);
+         myVs.onChange += myHandleChange;
      }

      protected override void myOnLastSubscriber()
      {
-         myVs.onChange -= new BCG.DataStream.ChangeDelegate (myHandleChange);;
+         myVs.onChange -= myHandleChange;
      }

      internal void myHandleChange (object newValue)
@@ -0,0 +1,230 @@
+using System;
+using System.Data;
+using System.Data.SQLite;
+using System.Diagnostics;
+using System.Collections;
+
+
+using BCG;
+using System.Linq;
+using System.Data.Entity;
+using System.Collections.Generic;
+
+namespace bts.SilverPine {
+
+   public class DataRowWrapperRow : IBlueRow {
+      private readonly DataRow _row;
+      private readonly IBlueTableSchema _schema;
+      private readonly int _id;
+      public DataRowWrapperRow(IBlueTableSchema schema, DataRow row, int id) {
+         _row = row;
+         _id = id;
+         _schema = schema;
+      }
+      public virtual IBlueTableSchema schema { get { return _schema; } }
+
+      public virtual object this[string col_name] {
+         get {
+            ColumnInfo ci = _schema[col_name];
+            return this[ci.colNo];
+         }
+      }
+      public virtual object this[int index] {
+         get {
+            if (_schema[index].colName == "id")
+               return _id;
+            return _row[index];
+         }
+      }
+
+      public virtual int Count { get { return _row.ItemArray.Length; } }
+
+      public virtual bool IsSynchronized { get { return false; } }
+
+      public virtual object SyncRoot { get { return this; } }
+
+      public virtual void CopyTo(Array array, int index) { _row.ItemArray.CopyTo(array, index); }
+
+      public virtual bool IsFixedSize { get { return true; } }
+
+      virtual public bool IsReadOnly { get { return true; } }
+
+      /// <summary>
+      /// The key can either be the string corrsponding to the column name or
+      /// the order of the column
+      /// </summary>
+      public virtual object this[object key] {
+         get {
+            if (key is string)
+               return this[(string)key];
+            else if (key is int)
+               return this[(int)key];
+            throw new BlueDataException("Wrong type while indexing the row");
+         }
+         set { throw new NotSupportedException("The IBlueRow is readonly"); }
+      }
+
+      public virtual ICollection Keys { get { return _schema.columnNames; } }
+
+      public virtual ICollection Values { get { return _row.ItemArray; } }
+
+      public virtual void Add(object key, object value) { throw new NotSupportedException("The IBlueRow is readonly"); }
+
+      public virtual void Clear() { throw new NotSupportedException("The IBlueRow is readonly"); }
+
+      public virtual bool Contains(object key) { return _schema[(string)key] != null; }
+
+      public IDictionaryEnumerator GetEnumerator() { return new BlueRowDictionaryEnumerator(this); }
+
+      IEnumerator IEnumerable.GetEnumerator() { return new BlueRowEnumerator(this); }
+
+      public virtual void Remove(object key) { throw new NotSupportedException("The IBlueRow is readonly"); }
+
+   }
+
+   public class DataTableWrapperTable : IBlueRowCollection {
+
+      public class DataTableWrapperTableEnumerator : IEnumerator {
+         private IEnumerator _enumerator;
+         private IBlueTableSchema _schema;
+         private int _id = 0;
+         public DataTableWrapperTableEnumerator(DataTable table) {
+            _enumerator = table.Rows.GetEnumerator();
+            _schema = BlueDataUtility.getSchema(table);
+
+         }
+         public object Current {
+            get {
+               DataRow r = _enumerator.Current as DataRow;
+               return new DataRowWrapperRow(_schema, r, _id);
+            }
+         }
+         public bool MoveNext() {
+            _id++;
+            return _enumerator.MoveNext();
+         }
+         public void Reset() {
+            _id = 0;
+            _enumerator.Reset();
+         }
+      }
+
+      private readonly DataTable _table;
+      private IBlueTableSchema _schema;
+      private IEnumerator _enumerator;
+
+      public DataTableWrapperTable(DataTable table) {
+         _table = table;
+         _schema = BlueDataUtility.getSchema(_table);
+         _enumerator = new DataTableWrapperTableEnumerator(table);
+      }
+      public virtual IBlueRow this[int id] {
+         get {
+            // its assumed the rows index == id
+            DataRow dr = _table.Rows[id];
+            return new DataRowWrapperRow(_schema, dr, id);
+         }
+         set {
+
+         }
+      }
+
+      public virtual IBlueTableSchema schema { get { return _schema; } }
+
+      public virtual bool containsID(int id) { return id < _table.Rows.Count; }
+
+      // private List<IBlueRow> _rows() {
+      //    List<IBlueRow> rv = new List<IBlueRow>();
+      //    int id = 0;
+      //    foreach (DataRow r in _table.Rows) {
+      //       rv.Add(new DataRowWrapperRow(r, id++));
+      //    }
+      //    return rv;
+      // }
+
+      public virtual IEnumerator GetEnumerator() {
+         // var rows = _rows();
+         // return rows.GetEnumerator();
+         return _enumerator;
+      }
+
+      public virtual bool IsSynchronized { get { return false; } }
+
+      public virtual object SyncRoot { get { return this; } }
+
+      public virtual void CopyTo(Array array, int index) {
+         Log.assert(false);
+         //_rows().CopyTo(array as IBlueRow[], index); 
+      }
+
+      public virtual int Count {
+         get { return _table.Rows.Count; }
+      }
+   }
+
+   public static class SQLiteHelper {
+      public static DataTable ExecuteSelectQuery(string connection_string, string query) {
+         // Create a new DataTable to store the results
+         DataTable dataTable = new DataTable();
+
+         // Create and open a SQLite connection using the provided connection string
+         using (SQLiteConnection connection = new SQLiteConnection(connection_string)) {
+            try {
+               connection.Open();
+
+               // Create a SQLite command with the provided query
+               using (SQLiteCommand command = new SQLiteCommand(query, connection)) {
+                  // Use SQLiteDataAdapter to fill the DataTable
+                  using (SQLiteDataAdapter adapter = new SQLiteDataAdapter(command)) {
+                     adapter.Fill(dataTable);
+                  }
+               }
+            } catch (Exception ex) {
+               Console.WriteLine("An error occurred: " + ex.Message);
+            }
+         }
+
+         // Return the populated DataTable
+         return dataTable;
+      }
+   }
+
+   [DataStreamUserRespresentation("SQLite")]
+   public class SQLiteStream : ViewStream {
+      private readonly string _connection_string, _query;
+      private Stopwatch _stopwatch = new Stopwatch();
+      public SQLiteStream(string connection_string, string query) {
+         _connection_string = connection_string;
+         _query = query;
+      }
+
+
+      private void _read_from_sqlite_nd_fire() {
+         _stopwatch.Start();
+         DataTable dt = SQLiteHelper.ExecuteSelectQuery(_connection_string, _query);
+         _stopwatch.Stop();
+         Log.Gen.info($"SQLiteStream._read_from_sqlite_and_fire time to read db: {_stopwatch.ElapsedMilliseconds} msecs");
+         _stopwatch.Start();
+         IBlueRowCollection rows = BCG.BlueDataUtility.toIBlueRowCollection(dt);
+         //IBlueRowCollection rows = new DataTableWrapperTable(dt);
+         _stopwatch.Stop();
+         Log.Gen.info($"SQLiteStream._read_from_sqlite_and_fire time to load into IBlueRowCollection: {_stopwatch.ElapsedMilliseconds} msecs");
+         DataPacket dp = BlueDataUtility.build_dp_inc(rows, myLastValue as DataPacket);
+         myFireEvent(dp);
+      }
+
+      protected override void myOnFirstSubscriber() {
+         _read_from_sqlite_nd_fire();
+      }
+
+      protected override void myOnLastSubscriber() {
+         // this value will be cleared by the base class
+         // but in this particular stream this behaviour
+         // is not usefull since any new subscribers 
+         // would force the query to be recreated
+         //myLastDP = myLastValue;
+      }
+
+   }
+}
+
@@ -3,12 +3,15 @@ using System.Collections.Generic;
 using System.Text;
 using System.IO;
 using BCG.Utilities;
+using System.Diagnostics;
+using Microsoft.Data.Analysis;
+using Parquet;
+using Parquet.Data;

-namespace BCG
-{
+
+namespace BCG {
   [DataStreamUserRespresentation("CSVSaver")]
-   public class CSVSaverStream : ViewStream, ISideEffectStream
-   {
+   public class CSVSaverStream : ViewStream, ISideEffectStream {
      private IBlueTableSchema myResultSchema = new BlueTableSchema("CSVResult", new ColumnInfo("id", 0, typeof(int)),
         new ColumnInfo("fileName", 1, typeof(string)),
         new ColumnInfo("nWrites", 2, typeof(int)),
@@ -20,11 +23,9 @@ namespace BCG
      private int myNWrites = 0, myNMaxWrites = Int32.MaxValue;

      public CSVSaverStream(ViewStream vs, string fullyQualifiedFileName)
-         : this(vs, fullyQualifiedFileName, true, Int32.MaxValue, false)
-      { }
+         : this(vs, fullyQualifiedFileName, true, Int32.MaxValue, false) { }

-      public CSVSaverStream(ViewStream vs, string fullyQualifiedFileName, bool overwrite, int nMaxWrites, bool fireResultSnapshot)
-      {
+      public CSVSaverStream(ViewStream vs, string fullyQualifiedFileName, bool overwrite, int nMaxWrites, bool fireResultSnapshot) {
         myStream = vs;
         myFullyQualifiedFileName = fullyQualifiedFileName;
         myOverwrite = overwrite;
@@ -32,22 +33,20 @@ namespace BCG
         myFireResultSnapshot = fireResultSnapshot;
      }

-      protected override void myOnFirstSubscriber() { myStream.onChange += new ChangeDelegate(myHandleChanges); }
+      protected override void myOnFirstSubscriber() =>
+         myStream.onChange += myHandleChanges;

-      protected override void myOnLastSubscriber() { myStream.onChange -= new ChangeDelegate(myHandleChanges); }
+      protected override void myOnLastSubscriber() =>
+         myStream.onChange -= myHandleChanges;

-      private void myHandleChanges(object newValue)
-      {
+      private void myHandleChanges(object newValue) {
         IBlueRowCollection totalState = (newValue as DataPacket).newRows;
-         if (totalState.Count > 0 && myNWrites < myNMaxWrites)
-         {
+         if (totalState.Count > 0 && myNWrites < myNMaxWrites) {
            myNWrites++;
            int i = 0;
            StreamWriter writer = new StreamWriter(new FileStream(myGetNewFileName(), FileMode.Create, FileAccess.Write));
-            foreach (IBlueRow r in totalState)
-            {
-               if (i++ == 0)
-               {
+            foreach (IBlueRow r in totalState) {
+               if (i++ == 0) {
                  string header = String.Empty;
                  foreach (string col in r.schema.columnNames)
                     if (col != "id")
@@ -59,20 +58,17 @@ namespace BCG
               writer.Flush();
            }
            writer.Close();
-            if (myFireResultSnapshot)
-            {
+            if (myFireResultSnapshot) {
               IBlueRow nr = new BlueRow(myResultSchema, 0, myFullyQualifiedFileName, myNWrites, totalState.Count);
               BlueTable nRows = new BlueTable(myResultSchema);
               nRows.addRow(nr);
               myFireEvent(new DataPacket(nRows, nRows, new BlueTable(myLastValue == null ? new BlueTable(myResultSchema) : (myLastValue as DataPacket).newRows)));
-            }
-            else
+            } else
               myFireEvent(newValue);
         }
      }

-      private string myGetNewFileName()
-      {
+      private string myGetNewFileName() {
         string[] tags = myFullyQualifiedFileName.Split('.');
         if (myOverwrite)
            return tags[0] + ".csv";
@@ -80,14 +76,11 @@ namespace BCG
            return tags[0] + "_" + myFormateDateTime(DateTime.Now) + ".csv";
      }

-      private static string myGetCSVRow(IBlueRow r)
-      {
+      private static string myGetCSVRow(IBlueRow r) {
         string rv = String.Empty;
         int i = 0;
-         foreach (string col in r.schema.columnNames)
-         {
-            if (col != "id")
-            {
+         foreach (string col in r.schema.columnNames) {
+            if (col != "id") {
               string colText = String.Empty;
               if (r[col] != null && r[col] != DBNull.Value)
                  colText = r[col].ToString();
@@ -99,8 +92,7 @@ namespace BCG
         return rv;
      }

-      private string myFormateDateTime(DateTime dt)
-      {
+      private string myFormateDateTime(DateTime dt) {
         return
         string.Format("{0}-{1}-{2}-{3}-{4}-{5}-{6}",
                        dt.Year.ToString("0000"),
@@ -114,52 +106,49 @@ namespace BCG
   }

   [DataStreamUserRespresentation("CSVReader")]
-   public class CSVReaderStream : ViewStream
-   {
-      private IBlueRowCollection myRows;
-
-      public CSVReaderStream(string fileAddress)
-      {
-         myRows = BlueDataUtility.createRowsFromCSVFileWithoutTypes(fileAddress);
+   public class CSVReaderStream : ViewStream {
+      private readonly string _file_address;
+      Stopwatch _stopwatch = new Stopwatch();
+      public CSVReaderStream(string fileAddress) {
+         _file_address = fileAddress;
      }

-      protected override void myOnFirstSubscriber()
-      {
-         myFireEvent(new DataPacket(myRows, myRows, new BlueTable(myRows.schema)));
+      protected override void myOnFirstSubscriber() {
+         _stopwatch.Start();
+         DataFrame df = DataFrame.LoadCsv(_file_address);
+         _stopwatch.Stop();
+         bts.Log.Gen.info($"CSVReaderStream.myOnFirstSubscriber stream csv load col_count: {df.Columns.Count} rows, time: {_stopwatch.ElapsedMilliseconds} msecs");
+         _stopwatch.Start();
+         IBlueRowCollection rows = BlueDataUtility.createRowsFromCSVFileWithoutTypes(_file_address);
+         _stopwatch.Stop();
+         bts.Log.Gen.info($"CSVReaderStream.myOnFirstSubscriber stream csv load count: {rows.Count} rows, time: {_stopwatch.ElapsedMilliseconds} msecs");
+         DataPacket dp_out = BlueDataUtility.build_dp_inc(rows, myLastValue as DataPacket);
+         myFireEvent(dp_out);
      }
   }

-   [DataStreamUserRespresentation ("CSVReaderStreamMapper")]
-   public class CSVReaderStreamMapperStream : ViewStream
-   {
-      public CSVReaderStreamMapperStream(ViewStream stream, string output_schema)
-      {
+   [DataStreamUserRespresentation("CSVReaderStreamMapper")]
+   public class CSVReaderStreamMapperStream : ViewStream {
+      public CSVReaderStreamMapperStream(ViewStream stream, string output_schema) {
         mySetRealSource(new MapStream(stream, new CSVReaderStreamMapper(output_schema)));
      }
   }

-   public class CSVReaderStreamMapper : IStreamMapper
-   {
+   public class CSVReaderStreamMapper : IStreamMapper {
      private IBlueTableSchema mySchema;
      private readonly string[] myOutputSchemaStringArray;

-      public CSVReaderStreamMapper(string outputSchema)
-      {
+      public CSVReaderStreamMapper(string outputSchema) {
         myOutputSchemaStringArray = outputSchema.Split(' ');
      }

-      public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema)
-      {
-         if (mySchema == null)
-         {
+      public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema) {
+         if (mySchema == null) {
            ColumnInfo[] ci = inputSchema.columnInfo;
-            foreach (string col in myOutputSchemaStringArray)
-            {
+            foreach (string col in myOutputSchemaStringArray) {
               string colName = col.Substring(0, col.IndexOf('\\')).ToLower();
-               for (int i = 0; i < ci.Length; i++)
-               {
-                  if (ci[i].colName.ToLower().Equals(colName))
-                  {
+               for (int i = 0; i < ci.Length; i++) {
+                  if (ci[i].colName.ToLower().Equals(colName)) {
                     Type t = myGetType(col.Substring(col.IndexOf('\\') + 1));
                     ci[i] = new ColumnInfo(ci[i].colName, i, t);
                     break;
@@ -171,74 +160,52 @@ namespace BCG
         return mySchema;
      }

-      public IBlueRow map(IBlueRow row)
-      {
+      public IBlueRow map(IBlueRow row) {
         object[] vals = new object[row.schema.columnInfo.Length];
         row.Values.CopyTo(vals, 0);
-         for (int i = 0; i < mySchema.columnInfo.Length; i++)
-         {
+         for (int i = 0; i < mySchema.columnInfo.Length; i++) {
            Type t = mySchema.columnInfo[i].colType;
-            try
-            {
+            try {
               vals[i] = System.Convert.ChangeType(vals[i], mySchema.columnInfo[i].colType, System.Globalization.CultureInfo.InvariantCulture);
-            }
-            catch
-            {
-               if (t.Equals(typeof(Int16)))
-               {
+            } catch {
+               if (t.Equals(typeof(Int16))) {
                  Int16 result;
-                  if (Int16.TryParse((string)vals[i], out result))
-                  {
+                  if (Int16.TryParse((string)vals[i], out result)) {
                     vals[i] = result;
                     continue;
                  }
                  double r2;
-                  if (Double.TryParse((string)vals[i], out r2))
-                  {
-                     try
-                     {
+                  if (Double.TryParse((string)vals[i], out r2)) {
+                     try {
                        vals[i] = (Int16)r2;
                        continue;
-                     }
-                     catch { }
+                     } catch { }
                  }
-               }
-               else if (t.Equals(typeof(Int32)))
-               {
+               } else if (t.Equals(typeof(Int32))) {
                  Int32 result;
-                  if (Int32.TryParse((string)vals[i], out result))
-                  {
+                  if (Int32.TryParse((string)vals[i], out result)) {
                     vals[i] = result;
                     continue;
                  }
                  double r2;
-                  if (Double.TryParse((string)vals[i], out r2))
-                  {
-                     try
-                     {
+                  if (Double.TryParse((string)vals[i], out r2)) {
+                     try {
                        vals[i] = (Int32)r2;
                        continue;
-                     }
-                     catch { }
+                     } catch { }
                  }
-               }
-               else if (t.Equals(typeof(Int64)))
-               {
+               } else if (t.Equals(typeof(Int64))) {
                  Int64 result;
-                  if (Int64.TryParse((string)vals[i], out result))
-                  {
+                  if (Int64.TryParse((string)vals[i], out result)) {
                     vals[i] = result;
                     continue;
                  }
                  double r2;
-                  if (Double.TryParse((string)vals[i], out r2))
-                  {
-                     try
-                     {
+                  if (Double.TryParse((string)vals[i], out r2)) {
+                     try {
                        vals[i] = (Int64)r2;
                        continue;
-                     }
-                     catch { }
+                     } catch { }
                  }
               }
               vals[i] = DBNull.Value;
@@ -247,8 +214,7 @@ namespace BCG
         return new BlueRow(mySchema, vals);
      }

-      private Type myGetType(string type)
-      {
+      private Type myGetType(string type) {
         type = type.ToLower();
         if (type.Contains("int"))
            return typeof(int);
@@ -1,31 +1,41 @@
 using System;
 using System.Collections;
 using System.Reflection;
+
 using BCG.Utilities;

-namespace BCG
-{
-   [DataStreamUserRespresentation (false)]
-   public class BlueRowCollectionStaticStream : ViewStream
-   {
-      public BlueRowCollectionStaticStream () {}
-
-      public BlueRowCollectionStaticStream (IBlueRowCollection rows) {loadCollection (rows);}
-
-      public void loadCollection (IBlueRowCollection rows)
-      {
-         myFireEvent (new DataPacket (rows, rows, myLastValue != null ?
-                                                   new BlueTable((myLastValue as DataPacket).newRows) : new BlueTable (rows.schema)));
+namespace BCG {
+   // if a table needs to be fired, it can be wrapped around this
+   public class BlueTableStream : ViewStream {
+      private IBlueRowCollection _rows;
+      public BlueTableStream(IBlueRowCollection rows) {
+         _rows = rows;
      }

-      public void crush_with_single_row(IBlueRow r)
-      {
+      protected override void myOnFirstSubscriber() {
+         var dp_out = BlueDataUtility.build_dp_inc(_rows, myLastValue as DataPacket);
+         myFireEvent(dp_out);
+      }
+   }
+
+   [DataStreamUserRespresentation(false)]
+   public class BlueRowCollectionStaticStream : ViewStream {
+      public BlueRowCollectionStaticStream() { }
+
+      public BlueRowCollectionStaticStream(IBlueRowCollection rows) { loadCollection(rows); }
+
+      public void loadCollection(IBlueRowCollection rows) {
+         myFireEvent(new DataPacket(rows, rows, myLastValue != null ?
+                                                   new BlueTable((myLastValue as DataPacket).newRows) : new BlueTable(rows.schema)));
+      }
+
+      public void crush_with_single_row(IBlueRow r) {
         BlueTable t = new BlueTable(r.schema);
         t.addRow(r);
         loadCollection(t);
      }

-      public void loadPacket (DataPacket dp) {myFireEvent (dp);}
+      public void loadPacket(DataPacket dp) { myFireEvent(dp); }

      /// <summary>
      /// Will work with streams that take ViewStream as first argument ex project, filter, distinct etc
@@ -35,17 +45,16 @@ namespace BCG
      /// <param name="streamType"></param>
      /// <param name="args"></param>
      /// <returns></returns>
-      public static IBlueRowCollection statelessOp (IBlueRowCollection rows, Type streamType, params object[] args)
-      {
-         ArrayList totArgs = new ArrayList (args);
-         BlueRowCollectionStaticStream v1 = new BlueRowCollectionStaticStream ();
-         totArgs.Insert (0, v1);
-         ViewStream retVal = Activator.CreateInstance (streamType, totArgs.ToArray()) as ViewStream;
-         v1.loadCollection (rows);
+      public static IBlueRowCollection statelessOp(IBlueRowCollection rows, Type streamType, params object[] args) {
+         ArrayList totArgs = new ArrayList(args);
+         BlueRowCollectionStaticStream v1 = new BlueRowCollectionStaticStream();
+         totArgs.Insert(0, v1);
+         ViewStream retVal = Activator.CreateInstance(streamType, totArgs.ToArray()) as ViewStream;
+         v1.loadCollection(rows);
         return retVal.rows;
      }

   }

-   
+
 }
@@ -35,14 +35,14 @@ namespace BCG
      #region ViewStream
      protected override void myOnFirstSubscriber()
      {
-         myStream.onChange +=new ChangeDelegate(myStreamChange);
+         myStream.onChange += myStreamChange;
         myVsd.addWorker(this);
      }

      protected override void  myOnLastSubscriber()
      {
         myVsd.removeWorker(this);
-         myStream.onChange -= new ChangeDelegate(myStreamChange);
+         myStream.onChange -= myStreamChange;
      }
      #endregion

@@ -28,7 +28,7 @@ namespace BCG

      protected override void myOnFirstSubscriber()
      {
-         myParentStream.onChange += new ChangeDelegate(myParentStreamChangeHandler);
+         myParentStream.onChange += myParentStreamChangeHandler;
         myVSD.addWorker(this);
      }

@@ -36,7 +36,7 @@ namespace BCG

      protected override void myOnLastSubscriber()
      {
-         myParentStream.onChange -= new ChangeDelegate(myParentStreamChangeHandler);
+         myParentStream.onChange -= myParentStreamChangeHandler;
         myVSD.removeWorker(this);
      }

@@ -53,11 +53,11 @@ namespace BCG
            if (r.is_subscription)
            {
               _current_source_subscription = r.subscription_id;
-               _source_stream.onChange += new ChangeDelegate(_source_stream_onChange);
+               _source_stream.onChange += _source_stream_onChange;
            }
            else
            {
-               _source_stream.onChange -= new ChangeDelegate(_source_stream_onChange);
+               _source_stream.onChange -= _source_stream_onChange;
            }
         }
      }
@@ -0,0 +1,34 @@
+namespace bts.SilverPine {
+   using bts;
+
+   using BCG;
+
+   using System;
+
+   public class DateTimeExpanstionStream : ViewStream {
+      public DateTimeExpanstionStream(ViewStream parent_stream, string inp_col) {
+         Func<IBlueTableSchema, IBlueTableSchema> get_schema = (s) => {
+            IBlueTableSchema rv = s.add_column(System.TypeCode.DateTime, $"{inp_col}_date");
+            rv = rv.add_column(System.TypeCode.String, $"{inp_col}_day");
+            rv = rv.add_column(System.TypeCode.String, $"{inp_col}_month");
+            rv = rv.add_column(System.TypeCode.String, $"{inp_col}_hour");
+            rv = rv.add_column(System.TypeCode.String, $"{inp_col}_min");
+            return rv;
+         };
+         ViewStream rs = new MapStream(parent_stream,
+         s => get_schema(s),
+         r => {
+            DateTime dt = r.get_datetime(inp_col);
+            IBlueTableSchema s = get_schema(r.schema);
+            MutableBlueRow rv = new MutableBlueRow(s, r);
+            rv.setValue($"{inp_col}_date", dt.Date);
+            rv.setValue($"{inp_col}_day", dt.DayOfWeek.ToString());
+            rv.setValue($"{inp_col}_month", dt.Date.Month.ToString());
+            rv.setValue($"{inp_col}_hour", dt.Hour);
+            rv.setValue($"{inp_col}_min", dt.Minute);
+            return rv;
+         });
+         mySetRealSource(rs);
+      }
+   }
+}
@@ -24,7 +24,7 @@ namespace BCG
        protected override void myOnFirstSubscriber()
        {
            myKeyToTimeToValue = new Dictionary<string, SortedList<DateTime, double>>();
-            myToAverage.onChange += new ChangeDelegate(myToAverage_onChange);
+            myToAverage.onChange += myToAverage_onChange;
        }


@@ -0,0 +1,111 @@
+namespace bts.SilverPine {
+   using BCG;
+
+   using bts;
+   using bts.collections;
+
+   using System;
+   using System.CodeDom;
+   using System.Collections.Generic;
+   using System.Linq;
+
+   // Given 2 filters, and 2 columns, finds the different in the value of the 2 columns from the 2 rows
+   // so:
+   // f1(base_stream.rows)[col1] - f1(base_stream.rows)[col2]
+   [DataStreamUserRespresentation("ColDiff")]
+   public class ColumnDifferenceStream : ViewStream {
+      private readonly ViewStream _base_stream;
+      private readonly FilterStream.OnFilter _f1, _f2;
+      private readonly string _c1, _c2;
+
+      public ColumnDifferenceStream(ViewStream base_stream, FilterStream.OnFilter f1, FilterStream.OnFilter f2, string col1, string col2) {
+         _base_stream = base_stream;
+         _f1 = f1;
+         _f2 = f2;
+         _c1 = col1;
+         _c2 = col2;
+      }
+
+      public ColumnDifferenceStream(ViewStream base_stream, string f1, string f2, string col1, string col2) {
+         FilterStream.OnFilter f1_del = r => {
+            BooleanExpressionFilter rv = new BooleanExpressionFilter(f1);
+            return rv.filter(r);
+         };
+         FilterStream.OnFilter f2_del = r => {
+            BooleanExpressionFilter rv = new BooleanExpressionFilter(f2);
+            return rv.filter(r);
+         };
+         ViewStream rs = new ColumnDifferenceStream(base_stream, f1_del, f2_del, col1, col2);
+         mySetRealSource(rs);
+      }
+
+      // for each row, does a row[new_ol_name] = row[col1] - rows[row_ffset][col2]
+      // we should generlize this for any math operation
+      public ColumnDifferenceStream(ViewStream parent_stream, string sort_col_name, int row_offset, string col1, string col2, string diff_col_name, double optional_perc_diff_filter = 0.0) {
+         // use GlobalMapStream
+         string diff_col_perc_name = $"{diff_col_name}_percentage";
+         System.Func<System.Collections.Generic.List<BCG.IBlueRow>, uint, BCG.IBlueRow> g_map_func = (rows, idx) => {
+            int minuend_idx = Convert.ToInt32(idx);
+            var inp_schema = rows[0].schema;
+            var out_schema = inp_schema.add_column(TypeCode.Double, diff_col_name);
+            out_schema = out_schema.add_column(TypeCode.Double, diff_col_perc_name);
+            // minuend - subtrahend
+            IBlueRow minuend_row = rows.ElementAt(minuend_idx);
+            MutableBlueRow rv = new MutableBlueRow(out_schema, minuend_row);
+            int subtrahend_index = minuend_idx + row_offset;
+            double diff_val = Double.NaN, diff_val_perc = Double.NaN;
+            if (subtrahend_index >= 0) {
+               try {
+                  IBlueRow subtrahend_row = rows.ElementAt(subtrahend_index);
+                  diff_val = minuend_row.get<double>(col1) - subtrahend_row.get<double>(col2);
+                  diff_val_perc = 100.0 * diff_val / subtrahend_row.get<double>(col1);
+               } catch (Exception ex) {
+                  Log.assert(false, ex.Message);
+               }
+            }
+            rv.setValue(diff_col_name, diff_val);
+            rv.setValue(diff_col_perc_name, diff_val_perc);
+            return rv;
+         };
+         ViewStream rs = new GlobalMapStream(parent_stream, sort_col_name, g_map_func);
+         if (optional_perc_diff_filter != 0) {
+            rs = new FilterStream(rs, r => {
+               double diff_val_perc = r.get<double>(diff_col_perc_name);
+               double abs_diff_val_perc = Math.Abs(diff_val_perc);
+               return abs_diff_val_perc >= optional_perc_diff_filter;
+            });
+         }
+         mySetRealSource(rs);
+      }
+
+      protected override void myOnFirstSubscriber() {
+         _base_stream.onChange += _handle_base_stream_update;
+      }
+
+      protected override void myOnLastSubscriber() {
+         _base_stream.onChange -= _handle_base_stream_update;
+      }
+
+      private void _handle_base_stream_update(object dp_obj) {
+         // could have been done with filter and cartesian prod, and then math
+         DataPacket dp = dp_obj as DataPacket;
+         List<IBlueRow> rows = dp.newRows.to_list();
+         var f1_rows = rows.Where(r => _f1(r));
+         Log.assert(f1_rows.Count() == 1);
+         var f2_rows = rows.Where(r => _f2(r));
+         Log.assert(f2_rows.Count() == 1);
+         IBlueRow f1_r = f1_rows.ElementAt(0);
+         IBlueRow f2_r = f2_rows.ElementAt(0);
+         double diff = f1_r.get<double>(_c1) - f2_r.get<double>(_c2);
+         IBlueTableSchema out_schema = f1_r.schema.add_columns(Tuple.Create(typeof(double), "diff"));
+         BlueTable out_table = new BlueTable(out_schema);
+         MutableBlueRow out_row = new MutableBlueRow(out_schema, f1_r);
+         out_row.setValue("diff", diff);
+         out_table.addRow(out_row);
+         DataPacket dp_out = BlueDataUtility.build_dp(out_table, myLastValue as DataPacket);
+         myFireEvent(dp_out);
+
+      }
+   }
+
+}
@@ -0,0 +1,51 @@
+namespace bts.SilverPine {
+   using BCG;
+
+   using bts;
+
+   using System.Collections.Generic;
+   using System.Linq;
+
+   // Given a sort order, an optional group by col, it observes the values of a column 
+   // within that group (or the whole table, if grouping not specified) and returns all rows
+   // where the value changes, ie row[current][col] != row[current-1][col]
+   // TODO: a more general way to express a filter where row[current][col] != row[previous][col],
+   // or even expand it to row[current][col_a] - row[previous][col_b] < 0.5
+   [DataStreamUserRespresentation("ColEvol")]
+   public class ColumnEvolutionStream : ViewStream {
+      private readonly ViewStream _base_stream;
+      private readonly string _sort_col;
+      private readonly string _evol_col;
+      public ColumnEvolutionStream(ViewStream base_stream, string sort_col, string evolution_col) {
+         _base_stream = base_stream;
+         _sort_col = sort_col;
+         _evol_col = evolution_col;
+      }
+
+      protected override void myOnFirstSubscriber() {
+         _base_stream.onChange += _handle_base_stream_update;
+      }
+
+      protected override void myOnLastSubscriber() {
+         base.myOnLastSubscriber();
+      }
+
+      private void _handle_base_stream_update(object dp_obj) {
+         DataPacket dp = dp_obj as DataPacket;
+         List<IBlueRow> rows = dp.newRows.to_list();
+         rows.Sort((r1, r2) => BlueDataUtility.compare_rows(r2, r1, _sort_col));
+         BlueTable rv = new BlueTable(dp.schema);
+         IBlueRow prev_row = rows.ElementAt(0);
+         foreach (IBlueRow r in rows) {
+            var cr = BlueDataUtility.compare_rows(r, prev_row, _evol_col);
+            if (cr != 0) {
+               rv.addRow(r);
+            }
+            prev_row = r;
+         }
+         DataPacket dp_out = BlueDataUtility.build_dp(rv, myLastValue as DataPacket);
+         myFireEvent(dp_out);
+      }
+   }
+
+}
@@ -12,10 +12,10 @@ namespace BCG {
         mySrc = src;
      }
      protected override void myOnFirstSubscriber() {
-         mySrc.onChange += new ChangeDelegate(mySrc_onChange);
+         mySrc.onChange += mySrc_onChange;
      }
      protected override void myOnLastSubscriber() {
-         mySrc.onChange -= new ChangeDelegate(mySrc_onChange);
+         mySrc.onChange -= mySrc_onChange;
      }
      private void mySrc_onChange(object newValue) {
         DataPacket dp = (DataPacket)newValue;
@@ -28,14 +28,14 @@ namespace BCG

      protected override void myOnFirstSubscriber()
      {
-         myLeftStream.onChange += new ChangeDelegate(myHandleLeftChanges);
-         myRightstream.onChange += new ChangeDelegate(myHandleRightChanges);
+         myLeftStream.onChange += myHandleLeftChanges;
+         myRightstream.onChange += myHandleRightChanges;
      }

      protected override void myOnLastSubscriber()
      {
-         myLeftStream.onChange -= new ChangeDelegate(myHandleLeftChanges);
-         myRightstream.onChange -= new ChangeDelegate(myHandleRightChanges);
+         myLeftStream.onChange -= myHandleLeftChanges;
+         myRightstream.onChange -= myHandleRightChanges;
      }

      private void myHandleLeftChanges (object newValue)
@@ -222,7 +222,7 @@ namespace BCG
            pj = new ProjectStream(pj, "!");
         myStreamIndex.Add(myMovingIndex, new Pair<ViewStream, ViewStream>(vs, pj));
         if (mySubscriptionActive)
-            pj.onChange += new ChangeDelegate(myHandleStreamChanges);
+            pj.onChange += myHandleStreamChanges;
      }

      public void removeStream(ViewStream vs)
@@ -235,7 +235,7 @@ namespace BCG
               index = p.Key;
               if (mySubscriptionActive)
               {
-                  p.Value.second.onChange -= new ChangeDelegate(myHandleStreamChanges);
+                  p.Value.second.onChange -= myHandleStreamChanges;
                  if (myTable != null)
                  {
                     IBlueRowCollection delRows = myTable.removeTable(p.Key);
@@ -254,14 +254,14 @@ namespace BCG
      {
         mySubscriptionActive = true;
         foreach (KeyValuePair<int, Pair<ViewStream, ViewStream>> p in myStreamIndex)
-            p.Value.second.onChange += new ChangeDelegate(myHandleStreamChanges);
+            p.Value.second.onChange += myHandleStreamChanges;
      }

      protected override void myOnLastSubscriber()
      {
         mySubscriptionActive = false;
         foreach (KeyValuePair<int, Pair<ViewStream, ViewStream>> p in myStreamIndex)
-            p.Value.second.onChange -= new ChangeDelegate(myHandleStreamChanges);
+            p.Value.second.onChange -= myHandleStreamChanges;
         if (myTable != null)
            myTable.clear();
      }
@@ -14,11 +14,11 @@ namespace BCG {
      }
      protected override void myOnFirstSubscriber()
      {
-         mySrc.onChange += new ChangeDelegate(mySrc_onChange);
+         mySrc.onChange += mySrc_onChange;
      }
      protected override void myOnLastSubscriber () 
      {
-         mySrc.onChange -= new ChangeDelegate (mySrc_onChange);
+         mySrc.onChange -= mySrc_onChange;
      }
      private void mySrc_onChange (object newValue) {
         DataPacket dp = (DataPacket)newValue;
@@ -21,15 +21,15 @@ namespace BCG
      {
         myLeftOp = new UnionOperand (true, this);
         myRightOp = new UnionOperand (false, this);
-         myLeftVs.onChange += new DataStream.ChangeDelegate (myLeftOp.myChangeHandler);
-         myRightVs.onChange += new DataStream.ChangeDelegate (myRightOp.myChangeHandler);
+         myLeftVs.onChange += myLeftOp.myChangeHandler;
+         myRightVs.onChange += myRightOp.myChangeHandler;
      }

      protected override void myOnLastSubscriber () 
      {
         myInit ();
-         myLeftVs.onChange -= new DataStream.ChangeDelegate (myLeftOp.myChangeHandler);
-         myRightVs.onChange -= new DataStream.ChangeDelegate (myRightOp.myChangeHandler);
+         myLeftVs.onChange -= myLeftOp.myChangeHandler;
+         myRightVs.onChange -= myRightOp.myChangeHandler;
      }

      public void startSession ()
@@ -116,4 +116,4 @@ namespace BCG
          receiveStream(dp, isLeft);
      }
   }
-}
+}
@@ -14,11 +14,11 @@ namespace BCG {

      protected override void myOnFirstSubscriber() {
         _vs_publisher = new ViewPublisher(_name, _broker, _main_stream);
-         _main_stream.onChange += new DataStream.ChangeDelegate(_handle_update);
+         _main_stream.onChange += _handle_update;
      }

      protected override void myOnLastSubscriber() {
-         _main_stream.onChange -= new DataStream.ChangeDelegate(_handle_update);
+         _main_stream.onChange -= _handle_update;
         _vs_publisher.Dispose();
      }

@@ -28,4 +28,4 @@ namespace BCG {


   }
-}
+}
@@ -20,11 +20,11 @@ namespace BCG {
      public ViewPublisher(string name, IPubTableBroker broker, ViewStream vs) {
         _stream = String.IsNullOrEmpty(name) ? vs : new ChangeTableNameStream(vs, name);
         _broker = broker;
-         _stream.onChange += new DataStream.ChangeDelegate(_handle_update);
+         _stream.onChange += _handle_update;
      }

      public void Dispose() {
-         _stream.onChange -= new DataStream.ChangeDelegate(_handle_update);
+         _stream.onChange -= _handle_update;
      }

      private void _handle_update(object o) {
@@ -15,10 +15,74 @@ using mbts;
 using BCG.Extensions;
 using System.Text.Json;
 using System.Text.Json.Nodes;
+using Newtonsoft.Json;
+using BCG;
+using System.Runtime.CompilerServices;
+
+namespace bts {
+
+   // public class CompositeRow : IBlueRow {
+   //    private IBlueRow _lrow, _rrow;
+   //    private BlueRow _out_row;
+   //    public CompositeRow(IBlueRow lrow, IBlueRow rrow) {
+   //       _lrow = lrow;
+   //       _rrow = rrow;
+   //       IBlueTableSchema schema = _lrow.schema;
+   //       List<object> vals = new List<object>();
+   //       foreach (var o in _lrow) {
+   //          vals.Add(o);
+   //       }
+   //       foreach (ColumnInfo ci in _rrow.schema.columnInfo) {
+   //          if (ci.colName == "id") {
+   //             continue;
+   //          }
+   //          vals.Add(_rrow[ci.colNo]);
+   //          schema = schema.add_column(ci.type_code(), ci.colName);
+   //       }
+   //       _out_row = new BlueRow(schema, vals.ToArray());
+   //    }
+   //    public IBlueTableSchema schema {
+   //       get { return _out_row.schema; }
+   //    }
+   //    public object? this[int index] {
+   //       get { return _out_row[index]; }
+   //    }
+   //    public object? this[string colName] {
+   //       get {
+   //          return _out_row[colName];
+   //       }
+   //    }
+
+   // }
+}

 namespace BCG {
   public static class BlueDataUtility {
+      public static IBlueRow create_composite_row(IBlueRow lr, IBlueRow rr, int id) {
+         IBlueTableSchema schema = lr.schema;
+         List<object> vals = new List<object>();
+         foreach (ColumnInfo ci in lr.schema.columnInfo) {
+            if (ci.colName == "id") {
+               vals.Add(id);
+            }
+            vals.Add(lr[ci.colName]);
+         }
+         foreach (ColumnInfo ci in rr.schema.columnInfo) {
+            if (ci.colName == "id") {
+               continue;
+            }
+            schema = schema.add_column(ci.type_code(), ci.colName);
+            vals.Add(rr[ci.colName]);
+         }
+         IBlueRow r = new BlueRow(schema, vals.ToArray());
+         return r;
+      }
      #region Extensions
+      public static bool is_null(this IBlueRow r, string col_name) {
+         object col_val = r[col_name];
+         return col_val == null || DBNull.Value.Equals(col_val);
+      }
+
      public static DateTime get_datetime(this IBlueRow r, string col_name, DateTime err_value = default(DateTime)) {
         return BlueDataUtility.get_row_datetime_value(r, col_name, err_value);
      }
@@ -49,6 +113,17 @@ namespace BCG {
            rv.Add(r.get<T>(col));
         return rv;
      }
+      public static IEnumerable<T> get_colvalues<T>(IEnumerable<IBlueRow> rows, string col, int start_index, int count) {
+         List<T> rv = new List<T>();
+         int i = 0;
+         int n_copied = 0;
+         foreach (IBlueRow r in rows)
+            if (i++ >= start_index && n_copied < count) {
+               rv.Add(r.get<T>(col));
+               n_copied++;
+            }
+         return rv;
+      }

      public static IDictionary<T, IEnumerable<IBlueRow>> group_by<T>(IBlueRowCollection rows, string gp_col) {
         Dictionary<T, List<IBlueRow>> rv = new Dictionary<T, List<IBlueRow>>();
@@ -112,13 +187,17 @@ namespace BCG {
         }
      }

-      public static IEnumerable<IBlueRow> to_ienum(this IBlueRowCollection rows) {
+      public static List<IBlueRow> to_list(this IBlueRowCollection rows) {
         List<IBlueRow> rv = new List<IBlueRow>();
         foreach (IBlueRow r in rows)
            rv.Add(r);
         return rv;
      }

+      public static IEnumerable<IBlueRow> to_ienum(this IBlueRowCollection rows) {
+         return rows.to_list();
+      }
+
      public static IDictionary<int, IBlueRow> to_map(this IBlueRowCollection rows) {
         Dictionary<int, IBlueRow> rv = new Dictionary<int, IBlueRow>();
         foreach (IBlueRow r in rows) {
@@ -135,6 +214,11 @@ namespace BCG {
         return new BlueTableSchema(input_schema.tableName, cis.ToArray());
      }

+      public static IBlueTableSchema add_column(this IBlueTableSchema input_schema, TypeCode type_arg, string col_name) {
+         Tuple<Type, string> tup = new Tuple<Type, string>(_typecode_to_type[type_arg], col_name);
+         return add_columns(input_schema, tup);
+      }
+
      public static IBlueRowCollection to_collection(this IBlueRow r) {
         if (r == null)
            return null;
@@ -191,6 +275,18 @@ namespace BCG {

      #region json

+      public static Dictionary<string, string> convert_json_to_dictionary(string json_string) {
+         Dictionary<string, string> dictionary = JsonConvert.DeserializeObject<Dictionary<string, string>>(json_string);
+         return dictionary;
+      }
+      public static IBlueRow convert_json_to_row(string json_string, int id = 0) {
+         Dictionary<string, object> dictionary = JsonConvert.DeserializeObject<Dictionary<string, object>>(json_string);
+         if (!dictionary.ContainsKey("id")) {
+            dictionary.Add("id", id);
+         }
+         return new BlueRow(dictionary);
+      }
+
      public static List<JsonNode> json_to_json_node_list(string jsonString) {
         var jsonNodeList = new List<JsonNode>();
         bool noe = String.IsNullOrEmpty(jsonString);
@@ -209,7 +305,7 @@ namespace BCG {
            foreach (JsonNode jsonNode in jsonArray) {
               jsonNodeList.Add(jsonNode);
            }
-         } catch (JsonException ex) {
+         } catch (System.Text.Json.JsonException ex) {
            Console.WriteLine($"Invalid JSON: {ex.Message}");
         }

@@ -486,7 +582,16 @@ namespace BCG {
         ColumnInfo[] cis = new ColumnInfo[table.Columns.Count];
         foreach (DataColumn col in table.Columns)
            cis[col.Ordinal] = new ColumnInfo(col.ColumnName, col.Ordinal, col.DataType);
-         return new BlueTableSchema(cis, table.TableName);
+         IBlueTableSchema rv = new BlueTableSchema(cis, table.TableName);
+         rv = rv.add_id_col_if_missing();
+         return rv;
+      }
+      public static IBlueTableSchema add_id_col_if_missing(this IBlueTableSchema schema) {
+         IBlueTableSchema rv = schema;
+         if (!schema.columnNames.Contains("id")) {
+            rv = schema.add_columns(Tuple.Create(typeof(int), "id"));
+         }
+         return rv;
      }

      /// <summary>
@@ -744,10 +849,16 @@ namespace BCG {
      }

      public static IBlueRowCollection toIBlueRowCollection(DataTable dt) {
-         BlueTable retVal = new BlueTable(getSchema(dt));
-         foreach (DataRow dr in dt.Rows)
-            retVal.addRow(dr);
-         return retVal;
+         IBlueTableSchema schema = getSchema(dt);
+         BlueTable rv = new BlueTable(schema);
+         foreach (DataRow dr in dt.Rows) {
+            MutableBlueRow nr = new MutableBlueRow(schema);
+            foreach (DataColumn col in dr.Table.Columns)
+               nr.setValue(col.ColumnName, dr[col.ColumnName]);
+            nr.setValue("id", rv.Count);
+            rv.addRow(nr);
+         }
+         return rv;
      }

      public static IBlueRowCollection toIBlueRowCollection(DataRow[] rows) {
@@ -819,20 +930,20 @@ namespace BCG {
      /// </summary>
      /// <param name="r1"></param>
      /// <param name="r2"></param>
-      /// <param name="colNames"></param>
+      /// <param name="col_names"></param>
      /// <returns></returns>
      public static int compare_rows(IBlueRow r1,
         IBlueRow r2,
-         params string[] colNames) {
-         int retVal = 0;
-         foreach (string colName in colNames) {
-            object val1 = r1[colName];
-            object val2 = r2[colName];
-            retVal = _compare_row_cols(val1, val2);
-            if (retVal != 0)
+         params string[] col_names) {
+         int comp_res = 0;
+         foreach (string cn in col_names) {
+            object val1 = r1[cn];
+            object val2 = r2[cn];
+            comp_res = _compare_row_cols(val1, val2);
+            if (comp_res != 0)
               break;
         }
-         return retVal;
+         return comp_res;
      }

      private static int _compare_row_cols(object v1, object v2) {
@@ -96,3 +96,107 @@ public class BlueTableImmutable : IBlueRowCollection, IImmutableRowCollection {
         new(schema, table.ToImmutable());
   }
 }
+
+public class BlueTableImmutable<T> : IBlueRowCollection<T> where T : IBlueRow {
+   readonly ImmutableDictionary<int, T> rows;
+
+   public BlueTableImmutable(IBlueTableSchema schema)
+      : this(schema, ImmutableDictionary<int, T>.Empty) { }
+
+   public BlueTableImmutable(IBlueTableSchema schema, ImmutableDictionary<int, T> rows) {
+      this.schema = schema;
+      this.rows = rows;
+   }
+
+   public int Count => rows.Count;
+   public bool IsSynchronized => false;
+   public IBlueTableSchema schema { get; }
+   public object SyncRoot => this;
+
+   public T this[int id] => rows[id];
+
+   public bool containsID(int id) =>
+      rows.ContainsKey(id);
+
+   public void CopyTo(Array array, int index) {
+      foreach (var r in rows.Values) {
+         array.SetValue(r, index++);
+      }
+   }
+
+   public IEnumerator<T> GetEnumerator() =>
+      rows.Values.GetEnumerator();
+
+   IEnumerator IEnumerable.GetEnumerator() =>
+      GetEnumerator();
+
+   public DataPacketBuilder ToDataPacketBuilder() =>
+      new(this);
+
+   public class DataPacketBuilder {
+      readonly IBlueTableSchema schema;
+      readonly ImmutableDictionary<int, T>.Builder table;
+      readonly ImmutableDictionary<int, T>.Builder added =
+         ImmutableDictionary<int, T>.Empty.ToBuilder();
+      readonly ImmutableDictionary<int, T>.Builder removed =
+         ImmutableDictionary<int, T>.Empty.ToBuilder();
+
+      public DataPacketBuilder(BlueTableImmutable<T> table) {
+         schema = table.schema;
+         this.table = table.rows.ToBuilder();
+      }
+
+      public void AddOrUpdate(T row, IEqualityComparer<IBlueRow>? comparer = null) {
+         var id = row.Id();
+         if (table.TryGetValue(id, out var orig)) {
+            if (comparer?.Equals(row, orig) ?? false) {
+               // No op: This row was already in the table (according to 'comparer').
+               // Remove it from 'added' and 'removed'.
+               added.Remove(id);
+               removed.Remove(id);
+               return;
+            }
+            // Change op: A row with this id was already in the table,
+            // but its values have changed (according to 'comparer').
+            removed[id] = orig;
+         }
+         added[id] = row;
+      }
+
+      public void Remove(int id) {
+         added.Remove(id);
+         if (table.TryGetValue(id, out var orig)) {
+            removed[id] = orig;
+         }
+      }
+
+      public DataPacket ToDataPacket(bool reset = true) {
+         table.RemoveRange(removed.Keys);
+         table.AddRange(added);
+         var newRows = new BlueTableImmutable<T>(schema, table.ToImmutable());
+         var addedRows = new BlueTableImmutable<T>(schema, added.ToImmutable());
+         var removedRows = new BlueTableImmutable<T>(schema, removed.ToImmutable());
+         if (reset) {
+            added.Clear();
+            removed.Clear();
+         }
+         return new(newRows, addedRows, removedRows);
+      }
+
+      public IDataPacket<T> ToDataPacketT(bool reset = true) {
+         table.RemoveRange(removed.Keys);
+         table.AddRange(added);
+         var newRows = new BlueTableImmutable<T>(schema, table.ToImmutable());
+         var addedRows = new BlueTableImmutable<T>(schema, added.ToImmutable());
+         var removedRows = new BlueTableImmutable<T>(schema, removed.ToImmutable());
+         if (reset) {
+            added.Clear();
+            removed.Clear();
+         }
+         return new DataPacket<T>(newRows, addedRows, removedRows);
+      }
+
+      public BlueTableImmutable<T> ToImmutable() =>
+         new(schema, table.ToImmutable());
+   }
+}
@@ -24,7 +24,16 @@ namespace BCG
      /// </summary>
      object? this[int index] {get;}
      object? this[string colName] {get;}
+      bool IDictionary.IsFixedSize => true;
+      bool IDictionary.IsReadOnly => true;
+      object ICollection.SyncRoot => this;
+      bool ICollection.IsSynchronized => false;
   }
+
+   // public interface IBlueRow<T> : IBlueRow where T : IBlueRow {
+   //    T Value => (T)this;
+   // }
+
   namespace Extensions
   {
      public static class IBlueRowExtensions
@@ -1,5 +1,6 @@
 using System;
 using System.Collections;
+using System.Collections.Generic;

 namespace BCG
 {
@@ -16,4 +17,10 @@ namespace BCG
      IBlueTableSchema schema {get;}
      bool containsID (int id);
   }
+
+   public interface IBlueRowCollection<T> : IBlueRowCollection, IReadOnlyCollection<T> where T : IBlueRow {
+      new T this[int id] { get; }
+
+      IBlueRow IBlueRowCollection.this[int id] => this[id];
+   }
 }
@@ -314,6 +314,18 @@ namespace BCG
      }
   }

+   public class DataPacket<T> : IDataPacket<T> where T : IBlueRow {
+      public DataPacket(IBlueRowCollection<T> newRows, IBlueRowCollection<T> addedRows, IBlueRowCollection<T> deletedRows) {
+         this.newRows = newRows;
+         this.addedRows = addedRows;
+         this.deletedRows = deletedRows;
+      }
+
+      public IBlueRowCollection<T> newRows { get; }
+      public IBlueRowCollection<T> addedRows { get; }
+      public IBlueRowCollection<T> deletedRows { get; }
+   }
+
   public static class DataPacketExtensions {
      public static bool HasChanges(this DataPacket dataPacket) =>
         dataPacket.addedRows.Count > 0 || dataPacket.deletedRows.Count > 0;
@@ -53,16 +53,15 @@ namespace BCG

      public event Action<Exception> ErrorChanged;

-      public delegate void ChangeDelegate(object newValue);
-      public event ChangeDelegate onChange
+      public event DataStreamChangeDelegate onChange
      {
         add 
         {
-            ChangeDelegate currValue = value;
+            DataStreamChangeDelegate currValue = value;
            if (myTestMode)
            {
               ChangeDelegateWrapper w = new ChangeDelegateWrapper (value);
-               currValue = new ChangeDelegate (w.wrapperFunction);
+               currValue = new DataStreamChangeDelegate (w.wrapperFunction);
               myDelMap.Add (value, currValue);
            }
            if (myRealSource != null)
@@ -70,7 +69,7 @@ namespace BCG
            else 
            {
               myOnChangeDelegate = 
-                  (ChangeDelegate) Delegate.Combine (myOnChangeDelegate,currValue);
+                  (DataStreamChangeDelegate) Delegate.Combine (myOnChangeDelegate,currValue);

               
               if (myOnChangeDelegate.GetInvocationList().Length == 1) // first subscriber 
@@ -95,11 +94,11 @@ namespace BCG
         }
         remove 
         {
-            ChangeDelegate currValue = value;
+            var currValue = value;
            // BlueTrace.WriteViewStreamUnsubscribe(this.ID);
            if (myTestMode)
            {
-               currValue = myDelMap[value] as ChangeDelegate;
+               currValue = myDelMap[value] as DataStreamChangeDelegate;
               myDelMap.Remove (value);
            }
            if (myRealSource != null)
@@ -107,7 +106,7 @@ namespace BCG
            else 
            {
               myOnChangeDelegate = 
-                  (ChangeDelegate) Delegate.Remove (myOnChangeDelegate,currValue);
+                  (DataStreamChangeDelegate) Delegate.Remove (myOnChangeDelegate,currValue);

               // this is need to avoid a callback in a reentrant add.
               if (myPendingAdds.Count > 0)
@@ -175,7 +174,7 @@ namespace BCG

      // this was made internal for testing purposes only.
      // don't mess with it.
-      internal ChangeDelegate myOnChangeDelegate;
+      internal DataStreamChangeDelegate myOnChangeDelegate;
      protected object myLastValue;
      protected DataStream myRealSource;
      private Stack myPendingAdds;
@@ -184,9 +183,9 @@ namespace BCG
   internal class ChangeDelegateWrapper 
   {
      readonly Stopwatch timer = new();
-      private DataStream.ChangeDelegate myDel;
+      private DataStreamChangeDelegate myDel;
      
-      internal ChangeDelegateWrapper (DataStream.ChangeDelegate del) {myDel = del;}
+      internal ChangeDelegateWrapper (DataStreamChangeDelegate del) {myDel = del;}

      internal void wrapperFunction (object o)
      {
@@ -0,0 +1,4 @@
+namespace BCG;
+
+public delegate void DataStreamChangeDelegate(object newValue);
+public delegate void DataStreamChangeDelegate<T>(IDataPacket<T> dp) where T : IBlueRow;
@@ -9,6 +9,9 @@
  <ItemGroup>
    <PackageReference Include="Google.Protobuf" Version="3.24.3" />
    <PackageReference Include="Microsoft.Extensions.Hosting" Version="7.0.1" />
+    <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
+    <PackageReference Include="OpenAI" Version="2.0.0" />
+    <PackageReference Include="SimMetrics.Net" Version="1.0.5" />
  </ItemGroup>

  <ItemGroup>
@@ -17,8 +20,8 @@
  </ItemGroup>

  <ItemGroup>
-    <Proto Include="../../lib/config/protos/data_stream_config.proto" CSName="DataStreamConfig">
-      <OutDir>$(ProtoOutputPath)/lib/config/protos</OutDir>
+    <Proto Include="../../lib_spark/spark/config/protos/data_stream_config.proto" CSName="DataStreamConfig">
+      <OutDir>$(ProtoOutputPath)/spark/config/protos</OutDir>
    </Proto>
  </ItemGroup>

@@ -8,11 +8,12 @@ namespace BCG {
   using System.Collections;
   using System.Reflection;
   using System.IO;
+   using System.Collections.Generic;

   public class PrimitiveManager {
-      private IDictionary myTypeLookupTable = new Hashtable();
      private static PrimitiveManager mySelf;
-      private ArrayList myAssemblyNames = new ArrayList();
+      readonly List<string> myAssemblyNames = new();
+      readonly Dictionary<string, Type> myTypeLookupTable = new();

      private PrimitiveManager() {
         myLoadAllTypes();
@@ -95,15 +96,14 @@ namespace BCG {
      }

      /// <summary>
-      /// Returns the Type name corresponding to the primitive name
+      /// Returns the Type corresponding to the primitive name
      /// </summary>
      /// <param name="friendlyName"></param>
      /// <returns></returns>
-      public string find(string friendlyName) {
-         if (myTypeLookupTable[friendlyName] == null)
-            throw new ApplicationException(friendlyName + " is not the friendly name of any of the loaded types");
-         return (myTypeLookupTable[friendlyName] as Type).Name;
-      }
+      public Type find(string friendlyName) =>
+         myTypeLookupTable.TryGetValue(friendlyName, out var t)
+            ? t
+            : throw new ArgumentException($"'{friendlyName}' is not the friendly name of any of the loaded types", nameof(friendlyName));

      /// <summary>
      /// Constructs and returns a ViewStream of type corresponding to the friendly name
@@ -118,12 +118,8 @@ namespace BCG {
            constructorArgs = new object[args.Count];
            args.CopyTo(constructorArgs, 0);
         }
-         if (myTypeLookupTable[friendlyName] == null)
-            throw new ApplicationException(friendlyName + " is not the friendly name of any of the loaded types");
-         return
-            constructorArgs == null ?
-            Activator.CreateInstance(myTypeLookupTable[friendlyName] as Type) as ViewStream :
-            Activator.CreateInstance(myTypeLookupTable[friendlyName] as Type, constructorArgs) as ViewStream;
+         var type = find(friendlyName);
+         return Activator.CreateInstance(type, constructorArgs) as ViewStream;
      }

      /// <summary>
@@ -12,4 +12,16 @@ namespace BCG
      //this is an ICollection of IDictionary
      void get_new_added_deleted(out ICollection new_rows, out ICollection added_rows, out ICollection deleted_rows);
   }
+
+   public interface IDataPacket<T> : IDataPacket where T : IBlueRow {
+      IBlueRowCollection<T> newRows { get; }
+      IBlueRowCollection<T> addedRows { get; }
+      IBlueRowCollection<T> deletedRows { get; }
+
+      void IDataPacket.get_new_added_deleted(out ICollection new_rows, out ICollection added_rows, out ICollection deleted_rows) {
+         new_rows = newRows;
+         added_rows = addedRows;
+         deleted_rows = deletedRows;
+      }
+   }
 }