build: Clean up client CMake

webapp check pt?
cleanup: Remove empty header FTXSecurity.h
2024-09-03 12:39:02 -04:00 · 2024-09-01 20:07:31 +00:00 · 2024-09-01 20:07:31 +00:00 · 2024-09-01 20:07:31 +00:00 · 2024-09-01 20:07:31 +00:00 · 2024-09-01 20:07:31 +00:00
951 changed files with 30856 additions and 26468 deletions
@@ -87,10 +87,6 @@ RUN dpkg -i /emumba/emumba-deribit-connector-3.3.1-Linux.LICENSE_DISABLED.deb \
 # Pcap
 RUN apt-get update && apt-get install -y libpcap0.8-dev

-# Databento (deps)
-RUN apt-get update && apt-get -y install libzstd-dev
-
-# Docker in Docker
 RUN apt-get update && apt-get install -y \
    docker.io \
    docker-compose \
@@ -16,20 +16,11 @@
        "program": "spark_api_server.exe",
        "args": [
          "run",
+          "--sparkd",
+          "http://127.0.0.1:8000/config"
        ],
        "cwd": "build/Debug/bin/net7.0",
      },
-      "linux": {
-        "preLaunchTask": "Build Api Server (Debug)",
-        "program": "spark_api_server",
-        "args": [
-          "run",
-        ],
-        "cwd": "build/Debug/bin/net7.0",
-      },
-      "env": {
-        "ASPNETCORE_ENVIRONMENT": "Development",
-      },
      "stopAtEntry": false,
      "console": "integratedTerminal"
    },
@@ -42,19 +33,11 @@
        "program": "spark_api_server.exe",
        "args": [
          "run",
+          "--sparkd",
+          "http://127.0.0.1:8000/config"
        ],
        "cwd": "build/Debug/bin/net7.0",
      },
-      "linux": {
-        "program": "spark_api_server",
-        "args": [
-          "run",
-        ],
-        "cwd": "build/Debug/bin/net7.0",
-      },
-      "env": {
-        "ASPNETCORE_ENVIRONMENT": "Development",
-      },
      "stopAtEntry": false,
      "console": "integratedTerminal"
    },
@@ -122,18 +105,6 @@
      "stopAtEntry": false,
      "console": "integratedTerminal"
    },
-    {
-      "name": "(gdb) Launch",
-      "type": "cppdbg",
-      "request": "launch",
-      "program": "/home/bts/spark/build/sparkd-debug-clang/server/sparkd",
-      "args": ["run"],
-      "stopAtEntry": false,
-      "cwd": "/home/bts/spark/build/sparkd-debug-clang/server",
-      "environment": [],
-      "externalConsole": false,
-      "MIMode": "gdb",
-    },
    {
      "name": "DataStreamQueryAnalyzer",
      "type": "coreclr",
@@ -263,35 +234,7 @@
        "program": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd/sparkd",
      },
      "args": [
-        "run"
-      ],
-      "cwd": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd",
-      "stopAtEntry": false,
-      "externalConsole": false,
-      "MIMode": "gdb",
-      "setupCommands": [
-        {
-          "description": "Enable pretty-printing for gdb",
-          "text": "-enable-pretty-printing",
-          "ignoreFailures": true
-        },
-        {
-          "description": "Set Disassembly Flavor to Intel",
-          "text": "-gdb-set disassembly-flavor intel",
-          "ignoreFailures": true
-        }
-      ]
-    },
-    {
-      "name": "Sparkd Web (Skip Build)",
-      "type": "cppdbg",
-      "request": "launch",
-      "program": "",
-      "linux": {
-        "program": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd/sparkd",
-      },
-      "args": [
-        "web"
+        "test-py-md-provider"
      ],
      "cwd": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd",
      "stopAtEntry": false,
@@ -1,5 +1,5 @@
 cmake_minimum_required(VERSION 3.12)
-project(spark)
+project(spark_root)
 set(CMAKE_CXX_STANDARD 20)

 if(WIN32)
@@ -33,85 +33,91 @@ else()
  set(BUILD_ADMIN_WEB_APP ON CACHE BOOL "Build the Admin WebApp")
 endif()

+include_directories(BEFORE SYSTEM ${EXTERNALS_INCLUDE} ${EXTERNALS_ROOT}/include/zlib)
+link_directories(${EXTERNALS_LIB})
+
 add_compile_definitions(SHARED_MEMORY_PREFIX="${SHARED_MEMORY_PREFIX}")
 if(DEFINED APP_DATA_DIR)
  add_compile_definitions(APP_DATA_DIR="${APP_DATA_DIR}")
 endif()
 add_compile_definitions(EXPORT_MD_CONSUMER="${EXPORT_MD_CONSUMER}")
+if(WIN32)
+  # subdirs
+  add_subdirectory(src/lib_spark)
+else()
+  if(BUILD_SPARKD)
+    # Note: All macros that begin w/ 'CCAPI' are ccapi's macros (defined and/or
+    # read by their code). We override some below.
+    add_compile_definitions(SPARK_ENABLE_CCAPI)

-if(BUILD_SPARKD)
-  # Note: All macros that begin w/ 'CCAPI' are ccapi's macros (defined and/or
-  # read by their code). We override some below.
-  add_compile_definitions(SPARK_ENABLE_CCAPI)
+    # This allows us to override hardcoded exchange-urls with ones read from
+    # config (i.e., resolved at runtime rather than compile-time)
+    add_compile_definitions(CCAPI_EXPOSE_INTERNAL)

-  # This allows us to override hardcoded exchange-urls with ones read from
-  # config (i.e., resolved at runtime rather than compile-time)
-  add_compile_definitions(CCAPI_EXPOSE_INTERNAL)
+    # Enable ccapi services
+    add_compile_definitions(CCAPI_ENABLE_SERVICE_MARKET_DATA)
+    add_compile_definitions(CCAPI_ENABLE_SERVICE_EXECUTION_MANAGEMENT)
+    # Enable ccapi exchanges
+    add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE)
+    add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE_COIN_FUTURES)
+    add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE_US)
+    add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE_USDS_FUTURES)
+    add_compile_definitions(CCAPI_ENABLE_EXCHANGE_COINBASE)
+    add_compile_definitions(CCAPI_ENABLE_EXCHANGE_DERIBIT)
+    add_compile_definitions(CCAPI_ENABLE_EXCHANGE_OKX)

-  # Enable ccapi services
-  add_compile_definitions(CCAPI_ENABLE_SERVICE_MARKET_DATA)
-  add_compile_definitions(CCAPI_ENABLE_SERVICE_EXECUTION_MANAGEMENT)
-  # Enable ccapi exchanges
-  add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE)
-  add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE_COIN_FUTURES)
-  add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE_US)
-  add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE_USDS_FUTURES)
-  add_compile_definitions(CCAPI_ENABLE_EXCHANGE_COINBASE)
-  add_compile_definitions(CCAPI_ENABLE_EXCHANGE_DERIBIT)
-  add_compile_definitions(CCAPI_ENABLE_EXCHANGE_OKX)
+    set(CCAPI_LOG_LEVEL
+      "Warn"
+      CACHE STRING "<Fatal|Error|Warn|Info|Debug|Trace> Log level for ccapi")
+    string(TOUPPER "${CCAPI_LOG_LEVEL}" _ccapi_log_level_upper)
+    add_compile_definitions("CCAPI_ENABLE_LOG_${_ccapi_log_level_upper}")

-  set(CCAPI_LOG_LEVEL
-    "Warn"
-    CACHE STRING "<Fatal|Error|Warn|Info|Debug|Trace> Log level for ccapi")
-  string(TOUPPER "${CCAPI_LOG_LEVEL}" _ccapi_log_level_upper)
-  add_compile_definitions("CCAPI_ENABLE_LOG_${_ccapi_log_level_upper}")
+    # ** Binance **
+    add_compile_definitions(CCAPI_BINANCE_URL_REST_BASE="https://api.binance.com")
+    add_compile_definitions(BINANCE_INSTRUMENTS_PATH="/api/v3/exchangeInfo")
+    add_compile_definitions(BINANCE_PRICE_STATS_PATH="/api/v3/ticker/24hr")

-  # ** Binance **
-  add_compile_definitions(CCAPI_BINANCE_URL_REST_BASE="https://api.binance.com")
-  add_compile_definitions(BINANCE_INSTRUMENTS_PATH="/api/v3/exchangeInfo")
-  add_compile_definitions(BINANCE_PRICE_STATS_PATH="/api/v3/ticker/24hr")
+    # ** Binance US **
+    add_compile_definitions(BINANCE_US_INSTRUMENTS_PATH="/api/v3/exchangeInfo")
+    add_compile_definitions(BINANCE_US_PRICE_STATS_PATH="/api/v3/ticker/24hr")

-  # ** Binance US **
-  add_compile_definitions(BINANCE_US_INSTRUMENTS_PATH="/api/v3/exchangeInfo")
-  add_compile_definitions(BINANCE_US_PRICE_STATS_PATH="/api/v3/ticker/24hr")
+    # ** Binance USDS Futures **
+    add_compile_definitions(CCAPI_BINANCE_USDS_FUTURES_URL_REST_BASE="https://dapi.binance.com")
+    add_compile_definitions(BINANCE_USDS_FUTURES_INSTRUMENTS_PATH="/dapi/v1/exchangeInfo")
+    add_compile_definitions(BINANCE_USDS_FUTURES_PRICE_STATS_PATH="/dapi/v1/ticker/24hr")

-  # ** Binance USDS Futures **
-  add_compile_definitions(CCAPI_BINANCE_USDS_FUTURES_URL_REST_BASE="https://dapi.binance.com")
-  add_compile_definitions(BINANCE_USDS_FUTURES_INSTRUMENTS_PATH="/dapi/v1/exchangeInfo")
-  add_compile_definitions(BINANCE_USDS_FUTURES_PRICE_STATS_PATH="/dapi/v1/ticker/24hr")
+    # ** Deribit **
+    add_compile_definitions(DERIBIT_INSTRUMENT_PATH="/api/v2/public/get_instrument")
+    add_compile_definitions(DERIBIT_INSTRUMENTS_PATH="/api/v2/public/get_instruments")
+    add_compile_definitions(DERIBIT_COMBOS_PATH="/api/v2/public/get_combos")
+    add_compile_definitions(DERIBIT_TICKER_PATH="/api/v2/public/ticker")
+    add_compile_definitions(DERIBIT_CURRENCIES_METHOD="public/get_currencies")
+    add_compile_definitions(DERIBIT_INSTRUMENT_METHOD="public/get_instrument")
+    add_compile_definitions(DERIBIT_INSTRUMENTS_METHOD="public/get_instruments")
+    add_compile_definitions(DERIBIT_COMBO_DETAILS_METHOD="public/get_combo_details")
+    add_compile_definitions(DERIBIT_COMBOS_METHOD="public/get_combos")

-  # ** Deribit **
-  add_compile_definitions(DERIBIT_INSTRUMENT_PATH="/api/v2/public/get_instrument")
-  add_compile_definitions(DERIBIT_INSTRUMENTS_PATH="/api/v2/public/get_instruments")
-  add_compile_definitions(DERIBIT_COMBOS_PATH="/api/v2/public/get_combos")
-  add_compile_definitions(DERIBIT_TICKER_PATH="/api/v2/public/ticker")
-  add_compile_definitions(DERIBIT_CURRENCIES_METHOD="public/get_currencies")
-  add_compile_definitions(DERIBIT_INSTRUMENT_METHOD="public/get_instrument")
-  add_compile_definitions(DERIBIT_INSTRUMENTS_METHOD="public/get_instruments")
-  add_compile_definitions(DERIBIT_COMBO_DETAILS_METHOD="public/get_combo_details")
-  add_compile_definitions(DERIBIT_COMBOS_METHOD="public/get_combos")
+    # ** Okx **
+    add_compile_definitions(OKX_INSTRUMENTS_PATH="/api/v5/public/instruments")
+    add_compile_definitions(OKX_UNDERLYING_PATH="/api/v5/public/underlying")
+  endif()

-  # ** Okx **
-  add_compile_definitions(OKX_INSTRUMENTS_PATH="/api/v5/public/instruments")
-  add_compile_definitions(OKX_UNDERLYING_PATH="/api/v5/public/underlying")
-endif()
-
-# subdirs
-add_subdirectory(src/lib_system)
-add_subdirectory(src/lib_domain)
-add_subdirectory(src/lib_http)
-if (BUILD_SPARKD)
-  add_subdirectory(src/lib_python_md)
-  add_subdirectory(src/app_sparkd)
-  add_subdirectory(src/data_files)
-endif()
-add_subdirectory(src/lib_spark)
-if (BUILD_ADMIN_WEB_APP)
-  add_subdirectory(src/Spark.Admin.WebApp)
-endif()
-if(BUILD_DOCS)
-  add_subdirectory(doc/user)
-endif()
-if(BUILD_SDK)
-  add_subdirectory(src/sdk)
+  # subdirs
+  if (BUILD_SPARKD)
+    add_subdirectory(src/lib_system)
+    add_subdirectory(src/lib_domain)
+    add_subdirectory(src/lib_python_md)
+    add_subdirectory(src/app_sparkd)
+    add_subdirectory(src/data_files)
+  endif()
+  add_subdirectory(src/lib_spark)
+  if (BUILD_ADMIN_WEB_APP)
+    add_subdirectory(src/Spark.Admin.WebApp)
+  endif()
+  if(BUILD_DOCS)
+    add_subdirectory(doc/user)
+  endif()
+  if(BUILD_SDK)
+    add_subdirectory(src/sdk)
+  endif()
 endif()
@@ -28,7 +28,7 @@
        "CMAKE_SHARED_LINKER_FLAGS_RELEASE": "/OPT:REF /OPT:ICF /LTCG",
        "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
        "EXTERNALS_ROOT": "${sourceDir}/externals/windows",
-        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf",
+        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0;${sourceDir}/build/$<CONFIG>/bin/net7.0-windows",
        "EXPORT_MD_CONSUMER": {
          "type": "BOOL",
@@ -53,7 +53,6 @@
        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
        "CMAKE_PREFIX_PATH": "${sourceDir}/cmake/;${sourceDir}/.venv/lib/python3.10/site-packages/pybind11/share/cmake/pybind11",
        "CMAKE_EXE_LINKER_FLAGS": "-Wno-undef",
-        "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
        "EXTERNALS_ROOT": "${sourceDir}/externals/linux",
        "BUILD_SDK": {
          "type": "BOOL",
@@ -77,7 +76,7 @@
          "value": "ON"
        },
        "CCAPI_LOG_LEVEL": "Info",
-        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/md_dev/*.conf;spark/config/profiles/default/*.conf"
+        "CONFIG_DEFAULTS_PATH": "config/profiles/md_dev/*.conf;config/profiles/default/*.conf"
      }
    },

@@ -88,7 +87,7 @@
      "cacheVariables": {
        "CMAKE_BUILD_TYPE": "Release",
        "CCAPI_LOG_LEVEL": "Warn",
-        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/md_prod/*.conf;spark/config/profiles/default/*.conf"
+        "CONFIG_DEFAULTS_PATH": "config/profiles/md_prod/*.conf;config/profiles/default/*.conf"
      }
    },

@@ -168,8 +167,8 @@
          "type": "BOOL",
          "value": "ON"
        },
-        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
+        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/Spark.Api.Server/$<CONFIG>/bin/net7.0",
        "EXPORT_MD_CONSUMER": {
          "type": "BOOL",
          "value": "ON"
@@ -207,7 +206,7 @@
      "description": "Data Stream utils ported over from Edge",
      "binaryDir": "${sourceDir}/build/${presetName}",
      "cacheVariables": {
-        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf;spark/config/profiles/data_stream/*.conf",
+        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf",
        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0;${sourceDir}/build/$<CONFIG>/bin/net7.0-windows"
      }
    },
@@ -240,7 +239,8 @@
        "CMAKE_CXX_COMPILER": "clang++",
        "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
-        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf;spark/config/profiles/data_stream/*.conf",
+        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf",
+        "CONFIG_PROFILES": "default;data_stream",
        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0;${sourceDir}/build/$<CONFIG>/bin/net8.0",
        "EXPORT_MD_CONSUMER": {
          "type": "BOOL",
@@ -14,16 +14,6 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "spark_test", "src\client\sp
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Spark.Console", "src\client\Spark.Console\Spark.Console.csproj", "{89D666E6-B500-498F-B026-4D167F68ADFC}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Net", "src\DataStreamSuite\Bts.Net\Bts.Net.csproj", "{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamCore", "src\DataStreamSuite\DataStreamCore\DataStreamCore.csproj", "{CB62D5D2-3495-4488-BFBC-7C7A3725A1EE}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStream", "src\DataStreamSuite\DataStream\DataStream.csproj", "{05995ED5-650A-4955-9DFC-3242B737E77A}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SharedStreams", "src\DataStreamSuite\SharedStreams\SharedStreams.csproj", "{A8B861DE-E098-4291-98EE-76AA0F5F5A7B}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PubSub", "src\DataStreamSuite\PubSub\PubSub.csproj", "{EFA9F78D-083E-457A-BABB-9386C88C32C0}"
-EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|x64 = Debug|x64
@@ -80,46 +70,6 @@ Global
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.Release|x64.Build.0 = Release|x64
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|x64.ActiveCfg = Debug|x64
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|x64.Build.0 = Debug|x64
-		{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.Debug|x64.Build.0 = Debug|Any CPU
-		{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.Release|x64.ActiveCfg = Release|Any CPU
-		{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.Release|x64.Build.0 = Release|Any CPU
-		{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{CB62D5D2-3495-4488-BFBC-7C7A3725A1EE}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{CB62D5D2-3495-4488-BFBC-7C7A3725A1EE}.Debug|x64.Build.0 = Debug|Any CPU
-		{CB62D5D2-3495-4488-BFBC-7C7A3725A1EE}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{CB62D5D2-3495-4488-BFBC-7C7A3725A1EE}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{CB62D5D2-3495-4488-BFBC-7C7A3725A1EE}.Release|x64.ActiveCfg = Release|Any CPU
-		{CB62D5D2-3495-4488-BFBC-7C7A3725A1EE}.Release|x64.Build.0 = Release|Any CPU
-		{CB62D5D2-3495-4488-BFBC-7C7A3725A1EE}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{CB62D5D2-3495-4488-BFBC-7C7A3725A1EE}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{05995ED5-650A-4955-9DFC-3242B737E77A}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{05995ED5-650A-4955-9DFC-3242B737E77A}.Debug|x64.Build.0 = Debug|Any CPU
-		{05995ED5-650A-4955-9DFC-3242B737E77A}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{05995ED5-650A-4955-9DFC-3242B737E77A}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{05995ED5-650A-4955-9DFC-3242B737E77A}.Release|x64.ActiveCfg = Release|Any CPU
-		{05995ED5-650A-4955-9DFC-3242B737E77A}.Release|x64.Build.0 = Release|Any CPU
-		{05995ED5-650A-4955-9DFC-3242B737E77A}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{05995ED5-650A-4955-9DFC-3242B737E77A}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{A8B861DE-E098-4291-98EE-76AA0F5F5A7B}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{A8B861DE-E098-4291-98EE-76AA0F5F5A7B}.Debug|x64.Build.0 = Debug|Any CPU
-		{A8B861DE-E098-4291-98EE-76AA0F5F5A7B}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{A8B861DE-E098-4291-98EE-76AA0F5F5A7B}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{A8B861DE-E098-4291-98EE-76AA0F5F5A7B}.Release|x64.ActiveCfg = Release|Any CPU
-		{A8B861DE-E098-4291-98EE-76AA0F5F5A7B}.Release|x64.Build.0 = Release|Any CPU
-		{A8B861DE-E098-4291-98EE-76AA0F5F5A7B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{A8B861DE-E098-4291-98EE-76AA0F5F5A7B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{EFA9F78D-083E-457A-BABB-9386C88C32C0}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{EFA9F78D-083E-457A-BABB-9386C88C32C0}.Debug|x64.Build.0 = Debug|Any CPU
-		{EFA9F78D-083E-457A-BABB-9386C88C32C0}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{EFA9F78D-083E-457A-BABB-9386C88C32C0}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{EFA9F78D-083E-457A-BABB-9386C88C32C0}.Release|x64.ActiveCfg = Release|Any CPU
-		{EFA9F78D-083E-457A-BABB-9386C88C32C0}.Release|x64.Build.0 = Release|Any CPU
-		{EFA9F78D-083E-457A-BABB-9386C88C32C0}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{EFA9F78D-083E-457A-BABB-9386C88C32C0}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -1,5 +1,3 @@
-include_guard(GLOBAL)
-
 macro(export_to_parent)
  foreach(var IN ITEMS ${ARGN})
    set(${var} ${${var}} PARENT_SCOPE)
@@ -1,142 +1,66 @@
 include_guard(DIRECTORY)
 include(${CMAKE_SOURCE_DIR}/cmake/ExportToParent.cmake)
-include(${CMAKE_SOURCE_DIR}/cmake/InstallBasic.cmake)
-include(${CMAKE_SOURCE_DIR}/cmake/ProtobufGenerateCpp.cmake)
-include(${CMAKE_SOURCE_DIR}/cmake/SetFirstDefined.cmake)
-include(${CMAKE_SOURCE_DIR}/cmake/safe_dep_glob.cmake)
+include(${CMAKE_SOURCE_DIR}/cmake/TernaryOperator.cmake)

 function(init_library)
  set(_options SHARED STATIC)
-  set(_one_value_args
-    NAME
-    SOURCE_DIR
-    INCLUDE_DIR
-  )
+  set(_one_value_args SOURCE_DIR)
  set(_multi_value_args)
  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+
  if (_ARGS_SHARED AND _ARGS_STATIC)
    message(FATAL_ERROR "Invalid args passed to init_library: cannot specify STATIC and SHARED")
  endif()
-  set_first_defined(_name _ARGS_NAME PROJECT_NAME)
-  set_first_defined(_source_dir _ARGS_SOURCE_DIR _name)
-  set_first_defined(_include_dir _ARGS_INCLUDE_DIR _source_dir)

-  # Cpp sources
+  set_ternary(_source_dir _ARGS_SOURCE_DIR _ARGS_SOURCE_DIR ${PROJECT_NAME})
+  # Glob cpp sources
  file(
-    GLOB_RECURSE ${_name}_SOURCES
+    GLOB_RECURSE ${PROJECT_NAME}_SOURCES
    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
    ${_source_dir}/*.cpp
  )
  # Note: cpp files inside 'tests' dir(s) are considered tests.
  # Remove tests from main sources, and add them to test sources
-  foreach(item ${${_name}_SOURCES})
-    if(${item} MATCHES "tests/.*\.cpp$")
-      list(APPEND ${_name}_TEST_SOURCES ${item})
+  foreach(item ${${PROJECT_NAME}_SOURCES})
+    if(${item} MATCHES "tests/*.cpp")
+      list(REMOVE_ITEM ${PROJECT_NAME}_SOURCES ${item})
+      list(APPEND ${PROJECT_NAME}_TEST_SOURCES ${item})
    endif()
  endforeach()
-  foreach(item ${${_name}_TEST_SOURCES})
-    list(REMOVE_ITEM ${_name}_SOURCES ${item})
-  endforeach()
-
-  # Proto sources 
-  file(
-    GLOB_RECURSE ${_name}_PROTO_SOURCES
-    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
-    ${_source_dir}/protobufs/*.proto
-  )
-  update_deps_file("${${_name}_PROTO_SOURCES};")
-  protobuf_generate_cpp(
-    CXX_SOURCES ${_name}_CXX_PROTO_SOURCES
-    CXX_HEADERS ${_name}_CXX_PROTO_HEADERS
-    CXX_OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}
-    PROTO_INCLUDE_DIRS
-      ${EXTERNALS_INCLUDE}
-      ${CMAKE_CURRENT_SOURCE_DIR}
-    PROTO_SOURCES
-      ${${_name}_PROTO_SOURCES}
-  )

  export_to_parent(
-    ${_name}_SOURCES
-    ${_name}_TEST_SOURCES
-    ${_name}_PROTO_SOURCES
-    ${_name}_CXX_PROTO_SOURCES
-    ${_name}_CXX_PROTO_HEADERS
+    ${PROJECT_NAME}_SOURCES
+    ${PROJECT_NAME}_TEST_SOURCES
  )

-  # Add main lib
+  # Add main library
  if (_ARGS_STATIC)
-    add_library(${_name} STATIC ${${_name}_SOURCES} ${${_name}_CXX_PROTO_SOURCES})
+    add_library(${PROJECT_NAME} STATIC ${${PROJECT_NAME}_SOURCES})
  elseif(_ARGS_SHARED)
-    add_library(${_name} SHARED ${${_name}_SOURCES} ${${_name}_CXX_PROTO_SOURCES})
+    add_library(${PROJECT_NAME} SHARED ${${PROJECT_NAME}_SOURCES})
  else()
-    add_library(${_name} ${${_name}_SOURCES} ${${_name}_CXX_PROTO_SOURCES})
+    add_library(${PROJECT_NAME} ${${PROJECT_NAME}_SOURCES})
  endif()
+
  target_include_directories(
-    ${_name}
-    SYSTEM
-    BEFORE
-    PUBLIC
-      $<BUILD_INTERFACE:${EXTERNALS_ROOT}/include>
-      $<BUILD_INTERFACE:${EXTERNALS_ROOT}/include/zlib>
-  )
-  target_include_directories(
-    ${_name}
+    ${PROJECT_NAME}
    PUBLIC
      $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
      $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>
  )
-  # Include protos (if any)
-  if (${_name}_PROTO_SOURCES)
-    target_include_directories(
-      ${_name}
-      PUBLIC
-        $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
-    )
-  endif()
  target_link_directories(
-    ${_name}
+    ${PROJECT_NAME}
    PUBLIC
      $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
-      $<BUILD_INTERFACE:${EXTERNALS_ROOT}/lib/$<CONFIG>>
      $<INSTALL_INTERFACE:${CMAKE_INSTALL_LIBDIR}>
  )
-  # Install main lib
-  foreach(_hdr ${${_name}_CXX_PROTO_HEADERS})
-    file(RELATIVE_PATH _relative_path ${CMAKE_CURRENT_BINARY_DIR} ${_hdr})
-    install(FILES ${_hdr} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/${_relative_path})
-  endforeach()
-  install_basic(
-    NAME ${_name}
-    INCLUDE_DIR ${_include_dir}
-  )

  # Add tests executable (if there are any)
-  if (${_name}_TEST_SOURCES)
-    add_executable(${_name}-tests ${${_name}_TEST_SOURCES})
-
-    get_target_property(_lib_type ${_name} TYPE)
-    if (_lib_type STREQUAL "SHARED_LIBRARY")
-      # Set RPATH so the tests exe knows where to find the shared lib.
-      set_target_properties(
-        ${_name}-tests
-        PROPERTIES
-          BUILD_RPATH "$ORIGIN"
-          INSTALL_RPATH "$ORIGIN/../${CMAKE_INSTALL_LIBDIR}"
-      )
-    endif()
-
-    # Link main lib
-    target_link_directories(
-      ${_name}-tests
-      PRIVATE
-        $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
-        $<INSTALL_INTERFACE:${CMAKE_INSTALL_LIBDIR}>
-    )
-    target_link_libraries(${_name}-tests PRIVATE ${_name})
-
+  if (${PROJECT_NAME}_TEST_SOURCES)
+    add_executable(${PROJECT_NAME}-tests ${${PROJECT_NAME}_TEST_SOURCES})
+    target_link_libraries(${PROJECT_NAME}-tests PRIVATE ${PROJECT_NAME})
    # Catch2
    find_package(Catch2 CONFIG REQUIRED COMPONENTS Catch2WithMain)
-    target_link_libraries(${_name}-tests PRIVATE Catch2::Catch2WithMain)
+    target_link_libraries(${PROJECT_NAME}-tests PRIVATE Catch2::Catch2WithMain)
  endif()
 endfunction()
@@ -0,0 +1,55 @@
+function(init_shared_library)
+  include(${CMAKE_SOURCE_DIR}/cmake/ExportToParent.cmake)
+  set(source_dir)
+  if (${ARGC} EQUAL 0)
+    set(source_dir ${PROJECT_NAME})
+  elseif(${ARGC} GREATER 1)
+    message(FATAL_ERROR "init_shared_library received invalid arguments. Usage: init_shared_library(<OPTIONAL:source_dir>)")
+  else()
+    set(source_dir ${ARGV})
+  endif()
+
+  # Glob cpp sources
+  file(
+    GLOB_RECURSE ${PROJECT_NAME}_SOURCES
+    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
+    ${source_dir}/*.cpp
+  )
+  # Note: cpp files inside 'tests' dir(s) are considered tests.
+  # Remove tests from main sources, and add them to test sources
+  foreach(item ${${PROJECT_NAME}_SOURCES})
+    if(${item} MATCHES "tests/*.cpp")
+      list(REMOVE_ITEM ${PROJECT_NAME}_SOURCES ${item})
+      list(APPEND ${PROJECT_NAME}_TEST_SOURCES ${item})
+    endif()
+  endforeach()
+
+  export_to_parent(
+    ${PROJECT_NAME}_SOURCES
+    ${PROJECT_NAME}_TEST_SOURCES
+  )
+
+  # Add main library
+  add_library(${PROJECT_NAME} SHARED ${${PROJECT_NAME}_SOURCES})
+  target_include_directories(
+    ${PROJECT_NAME}
+    PUBLIC
+      $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
+      $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>
+  )
+  target_link_directories(
+    ${PROJECT_NAME}
+    PUBLIC
+      $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
+      $<INSTALL_INTERFACE:${CMAKE_INSTALL_LIBDIR}>
+  )
+
+  # Add tests executable (if there are any)
+  if (${PROJECT_NAME}_TEST_SOURCES)
+    add_executable(${PROJECT_NAME}-tests ${${PROJECT_NAME}_TEST_SOURCES})
+    target_link_libraries(${PROJECT_NAME}-tests PRIVATE ${PROJECT_NAME})
+    # Catch2
+    find_package(Catch2 CONFIG REQUIRED COMPONENTS Catch2WithMain)
+    target_link_libraries(${PROJECT_NAME}-tests PRIVATE Catch2::Catch2WithMain)
+  endif()
+endfunction(init_shared_library)
@@ -1,39 +0,0 @@
-include_guard(DIRECTORY)
-include(GNUInstallDirs)
-include(${CMAKE_SOURCE_DIR}/cmake/PackBasic.cmake)
-include(${CMAKE_SOURCE_DIR}/cmake/SetFirstDefined.cmake)
-
-function(install_basic)
-  set(_options)
-  set(_one_value_args NAME INCLUDE_DIR)
-  set(_multi_value_args)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-  set_first_defined(_name _ARGS_NAME PROJECT_NAME)
-  set_first_defined(_include_dir _ARGS_INCLUDE_DIR _name)
-
-  file(
-    GLOB_RECURSE ${_name}_HEADERS
-    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
-    ${_include_dir}/*.h)
-
-  target_sources(${_name}
-    PUBLIC
-      FILE_SET HEADERS FILES ${${_name}_HEADERS})
-
-  install(
-    TARGETS ${_name}
-    EXPORT ${_name}Targets
-    RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
-    LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
-    ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
-    FILE_SET HEADERS DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}
-  )
-
-  install(
-    EXPORT ${_name}Targets
-    FILE ${_name}Targets.cmake
-    NAMESPACE ${_name}::
-    DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name})
-
-  pack_basic()
-endfunction()
@@ -0,0 +1,30 @@
+include(${CMAKE_SOURCE_DIR}/cmake/PackBasic.cmake)
+include(GNUInstallDirs)
+
+function(install_basic_config)
+  file(
+    GLOB_RECURSE ${PROJECT_NAME}_HEADERS
+    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
+    *.h)
+
+  target_sources(${PROJECT_NAME}
+    PUBLIC
+      FILE_SET HEADERS FILES ${${PROJECT_NAME}_HEADERS})
+
+  install(
+    TARGETS ${PROJECT_NAME}
+    EXPORT ${PROJECT_NAME}Targets
+    RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
+    LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
+    ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
+    FILE_SET HEADERS DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}
+  )
+
+  install(
+    EXPORT ${PROJECT_NAME}Targets
+    FILE ${PROJECT_NAME}Targets.cmake
+    NAMESPACE ${PROJECT_NAME}::
+    DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME})
+
+  pack_basic()
+endfunction()
@@ -1,14 +1,6 @@
-include_guard(DIRECTORY)
 include(CMakePackageConfigHelpers)
-include(${CMAKE_SOURCE_DIR}/cmake/SetFirstDefined.cmake)

 function(pack_basic)
-  set(_options)
-  set(_one_value_args NAME)
-  set(_multi_value_args)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-  set_first_defined(_name _ARGS_NAME PROJECT_NAME)
-
  find_package(Git QUIET)

  if(GIT_FOUND)
@@ -25,20 +17,20 @@ function(pack_basic)
  endif()

  write_basic_package_version_file(
-    "${CMAKE_CURRENT_BINARY_DIR}/${_name}ConfigVersion.cmake"
+    "${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}ConfigVersion.cmake"
    VERSION ${PROJECT_VERSION}
    COMPATIBILITY AnyNewerVersion
  )

  configure_package_config_file(
    "${CMAKE_CURRENT_SOURCE_DIR}/Config.cmake.in"
-    "${CMAKE_CURRENT_BINARY_DIR}/${_name}Config.cmake"
-    INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name}
+    "${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}Config.cmake"
+    INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}
  )

  install(
    FILES
-      "${CMAKE_CURRENT_BINARY_DIR}/${_name}Config.cmake"
-      "${CMAKE_CURRENT_BINARY_DIR}/${_name}ConfigVersion.cmake"
-    DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name})
+      "${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}Config.cmake"
+      "${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}ConfigVersion.cmake"
+    DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME})
 endfunction()
@@ -2,16 +2,8 @@ include_guard(GLOBAL)

 function(protobuf_generate_cpp)
  set(_options)
-  set(_one_value_args
-    CXX_SOURCES
-    CXX_HEADERS
-    CXX_OUT_DIR
-    PROTOC
-  )
-  set(_multi_value_args
-    PROTO_INCLUDE_DIRS
-    PROTO_SOURCES
-  )
+  set(_one_value_args CXX_SOURCES CXX_HEADERS CXX_OUT_DIR PROTOC)
+  set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})

  if (NOT DEFINED _ARGS_PROTOC)
@@ -34,7 +26,6 @@ function(protobuf_generate_cpp)
    endif()
    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.cc")
    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.h")
-    message(STATUS "add_custom_command for ${_fil} -> ${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.h")
    add_custom_command(
      OUTPUT
        "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.cc"
@@ -42,7 +33,7 @@ function(protobuf_generate_cpp)
      COMMAND
        ${_ARGS_PROTOC}
      ARGS
-        --cpp_out=${_ARGS_CXX_OUT_DIR} ${_proto_include_cli_args} ${_fil}
+        --cpp_out=${_ARGS_CXX_OUT_DIR} ${_proto_include_cli_args} ${_abs_fil}
      DEPENDS ${_abs_fil}
      COMMENT "Running C++ protocol buffer compiler on ${_fil}"
      VERBATIM
@@ -1,58 +0,0 @@
-function(protobuf_generate_cpp_config)
-  set(_options)
-  set(_one_value_args
-    PROTO_SOURCES_HEADER
-    PROTO_DEFAULTS_HEADER
-  )
-  set(_multi_value_args
-    PROTO_SOURCES
-    PROTO_DEFAULTS
-  )
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-
-  # Configure the 'config_proto_files' header. It provides an array of strings populated with the
-  # files in PROTO_SOURCES. At runtime, we use this array to generate a special 'config' proto
-  # message with fields for each message defined in said files. A global instance of this 'config'
-  # proto is then constructed, and populated with [compile-time] defaults, and [runtime] overrides
-  # (i.e., in spark.conf). Calls to `bts::config::get<T>()` retrieve the value from the field
-  # associated with type `T` in this global instance.
-  message(STATUS "PROTO_SOURCES: ${_ARGS_PROTO_SOURCES}")
-  message(STATUS "PROTO_SOURCES_HEADER: ${_ARGS_PROTO_SOURCES_HEADER}")
-  message(STATUS "PROTO_DEFAULTS_HEADER: ${_ARGS_PROTO_DEFAULTS_HEADER}")
-  if (_ARGS_PROTO_SOURCES_HEADER)
-    set(PROTO_SOURCES)
-    set(PROTO_SOURCES_CXX_ARRAY_ITEMS "")
-    foreach(_proto_path ${_ARGS_PROTO_SOURCES})
-      list(APPEND PROTO_SOURCES ${_proto_path})
-      string(APPEND PROTO_SOURCES_CXX_ARRAY_ITEMS "\"${_proto_path}\",\n")
-    endforeach()
-    list(LENGTH PROTO_SOURCES PROTO_SOURCES_LENGTH)
-    message(STATUS "CONFIGURING PROTO_SOURCES_HEADER): ${_ARGS_PROTO_SOURCES_HEADER}")
-    configure_file(${CMAKE_SOURCE_DIR}/cmake/config_proto_files.h.in ${_ARGS_PROTO_SOURCES_HEADER})
-  endif()
-
-  # Generate static defaults. Ideally, this would be a single string, but we use an array because
-  # the compiler complains if the string is longer than 65k.
-  set(_cfg_defaults_str "inline const char * config_static_defaults[] = { ")
-
-  foreach(_cfg_file_path ${_ARGS_PROTO_DEFAULTS})
-    file(READ ${_cfg_file_path} _cfg_file_content)
-    string(REPLACE "\"" "\\\"" _cfg_file_content_escaped ${_cfg_file_content})
-    string(REPLACE "\n" ";" _cfg_lines ${_cfg_file_content_escaped})
-
-    foreach(_line ${_cfg_lines})
-      string(APPEND _cfg_defaults_str "\n  \"${_line}\",")
-    endforeach(_line)
-  endforeach(_cfg_file_path)
-
-  string(APPEND _cfg_defaults_str "};")
-  set(_old_cfg_defaults_str "")
-
-  if(EXISTS ${_ARGS_PROTO_DEFAULTS_HEADER})
-    file(READ ${_ARGS_PROTO_DEFAULTS_HEADER} _old_cfg_defaults_str)
-  endif()
-
-  if(NOT _cfg_defaults_str STREQUAL _old_cfg_defaults_str)
-    file(WRITE "${_ARGS_PROTO_DEFAULTS_HEADER}" "${_cfg_defaults_str}")
-  endif()
-endfunction()
@@ -1,10 +0,0 @@
-include_guard(GLOBAL)
-
-macro(set_first_defined RESULT)
-  foreach(_arg ${ARGN})
-    if (DEFINED ${_arg})
-      set(${RESULT} ${${_arg}})
-      break()
-    endif()
-  endforeach()
-endmacro()
@@ -20,14 +20,16 @@ endfunction()

 function(sym_link_file)
  set(_options)
-  set(_one_value_args SOURCE DEST INSTALL_DEST)
+  set(_one_value_args TARGET_NAME SOURCE DEST INSTALL_DEST)
  set(_multi_value_args)
  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-  file(
-    CREATE_LINK
-      ${_ARGS_SOURCE}
-      ${_ARGS_DEST}
-    SYMBOLIC
+  add_custom_command(
+    OUTPUT ${_ARGS_DEST}
+    COMMAND ln ARGS -s ${_ARGS_SOURCE} ${_ARGS_DEST}
+  )
+  add_custom_target(
+    ${_ARGS_TARGET_NAME} ALL
+    DEPENDS ${_ARGS_DEST}
  )
  if (DEFINED _ARGS_INSTALL_DEST)
    install(
@@ -1,11 +0,0 @@
-#pragma once
-
-#include <array>
-
-namespace bts::@PROJECT_NAME@::protobufs {
-
-inline std::array<const char *, @PROTO_SOURCES_LENGTH@> config_proto_files = {
-@PROTO_SOURCES_CXX_ARRAY_ITEMS@
-};
-
-} // namespace bts::@PROJECT_NAME@::protobufs
@@ -1,24 +1,15 @@
 services:
  sparkd:
    build:
-      dockerfile: src/app_sparkd/Dockerfile
-      args:
-        - SOURCE_DIR=install/sparkd-release-gcc
-        - CONF_FILE=src/app_sparkd/spark.conf
+      dockerfile: src/server/sparkd/Dockerfile
    command: [ "./sparkd", "run" ]
-    ports:
-      - 8000:8000
-      - 8001:8001
-      - 8002:8002
+    ports: [ "8000:8000", "8001:8001", "8002:8002" ]

  pubsub:
    environment:
      - DOTNET_ENVIRONMENT=Staging
    build:
      dockerfile: src/DataStreamSuite/PubSubServer/Dockerfile
-      args:
-        - SOURCE_DIR=build/Release/bin/net7.0/linux-x64/publish
-        - CONF_FILE=src/DataStreamSuite/PubSubServer/spark.conf
    ports:
      - 22019:22019
    command: [ "./PubSubServer" ]
@@ -26,13 +17,20 @@ services:
  md-publisher:
    build:
      dockerfile: src/DataStreamSuite/DataStreamPublisher/Dockerfile
-      args:
-        - SOURCE_DIR=build/Release/bin/net7.0/linux-x64/publish
-        - CONF_FILE=src/DataStreamSuite/DataStreamPublisher/spark.conf
    depends_on:
      - pubsub
      - sparkd
-    command: [ "dockerize", "-wait", "tcp://pubsub:22019", "-wait", "tcp://sparkd:8000", "-timeout", "10s", "./DataStreamPublisher" ]
+    command:
+      [
+        "dockerize",
+        "-wait",
+        "tcp://pubsub:22019",
+        "-wait",
+        "tcp://sparkd:8000",
+        "-timeout",
+        "10s",
+        "./DataStreamPublisher"
+      ]
    environment:
      - DOTNET_ENVIRONMENT=MarketData

@@ -42,15 +40,13 @@ services:
      - 8443:8443
    build:
      dockerfile: src/DataStreamSuite/Bts.DataStreamWebApp.Server/Dockerfile
-      args:
-        - SOURCE_DIR=build/Release/bin/net7.0/linux-x64/publish
-        - WEB_ROOT=build/Bts.DataStreamWebApp.Client/production/dist
-        - CONF_FILE=src/DataStreamSuite/Bts.DataStreamWebApp.Server/spark.conf
    command: [ "./Bts.DataStreamWebApp.Server" ]
    environment:
      - ASPNETCORE_ENVIRONMENT=Staging

  api-server:
+    profiles:
+      - api-server
    depends_on:
      - sparkd
    ports:
@@ -58,7 +54,7 @@ services:
      - 9443:9443
    build:
      dockerfile: src/client/spark_api_server/Dockerfile
-    command: [ "dockerize", "-wait", "tcp://sparkd:8000", "-timeout", "10s", "./spark_api_server", "run" ]
+    command: [ "./spark_api_server", "run" ]
    environment:
      - ASPNETCORE_ENVIRONMENT=Staging

@@ -24,6 +24,7 @@ RUN locale-gen en_US.UTF-8
 ARG docker_user="bts"
 ARG docker_uid="1000"
 ARG docker_pass="bts"
+# ARG entrypoint

 RUN \
  groupadd -g "$docker_uid" "$docker_user" \
@@ -1 +1 @@
-VITE_DATASTREAM_WEBSOCKET_URL=wss://localhost:8443/ws
+VITE_DATASTREAM_WEBSOCKET_URL=wss://localhost:443
@@ -1 +1 @@
-VITE_DATASTREAM_WEBSOCKET_URL=wss://localhost:8443/ws
+VITE_DATASTREAM_WEBSOCKET_URL=wss://localhost:443
@@ -0,0 +1,32 @@
+module.exports = {
+  root: true,
+  env: { browser: true, es2020: true },
+  extends: [
+    'eslint:recommended',
+    'plugin:react/recommended',
+    'plugin:react/jsx-runtime',
+    'plugin:react-hooks/recommended',
+  ],
+  ignorePatterns: ['dist', '.eslintrc.cjs'],
+  parserOptions: { ecmaVersion: 'latest', sourceType: 'module' },
+  settings: { react: { version: '18.3.1' } },
+  plugins: ['react-refresh'],
+  rules: {
+    "import/order": [
+      "error",
+      {
+        "groups": [
+          "builtin",
+          "external",
+          "internal",
+          ["parent", "sibling", "index"]
+        ],
+        "newlines-between": "always"
+      }
+    ],
+    'react-refresh/only-export-components': [
+      'warn',
+      { allowConstantExport: true },
+    ],
+  },
+}
@@ -1,36 +0,0 @@
-import importPlugin from "eslint-plugin-import";
-import reactPkg from "eslint-plugin-react";
-const { configs: reactConfigs } = reactPkg;
-import js from "@eslint/js";
-import defineConfig from "./vite.config.js";
-
-export default [
-  js.configs.recommended,
-  importPlugin.flatConfigs.recommended,
-  reactConfigs.flat.recommended,
-  reactConfigs.flat["jsx-runtime"],
-  {
-    files: ["**/*.{js,jsx}"],
-    languageOptions: {
-      ecmaVersion: "latest",
-      sourceType: "module",
-      globals: {
-        console: "readonly"
-      }
-    },
-    settings: {
-      "import/resolver": {
-        vite: {
-          viteConfig: defineConfig({ mode: "development" })
-        }
-      }
-    },
-    rules: {
-      semi: "error",
-      "prefer-const": "error",
-      "no-unused-vars": "warn",
-      "import/no-dynamic-require": "warn",
-      "import/no-nodejs-modules": "warn",
-    }
-  }
-];
@@ -10,37 +10,35 @@
    "preview": "vite preview"
  },
  "dependencies": {
-    "@angular/material": "^18.2.8",
-    "@emotion/react": "^11.13.3",
-    "@emotion/styled": "^11.13.0",
-    "@fontsource/noto-sans": "^5.1.0",
-    "@material/data-table": "^15.0.0-canary.423edc3dc.0",
-    "@mui/icons-material": "^6.1.4",
-    "@mui/material": "^6.1.4",
-    "@mui/x-data-grid": "^7.20.0",
-    "bulma": "^1.0.2",
+    "@angular/material": "^18.0.4",
+    "@emotion/react": "^11.11.4",
+    "@emotion/styled": "^11.11.5",
+    "@fontsource/noto-sans": "^5.0.22",
+    "@material/data-table": "^15.0.0-canary.7f224ddd4.0",
+    "@mui/icons-material": "^5.15.20",
+    "@mui/material": "^5.15.20",
+    "@mui/x-data-grid": "^7.7.1",
+    "bulma": "^1.0.1",
    "classnames": "^2.5.1",
    "react": "^18.3.1",
-    "react-data-grid": "^7.0.0-beta.47",
+    "react-data-grid": "^7.0.0-beta.44",
    "react-data-table-component": "^7.6.2",
    "react-dom": "^18.3.1",
-    "react-json-view-lite": "^1.5.0",
+    "react-json-view-lite": "^1.4.0",
    "react-pro-sidebar": "^1.1.0",
-    "react-use-websocket": "^4.9.0",
-    "rpc-websockets": "^9.0.4"
+    "react-use-websocket": "^4.8.1",
+    "rpc-websockets": "^7.11.2"
  },
  "devDependencies": {
-    "@types/react": "^18.3.11",
-    "@types/react-dom": "^18.3.1",
-    "@vitejs/plugin-react": "^4.3.2",
-    "eslint": "^9.12.0",
-    "@eslint/js": "^9.12.0",
-    "eslint-plugin-import": "^2.31.0",
-    "eslint-plugin-react": "^7.37.1",
-    "eslint-plugin-react-hooks": "^5.0.0",
-    "eslint-plugin-react-refresh": "^0.4.12",
-    "eslint-import-resolver-vite": "^2.0.1",
-    "react-mui-sidebar": "^1.3.8",
-    "vite": "^5.4.9"
+    "@types/react": "^18.3.3",
+    "@types/react-dom": "^18.3.0",
+    "@vitejs/plugin-react": "^4.3.1",
+    "eslint": "^8.57.0",
+    "eslint-plugin-import": "^2.29.1",
+    "eslint-plugin-react": "^7.34.2",
+    "eslint-plugin-react-hooks": "^4.6.2",
+    "eslint-plugin-react-refresh": "^0.4.7",
+    "react-mui-sidebar": "^1.3.5",
+    "vite": "^5.2.13"
  }
 }
@@ -1,5 +1,5 @@
 import { fileURLToPath, URL } from 'node:url';
-import { defineConfig } from 'vite';
+import { defineConfig, loadEnv } from 'vite';
 import plugin from '@vitejs/plugin-react';
 import fs from 'fs';
 import path from 'path';
@@ -1,13 +1,7 @@
 FROM bluetradingsystems/linux_dotnet7

-ARG CS_SOURCE_DIR="build/Release/bin/net7.0/linux-x64/publish"
-ARG CPP_SOURCE_DIR="install/data-stream-suite-linux"
-ARG WEB_ROOT="build/Bts.DataStreamWebApp.Client/production/dist"
-ARG CONF_FILE="src/DataStreamSuite/Bts.DataStreamWebApp.Server/spark.conf"
-
 RUN dotnet dev-certs https --trust
-COPY --chown=bts:bts "${CPP_SOURCE_DIR}/lib" "bin"
-COPY --chown=bts:bts "${CS_SOURCE_DIR}" "bin"
-COPY --chown=bts:bts "${WEB_ROOT}" "bin/wwwroot"
-COPY --chown=bts:bts "${CONF_FILE}" ".local/share/bts/Bts.DataStreamWebApp.Server/conf/spark.conf"
+COPY --chown=bts:bts "build/Release/bin/net7.0/linux-x64/publish" "bin"
+COPY --chown=bts:bts "build/Bts.DataStreamWebApp.Client/production/dist" "bin/wwwroot"
+COPY --chown=bts:bts "src/DataStreamSuite/Bts.DataStreamWebApp.Server/spark.conf" ".local/share/bts/Bts.DataStreamWebApp.Server/conf/spark.conf"
 WORKDIR "/home/bts/bin"
@@ -1,4 +1,14 @@
 {
+  "Kestrel": {
+    "Endpoints": {
+      "Http": {
+        "Url": "http://localhost:8080"
+      },
+      "Https": {
+        "Url": "https://localhost:8443"
+      }
+    }
+  },
  "PubSub": {
    "Routing": {
      "PublisherRoutes": [
@@ -30,15 +30,5 @@
        "benzinga_news"
      ]
    }
-  ],
-  "Spark": {
-    "ConfigServer": "http://sparkd:8000/config"
-  },
-  "SparkApi": {
-    "IsEnabled": true,
-    "Credentials": {
-      "User": "bts",
-      "Password": ""
-    }
-  }
+  ]
 }
@@ -0,0 +1,6 @@
+FROM bluetradingsystems/linux_dotnet7
+
+RUN dotnet dev-certs https --trust
+COPY --chown=bts:bts "bin" "bin"
+COPY --chown=bts:bts "webapp.spark.conf" ".local/share/DataStream/conf/spark.conf"
+WORKDIR "/home/bts/bin"
@@ -18,11 +18,8 @@

  <ItemGroup>
    <PackageReference Include="HtmlAgilityPack" Version="1.11.57" />
-    <PackageReference Include="Microsoft.Data.Analysis" Version="0.21.1" />
    <PackageReference Include="MongoDB.Bson" Version="2.23.1" />
    <PackageReference Include="MongoDB.Driver" Version="2.23.1" />
-    <PackageReference Include="Parquet.Net" Version="5.0.1" />
-    <PackageReference Include="System.Data.SQLite" Version="1.0.118" />
  </ItemGroup>

 </Project>
@@ -11,9 +11,6 @@ public static class DataStreamExtensions {
   public static ViewStream AsViewStream(this IObservable<DataPacket> source) =>
      source as ViewStream ?? new ObservableViewStream(source);

-   public static ViewStream AsViewStream<T>(this IObservable<IDataPacket<T>> source) where T : IBlueRow =>
-      source as ViewStream ?? new ObservableViewStream(source.Select(dp => new DataPacket(dp.newRows, dp.addedRows, dp.deletedRows)));
-
   public static IObservable<DataPacket> AsObservable(this ViewStream source) {
      return source as IObservable<DataPacket> ?? Observable.Create<DataPacket>(Subscribe);

@@ -1,32 +0,0 @@
-namespace bts.SilverPine {
-   using System;
-   using BCG;
-   using System.Collections.Generic;
-   using Bts.DataStreamExpressions;
-
-   // All this does is take in a dsl expression, parse it and fire the data that come out of running it.
-   // So for example, if an expresison is passed 
-   [DataStreamUserRespresentation("dsl")]
-   public class DSLStream : ViewStream {
-      private readonly string _dsl;
-      public DSLStream(string dsl) {
-         Log.Gen.info($"DSLStream.cons dsl: {dsl}");
-         _dsl = dsl;
-         ViewStream rs = build_stream(dsl);
-         rs = rs ?? new LiteralStream("error", $"msg:invalid dsl \"{dsl}\"");
-         mySetRealSource(rs);
-      }
-
-      static ViewStream? build_stream(string stream_expression) {
-         try {
-            var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression);
-            var rv = exp.getView(new Dictionary<string, StatementExpression>(0));
-            return rv;
-
-         } catch (Exception ex) {
-            Log.Gen.error($"Could not build stream: {stream_expression} ex: {ex}");
-            return null;
-         }
-      }
-   }
-}
@@ -25,12 +25,12 @@ namespace BCG {
         byte[] rem_req_bytes = rem_req.serialize();
         _publisher_core.publish("System", "RemoteRequest", rem_req_bytes);
         _source_str = new TableStream(_name);
-         _source_str.onChange += _handle_data;
+         _source_str.onChange += new ChangeDelegate(_handle_data);
      }

      protected override void myOnLastSubscriber() {
         //maybe there should be a way to turn remote requests off ?
-         _source_str.onChange -= _handle_data;
+         _source_str.onChange -= new ChangeDelegate(_handle_data);
      }

      static string get_expression(params string[] lines) {
@@ -21,11 +21,13 @@ namespace BCG
         : this (new TableStream (tableName))
      {}

-      protected override void myOnFirstSubscriber () =>
-         myVS.onChange += myHandleChanges;
+      protected override void myOnFirstSubscriber () {
+         myVS.onChange += new ChangeDelegate(myHandleChanges);
+      }
+      protected override void myOnLastSubscriber () {
+         myVS.onChange -= new ChangeDelegate(myHandleChanges);
+      }

-      protected override void myOnLastSubscriber () =>
-         myVS.onChange -= myHandleChanges;

      private void myHandleChanges(object newValue)
      {
@@ -30,7 +30,7 @@ namespace BCG

        protected override void myOnFirstSubscriber()
        {
-            myToCrush.onChange += myToCrush_onChange;
+            myToCrush.onChange += new ChangeDelegate(myToCrush_onChange);
        }

        protected override void myOnLastSubscriber()
@@ -15,11 +15,11 @@ namespace BCG
            
            //set up an anonymous delegate for each of the streams
            //we do this so that we can unsubscribe
-            myDelegates = new DataStreamChangeDelegate[myToCrush.Length];
+            myDelegates = new ChangeDelegate[myToCrush.Length];
            for (int i = 0; i < myToCrush.Length; ++i)
            {
                int tempI = i;
-                myDelegates[tempI] = new((o) => { myHandleChange(tempI, ((DataPacket)o).newRows); });
+                myDelegates[tempI] = new ChangeDelegate((o) => { myHandleChange(tempI, ((DataPacket)o).newRows); });
            }
        }

@@ -27,7 +27,7 @@ namespace BCG
        private ViewStream[] myToCrush;
        private ITableCrusher myCrusher;

-        private DataStreamChangeDelegate[] myDelegates;
+        private ChangeDelegate[] myDelegates;

        protected override void myOnFirstSubscriber()
        {
@@ -28,7 +28,7 @@ namespace BCG

      protected override void myOnFirstSubscriber()
      {
-         myDataSource.onChange += myDataSource_onChange;
+         myDataSource.onChange += new ChangeDelegate(myDataSource_onChange);
      }

      protected override void myOnLastSubscriber()
@@ -0,0 +1,49 @@
+// using System;
+// using System.Collections.Generic;
+// using System.Text;
+// using System.IO;
+
+// using System.Runtime.Serialization.Formatters.Binary;
+
+// namespace BCG
+// {
+//    public class DumpLoaderStream : ViewStream, ISideEffectStream
+//    {
+//       //wtf
+//       //public class Exception : System.Exception
+//       //{public Exception(string msg) : base(msg) { }}
+
+//       public DumpLoaderStream(string fileName, int secondsBetweenPolls)
+//       {
+//          mySetRealSource(
+//             new PollStream(() => new LoaderStream(fileName).rows, secondsBetweenPolls, 
+//                ViewStreamDatabase.instance));
+//       }
+
+//       public DumpLoaderStream(string fileName)
+//       {
+//         if (fileName == null)
+//            throw new LoaderStream.Exception("File name cannot be null");
+
+//         for (int i = 0; i < 5; i++)
+//            try
+//            {
+//               using (FileStream fs = new FileStream(fileName, FileMode.Open, FileAccess.Read))
+//               {
+//                  IClientDeserializer deser = new ClientUniversalDeserializer();
+//                  byte[] the_ops;
+//                  Array the_rows;
+//                  deser.deserialize(new Stream[] { fs }, out the_ops, out the_rows);
+
+//                  mySetRealSource(new BlueRowCollectionStaticStream((IBlueRowCollection)the_rows.GetValue(0)));
+//               }
+//               break;
+//            }
+//            catch (IOException)
+//            {
+//               System.Threading.Thread.Sleep(50);
+//            }
+
+//       }
+//    }
+// }
@@ -1,7 +1,6 @@
 using BCG;

 namespace bts {
-   // This is the other side of the SaverStream, its "loads" the stream and fires once
   [DataStreamUserRespresentation("loader")]
   public class LoaderStream : ViewStream {

@@ -5,11 +5,10 @@ using System.Text;
 using System.IO;
 using Google.Protobuf;
 using System.Globalization;
-using BCG;
+using bts;

-namespace bts {
-   // Writes all the udpates to a stream in a file or multiples files dependong on the rollover size (as opposed to Saver
-   // which just writes one update, as a create)
+namespace BCG {
+   // Writes all the udpates to a stream in a file.
   // File naming:
   // token_epoch_yyyyMMdd_HHmmss
   // File location: directory(passed)\token\
@@ -95,10 +94,10 @@ namespace bts {
      void _on_change(object new_value) {
         DataPacket dp = new_value as DataPacket;
         byte[] serialized_update = null;
-         bool first_write = _serializer == null || _n_updates_in_current_file == 0;
+         bool _write_create = _serializer == null || _n_updates_in_current_file == 0;
         _serializer = _serializer ?? new UniversalStreamIOSerializer();
-         if (first_write) {
-            // first file always starts with create
+         if (_write_create) {
+            // new files always starts with create
            serialized_update = _serializer.serialize(new bts.StreamIOMessage(new bts.StreamIOMessage.CreateMessage(dp.newRows)));
         } else
            serialized_update = _serializer.serialize(new bts.StreamIOMessage(new bts.StreamIOMessage.MultipleRemoveAddMessage(dp.deletedRows, dp.addedRows)));
@@ -106,8 +105,8 @@ namespace bts {
         long utc_epoch = epoch_now();

         //time, length of data, data
-         //bw.Write(utc_epoch);
-         //bw.Write(serialized_update.Length);
+         bw.Write(utc_epoch);
+         bw.Write(serialized_update.Length);
         bw.Write(serialized_update);
         _out_fs.Flush();
         myFireEvent(dp);
@@ -0,0 +1,190 @@
+// using System;
+// using System.Collections.Generic;
+// using System.Linq;
+// using System.Text;
+// using System.IO;
+// using System.Runtime.Serialization.Formatters.Binary;
+// using BCG.Utilities.General;
+// using bts.collections;
+// using System.Reflection.Metadata;
+
+
+// namespace BCG {
+
+//    public class PlaybackStream : ViewStream, IWorker {
+//       private readonly PlaybackType _type;
+//       private BlueTable _current_table;
+//       private IClientDeserializer _deserilializer = new ClientUniversalDeserializer();
+//       // these are the packets already downloaded from the current file
+//       private List<PlaybackPacket> _packets = new List<PlaybackPacket>();
+//       //private IEnumerable<FileStream> _file_streams = new List<FileStream>();
+//       private IEnumerable<string> _file_names;
+//       private Tuple<int, FileStream> _current_fs = new Tuple<int, FileStream>(-1, null);
+//       private DateTime _playback_t0, _playback_t1;
+//       private bool _all_files_read = false;
+//       SnapshotData _snapshot_data = null; // only initialized when in snapshot mode
+
+//       public PlaybackStream(string datetime_start, string datetime_end, string directory_name, string token) {
+//          _type = PlaybackType.Streaming;
+//          _playback_t0 = PersistorStream.parse_datetime(datetime_start);
+//          _playback_t1 = PersistorStream.parse_datetime(datetime_end);
+//          string token_directory_name = $"{directory_name}\\{token}";
+//          bts.Log.Gen.info($"PlaybackStream t0: {_playback_t0} t1: {_playback_t1} dir: {token_directory_name}");
+//          List<string> files_in_range = new List<string>();
+//          if (Directory.Exists(token_directory_name)) {
+//             _file_names =
+//          Directory.GetFiles(token_directory_name).Where(f => {
+//             DateTime file_time = PersistorStream.parse_datetime_from_filename(f);
+//             return file_time > _playback_t0 && file_time < _playback_t1;
+//          });
+//          }
+//       }
+
+//       protected override void myOnFirstSubscriber() {
+//          FileStream fs = new FileStream(_snapshot_data.snapshot_file, FileMode.Open, FileAccess.Read);
+//          _snapshot_data.packets = StreamPersistorUtils.load_packets_from_file(fs);
+//          PlaybackPacket snapshot_packet = StreamPersistorUtils.find_packet_at_time(_snapshot_data.packets, _snapshot_data.snapshot_time);
+//          if (snapshot_packet != null) {
+//             foreach (PlaybackPacket p in _snapshot_data.packets) {
+//                byte[] ops;
+//                Array row_array;
+//                _deserilializer.deserialize(new Stream[] { new MemoryStream(p.data) }, out ops, out row_array);
+//                for (int i = 0; i < ops.Length; i++) {
+//                   IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
+//                   byte op = ops[i];
+//                   if (op == StreamOpCode.create) {
+//                      IBlueRowCollection full_table = rows_args[0];
+//                      _current_table = new BlueTable(full_table);
+//                   } else if (op == StreamOpCode.multipleRemoveAdd) {
+//                      //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
+//                      IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
+//                      IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
+//                      BlueDataUtility.absorbChanges(_current_table, dels, adds);
+//                   } else {
+//                      bts.Log.assert(false, $"Oprator: {op} is not expected");
+//                   }
+//                   if (p.idx == snapshot_packet.idx) {
+//                      myFireEvent(new DataPacket(_current_table, _current_table, new BlueTable(_current_table.schema)));
+//                      break;
+//                   }
+//                }
+//             }
+//          }
+//       }
+
+
+//       private void _load_packets_from_current_file() {
+//          int curr_idx = _current_fs.Item1;
+//          FileStream curr_fs = _current_fs.Item2;
+//          _packets = new List<PlaybackPacket>();
+//          _packets.AddRange(StreamPersistorUtils.load_packets_from_file(curr_fs));
+//       }
+
+//       private bool current_file_all_replayed {
+//          get {
+//             bool rv = true;
+//             _packets.ForEach(p => rv = rv && p.replayed);
+//             return rv;
+//          }
+//       }
+
+//       protected override void myOnLastSubscriber() {
+//          var fs = _current_fs.Item2;
+//          if (fs != null) {
+//             fs.Close();
+//          }
+//          ViewStreamDatabase.instance.removeWorker(this);
+//       }
+
+//       // public event PlaybackStreamProgress onProgress;
+//       // public event Action onFileExhausted;
+//       private PlaybackPacket _locate_packet_to_replay() {
+//          foreach (var p in _packets) {
+//             // returns the first non replayed packet
+//             if (!p.replayed)
+//                return p;
+//          }
+//          //else
+//          {
+//             double secondsElapsed = new BinaryReader(myFileReader).ReadDouble();
+//             myNextPacketTime = myPlaybackStart + new TimeSpan(0, 0, 0, 0, (int)(1000 * secondsElapsed / myPlaybackSpeed));
+//             if (onProgress != null)
+//                onProgress(myFileReader.Position, myFileReader.Length);
+//          }
+//       }
+
+//       #region IWorker Members
+
+//       public void doWork() {
+//          if (_all_files_read)
+//             return;
+//          if (current_file_all_replayed) {
+//             _load_next_file();
+//          }
+//          PlaybackPacket curr_packet = _locate_packet_to_replay();
+//          if (curr_packet.idx == 0) { // first packet in the file, fire it rightaway 
+//             byte[] ops;
+//             Array row_array;
+//             // what a stupid interface!
+//             _deserilializer.deserialize(new Stream[] { new MemoryStream(curr_packet.data) }, out ops, out row_array);
+//             _fire_packet(ops, row_array);
+//             curr_packet.replay_timestamp = PersistorStream.epoch_now();
+//          } else {
+//             PlaybackPacket prev_packet = _get_previous_packet(curr_packet);
+//             // this is the time we need to fire (replay) this packet
+//             long time_to_fire = prev_packet.replay_timestamp + curr_packet.offset_from_previous;
+//             long now_epoch = PersistorStream.epoch_now();
+//             // fire when we are past the 
+//             if (now_epoch >= time_to_fire) {
+//                byte[] ops;
+//                Array row_array;
+//                _deserilializer.deserialize(new Stream[] { new MemoryStream(curr_packet.data) }, out ops, out row_array);
+//                _fire_packet(ops, row_array);
+//                curr_packet.replay_timestamp = PersistorStream.epoch_now();
+//             }
+//          }
+//       }
+
+//       private void _fire_packet(byte[] ops, Array row_array) {
+//          for (int i = 0; i < ops.Length; i++) {
+//             IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
+//             byte op = ops[i];
+//             if (op == StreamOpCode.create) {
+//                IBlueRowCollection full_table = rows_args[0];
+//                _current_table = new BlueTable(full_table);
+//                //_current_table.addRows(full_table);
+//                myFireEvent(new DataPacket(full_table, full_table, myLastValue == null ? new BlueTable(full_table.schema) :
+//                                           (myLastValue as DataPacket).newRows));
+//             } else if (op == StreamOpCode.multipleRemoveAdd) {
+//                //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
+//                IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
+//                IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
+//                BlueDataUtility.absorbChanges(_current_table, dels, adds);
+//                myFireEvent(new DataPacket(_current_table, adds, dels));
+//             } else {
+//                bts.Log.assert(false, $"Oprator: {op} is not expected");
+//             }
+
+//          }
+//       }
+
+//       private void _load_next_file() {
+//          var new_idx = _current_fs.Item1 + 1;
+//          if (new_idx >= _file_names.Count()) {
+//             _all_files_read = true;
+//             bts.Log.Gen.info("All files read");
+//             return;
+//          }
+//          var prev_fs = _current_fs.Item2;
+//          if (prev_fs != null)
+//             prev_fs.Close();
+//          var new_filename = _file_names.ElementAt(new_idx);
+//          bts.Log.Gen.info("PlaybackStream_load_next_file new_filename=: {new_filename}");
+//          var new_fs = new FileStream(new_filename, FileMode.Open, FileAccess.Read);
+//          _current_fs = new Tuple<int, FileStream>(new_idx, new_fs);
+//          _load_packets_from_current_file();
+//       }
+
+//       #endregion
+//    }
+// }
@@ -0,0 +1,68 @@
+// using System;
+// using System.Collections.Generic;
+// using System.Linq;
+// using System.Text;
+// using System.IO;
+
+// namespace BCG
+// {
+//    public class RecorderStream : ViewStream
+//    {
+//       private ViewStream mySource;
+//       private DateTime myStartTime;
+//       private FileStream myOutFile;
+//       private string myFileName;
+//       private IBlueDataSerializer mySerializer;
+
+//       public RecorderStream(ViewStream src, string recordFile)
+//       {
+//          mySource = src;
+//          myFileName = recordFile;
+
+//          myFileName = myFileName.Replace("%DATE%", DateTime.Now.ToString("yyyyMMdd_HHmmss")); 
+//       }
+
+//       protected override void myOnFirstSubscriber()
+//       {
+//          myStartTime = DateTime.Now;         
+//          mySource.onChange += new ChangeDelegate(mySource_onChange);
+//          myOutFile = new FileStream(myFileName, FileMode.OpenOrCreate);
+//       }
+
+//       protected override void myOnLastSubscriber()
+//       {
+//          mySource.onChange -= mySource_onChange;
+//          myOutFile.Close();
+//       }
+
+//       void mySource_onChange(object newValue)
+//       {
+//          DataPacket dp = (DataPacket)newValue;         
+
+//          BlueTable toWrite = new BlueTable(dp.schema);
+
+//          if (mySerializer == null)
+//          {
+//             mySerializer = new UniversalSerializer(dp.schema);
+//          }
+//          foreach (IBlueRow r in dp.newRows)
+//             toWrite.addRow(r);
+
+//          byte[] result;
+
+//          if (myLastValue == null)
+//             mySerializer.serialize(StreamOpCode.create, new IBlueRowCollection[] { dp.newRows }, out result);
+//          else
+//             mySerializer.serialize(StreamOpCode.multipleRemoveAdd, new IBlueRowCollection[] { dp.deletedRows, dp.addedRows }, out result);
+
+//          BinaryWriter bw = new BinaryWriter(myOutFile);
+//          //write down what time this happened
+//          bw.Write((DateTime.Now - myStartTime).TotalSeconds);
+//          bw.Write(result.Length);
+//          bw.Write(result);         
+//          myOutFile.Flush();
+
+//          myFireEvent(dp);
+//       }
+//    }
+// }
@@ -5,8 +5,7 @@ using System.IO;
 using BCG;

 namespace bts {
-   // this saves the updates as a Create on the same file, each update calls -save, which
-   // reqrites the whole data to the packet. The other side of this stream is the Loader
+   // this saves the total state in the filr provided in the cons
   [DataStreamUserRespresentation("saver")]
   public class SaverStream : ViewStream {
      private ViewStream _stream;
@@ -18,8 +17,7 @@ namespace bts {
         _filename = filename;
      }

-      protected override void myOnFirstSubscriber() =>
-         _stream.onChange += _handle_changes;
+      protected override void myOnFirstSubscriber() { _stream.onChange += new ChangeDelegate(_handle_changes); }

      void _handle_changes(object dp_obj) {
         DataPacket dp = dp_obj as DataPacket;
@@ -27,13 +25,13 @@ namespace bts {
         myFireEvent(dp_obj);
      }

-      protected override void myOnLastSubscriber() =>
-         _stream.onChange -= _handle_changes;
+      protected override void myOnLastSubscriber() { _stream.onChange -= new ChangeDelegate(_handle_changes); }

      protected void _save(DataPacket dp) {
         StreamIOMessage.CreateMessage cr_msg = new StreamIOMessage.CreateMessage(dp.newRows);
         byte[] str_bytes = _serializer.serialize(new StreamIOMessage(cr_msg));
         StreamSerializationUtils.write_to_file(str_bytes, _filename);
+
      }
   }
 }
@@ -3,7 +3,6 @@ using System.Collections.Generic;
 using System.Linq;
 using System.Globalization;
 using System.IO;
-using bts;


 namespace BCG {
@@ -37,12 +37,12 @@ namespace BCG.StreamWorld.Addition

      protected override void myOnFirstSubscriber()
      {
-         myBaseStream.onChange += myHandleChanges;
+         myBaseStream.onChange += new ChangeDelegate(myHandleChanges);
      }

      protected override void myOnLastSubscriber()
      {
-         myBaseStream.onChange -= myHandleChanges;
+         myBaseStream.onChange -= new ChangeDelegate(myHandleChanges);
         myBaseTable.clear();
         myStateTable.clear();
      }
@@ -228,13 +228,13 @@ namespace BCG
      #endregion

      #region overridden methods
-      protected override void myOnFirstSubscriber() =>
-         myParentStream.onChange += myHandleCallbacks;
+      protected override void myOnFirstSubscriber()
+      { myParentStream.onChange += new ChangeDelegate(myHandleCallbacks); }

      protected override void myOnLastSubscriber()
      {
         myLastFiredRows = null;
-         myParentStream.onChange -= myHandleCallbacks;
+         myParentStream.onChange -= new ChangeDelegate(myHandleCallbacks);
      }

      protected override void myFireEvent(object o)
@@ -1,22 +0,0 @@
-namespace bts.SilverPine {
-   using BCG;
-
-   using System;
-
-   public class NullFilterStream : ViewStream {
-      public NullFilterStream(ViewStream parent_str, string col_name, bool filter_out_doublenans = false) {
-         ViewStream rs = new FilterStream(parent_str,
-         r => {
-            bool is_dbnull = r.is_null(col_name);
-            if (is_dbnull) {
-               return false;
-            }
-            if (r.schema[col_name].type_code() == TypeCode.Double) {
-               return !Double.IsNaN(r.get<double>(col_name));
-            }
-            return true;
-         });
-         mySetRealSource(rs);
-      }
-   }
-}
@@ -36,11 +36,11 @@ namespace BCG
         get{return myIsFrozen;}
      }

-      protected override void myOnFirstSubscriber () =>
-         myStreamToFreeze.onChange += myCallBack;
+      protected override void myOnFirstSubscriber ()
+      {myStreamToFreeze.onChange += new DataStream.ChangeDelegate (myCallBack);}

-      protected override void myOnLastSubscriber () =>
-         myStreamToFreeze.onChange -= myCallBack;
+      protected override void myOnLastSubscriber ()
+      {myStreamToFreeze.onChange -= new DataStream.ChangeDelegate (myCallBack);}

      internal void myCallBack(object o)
      {
@@ -96,15 +96,15 @@ namespace BCG

      protected override void myOnFirstSubscriber()
      {
-         myLeftSource.onChange += myLeft_onChange;
-         myRightSource.onChange += myRight_onChange;
+         this.myLeftSource.onChange += new ChangeDelegate(myLeft_onChange);
+         this.myRightSource.onChange += new ChangeDelegate(myRight_onChange);
         this.myVsd.addWorker(this);
      }

      protected override void myOnLastSubscriber()
      {
-         myLeftSource.onChange -= myLeft_onChange;
-         myRightSource.onChange -= myRight_onChange;
+         this.myLeftSource.onChange -= new ChangeDelegate(myLeft_onChange);
+         this.myRightSource.onChange -= new ChangeDelegate(myRight_onChange);
         this.myVsd.removeWorker(this);
      }

@@ -1,102 +1,102 @@
-// using System;
-// using System.Collections.Generic;
-// using System.Linq;
-// using System.Text;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;

-// namespace BCG
-// {
+namespace BCG
+{
   
-//     public class AverageGroupsStream : ViewStream
-//     {
-//         public AverageGroupsStream(ViewStream input, string colToAverage, string keyColumn)
-//         {
+    public class AverageGroupsStream : ViewStream
+    {
+        public AverageGroupsStream(ViewStream input, string colToAverage, string keyColumn)
+        {
            
-//             myColToAverage = colToAverage;
-//             myInput = input;
-//             myKeyColumn = keyColumn;
-//         }
+            myColToAverage = colToAverage;
+            myInput = input;
+            myKeyColumn = keyColumn;
+        }

-//         private ViewStream myInput;
-//         private string myColToAverage;
-//         private BlueTableSchema mySchema;
-//         private string myKeyColumn;
+        private ViewStream myInput;
+        private string myColToAverage;
+        private BlueTableSchema mySchema;
+        private string myKeyColumn;

-//         protected override void myOnFirstSubscriber()
-//         {
-//             myInput.onChange += new ChangeDelegate(myInput_onChange);
-//         }
-//         protected override void myOnLastSubscriber()
-//         {
-//             myInput.onChange -= myInput_onChange;
-//         }
-//         private void myInput_onChange(object newValue)
-//         {
-//             if (mySchema == null)
-//                 myBuildSchema(((DataPacket)newValue).schema);
+        protected override void myOnFirstSubscriber()
+        {
+            myInput.onChange += new ChangeDelegate(myInput_onChange);
+        }
+        protected override void myOnLastSubscriber()
+        {
+            myInput.onChange -= myInput_onChange;
+        }
+        private void myInput_onChange(object newValue)
+        {
+            if (mySchema == null)
+                myBuildSchema(((DataPacket)newValue).schema);

-//             IBlueRowCollection newRows = myEvaluate((DataPacket)newValue);
+            IBlueRowCollection newRows = myEvaluate((DataPacket)newValue);

-//             if (myLastValue == null)
-//                 myFireEvent(new DataPacket(newRows, newRows, new BlueTable(mySchema)));
-//             else
-//             {
-//                 IBlueRowCollection added, deleted;
-//                 BlueDataUtility.computeDifferences(newRows, ((DataPacket) myLastValue).newRows, out deleted, out added);
-//                 myFireEvent(new DataPacket(newRows, added, deleted));
-//             }
-//         }
+            if (myLastValue == null)
+                myFireEvent(new DataPacket(newRows, newRows, new BlueTable(mySchema)));
+            else
+            {
+                IBlueRowCollection added, deleted;
+                BlueDataUtility.computeDifferences(newRows, ((DataPacket) myLastValue).newRows, out deleted, out added);
+                myFireEvent(new DataPacket(newRows, added, deleted));
+            }
+        }

-//         private void myBuildSchema(IBlueTableSchema schema)
-//         {
-//             List<ColumnInfo> columns = new List<ColumnInfo>();
+        private void myBuildSchema(IBlueTableSchema schema)
+        {
+            List<ColumnInfo> columns = new List<ColumnInfo>();

-//             foreach (ColumnInfo ci in schema.columnInfo)
-//                 columns.Add(ci);
+            foreach (ColumnInfo ci in schema.columnInfo)
+                columns.Add(ci);

-//             columns.Add(new ColumnInfo("average_"+myColToAverage, columns.Count, typeof(double)));
-//             mySchema = new BlueTableSchema("average_of_" + schema.tableName, columns.ToArray());
-//         }
+            columns.Add(new ColumnInfo("average_"+myColToAverage, columns.Count, typeof(double)));
+            mySchema = new BlueTableSchema("average_of_" + schema.tableName, columns.ToArray());
+        }



-//         private IBlueRowCollection myEvaluate(DataPacket packet)
-//         {
+        private IBlueRowCollection myEvaluate(DataPacket packet)
+        {
         
-//             IDictionary<string, double> colToRunningTotal = new Dictionary<string, double>();
-//             IDictionary<string, int> colToCount = new Dictionary<string, int>();
-//             IDictionary<string, IBlueRow> colToLastRow = new Dictionary<string, IBlueRow>();
+            IDictionary<string, double> colToRunningTotal = new Dictionary<string, double>();
+            IDictionary<string, int> colToCount = new Dictionary<string, int>();
+            IDictionary<string, IBlueRow> colToLastRow = new Dictionary<string, IBlueRow>();

-//             foreach (IBlueRow r in packet.newRows)
-//             {
-//                 double thisVal = Convert.ToDouble(r[myColToAverage]);
+            foreach (IBlueRow r in packet.newRows)
+            {
+                double thisVal = Convert.ToDouble(r[myColToAverage]);

-//                 string keyVal = r[myKeyColumn].ToString();
-//                 if (!colToCount.ContainsKey(keyVal))
-//                 {
-//                     colToRunningTotal[keyVal] = thisVal;
-//                     colToCount[keyVal] = 1;
-//                 }
-//                 else
-//                 {
-//                     colToRunningTotal[keyVal] += thisVal;
-//                     colToCount[keyVal] += 1;
-//                 }
-//                 colToLastRow[keyVal] = r;
-//             }
+                string keyVal = r[myKeyColumn].ToString();
+                if (!colToCount.ContainsKey(keyVal))
+                {
+                    colToRunningTotal[keyVal] = thisVal;
+                    colToCount[keyVal] = 1;
+                }
+                else
+                {
+                    colToRunningTotal[keyVal] += thisVal;
+                    colToCount[keyVal] += 1;
+                }
+                colToLastRow[keyVal] = r;
+            }

-//             BlueTable result = new BlueTable(mySchema);
-//             foreach (string key in colToCount.Keys)
-//             {
-//                 object[] values = new object[mySchema.columnInfo.Length];
+            BlueTable result = new BlueTable(mySchema);
+            foreach (string key in colToCount.Keys)
+            {
+                object[] values = new object[mySchema.columnInfo.Length];

-//                 colToLastRow[key].CopyTo(values, 0);
-//                 values[values.Length - 1] = packet.newRows.Count > 0 ? colToRunningTotal[key] / packet.newRows.Count : 0.0;
-//                 result.addRow(new BlueRow(mySchema, values));
-//             }
+                colToLastRow[key].CopyTo(values, 0);
+                values[values.Length - 1] = packet.newRows.Count > 0 ? colToRunningTotal[key] / packet.newRows.Count : 0.0;
+                result.addRow(new BlueRow(mySchema, values));
+            }

-//             return result;
-//         }
+            return result;
+        }

      
-//     }
-// }
+    }
+}
@@ -1,31 +0,0 @@
-namespace bts.SilverPine {
-   using BCG;
-
-   using System;
-   using System.Collections.Generic;
-   using System.Linq;
-
-   [DataStreamUserRespresentation("Average")]
-   public class AverageNonIncGroupsStream : ViewStream {
-      public AverageNonIncGroupsStream(ViewStream parent_stream, string inp_col_name, params string[] group_by_cols) {
-         NonIncGroupByStream rs = new NonIncGroupByStream(parent_stream, () => group_by_cols,
-                                                         (g_rows) => _find_mean(inp_col_name, g_rows));
-         mySetRealSource(rs);
-      }
-
-      private List<IBlueRow> _find_mean(string inp_col_name, List<IBlueRow> g_rows) {
-         IBlueRow inp_row = g_rows.ElementAt(0);
-         IEnumerable<double> inp_vals = BlueDataUtility.get_colvalues<double>(g_rows, inp_col_name);
-         double sum = 0.0;
-         inp_vals.ToList().ForEach(d => sum += d);
-         string out_col_name = $"average_{inp_col_name}";
-         double average = sum / inp_vals.Count();
-         IBlueTableSchema schema_out = inp_row.schema.add_column(TypeCode.Double, out_col_name);
-         MutableBlueRow out_row = new MutableBlueRow(schema_out, inp_row);
-         out_row.setValue(out_col_name, average);
-         List<IBlueRow> rv = new List<IBlueRow>();
-         rv.Add(out_row);
-         return rv;
-      }
-   }
-}
@@ -175,7 +175,7 @@ namespace BCG {
         //gets resubscribed, we start the process of building data again, and treat the callback as the first 
         //callback.
         myInit();
-         myStream.onChange -= myChangeHandler;
+         myStream.onChange -= new DataStream.ChangeDelegate(myChangeHandler);
      }

      protected override void myInit() {
@@ -10,6 +10,6 @@ namespace BCG {
        }

        public MaxStream(ViewStream stream, string outputCol)
-           : this(new AddColumnsStream(stream, "fake_col: 0"), outputCol, new string[] { "fake_col" }) { }
+           : this(new AddColumnsStream(stream, "fakeCol=0"), outputCol, new string[] { "fakeCol" }) { }
    }
 }
@@ -82,12 +82,12 @@ namespace BCG

      protected override void myOnFirstSubscriber() 
      {
-         myStream.onChange += myChangeHandler;
+         myStream.onChange += new DataStream.ChangeDelegate (myChangeHandler);
      }

      protected override void myOnLastSubscriber () 
      {
-         myStream.onChange -= myChangeHandler;
+         myStream.onChange -= new DataStream.ChangeDelegate (myChangeHandler);
      }
   }
 }
@@ -1,37 +1,13 @@
 using System;
 using System.Collections.Generic;
 using System.Diagnostics;
-using System.Linq;

 using bts;

-namespace BCG {
-   public class NonIncGroupByStream : ViewStream {
-      private class DelNonIncGroupByEvaluator : INonIncGroupByEvaluator {
-         Func<string[]> _groupby_cols_func;
-         Func<List<IBlueRow>, List<IBlueRow>> _evaluate_func;
-         public DelNonIncGroupByEvaluator(Func<string[]> groupby_cols_func,
-                                          Func<List<IBlueRow>, List<IBlueRow>> evaluate_func) {
-            _groupby_cols_func = groupby_cols_func;
-            _evaluate_func = evaluate_func;
-         }
-         public IBlueRowCollection evaluate(IDictionary<int, List<IBlueRow>> grouped_rows) {
-            BlueTable rv = null;
-            foreach (var kv in grouped_rows) {
-               List<IBlueRow> g_rows = kv.Value;
-               List<IBlueRow> out_rows = _evaluate_func.Invoke(g_rows);
-               if (out_rows != null && out_rows.Count > 0) {
-                  rv = rv ?? new BlueTable(out_rows.ElementAt(0).schema);
-               }
-               rv.addRows(out_rows);
-            }
-            return rv;
-         }
-         public string[] get_group_by_cols() {
-            return _groupby_cols_func();
-         }
-
-      }
+namespace BCG
+{
+   public class NonIncGroupByStream : ViewStream
+   {
      private ViewStream _main_stream;
      private INonIncGroupByEvaluator _groupby_func;
      private BlueRowHasher _row_hasher;
@@ -39,29 +15,24 @@ namespace BCG {
      private Stopwatch _timer = new();
      private int _n_total_fires = 0, _n_expensive_fires = 0;

-      public NonIncGroupByStream(ViewStream main_stream, INonIncGroupByEvaluator groupby_func) {
+      public NonIncGroupByStream(ViewStream main_stream, INonIncGroupByEvaluator groupby_func)
+      {
         _main_stream = main_stream;
         _groupby_func = groupby_func;
         _groupby_cols = _groupby_func.get_group_by_cols();
         _row_hasher = new BlueRowHasher(_groupby_cols.Length);
      }

-      public NonIncGroupByStream(ViewStream main_stream, Func<string[]> groupby_cols_func,
-                                 Func<List<IBlueRow>, List<IBlueRow>> evaluate_func)
-                                 : this(main_stream, new DelNonIncGroupByEvaluator(groupby_cols_func, evaluate_func)) { }
-
-
      protected override void myOnFirstSubscriber() { _main_stream.onChange += _main_stream_onchange; }

      protected override void myOnLastSubscriber() { _main_stream.onChange -= _main_stream_onchange; }

-      private void _main_stream_onchange(object nv) {
+      private void _main_stream_onchange(object nv)
+      {
         DataPacket dp = nv as DataPacket;
         ///grouping together
         _timer.Restart();
-         IDictionary<int, List<IBlueRow>> grouped_rows = (_groupby_cols == null || _groupby_cols.Length == 0) ?
-         new Dictionary<int, List<IBlueRow>>() { { 0, dp.newRows.to_list() } } :
-         BlueDataUtility.group_together(dp.newRows, _groupby_cols, ref _row_hasher);
+         IDictionary<int, List<IBlueRow>> grouped_rows = BlueDataUtility.group_together (dp.newRows, _groupby_cols, ref _row_hasher);
         _timer.Stop();
         var groupElapsedMs = _timer.ElapsedMilliseconds;
         if (groupElapsedMs >= 50) {
@@ -77,13 +48,14 @@ namespace BCG {
            return;
         }
         var applyElapsedMs = _timer.ElapsedMilliseconds;
-         _n_total_fires++;
-         if (applyElapsedMs >= 50) {
+         _n_total_fires++;         
+         if (applyElapsedMs >= 50)
+         {
            _n_expensive_fires++;
            Log.DataFlow.info(
               $"Time to evaluate {grouped_rows.Count} groups {_timer.ElapsedMilliseconds}ms, ratio {_n_expensive_fires}/{_n_total_fires}");
-         }
-         myFireEvent(new DataPacket(total_state, total_state, myLastValue == null ? new BlueTable(total_state.schema) : (myLastValue as DataPacket).newRows));
+         }         
+         myFireEvent(new DataPacket(total_state, total_state, myLastValue == null ? new BlueTable(total_state.schema) : (myLastValue as DataPacket).newRows));             
      }
   }
 }
@@ -31,15 +31,15 @@ namespace BCG
      {
         myLeftOperand.master = this;
         myRightOperand.master = this;
-         myLeftStream.onChange += myLeftOperand.changeHandler;
-         myRightStream.onChange += myRightOperand.changeHandler;
+         myLeftStream.onChange += new DataStream.ChangeDelegate (myLeftOperand.changeHandler);
+         myRightStream.onChange += new DataStream.ChangeDelegate (myRightOperand.changeHandler);
      }
      
      protected override void myOnLastSubscriber () 
      {
         myInit ();
-         myLeftStream.onChange -= myLeftOperand.changeHandler;
-         myRightStream.onChange -= myRightOperand.changeHandler;
+         myLeftStream.onChange -= new DataStream.ChangeDelegate (myLeftOperand.changeHandler);
+         myRightStream.onChange -= new DataStream.ChangeDelegate (myRightOperand.changeHandler);
      }

      public JoinOperand getOtherOperand (JoinOperand jo)
@@ -116,7 +116,7 @@ namespace BCG
               IDictionary  map = resolver.getMap ();
               map["id"] = "id";
               myPs = new ProjectStream (myIjs, map);
-               myPs.onChange += myProjectChangeHandler;
+               myPs.onChange += new DataStream.ChangeDelegate (myProjectChangeHandler);
               return;
            }
            else
@@ -129,26 +129,26 @@ namespace BCG
      private void myProjectChangeHandler (object o)
      {
         if (myIjs.myOnChangeDelegate.GetInvocationList().Length > 1)
-            myIjs.onChange -= myInternalJoinChangeHandler;
+            myIjs.onChange -= new DataStream.ChangeDelegate (myInternalJoinChangeHandler);
         myFireEvent ((o as ICloneable).Clone());
         
      }

      protected override void myOnFirstSubscriber ()
      {
-         myIjs.onChange += myInternalJoinChangeHandler;
+         myIjs.onChange += new DataStream.ChangeDelegate (myInternalJoinChangeHandler);
      }

      protected override void myOnLastSubscriber ()
      {
         // how does this relate to the -= in myProjectChangeHandler?
-         myIjs.onChange -= myInternalJoinChangeHandler;
+         myIjs.onChange -= new DataStream.ChangeDelegate (myInternalJoinChangeHandler);

         // if there is a  project after the ijs we need to 
         // unsubscribe to it otherwise references will be kept and 
         // a memory leak will occur
         if (myPs != null)
-            myPs.onChange -= myProjectChangeHandler;
+            myPs.onChange -= new DataStream.ChangeDelegate (myProjectChangeHandler);
         myPs = null;
      }

@@ -5,10 +5,11 @@ using mbts;
 using System;
 using System.Collections.Generic;
 using System.Diagnostics;
-using System.Security.Cryptography;

-namespace BCG {
-   public class NonIncJoinBookkeeper : IDisposable {
+namespace BCG
+{
+   public class NonIncJoinBookkeeper : IDisposable
+   {
      public Action on_new_data;
      private readonly NonIncJoinSpec _join_spec;
      private readonly int _join_position;
@@ -25,38 +26,41 @@ namespace BCG {

      private int[] _join_col_indices = null,
                     _data_col_indices = null;
-
-      public NonIncJoinBookkeeper(NonIncJoinSpec the_join_spec, int join_position, int start_index_in_output, BlueRowHasher row_hasher) {
+      
+      public NonIncJoinBookkeeper(NonIncJoinSpec the_join_spec, int join_position, int start_index_in_output, BlueRowHasher row_hasher)
+      {
         _join_spec = the_join_spec;
         _join_position = join_position;
         _start_index_in_output = start_index_in_output;
         _row_hasher = row_hasher;
         _is_left_most = _join_position == 0;
         _is_tag_along = _join_spec.join_col_names.Length == 0;
-      }
-
-      public void init() {
         _join_spec.stream.onChange += _on_stream_cb;
      }

-      public void _on_stream_cb(object dp_obj) {
-         Log.Gen.info($"NonIncJoinBookkeeper._on_stream_cb dp_obj: {dp_obj}");
+      public void _on_stream_cb(object dp_obj)
+      {
         var timer = Stopwatch.StartNew();
         DataPacket dp = (DataPacket)dp_obj;
-         if (_schema == null) {
+         if (_schema == null)
+         {
            _schema = dp.schema;
            _join_col_indices = BlueDataUtility.get_column_indices(_schema, _join_spec.join_col_names);
            _data_col_indices = BlueDataUtility.get_column_indices(_schema, _join_spec.data_col_names);
         }

-         if (_is_left_most) {
+         if (_is_left_most)
+         {
            _row_list.Clear();
            foreach (IBlueRow r in dp.newRows)
-               _row_list.Add(new Pair<int, IBlueRow>(_row_hasher.get_hash(r, _join_col_indices), r));
-         } else {
+               _row_list.Add(new Pair<int, IBlueRow>(_row_hasher.get_hash(r, _join_col_indices), r));            
+         }
+         else
+         {
            if (_is_tag_along)
               _solitary_row = BlueDataUtility.extractRow(dp.newRows, 0);
-            else {
+            else
+            {
               _row_map.Clear();
               foreach (IBlueRow r in dp.newRows)
                  _row_map[_row_hasher.get_hash(r, _join_col_indices)] = r;
@@ -75,25 +79,31 @@ namespace BCG {

      public bool is_ready { get { return _schema != null; } }

-      public void constribute_cols_to_schema(List<ColumnInfo> cis) {
+      public void constribute_cols_to_schema(List<ColumnInfo> cis)
+      {
         for (int i = 0; i < _join_spec.data_col_names.Length; i++)
            cis.Add(new ColumnInfo(_join_spec.projected_col_names[i], cis.Count, _schema[_join_spec.data_col_names[i]].colType));
      }

      public List<Pair<int, IBlueRow>> get_list() { return _row_list; }

-      public void copy_master_row_to_output_row(int row_index, object[] output_row) {
+      public void copy_master_row_to_output_row(int row_index, object[] output_row)
+      {
         IBlueRow this_row = _row_list[row_index].second;
         BlueDataUtility.copy_to_row(this_row, _data_col_indices, output_row, _start_index_in_output);
      }

-      public void copy_matching_row_to_output_row(int row_hash, object[] output_row) {
-         if (_is_tag_along) {
+      public void copy_matching_row_to_output_row(int row_hash, object[] output_row)
+      {
+         if (_is_tag_along)
+         {
            if (_solitary_row != null)
               BlueDataUtility.copy_to_row(_solitary_row, _data_col_indices, output_row, _start_index_in_output);
            else
               copy_dbnull_to_output_row(output_row);
-         } else {
+         }
+         else
+         {
            IBlueRow this_row = null;
            if (_row_map.TryGetValue(row_hash, out this_row))
               BlueDataUtility.copy_to_row(this_row, _data_col_indices, output_row, _start_index_in_output);
@@ -102,7 +112,8 @@ namespace BCG {
         }
      }

-      private void copy_dbnull_to_output_row(object[] output_row) {
+      private void copy_dbnull_to_output_row(object[] output_row)
+      {
         for (int i = 0; i < _data_col_indices.Length; i++)
            output_row[i + _start_index_in_output] = DBNull.Value;
      }
@@ -3,62 +3,63 @@ using System.Collections.Generic;
 using System.Linq;
 using System.Text;
 using System.Threading.Tasks;
-
 using mbts;

-namespace BCG {
-   public class NonIncJoinSpec {
+namespace BCG
+{
+   public class NonIncJoinSpec
+   {
      public ViewStream stream;
-      // join col names are the names on which the streams are joined (for equality),
-      // data col names are the names in the original stream
-      // projected col naes are their new names, ie if its x:l_x, then x is in the 
-      // data_col_names and l_x in projected cols
      public string[] join_col_names, data_col_names, projected_col_names;

      public NonIncJoinSpec() { }

      public NonIncJoinSpec(ViewStream the_stream, string[] the_join_col_names, string[] the_data_col_names)
-         : this(the_stream, the_join_col_names, the_data_col_names, the_data_col_names) { }
+         : this (the_stream, the_join_col_names, the_data_col_names, the_data_col_names)
+      { }

-      public NonIncJoinSpec(ViewStream the_stream, string[] the_join_col_names, string[] the_data_col_names, string[] the_projected_col_names) {
+      public NonIncJoinSpec(ViewStream the_stream, string[] the_join_col_names, string[] the_data_col_names, string[] the_projected_col_names)
+      {
         stream = the_stream;
         join_col_names = the_join_col_names;
         data_col_names = the_data_col_names;
         projected_col_names = the_projected_col_names;
      }

-      public static NonIncJoinSpec[] from_triplet(params Triplet<ViewStream, string, string>[] join_specs) {
+      public static NonIncJoinSpec[] from_triplet(params Triplet<ViewStream, string, string>[] join_specs)
+      {
         List<NonIncJoinSpec> rv = new List<NonIncJoinSpec>();
-         foreach (Triplet<ViewStream, string, string> r in join_specs) {
-            string[] keys = String.IsNullOrEmpty(r.second) ? new string[] { } : r.second.Split(' ');
+         foreach (Triplet<ViewStream, string, string> r in join_specs)
+         {
+            string[] keys = String.IsNullOrEmpty (r.second) ? new string[] {} : r.second.Split(' ');
            string[] visible_cols = r.third.Split(' ');
            List<string> data_col_names = new List<string>(), mapped_col_names = new List<string>();
-            foreach (string s in visible_cols) {
+            foreach (string s in visible_cols)
+            {
               string[] toks = s.Split(':');
-               if (toks.Length == 1) {
+               if (toks.Length == 1)
+               {
                  data_col_names.Add(toks[0]);
                  mapped_col_names.Add(toks[0]);
-               } else if (toks.Length == 2) {
+               }
+               else if (toks.Length == 2)
+               {
                  data_col_names.Add(toks[0]);
                  mapped_col_names.Add(toks[1]);
               }
            }
-            rv.Add(new NonIncJoinSpec(r.first, keys, data_col_names.ToArray(), mapped_col_names.ToArray()));
+            rv.Add (new NonIncJoinSpec (r.first, keys, data_col_names.ToArray (), mapped_col_names.ToArray()));
         }
-         return rv.ToArray();
-      }
-
-      public override string ToString() {
-         Func<string[], string> arr_txt = (a) => {
-            string rv = "";
-            foreach (string s in a) {
-               rv += "{s} ";
-            }
-            rv = rv.Trim();
-            return rv;
-         };
-         string rv = $"stream: {stream} join_col_names: {arr_txt(join_col_names)} data_col_names: {arr_txt(data_col_names)} projected_col_names: {arr_txt(projected_col_names)}";
-         return rv;
+         return rv.ToArray ();
      }
   }
+
+   //[DataStreamUserRespresentation("ninctest")]
+   //public class NonIncJoinTestStream : ViewStream
+   //{
+   //   public NonIncJoinTestStream (string key_cols, string data_cols)
+   //   {
+   //      NonIncJoinSpec.from_triplet(new Triplet<ViewStream, string, string>(null, key_cols, data_cols));
+   //   }
+   //}
 }
@@ -6,9 +6,11 @@ using System;
 using System.Collections.Generic;
 using System.Diagnostics;

-namespace BCG {
+namespace BCG
+{
   [DataStreamUserRespresentation("fastJoin")]
-   public class NonIncJoinStream : ViewStream, BCG.Utilities.General.IWorker {
+   public class NonIncJoinStream : ViewStream, BCG.Utilities.General.IWorker
+   {
      private readonly NonIncJoinSpec[] _join_specs;
      private readonly INonIncJoinInlineMapper _output_mapper;
      private readonly BlueRowHasher _row_hasher;
@@ -26,19 +28,24 @@ namespace BCG {
               {
                  new Triplet<ViewStream, string, string> (left, left_join_cols, left_proj),
                  new Triplet<ViewStream, string, string> (right, right_join_cols, right_proj)
-               }) { }
+               })
+      { }

      public NonIncJoinStream(params Triplet<ViewStream, string, string>[] join_specs)
-         : this(NonIncJoinSpec.from_triplet(join_specs), null, 0) { }
+         : this(NonIncJoinSpec.from_triplet(join_specs), null, 0)
+      { }

      public NonIncJoinStream(ViewStreamDatabase vsd, INonIncJoinInlineMapper output_mapper, int throttles_ms,
                              params Triplet<ViewStream, string, string>[] join_specs)
-         : this(vsd, NonIncJoinSpec.from_triplet(join_specs), output_mapper, throttles_ms) { }
+         : this(vsd, NonIncJoinSpec.from_triplet(join_specs), output_mapper, throttles_ms)
+      { }

      public NonIncJoinStream(NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper output_mapper, int throttles_ms = 0)
-         : this(ViewStreamDatabase.instance, join_specs, output_mapper, throttles_ms) { }
+         : this(ViewStreamDatabase.instance, join_specs, output_mapper, throttles_ms)
+      { }

-      public NonIncJoinStream(ViewStreamDatabase vsd, NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper output_mapper, int throttles_ms = 0) {
+      public NonIncJoinStream(ViewStreamDatabase vsd, NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper output_mapper, int throttles_ms = 0)
+      {
         _join_specs = join_specs;
         _output_mapper = output_mapper;
         _row_hasher = new BlueRowHasher(_join_specs[0].join_col_names.Length);
@@ -46,11 +53,12 @@ namespace BCG {
         _vsd = vsd;
      }

-      protected override void myOnFirstSubscriber() {
+      protected override void myOnFirstSubscriber()
+      {
         int data_col_no = 1;
-         for (int jp = 0; jp < _join_specs.Length; jp++) {
+         for (int jp = 0; jp < _join_specs.Length; jp++)
+         {
            NonIncJoinBookkeeper s = new NonIncJoinBookkeeper(_join_specs[jp], jp, data_col_no, _row_hasher);
-            Log.Gen.info($"NonIncJoinStream.myOnFirstSubscriber jp: {jp} spec: {_join_specs[jp]}");
            data_col_no += _join_specs[jp].data_col_names.Length;
            s.on_new_data += _stream_callback;
            _book_keeping_data.Add(s);
@@ -58,11 +66,10 @@ namespace BCG {
         if (_output_mapper != null)
            _output_mapper.onMapperChange += _on_mapper_change;
         _vsd.addWorker(this);
-         foreach (var bkd in _book_keeping_data)
-            bkd.init();
      }

-      protected override void myOnLastSubscriber() {
+      protected override void myOnLastSubscriber()
+      {
         foreach (NonIncJoinBookkeeper d in _book_keeping_data)
            d.Dispose();
         if (_output_mapper != null)
@@ -70,26 +77,30 @@ namespace BCG {
         _vsd.removeWorker(this);
      }

-      private void _stream_callback() {
-         Log.Gen.info("NonIncJoinStream._stream_callback received callback");
+      private void _stream_callback()
+      {
         _is_dirty = true;
         doWork();
      }

-      private void _on_mapper_change() {
+      private void _on_mapper_change()
+      {
         _is_dirty = true;
         doWork();
      }

-      private bool _is_ready() {
+      private bool _is_ready()
+      {
         bool rv = true;
         foreach (NonIncJoinBookkeeper d in _book_keeping_data)
            rv &= d.is_ready;
         return rv;
      }

-      private void _construct_schema() {
-         if (_output_schema == null) {
+      private void _construct_schema()
+      {
+         if (_output_schema == null)
+         {
            List<ColumnInfo> cis = new List<ColumnInfo>(new ColumnInfo[] { new ColumnInfo("id", 0, typeof(int)) });
            foreach (NonIncJoinBookkeeper d in _book_keeping_data)
               d.constribute_cols_to_schema(cis);
@@ -98,23 +109,27 @@ namespace BCG {
      }

      private int _n_total_fires = 0, _n_expensive_fires = 0;
-      private void _form_and_fire() {
-         Log.Gen.info($"NonIncJoinStream._form_and_fire _is_ready(): {_is_ready()}");
-         if (_is_ready()) {
+      private void _form_and_fire()
+      {
+         if (_is_ready())
+         {
            _construct_schema();
            var timer = Stopwatch.StartNew();
            IList<Pair<int, IBlueRow>> left_row_list = _book_keeping_data[0].get_list();
            List<object[]> output_rows = new List<object[]>();

-            for (int i = 0; i < left_row_list.Count; i++) {
+            for (int i = 0; i < left_row_list.Count; i++)
+            {
               object[] or = new object[_output_schema.columnInfo.Length];
               or[0] = i;
               output_rows.Add(or);
            }

-            for (int ji = 0; ji < _book_keeping_data.Count; ji++) {
+            for (int ji = 0; ji < _book_keeping_data.Count; ji++)
+            {
               NonIncJoinBookkeeper bk_data = _book_keeping_data[ji];
-               for (int i = 0; i < left_row_list.Count; i++) {
+               for (int i = 0; i < left_row_list.Count; i++)
+               {
                  if (ji == 0)
                     bk_data.copy_master_row_to_output_row(i, output_rows[i]);
                  else
@@ -124,20 +139,19 @@ namespace BCG {
            IBlueRowCollection dp_output_rows = _apply_map(new SequentialIDBlueTable(_output_schema, output_rows));
            timer.Stop();
            _n_total_fires++;
-            //if (timer.ElapsedMilliseconds >= 250) {
-            _n_expensive_fires++;
-            string log_msg = $"Join and map time for {_book_keeping_data.Count} "
-               + $"tables {timer.Elapsed.TotalMilliseconds} msecs, ratio {_n_expensive_fires}/{_n_total_fires}";
-            Log.Gen.info(log_msg);
-            //}
-            var dp_out = BlueDataUtility.build_dp_inc(dp_output_rows, myLastValue as DataPacket);
-            myFireEvent(dp_out);
-            //myFireEvent(new DataPacket(dp_output_rows, dp_output_rows, myLastValue == null ? new BlueTable(_output_schema) : (myLastValue as DataPacket).newRows));
+            if (timer.ElapsedMilliseconds >= 250) {
+               _n_expensive_fires++;
+               string log_msg = $"Join and map time for {_book_keeping_data.Count} "
+                  + $"tables {timer.Elapsed} ratio {_n_expensive_fires}/{_n_total_fires}";
+            }
+            myFireEvent(new DataPacket(dp_output_rows, dp_output_rows, myLastValue == null ? new BlueTable(_output_schema) : (myLastValue as DataPacket).newRows));
         }
      }

-      private IBlueRowCollection _apply_map(IBlueRowCollection unmapped_rows) {
-         if (_output_mapper != null) {
+      private IBlueRowCollection _apply_map(IBlueRowCollection unmapped_rows)
+      {
+         if (_output_mapper != null)
+         {
            IBlueRowCollection output_rows;
            var timer = Stopwatch.StartNew();
            output_rows = _output_mapper.map_rows(unmapped_rows);
@@ -146,14 +160,17 @@ namespace BCG {
               Log.Gen.info($"Time to map {unmapped_rows.Count} rows into {output_rows.Count} rows {timer.Elapsed}");
            }
            return output_rows;
-         } else {
+         }
+         else
+         {
            return unmapped_rows;
         }
      }

-      public void doWork() {
-         Log.Gen.info($"NonIncJoinStream.doWork _last_fire_time: {_last_fire_time} _is_dirty: {_is_dirty}");
-         if (DateTime.Now - _last_fire_time >= _throttle_period && _is_dirty) {
+      public void doWork()
+      {
+         if (DateTime.Now - _last_fire_time >= _throttle_period && _is_dirty)
+         {
            _is_dirty = false;
            _last_fire_time = DateTime.Now;
            _form_and_fire();
@@ -181,8 +181,8 @@ namespace BCG

       protected override void myOnFirstSubscriber()
       {
-           myVLeft.onChange += myVLeft_onChange;
-           myVRight.onChange += myVRight_onChange;
+           myVLeft.onChange += new ChangeDelegate(myVLeft_onChange);
+           myVRight.onChange += new ChangeDelegate(myVRight_onChange);
       }

       void myVRight_onChange(object newValue)
@@ -1,68 +0,0 @@
-namespace bts {
-   using BCG;
-   using BCG.Utilities.General;
-
-   using System;
-   using System.Diagnostics;
-
-   using ZstdSharp.Unsafe;
-
-   // functon that joins the 2 rows and returns the outputput row. The function
-   // can return a null which means the 2 rows dont need to be joined
-   using JoinFunc = System.Func<BCG.IBlueRow, BCG.IBlueRow, BCG.IBlueRow>;
-   public class ProximityJoinStream : ViewStream, IWorker {
-      private ViewStream _lvs, _rvs;
-      private JoinFunc _join_func;
-      private bool _dirty = false;
-      private Stopwatch _stopwatch = new Stopwatch();
-      private ViewStreamDatabase _vsd = ViewStreamDatabase.instance;
-      public ProximityJoinStream(ViewStream lvs, ViewStream rvs, JoinFunc join_func) {
-         _lvs = lvs;
-         _rvs = rvs;
-         _join_func = join_func;
-      }
-
-      protected override void myOnFirstSubscriber() {
-         _lvs.onChange += _handle_data;
-         _rvs.onChange += _handle_data;
-         _vsd.addWorker(this);
-      }
-
-      protected override void myOnLastSubscriber() {
-         _lvs.onChange -= _handle_data;
-         _rvs.onChange -= _handle_data;
-         _vsd.removeWorker(this);
-      }
-
-      private void _handle_data(object dp_obj) {
-         _dirty = true;
-         doWork();
-      }
-
-      public void doWork() {
-         Func<bool> all_data_revcd = () => {
-            return _lvs.rows != null && _rvs.rows != null;
-         };
-         _stopwatch.Start();
-         if (_dirty && all_data_revcd()) {
-            _dirty = false;
-            BlueTable rv = null;
-            foreach (IBlueRow lr in _lvs.rows) {
-               foreach (IBlueRow rr in _rvs.rows) {
-                  IBlueRow resr = _join_func(lr, rr);
-                  if (resr == null)
-                     continue;
-                  rv = rv ?? new BlueTable(resr.schema);
-                  rv.addRow(resr);
-               }
-            }
-            _stopwatch.Stop();
-            Log.Gen.info($"ProximityJoinStream left_count: {_lvs.rows.Count} right_count: {_rvs.rows.Count} result_count: {rv.Count} time: {_stopwatch.ElapsedMilliseconds} msecs");
-            var dp = BlueDataUtility.build_dp_inc(rv, myLastValue as DataPacket);
-            myFireEvent(dp);
-         }
-
-      }
-
-   }
-}
@@ -1,47 +0,0 @@
-namespace bts.SilverPine {
-   using BCG;
-
-   using System;
-   using System.CodeDom;
-   using System.Configuration;
-   using System.Reactive.Disposables;
-
-   public class TimeProximityJoinStream : ViewStream {
-      private int _id = 0;
-
-      public TimeProximityJoinStream(ViewStream ls, string l_time_col, ViewStream rs, string r_time_col,
-                                       string pre_join_period, string post_join_period) :
-            this(ls, l_time_col, rs, r_time_col, TimeBasedFilter.parse_duration(pre_join_period),
-            TimeBasedFilter.parse_duration(post_join_period)) { }
-
-
-      public TimeProximityJoinStream(ViewStream ls, string l_time_col, ViewStream rs, string r_time_col,
-                                       TimeSpan pre_join_period, TimeSpan post_join_period) {
-         Func<IBlueRow, IBlueRow, IBlueRow> create_row = (lr, rr) => {
-            var id = 0;
-            lock (this) {
-               id = _id++;
-            }
-            IBlueRow rv = BlueDataUtility.create_composite_row(lr, rr, id);
-            return rv;
-         };
-         Func<IBlueRow, IBlueRow, IBlueRow> join_func = (lr, rr) => {
-            DateTime ldt = lr.get_datetime(l_time_col);
-            DateTime rdt = rr.get_datetime(r_time_col);
-            TimeSpan diff = ldt - rdt;
-            if (ldt < rdt) {
-               if (diff.Duration() > pre_join_period) {
-                  return null;
-               }
-            } else {
-               if (diff > post_join_period)
-                  return null;
-            }
-            return create_row(lr, rr);
-         };
-
-         ViewStream rstr = new ProximityJoinStream(ls, rs, join_func);
-         mySetRealSource(rstr);
-      }
-   }
-}
@@ -327,14 +327,14 @@ namespace BCG

      protected override void myOnFirstSubscriber()
      {
-         myLeftStream.onChange += myHandleLeftStreamChanges;
-         myRightStream.onChange += myHandleRightStreamChanges;
+         myLeftStream.onChange += new ChangeDelegate(myHandleLeftStreamChanges);
+         myRightStream.onChange += new ChangeDelegate(myHandleRightStreamChanges);
      }

      protected override void myOnLastSubscriber()
      {
-         myLeftStream.onChange -= myHandleLeftStreamChanges;
-         myRightStream.onChange -= myHandleRightStreamChanges;
+         myLeftStream.onChange -= new ChangeDelegate(myHandleLeftStreamChanges);
+         myRightStream.onChange -= new ChangeDelegate(myHandleRightStreamChanges);
         myLeftKeyToLeftRows.Clear();
         myRightKeyToRightRows.Clear();
         myLeftIDToResultRows.Clear();
@@ -23,15 +23,15 @@ namespace BCG
         myRightOperand.master = this;
         //the events are not hooked onto the operands delegate, but rather a level of indirection
         //added in the form on myLeftChangeHandler
-         myLeftStream.onChange += myLeftChangeHandler;
-         myRightStream.onChange += myRightChangeHandler;
+         myLeftStream.onChange += new DataStream.ChangeDelegate (myLeftChangeHandler);
+         myRightStream.onChange += new DataStream.ChangeDelegate (myRightChangeHandler);
      }
       
      protected override void myOnLastSubscriber () 
      {
         myInit ();
-         myLeftStream.onChange -= myLeftChangeHandler;
-         myRightStream.onChange -= myRightChangeHandler;
+         myLeftStream.onChange -= new DataStream.ChangeDelegate (myLeftChangeHandler);
+         myRightStream.onChange -= new DataStream.ChangeDelegate (myRightChangeHandler);
      }

      protected override void myInit()
@@ -2,13 +2,9 @@ using System;
 using System.Collections;
 using System.Collections.Generic;

-using Amazon.Runtime.Internal.Util;
-
 using BCG;
 using BCG.Utilities;

-using bts;
-
 namespace BCG {
   public class LiteralStream : ViewStream {

@@ -26,34 +22,6 @@ namespace BCG {
         return rv;
      }

-      //literal("table1", "x:123, y:1.001, z:abc;x:123, y:1.001, z:abc")
-      public LiteralStream(string table_name, string rows) {
-         Func<string, int, IBlueRow> get_row = (json_row, id) => {
-            Dictionary<string, object> map = new Dictionary<string, object>();
-            string[] toks = json_row.Split(',', StringSplitOptions.TrimEntries);
-            //Logger.assert(col_vals.Length == 2);
-            foreach (string t in toks) {
-               var col_vals = t.Split(':', StringSplitOptions.TrimEntries);
-               string col = col_vals[0];
-               string val = col_vals[1];
-               map.Add(col, val);
-            }
-            map["id"] = id;
-            return new BlueRow(map);
-         };
-         string[] row_arr = rows.Split(';', StringSplitOptions.TrimEntries);
-         BlueTable table = null;
-         foreach (string json_string in row_arr) {
-            IBlueRow r = //BlueDataUtility.convert_json_to_row(json_string, table == null ? 0 : table.Count);
-            get_row(json_string, table == null ? 0 : table.Count);
-            table = table ?? new BlueTable(r.schema);
-            table.addRow(r);
-         }
-         ViewStream str = new BlueTableStream(table);
-         str = new SchemaGuesserStream(str);
-         mySetRealSource(str);
-      }
-
      public LiteralStream(string tableName, string schema, string value) {
         string[] columns = schema.Split(' ');

@@ -87,7 +55,7 @@ namespace BCG {
         mySetRealSource(new BlueRowCollectionStaticStream(t));
      }

-      // public LiteralStream(string schema, string value) : this("literal", schema, value) { }
+      public LiteralStream(string schema, string value) : this("literal", schema, value) { }

      // public LiteralStream(string value) {

@@ -1,72 +0,0 @@
-using BCG;
-
-using MongoDB.Driver.Core.WireProtocol.Messages;
-
-using System;
-using System.Collections.Generic;
-using System.Linq;
-
-// the first arg is all the rows, the second arg is the index of the row that needs mapping, and 
-// the func returns the mapped row (ie mapped version of the index row)
-using GlobalMap = System.Func<System.Collections.Generic.List<BCG.IBlueRow>, uint, BCG.IBlueRow>;
-
-namespace bts {
-   // In GlobalMap, each row is mapped with the full view of all rows, ie the map functions for a row
-   // can use the state fo all rows.
-   public class GlobalMapStream : ViewStream {
-      private ViewStream _base_stream;
-      private readonly string _sort_col_name, _group_by_col;
-      private GlobalMap _map_func;
-      public GlobalMapStream(ViewStream base_stream, string sort_col_name, string group_by_col, GlobalMap map_func) {
-         _base_stream = base_stream;
-         _sort_col_name = sort_col_name;
-         _group_by_col = group_by_col;
-         _map_func = map_func;
-      }
-
-      public GlobalMapStream(ViewStream base_stream, string sort_col_name, GlobalMap map_func) :
-      this(base_stream, sort_col_name, null, map_func) {
-      }
-
-      protected override void myOnFirstSubscriber() {
-         _base_stream.onChange += _handle_stream_update;
-      }
-
-      protected override void myOnLastSubscriber() {
-         _base_stream.onChange -= _handle_stream_update;
-      }
-
-      void _handle_stream_update(object dp_obj) {
-         DataPacket dp = dp_obj as DataPacket;
-         IDictionary<int, List<IBlueRow>> groups = !String.IsNullOrEmpty(_group_by_col) ?
-         BlueDataUtility.group_together(dp.newRows, new string[] { _group_by_col })
-         :
-         new Dictionary<int, List<IBlueRow>>() { { 0, dp.newRows.to_list() } };
-         BlueTable out_table = null;
-         foreach (var kv in groups) {
-            List<IBlueRow> group_rows = kv.Value;
-            group_rows.Sort((r1, r2) => BlueDataUtility.compare_rows(r2, r1, _sort_col_name));
-            uint i = 0;
-            foreach (IBlueRow r in group_rows) {
-               IBlueRow nr = _map_func(group_rows, i++);
-               out_table = out_table ?? new BlueTable(nr.schema);
-               out_table.addRow(nr);
-            }
-         }
-
-
-         // List<IBlueRow> rows = dp.newRows.to_ienum().ToList();
-         // rows.Sort((r1, r2) => BlueDataUtility.compare_rows(r1, r2, _sort_col_name));
-         // uint i = 0;
-
-         // BlueTable out_table = null;
-         // foreach (IBlueRow r in rows) {
-         //    IBlueRow nr = _map_func(rows, i++);
-         //    out_table = out_table ?? new BlueTable(nr.schema);
-         //    out_table.addRow(nr);
-         // }
-         DataPacket dp_out = BlueDataUtility.build_dp_inc(out_table, myLastValue as DataPacket);
-         myFireEvent(dp_out);
-      }
-   }
-}
@@ -119,13 +119,13 @@ namespace BCG {


      protected override void myOnFirstSubscriber() {
-         myStream.onChange += myChangeHandler;
+         myStream.onChange += new DataStream.ChangeDelegate(myChangeHandler);
      }

      protected override void myOnLastSubscriber() {
         myInit();
         if (myStream != null)
-            myStream.onChange -= myChangeHandler;
+            myStream.onChange -= new DataStream.ChangeDelegate(myChangeHandler);
      }

      protected override void myOnTableClear() {
@@ -83,11 +83,10 @@ namespace BCG {
         myMapper.onMapperChange += new MapperChangeDelegate(myHandleMapChange);
      }

-      protected override void myOnFirstSubscriber() =>
-         myStream.onChange += myHandleData;
+      protected override void myOnFirstSubscriber() { myStream.onChange += new ChangeDelegate(myHandleData); }

      protected override void myOnLastSubscriber() {
-         myStream.onChange -= myHandleData;
+         myStream.onChange -= new ChangeDelegate(myHandleData);
         myRows = null;
         myBaseRows = null;
         myMap.Clear();
@@ -16,10 +16,10 @@ namespace BCG {
      }

      protected override void myOnFirstSubscriber() {
-         _stream.onChange += _onstream_change;
+         _stream.onChange += new ChangeDelegate(_onstream_change);
      }
      protected override void myOnLastSubscriber() {
-         _stream.onChange -= _onstream_change;
+         _stream.onChange -= new ChangeDelegate(_onstream_change);
      }

      private void _onstream_change(object o) {
@@ -45,4 +45,4 @@ namespace BCG {

      }
   }
-}
+}
@@ -17,10 +17,10 @@ namespace BCG {
      }

      protected override void myOnFirstSubscriber() {
-         _stream.onChange += _onstream_change;
+         _stream.onChange += new ChangeDelegate(_onstream_change);
      }
      protected override void myOnLastSubscriber() {
-         _stream.onChange -= _onstream_change;
+         _stream.onChange -= new ChangeDelegate(_onstream_change);
      }


@@ -41,6 +41,7 @@ namespace BCG {
         }
         _table.addRows(added_rows);
         myFireEvent(new DataPacket(_table, added_rows, new BlueTable(_table.schema)));
+
      }
   }

@@ -137,24 +137,10 @@ namespace bts {
         HashSet<string> removed_cols = new HashSet<string>();
         Dictionary<string, string> mapped_to_orig_map = new Dictionary<string, string>();
         Dictionary<string, string> orig_to_mapped = new Dictionary<string, string>();
-         // by default every col will show up, so if we have the projection as (a:a1)
-         // it will convert a to a1, 
-         // examples: if a schema has cols a b c
-         // project(parent_str, "a:aa ") -> aa b c
-         // project(parent_str, "a:aa !c") -> aa b
-         // project(parent_str, "a:aa !") -> aa
-         // project(parent_str, "a:aa b = (aa*aa)!") -> aa b
-         bool hide_all = false;
         foreach (string p_tok in project_toks) {
            if (p_tok.StartsWith("!")) {
-               if (p_tok.Length == 1) {
-                  // if it is a "blanket "!"=> hide all cols except the one that
-                  // are stated explicitly
-                  hide_all = true;
-               } else {
-                  string removed_col = p_tok.Substring(1);
-                  removed_cols.Add(removed_col);
-               }
+               string removed_col = p_tok.Substring(1);
+               removed_cols.Add(removed_col);
            } else if (p_tok.Contains(":")) {
               string[] map_toks = p_tok.Split(":");
               string orig_col = map_toks[0];
@@ -169,11 +155,12 @@ namespace bts {
               }
            }
         }
-
         // make the new schema
         List<ColumnInfo> new_cols = new List<ColumnInfo>();
         foreach (ColumnInfo ci in input_schema.columnInfo) {
            if (ci.colName == "id") {
+               // if there's a hidden col, other columns will show by default, but if cols are being
+               // projected epxlicitely, than others wil be hidden
               ColumnInfo nc = new ColumnInfo(ci.colName, new_cols.Count, ci.colType);
               new_cols.Add(nc);
               continue;
@@ -185,13 +172,12 @@ namespace bts {
               ColumnInfo nc = new ColumnInfo(mapped_name, new_cols.Count, ci.colType);
               new_cols.Add(nc);
            } else {
-               if (hide_all)
-                  continue;
-               //if (ci.colName == "id" || removed_cols.Count > 0) {
-               // unless columns are  explicitely hidden, all columns show up
-               ColumnInfo nc = new ColumnInfo(ci.colName, new_cols.Count, ci.colType);
-               new_cols.Add(nc);
-               //}
+               if (ci.colName == "id" || removed_cols.Count > 0) {
+                  // if there's a hidden col, other columns will show by default, but if cols are being
+                  // projected epxlicitely, than others wil be hidden
+                  ColumnInfo nc = new ColumnInfo(ci.colName, new_cols.Count, ci.colType);
+                  new_cols.Add(nc);
+               }
            }
         }
         IBlueTableSchema new_schema = new BlueTableSchema(input_schema.tableName, new_cols.ToArray());
@@ -54,12 +54,12 @@ namespace BCG {

      protected override void myOnFirstSubscriber()
      {
-         myVs.onChange += myHandleChange;
+         myVs.onChange += new BCG.DataStream.ChangeDelegate (myHandleChange);
      }

      protected override void myOnLastSubscriber()
      {
-         myVs.onChange -= myHandleChange;
+         myVs.onChange -= new BCG.DataStream.ChangeDelegate (myHandleChange);;
      }

      internal void myHandleChange (object newValue)
@@ -1,230 +0,0 @@
-using System;
-using System.Data;
-using System.Data.SQLite;
-using System.Diagnostics;
-using System.Collections;
-
-
-using BCG;
-using System.Linq;
-using System.Data.Entity;
-using System.Collections.Generic;
-
-namespace bts.SilverPine {
-
-   public class DataRowWrapperRow : IBlueRow {
-      private readonly DataRow _row;
-      private readonly IBlueTableSchema _schema;
-      private readonly int _id;
-      public DataRowWrapperRow(IBlueTableSchema schema, DataRow row, int id) {
-         _row = row;
-         _id = id;
-         _schema = schema;
-      }
-      public virtual IBlueTableSchema schema { get { return _schema; } }
-
-      public virtual object this[string col_name] {
-         get {
-            ColumnInfo ci = _schema[col_name];
-            return this[ci.colNo];
-         }
-      }
-      public virtual object this[int index] {
-         get {
-            if (_schema[index].colName == "id")
-               return _id;
-            return _row[index];
-         }
-      }
-
-      public virtual int Count { get { return _row.ItemArray.Length; } }
-
-      public virtual bool IsSynchronized { get { return false; } }
-
-      public virtual object SyncRoot { get { return this; } }
-
-      public virtual void CopyTo(Array array, int index) { _row.ItemArray.CopyTo(array, index); }
-
-      public virtual bool IsFixedSize { get { return true; } }
-
-      virtual public bool IsReadOnly { get { return true; } }
-
-      /// <summary>
-      /// The key can either be the string corrsponding to the column name or
-      /// the order of the column
-      /// </summary>
-      public virtual object this[object key] {
-         get {
-            if (key is string)
-               return this[(string)key];
-            else if (key is int)
-               return this[(int)key];
-            throw new BlueDataException("Wrong type while indexing the row");
-         }
-         set { throw new NotSupportedException("The IBlueRow is readonly"); }
-      }
-
-      public virtual ICollection Keys { get { return _schema.columnNames; } }
-
-      public virtual ICollection Values { get { return _row.ItemArray; } }
-
-      public virtual void Add(object key, object value) { throw new NotSupportedException("The IBlueRow is readonly"); }
-
-      public virtual void Clear() { throw new NotSupportedException("The IBlueRow is readonly"); }
-
-      public virtual bool Contains(object key) { return _schema[(string)key] != null; }
-
-      public IDictionaryEnumerator GetEnumerator() { return new BlueRowDictionaryEnumerator(this); }
-
-      IEnumerator IEnumerable.GetEnumerator() { return new BlueRowEnumerator(this); }
-
-      public virtual void Remove(object key) { throw new NotSupportedException("The IBlueRow is readonly"); }
-
-   }
-
-   public class DataTableWrapperTable : IBlueRowCollection {
-
-      public class DataTableWrapperTableEnumerator : IEnumerator {
-         private IEnumerator _enumerator;
-         private IBlueTableSchema _schema;
-         private int _id = 0;
-         public DataTableWrapperTableEnumerator(DataTable table) {
-            _enumerator = table.Rows.GetEnumerator();
-            _schema = BlueDataUtility.getSchema(table);
-
-         }
-         public object Current {
-            get {
-               DataRow r = _enumerator.Current as DataRow;
-               return new DataRowWrapperRow(_schema, r, _id);
-            }
-         }
-         public bool MoveNext() {
-            _id++;
-            return _enumerator.MoveNext();
-         }
-         public void Reset() {
-            _id = 0;
-            _enumerator.Reset();
-         }
-      }
-
-      private readonly DataTable _table;
-      private IBlueTableSchema _schema;
-      private IEnumerator _enumerator;
-
-      public DataTableWrapperTable(DataTable table) {
-         _table = table;
-         _schema = BlueDataUtility.getSchema(_table);
-         _enumerator = new DataTableWrapperTableEnumerator(table);
-      }
-      public virtual IBlueRow this[int id] {
-         get {
-            // its assumed the rows index == id
-            DataRow dr = _table.Rows[id];
-            return new DataRowWrapperRow(_schema, dr, id);
-         }
-         set {
-
-         }
-      }
-
-      public virtual IBlueTableSchema schema { get { return _schema; } }
-
-      public virtual bool containsID(int id) { return id < _table.Rows.Count; }
-
-      // private List<IBlueRow> _rows() {
-      //    List<IBlueRow> rv = new List<IBlueRow>();
-      //    int id = 0;
-      //    foreach (DataRow r in _table.Rows) {
-      //       rv.Add(new DataRowWrapperRow(r, id++));
-      //    }
-      //    return rv;
-      // }
-
-      public virtual IEnumerator GetEnumerator() {
-         // var rows = _rows();
-         // return rows.GetEnumerator();
-         return _enumerator;
-      }
-
-      public virtual bool IsSynchronized { get { return false; } }
-
-      public virtual object SyncRoot { get { return this; } }
-
-      public virtual void CopyTo(Array array, int index) {
-         Log.assert(false);
-         //_rows().CopyTo(array as IBlueRow[], index); 
-      }
-
-      public virtual int Count {
-         get { return _table.Rows.Count; }
-      }
-   }
-
-   public static class SQLiteHelper {
-      public static DataTable ExecuteSelectQuery(string connection_string, string query) {
-         // Create a new DataTable to store the results
-         DataTable dataTable = new DataTable();
-
-         // Create and open a SQLite connection using the provided connection string
-         using (SQLiteConnection connection = new SQLiteConnection(connection_string)) {
-            try {
-               connection.Open();
-
-               // Create a SQLite command with the provided query
-               using (SQLiteCommand command = new SQLiteCommand(query, connection)) {
-                  // Use SQLiteDataAdapter to fill the DataTable
-                  using (SQLiteDataAdapter adapter = new SQLiteDataAdapter(command)) {
-                     adapter.Fill(dataTable);
-                  }
-               }
-            } catch (Exception ex) {
-               Console.WriteLine("An error occurred: " + ex.Message);
-            }
-         }
-
-         // Return the populated DataTable
-         return dataTable;
-      }
-   }
-
-   [DataStreamUserRespresentation("SQLite")]
-   public class SQLiteStream : ViewStream {
-      private readonly string _connection_string, _query;
-      private Stopwatch _stopwatch = new Stopwatch();
-      public SQLiteStream(string connection_string, string query) {
-         _connection_string = connection_string;
-         _query = query;
-      }
-
-
-      private void _read_from_sqlite_nd_fire() {
-         _stopwatch.Start();
-         DataTable dt = SQLiteHelper.ExecuteSelectQuery(_connection_string, _query);
-         _stopwatch.Stop();
-         Log.Gen.info($"SQLiteStream._read_from_sqlite_and_fire time to read db: {_stopwatch.ElapsedMilliseconds} msecs");
-         _stopwatch.Start();
-         IBlueRowCollection rows = BCG.BlueDataUtility.toIBlueRowCollection(dt);
-         //IBlueRowCollection rows = new DataTableWrapperTable(dt);
-         _stopwatch.Stop();
-         Log.Gen.info($"SQLiteStream._read_from_sqlite_and_fire time to load into IBlueRowCollection: {_stopwatch.ElapsedMilliseconds} msecs");
-         DataPacket dp = BlueDataUtility.build_dp_inc(rows, myLastValue as DataPacket);
-         myFireEvent(dp);
-      }
-
-      protected override void myOnFirstSubscriber() {
-         _read_from_sqlite_nd_fire();
-      }
-
-      protected override void myOnLastSubscriber() {
-         // this value will be cleared by the base class
-         // but in this particular stream this behaviour
-         // is not usefull since any new subscribers 
-         // would force the query to be recreated
-         //myLastDP = myLastValue;
-      }
-
-   }
-}
-
@@ -3,15 +3,12 @@ using System.Collections.Generic;
 using System.Text;
 using System.IO;
 using BCG.Utilities;
-using System.Diagnostics;
-using Microsoft.Data.Analysis;
-using Parquet;
-using Parquet.Data;

-
-namespace BCG {
+namespace BCG
+{
   [DataStreamUserRespresentation("CSVSaver")]
-   public class CSVSaverStream : ViewStream, ISideEffectStream {
+   public class CSVSaverStream : ViewStream, ISideEffectStream
+   {
      private IBlueTableSchema myResultSchema = new BlueTableSchema("CSVResult", new ColumnInfo("id", 0, typeof(int)),
         new ColumnInfo("fileName", 1, typeof(string)),
         new ColumnInfo("nWrites", 2, typeof(int)),
@@ -23,9 +20,11 @@ namespace BCG {
      private int myNWrites = 0, myNMaxWrites = Int32.MaxValue;

      public CSVSaverStream(ViewStream vs, string fullyQualifiedFileName)
-         : this(vs, fullyQualifiedFileName, true, Int32.MaxValue, false) { }
+         : this(vs, fullyQualifiedFileName, true, Int32.MaxValue, false)
+      { }

-      public CSVSaverStream(ViewStream vs, string fullyQualifiedFileName, bool overwrite, int nMaxWrites, bool fireResultSnapshot) {
+      public CSVSaverStream(ViewStream vs, string fullyQualifiedFileName, bool overwrite, int nMaxWrites, bool fireResultSnapshot)
+      {
         myStream = vs;
         myFullyQualifiedFileName = fullyQualifiedFileName;
         myOverwrite = overwrite;
@@ -33,20 +32,22 @@ namespace BCG {
         myFireResultSnapshot = fireResultSnapshot;
      }

-      protected override void myOnFirstSubscriber() =>
-         myStream.onChange += myHandleChanges;
+      protected override void myOnFirstSubscriber() { myStream.onChange += new ChangeDelegate(myHandleChanges); }

-      protected override void myOnLastSubscriber() =>
-         myStream.onChange -= myHandleChanges;
+      protected override void myOnLastSubscriber() { myStream.onChange -= new ChangeDelegate(myHandleChanges); }

-      private void myHandleChanges(object newValue) {
+      private void myHandleChanges(object newValue)
+      {
         IBlueRowCollection totalState = (newValue as DataPacket).newRows;
-         if (totalState.Count > 0 && myNWrites < myNMaxWrites) {
+         if (totalState.Count > 0 && myNWrites < myNMaxWrites)
+         {
            myNWrites++;
            int i = 0;
            StreamWriter writer = new StreamWriter(new FileStream(myGetNewFileName(), FileMode.Create, FileAccess.Write));
-            foreach (IBlueRow r in totalState) {
-               if (i++ == 0) {
+            foreach (IBlueRow r in totalState)
+            {
+               if (i++ == 0)
+               {
                  string header = String.Empty;
                  foreach (string col in r.schema.columnNames)
                     if (col != "id")
@@ -58,17 +59,20 @@ namespace BCG {
               writer.Flush();
            }
            writer.Close();
-            if (myFireResultSnapshot) {
+            if (myFireResultSnapshot)
+            {
               IBlueRow nr = new BlueRow(myResultSchema, 0, myFullyQualifiedFileName, myNWrites, totalState.Count);
               BlueTable nRows = new BlueTable(myResultSchema);
               nRows.addRow(nr);
               myFireEvent(new DataPacket(nRows, nRows, new BlueTable(myLastValue == null ? new BlueTable(myResultSchema) : (myLastValue as DataPacket).newRows)));
-            } else
+            }
+            else
               myFireEvent(newValue);
         }
      }

-      private string myGetNewFileName() {
+      private string myGetNewFileName()
+      {
         string[] tags = myFullyQualifiedFileName.Split('.');
         if (myOverwrite)
            return tags[0] + ".csv";
@@ -76,11 +80,14 @@ namespace BCG {
            return tags[0] + "_" + myFormateDateTime(DateTime.Now) + ".csv";
      }

-      private static string myGetCSVRow(IBlueRow r) {
+      private static string myGetCSVRow(IBlueRow r)
+      {
         string rv = String.Empty;
         int i = 0;
-         foreach (string col in r.schema.columnNames) {
-            if (col != "id") {
+         foreach (string col in r.schema.columnNames)
+         {
+            if (col != "id")
+            {
               string colText = String.Empty;
               if (r[col] != null && r[col] != DBNull.Value)
                  colText = r[col].ToString();
@@ -92,7 +99,8 @@ namespace BCG {
         return rv;
      }

-      private string myFormateDateTime(DateTime dt) {
+      private string myFormateDateTime(DateTime dt)
+      {
         return
         string.Format("{0}-{1}-{2}-{3}-{4}-{5}-{6}",
                        dt.Year.ToString("0000"),
@@ -106,49 +114,52 @@ namespace BCG {
   }

   [DataStreamUserRespresentation("CSVReader")]
-   public class CSVReaderStream : ViewStream {
-      private readonly string _file_address;
-      Stopwatch _stopwatch = new Stopwatch();
-      public CSVReaderStream(string fileAddress) {
-         _file_address = fileAddress;
+   public class CSVReaderStream : ViewStream
+   {
+      private IBlueRowCollection myRows;
+
+      public CSVReaderStream(string fileAddress)
+      {
+         myRows = BlueDataUtility.createRowsFromCSVFileWithoutTypes(fileAddress);
      }

-      protected override void myOnFirstSubscriber() {
-         _stopwatch.Start();
-         DataFrame df = DataFrame.LoadCsv(_file_address);
-         _stopwatch.Stop();
-         bts.Log.Gen.info($"CSVReaderStream.myOnFirstSubscriber stream csv load col_count: {df.Columns.Count} rows, time: {_stopwatch.ElapsedMilliseconds} msecs");
-         _stopwatch.Start();
-         IBlueRowCollection rows = BlueDataUtility.createRowsFromCSVFileWithoutTypes(_file_address);
-         _stopwatch.Stop();
-         bts.Log.Gen.info($"CSVReaderStream.myOnFirstSubscriber stream csv load count: {rows.Count} rows, time: {_stopwatch.ElapsedMilliseconds} msecs");
-         DataPacket dp_out = BlueDataUtility.build_dp_inc(rows, myLastValue as DataPacket);
-         myFireEvent(dp_out);
+      protected override void myOnFirstSubscriber()
+      {
+         myFireEvent(new DataPacket(myRows, myRows, new BlueTable(myRows.schema)));
      }
   }

-   [DataStreamUserRespresentation("CSVReaderStreamMapper")]
-   public class CSVReaderStreamMapperStream : ViewStream {
-      public CSVReaderStreamMapperStream(ViewStream stream, string output_schema) {
+   [DataStreamUserRespresentation ("CSVReaderStreamMapper")]
+   public class CSVReaderStreamMapperStream : ViewStream
+   {
+      public CSVReaderStreamMapperStream(ViewStream stream, string output_schema)
+      {
         mySetRealSource(new MapStream(stream, new CSVReaderStreamMapper(output_schema)));
      }
   }

-   public class CSVReaderStreamMapper : IStreamMapper {
+   public class CSVReaderStreamMapper : IStreamMapper
+   {
      private IBlueTableSchema mySchema;
      private readonly string[] myOutputSchemaStringArray;

-      public CSVReaderStreamMapper(string outputSchema) {
+      public CSVReaderStreamMapper(string outputSchema)
+      {
         myOutputSchemaStringArray = outputSchema.Split(' ');
      }

-      public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema) {
-         if (mySchema == null) {
+      public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema)
+      {
+         if (mySchema == null)
+         {
            ColumnInfo[] ci = inputSchema.columnInfo;
-            foreach (string col in myOutputSchemaStringArray) {
+            foreach (string col in myOutputSchemaStringArray)
+            {
               string colName = col.Substring(0, col.IndexOf('\\')).ToLower();
-               for (int i = 0; i < ci.Length; i++) {
-                  if (ci[i].colName.ToLower().Equals(colName)) {
+               for (int i = 0; i < ci.Length; i++)
+               {
+                  if (ci[i].colName.ToLower().Equals(colName))
+                  {
                     Type t = myGetType(col.Substring(col.IndexOf('\\') + 1));
                     ci[i] = new ColumnInfo(ci[i].colName, i, t);
                     break;
@@ -160,52 +171,74 @@ namespace BCG {
         return mySchema;
      }

-      public IBlueRow map(IBlueRow row) {
+      public IBlueRow map(IBlueRow row)
+      {
         object[] vals = new object[row.schema.columnInfo.Length];
         row.Values.CopyTo(vals, 0);
-         for (int i = 0; i < mySchema.columnInfo.Length; i++) {
+         for (int i = 0; i < mySchema.columnInfo.Length; i++)
+         {
            Type t = mySchema.columnInfo[i].colType;
-            try {
+            try
+            {
               vals[i] = System.Convert.ChangeType(vals[i], mySchema.columnInfo[i].colType, System.Globalization.CultureInfo.InvariantCulture);
-            } catch {
-               if (t.Equals(typeof(Int16))) {
+            }
+            catch
+            {
+               if (t.Equals(typeof(Int16)))
+               {
                  Int16 result;
-                  if (Int16.TryParse((string)vals[i], out result)) {
+                  if (Int16.TryParse((string)vals[i], out result))
+                  {
                     vals[i] = result;
                     continue;
                  }
                  double r2;
-                  if (Double.TryParse((string)vals[i], out r2)) {
-                     try {
+                  if (Double.TryParse((string)vals[i], out r2))
+                  {
+                     try
+                     {
                        vals[i] = (Int16)r2;
                        continue;
-                     } catch { }
+                     }
+                     catch { }
                  }
-               } else if (t.Equals(typeof(Int32))) {
+               }
+               else if (t.Equals(typeof(Int32)))
+               {
                  Int32 result;
-                  if (Int32.TryParse((string)vals[i], out result)) {
+                  if (Int32.TryParse((string)vals[i], out result))
+                  {
                     vals[i] = result;
                     continue;
                  }
                  double r2;
-                  if (Double.TryParse((string)vals[i], out r2)) {
-                     try {
+                  if (Double.TryParse((string)vals[i], out r2))
+                  {
+                     try
+                     {
                        vals[i] = (Int32)r2;
                        continue;
-                     } catch { }
+                     }
+                     catch { }
                  }
-               } else if (t.Equals(typeof(Int64))) {
+               }
+               else if (t.Equals(typeof(Int64)))
+               {
                  Int64 result;
-                  if (Int64.TryParse((string)vals[i], out result)) {
+                  if (Int64.TryParse((string)vals[i], out result))
+                  {
                     vals[i] = result;
                     continue;
                  }
                  double r2;
-                  if (Double.TryParse((string)vals[i], out r2)) {
-                     try {
+                  if (Double.TryParse((string)vals[i], out r2))
+                  {
+                     try
+                     {
                        vals[i] = (Int64)r2;
                        continue;
-                     } catch { }
+                     }
+                     catch { }
                  }
               }
               vals[i] = DBNull.Value;
@@ -214,7 +247,8 @@ namespace BCG {
         return new BlueRow(mySchema, vals);
      }

-      private Type myGetType(string type) {
+      private Type myGetType(string type)
+      {
         type = type.ToLower();
         if (type.Contains("int"))
            return typeof(int);
@@ -1,41 +1,31 @@
 using System;
 using System.Collections;
 using System.Reflection;
-
 using BCG.Utilities;

-namespace BCG {
-   // if a table needs to be fired, it can be wrapped around this
-   public class BlueTableStream : ViewStream {
-      private IBlueRowCollection _rows;
-      public BlueTableStream(IBlueRowCollection rows) {
-         _rows = rows;
+namespace BCG
+{
+   [DataStreamUserRespresentation (false)]
+   public class BlueRowCollectionStaticStream : ViewStream
+   {
+      public BlueRowCollectionStaticStream () {}
+
+      public BlueRowCollectionStaticStream (IBlueRowCollection rows) {loadCollection (rows);}
+
+      public void loadCollection (IBlueRowCollection rows)
+      {
+         myFireEvent (new DataPacket (rows, rows, myLastValue != null ?
+                                                   new BlueTable((myLastValue as DataPacket).newRows) : new BlueTable (rows.schema)));
      }

-      protected override void myOnFirstSubscriber() {
-         var dp_out = BlueDataUtility.build_dp_inc(_rows, myLastValue as DataPacket);
-         myFireEvent(dp_out);
-      }
-   }
-
-   [DataStreamUserRespresentation(false)]
-   public class BlueRowCollectionStaticStream : ViewStream {
-      public BlueRowCollectionStaticStream() { }
-
-      public BlueRowCollectionStaticStream(IBlueRowCollection rows) { loadCollection(rows); }
-
-      public void loadCollection(IBlueRowCollection rows) {
-         myFireEvent(new DataPacket(rows, rows, myLastValue != null ?
-                                                   new BlueTable((myLastValue as DataPacket).newRows) : new BlueTable(rows.schema)));
-      }
-
-      public void crush_with_single_row(IBlueRow r) {
+      public void crush_with_single_row(IBlueRow r)
+      {
         BlueTable t = new BlueTable(r.schema);
         t.addRow(r);
         loadCollection(t);
      }

-      public void loadPacket(DataPacket dp) { myFireEvent(dp); }
+      public void loadPacket (DataPacket dp) {myFireEvent (dp);}

      /// <summary>
      /// Will work with streams that take ViewStream as first argument ex project, filter, distinct etc
@@ -45,16 +35,17 @@ namespace BCG {
      /// <param name="streamType"></param>
      /// <param name="args"></param>
      /// <returns></returns>
-      public static IBlueRowCollection statelessOp(IBlueRowCollection rows, Type streamType, params object[] args) {
-         ArrayList totArgs = new ArrayList(args);
-         BlueRowCollectionStaticStream v1 = new BlueRowCollectionStaticStream();
-         totArgs.Insert(0, v1);
-         ViewStream retVal = Activator.CreateInstance(streamType, totArgs.ToArray()) as ViewStream;
-         v1.loadCollection(rows);
+      public static IBlueRowCollection statelessOp (IBlueRowCollection rows, Type streamType, params object[] args)
+      {
+         ArrayList totArgs = new ArrayList (args);
+         BlueRowCollectionStaticStream v1 = new BlueRowCollectionStaticStream ();
+         totArgs.Insert (0, v1);
+         ViewStream retVal = Activator.CreateInstance (streamType, totArgs.ToArray()) as ViewStream;
+         v1.loadCollection (rows);
         return retVal.rows;
      }

   }

-
+   
 }
@@ -35,14 +35,14 @@ namespace BCG
      #region ViewStream
      protected override void myOnFirstSubscriber()
      {
-         myStream.onChange += myStreamChange;
+         myStream.onChange +=new ChangeDelegate(myStreamChange);
         myVsd.addWorker(this);
      }

      protected override void  myOnLastSubscriber()
      {
         myVsd.removeWorker(this);
-         myStream.onChange -= myStreamChange;
+         myStream.onChange -= new ChangeDelegate(myStreamChange);
      }
      #endregion

@@ -28,7 +28,7 @@ namespace BCG

      protected override void myOnFirstSubscriber()
      {
-         myParentStream.onChange += myParentStreamChangeHandler;
+         myParentStream.onChange += new ChangeDelegate(myParentStreamChangeHandler);
         myVSD.addWorker(this);
      }

@@ -36,7 +36,7 @@ namespace BCG

      protected override void myOnLastSubscriber()
      {
-         myParentStream.onChange -= myParentStreamChangeHandler;
+         myParentStream.onChange -= new ChangeDelegate(myParentStreamChangeHandler);
         myVSD.removeWorker(this);
      }

@@ -53,11 +53,11 @@ namespace BCG
            if (r.is_subscription)
            {
               _current_source_subscription = r.subscription_id;
-               _source_stream.onChange += _source_stream_onChange;
+               _source_stream.onChange += new ChangeDelegate(_source_stream_onChange);
            }
            else
            {
-               _source_stream.onChange -= _source_stream_onChange;
+               _source_stream.onChange -= new ChangeDelegate(_source_stream_onChange);
            }
         }
      }
@@ -1,34 +0,0 @@
-namespace bts.SilverPine {
-   using bts;
-
-   using BCG;
-
-   using System;
-
-   public class DateTimeExpanstionStream : ViewStream {
-      public DateTimeExpanstionStream(ViewStream parent_stream, string inp_col) {
-         Func<IBlueTableSchema, IBlueTableSchema> get_schema = (s) => {
-            IBlueTableSchema rv = s.add_column(System.TypeCode.DateTime, $"{inp_col}_date");
-            rv = rv.add_column(System.TypeCode.String, $"{inp_col}_day");
-            rv = rv.add_column(System.TypeCode.String, $"{inp_col}_month");
-            rv = rv.add_column(System.TypeCode.String, $"{inp_col}_hour");
-            rv = rv.add_column(System.TypeCode.String, $"{inp_col}_min");
-            return rv;
-         };
-         ViewStream rs = new MapStream(parent_stream,
-         s => get_schema(s),
-         r => {
-            DateTime dt = r.get_datetime(inp_col);
-            IBlueTableSchema s = get_schema(r.schema);
-            MutableBlueRow rv = new MutableBlueRow(s, r);
-            rv.setValue($"{inp_col}_date", dt.Date);
-            rv.setValue($"{inp_col}_day", dt.DayOfWeek.ToString());
-            rv.setValue($"{inp_col}_month", dt.Date.Month.ToString());
-            rv.setValue($"{inp_col}_hour", dt.Hour);
-            rv.setValue($"{inp_col}_min", dt.Minute);
-            return rv;
-         });
-         mySetRealSource(rs);
-      }
-   }
-}
@@ -24,7 +24,7 @@ namespace BCG
        protected override void myOnFirstSubscriber()
        {
            myKeyToTimeToValue = new Dictionary<string, SortedList<DateTime, double>>();
-            myToAverage.onChange += myToAverage_onChange;
+            myToAverage.onChange += new ChangeDelegate(myToAverage_onChange);
        }


@@ -1,111 +0,0 @@
-namespace bts.SilverPine {
-   using BCG;
-
-   using bts;
-   using bts.collections;
-
-   using System;
-   using System.CodeDom;
-   using System.Collections.Generic;
-   using System.Linq;
-
-   // Given 2 filters, and 2 columns, finds the different in the value of the 2 columns from the 2 rows
-   // so:
-   // f1(base_stream.rows)[col1] - f1(base_stream.rows)[col2]
-   [DataStreamUserRespresentation("ColDiff")]
-   public class ColumnDifferenceStream : ViewStream {
-      private readonly ViewStream _base_stream;
-      private readonly FilterStream.OnFilter _f1, _f2;
-      private readonly string _c1, _c2;
-
-      public ColumnDifferenceStream(ViewStream base_stream, FilterStream.OnFilter f1, FilterStream.OnFilter f2, string col1, string col2) {
-         _base_stream = base_stream;
-         _f1 = f1;
-         _f2 = f2;
-         _c1 = col1;
-         _c2 = col2;
-      }
-
-      public ColumnDifferenceStream(ViewStream base_stream, string f1, string f2, string col1, string col2) {
-         FilterStream.OnFilter f1_del = r => {
-            BooleanExpressionFilter rv = new BooleanExpressionFilter(f1);
-            return rv.filter(r);
-         };
-         FilterStream.OnFilter f2_del = r => {
-            BooleanExpressionFilter rv = new BooleanExpressionFilter(f2);
-            return rv.filter(r);
-         };
-         ViewStream rs = new ColumnDifferenceStream(base_stream, f1_del, f2_del, col1, col2);
-         mySetRealSource(rs);
-      }
-
-      // for each row, does a row[new_ol_name] = row[col1] - rows[row_ffset][col2]
-      // we should generlize this for any math operation
-      public ColumnDifferenceStream(ViewStream parent_stream, string sort_col_name, int row_offset, string col1, string col2, string diff_col_name, double optional_perc_diff_filter = 0.0) {
-         // use GlobalMapStream
-         string diff_col_perc_name = $"{diff_col_name}_percentage";
-         System.Func<System.Collections.Generic.List<BCG.IBlueRow>, uint, BCG.IBlueRow> g_map_func = (rows, idx) => {
-            int minuend_idx = Convert.ToInt32(idx);
-            var inp_schema = rows[0].schema;
-            var out_schema = inp_schema.add_column(TypeCode.Double, diff_col_name);
-            out_schema = out_schema.add_column(TypeCode.Double, diff_col_perc_name);
-            // minuend - subtrahend
-            IBlueRow minuend_row = rows.ElementAt(minuend_idx);
-            MutableBlueRow rv = new MutableBlueRow(out_schema, minuend_row);
-            int subtrahend_index = minuend_idx + row_offset;
-            double diff_val = Double.NaN, diff_val_perc = Double.NaN;
-            if (subtrahend_index >= 0) {
-               try {
-                  IBlueRow subtrahend_row = rows.ElementAt(subtrahend_index);
-                  diff_val = minuend_row.get<double>(col1) - subtrahend_row.get<double>(col2);
-                  diff_val_perc = 100.0 * diff_val / subtrahend_row.get<double>(col1);
-               } catch (Exception ex) {
-                  Log.assert(false, ex.Message);
-               }
-            }
-            rv.setValue(diff_col_name, diff_val);
-            rv.setValue(diff_col_perc_name, diff_val_perc);
-            return rv;
-         };
-         ViewStream rs = new GlobalMapStream(parent_stream, sort_col_name, g_map_func);
-         if (optional_perc_diff_filter != 0) {
-            rs = new FilterStream(rs, r => {
-               double diff_val_perc = r.get<double>(diff_col_perc_name);
-               double abs_diff_val_perc = Math.Abs(diff_val_perc);
-               return abs_diff_val_perc >= optional_perc_diff_filter;
-            });
-         }
-         mySetRealSource(rs);
-      }
-
-      protected override void myOnFirstSubscriber() {
-         _base_stream.onChange += _handle_base_stream_update;
-      }
-
-      protected override void myOnLastSubscriber() {
-         _base_stream.onChange -= _handle_base_stream_update;
-      }
-
-      private void _handle_base_stream_update(object dp_obj) {
-         // could have been done with filter and cartesian prod, and then math
-         DataPacket dp = dp_obj as DataPacket;
-         List<IBlueRow> rows = dp.newRows.to_list();
-         var f1_rows = rows.Where(r => _f1(r));
-         Log.assert(f1_rows.Count() == 1);
-         var f2_rows = rows.Where(r => _f2(r));
-         Log.assert(f2_rows.Count() == 1);
-         IBlueRow f1_r = f1_rows.ElementAt(0);
-         IBlueRow f2_r = f2_rows.ElementAt(0);
-         double diff = f1_r.get<double>(_c1) - f2_r.get<double>(_c2);
-         IBlueTableSchema out_schema = f1_r.schema.add_columns(Tuple.Create(typeof(double), "diff"));
-         BlueTable out_table = new BlueTable(out_schema);
-         MutableBlueRow out_row = new MutableBlueRow(out_schema, f1_r);
-         out_row.setValue("diff", diff);
-         out_table.addRow(out_row);
-         DataPacket dp_out = BlueDataUtility.build_dp(out_table, myLastValue as DataPacket);
-         myFireEvent(dp_out);
-
-      }
-   }
-
-}
@@ -1,51 +0,0 @@
-namespace bts.SilverPine {
-   using BCG;
-
-   using bts;
-
-   using System.Collections.Generic;
-   using System.Linq;
-
-   // Given a sort order, an optional group by col, it observes the values of a column 
-   // within that group (or the whole table, if grouping not specified) and returns all rows
-   // where the value changes, ie row[current][col] != row[current-1][col]
-   // TODO: a more general way to express a filter where row[current][col] != row[previous][col],
-   // or even expand it to row[current][col_a] - row[previous][col_b] < 0.5
-   [DataStreamUserRespresentation("ColEvol")]
-   public class ColumnEvolutionStream : ViewStream {
-      private readonly ViewStream _base_stream;
-      private readonly string _sort_col;
-      private readonly string _evol_col;
-      public ColumnEvolutionStream(ViewStream base_stream, string sort_col, string evolution_col) {
-         _base_stream = base_stream;
-         _sort_col = sort_col;
-         _evol_col = evolution_col;
-      }
-
-      protected override void myOnFirstSubscriber() {
-         _base_stream.onChange += _handle_base_stream_update;
-      }
-
-      protected override void myOnLastSubscriber() {
-         base.myOnLastSubscriber();
-      }
-
-      private void _handle_base_stream_update(object dp_obj) {
-         DataPacket dp = dp_obj as DataPacket;
-         List<IBlueRow> rows = dp.newRows.to_list();
-         rows.Sort((r1, r2) => BlueDataUtility.compare_rows(r2, r1, _sort_col));
-         BlueTable rv = new BlueTable(dp.schema);
-         IBlueRow prev_row = rows.ElementAt(0);
-         foreach (IBlueRow r in rows) {
-            var cr = BlueDataUtility.compare_rows(r, prev_row, _evol_col);
-            if (cr != 0) {
-               rv.addRow(r);
-            }
-            prev_row = r;
-         }
-         DataPacket dp_out = BlueDataUtility.build_dp(rv, myLastValue as DataPacket);
-         myFireEvent(dp_out);
-      }
-   }
-
-}
@@ -12,10 +12,10 @@ namespace BCG {
         mySrc = src;
      }
      protected override void myOnFirstSubscriber() {
-         mySrc.onChange += mySrc_onChange;
+         mySrc.onChange += new ChangeDelegate(mySrc_onChange);
      }
      protected override void myOnLastSubscriber() {
-         mySrc.onChange -= mySrc_onChange;
+         mySrc.onChange -= new ChangeDelegate(mySrc_onChange);
      }
      private void mySrc_onChange(object newValue) {
         DataPacket dp = (DataPacket)newValue;
@@ -28,14 +28,14 @@ namespace BCG

      protected override void myOnFirstSubscriber()
      {
-         myLeftStream.onChange += myHandleLeftChanges;
-         myRightstream.onChange += myHandleRightChanges;
+         myLeftStream.onChange += new ChangeDelegate(myHandleLeftChanges);
+         myRightstream.onChange += new ChangeDelegate(myHandleRightChanges);
      }

      protected override void myOnLastSubscriber()
      {
-         myLeftStream.onChange -= myHandleLeftChanges;
-         myRightstream.onChange -= myHandleRightChanges;
+         myLeftStream.onChange -= new ChangeDelegate(myHandleLeftChanges);
+         myRightstream.onChange -= new ChangeDelegate(myHandleRightChanges);
      }

      private void myHandleLeftChanges (object newValue)
@@ -222,7 +222,7 @@ namespace BCG
            pj = new ProjectStream(pj, "!");
         myStreamIndex.Add(myMovingIndex, new Pair<ViewStream, ViewStream>(vs, pj));
         if (mySubscriptionActive)
-            pj.onChange += myHandleStreamChanges;
+            pj.onChange += new ChangeDelegate(myHandleStreamChanges);
      }

      public void removeStream(ViewStream vs)
@@ -235,7 +235,7 @@ namespace BCG
               index = p.Key;
               if (mySubscriptionActive)
               {
-                  p.Value.second.onChange -= myHandleStreamChanges;
+                  p.Value.second.onChange -= new ChangeDelegate(myHandleStreamChanges);
                  if (myTable != null)
                  {
                     IBlueRowCollection delRows = myTable.removeTable(p.Key);
@@ -254,14 +254,14 @@ namespace BCG
      {
         mySubscriptionActive = true;
         foreach (KeyValuePair<int, Pair<ViewStream, ViewStream>> p in myStreamIndex)
-            p.Value.second.onChange += myHandleStreamChanges;
+            p.Value.second.onChange += new ChangeDelegate(myHandleStreamChanges);
      }

      protected override void myOnLastSubscriber()
      {
         mySubscriptionActive = false;
         foreach (KeyValuePair<int, Pair<ViewStream, ViewStream>> p in myStreamIndex)
-            p.Value.second.onChange -= myHandleStreamChanges;
+            p.Value.second.onChange -= new ChangeDelegate(myHandleStreamChanges);
         if (myTable != null)
            myTable.clear();
      }
@@ -14,11 +14,11 @@ namespace BCG {
      }
      protected override void myOnFirstSubscriber()
      {
-         mySrc.onChange += mySrc_onChange;
+         mySrc.onChange += new ChangeDelegate(mySrc_onChange);
      }
      protected override void myOnLastSubscriber () 
      {
-         mySrc.onChange -= mySrc_onChange;
+         mySrc.onChange -= new ChangeDelegate (mySrc_onChange);
      }
      private void mySrc_onChange (object newValue) {
         DataPacket dp = (DataPacket)newValue;
@@ -21,15 +21,15 @@ namespace BCG
      {
         myLeftOp = new UnionOperand (true, this);
         myRightOp = new UnionOperand (false, this);
-         myLeftVs.onChange += myLeftOp.myChangeHandler;
-         myRightVs.onChange += myRightOp.myChangeHandler;
+         myLeftVs.onChange += new DataStream.ChangeDelegate (myLeftOp.myChangeHandler);
+         myRightVs.onChange += new DataStream.ChangeDelegate (myRightOp.myChangeHandler);
      }

      protected override void myOnLastSubscriber () 
      {
         myInit ();
-         myLeftVs.onChange -= myLeftOp.myChangeHandler;
-         myRightVs.onChange -= myRightOp.myChangeHandler;
+         myLeftVs.onChange -= new DataStream.ChangeDelegate (myLeftOp.myChangeHandler);
+         myRightVs.onChange -= new DataStream.ChangeDelegate (myRightOp.myChangeHandler);
      }

      public void startSession ()
@@ -116,4 +116,4 @@ namespace BCG
          receiveStream(dp, isLeft);
      }
   }
-}
+}
@@ -14,11 +14,11 @@ namespace BCG {

      protected override void myOnFirstSubscriber() {
         _vs_publisher = new ViewPublisher(_name, _broker, _main_stream);
-         _main_stream.onChange += _handle_update;
+         _main_stream.onChange += new DataStream.ChangeDelegate(_handle_update);
      }

      protected override void myOnLastSubscriber() {
-         _main_stream.onChange -= _handle_update;
+         _main_stream.onChange -= new DataStream.ChangeDelegate(_handle_update);
         _vs_publisher.Dispose();
      }

@@ -28,4 +28,4 @@ namespace BCG {


   }
-}
+}
@@ -20,11 +20,11 @@ namespace BCG {
      public ViewPublisher(string name, IPubTableBroker broker, ViewStream vs) {
         _stream = String.IsNullOrEmpty(name) ? vs : new ChangeTableNameStream(vs, name);
         _broker = broker;
-         _stream.onChange += _handle_update;
+         _stream.onChange += new DataStream.ChangeDelegate(_handle_update);
      }

      public void Dispose() {
-         _stream.onChange -= _handle_update;
+         _stream.onChange -= new DataStream.ChangeDelegate(_handle_update);
      }

      private void _handle_update(object o) {
@@ -15,74 +15,10 @@ using mbts;
 using BCG.Extensions;
 using System.Text.Json;
 using System.Text.Json.Nodes;
-using Newtonsoft.Json;
-using BCG;
-using System.Runtime.CompilerServices;
-
-namespace bts {
-
-   // public class CompositeRow : IBlueRow {
-   //    private IBlueRow _lrow, _rrow;
-   //    private BlueRow _out_row;
-   //    public CompositeRow(IBlueRow lrow, IBlueRow rrow) {
-   //       _lrow = lrow;
-   //       _rrow = rrow;
-   //       IBlueTableSchema schema = _lrow.schema;
-   //       List<object> vals = new List<object>();
-   //       foreach (var o in _lrow) {
-   //          vals.Add(o);
-   //       }
-   //       foreach (ColumnInfo ci in _rrow.schema.columnInfo) {
-   //          if (ci.colName == "id") {
-   //             continue;
-   //          }
-   //          vals.Add(_rrow[ci.colNo]);
-   //          schema = schema.add_column(ci.type_code(), ci.colName);
-   //       }
-   //       _out_row = new BlueRow(schema, vals.ToArray());
-   //    }
-   //    public IBlueTableSchema schema {
-   //       get { return _out_row.schema; }
-   //    }
-   //    public object? this[int index] {
-   //       get { return _out_row[index]; }
-   //    }
-   //    public object? this[string colName] {
-   //       get {
-   //          return _out_row[colName];
-   //       }
-   //    }
-
-   // }
-}

 namespace BCG {
   public static class BlueDataUtility {
-      public static IBlueRow create_composite_row(IBlueRow lr, IBlueRow rr, int id) {
-         IBlueTableSchema schema = lr.schema;
-         List<object> vals = new List<object>();
-         foreach (ColumnInfo ci in lr.schema.columnInfo) {
-            if (ci.colName == "id") {
-               vals.Add(id);
-            }
-            vals.Add(lr[ci.colName]);
-         }
-         foreach (ColumnInfo ci in rr.schema.columnInfo) {
-            if (ci.colName == "id") {
-               continue;
-            }
-            schema = schema.add_column(ci.type_code(), ci.colName);
-            vals.Add(rr[ci.colName]);
-         }
-         IBlueRow r = new BlueRow(schema, vals.ToArray());
-         return r;
-      }
      #region Extensions
-      public static bool is_null(this IBlueRow r, string col_name) {
-         object col_val = r[col_name];
-         return col_val == null || DBNull.Value.Equals(col_val);
-      }
-
      public static DateTime get_datetime(this IBlueRow r, string col_name, DateTime err_value = default(DateTime)) {
         return BlueDataUtility.get_row_datetime_value(r, col_name, err_value);
      }
@@ -113,17 +49,6 @@ namespace BCG {
            rv.Add(r.get<T>(col));
         return rv;
      }
-      public static IEnumerable<T> get_colvalues<T>(IEnumerable<IBlueRow> rows, string col, int start_index, int count) {
-         List<T> rv = new List<T>();
-         int i = 0;
-         int n_copied = 0;
-         foreach (IBlueRow r in rows)
-            if (i++ >= start_index && n_copied < count) {
-               rv.Add(r.get<T>(col));
-               n_copied++;
-            }
-         return rv;
-      }

      public static IDictionary<T, IEnumerable<IBlueRow>> group_by<T>(IBlueRowCollection rows, string gp_col) {
         Dictionary<T, List<IBlueRow>> rv = new Dictionary<T, List<IBlueRow>>();
@@ -187,17 +112,13 @@ namespace BCG {
         }
      }

-      public static List<IBlueRow> to_list(this IBlueRowCollection rows) {
+      public static IEnumerable<IBlueRow> to_ienum(this IBlueRowCollection rows) {
         List<IBlueRow> rv = new List<IBlueRow>();
         foreach (IBlueRow r in rows)
            rv.Add(r);
         return rv;
      }

-      public static IEnumerable<IBlueRow> to_ienum(this IBlueRowCollection rows) {
-         return rows.to_list();
-      }
-
      public static IDictionary<int, IBlueRow> to_map(this IBlueRowCollection rows) {
         Dictionary<int, IBlueRow> rv = new Dictionary<int, IBlueRow>();
         foreach (IBlueRow r in rows) {
@@ -214,11 +135,6 @@ namespace BCG {
         return new BlueTableSchema(input_schema.tableName, cis.ToArray());
      }

-      public static IBlueTableSchema add_column(this IBlueTableSchema input_schema, TypeCode type_arg, string col_name) {
-         Tuple<Type, string> tup = new Tuple<Type, string>(_typecode_to_type[type_arg], col_name);
-         return add_columns(input_schema, tup);
-      }
-
      public static IBlueRowCollection to_collection(this IBlueRow r) {
         if (r == null)
            return null;
@@ -275,18 +191,6 @@ namespace BCG {

      #region json

-      public static Dictionary<string, string> convert_json_to_dictionary(string json_string) {
-         Dictionary<string, string> dictionary = JsonConvert.DeserializeObject<Dictionary<string, string>>(json_string);
-         return dictionary;
-      }
-      public static IBlueRow convert_json_to_row(string json_string, int id = 0) {
-         Dictionary<string, object> dictionary = JsonConvert.DeserializeObject<Dictionary<string, object>>(json_string);
-         if (!dictionary.ContainsKey("id")) {
-            dictionary.Add("id", id);
-         }
-         return new BlueRow(dictionary);
-      }
-
      public static List<JsonNode> json_to_json_node_list(string jsonString) {
         var jsonNodeList = new List<JsonNode>();
         bool noe = String.IsNullOrEmpty(jsonString);
@@ -305,7 +209,7 @@ namespace BCG {
            foreach (JsonNode jsonNode in jsonArray) {
               jsonNodeList.Add(jsonNode);
            }
-         } catch (System.Text.Json.JsonException ex) {
+         } catch (JsonException ex) {
            Console.WriteLine($"Invalid JSON: {ex.Message}");
         }

@@ -582,16 +486,7 @@ namespace BCG {
         ColumnInfo[] cis = new ColumnInfo[table.Columns.Count];
         foreach (DataColumn col in table.Columns)
            cis[col.Ordinal] = new ColumnInfo(col.ColumnName, col.Ordinal, col.DataType);
-         IBlueTableSchema rv = new BlueTableSchema(cis, table.TableName);
-         rv = rv.add_id_col_if_missing();
-         return rv;
-      }
-      public static IBlueTableSchema add_id_col_if_missing(this IBlueTableSchema schema) {
-         IBlueTableSchema rv = schema;
-         if (!schema.columnNames.Contains("id")) {
-            rv = schema.add_columns(Tuple.Create(typeof(int), "id"));
-         }
-         return rv;
+         return new BlueTableSchema(cis, table.TableName);
      }

      /// <summary>
@@ -849,16 +744,10 @@ namespace BCG {
      }

      public static IBlueRowCollection toIBlueRowCollection(DataTable dt) {
-         IBlueTableSchema schema = getSchema(dt);
-         BlueTable rv = new BlueTable(schema);
-         foreach (DataRow dr in dt.Rows) {
-            MutableBlueRow nr = new MutableBlueRow(schema);
-            foreach (DataColumn col in dr.Table.Columns)
-               nr.setValue(col.ColumnName, dr[col.ColumnName]);
-            nr.setValue("id", rv.Count);
-            rv.addRow(nr);
-         }
-         return rv;
+         BlueTable retVal = new BlueTable(getSchema(dt));
+         foreach (DataRow dr in dt.Rows)
+            retVal.addRow(dr);
+         return retVal;
      }

      public static IBlueRowCollection toIBlueRowCollection(DataRow[] rows) {
@@ -930,20 +819,20 @@ namespace BCG {
      /// </summary>
      /// <param name="r1"></param>
      /// <param name="r2"></param>
-      /// <param name="col_names"></param>
+      /// <param name="colNames"></param>
      /// <returns></returns>
      public static int compare_rows(IBlueRow r1,
         IBlueRow r2,
-         params string[] col_names) {
-         int comp_res = 0;
-         foreach (string cn in col_names) {
-            object val1 = r1[cn];
-            object val2 = r2[cn];
-            comp_res = _compare_row_cols(val1, val2);
-            if (comp_res != 0)
+         params string[] colNames) {
+         int retVal = 0;
+         foreach (string colName in colNames) {
+            object val1 = r1[colName];
+            object val2 = r2[colName];
+            retVal = _compare_row_cols(val1, val2);
+            if (retVal != 0)
               break;
         }
-         return comp_res;
+         return retVal;
      }

      private static int _compare_row_cols(object v1, object v2) {
@@ -96,107 +96,3 @@ public class BlueTableImmutable : IBlueRowCollection, IImmutableRowCollection {
         new(schema, table.ToImmutable());
   }
 }
-
-public class BlueTableImmutable<T> : IBlueRowCollection<T> where T : IBlueRow {
-   readonly ImmutableDictionary<int, T> rows;
-
-   public BlueTableImmutable(IBlueTableSchema schema)
-      : this(schema, ImmutableDictionary<int, T>.Empty) { }
-
-   public BlueTableImmutable(IBlueTableSchema schema, ImmutableDictionary<int, T> rows) {
-      this.schema = schema;
-      this.rows = rows;
-   }
-
-   public int Count => rows.Count;
-   public bool IsSynchronized => false;
-   public IBlueTableSchema schema { get; }
-   public object SyncRoot => this;
-
-   public T this[int id] => rows[id];
-
-   public bool containsID(int id) =>
-      rows.ContainsKey(id);
-
-   public void CopyTo(Array array, int index) {
-      foreach (var r in rows.Values) {
-         array.SetValue(r, index++);
-      }
-   }
-
-   public IEnumerator<T> GetEnumerator() =>
-      rows.Values.GetEnumerator();
-
-   IEnumerator IEnumerable.GetEnumerator() =>
-      GetEnumerator();
-
-   public DataPacketBuilder ToDataPacketBuilder() =>
-      new(this);
-
-   public class DataPacketBuilder {
-      readonly IBlueTableSchema schema;
-      readonly ImmutableDictionary<int, T>.Builder table;
-      readonly ImmutableDictionary<int, T>.Builder added =
-         ImmutableDictionary<int, T>.Empty.ToBuilder();
-      readonly ImmutableDictionary<int, T>.Builder removed =
-         ImmutableDictionary<int, T>.Empty.ToBuilder();
-
-      public DataPacketBuilder(BlueTableImmutable<T> table) {
-         schema = table.schema;
-         this.table = table.rows.ToBuilder();
-      }
-
-      public void AddOrUpdate(T row, IEqualityComparer<IBlueRow>? comparer = null) {
-         var id = row.Id();
-         if (table.TryGetValue(id, out var orig)) {
-            if (comparer?.Equals(row, orig) ?? false) {
-               // No op: This row was already in the table (according to 'comparer').
-               // Remove it from 'added' and 'removed'.
-               added.Remove(id);
-               removed.Remove(id);
-               return;
-            }
-            // Change op: A row with this id was already in the table,
-            // but its values have changed (according to 'comparer').
-            removed[id] = orig;
-         }
-         added[id] = row;
-      }
-
-      public void Remove(int id) {
-         added.Remove(id);
-         if (table.TryGetValue(id, out var orig)) {
-            removed[id] = orig;
-         }
-      }
-
-      public DataPacket ToDataPacket(bool reset = true) {
-         table.RemoveRange(removed.Keys);
-         table.AddRange(added);
-         var newRows = new BlueTableImmutable<T>(schema, table.ToImmutable());
-         var addedRows = new BlueTableImmutable<T>(schema, added.ToImmutable());
-         var removedRows = new BlueTableImmutable<T>(schema, removed.ToImmutable());
-         if (reset) {
-            added.Clear();
-            removed.Clear();
-         }
-         return new(newRows, addedRows, removedRows);
-      }
-
-      public IDataPacket<T> ToDataPacketT(bool reset = true) {
-         table.RemoveRange(removed.Keys);
-         table.AddRange(added);
-         var newRows = new BlueTableImmutable<T>(schema, table.ToImmutable());
-         var addedRows = new BlueTableImmutable<T>(schema, added.ToImmutable());
-         var removedRows = new BlueTableImmutable<T>(schema, removed.ToImmutable());
-         if (reset) {
-            added.Clear();
-            removed.Clear();
-         }
-         return new DataPacket<T>(newRows, addedRows, removedRows);
-      }
-
-      public BlueTableImmutable<T> ToImmutable() =>
-         new(schema, table.ToImmutable());
-   }
-}
@@ -24,16 +24,7 @@ namespace BCG
      /// </summary>
      object? this[int index] {get;}
      object? this[string colName] {get;}
-      bool IDictionary.IsFixedSize => true;
-      bool IDictionary.IsReadOnly => true;
-      object ICollection.SyncRoot => this;
-      bool ICollection.IsSynchronized => false;
   }
-
-   // public interface IBlueRow<T> : IBlueRow where T : IBlueRow {
-   //    T Value => (T)this;
-   // }
-
   namespace Extensions
   {
      public static class IBlueRowExtensions
@@ -1,6 +1,5 @@
 using System;
 using System.Collections;
-using System.Collections.Generic;

 namespace BCG
 {
@@ -17,10 +16,4 @@ namespace BCG
      IBlueTableSchema schema {get;}
      bool containsID (int id);
   }
-
-   public interface IBlueRowCollection<T> : IBlueRowCollection, IReadOnlyCollection<T> where T : IBlueRow {
-      new T this[int id] { get; }
-
-      IBlueRow IBlueRowCollection.this[int id] => this[id];
-   }
 }
@@ -314,18 +314,6 @@ namespace BCG
      }
   }

-   public class DataPacket<T> : IDataPacket<T> where T : IBlueRow {
-      public DataPacket(IBlueRowCollection<T> newRows, IBlueRowCollection<T> addedRows, IBlueRowCollection<T> deletedRows) {
-         this.newRows = newRows;
-         this.addedRows = addedRows;
-         this.deletedRows = deletedRows;
-      }
-
-      public IBlueRowCollection<T> newRows { get; }
-      public IBlueRowCollection<T> addedRows { get; }
-      public IBlueRowCollection<T> deletedRows { get; }
-   }
-
   public static class DataPacketExtensions {
      public static bool HasChanges(this DataPacket dataPacket) =>
         dataPacket.addedRows.Count > 0 || dataPacket.deletedRows.Count > 0;
@@ -53,15 +53,16 @@ namespace BCG

      public event Action<Exception> ErrorChanged;

-      public event DataStreamChangeDelegate onChange
+      public delegate void ChangeDelegate(object newValue);
+      public event ChangeDelegate onChange
      {
         add 
         {
-            DataStreamChangeDelegate currValue = value;
+            ChangeDelegate currValue = value;
            if (myTestMode)
            {
               ChangeDelegateWrapper w = new ChangeDelegateWrapper (value);
-               currValue = new DataStreamChangeDelegate (w.wrapperFunction);
+               currValue = new ChangeDelegate (w.wrapperFunction);
               myDelMap.Add (value, currValue);
            }
            if (myRealSource != null)
@@ -69,7 +70,7 @@ namespace BCG
            else 
            {
               myOnChangeDelegate = 
-                  (DataStreamChangeDelegate) Delegate.Combine (myOnChangeDelegate,currValue);
+                  (ChangeDelegate) Delegate.Combine (myOnChangeDelegate,currValue);

               
               if (myOnChangeDelegate.GetInvocationList().Length == 1) // first subscriber 
@@ -94,11 +95,11 @@ namespace BCG
         }
         remove 
         {
-            var currValue = value;
+            ChangeDelegate currValue = value;
            // BlueTrace.WriteViewStreamUnsubscribe(this.ID);
            if (myTestMode)
            {
-               currValue = myDelMap[value] as DataStreamChangeDelegate;
+               currValue = myDelMap[value] as ChangeDelegate;
               myDelMap.Remove (value);
            }
            if (myRealSource != null)
@@ -106,7 +107,7 @@ namespace BCG
            else 
            {
               myOnChangeDelegate = 
-                  (DataStreamChangeDelegate) Delegate.Remove (myOnChangeDelegate,currValue);
+                  (ChangeDelegate) Delegate.Remove (myOnChangeDelegate,currValue);

               // this is need to avoid a callback in a reentrant add.
               if (myPendingAdds.Count > 0)
@@ -174,7 +175,7 @@ namespace BCG

      // this was made internal for testing purposes only.
      // don't mess with it.
-      internal DataStreamChangeDelegate myOnChangeDelegate;
+      internal ChangeDelegate myOnChangeDelegate;
      protected object myLastValue;
      protected DataStream myRealSource;
      private Stack myPendingAdds;
@@ -183,9 +184,9 @@ namespace BCG
   internal class ChangeDelegateWrapper 
   {
      readonly Stopwatch timer = new();
-      private DataStreamChangeDelegate myDel;
+      private DataStream.ChangeDelegate myDel;
      
-      internal ChangeDelegateWrapper (DataStreamChangeDelegate del) {myDel = del;}
+      internal ChangeDelegateWrapper (DataStream.ChangeDelegate del) {myDel = del;}

      internal void wrapperFunction (object o)
      {
@@ -1,4 +0,0 @@
-namespace BCG;
-
-public delegate void DataStreamChangeDelegate(object newValue);
-public delegate void DataStreamChangeDelegate<T>(IDataPacket<T> dp) where T : IBlueRow;
@@ -9,9 +9,6 @@
  <ItemGroup>
    <PackageReference Include="Google.Protobuf" Version="3.24.3" />
    <PackageReference Include="Microsoft.Extensions.Hosting" Version="7.0.1" />
-    <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
-    <PackageReference Include="OpenAI" Version="2.0.0" />
-    <PackageReference Include="SimMetrics.Net" Version="1.0.5" />
  </ItemGroup>

  <ItemGroup>
@@ -20,8 +17,8 @@
  </ItemGroup>

  <ItemGroup>
-    <Proto Include="../../lib_spark/spark/config/protos/data_stream_config.proto" CSName="DataStreamConfig">
-      <OutDir>$(ProtoOutputPath)/spark/config/protos</OutDir>
+    <Proto Include="../../lib/config/protos/data_stream_config.proto" CSName="DataStreamConfig">
+      <OutDir>$(ProtoOutputPath)/lib/config/protos</OutDir>
    </Proto>
  </ItemGroup>

@@ -8,12 +8,11 @@ namespace BCG {
   using System.Collections;
   using System.Reflection;
   using System.IO;
-   using System.Collections.Generic;

   public class PrimitiveManager {
+      private IDictionary myTypeLookupTable = new Hashtable();
      private static PrimitiveManager mySelf;
-      readonly List<string> myAssemblyNames = new();
-      readonly Dictionary<string, Type> myTypeLookupTable = new();
+      private ArrayList myAssemblyNames = new ArrayList();

      private PrimitiveManager() {
         myLoadAllTypes();
@@ -96,14 +95,15 @@ namespace BCG {
      }

      /// <summary>
-      /// Returns the Type corresponding to the primitive name
+      /// Returns the Type name corresponding to the primitive name
      /// </summary>
      /// <param name="friendlyName"></param>
      /// <returns></returns>
-      public Type find(string friendlyName) =>
-         myTypeLookupTable.TryGetValue(friendlyName, out var t)
-            ? t
-            : throw new ArgumentException($"'{friendlyName}' is not the friendly name of any of the loaded types", nameof(friendlyName));
+      public string find(string friendlyName) {
+         if (myTypeLookupTable[friendlyName] == null)
+            throw new ApplicationException(friendlyName + " is not the friendly name of any of the loaded types");
+         return (myTypeLookupTable[friendlyName] as Type).Name;
+      }

      /// <summary>
      /// Constructs and returns a ViewStream of type corresponding to the friendly name
@@ -118,8 +118,12 @@ namespace BCG {
            constructorArgs = new object[args.Count];
            args.CopyTo(constructorArgs, 0);
         }
-         var type = find(friendlyName);
-         return Activator.CreateInstance(type, constructorArgs) as ViewStream;
+         if (myTypeLookupTable[friendlyName] == null)
+            throw new ApplicationException(friendlyName + " is not the friendly name of any of the loaded types");
+         return
+            constructorArgs == null ?
+            Activator.CreateInstance(myTypeLookupTable[friendlyName] as Type) as ViewStream :
+            Activator.CreateInstance(myTypeLookupTable[friendlyName] as Type, constructorArgs) as ViewStream;
      }

      /// <summary>
@@ -12,16 +12,4 @@ namespace BCG
      //this is an ICollection of IDictionary
      void get_new_added_deleted(out ICollection new_rows, out ICollection added_rows, out ICollection deleted_rows);
   }
-
-   public interface IDataPacket<T> : IDataPacket where T : IBlueRow {
-      IBlueRowCollection<T> newRows { get; }
-      IBlueRowCollection<T> addedRows { get; }
-      IBlueRowCollection<T> deletedRows { get; }
-
-      void IDataPacket.get_new_added_deleted(out ICollection new_rows, out ICollection added_rows, out ICollection deleted_rows) {
-         new_rows = newRows;
-         added_rows = addedRows;
-         deleted_rows = deletedRows;
-      }
-   }
 }