refactor: Use generic collections in PrimitiveManager

The class that instantiates ViewStreams via reflection
Fix typo in NLStream's namespace
2024-11-02 16:48:42 +00:00 · 2024-11-02 02:57:19 +00:00 · 2024-10-30 20:42:14 +00:00 · 2024-10-30 20:13:40 +00:00 · 2024-10-30 19:37:48 +00:00 · 2024-10-30 19:37:48 +00:00
256 changed files with 10616 additions and 6507 deletions
@@ -16,11 +16,20 @@
        "program": "spark_api_server.exe",
        "args": [
          "run",
-          "--sparkd",
-          "http://127.0.0.1:8000/config"
        ],
        "cwd": "build/Debug/bin/net7.0",
      },
+      "linux": {
+        "preLaunchTask": "Build Api Server (Debug)",
+        "program": "spark_api_server",
+        "args": [
+          "run",
+        ],
+        "cwd": "build/Debug/bin/net7.0",
+      },
+      "env": {
+        "ASPNETCORE_ENVIRONMENT": "Development",
+      },
      "stopAtEntry": false,
      "console": "integratedTerminal"
    },
@@ -33,11 +42,19 @@
        "program": "spark_api_server.exe",
        "args": [
          "run",
-          "--sparkd",
-          "http://127.0.0.1:8000/config"
        ],
        "cwd": "build/Debug/bin/net7.0",
      },
+      "linux": {
+        "program": "spark_api_server",
+        "args": [
+          "run",
+        ],
+        "cwd": "build/Debug/bin/net7.0",
+      },
+      "env": {
+        "ASPNETCORE_ENVIRONMENT": "Development",
+      },
      "stopAtEntry": false,
      "console": "integratedTerminal"
    },
@@ -105,6 +122,18 @@
      "stopAtEntry": false,
      "console": "integratedTerminal"
    },
+    {
+      "name": "(gdb) Launch",
+      "type": "cppdbg",
+      "request": "launch",
+      "program": "/home/bts/spark/build/sparkd-debug-clang/server/sparkd",
+      "args": ["run"],
+      "stopAtEntry": false,
+      "cwd": "/home/bts/spark/build/sparkd-debug-clang/server",
+      "environment": [],
+      "externalConsole": false,
+      "MIMode": "gdb",
+    },
    {
      "name": "DataStreamQueryAnalyzer",
      "type": "coreclr",
@@ -253,6 +282,34 @@
        }
      ]
    },
+    {
+      "name": "Sparkd Web (Skip Build)",
+      "type": "cppdbg",
+      "request": "launch",
+      "program": "",
+      "linux": {
+        "program": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd/sparkd",
+      },
+      "args": [
+        "web"
+      ],
+      "cwd": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd",
+      "stopAtEntry": false,
+      "externalConsole": false,
+      "MIMode": "gdb",
+      "setupCommands": [
+        {
+          "description": "Enable pretty-printing for gdb",
+          "text": "-enable-pretty-printing",
+          "ignoreFailures": true
+        },
+        {
+          "description": "Set Disassembly Flavor to Intel",
+          "text": "-gdb-set disassembly-flavor intel",
+          "ignoreFailures": true
+        }
+      ]
+    },
  ],
  "inputs": [
    {
@@ -33,92 +33,85 @@ else()
  set(BUILD_ADMIN_WEB_APP ON CACHE BOOL "Build the Admin WebApp")
 endif()

-include_directories(BEFORE SYSTEM ${EXTERNALS_INCLUDE} ${EXTERNALS_ROOT}/include/zlib)
-link_directories(${EXTERNALS_LIB})
-
 add_compile_definitions(SHARED_MEMORY_PREFIX="${SHARED_MEMORY_PREFIX}")
 if(DEFINED APP_DATA_DIR)
  add_compile_definitions(APP_DATA_DIR="${APP_DATA_DIR}")
 endif()
 add_compile_definitions(EXPORT_MD_CONSUMER="${EXPORT_MD_CONSUMER}")
-if(WIN32)
-  # subdirs
-  add_subdirectory(src/lib_spark)
-else()
-  if(BUILD_SPARKD)
-    # Note: All macros that begin w/ 'CCAPI' are ccapi's macros (defined and/or
-    # read by their code). We override some below.
-    add_compile_definitions(SPARK_ENABLE_CCAPI)

-    # This allows us to override hardcoded exchange-urls with ones read from
-    # config (i.e., resolved at runtime rather than compile-time)
-    add_compile_definitions(CCAPI_EXPOSE_INTERNAL)
+if(BUILD_SPARKD)
+  # Note: All macros that begin w/ 'CCAPI' are ccapi's macros (defined and/or
+  # read by their code). We override some below.
+  add_compile_definitions(SPARK_ENABLE_CCAPI)

-    # Enable ccapi services
-    add_compile_definitions(CCAPI_ENABLE_SERVICE_MARKET_DATA)
-    add_compile_definitions(CCAPI_ENABLE_SERVICE_EXECUTION_MANAGEMENT)
-    # Enable ccapi exchanges
-    add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE)
-    add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE_COIN_FUTURES)
-    add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE_US)
-    add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE_USDS_FUTURES)
-    add_compile_definitions(CCAPI_ENABLE_EXCHANGE_COINBASE)
-    add_compile_definitions(CCAPI_ENABLE_EXCHANGE_DERIBIT)
-    add_compile_definitions(CCAPI_ENABLE_EXCHANGE_OKX)
+  # This allows us to override hardcoded exchange-urls with ones read from
+  # config (i.e., resolved at runtime rather than compile-time)
+  add_compile_definitions(CCAPI_EXPOSE_INTERNAL)

-    set(CCAPI_LOG_LEVEL
-      "Warn"
-      CACHE STRING "<Fatal|Error|Warn|Info|Debug|Trace> Log level for ccapi")
-    string(TOUPPER "${CCAPI_LOG_LEVEL}" _ccapi_log_level_upper)
-    add_compile_definitions("CCAPI_ENABLE_LOG_${_ccapi_log_level_upper}")
+  # Enable ccapi services
+  add_compile_definitions(CCAPI_ENABLE_SERVICE_MARKET_DATA)
+  add_compile_definitions(CCAPI_ENABLE_SERVICE_EXECUTION_MANAGEMENT)
+  # Enable ccapi exchanges
+  add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE)
+  add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE_COIN_FUTURES)
+  add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE_US)
+  add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE_USDS_FUTURES)
+  add_compile_definitions(CCAPI_ENABLE_EXCHANGE_COINBASE)
+  add_compile_definitions(CCAPI_ENABLE_EXCHANGE_DERIBIT)
+  add_compile_definitions(CCAPI_ENABLE_EXCHANGE_OKX)

-    # ** Binance **
-    add_compile_definitions(CCAPI_BINANCE_URL_REST_BASE="https://api.binance.com")
-    add_compile_definitions(BINANCE_INSTRUMENTS_PATH="/api/v3/exchangeInfo")
-    add_compile_definitions(BINANCE_PRICE_STATS_PATH="/api/v3/ticker/24hr")
+  set(CCAPI_LOG_LEVEL
+    "Warn"
+    CACHE STRING "<Fatal|Error|Warn|Info|Debug|Trace> Log level for ccapi")
+  string(TOUPPER "${CCAPI_LOG_LEVEL}" _ccapi_log_level_upper)
+  add_compile_definitions("CCAPI_ENABLE_LOG_${_ccapi_log_level_upper}")

-    # ** Binance US **
-    add_compile_definitions(BINANCE_US_INSTRUMENTS_PATH="/api/v3/exchangeInfo")
-    add_compile_definitions(BINANCE_US_PRICE_STATS_PATH="/api/v3/ticker/24hr")
+  # ** Binance **
+  add_compile_definitions(CCAPI_BINANCE_URL_REST_BASE="https://api.binance.com")
+  add_compile_definitions(BINANCE_INSTRUMENTS_PATH="/api/v3/exchangeInfo")
+  add_compile_definitions(BINANCE_PRICE_STATS_PATH="/api/v3/ticker/24hr")

-    # ** Binance USDS Futures **
-    add_compile_definitions(CCAPI_BINANCE_USDS_FUTURES_URL_REST_BASE="https://dapi.binance.com")
-    add_compile_definitions(BINANCE_USDS_FUTURES_INSTRUMENTS_PATH="/dapi/v1/exchangeInfo")
-    add_compile_definitions(BINANCE_USDS_FUTURES_PRICE_STATS_PATH="/dapi/v1/ticker/24hr")
+  # ** Binance US **
+  add_compile_definitions(BINANCE_US_INSTRUMENTS_PATH="/api/v3/exchangeInfo")
+  add_compile_definitions(BINANCE_US_PRICE_STATS_PATH="/api/v3/ticker/24hr")

-    # ** Deribit **
-    add_compile_definitions(DERIBIT_INSTRUMENT_PATH="/api/v2/public/get_instrument")
-    add_compile_definitions(DERIBIT_INSTRUMENTS_PATH="/api/v2/public/get_instruments")
-    add_compile_definitions(DERIBIT_COMBOS_PATH="/api/v2/public/get_combos")
-    add_compile_definitions(DERIBIT_TICKER_PATH="/api/v2/public/ticker")
-    add_compile_definitions(DERIBIT_CURRENCIES_METHOD="public/get_currencies")
-    add_compile_definitions(DERIBIT_INSTRUMENT_METHOD="public/get_instrument")
-    add_compile_definitions(DERIBIT_INSTRUMENTS_METHOD="public/get_instruments")
-    add_compile_definitions(DERIBIT_COMBO_DETAILS_METHOD="public/get_combo_details")
-    add_compile_definitions(DERIBIT_COMBOS_METHOD="public/get_combos")
+  # ** Binance USDS Futures **
+  add_compile_definitions(CCAPI_BINANCE_USDS_FUTURES_URL_REST_BASE="https://dapi.binance.com")
+  add_compile_definitions(BINANCE_USDS_FUTURES_INSTRUMENTS_PATH="/dapi/v1/exchangeInfo")
+  add_compile_definitions(BINANCE_USDS_FUTURES_PRICE_STATS_PATH="/dapi/v1/ticker/24hr")

-    # ** Okx **
-    add_compile_definitions(OKX_INSTRUMENTS_PATH="/api/v5/public/instruments")
-    add_compile_definitions(OKX_UNDERLYING_PATH="/api/v5/public/underlying")
-  endif()
+  # ** Deribit **
+  add_compile_definitions(DERIBIT_INSTRUMENT_PATH="/api/v2/public/get_instrument")
+  add_compile_definitions(DERIBIT_INSTRUMENTS_PATH="/api/v2/public/get_instruments")
+  add_compile_definitions(DERIBIT_COMBOS_PATH="/api/v2/public/get_combos")
+  add_compile_definitions(DERIBIT_TICKER_PATH="/api/v2/public/ticker")
+  add_compile_definitions(DERIBIT_CURRENCIES_METHOD="public/get_currencies")
+  add_compile_definitions(DERIBIT_INSTRUMENT_METHOD="public/get_instrument")
+  add_compile_definitions(DERIBIT_INSTRUMENTS_METHOD="public/get_instruments")
+  add_compile_definitions(DERIBIT_COMBO_DETAILS_METHOD="public/get_combo_details")
+  add_compile_definitions(DERIBIT_COMBOS_METHOD="public/get_combos")

-  # subdirs
-  if (BUILD_SPARKD)
-    add_subdirectory(src/lib_system)
-    add_subdirectory(src/lib_http)
-    add_subdirectory(src/lib_domain)
-    add_subdirectory(src/lib_python_md)
-    add_subdirectory(src/app_sparkd)
-    add_subdirectory(src/data_files)
-  endif()
-  add_subdirectory(src/lib_spark)
-  if (BUILD_ADMIN_WEB_APP)
-    add_subdirectory(src/Spark.Admin.WebApp)
-  endif()
-  if(BUILD_DOCS)
-    add_subdirectory(doc/user)
-  endif()
-  if(BUILD_SDK)
-    add_subdirectory(src/sdk)
-  endif()
+  # ** Okx **
+  add_compile_definitions(OKX_INSTRUMENTS_PATH="/api/v5/public/instruments")
+  add_compile_definitions(OKX_UNDERLYING_PATH="/api/v5/public/underlying")
+endif()
+
+# subdirs
+add_subdirectory(src/lib_system)
+add_subdirectory(src/lib_domain)
+add_subdirectory(src/lib_http)
+if (BUILD_SPARKD)
+  add_subdirectory(src/lib_python_md)
+  add_subdirectory(src/app_sparkd)
+  add_subdirectory(src/data_files)
+endif()
+add_subdirectory(src/lib_spark)
+if (BUILD_ADMIN_WEB_APP)
+  add_subdirectory(src/Spark.Admin.WebApp)
+endif()
+if(BUILD_DOCS)
+  add_subdirectory(doc/user)
+endif()
+if(BUILD_SDK)
+  add_subdirectory(src/sdk)
 endif()
@@ -28,7 +28,7 @@
        "CMAKE_SHARED_LINKER_FLAGS_RELEASE": "/OPT:REF /OPT:ICF /LTCG",
        "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
        "EXTERNALS_ROOT": "${sourceDir}/externals/windows",
-        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
+        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf",
        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0;${sourceDir}/build/$<CONFIG>/bin/net7.0-windows",
        "EXPORT_MD_CONSUMER": {
          "type": "BOOL",
@@ -168,8 +168,8 @@
          "type": "BOOL",
          "value": "ON"
        },
-        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/Spark.Api.Server/$<CONFIG>/bin/net7.0",
+        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf",
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
        "EXPORT_MD_CONSUMER": {
          "type": "BOOL",
          "value": "ON"
@@ -207,7 +207,7 @@
      "description": "Data Stream utils ported over from Edge",
      "binaryDir": "${sourceDir}/build/${presetName}",
      "cacheVariables": {
-        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf",
+        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf;spark/config/profiles/data_stream/*.conf",
        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0;${sourceDir}/build/$<CONFIG>/bin/net7.0-windows"
      }
    },
@@ -240,8 +240,7 @@
        "CMAKE_CXX_COMPILER": "clang++",
        "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
-        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf",
-        "CONFIG_PROFILES": "default;data_stream",
+        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf;spark/config/profiles/data_stream/*.conf",
        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0;${sourceDir}/build/$<CONFIG>/bin/net8.0",
        "EXPORT_MD_CONSUMER": {
          "type": "BOOL",
@@ -14,6 +14,16 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "spark_test", "src\client\sp
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Spark.Console", "src\client\Spark.Console\Spark.Console.csproj", "{89D666E6-B500-498F-B026-4D167F68ADFC}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Net", "src\DataStreamSuite\Bts.Net\Bts.Net.csproj", "{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamCore", "src\DataStreamSuite\DataStreamCore\DataStreamCore.csproj", "{CB62D5D2-3495-4488-BFBC-7C7A3725A1EE}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStream", "src\DataStreamSuite\DataStream\DataStream.csproj", "{05995ED5-650A-4955-9DFC-3242B737E77A}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SharedStreams", "src\DataStreamSuite\SharedStreams\SharedStreams.csproj", "{A8B861DE-E098-4291-98EE-76AA0F5F5A7B}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PubSub", "src\DataStreamSuite\PubSub\PubSub.csproj", "{EFA9F78D-083E-457A-BABB-9386C88C32C0}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|x64 = Debug|x64
@@ -70,6 +80,46 @@ Global
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.Release|x64.Build.0 = Release|x64
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|x64.ActiveCfg = Debug|x64
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|x64.Build.0 = Debug|x64
+		{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.Debug|x64.Build.0 = Debug|Any CPU
+		{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.Release|x64.ActiveCfg = Release|Any CPU
+		{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.Release|x64.Build.0 = Release|Any CPU
+		{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{CB62D5D2-3495-4488-BFBC-7C7A3725A1EE}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{CB62D5D2-3495-4488-BFBC-7C7A3725A1EE}.Debug|x64.Build.0 = Debug|Any CPU
+		{CB62D5D2-3495-4488-BFBC-7C7A3725A1EE}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{CB62D5D2-3495-4488-BFBC-7C7A3725A1EE}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{CB62D5D2-3495-4488-BFBC-7C7A3725A1EE}.Release|x64.ActiveCfg = Release|Any CPU
+		{CB62D5D2-3495-4488-BFBC-7C7A3725A1EE}.Release|x64.Build.0 = Release|Any CPU
+		{CB62D5D2-3495-4488-BFBC-7C7A3725A1EE}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{CB62D5D2-3495-4488-BFBC-7C7A3725A1EE}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{05995ED5-650A-4955-9DFC-3242B737E77A}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{05995ED5-650A-4955-9DFC-3242B737E77A}.Debug|x64.Build.0 = Debug|Any CPU
+		{05995ED5-650A-4955-9DFC-3242B737E77A}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{05995ED5-650A-4955-9DFC-3242B737E77A}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{05995ED5-650A-4955-9DFC-3242B737E77A}.Release|x64.ActiveCfg = Release|Any CPU
+		{05995ED5-650A-4955-9DFC-3242B737E77A}.Release|x64.Build.0 = Release|Any CPU
+		{05995ED5-650A-4955-9DFC-3242B737E77A}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{05995ED5-650A-4955-9DFC-3242B737E77A}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{A8B861DE-E098-4291-98EE-76AA0F5F5A7B}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{A8B861DE-E098-4291-98EE-76AA0F5F5A7B}.Debug|x64.Build.0 = Debug|Any CPU
+		{A8B861DE-E098-4291-98EE-76AA0F5F5A7B}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{A8B861DE-E098-4291-98EE-76AA0F5F5A7B}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{A8B861DE-E098-4291-98EE-76AA0F5F5A7B}.Release|x64.ActiveCfg = Release|Any CPU
+		{A8B861DE-E098-4291-98EE-76AA0F5F5A7B}.Release|x64.Build.0 = Release|Any CPU
+		{A8B861DE-E098-4291-98EE-76AA0F5F5A7B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{A8B861DE-E098-4291-98EE-76AA0F5F5A7B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{EFA9F78D-083E-457A-BABB-9386C88C32C0}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{EFA9F78D-083E-457A-BABB-9386C88C32C0}.Debug|x64.Build.0 = Debug|Any CPU
+		{EFA9F78D-083E-457A-BABB-9386C88C32C0}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{EFA9F78D-083E-457A-BABB-9386C88C32C0}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{EFA9F78D-083E-457A-BABB-9386C88C32C0}.Release|x64.ActiveCfg = Release|Any CPU
+		{EFA9F78D-083E-457A-BABB-9386C88C32C0}.Release|x64.Build.0 = Release|Any CPU
+		{EFA9F78D-083E-457A-BABB-9386C88C32C0}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{EFA9F78D-083E-457A-BABB-9386C88C32C0}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -72,6 +72,14 @@ function(init_library)
  else()
    add_library(${_name} ${${_name}_SOURCES} ${${_name}_CXX_PROTO_SOURCES})
  endif()
+  target_include_directories(
+    ${_name}
+    SYSTEM
+    BEFORE
+    PUBLIC
+      $<BUILD_INTERFACE:${EXTERNALS_ROOT}/include>
+      $<BUILD_INTERFACE:${EXTERNALS_ROOT}/include/zlib>
+  )
  target_include_directories(
    ${_name}
    PUBLIC
@@ -90,6 +98,7 @@ function(init_library)
    ${_name}
    PUBLIC
      $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
+      $<BUILD_INTERFACE:${EXTERNALS_ROOT}/lib/$<CONFIG>>
      $<INSTALL_INTERFACE:${CMAKE_INSTALL_LIBDIR}>
  )
  # Install main lib
@@ -33,7 +33,7 @@ function(protobuf_generate_cpp_config)

  # Generate static defaults. Ideally, this would be a single string, but we use an array because
  # the compiler complains if the string is longer than 65k.
-  set(_cfg_defaults_str "const char * config_static_defaults[] = { ")
+  set(_cfg_defaults_str "inline const char * config_static_defaults[] = { ")

  foreach(_cfg_file_path ${_ARGS_PROTO_DEFAULTS})
    file(READ ${_cfg_file_path} _cfg_file_content)
@@ -20,16 +20,14 @@ endfunction()

 function(sym_link_file)
  set(_options)
-  set(_one_value_args TARGET_NAME SOURCE DEST INSTALL_DEST)
+  set(_one_value_args SOURCE DEST INSTALL_DEST)
  set(_multi_value_args)
  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-  add_custom_command(
-    OUTPUT ${_ARGS_DEST}
-    COMMAND ln ARGS -s ${_ARGS_SOURCE} ${_ARGS_DEST}
-  )
-  add_custom_target(
-    ${_ARGS_TARGET_NAME} ALL
-    DEPENDS ${_ARGS_DEST}
+  file(
+    CREATE_LINK
+      ${_ARGS_SOURCE}
+      ${_ARGS_DEST}
+    SYMBOLIC
  )
  if (DEFINED _ARGS_INSTALL_DEST)
    install(
@@ -1,15 +1,24 @@
 services:
  sparkd:
    build:
-      dockerfile: src/server/sparkd/Dockerfile
+      dockerfile: src/app_sparkd/Dockerfile
+      args:
+        - SOURCE_DIR=install/sparkd-release-gcc
+        - CONF_FILE=src/app_sparkd/spark.conf
    command: [ "./sparkd", "run" ]
-    ports: [ "8000:8000", "8001:8001", "8002:8002" ]
+    ports:
+      - 8000:8000
+      - 8001:8001
+      - 8002:8002

  pubsub:
    environment:
      - DOTNET_ENVIRONMENT=Staging
    build:
      dockerfile: src/DataStreamSuite/PubSubServer/Dockerfile
+      args:
+        - SOURCE_DIR=build/Release/bin/net7.0/linux-x64/publish
+        - CONF_FILE=src/DataStreamSuite/PubSubServer/spark.conf
    ports:
      - 22019:22019
    command: [ "./PubSubServer" ]
@@ -17,20 +26,13 @@ services:
  md-publisher:
    build:
      dockerfile: src/DataStreamSuite/DataStreamPublisher/Dockerfile
+      args:
+        - SOURCE_DIR=build/Release/bin/net7.0/linux-x64/publish
+        - CONF_FILE=src/DataStreamSuite/DataStreamPublisher/spark.conf
    depends_on:
      - pubsub
      - sparkd
-    command:
-      [
-        "dockerize",
-        "-wait",
-        "tcp://pubsub:22019",
-        "-wait",
-        "tcp://sparkd:8000",
-        "-timeout",
-        "10s",
-        "./DataStreamPublisher"
-      ]
+    command: [ "dockerize", "-wait", "tcp://pubsub:22019", "-wait", "tcp://sparkd:8000", "-timeout", "10s", "./DataStreamPublisher" ]
    environment:
      - DOTNET_ENVIRONMENT=MarketData

@@ -40,13 +42,15 @@ services:
      - 8443:8443
    build:
      dockerfile: src/DataStreamSuite/Bts.DataStreamWebApp.Server/Dockerfile
+      args:
+        - SOURCE_DIR=build/Release/bin/net7.0/linux-x64/publish
+        - WEB_ROOT=build/Bts.DataStreamWebApp.Client/production/dist
+        - CONF_FILE=src/DataStreamSuite/Bts.DataStreamWebApp.Server/spark.conf
    command: [ "./Bts.DataStreamWebApp.Server" ]
    environment:
      - ASPNETCORE_ENVIRONMENT=Staging

  api-server:
-    profiles:
-      - api-server
    depends_on:
      - sparkd
    ports:
@@ -54,7 +58,7 @@ services:
      - 9443:9443
    build:
      dockerfile: src/client/spark_api_server/Dockerfile
-    command: [ "./spark_api_server", "run" ]
+    command: [ "dockerize", "-wait", "tcp://sparkd:8000", "-timeout", "10s", "./spark_api_server", "run" ]
    environment:
      - ASPNETCORE_ENVIRONMENT=Staging

@@ -1 +1 @@
-VITE_DATASTREAM_WEBSOCKET_URL=wss://localhost:443
+VITE_DATASTREAM_WEBSOCKET_URL=wss://localhost:8443/ws
@@ -1 +1 @@
-VITE_DATASTREAM_WEBSOCKET_URL=wss://localhost:443
+VITE_DATASTREAM_WEBSOCKET_URL=wss://localhost:8443/ws
@@ -1,32 +0,0 @@
-module.exports = {
-  root: true,
-  env: { browser: true, es2020: true },
-  extends: [
-    'eslint:recommended',
-    'plugin:react/recommended',
-    'plugin:react/jsx-runtime',
-    'plugin:react-hooks/recommended',
-  ],
-  ignorePatterns: ['dist', '.eslintrc.cjs'],
-  parserOptions: { ecmaVersion: 'latest', sourceType: 'module' },
-  settings: { react: { version: '18.3.1' } },
-  plugins: ['react-refresh'],
-  rules: {
-    "import/order": [
-      "error",
-      {
-        "groups": [
-          "builtin",
-          "external",
-          "internal",
-          ["parent", "sibling", "index"]
-        ],
-        "newlines-between": "always"
-      }
-    ],
-    'react-refresh/only-export-components': [
-      'warn',
-      { allowConstantExport: true },
-    ],
-  },
-}
@@ -0,0 +1,36 @@
+import importPlugin from "eslint-plugin-import";
+import reactPkg from "eslint-plugin-react";
+const { configs: reactConfigs } = reactPkg;
+import js from "@eslint/js";
+import defineConfig from "./vite.config.js";
+
+export default [
+  js.configs.recommended,
+  importPlugin.flatConfigs.recommended,
+  reactConfigs.flat.recommended,
+  reactConfigs.flat["jsx-runtime"],
+  {
+    files: ["**/*.{js,jsx}"],
+    languageOptions: {
+      ecmaVersion: "latest",
+      sourceType: "module",
+      globals: {
+        console: "readonly"
+      }
+    },
+    settings: {
+      "import/resolver": {
+        vite: {
+          viteConfig: defineConfig({ mode: "development" })
+        }
+      }
+    },
+    rules: {
+      semi: "error",
+      "prefer-const": "error",
+      "no-unused-vars": "warn",
+      "import/no-dynamic-require": "warn",
+      "import/no-nodejs-modules": "warn",
+    }
+  }
+];
@@ -10,35 +10,37 @@
    "preview": "vite preview"
  },
  "dependencies": {
-    "@angular/material": "^18.0.4",
-    "@emotion/react": "^11.11.4",
-    "@emotion/styled": "^11.11.5",
-    "@fontsource/noto-sans": "^5.0.22",
-    "@material/data-table": "^15.0.0-canary.7f224ddd4.0",
-    "@mui/icons-material": "^5.15.20",
-    "@mui/material": "^5.15.20",
-    "@mui/x-data-grid": "^7.7.1",
-    "bulma": "^1.0.1",
+    "@angular/material": "^18.2.8",
+    "@emotion/react": "^11.13.3",
+    "@emotion/styled": "^11.13.0",
+    "@fontsource/noto-sans": "^5.1.0",
+    "@material/data-table": "^15.0.0-canary.423edc3dc.0",
+    "@mui/icons-material": "^6.1.4",
+    "@mui/material": "^6.1.4",
+    "@mui/x-data-grid": "^7.20.0",
+    "bulma": "^1.0.2",
    "classnames": "^2.5.1",
    "react": "^18.3.1",
-    "react-data-grid": "^7.0.0-beta.44",
+    "react-data-grid": "^7.0.0-beta.47",
    "react-data-table-component": "^7.6.2",
    "react-dom": "^18.3.1",
-    "react-json-view-lite": "^1.4.0",
+    "react-json-view-lite": "^1.5.0",
    "react-pro-sidebar": "^1.1.0",
-    "react-use-websocket": "^4.8.1",
-    "rpc-websockets": "^7.11.2"
+    "react-use-websocket": "^4.9.0",
+    "rpc-websockets": "^9.0.4"
  },
  "devDependencies": {
-    "@types/react": "^18.3.3",
-    "@types/react-dom": "^18.3.0",
-    "@vitejs/plugin-react": "^4.3.1",
-    "eslint": "^8.57.0",
-    "eslint-plugin-import": "^2.29.1",
-    "eslint-plugin-react": "^7.34.2",
-    "eslint-plugin-react-hooks": "^4.6.2",
-    "eslint-plugin-react-refresh": "^0.4.7",
-    "react-mui-sidebar": "^1.3.5",
-    "vite": "^5.2.13"
+    "@types/react": "^18.3.11",
+    "@types/react-dom": "^18.3.1",
+    "@vitejs/plugin-react": "^4.3.2",
+    "eslint": "^9.12.0",
+    "@eslint/js": "^9.12.0",
+    "eslint-plugin-import": "^2.31.0",
+    "eslint-plugin-react": "^7.37.1",
+    "eslint-plugin-react-hooks": "^5.0.0",
+    "eslint-plugin-react-refresh": "^0.4.12",
+    "eslint-import-resolver-vite": "^2.0.1",
+    "react-mui-sidebar": "^1.3.8",
+    "vite": "^5.4.9"
  }
 }
@@ -1,5 +1,5 @@
 import { fileURLToPath, URL } from 'node:url';
-import { defineConfig, loadEnv } from 'vite';
+import { defineConfig } from 'vite';
 import plugin from '@vitejs/plugin-react';
 import fs from 'fs';
 import path from 'path';
@@ -1,11 +1,13 @@
 FROM bluetradingsystems/linux_dotnet7

-ARG SOURCE_DIR="build/Release/bin/net7.0/linux-x64/publish"
+ARG CS_SOURCE_DIR="build/Release/bin/net7.0/linux-x64/publish"
+ARG CPP_SOURCE_DIR="install/data-stream-suite-linux"
 ARG WEB_ROOT="build/Bts.DataStreamWebApp.Client/production/dist"
 ARG CONF_FILE="src/DataStreamSuite/Bts.DataStreamWebApp.Server/spark.conf"

 RUN dotnet dev-certs https --trust
-COPY --chown=bts:bts "${SOURCE_DIR}" "bin"
+COPY --chown=bts:bts "${CPP_SOURCE_DIR}/lib" "bin"
+COPY --chown=bts:bts "${CS_SOURCE_DIR}" "bin"
 COPY --chown=bts:bts "${WEB_ROOT}" "bin/wwwroot"
 COPY --chown=bts:bts "${CONF_FILE}" ".local/share/bts/Bts.DataStreamWebApp.Server/conf/spark.conf"
 WORKDIR "/home/bts/bin"
@@ -1,14 +1,4 @@
 {
-  "Kestrel": {
-    "Endpoints": {
-      "Http": {
-        "Url": "http://localhost:8080"
-      },
-      "Https": {
-        "Url": "https://localhost:8443"
-      }
-    }
-  },
  "PubSub": {
    "Routing": {
      "PublisherRoutes": [
@@ -30,5 +30,15 @@
        "benzinga_news"
      ]
    }
-  ]
+  ],
+  "Spark": {
+    "ConfigServer": "http://sparkd:8000/config"
+  },
+  "SparkApi": {
+    "IsEnabled": true,
+    "Credentials": {
+      "User": "bts",
+      "Password": ""
+    }
+  }
 }
@@ -18,8 +18,11 @@

  <ItemGroup>
    <PackageReference Include="HtmlAgilityPack" Version="1.11.57" />
+    <PackageReference Include="Microsoft.Data.Analysis" Version="0.21.1" />
    <PackageReference Include="MongoDB.Bson" Version="2.23.1" />
    <PackageReference Include="MongoDB.Driver" Version="2.23.1" />
+    <PackageReference Include="Parquet.Net" Version="5.0.1" />
+    <PackageReference Include="System.Data.SQLite" Version="1.0.118" />
  </ItemGroup>

 </Project>
@@ -11,6 +11,9 @@ public static class DataStreamExtensions {
   public static ViewStream AsViewStream(this IObservable<DataPacket> source) =>
      source as ViewStream ?? new ObservableViewStream(source);

+   public static ViewStream AsViewStream<T>(this IObservable<IDataPacket<T>> source) where T : IBlueRow =>
+      source as ViewStream ?? new ObservableViewStream(source.Select(dp => new DataPacket(dp.newRows, dp.addedRows, dp.deletedRows)));
+
   public static IObservable<DataPacket> AsObservable(this ViewStream source) {
      return source as IObservable<DataPacket> ?? Observable.Create<DataPacket>(Subscribe);

@@ -0,0 +1,32 @@
+namespace bts.SilverPine {
+   using System;
+   using BCG;
+   using System.Collections.Generic;
+   using Bts.DataStreamExpressions;
+
+   // All this does is take in a dsl expression, parse it and fire the data that come out of running it.
+   // So for example, if an expresison is passed 
+   [DataStreamUserRespresentation("dsl")]
+   public class DSLStream : ViewStream {
+      private readonly string _dsl;
+      public DSLStream(string dsl) {
+         Log.Gen.info($"DSLStream.cons dsl: {dsl}");
+         _dsl = dsl;
+         ViewStream rs = build_stream(dsl);
+         rs = rs ?? new LiteralStream("error", $"msg:invalid dsl \"{dsl}\"");
+         mySetRealSource(rs);
+      }
+
+      static ViewStream? build_stream(string stream_expression) {
+         try {
+            var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression);
+            var rv = exp.getView(new Dictionary<string, StatementExpression>(0));
+            return rv;
+
+         } catch (Exception ex) {
+            Log.Gen.error($"Could not build stream: {stream_expression} ex: {ex}");
+            return null;
+         }
+      }
+   }
+}
@@ -25,12 +25,12 @@ namespace BCG {
         byte[] rem_req_bytes = rem_req.serialize();
         _publisher_core.publish("System", "RemoteRequest", rem_req_bytes);
         _source_str = new TableStream(_name);
-         _source_str.onChange += new ChangeDelegate(_handle_data);
+         _source_str.onChange += _handle_data;
      }

      protected override void myOnLastSubscriber() {
         //maybe there should be a way to turn remote requests off ?
-         _source_str.onChange -= new ChangeDelegate(_handle_data);
+         _source_str.onChange -= _handle_data;
      }

      static string get_expression(params string[] lines) {
@@ -21,13 +21,11 @@ namespace BCG
         : this (new TableStream (tableName))
      {}

-      protected override void myOnFirstSubscriber () {
-         myVS.onChange += new ChangeDelegate(myHandleChanges);
-      }
-      protected override void myOnLastSubscriber () {
-         myVS.onChange -= new ChangeDelegate(myHandleChanges);
-      }
+      protected override void myOnFirstSubscriber () =>
+         myVS.onChange += myHandleChanges;

+      protected override void myOnLastSubscriber () =>
+         myVS.onChange -= myHandleChanges;

      private void myHandleChanges(object newValue)
      {
@@ -30,7 +30,7 @@ namespace BCG

        protected override void myOnFirstSubscriber()
        {
-            myToCrush.onChange += new ChangeDelegate(myToCrush_onChange);
+            myToCrush.onChange += myToCrush_onChange;
        }

        protected override void myOnLastSubscriber()
@@ -15,11 +15,11 @@ namespace BCG
            
            //set up an anonymous delegate for each of the streams
            //we do this so that we can unsubscribe
-            myDelegates = new ChangeDelegate[myToCrush.Length];
+            myDelegates = new DataStreamChangeDelegate[myToCrush.Length];
            for (int i = 0; i < myToCrush.Length; ++i)
            {
                int tempI = i;
-                myDelegates[tempI] = new ChangeDelegate((o) => { myHandleChange(tempI, ((DataPacket)o).newRows); });
+                myDelegates[tempI] = new((o) => { myHandleChange(tempI, ((DataPacket)o).newRows); });
            }
        }

@@ -27,7 +27,7 @@ namespace BCG
        private ViewStream[] myToCrush;
        private ITableCrusher myCrusher;

-        private ChangeDelegate[] myDelegates;
+        private DataStreamChangeDelegate[] myDelegates;

        protected override void myOnFirstSubscriber()
        {
@@ -28,7 +28,7 @@ namespace BCG

      protected override void myOnFirstSubscriber()
      {
-         myDataSource.onChange += new ChangeDelegate(myDataSource_onChange);
+         myDataSource.onChange += myDataSource_onChange;
      }

      protected override void myOnLastSubscriber()
@@ -18,7 +18,8 @@ namespace bts {
         _filename = filename;
      }

-      protected override void myOnFirstSubscriber() { _stream.onChange += new ChangeDelegate(_handle_changes); }
+      protected override void myOnFirstSubscriber() =>
+         _stream.onChange += _handle_changes;

      void _handle_changes(object dp_obj) {
         DataPacket dp = dp_obj as DataPacket;
@@ -26,7 +27,8 @@ namespace bts {
         myFireEvent(dp_obj);
      }

-      protected override void myOnLastSubscriber() { _stream.onChange -= new ChangeDelegate(_handle_changes); }
+      protected override void myOnLastSubscriber() =>
+         _stream.onChange -= _handle_changes;

      protected void _save(DataPacket dp) {
         StreamIOMessage.CreateMessage cr_msg = new StreamIOMessage.CreateMessage(dp.newRows);
@@ -37,12 +37,12 @@ namespace BCG.StreamWorld.Addition

      protected override void myOnFirstSubscriber()
      {
-         myBaseStream.onChange += new ChangeDelegate(myHandleChanges);
+         myBaseStream.onChange += myHandleChanges;
      }

      protected override void myOnLastSubscriber()
      {
-         myBaseStream.onChange -= new ChangeDelegate(myHandleChanges);
+         myBaseStream.onChange -= myHandleChanges;
         myBaseTable.clear();
         myStateTable.clear();
      }
@@ -228,13 +228,13 @@ namespace BCG
      #endregion

      #region overridden methods
-      protected override void myOnFirstSubscriber()
-      { myParentStream.onChange += new ChangeDelegate(myHandleCallbacks); }
+      protected override void myOnFirstSubscriber() =>
+         myParentStream.onChange += myHandleCallbacks;

      protected override void myOnLastSubscriber()
      {
         myLastFiredRows = null;
-         myParentStream.onChange -= new ChangeDelegate(myHandleCallbacks);
+         myParentStream.onChange -= myHandleCallbacks;
      }

      protected override void myFireEvent(object o)
@@ -0,0 +1,22 @@
+namespace bts.SilverPine {
+   using BCG;
+
+   using System;
+
+   public class NullFilterStream : ViewStream {
+      public NullFilterStream(ViewStream parent_str, string col_name, bool filter_out_doublenans = false) {
+         ViewStream rs = new FilterStream(parent_str,
+         r => {
+            bool is_dbnull = r.is_null(col_name);
+            if (is_dbnull) {
+               return false;
+            }
+            if (r.schema[col_name].type_code() == TypeCode.Double) {
+               return !Double.IsNaN(r.get<double>(col_name));
+            }
+            return true;
+         });
+         mySetRealSource(rs);
+      }
+   }
+}
@@ -36,11 +36,11 @@ namespace BCG
         get{return myIsFrozen;}
      }

-      protected override void myOnFirstSubscriber ()
-      {myStreamToFreeze.onChange += new DataStream.ChangeDelegate (myCallBack);}
+      protected override void myOnFirstSubscriber () =>
+         myStreamToFreeze.onChange += myCallBack;

-      protected override void myOnLastSubscriber ()
-      {myStreamToFreeze.onChange -= new DataStream.ChangeDelegate (myCallBack);}
+      protected override void myOnLastSubscriber () =>
+         myStreamToFreeze.onChange -= myCallBack;

      internal void myCallBack(object o)
      {
@@ -96,15 +96,15 @@ namespace BCG

      protected override void myOnFirstSubscriber()
      {
-         this.myLeftSource.onChange += new ChangeDelegate(myLeft_onChange);
-         this.myRightSource.onChange += new ChangeDelegate(myRight_onChange);
+         myLeftSource.onChange += myLeft_onChange;
+         myRightSource.onChange += myRight_onChange;
         this.myVsd.addWorker(this);
      }

      protected override void myOnLastSubscriber()
      {
-         this.myLeftSource.onChange -= new ChangeDelegate(myLeft_onChange);
-         this.myRightSource.onChange -= new ChangeDelegate(myRight_onChange);
+         myLeftSource.onChange -= myLeft_onChange;
+         myRightSource.onChange -= myRight_onChange;
         this.myVsd.removeWorker(this);
      }

@@ -1,102 +1,102 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
+// using System;
+// using System.Collections.Generic;
+// using System.Linq;
+// using System.Text;

-namespace BCG
-{
+// namespace BCG
+// {
   
-    public class AverageGroupsStream : ViewStream
-    {
-        public AverageGroupsStream(ViewStream input, string colToAverage, string keyColumn)
-        {
+//     public class AverageGroupsStream : ViewStream
+//     {
+//         public AverageGroupsStream(ViewStream input, string colToAverage, string keyColumn)
+//         {
            
-            myColToAverage = colToAverage;
-            myInput = input;
-            myKeyColumn = keyColumn;
-        }
+//             myColToAverage = colToAverage;
+//             myInput = input;
+//             myKeyColumn = keyColumn;
+//         }

-        private ViewStream myInput;
-        private string myColToAverage;
-        private BlueTableSchema mySchema;
-        private string myKeyColumn;
+//         private ViewStream myInput;
+//         private string myColToAverage;
+//         private BlueTableSchema mySchema;
+//         private string myKeyColumn;

-        protected override void myOnFirstSubscriber()
-        {
-            myInput.onChange += new ChangeDelegate(myInput_onChange);
-        }
-        protected override void myOnLastSubscriber()
-        {
-            myInput.onChange -= myInput_onChange;
-        }
-        private void myInput_onChange(object newValue)
-        {
-            if (mySchema == null)
-                myBuildSchema(((DataPacket)newValue).schema);
+//         protected override void myOnFirstSubscriber()
+//         {
+//             myInput.onChange += new ChangeDelegate(myInput_onChange);
+//         }
+//         protected override void myOnLastSubscriber()
+//         {
+//             myInput.onChange -= myInput_onChange;
+//         }
+//         private void myInput_onChange(object newValue)
+//         {
+//             if (mySchema == null)
+//                 myBuildSchema(((DataPacket)newValue).schema);

-            IBlueRowCollection newRows = myEvaluate((DataPacket)newValue);
+//             IBlueRowCollection newRows = myEvaluate((DataPacket)newValue);

-            if (myLastValue == null)
-                myFireEvent(new DataPacket(newRows, newRows, new BlueTable(mySchema)));
-            else
-            {
-                IBlueRowCollection added, deleted;
-                BlueDataUtility.computeDifferences(newRows, ((DataPacket) myLastValue).newRows, out deleted, out added);
-                myFireEvent(new DataPacket(newRows, added, deleted));
-            }
-        }
+//             if (myLastValue == null)
+//                 myFireEvent(new DataPacket(newRows, newRows, new BlueTable(mySchema)));
+//             else
+//             {
+//                 IBlueRowCollection added, deleted;
+//                 BlueDataUtility.computeDifferences(newRows, ((DataPacket) myLastValue).newRows, out deleted, out added);
+//                 myFireEvent(new DataPacket(newRows, added, deleted));
+//             }
+//         }

-        private void myBuildSchema(IBlueTableSchema schema)
-        {
-            List<ColumnInfo> columns = new List<ColumnInfo>();
+//         private void myBuildSchema(IBlueTableSchema schema)
+//         {
+//             List<ColumnInfo> columns = new List<ColumnInfo>();

-            foreach (ColumnInfo ci in schema.columnInfo)
-                columns.Add(ci);
+//             foreach (ColumnInfo ci in schema.columnInfo)
+//                 columns.Add(ci);

-            columns.Add(new ColumnInfo("average_"+myColToAverage, columns.Count, typeof(double)));
-            mySchema = new BlueTableSchema("average_of_" + schema.tableName, columns.ToArray());
-        }
+//             columns.Add(new ColumnInfo("average_"+myColToAverage, columns.Count, typeof(double)));
+//             mySchema = new BlueTableSchema("average_of_" + schema.tableName, columns.ToArray());
+//         }



-        private IBlueRowCollection myEvaluate(DataPacket packet)
-        {
+//         private IBlueRowCollection myEvaluate(DataPacket packet)
+//         {
         
-            IDictionary<string, double> colToRunningTotal = new Dictionary<string, double>();
-            IDictionary<string, int> colToCount = new Dictionary<string, int>();
-            IDictionary<string, IBlueRow> colToLastRow = new Dictionary<string, IBlueRow>();
+//             IDictionary<string, double> colToRunningTotal = new Dictionary<string, double>();
+//             IDictionary<string, int> colToCount = new Dictionary<string, int>();
+//             IDictionary<string, IBlueRow> colToLastRow = new Dictionary<string, IBlueRow>();

-            foreach (IBlueRow r in packet.newRows)
-            {
-                double thisVal = Convert.ToDouble(r[myColToAverage]);
+//             foreach (IBlueRow r in packet.newRows)
+//             {
+//                 double thisVal = Convert.ToDouble(r[myColToAverage]);

-                string keyVal = r[myKeyColumn].ToString();
-                if (!colToCount.ContainsKey(keyVal))
-                {
-                    colToRunningTotal[keyVal] = thisVal;
-                    colToCount[keyVal] = 1;
-                }
-                else
-                {
-                    colToRunningTotal[keyVal] += thisVal;
-                    colToCount[keyVal] += 1;
-                }
-                colToLastRow[keyVal] = r;
-            }
+//                 string keyVal = r[myKeyColumn].ToString();
+//                 if (!colToCount.ContainsKey(keyVal))
+//                 {
+//                     colToRunningTotal[keyVal] = thisVal;
+//                     colToCount[keyVal] = 1;
+//                 }
+//                 else
+//                 {
+//                     colToRunningTotal[keyVal] += thisVal;
+//                     colToCount[keyVal] += 1;
+//                 }
+//                 colToLastRow[keyVal] = r;
+//             }

-            BlueTable result = new BlueTable(mySchema);
-            foreach (string key in colToCount.Keys)
-            {
-                object[] values = new object[mySchema.columnInfo.Length];
+//             BlueTable result = new BlueTable(mySchema);
+//             foreach (string key in colToCount.Keys)
+//             {
+//                 object[] values = new object[mySchema.columnInfo.Length];

-                colToLastRow[key].CopyTo(values, 0);
-                values[values.Length - 1] = packet.newRows.Count > 0 ? colToRunningTotal[key] / packet.newRows.Count : 0.0;
-                result.addRow(new BlueRow(mySchema, values));
-            }
+//                 colToLastRow[key].CopyTo(values, 0);
+//                 values[values.Length - 1] = packet.newRows.Count > 0 ? colToRunningTotal[key] / packet.newRows.Count : 0.0;
+//                 result.addRow(new BlueRow(mySchema, values));
+//             }

-            return result;
-        }
+//             return result;
+//         }

      
-    }
-}
+//     }
+// }
@@ -0,0 +1,31 @@
+namespace bts.SilverPine {
+   using BCG;
+
+   using System;
+   using System.Collections.Generic;
+   using System.Linq;
+
+   [DataStreamUserRespresentation("Average")]
+   public class AverageNonIncGroupsStream : ViewStream {
+      public AverageNonIncGroupsStream(ViewStream parent_stream, string inp_col_name, params string[] group_by_cols) {
+         NonIncGroupByStream rs = new NonIncGroupByStream(parent_stream, () => group_by_cols,
+                                                         (g_rows) => _find_mean(inp_col_name, g_rows));
+         mySetRealSource(rs);
+      }
+
+      private List<IBlueRow> _find_mean(string inp_col_name, List<IBlueRow> g_rows) {
+         IBlueRow inp_row = g_rows.ElementAt(0);
+         IEnumerable<double> inp_vals = BlueDataUtility.get_colvalues<double>(g_rows, inp_col_name);
+         double sum = 0.0;
+         inp_vals.ToList().ForEach(d => sum += d);
+         string out_col_name = $"average_{inp_col_name}";
+         double average = sum / inp_vals.Count();
+         IBlueTableSchema schema_out = inp_row.schema.add_column(TypeCode.Double, out_col_name);
+         MutableBlueRow out_row = new MutableBlueRow(schema_out, inp_row);
+         out_row.setValue(out_col_name, average);
+         List<IBlueRow> rv = new List<IBlueRow>();
+         rv.Add(out_row);
+         return rv;
+      }
+   }
+}
@@ -175,7 +175,7 @@ namespace BCG {
         //gets resubscribed, we start the process of building data again, and treat the callback as the first 
         //callback.
         myInit();
-         myStream.onChange -= new DataStream.ChangeDelegate(myChangeHandler);
+         myStream.onChange -= myChangeHandler;
      }

      protected override void myInit() {
@@ -10,6 +10,6 @@ namespace BCG {
        }

        public MaxStream(ViewStream stream, string outputCol)
-           : this(new AddColumnsStream(stream, "fakeCol=0"), outputCol, new string[] { "fakeCol" }) { }
+           : this(new AddColumnsStream(stream, "fake_col: 0"), outputCol, new string[] { "fake_col" }) { }
    }
 }
@@ -82,12 +82,12 @@ namespace BCG

      protected override void myOnFirstSubscriber() 
      {
-         myStream.onChange += new DataStream.ChangeDelegate (myChangeHandler);
+         myStream.onChange += myChangeHandler;
      }

      protected override void myOnLastSubscriber () 
      {
-         myStream.onChange -= new DataStream.ChangeDelegate (myChangeHandler);
+         myStream.onChange -= myChangeHandler;
      }
   }
 }
@@ -1,13 +1,37 @@
 using System;
 using System.Collections.Generic;
 using System.Diagnostics;
+using System.Linq;

 using bts;

-namespace BCG
-{
-   public class NonIncGroupByStream : ViewStream
-   {
+namespace BCG {
+   public class NonIncGroupByStream : ViewStream {
+      private class DelNonIncGroupByEvaluator : INonIncGroupByEvaluator {
+         Func<string[]> _groupby_cols_func;
+         Func<List<IBlueRow>, List<IBlueRow>> _evaluate_func;
+         public DelNonIncGroupByEvaluator(Func<string[]> groupby_cols_func,
+                                          Func<List<IBlueRow>, List<IBlueRow>> evaluate_func) {
+            _groupby_cols_func = groupby_cols_func;
+            _evaluate_func = evaluate_func;
+         }
+         public IBlueRowCollection evaluate(IDictionary<int, List<IBlueRow>> grouped_rows) {
+            BlueTable rv = null;
+            foreach (var kv in grouped_rows) {
+               List<IBlueRow> g_rows = kv.Value;
+               List<IBlueRow> out_rows = _evaluate_func.Invoke(g_rows);
+               if (out_rows != null && out_rows.Count > 0) {
+                  rv = rv ?? new BlueTable(out_rows.ElementAt(0).schema);
+               }
+               rv.addRows(out_rows);
+            }
+            return rv;
+         }
+         public string[] get_group_by_cols() {
+            return _groupby_cols_func();
+         }
+
+      }
      private ViewStream _main_stream;
      private INonIncGroupByEvaluator _groupby_func;
      private BlueRowHasher _row_hasher;
@@ -15,24 +39,29 @@ namespace BCG
      private Stopwatch _timer = new();
      private int _n_total_fires = 0, _n_expensive_fires = 0;

-      public NonIncGroupByStream(ViewStream main_stream, INonIncGroupByEvaluator groupby_func)
-      {
+      public NonIncGroupByStream(ViewStream main_stream, INonIncGroupByEvaluator groupby_func) {
         _main_stream = main_stream;
         _groupby_func = groupby_func;
         _groupby_cols = _groupby_func.get_group_by_cols();
         _row_hasher = new BlueRowHasher(_groupby_cols.Length);
      }

+      public NonIncGroupByStream(ViewStream main_stream, Func<string[]> groupby_cols_func,
+                                 Func<List<IBlueRow>, List<IBlueRow>> evaluate_func)
+                                 : this(main_stream, new DelNonIncGroupByEvaluator(groupby_cols_func, evaluate_func)) { }
+
+
      protected override void myOnFirstSubscriber() { _main_stream.onChange += _main_stream_onchange; }

      protected override void myOnLastSubscriber() { _main_stream.onChange -= _main_stream_onchange; }

-      private void _main_stream_onchange(object nv)
-      {
+      private void _main_stream_onchange(object nv) {
         DataPacket dp = nv as DataPacket;
         ///grouping together
         _timer.Restart();
-         IDictionary<int, List<IBlueRow>> grouped_rows = BlueDataUtility.group_together (dp.newRows, _groupby_cols, ref _row_hasher);
+         IDictionary<int, List<IBlueRow>> grouped_rows = (_groupby_cols == null || _groupby_cols.Length == 0) ?
+         new Dictionary<int, List<IBlueRow>>() { { 0, dp.newRows.to_list() } } :
+         BlueDataUtility.group_together(dp.newRows, _groupby_cols, ref _row_hasher);
         _timer.Stop();
         var groupElapsedMs = _timer.ElapsedMilliseconds;
         if (groupElapsedMs >= 50) {
@@ -48,14 +77,13 @@ namespace BCG
            return;
         }
         var applyElapsedMs = _timer.ElapsedMilliseconds;
-         _n_total_fires++;         
-         if (applyElapsedMs >= 50)
-         {
+         _n_total_fires++;
+         if (applyElapsedMs >= 50) {
            _n_expensive_fires++;
            Log.DataFlow.info(
               $"Time to evaluate {grouped_rows.Count} groups {_timer.ElapsedMilliseconds}ms, ratio {_n_expensive_fires}/{_n_total_fires}");
-         }         
-         myFireEvent(new DataPacket(total_state, total_state, myLastValue == null ? new BlueTable(total_state.schema) : (myLastValue as DataPacket).newRows));             
+         }
+         myFireEvent(new DataPacket(total_state, total_state, myLastValue == null ? new BlueTable(total_state.schema) : (myLastValue as DataPacket).newRows));
      }
   }
 }
@@ -31,15 +31,15 @@ namespace BCG
      {
         myLeftOperand.master = this;
         myRightOperand.master = this;
-         myLeftStream.onChange += new DataStream.ChangeDelegate (myLeftOperand.changeHandler);
-         myRightStream.onChange += new DataStream.ChangeDelegate (myRightOperand.changeHandler);
+         myLeftStream.onChange += myLeftOperand.changeHandler;
+         myRightStream.onChange += myRightOperand.changeHandler;
      }
      
      protected override void myOnLastSubscriber () 
      {
         myInit ();
-         myLeftStream.onChange -= new DataStream.ChangeDelegate (myLeftOperand.changeHandler);
-         myRightStream.onChange -= new DataStream.ChangeDelegate (myRightOperand.changeHandler);
+         myLeftStream.onChange -= myLeftOperand.changeHandler;
+         myRightStream.onChange -= myRightOperand.changeHandler;
      }

      public JoinOperand getOtherOperand (JoinOperand jo)
@@ -116,7 +116,7 @@ namespace BCG
               IDictionary  map = resolver.getMap ();
               map["id"] = "id";
               myPs = new ProjectStream (myIjs, map);
-               myPs.onChange += new DataStream.ChangeDelegate (myProjectChangeHandler);
+               myPs.onChange += myProjectChangeHandler;
               return;
            }
            else
@@ -129,26 +129,26 @@ namespace BCG
      private void myProjectChangeHandler (object o)
      {
         if (myIjs.myOnChangeDelegate.GetInvocationList().Length > 1)
-            myIjs.onChange -= new DataStream.ChangeDelegate (myInternalJoinChangeHandler);
+            myIjs.onChange -= myInternalJoinChangeHandler;
         myFireEvent ((o as ICloneable).Clone());
         
      }

      protected override void myOnFirstSubscriber ()
      {
-         myIjs.onChange += new DataStream.ChangeDelegate (myInternalJoinChangeHandler);
+         myIjs.onChange += myInternalJoinChangeHandler;
      }

      protected override void myOnLastSubscriber ()
      {
         // how does this relate to the -= in myProjectChangeHandler?
-         myIjs.onChange -= new DataStream.ChangeDelegate (myInternalJoinChangeHandler);
+         myIjs.onChange -= myInternalJoinChangeHandler;

         // if there is a  project after the ijs we need to 
         // unsubscribe to it otherwise references will be kept and 
         // a memory leak will occur
         if (myPs != null)
-            myPs.onChange -= new DataStream.ChangeDelegate (myProjectChangeHandler);
+            myPs.onChange -= myProjectChangeHandler;
         myPs = null;
      }

@@ -5,11 +5,10 @@ using mbts;
 using System;
 using System.Collections.Generic;
 using System.Diagnostics;
+using System.Security.Cryptography;

-namespace BCG
-{
-   public class NonIncJoinBookkeeper : IDisposable
-   {
+namespace BCG {
+   public class NonIncJoinBookkeeper : IDisposable {
      public Action on_new_data;
      private readonly NonIncJoinSpec _join_spec;
      private readonly int _join_position;
@@ -26,41 +25,38 @@ namespace BCG

      private int[] _join_col_indices = null,
                     _data_col_indices = null;
-      
-      public NonIncJoinBookkeeper(NonIncJoinSpec the_join_spec, int join_position, int start_index_in_output, BlueRowHasher row_hasher)
-      {
+
+      public NonIncJoinBookkeeper(NonIncJoinSpec the_join_spec, int join_position, int start_index_in_output, BlueRowHasher row_hasher) {
         _join_spec = the_join_spec;
         _join_position = join_position;
         _start_index_in_output = start_index_in_output;
         _row_hasher = row_hasher;
         _is_left_most = _join_position == 0;
         _is_tag_along = _join_spec.join_col_names.Length == 0;
+      }
+
+      public void init() {
         _join_spec.stream.onChange += _on_stream_cb;
      }

-      public void _on_stream_cb(object dp_obj)
-      {
+      public void _on_stream_cb(object dp_obj) {
+         Log.Gen.info($"NonIncJoinBookkeeper._on_stream_cb dp_obj: {dp_obj}");
         var timer = Stopwatch.StartNew();
         DataPacket dp = (DataPacket)dp_obj;
-         if (_schema == null)
-         {
+         if (_schema == null) {
            _schema = dp.schema;
            _join_col_indices = BlueDataUtility.get_column_indices(_schema, _join_spec.join_col_names);
            _data_col_indices = BlueDataUtility.get_column_indices(_schema, _join_spec.data_col_names);
         }

-         if (_is_left_most)
-         {
+         if (_is_left_most) {
            _row_list.Clear();
            foreach (IBlueRow r in dp.newRows)
-               _row_list.Add(new Pair<int, IBlueRow>(_row_hasher.get_hash(r, _join_col_indices), r));            
-         }
-         else
-         {
+               _row_list.Add(new Pair<int, IBlueRow>(_row_hasher.get_hash(r, _join_col_indices), r));
+         } else {
            if (_is_tag_along)
               _solitary_row = BlueDataUtility.extractRow(dp.newRows, 0);
-            else
-            {
+            else {
               _row_map.Clear();
               foreach (IBlueRow r in dp.newRows)
                  _row_map[_row_hasher.get_hash(r, _join_col_indices)] = r;
@@ -79,31 +75,25 @@ namespace BCG

      public bool is_ready { get { return _schema != null; } }

-      public void constribute_cols_to_schema(List<ColumnInfo> cis)
-      {
+      public void constribute_cols_to_schema(List<ColumnInfo> cis) {
         for (int i = 0; i < _join_spec.data_col_names.Length; i++)
            cis.Add(new ColumnInfo(_join_spec.projected_col_names[i], cis.Count, _schema[_join_spec.data_col_names[i]].colType));
      }

      public List<Pair<int, IBlueRow>> get_list() { return _row_list; }

-      public void copy_master_row_to_output_row(int row_index, object[] output_row)
-      {
+      public void copy_master_row_to_output_row(int row_index, object[] output_row) {
         IBlueRow this_row = _row_list[row_index].second;
         BlueDataUtility.copy_to_row(this_row, _data_col_indices, output_row, _start_index_in_output);
      }

-      public void copy_matching_row_to_output_row(int row_hash, object[] output_row)
-      {
-         if (_is_tag_along)
-         {
+      public void copy_matching_row_to_output_row(int row_hash, object[] output_row) {
+         if (_is_tag_along) {
            if (_solitary_row != null)
               BlueDataUtility.copy_to_row(_solitary_row, _data_col_indices, output_row, _start_index_in_output);
            else
               copy_dbnull_to_output_row(output_row);
-         }
-         else
-         {
+         } else {
            IBlueRow this_row = null;
            if (_row_map.TryGetValue(row_hash, out this_row))
               BlueDataUtility.copy_to_row(this_row, _data_col_indices, output_row, _start_index_in_output);
@@ -112,8 +102,7 @@ namespace BCG
         }
      }

-      private void copy_dbnull_to_output_row(object[] output_row)
-      {
+      private void copy_dbnull_to_output_row(object[] output_row) {
         for (int i = 0; i < _data_col_indices.Length; i++)
            output_row[i + _start_index_in_output] = DBNull.Value;
      }
@@ -3,63 +3,62 @@ using System.Collections.Generic;
 using System.Linq;
 using System.Text;
 using System.Threading.Tasks;
+
 using mbts;

-namespace BCG
-{
-   public class NonIncJoinSpec
-   {
+namespace BCG {
+   public class NonIncJoinSpec {
      public ViewStream stream;
+      // join col names are the names on which the streams are joined (for equality),
+      // data col names are the names in the original stream
+      // projected col naes are their new names, ie if its x:l_x, then x is in the 
+      // data_col_names and l_x in projected cols
      public string[] join_col_names, data_col_names, projected_col_names;

      public NonIncJoinSpec() { }

      public NonIncJoinSpec(ViewStream the_stream, string[] the_join_col_names, string[] the_data_col_names)
-         : this (the_stream, the_join_col_names, the_data_col_names, the_data_col_names)
-      { }
+         : this(the_stream, the_join_col_names, the_data_col_names, the_data_col_names) { }

-      public NonIncJoinSpec(ViewStream the_stream, string[] the_join_col_names, string[] the_data_col_names, string[] the_projected_col_names)
-      {
+      public NonIncJoinSpec(ViewStream the_stream, string[] the_join_col_names, string[] the_data_col_names, string[] the_projected_col_names) {
         stream = the_stream;
         join_col_names = the_join_col_names;
         data_col_names = the_data_col_names;
         projected_col_names = the_projected_col_names;
      }

-      public static NonIncJoinSpec[] from_triplet(params Triplet<ViewStream, string, string>[] join_specs)
-      {
+      public static NonIncJoinSpec[] from_triplet(params Triplet<ViewStream, string, string>[] join_specs) {
         List<NonIncJoinSpec> rv = new List<NonIncJoinSpec>();
-         foreach (Triplet<ViewStream, string, string> r in join_specs)
-         {
-            string[] keys = String.IsNullOrEmpty (r.second) ? new string[] {} : r.second.Split(' ');
+         foreach (Triplet<ViewStream, string, string> r in join_specs) {
+            string[] keys = String.IsNullOrEmpty(r.second) ? new string[] { } : r.second.Split(' ');
            string[] visible_cols = r.third.Split(' ');
            List<string> data_col_names = new List<string>(), mapped_col_names = new List<string>();
-            foreach (string s in visible_cols)
-            {
+            foreach (string s in visible_cols) {
               string[] toks = s.Split(':');
-               if (toks.Length == 1)
-               {
+               if (toks.Length == 1) {
                  data_col_names.Add(toks[0]);
                  mapped_col_names.Add(toks[0]);
-               }
-               else if (toks.Length == 2)
-               {
+               } else if (toks.Length == 2) {
                  data_col_names.Add(toks[0]);
                  mapped_col_names.Add(toks[1]);
               }
            }
-            rv.Add (new NonIncJoinSpec (r.first, keys, data_col_names.ToArray (), mapped_col_names.ToArray()));
+            rv.Add(new NonIncJoinSpec(r.first, keys, data_col_names.ToArray(), mapped_col_names.ToArray()));
         }
-         return rv.ToArray ();
+         return rv.ToArray();
+      }
+
+      public override string ToString() {
+         Func<string[], string> arr_txt = (a) => {
+            string rv = "";
+            foreach (string s in a) {
+               rv += "{s} ";
+            }
+            rv = rv.Trim();
+            return rv;
+         };
+         string rv = $"stream: {stream} join_col_names: {arr_txt(join_col_names)} data_col_names: {arr_txt(data_col_names)} projected_col_names: {arr_txt(projected_col_names)}";
+         return rv;
      }
   }
-
-   //[DataStreamUserRespresentation("ninctest")]
-   //public class NonIncJoinTestStream : ViewStream
-   //{
-   //   public NonIncJoinTestStream (string key_cols, string data_cols)
-   //   {
-   //      NonIncJoinSpec.from_triplet(new Triplet<ViewStream, string, string>(null, key_cols, data_cols));
-   //   }
-   //}
 }
@@ -6,11 +6,9 @@ using System;
 using System.Collections.Generic;
 using System.Diagnostics;

-namespace BCG
-{
+namespace BCG {
   [DataStreamUserRespresentation("fastJoin")]
-   public class NonIncJoinStream : ViewStream, BCG.Utilities.General.IWorker
-   {
+   public class NonIncJoinStream : ViewStream, BCG.Utilities.General.IWorker {
      private readonly NonIncJoinSpec[] _join_specs;
      private readonly INonIncJoinInlineMapper _output_mapper;
      private readonly BlueRowHasher _row_hasher;
@@ -28,24 +26,19 @@ namespace BCG
               {
                  new Triplet<ViewStream, string, string> (left, left_join_cols, left_proj),
                  new Triplet<ViewStream, string, string> (right, right_join_cols, right_proj)
-               })
-      { }
+               }) { }

      public NonIncJoinStream(params Triplet<ViewStream, string, string>[] join_specs)
-         : this(NonIncJoinSpec.from_triplet(join_specs), null, 0)
-      { }
+         : this(NonIncJoinSpec.from_triplet(join_specs), null, 0) { }

      public NonIncJoinStream(ViewStreamDatabase vsd, INonIncJoinInlineMapper output_mapper, int throttles_ms,
                              params Triplet<ViewStream, string, string>[] join_specs)
-         : this(vsd, NonIncJoinSpec.from_triplet(join_specs), output_mapper, throttles_ms)
-      { }
+         : this(vsd, NonIncJoinSpec.from_triplet(join_specs), output_mapper, throttles_ms) { }

      public NonIncJoinStream(NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper output_mapper, int throttles_ms = 0)
-         : this(ViewStreamDatabase.instance, join_specs, output_mapper, throttles_ms)
-      { }
+         : this(ViewStreamDatabase.instance, join_specs, output_mapper, throttles_ms) { }

-      public NonIncJoinStream(ViewStreamDatabase vsd, NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper output_mapper, int throttles_ms = 0)
-      {
+      public NonIncJoinStream(ViewStreamDatabase vsd, NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper output_mapper, int throttles_ms = 0) {
         _join_specs = join_specs;
         _output_mapper = output_mapper;
         _row_hasher = new BlueRowHasher(_join_specs[0].join_col_names.Length);
@@ -53,12 +46,11 @@ namespace BCG
         _vsd = vsd;
      }

-      protected override void myOnFirstSubscriber()
-      {
+      protected override void myOnFirstSubscriber() {
         int data_col_no = 1;
-         for (int jp = 0; jp < _join_specs.Length; jp++)
-         {
+         for (int jp = 0; jp < _join_specs.Length; jp++) {
            NonIncJoinBookkeeper s = new NonIncJoinBookkeeper(_join_specs[jp], jp, data_col_no, _row_hasher);
+            Log.Gen.info($"NonIncJoinStream.myOnFirstSubscriber jp: {jp} spec: {_join_specs[jp]}");
            data_col_no += _join_specs[jp].data_col_names.Length;
            s.on_new_data += _stream_callback;
            _book_keeping_data.Add(s);
@@ -66,10 +58,11 @@ namespace BCG
         if (_output_mapper != null)
            _output_mapper.onMapperChange += _on_mapper_change;
         _vsd.addWorker(this);
+         foreach (var bkd in _book_keeping_data)
+            bkd.init();
      }

-      protected override void myOnLastSubscriber()
-      {
+      protected override void myOnLastSubscriber() {
         foreach (NonIncJoinBookkeeper d in _book_keeping_data)
            d.Dispose();
         if (_output_mapper != null)
@@ -77,30 +70,26 @@ namespace BCG
         _vsd.removeWorker(this);
      }

-      private void _stream_callback()
-      {
+      private void _stream_callback() {
+         Log.Gen.info("NonIncJoinStream._stream_callback received callback");
         _is_dirty = true;
         doWork();
      }

-      private void _on_mapper_change()
-      {
+      private void _on_mapper_change() {
         _is_dirty = true;
         doWork();
      }

-      private bool _is_ready()
-      {
+      private bool _is_ready() {
         bool rv = true;
         foreach (NonIncJoinBookkeeper d in _book_keeping_data)
            rv &= d.is_ready;
         return rv;
      }

-      private void _construct_schema()
-      {
-         if (_output_schema == null)
-         {
+      private void _construct_schema() {
+         if (_output_schema == null) {
            List<ColumnInfo> cis = new List<ColumnInfo>(new ColumnInfo[] { new ColumnInfo("id", 0, typeof(int)) });
            foreach (NonIncJoinBookkeeper d in _book_keeping_data)
               d.constribute_cols_to_schema(cis);
@@ -109,27 +98,23 @@ namespace BCG
      }

      private int _n_total_fires = 0, _n_expensive_fires = 0;
-      private void _form_and_fire()
-      {
-         if (_is_ready())
-         {
+      private void _form_and_fire() {
+         Log.Gen.info($"NonIncJoinStream._form_and_fire _is_ready(): {_is_ready()}");
+         if (_is_ready()) {
            _construct_schema();
            var timer = Stopwatch.StartNew();
            IList<Pair<int, IBlueRow>> left_row_list = _book_keeping_data[0].get_list();
            List<object[]> output_rows = new List<object[]>();

-            for (int i = 0; i < left_row_list.Count; i++)
-            {
+            for (int i = 0; i < left_row_list.Count; i++) {
               object[] or = new object[_output_schema.columnInfo.Length];
               or[0] = i;
               output_rows.Add(or);
            }

-            for (int ji = 0; ji < _book_keeping_data.Count; ji++)
-            {
+            for (int ji = 0; ji < _book_keeping_data.Count; ji++) {
               NonIncJoinBookkeeper bk_data = _book_keeping_data[ji];
-               for (int i = 0; i < left_row_list.Count; i++)
-               {
+               for (int i = 0; i < left_row_list.Count; i++) {
                  if (ji == 0)
                     bk_data.copy_master_row_to_output_row(i, output_rows[i]);
                  else
@@ -139,19 +124,20 @@ namespace BCG
            IBlueRowCollection dp_output_rows = _apply_map(new SequentialIDBlueTable(_output_schema, output_rows));
            timer.Stop();
            _n_total_fires++;
-            if (timer.ElapsedMilliseconds >= 250) {
-               _n_expensive_fires++;
-               string log_msg = $"Join and map time for {_book_keeping_data.Count} "
-                  + $"tables {timer.Elapsed} ratio {_n_expensive_fires}/{_n_total_fires}";
-            }
-            myFireEvent(new DataPacket(dp_output_rows, dp_output_rows, myLastValue == null ? new BlueTable(_output_schema) : (myLastValue as DataPacket).newRows));
+            //if (timer.ElapsedMilliseconds >= 250) {
+            _n_expensive_fires++;
+            string log_msg = $"Join and map time for {_book_keeping_data.Count} "
+               + $"tables {timer.Elapsed.TotalMilliseconds} msecs, ratio {_n_expensive_fires}/{_n_total_fires}";
+            Log.Gen.info(log_msg);
+            //}
+            var dp_out = BlueDataUtility.build_dp_inc(dp_output_rows, myLastValue as DataPacket);
+            myFireEvent(dp_out);
+            //myFireEvent(new DataPacket(dp_output_rows, dp_output_rows, myLastValue == null ? new BlueTable(_output_schema) : (myLastValue as DataPacket).newRows));
         }
      }

-      private IBlueRowCollection _apply_map(IBlueRowCollection unmapped_rows)
-      {
-         if (_output_mapper != null)
-         {
+      private IBlueRowCollection _apply_map(IBlueRowCollection unmapped_rows) {
+         if (_output_mapper != null) {
            IBlueRowCollection output_rows;
            var timer = Stopwatch.StartNew();
            output_rows = _output_mapper.map_rows(unmapped_rows);
@@ -160,17 +146,14 @@ namespace BCG
               Log.Gen.info($"Time to map {unmapped_rows.Count} rows into {output_rows.Count} rows {timer.Elapsed}");
            }
            return output_rows;
-         }
-         else
-         {
+         } else {
            return unmapped_rows;
         }
      }

-      public void doWork()
-      {
-         if (DateTime.Now - _last_fire_time >= _throttle_period && _is_dirty)
-         {
+      public void doWork() {
+         Log.Gen.info($"NonIncJoinStream.doWork _last_fire_time: {_last_fire_time} _is_dirty: {_is_dirty}");
+         if (DateTime.Now - _last_fire_time >= _throttle_period && _is_dirty) {
            _is_dirty = false;
            _last_fire_time = DateTime.Now;
            _form_and_fire();
@@ -181,8 +181,8 @@ namespace BCG

       protected override void myOnFirstSubscriber()
       {
-           myVLeft.onChange += new ChangeDelegate(myVLeft_onChange);
-           myVRight.onChange += new ChangeDelegate(myVRight_onChange);
+           myVLeft.onChange += myVLeft_onChange;
+           myVRight.onChange += myVRight_onChange;
       }

       void myVRight_onChange(object newValue)
@@ -0,0 +1,68 @@
+namespace bts {
+   using BCG;
+   using BCG.Utilities.General;
+
+   using System;
+   using System.Diagnostics;
+
+   using ZstdSharp.Unsafe;
+
+   // functon that joins the 2 rows and returns the outputput row. The function
+   // can return a null which means the 2 rows dont need to be joined
+   using JoinFunc = System.Func<BCG.IBlueRow, BCG.IBlueRow, BCG.IBlueRow>;
+   public class ProximityJoinStream : ViewStream, IWorker {
+      private ViewStream _lvs, _rvs;
+      private JoinFunc _join_func;
+      private bool _dirty = false;
+      private Stopwatch _stopwatch = new Stopwatch();
+      private ViewStreamDatabase _vsd = ViewStreamDatabase.instance;
+      public ProximityJoinStream(ViewStream lvs, ViewStream rvs, JoinFunc join_func) {
+         _lvs = lvs;
+         _rvs = rvs;
+         _join_func = join_func;
+      }
+
+      protected override void myOnFirstSubscriber() {
+         _lvs.onChange += _handle_data;
+         _rvs.onChange += _handle_data;
+         _vsd.addWorker(this);
+      }
+
+      protected override void myOnLastSubscriber() {
+         _lvs.onChange -= _handle_data;
+         _rvs.onChange -= _handle_data;
+         _vsd.removeWorker(this);
+      }
+
+      private void _handle_data(object dp_obj) {
+         _dirty = true;
+         doWork();
+      }
+
+      public void doWork() {
+         Func<bool> all_data_revcd = () => {
+            return _lvs.rows != null && _rvs.rows != null;
+         };
+         _stopwatch.Start();
+         if (_dirty && all_data_revcd()) {
+            _dirty = false;
+            BlueTable rv = null;
+            foreach (IBlueRow lr in _lvs.rows) {
+               foreach (IBlueRow rr in _rvs.rows) {
+                  IBlueRow resr = _join_func(lr, rr);
+                  if (resr == null)
+                     continue;
+                  rv = rv ?? new BlueTable(resr.schema);
+                  rv.addRow(resr);
+               }
+            }
+            _stopwatch.Stop();
+            Log.Gen.info($"ProximityJoinStream left_count: {_lvs.rows.Count} right_count: {_rvs.rows.Count} result_count: {rv.Count} time: {_stopwatch.ElapsedMilliseconds} msecs");
+            var dp = BlueDataUtility.build_dp_inc(rv, myLastValue as DataPacket);
+            myFireEvent(dp);
+         }
+
+      }
+
+   }
+}
@@ -0,0 +1,47 @@
+namespace bts.SilverPine {
+   using BCG;
+
+   using System;
+   using System.CodeDom;
+   using System.Configuration;
+   using System.Reactive.Disposables;
+
+   public class TimeProximityJoinStream : ViewStream {
+      private int _id = 0;
+
+      public TimeProximityJoinStream(ViewStream ls, string l_time_col, ViewStream rs, string r_time_col,
+                                       string pre_join_period, string post_join_period) :
+            this(ls, l_time_col, rs, r_time_col, TimeBasedFilter.parse_duration(pre_join_period),
+            TimeBasedFilter.parse_duration(post_join_period)) { }
+
+
+      public TimeProximityJoinStream(ViewStream ls, string l_time_col, ViewStream rs, string r_time_col,
+                                       TimeSpan pre_join_period, TimeSpan post_join_period) {
+         Func<IBlueRow, IBlueRow, IBlueRow> create_row = (lr, rr) => {
+            var id = 0;
+            lock (this) {
+               id = _id++;
+            }
+            IBlueRow rv = BlueDataUtility.create_composite_row(lr, rr, id);
+            return rv;
+         };
+         Func<IBlueRow, IBlueRow, IBlueRow> join_func = (lr, rr) => {
+            DateTime ldt = lr.get_datetime(l_time_col);
+            DateTime rdt = rr.get_datetime(r_time_col);
+            TimeSpan diff = ldt - rdt;
+            if (ldt < rdt) {
+               if (diff.Duration() > pre_join_period) {
+                  return null;
+               }
+            } else {
+               if (diff > post_join_period)
+                  return null;
+            }
+            return create_row(lr, rr);
+         };
+
+         ViewStream rstr = new ProximityJoinStream(ls, rs, join_func);
+         mySetRealSource(rstr);
+      }
+   }
+}
@@ -327,14 +327,14 @@ namespace BCG

      protected override void myOnFirstSubscriber()
      {
-         myLeftStream.onChange += new ChangeDelegate(myHandleLeftStreamChanges);
-         myRightStream.onChange += new ChangeDelegate(myHandleRightStreamChanges);
+         myLeftStream.onChange += myHandleLeftStreamChanges;
+         myRightStream.onChange += myHandleRightStreamChanges;
      }

      protected override void myOnLastSubscriber()
      {
-         myLeftStream.onChange -= new ChangeDelegate(myHandleLeftStreamChanges);
-         myRightStream.onChange -= new ChangeDelegate(myHandleRightStreamChanges);
+         myLeftStream.onChange -= myHandleLeftStreamChanges;
+         myRightStream.onChange -= myHandleRightStreamChanges;
         myLeftKeyToLeftRows.Clear();
         myRightKeyToRightRows.Clear();
         myLeftIDToResultRows.Clear();
@@ -23,15 +23,15 @@ namespace BCG
         myRightOperand.master = this;
         //the events are not hooked onto the operands delegate, but rather a level of indirection
         //added in the form on myLeftChangeHandler
-         myLeftStream.onChange += new DataStream.ChangeDelegate (myLeftChangeHandler);
-         myRightStream.onChange += new DataStream.ChangeDelegate (myRightChangeHandler);
+         myLeftStream.onChange += myLeftChangeHandler;
+         myRightStream.onChange += myRightChangeHandler;
      }
       
      protected override void myOnLastSubscriber () 
      {
         myInit ();
-         myLeftStream.onChange -= new DataStream.ChangeDelegate (myLeftChangeHandler);
-         myRightStream.onChange -= new DataStream.ChangeDelegate (myRightChangeHandler);
+         myLeftStream.onChange -= myLeftChangeHandler;
+         myRightStream.onChange -= myRightChangeHandler;
      }

      protected override void myInit()
@@ -2,9 +2,13 @@ using System;
 using System.Collections;
 using System.Collections.Generic;

+using Amazon.Runtime.Internal.Util;
+
 using BCG;
 using BCG.Utilities;

+using bts;
+
 namespace BCG {
   public class LiteralStream : ViewStream {

@@ -22,6 +26,34 @@ namespace BCG {
         return rv;
      }

+      //literal("table1", "x:123, y:1.001, z:abc;x:123, y:1.001, z:abc")
+      public LiteralStream(string table_name, string rows) {
+         Func<string, int, IBlueRow> get_row = (json_row, id) => {
+            Dictionary<string, object> map = new Dictionary<string, object>();
+            string[] toks = json_row.Split(',', StringSplitOptions.TrimEntries);
+            //Logger.assert(col_vals.Length == 2);
+            foreach (string t in toks) {
+               var col_vals = t.Split(':', StringSplitOptions.TrimEntries);
+               string col = col_vals[0];
+               string val = col_vals[1];
+               map.Add(col, val);
+            }
+            map["id"] = id;
+            return new BlueRow(map);
+         };
+         string[] row_arr = rows.Split(';', StringSplitOptions.TrimEntries);
+         BlueTable table = null;
+         foreach (string json_string in row_arr) {
+            IBlueRow r = //BlueDataUtility.convert_json_to_row(json_string, table == null ? 0 : table.Count);
+            get_row(json_string, table == null ? 0 : table.Count);
+            table = table ?? new BlueTable(r.schema);
+            table.addRow(r);
+         }
+         ViewStream str = new BlueTableStream(table);
+         str = new SchemaGuesserStream(str);
+         mySetRealSource(str);
+      }
+
      public LiteralStream(string tableName, string schema, string value) {
         string[] columns = schema.Split(' ');

@@ -55,7 +87,7 @@ namespace BCG {
         mySetRealSource(new BlueRowCollectionStaticStream(t));
      }

-      public LiteralStream(string schema, string value) : this("literal", schema, value) { }
+      // public LiteralStream(string schema, string value) : this("literal", schema, value) { }

      // public LiteralStream(string value) {

@@ -0,0 +1,72 @@
+using BCG;
+
+using MongoDB.Driver.Core.WireProtocol.Messages;
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+
+// the first arg is all the rows, the second arg is the index of the row that needs mapping, and 
+// the func returns the mapped row (ie mapped version of the index row)
+using GlobalMap = System.Func<System.Collections.Generic.List<BCG.IBlueRow>, uint, BCG.IBlueRow>;
+
+namespace bts {
+   // In GlobalMap, each row is mapped with the full view of all rows, ie the map functions for a row
+   // can use the state fo all rows.
+   public class GlobalMapStream : ViewStream {
+      private ViewStream _base_stream;
+      private readonly string _sort_col_name, _group_by_col;
+      private GlobalMap _map_func;
+      public GlobalMapStream(ViewStream base_stream, string sort_col_name, string group_by_col, GlobalMap map_func) {
+         _base_stream = base_stream;
+         _sort_col_name = sort_col_name;
+         _group_by_col = group_by_col;
+         _map_func = map_func;
+      }
+
+      public GlobalMapStream(ViewStream base_stream, string sort_col_name, GlobalMap map_func) :
+      this(base_stream, sort_col_name, null, map_func) {
+      }
+
+      protected override void myOnFirstSubscriber() {
+         _base_stream.onChange += _handle_stream_update;
+      }
+
+      protected override void myOnLastSubscriber() {
+         _base_stream.onChange -= _handle_stream_update;
+      }
+
+      void _handle_stream_update(object dp_obj) {
+         DataPacket dp = dp_obj as DataPacket;
+         IDictionary<int, List<IBlueRow>> groups = !String.IsNullOrEmpty(_group_by_col) ?
+         BlueDataUtility.group_together(dp.newRows, new string[] { _group_by_col })
+         :
+         new Dictionary<int, List<IBlueRow>>() { { 0, dp.newRows.to_list() } };
+         BlueTable out_table = null;
+         foreach (var kv in groups) {
+            List<IBlueRow> group_rows = kv.Value;
+            group_rows.Sort((r1, r2) => BlueDataUtility.compare_rows(r2, r1, _sort_col_name));
+            uint i = 0;
+            foreach (IBlueRow r in group_rows) {
+               IBlueRow nr = _map_func(group_rows, i++);
+               out_table = out_table ?? new BlueTable(nr.schema);
+               out_table.addRow(nr);
+            }
+         }
+
+
+         // List<IBlueRow> rows = dp.newRows.to_ienum().ToList();
+         // rows.Sort((r1, r2) => BlueDataUtility.compare_rows(r1, r2, _sort_col_name));
+         // uint i = 0;
+
+         // BlueTable out_table = null;
+         // foreach (IBlueRow r in rows) {
+         //    IBlueRow nr = _map_func(rows, i++);
+         //    out_table = out_table ?? new BlueTable(nr.schema);
+         //    out_table.addRow(nr);
+         // }
+         DataPacket dp_out = BlueDataUtility.build_dp_inc(out_table, myLastValue as DataPacket);
+         myFireEvent(dp_out);
+      }
+   }
+}
@@ -119,13 +119,13 @@ namespace BCG {


      protected override void myOnFirstSubscriber() {
-         myStream.onChange += new DataStream.ChangeDelegate(myChangeHandler);
+         myStream.onChange += myChangeHandler;
      }

      protected override void myOnLastSubscriber() {
         myInit();
         if (myStream != null)
-            myStream.onChange -= new DataStream.ChangeDelegate(myChangeHandler);
+            myStream.onChange -= myChangeHandler;
      }

      protected override void myOnTableClear() {
@@ -83,10 +83,11 @@ namespace BCG {
         myMapper.onMapperChange += new MapperChangeDelegate(myHandleMapChange);
      }

-      protected override void myOnFirstSubscriber() { myStream.onChange += new ChangeDelegate(myHandleData); }
+      protected override void myOnFirstSubscriber() =>
+         myStream.onChange += myHandleData;

      protected override void myOnLastSubscriber() {
-         myStream.onChange -= new ChangeDelegate(myHandleData);
+         myStream.onChange -= myHandleData;
         myRows = null;
         myBaseRows = null;
         myMap.Clear();
@@ -16,10 +16,10 @@ namespace BCG {
      }

      protected override void myOnFirstSubscriber() {
-         _stream.onChange += new ChangeDelegate(_onstream_change);
+         _stream.onChange += _onstream_change;
      }
      protected override void myOnLastSubscriber() {
-         _stream.onChange -= new ChangeDelegate(_onstream_change);
+         _stream.onChange -= _onstream_change;
      }

      private void _onstream_change(object o) {
@@ -45,4 +45,4 @@ namespace BCG {

      }
   }
-}
+}
@@ -17,10 +17,10 @@ namespace BCG {
      }

      protected override void myOnFirstSubscriber() {
-         _stream.onChange += new ChangeDelegate(_onstream_change);
+         _stream.onChange += _onstream_change;
      }
      protected override void myOnLastSubscriber() {
-         _stream.onChange -= new ChangeDelegate(_onstream_change);
+         _stream.onChange -= _onstream_change;
      }


@@ -41,7 +41,6 @@ namespace BCG {
         }
         _table.addRows(added_rows);
         myFireEvent(new DataPacket(_table, added_rows, new BlueTable(_table.schema)));
-
      }
   }

@@ -137,10 +137,24 @@ namespace bts {
         HashSet<string> removed_cols = new HashSet<string>();
         Dictionary<string, string> mapped_to_orig_map = new Dictionary<string, string>();
         Dictionary<string, string> orig_to_mapped = new Dictionary<string, string>();
+         // by default every col will show up, so if we have the projection as (a:a1)
+         // it will convert a to a1, 
+         // examples: if a schema has cols a b c
+         // project(parent_str, "a:aa ") -> aa b c
+         // project(parent_str, "a:aa !c") -> aa b
+         // project(parent_str, "a:aa !") -> aa
+         // project(parent_str, "a:aa b = (aa*aa)!") -> aa b
+         bool hide_all = false;
         foreach (string p_tok in project_toks) {
            if (p_tok.StartsWith("!")) {
-               string removed_col = p_tok.Substring(1);
-               removed_cols.Add(removed_col);
+               if (p_tok.Length == 1) {
+                  // if it is a "blanket "!"=> hide all cols except the one that
+                  // are stated explicitly
+                  hide_all = true;
+               } else {
+                  string removed_col = p_tok.Substring(1);
+                  removed_cols.Add(removed_col);
+               }
            } else if (p_tok.Contains(":")) {
               string[] map_toks = p_tok.Split(":");
               string orig_col = map_toks[0];
@@ -155,12 +169,11 @@ namespace bts {
               }
            }
         }
+
         // make the new schema
         List<ColumnInfo> new_cols = new List<ColumnInfo>();
         foreach (ColumnInfo ci in input_schema.columnInfo) {
            if (ci.colName == "id") {
-               // if there's a hidden col, other columns will show by default, but if cols are being
-               // projected epxlicitely, than others wil be hidden
               ColumnInfo nc = new ColumnInfo(ci.colName, new_cols.Count, ci.colType);
               new_cols.Add(nc);
               continue;
@@ -172,12 +185,13 @@ namespace bts {
               ColumnInfo nc = new ColumnInfo(mapped_name, new_cols.Count, ci.colType);
               new_cols.Add(nc);
            } else {
-               if (ci.colName == "id" || removed_cols.Count > 0) {
-                  // if there's a hidden col, other columns will show by default, but if cols are being
-                  // projected epxlicitely, than others wil be hidden
-                  ColumnInfo nc = new ColumnInfo(ci.colName, new_cols.Count, ci.colType);
-                  new_cols.Add(nc);
-               }
+               if (hide_all)
+                  continue;
+               //if (ci.colName == "id" || removed_cols.Count > 0) {
+               // unless columns are  explicitely hidden, all columns show up
+               ColumnInfo nc = new ColumnInfo(ci.colName, new_cols.Count, ci.colType);
+               new_cols.Add(nc);
+               //}
            }
         }
         IBlueTableSchema new_schema = new BlueTableSchema(input_schema.tableName, new_cols.ToArray());
@@ -54,12 +54,12 @@ namespace BCG {

      protected override void myOnFirstSubscriber()
      {
-         myVs.onChange += new BCG.DataStream.ChangeDelegate (myHandleChange);
+         myVs.onChange += myHandleChange;
      }

      protected override void myOnLastSubscriber()
      {
-         myVs.onChange -= new BCG.DataStream.ChangeDelegate (myHandleChange);;
+         myVs.onChange -= myHandleChange;
      }

      internal void myHandleChange (object newValue)
@@ -0,0 +1,230 @@
+using System;
+using System.Data;
+using System.Data.SQLite;
+using System.Diagnostics;
+using System.Collections;
+
+
+using BCG;
+using System.Linq;
+using System.Data.Entity;
+using System.Collections.Generic;
+
+namespace bts.SilverPine {
+
+   public class DataRowWrapperRow : IBlueRow {
+      private readonly DataRow _row;
+      private readonly IBlueTableSchema _schema;
+      private readonly int _id;
+      public DataRowWrapperRow(IBlueTableSchema schema, DataRow row, int id) {
+         _row = row;
+         _id = id;
+         _schema = schema;
+      }
+      public virtual IBlueTableSchema schema { get { return _schema; } }
+
+      public virtual object this[string col_name] {
+         get {
+            ColumnInfo ci = _schema[col_name];
+            return this[ci.colNo];
+         }
+      }
+      public virtual object this[int index] {
+         get {
+            if (_schema[index].colName == "id")
+               return _id;
+            return _row[index];
+         }
+      }
+
+      public virtual int Count { get { return _row.ItemArray.Length; } }
+
+      public virtual bool IsSynchronized { get { return false; } }
+
+      public virtual object SyncRoot { get { return this; } }
+
+      public virtual void CopyTo(Array array, int index) { _row.ItemArray.CopyTo(array, index); }
+
+      public virtual bool IsFixedSize { get { return true; } }
+
+      virtual public bool IsReadOnly { get { return true; } }
+
+      /// <summary>
+      /// The key can either be the string corrsponding to the column name or
+      /// the order of the column
+      /// </summary>
+      public virtual object this[object key] {
+         get {
+            if (key is string)
+               return this[(string)key];
+            else if (key is int)
+               return this[(int)key];
+            throw new BlueDataException("Wrong type while indexing the row");
+         }
+         set { throw new NotSupportedException("The IBlueRow is readonly"); }
+      }
+
+      public virtual ICollection Keys { get { return _schema.columnNames; } }
+
+      public virtual ICollection Values { get { return _row.ItemArray; } }
+
+      public virtual void Add(object key, object value) { throw new NotSupportedException("The IBlueRow is readonly"); }
+
+      public virtual void Clear() { throw new NotSupportedException("The IBlueRow is readonly"); }
+
+      public virtual bool Contains(object key) { return _schema[(string)key] != null; }
+
+      public IDictionaryEnumerator GetEnumerator() { return new BlueRowDictionaryEnumerator(this); }
+
+      IEnumerator IEnumerable.GetEnumerator() { return new BlueRowEnumerator(this); }
+
+      public virtual void Remove(object key) { throw new NotSupportedException("The IBlueRow is readonly"); }
+
+   }
+
+   public class DataTableWrapperTable : IBlueRowCollection {
+
+      public class DataTableWrapperTableEnumerator : IEnumerator {
+         private IEnumerator _enumerator;
+         private IBlueTableSchema _schema;
+         private int _id = 0;
+         public DataTableWrapperTableEnumerator(DataTable table) {
+            _enumerator = table.Rows.GetEnumerator();
+            _schema = BlueDataUtility.getSchema(table);
+
+         }
+         public object Current {
+            get {
+               DataRow r = _enumerator.Current as DataRow;
+               return new DataRowWrapperRow(_schema, r, _id);
+            }
+         }
+         public bool MoveNext() {
+            _id++;
+            return _enumerator.MoveNext();
+         }
+         public void Reset() {
+            _id = 0;
+            _enumerator.Reset();
+         }
+      }
+
+      private readonly DataTable _table;
+      private IBlueTableSchema _schema;
+      private IEnumerator _enumerator;
+
+      public DataTableWrapperTable(DataTable table) {
+         _table = table;
+         _schema = BlueDataUtility.getSchema(_table);
+         _enumerator = new DataTableWrapperTableEnumerator(table);
+      }
+      public virtual IBlueRow this[int id] {
+         get {
+            // its assumed the rows index == id
+            DataRow dr = _table.Rows[id];
+            return new DataRowWrapperRow(_schema, dr, id);
+         }
+         set {
+
+         }
+      }
+
+      public virtual IBlueTableSchema schema { get { return _schema; } }
+
+      public virtual bool containsID(int id) { return id < _table.Rows.Count; }
+
+      // private List<IBlueRow> _rows() {
+      //    List<IBlueRow> rv = new List<IBlueRow>();
+      //    int id = 0;
+      //    foreach (DataRow r in _table.Rows) {
+      //       rv.Add(new DataRowWrapperRow(r, id++));
+      //    }
+      //    return rv;
+      // }
+
+      public virtual IEnumerator GetEnumerator() {
+         // var rows = _rows();
+         // return rows.GetEnumerator();
+         return _enumerator;
+      }
+
+      public virtual bool IsSynchronized { get { return false; } }
+
+      public virtual object SyncRoot { get { return this; } }
+
+      public virtual void CopyTo(Array array, int index) {
+         Log.assert(false);
+         //_rows().CopyTo(array as IBlueRow[], index); 
+      }
+
+      public virtual int Count {
+         get { return _table.Rows.Count; }
+      }
+   }
+
+   public static class SQLiteHelper {
+      public static DataTable ExecuteSelectQuery(string connection_string, string query) {
+         // Create a new DataTable to store the results
+         DataTable dataTable = new DataTable();
+
+         // Create and open a SQLite connection using the provided connection string
+         using (SQLiteConnection connection = new SQLiteConnection(connection_string)) {
+            try {
+               connection.Open();
+
+               // Create a SQLite command with the provided query
+               using (SQLiteCommand command = new SQLiteCommand(query, connection)) {
+                  // Use SQLiteDataAdapter to fill the DataTable
+                  using (SQLiteDataAdapter adapter = new SQLiteDataAdapter(command)) {
+                     adapter.Fill(dataTable);
+                  }
+               }
+            } catch (Exception ex) {
+               Console.WriteLine("An error occurred: " + ex.Message);
+            }
+         }
+
+         // Return the populated DataTable
+         return dataTable;
+      }
+   }
+
+   [DataStreamUserRespresentation("SQLite")]
+   public class SQLiteStream : ViewStream {
+      private readonly string _connection_string, _query;
+      private Stopwatch _stopwatch = new Stopwatch();
+      public SQLiteStream(string connection_string, string query) {
+         _connection_string = connection_string;
+         _query = query;
+      }
+
+
+      private void _read_from_sqlite_nd_fire() {
+         _stopwatch.Start();
+         DataTable dt = SQLiteHelper.ExecuteSelectQuery(_connection_string, _query);
+         _stopwatch.Stop();
+         Log.Gen.info($"SQLiteStream._read_from_sqlite_and_fire time to read db: {_stopwatch.ElapsedMilliseconds} msecs");
+         _stopwatch.Start();
+         IBlueRowCollection rows = BCG.BlueDataUtility.toIBlueRowCollection(dt);
+         //IBlueRowCollection rows = new DataTableWrapperTable(dt);
+         _stopwatch.Stop();
+         Log.Gen.info($"SQLiteStream._read_from_sqlite_and_fire time to load into IBlueRowCollection: {_stopwatch.ElapsedMilliseconds} msecs");
+         DataPacket dp = BlueDataUtility.build_dp_inc(rows, myLastValue as DataPacket);
+         myFireEvent(dp);
+      }
+
+      protected override void myOnFirstSubscriber() {
+         _read_from_sqlite_nd_fire();
+      }
+
+      protected override void myOnLastSubscriber() {
+         // this value will be cleared by the base class
+         // but in this particular stream this behaviour
+         // is not usefull since any new subscribers 
+         // would force the query to be recreated
+         //myLastDP = myLastValue;
+      }
+
+   }
+}
+
@@ -3,12 +3,15 @@ using System.Collections.Generic;
 using System.Text;
 using System.IO;
 using BCG.Utilities;
+using System.Diagnostics;
+using Microsoft.Data.Analysis;
+using Parquet;
+using Parquet.Data;

-namespace BCG
-{
+
+namespace BCG {
   [DataStreamUserRespresentation("CSVSaver")]
-   public class CSVSaverStream : ViewStream, ISideEffectStream
-   {
+   public class CSVSaverStream : ViewStream, ISideEffectStream {
      private IBlueTableSchema myResultSchema = new BlueTableSchema("CSVResult", new ColumnInfo("id", 0, typeof(int)),
         new ColumnInfo("fileName", 1, typeof(string)),
         new ColumnInfo("nWrites", 2, typeof(int)),
@@ -20,11 +23,9 @@ namespace BCG
      private int myNWrites = 0, myNMaxWrites = Int32.MaxValue;

      public CSVSaverStream(ViewStream vs, string fullyQualifiedFileName)
-         : this(vs, fullyQualifiedFileName, true, Int32.MaxValue, false)
-      { }
+         : this(vs, fullyQualifiedFileName, true, Int32.MaxValue, false) { }

-      public CSVSaverStream(ViewStream vs, string fullyQualifiedFileName, bool overwrite, int nMaxWrites, bool fireResultSnapshot)
-      {
+      public CSVSaverStream(ViewStream vs, string fullyQualifiedFileName, bool overwrite, int nMaxWrites, bool fireResultSnapshot) {
         myStream = vs;
         myFullyQualifiedFileName = fullyQualifiedFileName;
         myOverwrite = overwrite;
@@ -32,22 +33,20 @@ namespace BCG
         myFireResultSnapshot = fireResultSnapshot;
      }

-      protected override void myOnFirstSubscriber() { myStream.onChange += new ChangeDelegate(myHandleChanges); }
+      protected override void myOnFirstSubscriber() =>
+         myStream.onChange += myHandleChanges;

-      protected override void myOnLastSubscriber() { myStream.onChange -= new ChangeDelegate(myHandleChanges); }
+      protected override void myOnLastSubscriber() =>
+         myStream.onChange -= myHandleChanges;

-      private void myHandleChanges(object newValue)
-      {
+      private void myHandleChanges(object newValue) {
         IBlueRowCollection totalState = (newValue as DataPacket).newRows;
-         if (totalState.Count > 0 && myNWrites < myNMaxWrites)
-         {
+         if (totalState.Count > 0 && myNWrites < myNMaxWrites) {
            myNWrites++;
            int i = 0;
            StreamWriter writer = new StreamWriter(new FileStream(myGetNewFileName(), FileMode.Create, FileAccess.Write));
-            foreach (IBlueRow r in totalState)
-            {
-               if (i++ == 0)
-               {
+            foreach (IBlueRow r in totalState) {
+               if (i++ == 0) {
                  string header = String.Empty;
                  foreach (string col in r.schema.columnNames)
                     if (col != "id")
@@ -59,20 +58,17 @@ namespace BCG
               writer.Flush();
            }
            writer.Close();
-            if (myFireResultSnapshot)
-            {
+            if (myFireResultSnapshot) {
               IBlueRow nr = new BlueRow(myResultSchema, 0, myFullyQualifiedFileName, myNWrites, totalState.Count);
               BlueTable nRows = new BlueTable(myResultSchema);
               nRows.addRow(nr);
               myFireEvent(new DataPacket(nRows, nRows, new BlueTable(myLastValue == null ? new BlueTable(myResultSchema) : (myLastValue as DataPacket).newRows)));
-            }
-            else
+            } else
               myFireEvent(newValue);
         }
      }

-      private string myGetNewFileName()
-      {
+      private string myGetNewFileName() {
         string[] tags = myFullyQualifiedFileName.Split('.');
         if (myOverwrite)
            return tags[0] + ".csv";
@@ -80,14 +76,11 @@ namespace BCG
            return tags[0] + "_" + myFormateDateTime(DateTime.Now) + ".csv";
      }

-      private static string myGetCSVRow(IBlueRow r)
-      {
+      private static string myGetCSVRow(IBlueRow r) {
         string rv = String.Empty;
         int i = 0;
-         foreach (string col in r.schema.columnNames)
-         {
-            if (col != "id")
-            {
+         foreach (string col in r.schema.columnNames) {
+            if (col != "id") {
               string colText = String.Empty;
               if (r[col] != null && r[col] != DBNull.Value)
                  colText = r[col].ToString();
@@ -99,8 +92,7 @@ namespace BCG
         return rv;
      }

-      private string myFormateDateTime(DateTime dt)
-      {
+      private string myFormateDateTime(DateTime dt) {
         return
         string.Format("{0}-{1}-{2}-{3}-{4}-{5}-{6}",
                        dt.Year.ToString("0000"),
@@ -114,52 +106,49 @@ namespace BCG
   }

   [DataStreamUserRespresentation("CSVReader")]
-   public class CSVReaderStream : ViewStream
-   {
-      private IBlueRowCollection myRows;
-
-      public CSVReaderStream(string fileAddress)
-      {
-         myRows = BlueDataUtility.createRowsFromCSVFileWithoutTypes(fileAddress);
+   public class CSVReaderStream : ViewStream {
+      private readonly string _file_address;
+      Stopwatch _stopwatch = new Stopwatch();
+      public CSVReaderStream(string fileAddress) {
+         _file_address = fileAddress;
      }

-      protected override void myOnFirstSubscriber()
-      {
-         myFireEvent(new DataPacket(myRows, myRows, new BlueTable(myRows.schema)));
+      protected override void myOnFirstSubscriber() {
+         _stopwatch.Start();
+         DataFrame df = DataFrame.LoadCsv(_file_address);
+         _stopwatch.Stop();
+         bts.Log.Gen.info($"CSVReaderStream.myOnFirstSubscriber stream csv load col_count: {df.Columns.Count} rows, time: {_stopwatch.ElapsedMilliseconds} msecs");
+         _stopwatch.Start();
+         IBlueRowCollection rows = BlueDataUtility.createRowsFromCSVFileWithoutTypes(_file_address);
+         _stopwatch.Stop();
+         bts.Log.Gen.info($"CSVReaderStream.myOnFirstSubscriber stream csv load count: {rows.Count} rows, time: {_stopwatch.ElapsedMilliseconds} msecs");
+         DataPacket dp_out = BlueDataUtility.build_dp_inc(rows, myLastValue as DataPacket);
+         myFireEvent(dp_out);
      }
   }

-   [DataStreamUserRespresentation ("CSVReaderStreamMapper")]
-   public class CSVReaderStreamMapperStream : ViewStream
-   {
-      public CSVReaderStreamMapperStream(ViewStream stream, string output_schema)
-      {
+   [DataStreamUserRespresentation("CSVReaderStreamMapper")]
+   public class CSVReaderStreamMapperStream : ViewStream {
+      public CSVReaderStreamMapperStream(ViewStream stream, string output_schema) {
         mySetRealSource(new MapStream(stream, new CSVReaderStreamMapper(output_schema)));
      }
   }

-   public class CSVReaderStreamMapper : IStreamMapper
-   {
+   public class CSVReaderStreamMapper : IStreamMapper {
      private IBlueTableSchema mySchema;
      private readonly string[] myOutputSchemaStringArray;

-      public CSVReaderStreamMapper(string outputSchema)
-      {
+      public CSVReaderStreamMapper(string outputSchema) {
         myOutputSchemaStringArray = outputSchema.Split(' ');
      }

-      public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema)
-      {
-         if (mySchema == null)
-         {
+      public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema) {
+         if (mySchema == null) {
            ColumnInfo[] ci = inputSchema.columnInfo;
-            foreach (string col in myOutputSchemaStringArray)
-            {
+            foreach (string col in myOutputSchemaStringArray) {
               string colName = col.Substring(0, col.IndexOf('\\')).ToLower();
-               for (int i = 0; i < ci.Length; i++)
-               {
-                  if (ci[i].colName.ToLower().Equals(colName))
-                  {
+               for (int i = 0; i < ci.Length; i++) {
+                  if (ci[i].colName.ToLower().Equals(colName)) {
                     Type t = myGetType(col.Substring(col.IndexOf('\\') + 1));
                     ci[i] = new ColumnInfo(ci[i].colName, i, t);
                     break;
@@ -171,74 +160,52 @@ namespace BCG
         return mySchema;
      }

-      public IBlueRow map(IBlueRow row)
-      {
+      public IBlueRow map(IBlueRow row) {
         object[] vals = new object[row.schema.columnInfo.Length];
         row.Values.CopyTo(vals, 0);
-         for (int i = 0; i < mySchema.columnInfo.Length; i++)
-         {
+         for (int i = 0; i < mySchema.columnInfo.Length; i++) {
            Type t = mySchema.columnInfo[i].colType;
-            try
-            {
+            try {
               vals[i] = System.Convert.ChangeType(vals[i], mySchema.columnInfo[i].colType, System.Globalization.CultureInfo.InvariantCulture);
-            }
-            catch
-            {
-               if (t.Equals(typeof(Int16)))
-               {
+            } catch {
+               if (t.Equals(typeof(Int16))) {
                  Int16 result;
-                  if (Int16.TryParse((string)vals[i], out result))
-                  {
+                  if (Int16.TryParse((string)vals[i], out result)) {
                     vals[i] = result;
                     continue;
                  }
                  double r2;
-                  if (Double.TryParse((string)vals[i], out r2))
-                  {
-                     try
-                     {
+                  if (Double.TryParse((string)vals[i], out r2)) {
+                     try {
                        vals[i] = (Int16)r2;
                        continue;
-                     }
-                     catch { }
+                     } catch { }
                  }
-               }
-               else if (t.Equals(typeof(Int32)))
-               {
+               } else if (t.Equals(typeof(Int32))) {
                  Int32 result;
-                  if (Int32.TryParse((string)vals[i], out result))
-                  {
+                  if (Int32.TryParse((string)vals[i], out result)) {
                     vals[i] = result;
                     continue;
                  }
                  double r2;
-                  if (Double.TryParse((string)vals[i], out r2))
-                  {
-                     try
-                     {
+                  if (Double.TryParse((string)vals[i], out r2)) {
+                     try {
                        vals[i] = (Int32)r2;
                        continue;
-                     }
-                     catch { }
+                     } catch { }
                  }
-               }
-               else if (t.Equals(typeof(Int64)))
-               {
+               } else if (t.Equals(typeof(Int64))) {
                  Int64 result;
-                  if (Int64.TryParse((string)vals[i], out result))
-                  {
+                  if (Int64.TryParse((string)vals[i], out result)) {
                     vals[i] = result;
                     continue;
                  }
                  double r2;
-                  if (Double.TryParse((string)vals[i], out r2))
-                  {
-                     try
-                     {
+                  if (Double.TryParse((string)vals[i], out r2)) {
+                     try {
                        vals[i] = (Int64)r2;
                        continue;
-                     }
-                     catch { }
+                     } catch { }
                  }
               }
               vals[i] = DBNull.Value;
@@ -247,8 +214,7 @@ namespace BCG
         return new BlueRow(mySchema, vals);
      }

-      private Type myGetType(string type)
-      {
+      private Type myGetType(string type) {
         type = type.ToLower();
         if (type.Contains("int"))
            return typeof(int);
@@ -1,31 +1,41 @@
 using System;
 using System.Collections;
 using System.Reflection;
+
 using BCG.Utilities;

-namespace BCG
-{
-   [DataStreamUserRespresentation (false)]
-   public class BlueRowCollectionStaticStream : ViewStream
-   {
-      public BlueRowCollectionStaticStream () {}
-
-      public BlueRowCollectionStaticStream (IBlueRowCollection rows) {loadCollection (rows);}
-
-      public void loadCollection (IBlueRowCollection rows)
-      {
-         myFireEvent (new DataPacket (rows, rows, myLastValue != null ?
-                                                   new BlueTable((myLastValue as DataPacket).newRows) : new BlueTable (rows.schema)));
+namespace BCG {
+   // if a table needs to be fired, it can be wrapped around this
+   public class BlueTableStream : ViewStream {
+      private IBlueRowCollection _rows;
+      public BlueTableStream(IBlueRowCollection rows) {
+         _rows = rows;
      }

-      public void crush_with_single_row(IBlueRow r)
-      {
+      protected override void myOnFirstSubscriber() {
+         var dp_out = BlueDataUtility.build_dp_inc(_rows, myLastValue as DataPacket);
+         myFireEvent(dp_out);
+      }
+   }
+
+   [DataStreamUserRespresentation(false)]
+   public class BlueRowCollectionStaticStream : ViewStream {
+      public BlueRowCollectionStaticStream() { }
+
+      public BlueRowCollectionStaticStream(IBlueRowCollection rows) { loadCollection(rows); }
+
+      public void loadCollection(IBlueRowCollection rows) {
+         myFireEvent(new DataPacket(rows, rows, myLastValue != null ?
+                                                   new BlueTable((myLastValue as DataPacket).newRows) : new BlueTable(rows.schema)));
+      }
+
+      public void crush_with_single_row(IBlueRow r) {
         BlueTable t = new BlueTable(r.schema);
         t.addRow(r);
         loadCollection(t);
      }

-      public void loadPacket (DataPacket dp) {myFireEvent (dp);}
+      public void loadPacket(DataPacket dp) { myFireEvent(dp); }

      /// <summary>
      /// Will work with streams that take ViewStream as first argument ex project, filter, distinct etc
@@ -35,17 +45,16 @@ namespace BCG
      /// <param name="streamType"></param>
      /// <param name="args"></param>
      /// <returns></returns>
-      public static IBlueRowCollection statelessOp (IBlueRowCollection rows, Type streamType, params object[] args)
-      {
-         ArrayList totArgs = new ArrayList (args);
-         BlueRowCollectionStaticStream v1 = new BlueRowCollectionStaticStream ();
-         totArgs.Insert (0, v1);
-         ViewStream retVal = Activator.CreateInstance (streamType, totArgs.ToArray()) as ViewStream;
-         v1.loadCollection (rows);
+      public static IBlueRowCollection statelessOp(IBlueRowCollection rows, Type streamType, params object[] args) {
+         ArrayList totArgs = new ArrayList(args);
+         BlueRowCollectionStaticStream v1 = new BlueRowCollectionStaticStream();
+         totArgs.Insert(0, v1);
+         ViewStream retVal = Activator.CreateInstance(streamType, totArgs.ToArray()) as ViewStream;
+         v1.loadCollection(rows);
         return retVal.rows;
      }

   }

-   
+
 }
@@ -35,14 +35,14 @@ namespace BCG
      #region ViewStream
      protected override void myOnFirstSubscriber()
      {
-         myStream.onChange +=new ChangeDelegate(myStreamChange);
+         myStream.onChange += myStreamChange;
         myVsd.addWorker(this);
      }

      protected override void  myOnLastSubscriber()
      {
         myVsd.removeWorker(this);
-         myStream.onChange -= new ChangeDelegate(myStreamChange);
+         myStream.onChange -= myStreamChange;
      }
      #endregion

@@ -28,7 +28,7 @@ namespace BCG

      protected override void myOnFirstSubscriber()
      {
-         myParentStream.onChange += new ChangeDelegate(myParentStreamChangeHandler);
+         myParentStream.onChange += myParentStreamChangeHandler;
         myVSD.addWorker(this);
      }

@@ -36,7 +36,7 @@ namespace BCG

      protected override void myOnLastSubscriber()
      {
-         myParentStream.onChange -= new ChangeDelegate(myParentStreamChangeHandler);
+         myParentStream.onChange -= myParentStreamChangeHandler;
         myVSD.removeWorker(this);
      }

@@ -53,11 +53,11 @@ namespace BCG
            if (r.is_subscription)
            {
               _current_source_subscription = r.subscription_id;
-               _source_stream.onChange += new ChangeDelegate(_source_stream_onChange);
+               _source_stream.onChange += _source_stream_onChange;
            }
            else
            {
-               _source_stream.onChange -= new ChangeDelegate(_source_stream_onChange);
+               _source_stream.onChange -= _source_stream_onChange;
            }
         }
      }
@@ -0,0 +1,34 @@
+namespace bts.SilverPine {
+   using bts;
+
+   using BCG;
+
+   using System;
+
+   public class DateTimeExpanstionStream : ViewStream {
+      public DateTimeExpanstionStream(ViewStream parent_stream, string inp_col) {
+         Func<IBlueTableSchema, IBlueTableSchema> get_schema = (s) => {
+            IBlueTableSchema rv = s.add_column(System.TypeCode.DateTime, $"{inp_col}_date");
+            rv = rv.add_column(System.TypeCode.String, $"{inp_col}_day");
+            rv = rv.add_column(System.TypeCode.String, $"{inp_col}_month");
+            rv = rv.add_column(System.TypeCode.String, $"{inp_col}_hour");
+            rv = rv.add_column(System.TypeCode.String, $"{inp_col}_min");
+            return rv;
+         };
+         ViewStream rs = new MapStream(parent_stream,
+         s => get_schema(s),
+         r => {
+            DateTime dt = r.get_datetime(inp_col);
+            IBlueTableSchema s = get_schema(r.schema);
+            MutableBlueRow rv = new MutableBlueRow(s, r);
+            rv.setValue($"{inp_col}_date", dt.Date);
+            rv.setValue($"{inp_col}_day", dt.DayOfWeek.ToString());
+            rv.setValue($"{inp_col}_month", dt.Date.Month.ToString());
+            rv.setValue($"{inp_col}_hour", dt.Hour);
+            rv.setValue($"{inp_col}_min", dt.Minute);
+            return rv;
+         });
+         mySetRealSource(rs);
+      }
+   }
+}
@@ -24,7 +24,7 @@ namespace BCG
        protected override void myOnFirstSubscriber()
        {
            myKeyToTimeToValue = new Dictionary<string, SortedList<DateTime, double>>();
-            myToAverage.onChange += new ChangeDelegate(myToAverage_onChange);
+            myToAverage.onChange += myToAverage_onChange;
        }


@@ -0,0 +1,111 @@
+namespace bts.SilverPine {
+   using BCG;
+
+   using bts;
+   using bts.collections;
+
+   using System;
+   using System.CodeDom;
+   using System.Collections.Generic;
+   using System.Linq;
+
+   // Given 2 filters, and 2 columns, finds the different in the value of the 2 columns from the 2 rows
+   // so:
+   // f1(base_stream.rows)[col1] - f1(base_stream.rows)[col2]
+   [DataStreamUserRespresentation("ColDiff")]
+   public class ColumnDifferenceStream : ViewStream {
+      private readonly ViewStream _base_stream;
+      private readonly FilterStream.OnFilter _f1, _f2;
+      private readonly string _c1, _c2;
+
+      public ColumnDifferenceStream(ViewStream base_stream, FilterStream.OnFilter f1, FilterStream.OnFilter f2, string col1, string col2) {
+         _base_stream = base_stream;
+         _f1 = f1;
+         _f2 = f2;
+         _c1 = col1;
+         _c2 = col2;
+      }
+
+      public ColumnDifferenceStream(ViewStream base_stream, string f1, string f2, string col1, string col2) {
+         FilterStream.OnFilter f1_del = r => {
+            BooleanExpressionFilter rv = new BooleanExpressionFilter(f1);
+            return rv.filter(r);
+         };
+         FilterStream.OnFilter f2_del = r => {
+            BooleanExpressionFilter rv = new BooleanExpressionFilter(f2);
+            return rv.filter(r);
+         };
+         ViewStream rs = new ColumnDifferenceStream(base_stream, f1_del, f2_del, col1, col2);
+         mySetRealSource(rs);
+      }
+
+      // for each row, does a row[new_ol_name] = row[col1] - rows[row_ffset][col2]
+      // we should generlize this for any math operation
+      public ColumnDifferenceStream(ViewStream parent_stream, string sort_col_name, int row_offset, string col1, string col2, string diff_col_name, double optional_perc_diff_filter = 0.0) {
+         // use GlobalMapStream
+         string diff_col_perc_name = $"{diff_col_name}_percentage";
+         System.Func<System.Collections.Generic.List<BCG.IBlueRow>, uint, BCG.IBlueRow> g_map_func = (rows, idx) => {
+            int minuend_idx = Convert.ToInt32(idx);
+            var inp_schema = rows[0].schema;
+            var out_schema = inp_schema.add_column(TypeCode.Double, diff_col_name);
+            out_schema = out_schema.add_column(TypeCode.Double, diff_col_perc_name);
+            // minuend - subtrahend
+            IBlueRow minuend_row = rows.ElementAt(minuend_idx);
+            MutableBlueRow rv = new MutableBlueRow(out_schema, minuend_row);
+            int subtrahend_index = minuend_idx + row_offset;
+            double diff_val = Double.NaN, diff_val_perc = Double.NaN;
+            if (subtrahend_index >= 0) {
+               try {
+                  IBlueRow subtrahend_row = rows.ElementAt(subtrahend_index);
+                  diff_val = minuend_row.get<double>(col1) - subtrahend_row.get<double>(col2);
+                  diff_val_perc = 100.0 * diff_val / subtrahend_row.get<double>(col1);
+               } catch (Exception ex) {
+                  Log.assert(false, ex.Message);
+               }
+            }
+            rv.setValue(diff_col_name, diff_val);
+            rv.setValue(diff_col_perc_name, diff_val_perc);
+            return rv;
+         };
+         ViewStream rs = new GlobalMapStream(parent_stream, sort_col_name, g_map_func);
+         if (optional_perc_diff_filter != 0) {
+            rs = new FilterStream(rs, r => {
+               double diff_val_perc = r.get<double>(diff_col_perc_name);
+               double abs_diff_val_perc = Math.Abs(diff_val_perc);
+               return abs_diff_val_perc >= optional_perc_diff_filter;
+            });
+         }
+         mySetRealSource(rs);
+      }
+
+      protected override void myOnFirstSubscriber() {
+         _base_stream.onChange += _handle_base_stream_update;
+      }
+
+      protected override void myOnLastSubscriber() {
+         _base_stream.onChange -= _handle_base_stream_update;
+      }
+
+      private void _handle_base_stream_update(object dp_obj) {
+         // could have been done with filter and cartesian prod, and then math
+         DataPacket dp = dp_obj as DataPacket;
+         List<IBlueRow> rows = dp.newRows.to_list();
+         var f1_rows = rows.Where(r => _f1(r));
+         Log.assert(f1_rows.Count() == 1);
+         var f2_rows = rows.Where(r => _f2(r));
+         Log.assert(f2_rows.Count() == 1);
+         IBlueRow f1_r = f1_rows.ElementAt(0);
+         IBlueRow f2_r = f2_rows.ElementAt(0);
+         double diff = f1_r.get<double>(_c1) - f2_r.get<double>(_c2);
+         IBlueTableSchema out_schema = f1_r.schema.add_columns(Tuple.Create(typeof(double), "diff"));
+         BlueTable out_table = new BlueTable(out_schema);
+         MutableBlueRow out_row = new MutableBlueRow(out_schema, f1_r);
+         out_row.setValue("diff", diff);
+         out_table.addRow(out_row);
+         DataPacket dp_out = BlueDataUtility.build_dp(out_table, myLastValue as DataPacket);
+         myFireEvent(dp_out);
+
+      }
+   }
+
+}
@@ -0,0 +1,51 @@
+namespace bts.SilverPine {
+   using BCG;
+
+   using bts;
+
+   using System.Collections.Generic;
+   using System.Linq;
+
+   // Given a sort order, an optional group by col, it observes the values of a column 
+   // within that group (or the whole table, if grouping not specified) and returns all rows
+   // where the value changes, ie row[current][col] != row[current-1][col]
+   // TODO: a more general way to express a filter where row[current][col] != row[previous][col],
+   // or even expand it to row[current][col_a] - row[previous][col_b] < 0.5
+   [DataStreamUserRespresentation("ColEvol")]
+   public class ColumnEvolutionStream : ViewStream {
+      private readonly ViewStream _base_stream;
+      private readonly string _sort_col;
+      private readonly string _evol_col;
+      public ColumnEvolutionStream(ViewStream base_stream, string sort_col, string evolution_col) {
+         _base_stream = base_stream;
+         _sort_col = sort_col;
+         _evol_col = evolution_col;
+      }
+
+      protected override void myOnFirstSubscriber() {
+         _base_stream.onChange += _handle_base_stream_update;
+      }
+
+      protected override void myOnLastSubscriber() {
+         base.myOnLastSubscriber();
+      }
+
+      private void _handle_base_stream_update(object dp_obj) {
+         DataPacket dp = dp_obj as DataPacket;
+         List<IBlueRow> rows = dp.newRows.to_list();
+         rows.Sort((r1, r2) => BlueDataUtility.compare_rows(r2, r1, _sort_col));
+         BlueTable rv = new BlueTable(dp.schema);
+         IBlueRow prev_row = rows.ElementAt(0);
+         foreach (IBlueRow r in rows) {
+            var cr = BlueDataUtility.compare_rows(r, prev_row, _evol_col);
+            if (cr != 0) {
+               rv.addRow(r);
+            }
+            prev_row = r;
+         }
+         DataPacket dp_out = BlueDataUtility.build_dp(rv, myLastValue as DataPacket);
+         myFireEvent(dp_out);
+      }
+   }
+
+}
@@ -12,10 +12,10 @@ namespace BCG {
         mySrc = src;
      }
      protected override void myOnFirstSubscriber() {
-         mySrc.onChange += new ChangeDelegate(mySrc_onChange);
+         mySrc.onChange += mySrc_onChange;
      }
      protected override void myOnLastSubscriber() {
-         mySrc.onChange -= new ChangeDelegate(mySrc_onChange);
+         mySrc.onChange -= mySrc_onChange;
      }
      private void mySrc_onChange(object newValue) {
         DataPacket dp = (DataPacket)newValue;
@@ -28,14 +28,14 @@ namespace BCG

      protected override void myOnFirstSubscriber()
      {
-         myLeftStream.onChange += new ChangeDelegate(myHandleLeftChanges);
-         myRightstream.onChange += new ChangeDelegate(myHandleRightChanges);
+         myLeftStream.onChange += myHandleLeftChanges;
+         myRightstream.onChange += myHandleRightChanges;
      }

      protected override void myOnLastSubscriber()
      {
-         myLeftStream.onChange -= new ChangeDelegate(myHandleLeftChanges);
-         myRightstream.onChange -= new ChangeDelegate(myHandleRightChanges);
+         myLeftStream.onChange -= myHandleLeftChanges;
+         myRightstream.onChange -= myHandleRightChanges;
      }

      private void myHandleLeftChanges (object newValue)
@@ -222,7 +222,7 @@ namespace BCG
            pj = new ProjectStream(pj, "!");
         myStreamIndex.Add(myMovingIndex, new Pair<ViewStream, ViewStream>(vs, pj));
         if (mySubscriptionActive)
-            pj.onChange += new ChangeDelegate(myHandleStreamChanges);
+            pj.onChange += myHandleStreamChanges;
      }

      public void removeStream(ViewStream vs)
@@ -235,7 +235,7 @@ namespace BCG
               index = p.Key;
               if (mySubscriptionActive)
               {
-                  p.Value.second.onChange -= new ChangeDelegate(myHandleStreamChanges);
+                  p.Value.second.onChange -= myHandleStreamChanges;
                  if (myTable != null)
                  {
                     IBlueRowCollection delRows = myTable.removeTable(p.Key);
@@ -254,14 +254,14 @@ namespace BCG
      {
         mySubscriptionActive = true;
         foreach (KeyValuePair<int, Pair<ViewStream, ViewStream>> p in myStreamIndex)
-            p.Value.second.onChange += new ChangeDelegate(myHandleStreamChanges);
+            p.Value.second.onChange += myHandleStreamChanges;
      }

      protected override void myOnLastSubscriber()
      {
         mySubscriptionActive = false;
         foreach (KeyValuePair<int, Pair<ViewStream, ViewStream>> p in myStreamIndex)
-            p.Value.second.onChange -= new ChangeDelegate(myHandleStreamChanges);
+            p.Value.second.onChange -= myHandleStreamChanges;
         if (myTable != null)
            myTable.clear();
      }
@@ -14,11 +14,11 @@ namespace BCG {
      }
      protected override void myOnFirstSubscriber()
      {
-         mySrc.onChange += new ChangeDelegate(mySrc_onChange);
+         mySrc.onChange += mySrc_onChange;
      }
      protected override void myOnLastSubscriber () 
      {
-         mySrc.onChange -= new ChangeDelegate (mySrc_onChange);
+         mySrc.onChange -= mySrc_onChange;
      }
      private void mySrc_onChange (object newValue) {
         DataPacket dp = (DataPacket)newValue;
@@ -21,15 +21,15 @@ namespace BCG
      {
         myLeftOp = new UnionOperand (true, this);
         myRightOp = new UnionOperand (false, this);
-         myLeftVs.onChange += new DataStream.ChangeDelegate (myLeftOp.myChangeHandler);
-         myRightVs.onChange += new DataStream.ChangeDelegate (myRightOp.myChangeHandler);
+         myLeftVs.onChange += myLeftOp.myChangeHandler;
+         myRightVs.onChange += myRightOp.myChangeHandler;
      }

      protected override void myOnLastSubscriber () 
      {
         myInit ();
-         myLeftVs.onChange -= new DataStream.ChangeDelegate (myLeftOp.myChangeHandler);
-         myRightVs.onChange -= new DataStream.ChangeDelegate (myRightOp.myChangeHandler);
+         myLeftVs.onChange -= myLeftOp.myChangeHandler;
+         myRightVs.onChange -= myRightOp.myChangeHandler;
      }

      public void startSession ()
@@ -116,4 +116,4 @@ namespace BCG
          receiveStream(dp, isLeft);
      }
   }
-}
+}
@@ -14,11 +14,11 @@ namespace BCG {

      protected override void myOnFirstSubscriber() {
         _vs_publisher = new ViewPublisher(_name, _broker, _main_stream);
-         _main_stream.onChange += new DataStream.ChangeDelegate(_handle_update);
+         _main_stream.onChange += _handle_update;
      }

      protected override void myOnLastSubscriber() {
-         _main_stream.onChange -= new DataStream.ChangeDelegate(_handle_update);
+         _main_stream.onChange -= _handle_update;
         _vs_publisher.Dispose();
      }

@@ -28,4 +28,4 @@ namespace BCG {


   }
-}
+}
@@ -20,11 +20,11 @@ namespace BCG {
      public ViewPublisher(string name, IPubTableBroker broker, ViewStream vs) {
         _stream = String.IsNullOrEmpty(name) ? vs : new ChangeTableNameStream(vs, name);
         _broker = broker;
-         _stream.onChange += new DataStream.ChangeDelegate(_handle_update);
+         _stream.onChange += _handle_update;
      }

      public void Dispose() {
-         _stream.onChange -= new DataStream.ChangeDelegate(_handle_update);
+         _stream.onChange -= _handle_update;
      }

      private void _handle_update(object o) {
@@ -15,10 +15,74 @@ using mbts;
 using BCG.Extensions;
 using System.Text.Json;
 using System.Text.Json.Nodes;
+using Newtonsoft.Json;
+using BCG;
+using System.Runtime.CompilerServices;
+
+namespace bts {
+
+   // public class CompositeRow : IBlueRow {
+   //    private IBlueRow _lrow, _rrow;
+   //    private BlueRow _out_row;
+   //    public CompositeRow(IBlueRow lrow, IBlueRow rrow) {
+   //       _lrow = lrow;
+   //       _rrow = rrow;
+   //       IBlueTableSchema schema = _lrow.schema;
+   //       List<object> vals = new List<object>();
+   //       foreach (var o in _lrow) {
+   //          vals.Add(o);
+   //       }
+   //       foreach (ColumnInfo ci in _rrow.schema.columnInfo) {
+   //          if (ci.colName == "id") {
+   //             continue;
+   //          }
+   //          vals.Add(_rrow[ci.colNo]);
+   //          schema = schema.add_column(ci.type_code(), ci.colName);
+   //       }
+   //       _out_row = new BlueRow(schema, vals.ToArray());
+   //    }
+   //    public IBlueTableSchema schema {
+   //       get { return _out_row.schema; }
+   //    }
+   //    public object? this[int index] {
+   //       get { return _out_row[index]; }
+   //    }
+   //    public object? this[string colName] {
+   //       get {
+   //          return _out_row[colName];
+   //       }
+   //    }
+
+   // }
+}

 namespace BCG {
   public static class BlueDataUtility {
+      public static IBlueRow create_composite_row(IBlueRow lr, IBlueRow rr, int id) {
+         IBlueTableSchema schema = lr.schema;
+         List<object> vals = new List<object>();
+         foreach (ColumnInfo ci in lr.schema.columnInfo) {
+            if (ci.colName == "id") {
+               vals.Add(id);
+            }
+            vals.Add(lr[ci.colName]);
+         }
+         foreach (ColumnInfo ci in rr.schema.columnInfo) {
+            if (ci.colName == "id") {
+               continue;
+            }
+            schema = schema.add_column(ci.type_code(), ci.colName);
+            vals.Add(rr[ci.colName]);
+         }
+         IBlueRow r = new BlueRow(schema, vals.ToArray());
+         return r;
+      }
      #region Extensions
+      public static bool is_null(this IBlueRow r, string col_name) {
+         object col_val = r[col_name];
+         return col_val == null || DBNull.Value.Equals(col_val);
+      }
+
      public static DateTime get_datetime(this IBlueRow r, string col_name, DateTime err_value = default(DateTime)) {
         return BlueDataUtility.get_row_datetime_value(r, col_name, err_value);
      }
@@ -49,6 +113,17 @@ namespace BCG {
            rv.Add(r.get<T>(col));
         return rv;
      }
+      public static IEnumerable<T> get_colvalues<T>(IEnumerable<IBlueRow> rows, string col, int start_index, int count) {
+         List<T> rv = new List<T>();
+         int i = 0;
+         int n_copied = 0;
+         foreach (IBlueRow r in rows)
+            if (i++ >= start_index && n_copied < count) {
+               rv.Add(r.get<T>(col));
+               n_copied++;
+            }
+         return rv;
+      }

      public static IDictionary<T, IEnumerable<IBlueRow>> group_by<T>(IBlueRowCollection rows, string gp_col) {
         Dictionary<T, List<IBlueRow>> rv = new Dictionary<T, List<IBlueRow>>();
@@ -112,13 +187,17 @@ namespace BCG {
         }
      }

-      public static IEnumerable<IBlueRow> to_ienum(this IBlueRowCollection rows) {
+      public static List<IBlueRow> to_list(this IBlueRowCollection rows) {
         List<IBlueRow> rv = new List<IBlueRow>();
         foreach (IBlueRow r in rows)
            rv.Add(r);
         return rv;
      }

+      public static IEnumerable<IBlueRow> to_ienum(this IBlueRowCollection rows) {
+         return rows.to_list();
+      }
+
      public static IDictionary<int, IBlueRow> to_map(this IBlueRowCollection rows) {
         Dictionary<int, IBlueRow> rv = new Dictionary<int, IBlueRow>();
         foreach (IBlueRow r in rows) {
@@ -135,6 +214,11 @@ namespace BCG {
         return new BlueTableSchema(input_schema.tableName, cis.ToArray());
      }

+      public static IBlueTableSchema add_column(this IBlueTableSchema input_schema, TypeCode type_arg, string col_name) {
+         Tuple<Type, string> tup = new Tuple<Type, string>(_typecode_to_type[type_arg], col_name);
+         return add_columns(input_schema, tup);
+      }
+
      public static IBlueRowCollection to_collection(this IBlueRow r) {
         if (r == null)
            return null;
@@ -191,6 +275,18 @@ namespace BCG {

      #region json

+      public static Dictionary<string, string> convert_json_to_dictionary(string json_string) {
+         Dictionary<string, string> dictionary = JsonConvert.DeserializeObject<Dictionary<string, string>>(json_string);
+         return dictionary;
+      }
+      public static IBlueRow convert_json_to_row(string json_string, int id = 0) {
+         Dictionary<string, object> dictionary = JsonConvert.DeserializeObject<Dictionary<string, object>>(json_string);
+         if (!dictionary.ContainsKey("id")) {
+            dictionary.Add("id", id);
+         }
+         return new BlueRow(dictionary);
+      }
+
      public static List<JsonNode> json_to_json_node_list(string jsonString) {
         var jsonNodeList = new List<JsonNode>();
         bool noe = String.IsNullOrEmpty(jsonString);
@@ -209,7 +305,7 @@ namespace BCG {
            foreach (JsonNode jsonNode in jsonArray) {
               jsonNodeList.Add(jsonNode);
            }
-         } catch (JsonException ex) {
+         } catch (System.Text.Json.JsonException ex) {
            Console.WriteLine($"Invalid JSON: {ex.Message}");
         }

@@ -486,7 +582,16 @@ namespace BCG {
         ColumnInfo[] cis = new ColumnInfo[table.Columns.Count];
         foreach (DataColumn col in table.Columns)
            cis[col.Ordinal] = new ColumnInfo(col.ColumnName, col.Ordinal, col.DataType);
-         return new BlueTableSchema(cis, table.TableName);
+         IBlueTableSchema rv = new BlueTableSchema(cis, table.TableName);
+         rv = rv.add_id_col_if_missing();
+         return rv;
+      }
+      public static IBlueTableSchema add_id_col_if_missing(this IBlueTableSchema schema) {
+         IBlueTableSchema rv = schema;
+         if (!schema.columnNames.Contains("id")) {
+            rv = schema.add_columns(Tuple.Create(typeof(int), "id"));
+         }
+         return rv;
      }

      /// <summary>
@@ -744,10 +849,16 @@ namespace BCG {
      }

      public static IBlueRowCollection toIBlueRowCollection(DataTable dt) {
-         BlueTable retVal = new BlueTable(getSchema(dt));
-         foreach (DataRow dr in dt.Rows)
-            retVal.addRow(dr);
-         return retVal;
+         IBlueTableSchema schema = getSchema(dt);
+         BlueTable rv = new BlueTable(schema);
+         foreach (DataRow dr in dt.Rows) {
+            MutableBlueRow nr = new MutableBlueRow(schema);
+            foreach (DataColumn col in dr.Table.Columns)
+               nr.setValue(col.ColumnName, dr[col.ColumnName]);
+            nr.setValue("id", rv.Count);
+            rv.addRow(nr);
+         }
+         return rv;
      }

      public static IBlueRowCollection toIBlueRowCollection(DataRow[] rows) {
@@ -819,20 +930,20 @@ namespace BCG {
      /// </summary>
      /// <param name="r1"></param>
      /// <param name="r2"></param>
-      /// <param name="colNames"></param>
+      /// <param name="col_names"></param>
      /// <returns></returns>
      public static int compare_rows(IBlueRow r1,
         IBlueRow r2,
-         params string[] colNames) {
-         int retVal = 0;
-         foreach (string colName in colNames) {
-            object val1 = r1[colName];
-            object val2 = r2[colName];
-            retVal = _compare_row_cols(val1, val2);
-            if (retVal != 0)
+         params string[] col_names) {
+         int comp_res = 0;
+         foreach (string cn in col_names) {
+            object val1 = r1[cn];
+            object val2 = r2[cn];
+            comp_res = _compare_row_cols(val1, val2);
+            if (comp_res != 0)
               break;
         }
-         return retVal;
+         return comp_res;
      }

      private static int _compare_row_cols(object v1, object v2) {
@@ -96,3 +96,107 @@ public class BlueTableImmutable : IBlueRowCollection, IImmutableRowCollection {
         new(schema, table.ToImmutable());
   }
 }
+
+public class BlueTableImmutable<T> : IBlueRowCollection<T> where T : IBlueRow {
+   readonly ImmutableDictionary<int, T> rows;
+
+   public BlueTableImmutable(IBlueTableSchema schema)
+      : this(schema, ImmutableDictionary<int, T>.Empty) { }
+
+   public BlueTableImmutable(IBlueTableSchema schema, ImmutableDictionary<int, T> rows) {
+      this.schema = schema;
+      this.rows = rows;
+   }
+
+   public int Count => rows.Count;
+   public bool IsSynchronized => false;
+   public IBlueTableSchema schema { get; }
+   public object SyncRoot => this;
+
+   public T this[int id] => rows[id];
+
+   public bool containsID(int id) =>
+      rows.ContainsKey(id);
+
+   public void CopyTo(Array array, int index) {
+      foreach (var r in rows.Values) {
+         array.SetValue(r, index++);
+      }
+   }
+
+   public IEnumerator<T> GetEnumerator() =>
+      rows.Values.GetEnumerator();
+
+   IEnumerator IEnumerable.GetEnumerator() =>
+      GetEnumerator();
+
+   public DataPacketBuilder ToDataPacketBuilder() =>
+      new(this);
+
+   public class DataPacketBuilder {
+      readonly IBlueTableSchema schema;
+      readonly ImmutableDictionary<int, T>.Builder table;
+      readonly ImmutableDictionary<int, T>.Builder added =
+         ImmutableDictionary<int, T>.Empty.ToBuilder();
+      readonly ImmutableDictionary<int, T>.Builder removed =
+         ImmutableDictionary<int, T>.Empty.ToBuilder();
+
+      public DataPacketBuilder(BlueTableImmutable<T> table) {
+         schema = table.schema;
+         this.table = table.rows.ToBuilder();
+      }
+
+      public void AddOrUpdate(T row, IEqualityComparer<IBlueRow>? comparer = null) {
+         var id = row.Id();
+         if (table.TryGetValue(id, out var orig)) {
+            if (comparer?.Equals(row, orig) ?? false) {
+               // No op: This row was already in the table (according to 'comparer').
+               // Remove it from 'added' and 'removed'.
+               added.Remove(id);
+               removed.Remove(id);
+               return;
+            }
+            // Change op: A row with this id was already in the table,
+            // but its values have changed (according to 'comparer').
+            removed[id] = orig;
+         }
+         added[id] = row;
+      }
+
+      public void Remove(int id) {
+         added.Remove(id);
+         if (table.TryGetValue(id, out var orig)) {
+            removed[id] = orig;
+         }
+      }
+
+      public DataPacket ToDataPacket(bool reset = true) {
+         table.RemoveRange(removed.Keys);
+         table.AddRange(added);
+         var newRows = new BlueTableImmutable<T>(schema, table.ToImmutable());
+         var addedRows = new BlueTableImmutable<T>(schema, added.ToImmutable());
+         var removedRows = new BlueTableImmutable<T>(schema, removed.ToImmutable());
+         if (reset) {
+            added.Clear();
+            removed.Clear();
+         }
+         return new(newRows, addedRows, removedRows);
+      }
+
+      public IDataPacket<T> ToDataPacketT(bool reset = true) {
+         table.RemoveRange(removed.Keys);
+         table.AddRange(added);
+         var newRows = new BlueTableImmutable<T>(schema, table.ToImmutable());
+         var addedRows = new BlueTableImmutable<T>(schema, added.ToImmutable());
+         var removedRows = new BlueTableImmutable<T>(schema, removed.ToImmutable());
+         if (reset) {
+            added.Clear();
+            removed.Clear();
+         }
+         return new DataPacket<T>(newRows, addedRows, removedRows);
+      }
+
+      public BlueTableImmutable<T> ToImmutable() =>
+         new(schema, table.ToImmutable());
+   }
+}
@@ -24,7 +24,16 @@ namespace BCG
      /// </summary>
      object? this[int index] {get;}
      object? this[string colName] {get;}
+      bool IDictionary.IsFixedSize => true;
+      bool IDictionary.IsReadOnly => true;
+      object ICollection.SyncRoot => this;
+      bool ICollection.IsSynchronized => false;
   }
+
+   // public interface IBlueRow<T> : IBlueRow where T : IBlueRow {
+   //    T Value => (T)this;
+   // }
+
   namespace Extensions
   {
      public static class IBlueRowExtensions
@@ -1,5 +1,6 @@
 using System;
 using System.Collections;
+using System.Collections.Generic;

 namespace BCG
 {
@@ -16,4 +17,10 @@ namespace BCG
      IBlueTableSchema schema {get;}
      bool containsID (int id);
   }
+
+   public interface IBlueRowCollection<T> : IBlueRowCollection, IReadOnlyCollection<T> where T : IBlueRow {
+      new T this[int id] { get; }
+
+      IBlueRow IBlueRowCollection.this[int id] => this[id];
+   }
 }
@@ -314,6 +314,18 @@ namespace BCG
      }
   }

+   public class DataPacket<T> : IDataPacket<T> where T : IBlueRow {
+      public DataPacket(IBlueRowCollection<T> newRows, IBlueRowCollection<T> addedRows, IBlueRowCollection<T> deletedRows) {
+         this.newRows = newRows;
+         this.addedRows = addedRows;
+         this.deletedRows = deletedRows;
+      }
+
+      public IBlueRowCollection<T> newRows { get; }
+      public IBlueRowCollection<T> addedRows { get; }
+      public IBlueRowCollection<T> deletedRows { get; }
+   }
+
   public static class DataPacketExtensions {
      public static bool HasChanges(this DataPacket dataPacket) =>
         dataPacket.addedRows.Count > 0 || dataPacket.deletedRows.Count > 0;
@@ -53,16 +53,15 @@ namespace BCG

      public event Action<Exception> ErrorChanged;

-      public delegate void ChangeDelegate(object newValue);
-      public event ChangeDelegate onChange
+      public event DataStreamChangeDelegate onChange
      {
         add 
         {
-            ChangeDelegate currValue = value;
+            DataStreamChangeDelegate currValue = value;
            if (myTestMode)
            {
               ChangeDelegateWrapper w = new ChangeDelegateWrapper (value);
-               currValue = new ChangeDelegate (w.wrapperFunction);
+               currValue = new DataStreamChangeDelegate (w.wrapperFunction);
               myDelMap.Add (value, currValue);
            }
            if (myRealSource != null)
@@ -70,7 +69,7 @@ namespace BCG
            else 
            {
               myOnChangeDelegate = 
-                  (ChangeDelegate) Delegate.Combine (myOnChangeDelegate,currValue);
+                  (DataStreamChangeDelegate) Delegate.Combine (myOnChangeDelegate,currValue);

               
               if (myOnChangeDelegate.GetInvocationList().Length == 1) // first subscriber 
@@ -95,11 +94,11 @@ namespace BCG
         }
         remove 
         {
-            ChangeDelegate currValue = value;
+            var currValue = value;
            // BlueTrace.WriteViewStreamUnsubscribe(this.ID);
            if (myTestMode)
            {
-               currValue = myDelMap[value] as ChangeDelegate;
+               currValue = myDelMap[value] as DataStreamChangeDelegate;
               myDelMap.Remove (value);
            }
            if (myRealSource != null)
@@ -107,7 +106,7 @@ namespace BCG
            else 
            {
               myOnChangeDelegate = 
-                  (ChangeDelegate) Delegate.Remove (myOnChangeDelegate,currValue);
+                  (DataStreamChangeDelegate) Delegate.Remove (myOnChangeDelegate,currValue);

               // this is need to avoid a callback in a reentrant add.
               if (myPendingAdds.Count > 0)
@@ -175,7 +174,7 @@ namespace BCG

      // this was made internal for testing purposes only.
      // don't mess with it.
-      internal ChangeDelegate myOnChangeDelegate;
+      internal DataStreamChangeDelegate myOnChangeDelegate;
      protected object myLastValue;
      protected DataStream myRealSource;
      private Stack myPendingAdds;
@@ -184,9 +183,9 @@ namespace BCG
   internal class ChangeDelegateWrapper 
   {
      readonly Stopwatch timer = new();
-      private DataStream.ChangeDelegate myDel;
+      private DataStreamChangeDelegate myDel;
      
-      internal ChangeDelegateWrapper (DataStream.ChangeDelegate del) {myDel = del;}
+      internal ChangeDelegateWrapper (DataStreamChangeDelegate del) {myDel = del;}

      internal void wrapperFunction (object o)
      {
@@ -0,0 +1,4 @@
+namespace BCG;
+
+public delegate void DataStreamChangeDelegate(object newValue);
+public delegate void DataStreamChangeDelegate<T>(IDataPacket<T> dp) where T : IBlueRow;
@@ -9,6 +9,9 @@
  <ItemGroup>
    <PackageReference Include="Google.Protobuf" Version="3.24.3" />
    <PackageReference Include="Microsoft.Extensions.Hosting" Version="7.0.1" />
+    <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
+    <PackageReference Include="OpenAI" Version="2.0.0" />
+    <PackageReference Include="SimMetrics.Net" Version="1.0.5" />
  </ItemGroup>

  <ItemGroup>
@@ -17,8 +20,8 @@
  </ItemGroup>

  <ItemGroup>
-    <Proto Include="../../lib/config/protos/data_stream_config.proto" CSName="DataStreamConfig">
-      <OutDir>$(ProtoOutputPath)/lib/config/protos</OutDir>
+    <Proto Include="../../lib_spark/spark/config/protos/data_stream_config.proto" CSName="DataStreamConfig">
+      <OutDir>$(ProtoOutputPath)/spark/config/protos</OutDir>
    </Proto>
  </ItemGroup>

@@ -8,11 +8,12 @@ namespace BCG {
   using System.Collections;
   using System.Reflection;
   using System.IO;
+   using System.Collections.Generic;

   public class PrimitiveManager {
-      private IDictionary myTypeLookupTable = new Hashtable();
      private static PrimitiveManager mySelf;
-      private ArrayList myAssemblyNames = new ArrayList();
+      readonly List<string> myAssemblyNames = new();
+      readonly Dictionary<string, Type> myTypeLookupTable = new();

      private PrimitiveManager() {
         myLoadAllTypes();
@@ -95,15 +96,14 @@ namespace BCG {
      }

      /// <summary>
-      /// Returns the Type name corresponding to the primitive name
+      /// Returns the Type corresponding to the primitive name
      /// </summary>
      /// <param name="friendlyName"></param>
      /// <returns></returns>
-      public string find(string friendlyName) {
-         if (myTypeLookupTable[friendlyName] == null)
-            throw new ApplicationException(friendlyName + " is not the friendly name of any of the loaded types");
-         return (myTypeLookupTable[friendlyName] as Type).Name;
-      }
+      public Type find(string friendlyName) =>
+         myTypeLookupTable.TryGetValue(friendlyName, out var t)
+            ? t
+            : throw new ArgumentException($"'{friendlyName}' is not the friendly name of any of the loaded types", nameof(friendlyName));

      /// <summary>
      /// Constructs and returns a ViewStream of type corresponding to the friendly name
@@ -118,12 +118,8 @@ namespace BCG {
            constructorArgs = new object[args.Count];
            args.CopyTo(constructorArgs, 0);
         }
-         if (myTypeLookupTable[friendlyName] == null)
-            throw new ApplicationException(friendlyName + " is not the friendly name of any of the loaded types");
-         return
-            constructorArgs == null ?
-            Activator.CreateInstance(myTypeLookupTable[friendlyName] as Type) as ViewStream :
-            Activator.CreateInstance(myTypeLookupTable[friendlyName] as Type, constructorArgs) as ViewStream;
+         var type = find(friendlyName);
+         return Activator.CreateInstance(type, constructorArgs) as ViewStream;
      }

      /// <summary>
@@ -12,4 +12,16 @@ namespace BCG
      //this is an ICollection of IDictionary
      void get_new_added_deleted(out ICollection new_rows, out ICollection added_rows, out ICollection deleted_rows);
   }
+
+   public interface IDataPacket<T> : IDataPacket where T : IBlueRow {
+      IBlueRowCollection<T> newRows { get; }
+      IBlueRowCollection<T> addedRows { get; }
+      IBlueRowCollection<T> deletedRows { get; }
+
+      void IDataPacket.get_new_added_deleted(out ICollection new_rows, out ICollection added_rows, out ICollection deleted_rows) {
+         new_rows = newRows;
+         added_rows = addedRows;
+         deleted_rows = deletedRows;
+      }
+   }
 }
@@ -0,0 +1,13 @@
+namespace BCG;
+
+public interface IViewStream {
+   event DataStreamChangeDelegate onChange;
+
+   IBlueRowCollection rows { get; }
+}
+
+public interface IViewStream<T> : IViewStream where T : IBlueRow {
+   new event DataStreamChangeDelegate<T> onChange;
+
+   new IBlueRowCollection<T> rows { get; }
+}
@@ -5,7 +5,7 @@ namespace BCG
    /// <summary>
    /// Marker abstract class
    /// </summary>
-   public abstract class ViewStream : DataStream
+   public abstract class ViewStream : DataStream, IViewStream
   {
      private IBlueRowCollection myLastValueForTestMode;
      // protected bool myUseUncopiedNewRowsInFireEvent =
@@ -45,7 +45,7 @@ namespace BCG
            {
               // used to force a computation of a stream which 
               // happens to have no subscribers
-               onChange += new DataStream.ChangeDelegate(myDummy);
+               onChange += new DataStreamChangeDelegate(myDummy);
               try 
               {

@@ -56,7 +56,7 @@ namespace BCG
               }
               finally 
               {
-                  onChange -= new DataStream.ChangeDelegate(myDummy);
+                  onChange -= new DataStreamChangeDelegate(myDummy);
               }
            }
         }
@@ -9,7 +9,15 @@
 Letter                   [a-zA-Z]
 BooleanConstant          'true'|'false'
 CSStringConstant         \"([^\"\\]*|\\.)*\"
-DateTimeConstant         ([1-9]|'10'|'11'|'12')\/([1-9]|([1-2][0-9])|'30'|'31')\/[0-9][0-9][0-9]*
+YearInt                  [0-9]{4}
+MonthInt                 [0-9]{1,2}
+DayInt                   [0-9]{1,2}
+/**
+ * DateTimeConstant supports American (mm/dd/yyyy) and European (yyyy/mm/dd)
+ * formats. Month and day can be 1 or 2 digits. It does not validate that the
+ * numbers represent an actual date.
+ */
+DateTimeConstant         ({MonthInt}\/{DayInt}\/{YearInt}|{YearInt}\/{MonthInt}\/{DayInt})
 IntegerConstant          -?[0-9]+
 NullConstant             'null'
 RealConstant             -?[0-9]+\.[0-9]+
@@ -20,7 +20,7 @@ public class PrimitiveExpression : StatementExpression {
   }

   public override string print(int startCol, IList<string> activeBindings) {
-      string prelude = "new " + PrimitiveManager.instance.find(myName) + " (";
+      string prelude = "new " + PrimitiveManager.instance.find(myName).Name + " (";

      return prelude + myPrintArgs(startCol + prelude.Length, activeBindings) + ")";
   }
@@ -1,9 +1,10 @@
 FROM bluetradingsystems/linux_dotnet7

-ARG SOURCE_DIR="build/Release/bin/net7.0/linux-x64/publish"
+ARG CS_SOURCE_DIR="build/Release/bin/net7.0/linux-x64/publish"
+ARG CPP_SOURCE_DIR="install/data-stream-suite-linux"
 ARG CONF_FILE="src/DataStreamSuite/DataStreamPublisher/spark.conf"

-ENV DOCKERIZE_VERSION v0.7.0
+ENV DOCKERIZE_VERSION=v0.7.0
 USER root
 RUN apt-get update \
    && apt-get install -y wget \
@@ -11,6 +12,7 @@ RUN apt-get update \
    && apt-get autoremove -yqq --purge wget && rm -rf /var/lib/apt/lists/*

 USER bts
-COPY --chown=bts:bts "${SOURCE_DIR}" "bin"
+COPY --chown=bts:bts "${CPP_SOURCE_DIR}/lib" "bin"
+COPY --chown=bts:bts "${CS_SOURCE_DIR}" "bin"
 COPY --chown=bts:bts "${CONF_FILE}" ".local/share/bts/DataStreamPublisher/conf/spark.conf"
 WORKDIR "/home/bts/bin"
@@ -1,4 +1,13 @@
 {
+  "Spark": {
+    "ConfigServer": "http://localhost:8000/config"
+  },
+  "SparkApi": {
+    "Credentials": {
+      "User": "bts",
+      "Password": ""
+    }
+  },
  "PubSub": {
    "Routing": {
      "PublisherRoutes": [
@@ -29,7 +38,7 @@
          },
          {
            "Name": "test_literal",
-            "Expression": "literal(\"x\", \"420.0\")",
+            "Expression": "literal(\"x\", \"x:420.0\")",
            "IsOnWebsocket": true
          }
        ]
@@ -48,12 +48,12 @@
          },
          {
            "Name": "test_literal",
-            "Expression": "literal(\"x\", \"420.0\")",
+            "Expression": "literal(\"x\", \"x:420.0\")",
            "IsOnWebsocket": true
          },
          {
            "Name": "test_not_on_websocket",
-            "Expression": "literal(\"x\", \"840.0\")",
+            "Expression": "literal(\"x\", \"x:840.0\")",
            "IsOnWebsocket": false
          }
        ]
@@ -73,7 +73,7 @@
        "Streams": [
          {
            "Name": "test_literal",
-            "Expression": "literal(\"x\", \"420.0\")",
+            "Expression": "literal(\"x\", \"x:420.0\")",
            "IsOnWebsocket": true
          }
        ]
@@ -169,8 +169,10 @@ namespace BCG
      }
      #endregion

+      private Stopwatch _stopwatch = new Stopwatch();
      private void myTryExpression(string expStr)
      {
+         _stopwatch.Restart();
         myFreezeView(false); // force unfreeze the view
         DataStreamExpression oldExp = myExp;
         try
@@ -344,6 +346,7 @@ namespace BCG
      // this is the stream callback
      private void myCallback(object o)
      {
+         _stopwatch.Stop();
         var timer = Stopwatch.StartNew();
         DataPacket dp = o as DataPacket;
         myCrono.Stop();
@@ -359,7 +362,8 @@ namespace BCG
         myToolStripStatusLabelRows.BackColor = Color.Black;
         myToolStripStatusLabelTime.ForeColor = Color.White;
         myToolStripStatusLabelRows.ForeColor = Color.White;
-         myToolStripStatusLabelTime.Text = "Last update: " + DateTime.Now.ToString("hh:mm:ss:fff tt");
+         string time_text = "Last update: " + DateTime.Now.ToString("hh:mm:ss:fff tt") + $" time: {_stopwatch.ElapsedMilliseconds} msecs\t\t";
+         myToolStripStatusLabelTime.Text = time_text;
         myToolStripStatusLabelRows.Text = "New = " + dp.newRows.Count + "  Added = " + dp.addedRows.Count + "  Deleted = " + dp.deletedRows.Count;
         Log.Ui.diag(Text);
         int hscrollval = myChangeStyle ? 0 : myDataGrid.HorizScrollBarValue;
@@ -1,8 +1,10 @@
 FROM bluetradingsystems/linux_dotnet7

-ARG SOURCE_DIR="build/Release/bin/net7.0/linux-x64/publish"
+ARG CS_SOURCE_DIR="build/Release/bin/net7.0/linux-x64/publish"
+ARG CPP_SOURCE_DIR="install/data-stream-suite-linux"
 ARG CONF_FILE="src/DataStreamSuite/PubSubServer/spark.conf"

-COPY --chown=bts:bts "${SOURCE_DIR}" "bin"
+COPY --chown=bts:bts "${CPP_SOURCE_DIR}/lib" "bin"
+COPY --chown=bts:bts "${CS_SOURCE_DIR}" "bin"
 COPY --chown=bts:bts "${CONF_FILE}" ".local/share/bts/PubSubServer/conf/spark.conf"
 WORKDIR "/home/bts/bin"
@@ -0,0 +1,182 @@
+using bts.SilverPine;
+
+namespace bts.SilverPine {
+   using bts.api;
+   using bts.spark;
+
+   using System;
+
+   using ZstdSharp.Unsafe;
+
+   public static class ContractUtils {
+      #region Parsing
+      public static FutureContract ParseFutureContract(string contract_txt, Exchange exchange = Exchange.CME) {
+         // Validate the future string length
+         if (!contract_txt.All(c => Char.IsLetterOrDigit(c)))
+            return null;
+         if (contract_txt.Length < 4) {
+            return null;
+         }
+
+         // Extract the symbol and the expiration month/year codes
+         string symbol = contract_txt.Substring(0, contract_txt.Length - 2);
+         char monthCode = contract_txt[contract_txt.Length - 2];
+         char yearCode = contract_txt[contract_txt.Length - 1];
+
+         // Map month code to month, return null if the month code is invalid
+         int month = monthCode switch {
+            'F' => 1,  // January
+            'G' => 2,  // February
+            'H' => 3,  // March
+            'J' => 4,  // April
+            'K' => 5,  // May
+            'M' => 6,  // June
+            'N' => 7,  // July
+            'Q' => 8,  // August
+            'U' => 9,  // September
+            'V' => 10, // October
+            'X' => 11, // November
+            'Z' => 12, // December
+            _ => 0 // Return 0 for invalid month code
+         };
+
+         if (month == 0) {
+            return null; // Invalid month code
+         }
+
+         // Calculate the year based on the current decade
+         int currentYear = DateTime.Now.Year;
+         int decadeStart = currentYear / 10 * 10;
+         int year = decadeStart + (yearCode - '0');
+
+         // If the year is in the past, assume the next decade
+         if (year < currentYear) {
+            year += 10;
+         }
+
+         // Create the FutureContract instance and set its properties
+         DateTime expiration = new DateTime(year, month, 1);
+         var futureContract = new FutureContract {
+            symbol = symbol,
+            expiration = expiration,
+            exchange = exchange
+         };
+
+         return futureContract;
+      }
+      // Method to parse a futures spread, e.g. "CLX4-CLZ4"
+      public static FutureSpreadContract ParseFutureSpreadContract(string spreadString, Exchange exchange = Exchange.CME) {
+         // Split the string by '-' to handle each leg
+         var spreadParts = spreadString.Split('-');
+         if (spreadParts.Length < 2) {
+            return null; // Not a valid spread format
+         }
+
+         var spreadContract = new FutureSpreadContract { legs = new List<Tuple<double, FutureContract>>() };
+
+         foreach (var part in spreadParts) {
+            // Split each leg by '*', assuming the format is weight*future or just future
+            string[] legParts = part.Split('*');
+            double weight = 1.0;
+            string futurePart;
+
+            if (legParts.Length == 2) {
+               if (!double.TryParse(legParts[0], out weight)) {
+                  return null; // Invalid weight, parsing failed
+               }
+               futurePart = legParts[1];
+            } else if (legParts.Length == 1) {
+               futurePart = legParts[0];
+            } else {
+               return null; // Invalid format, too many '*' parts
+            }
+
+            // Parse the future contract
+            var futureContract = ParseFutureContract(futurePart, exchange);
+            if (futureContract == null) {
+               return null; // Failed to parse the future part
+            }
+
+            // Add the leg to the spread contract
+            spreadContract.legs.Add(new Tuple<double, FutureContract>(weight, futureContract));
+         }
+
+         return spreadContract;
+      }
+
+      public static ContractType DetermineFutureOrSpread(string contract) {
+         if (contract.Contains("-")) {
+            return ContractType.FutureSpread;
+         } else {
+            return ContractType.Future;
+         }
+      }
+      #endregion
+
+      public static ContractType get_type(string contract_txt) {
+         FutureContract fc = ParseFutureContract(contract_txt);
+         if (fc != null) {
+            return ContractType.Future;
+         }
+         FutureSpreadContract sc = ParseFutureSpreadContract(contract_txt);
+         if (sc != null) {
+            return ContractType.FutureSpread;
+         }
+         // tehre are some non calendar spreads we cant parse
+         //Log.assert(false);
+         return bts.SilverPine.ContractUtils.ContractType.FutureSpread;
+      }
+
+      public static ContractCategory get_category(string symbol) {
+         var tp = get_symbol_category(symbol);
+         return tp.Item1;
+      }
+
+      public static Tuple<ContractCategory, string> get_symbol_category(string symbol) {
+         ContractUtils.FutureContract fc = ParseFutureContract(symbol);
+         if (fc != null) {
+            return Tuple.Create(ContractCategory.FutureString, fc.symbol);
+         }
+         var sc = ParseFutureSpreadContract(symbol);
+         if (sc != null) {
+            return Tuple.Create(ContractCategory.FutureString, sc.symbol);
+         }
+         return Tuple.Create(ContractCategory.Symbol, symbol);
+      }
+
+
+
+      public enum ContractType {
+         Spot,
+         Option,
+         Future,
+         FutureSpread,
+         OptionSpread
+      }
+      public enum ContractCategory {
+         Symbol = 0,
+         FutureString,
+         SpreadString
+      }
+      public enum CallPut {
+         Call,
+         Put
+      }
+      public enum Exchange {
+         CME
+      }
+      public class FutureContract {
+         public string symbol;
+         public DateTime expiration;
+         public Exchange exchange;
+      }
+      public class FutureSpreadContract {
+         public List<Tuple<double, bts.SilverPine.ContractUtils.FutureContract>> legs;
+         public string symbol {
+            get {
+               return legs == null || legs.Count == 0 ? String.Empty : legs.ElementAt(0).Item2.symbol;
+            }
+         }
+      }
+   }
+}
@@ -0,0 +1,23 @@
+namespace bts.SilverPine {
+   using BCG;
+
+   // Finds the regions around specified events 
+   [DataStreamUserRespresentation("EventsPriceAction")]
+   public class EventsPriceActionStream : ViewStream {
+      public EventsPriceActionStream(string symbol, string event_type, string pre_event_period,
+            string post_event_period, int n_events) :
+         this(symbol, event_type, TimeBasedFilter.parse_duration(pre_event_period), TimeBasedFilter.parse_duration(post_event_period),
+         n_events, ViewStreamDatabase.instance) { }
+
+      public EventsPriceActionStream(string symbol, string event_type, TimeSpan pre_event_period,
+      TimeSpan post_event_period, int n_events, ViewStreamDatabase vsd) {
+         // TODO: figure out which bars to use, 1 min or 1 day or live, based on pre_event_period / post_event_period
+         FuturesActiveContractStream olhcv_str = new FuturesActiveContractStream(symbol);
+         ViewStream events_str = new EventsStream(vsd, event_type);
+         // todo
+         //events_str = new TopStream(events_str, n_events, "-ts");
+         ViewStream rs = new TimeProximityJoinStream(olhcv_str, "ts_event", events_str, "ts", pre_event_period, post_event_period);
+         mySetRealSource(rs);
+      }
+   }
+}
@@ -0,0 +1,20 @@
+namespace bts.SilverPine {
+   using BCG;
+
+   public class IndicatorStream : ViewStream {
+      enum Indicator {
+         StandardDeviation = 0
+      }
+      public class IndicatorInfo {
+         Indicator indicator;
+         IDictionary<string, object> parameters;
+      }
+      public IndicatorStream(string symbol_name, string indicator_name, string indicator_params_json) {
+
+      }
+      public IndicatorStream(string symbol_name, IndicatorInfo info) {
+
+      }
+
+   }
+}
@@ -0,0 +1,57 @@
+namespace bts.SilverPine {
+   using BCG;
+
+   using ZstdSharp.Unsafe;
+
+   [DataStreamUserRespresentation("AbsVolume")]
+   // Calculates the volume within a time window (this does not do rates, like highest daily volume etc,
+   // but with some thought can perhaps be extended)
+   public class AbsoluteVolumeStream : ViewStream {
+      // TODO, abstract this out
+      // adds contract_type to OHLCV stream
+      private static ViewStream _get_ohlcv_steam(string symbol, DateTime d1, DateTime d2) {
+         var ohlcv_params = OHLCVStream.get_params(d1, d2);
+         ViewStream rs = new OHLCVStream(symbol, ohlcv_params);
+         Func<IBlueTableSchema, IBlueTableSchema> create_schema =
+         s => BlueDataUtility.add_columns(s, Tuple.Create(typeof(string), "contract_type"));
+         rs = new SuperMapStream(rs,
+         s => create_schema(s),
+         r => {
+            IBlueTableSchema out_schema = create_schema(r.schema);
+            string contract_txt = r.get<string>("contract");
+            var contr_type = ContractUtils.get_type(contract_txt);
+            MutableBlueRow out_row = new MutableBlueRow(out_schema, r);
+            out_row.setValue("contract_type", contr_type.ToString());
+            return new IBlueRow[] { out_row };
+         });
+         return rs;
+      }
+
+      public AbsoluteVolumeStream(string symbol, string ctype, DateTime d1)
+      : this(symbol, ctype, d1, d1) { }
+
+      public AbsoluteVolumeStream(string symbol, string ctype, DateTime d1, DateTime d2)
+      : this(symbol, Enum.Parse<ContractUtils.ContractType>(ctype), d1, d2) {
+      }
+
+      static void swap_if_necessary(ref DateTime dt1, ref DateTime dt2) {
+         if (dt1 < dt2) {
+            DateTime aux = dt1;
+            dt1 = dt2;
+            dt2 = aux;
+         }
+      }
+
+      private AbsoluteVolumeStream(string symbol, ContractUtils.ContractType ctype, DateTime d1, DateTime d2) {
+         swap_if_necessary(ref d1, ref d2);
+         ViewStream ohlcv_str = _get_ohlcv_steam(symbol, d1, d2);
+         ohlcv_str = new FilterStream(ohlcv_str,
+            r => r.get_datetime("ts_event") <= d1 && r.get_datetime("ts_event") >= d2);
+         ohlcv_str = new FilterStream(ohlcv_str, r => String.Compare(ctype.ToString(), r.get<string>("contract_type"), true) == 0);
+         ViewStream rs = new MaxStream(ohlcv_str, "volume");
+         rs = new NewProjectStream(rs, "volume:total_volume contract !");
+         mySetRealSource(rs);
+      }
+
+   }
+}
@@ -0,0 +1,152 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+
+using bts;
+
+namespace bts.SilverPine {
+   public static class IndicatorCalc {
+      public static double calculate_pearson_correlation(List<double> prices1, List<double> prices2) {
+         if (prices1 == null)
+            throw new ArgumentNullException(nameof(prices1), "First price list cannot be null.");
+         if (prices2 == null)
+            throw new ArgumentNullException(nameof(prices2), "Second price list cannot be null.");
+         if (prices1.Count != prices2.Count)
+            throw new ArgumentException("Price lists must have the same number of elements.");
+         if (prices1.Count < 2)
+            throw new ArgumentException("At least two data points are required to calculate correlation.");
+
+         int n = prices1.Count;
+         double sumX = 0;
+         double sumY = 0;
+         double sumX2 = 0;
+         double sumY2 = 0;
+         double sumXY = 0;
+
+         for (int i = 0; i < n; i++) {
+            double x = prices1[i];
+            double y = prices2[i];
+
+            sumX += x;
+            sumY += y;
+            sumX2 += x * x;
+            sumY2 += y * y;
+            sumXY += x * y;
+         }
+
+         double numerator = (n * sumXY) - (sumX * sumY);
+         double denominatorPart1 = (n * sumX2) - (sumX * sumX);
+         double denominatorPart2 = (n * sumY2) - (sumY * sumY);
+         double denominator = Math.Sqrt(denominatorPart1 * denominatorPart2);
+
+         if (denominator == 0)
+            throw new InvalidOperationException("Denominator is zero, correlation is undefined.");
+
+         return numerator / denominator;
+      }
+      public static double calculate_VWAP(List<Tuple<int, double>> volumePriceData) {
+         Log.Gen.info($"ChartingHelper.calculate_VWAP n_vals: {volumePriceData.Count}");
+         double totalVolume = 0;
+         double totalPriceVolume = 0;
+
+         foreach (var data in volumePriceData) {
+            int volume = data.Item1; // Volume
+            double price = data.Item2; // Closing price
+
+            // Accumulate total price-volume (price * volume) and total volume
+            totalPriceVolume += price * volume;
+            totalVolume += volume;
+         }
+
+         // VWAP is totalPriceVolume divided by totalVolume
+         return totalVolume > 0 ? totalPriceVolume / totalVolume : 0;
+      }
+
+      // Calculate Volatility (Standard Deviation of returns) over a given period
+      public static double calculate_volatility(IList<double> prices, int period = 252) {
+         if (prices == null || prices.Count < period + 1) {
+            throw new ArgumentException("Insufficient data to calculate volatility.");
+         }
+
+         // Calculate daily returns
+         var returns = new List<double>();
+         for (int i = 1; i < prices.Count; i++) {
+            double dailyReturn = (prices[i] - prices[i - 1]) / prices[i - 1];
+            returns.Add(dailyReturn);
+         }
+
+         // Get the returns over the desired period
+         var recentReturns = returns.Skip(returns.Count - period).Take(period).ToList();
+
+         // Calculate the mean of returns
+         double meanReturn = recentReturns.Average();
+
+         // Calculate the variance (sum of squared differences from the mean)
+         double variance = recentReturns.Sum(r => Math.Pow(r - meanReturn, 2)) / period;
+
+         // Volatility is the square root of variance (standard deviation)
+         double volatility = Math.Sqrt(variance);
+
+         // Annualize volatility by multiplying by the square root of 252 (number of trading days in a year)
+         return volatility * Math.Sqrt(252);
+      }
+
+      // Calculate SMA (Simple Moving Average) for a given period (e.g., 50, 100, or 200)
+      public static double calculate_SMA(IList<double> prices, int period = 50) {
+         if (prices == null || prices.Count < period) {
+            throw new ArgumentException("Insufficient data to calculate SMA.");
+         }
+
+         // Sum up the prices over the given period
+         double sum = prices.Skip(prices.Count - period).Take(period).Sum();
+
+         // Return the average
+         return sum / period;
+      }
+
+      public static double calculate_RSI(IList<double> prices, int period = 14) {
+         if (prices == null || prices.Count < period + 1) {
+            return Double.NaN;
+            //throw new ArgumentException("Insufficient data to calculate RSI.");
+         }
+
+         double gainSum = 0;
+         double lossSum = 0;
+
+         // Calculate the initial average gain and loss over the first 'period' days
+         for (int i = 1; i <= period; i++) {
+            double change = prices[i] - prices[i - 1];
+            if (change > 0) {
+               gainSum += change;
+            } else {
+               lossSum -= change; // Subtract since loss is negative
+            }
+         }
+
+         double avgGain = gainSum / period;
+         double avgLoss = lossSum / period;
+
+         // Calculate RSI using smoothed values
+         for (int i = period + 1; i < prices.Count; i++) {
+            double change = prices[i] - prices[i - 1];
+            if (change > 0) {
+               avgGain = (avgGain * (period - 1) + change) / period;
+               avgLoss = (avgLoss * (period - 1)) / period;
+            } else {
+               avgGain = (avgGain * (period - 1)) / period;
+               avgLoss = (avgLoss * (period - 1) - change) / period;
+            }
+         }
+
+         // Prevent division by zero in case of no losses
+         if (avgLoss == 0) {
+            return 100;
+         }
+
+         double rs = avgGain / avgLoss;
+         double rsi = 100 - (100 / (1 + rs));
+
+         return rsi;
+      }
+   }
+}
@@ -0,0 +1,31 @@
+namespace bts.SilverPine {
+   using BCG;
+
+   using bts;
+
+   [DataStreamUserRespresentation("DailyChange")]
+   // takes in the ohlcv stream and computes the closing vs closing difference
+   public class DailyChange : ViewStream {
+      public DailyChange(ViewStream ohlc_str) {
+         GlobalMapStream rs = new GlobalMapStream(ohlc_str, "ts_event", "contract",
+         (rows, idx) => {
+            double daily_change = Double.NaN;
+            double daily_change_perc = Double.NaN;
+            IBlueRow ir = rows.ElementAt((int)idx);
+            if (idx < rows.Count - 1) {
+               IBlueRow pr = rows.ElementAt((int)idx + 1);
+               double close = ir.get<double>("close"), prev_close = pr.get<double>("close");
+               daily_change = close - prev_close;
+               daily_change_perc = 100.0 * daily_change / prev_close;
+            }
+            IBlueTableSchema schema = ir.schema.add_columns(Tuple.Create(typeof(double), "daily_change"));
+            schema = schema.add_columns(Tuple.Create(typeof(double), "daily_change_perc"));
+            MutableBlueRow nr = new MutableBlueRow(schema, ir);
+            nr.setValue("daily_change", daily_change);
+            nr.setValue("daily_change_perc", daily_change_perc);
+            return nr;
+         });
+         mySetRealSource(rs);
+      }
+   }
+}
@@ -1,54 +0,0 @@
-
-using Google.Protobuf.WellKnownTypes;
-
-using System;
-using BCG.Utilities.General;
-using System.Data;
-using bts.api.domain;
-
-namespace BCG {
-    // Stream computes the EMA of a column in any stream. It takes in the base stream, the column name and a smoothing factor
-    // as args, and create one row with an ema column, which is the running ema of the column provided 
-    [DataStreamUserRespresentation("ema")]
-    public class EMAStream : ViewStream {
-        private ViewStream _stream;
-        private readonly string _col_name;
-        private EMACalculator _calc;
-        private readonly IBlueTableSchema _schema = BlueDataUtility.form_schema(
-            "SMA", Tuple.Create("id", TypeCode.Int32),
-            Tuple.Create("ema", TypeCode.Double));
-
-        public EMAStream(ViewStream stream, string col_name, double smoothing_factor) {
-            _col_name = col_name;
-            _stream = stream;
-            _calc = new EMACalculator(smoothing_factor);
-        }
-
-        protected override void myOnFirstSubscriber() {
-            _stream.onChange += new ChangeDelegate(_onstream_change);
-        }
-        protected override void myOnLastSubscriber() {
-            _stream.onChange -= new ChangeDelegate(_onstream_change);
-        }
-
-
-        private void _onstream_change(object o) {
-
-            DataPacket dp = o as DataPacket;
-            foreach (IBlueRow r in dp.addedRows) {
-                double price = r.get<double>(_col_name);
-                if (!Double.IsNaN(price))
-                    _calc.add(price);
-            }
-            IBlueRow nr = new BlueRow(_schema, 0, _calc.get_EMA());
-            BlueTable state = new BlueTable(_schema);
-            state.addRow(nr);
-            DataPacket dp_out = BlueDataUtility.build_dp_inc(state, myLastValue as DataPacket);
-            myFireEvent(dp_out);
-        }
-    }
-
-}
-
-
-