Add PUT method to http client

Where it belongs

.config/dotnet-tools.json

@@ -25,6 +25,12 @@
       "commands": [
         "fake"
       ]
+    },
+    "dotnet-t4": {
+      "version": "3.0.0",
+      "commands": [
+        "t4"
+      ]
     }
   }
-}
+}

.env

@@ -0,0 +1,81 @@
+# This file defines the global default configuration for docker deployments.
+# It is tracked in source control. For temporary modifications, please create
+# a separate file called ".env.local" (it's in .gitignore, so won't be tracked).
+# If a variable is defined in both files, the value in .env.local will be used.
+#
+# About the deployment procedure:
+#   We use Docker Compose to build images and launch containers for various services
+#   that make up our software suite:
+#     - sparkd
+#     - pubsub
+#     - md-publisher
+#     - webapp (DSQA on the web)
+#     - api-server (websocket based)
+#   From inside a devcontainer, you can launch all services in separate containers
+#   on your machine by running:
+#
+#     tools/deploy.sh
+#
+#   This script builds release versions for each service, packaged (generally) in
+#   self-contained directories. Next, it runs a few "docker compose" commands to
+#   build docker images loaded with the self-contained binaries from the previous
+#   step, and start containers with those images to run each service. To view the
+#   commands that deploy.sh runs internally (without actually running them), run:
+#
+#     tools/deploy.sh --dry-run
+#
+#   For additional information/help, run:
+#
+#     tools/deploy.sh --help
+#
+
+### Sparkd ###
+# The internal image for sparkd to use. For more info, view:
+#   src/app_sparkd/Dockerfile
+SPARKD__IMAGE=sparkd
+# The 'spark.conf' for this service to use
+SPARKD__CONF_FILE=src/app_sparkd/spark.conf
+# To run replay md, put recorded md files in SPARKD__MD_RECORDING_DIR, and use:
+#   SPARKD__IMAGE=sparkd_md_replay
+#   SPARKD__CONF_FILE=src/app_sparkd/spark.local.conf
+# (configure which md to replay in the new SPARKD__CONF_FILE)
+SPARKD__MD_RECORDING_DIR=.md_recording
+SPARKD__SOURCE_DIR=install/sparkd-release-gcc
+SPARKD__CONFIG_PORT=8000
+SPARKD__QUICKBUS_PORT=8001
+SPARKD__SIM_FIX_PORT=8002
+
+### Pubsub ###
+# The 'spark.conf' for this service to use
+PUBSUB__CONF_FILE=src/DataStreamSuite/PubSubServer/spark.conf
+PUBSUB__CS_SOURCE_DIR=build/Release/bin/net7.0/linux-x64/publish
+PUBSUB__CPP_SOURCE_DIR=install/data-stream-suite-linux
+# The appsettings to load at runtime. Expects a file named:
+#   src/DataStreamSuite/PubSub/appsettings.${PUBSUB__DOTNET_ENVIRONMENT}.json
+PUBSUB__DOTNET_ENVIRONMENT=Staging
+
+### Md Publisher ###
+# The 'spark.conf' for this service to use
+MD_PUBLISHER__CONF_FILE=src/DataStreamSuite/DataStreamPublisher/spark.conf
+MD_PUBLISHER__SOURCE_DIR=build/Release/bin/net7.0/linux-x64/publish
+# Determines which appsettings to load at runtime. Expects a file named:
+#   src/DataStreamSuite/DataStreamPublisher/appsettings.${MD_PUBLISHER__DOTNET_ENVIRONMENT}.json
+MD_PUBLISHER__DOTNET_ENVIRONMENT=MarketData
+
+### Webapp (DSQA on the web) ###
+# The 'spark.conf' for this service to use
+WEBAPP__CONF_FILE=src/client/spark_api_server/spark.conf
+WEBAPP__SOURCE_DIR=build/Release/bin/net7.0/linux-x64/publish
+WEBAPP__WEB_ROOT=build/Bts.DataStreamWebApp.Client/production/dist
+# Determines which appsettings to load at runtime. Expects a file named:
+#   src/DataStreamSuite/Bts.DataStreamWebApp.Server/appsettings.${WEBAPP__ASPNETCORE_ENVIRONMENT}.json
+WEBAPP__ASPNETCORE_ENVIRONMENT=Staging
+
+### Api Server ###
+# The 'spark.conf' for this service to use
+API_SERVER__CONF_FILE=src/client/spark_api_server/spark.conf
+API_SERVER__CS_SOURCE_DIR=build/Release/bin/net7.0/linux-x64/publish
+API_SERVER__CPP_SOURCE_DIR=install/spark-api-server-linux
+# Determines which appsettings to load at runtime. Expects a file named:
+#   src/client/spark_api_server/appsettings.${API_SERVER__ASPNETCORE_ENVIRONMENT}.json
+API_SERVER__ASPNETCORE_ENVIRONMENT=Staging

.gitignore

@@ -59,3 +59,6 @@ trace_data/
 .devcontainer/
 *.Generated.cs
 *.lst
+src/app_sparkd/spark.local.conf
+.env.local
+.md_recording

.vscode/launch.json

@@ -16,11 +16,20 @@
         "program": "spark_api_server.exe",
         "args": [
           "run",
-          "--sparkd",
-          "http://127.0.0.1:8000/config"
         ],
         "cwd": "build/Debug/bin/net7.0",
       },
+      "linux": {
+        "preLaunchTask": "Build Api Server (Debug)",
+        "program": "spark_api_server",
+        "args": [
+          "run",
+        ],
+        "cwd": "build/Debug/bin/net7.0",
+      },
+      "env": {
+        "ASPNETCORE_ENVIRONMENT": "Development",
+      },
       "stopAtEntry": false,
       "console": "integratedTerminal"
     },
@@ -33,11 +42,19 @@
         "program": "spark_api_server.exe",
         "args": [
           "run",
-          "--sparkd",
-          "http://127.0.0.1:8000/config"
         ],
         "cwd": "build/Debug/bin/net7.0",
       },
+      "linux": {
+        "program": "spark_api_server",
+        "args": [
+          "run",
+        ],
+        "cwd": "build/Debug/bin/net7.0",
+      },
+      "env": {
+        "ASPNETCORE_ENVIRONMENT": "Development",
+      },
       "stopAtEntry": false,
       "console": "integratedTerminal"
     },
@@ -45,7 +62,7 @@
       "name": "Bts.DataStreamWebApp.Server",
       "type": "coreclr",
       "request": "launch",
-      "preLaunchTask": "Build DataStream WebApp (Debug)",
+      "preLaunchTask": "Build DataStream (Debug)",
       "program": "${workspaceFolder}/build/Debug/bin/net7.0/Bts.DataStreamWebApp.Server.dll",
       "cwd": "${workspaceFolder}/build/Debug/bin/net7.0",
       "env": {
@@ -105,6 +122,18 @@
       "stopAtEntry": false,
       "console": "integratedTerminal"
     },
+    {
+      "name": "(gdb) Launch",
+      "type": "cppdbg",
+      "request": "launch",
+      "program": "/home/bts/spark/build/sparkd-debug-clang/server/sparkd",
+      "args": ["run"],
+      "stopAtEntry": false,
+      "cwd": "/home/bts/spark/build/sparkd-debug-clang/server",
+      "environment": [],
+      "externalConsole": false,
+      "MIMode": "gdb",
+    },
     {
       "name": "DataStreamQueryAnalyzer",
       "type": "coreclr",
@@ -253,6 +282,34 @@
         }
       ]
     },
+    {
+      "name": "Sparkd Web (Skip Build)",
+      "type": "cppdbg",
+      "request": "launch",
+      "program": "",
+      "linux": {
+        "program": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd/sparkd",
+      },
+      "args": [
+        "web"
+      ],
+      "cwd": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd",
+      "stopAtEntry": false,
+      "externalConsole": false,
+      "MIMode": "gdb",
+      "setupCommands": [
+        {
+          "description": "Enable pretty-printing for gdb",
+          "text": "-enable-pretty-printing",
+          "ignoreFailures": true
+        },
+        {
+          "description": "Set Disassembly Flavor to Intel",
+          "text": "-gdb-set disassembly-flavor intel",
+          "ignoreFailures": true
+        }
+      ]
+    },
   ],
   "inputs": [
     {

CMakeLists.txt

@@ -33,92 +33,85 @@ else()
   set(BUILD_ADMIN_WEB_APP ON CACHE BOOL "Build the Admin WebApp")
 endif()
 
-include_directories(BEFORE SYSTEM ${EXTERNALS_INCLUDE} ${EXTERNALS_ROOT}/include/zlib)
-link_directories(${EXTERNALS_LIB})
-
 add_compile_definitions(SHARED_MEMORY_PREFIX="${SHARED_MEMORY_PREFIX}")
 if(DEFINED APP_DATA_DIR)
   add_compile_definitions(APP_DATA_DIR="${APP_DATA_DIR}")
 endif()
 add_compile_definitions(EXPORT_MD_CONSUMER="${EXPORT_MD_CONSUMER}")
-if(WIN32)
-  # subdirs
-  add_subdirectory(src/lib_spark)
-else()
-  if(BUILD_SPARKD)
-    # Note: All macros that begin w/ 'CCAPI' are ccapi's macros (defined and/or
-    # read by their code). We override some below.
-    add_compile_definitions(SPARK_ENABLE_CCAPI)
 
-    # This allows us to override hardcoded exchange-urls with ones read from
-    # config (i.e., resolved at runtime rather than compile-time)
-    add_compile_definitions(CCAPI_EXPOSE_INTERNAL)
+if(BUILD_SPARKD)
+  # Note: All macros that begin w/ 'CCAPI' are ccapi's macros (defined and/or
+  # read by their code). We override some below.
+  add_compile_definitions(SPARK_ENABLE_CCAPI)
 
-    # Enable ccapi services
-    add_compile_definitions(CCAPI_ENABLE_SERVICE_MARKET_DATA)
-    add_compile_definitions(CCAPI_ENABLE_SERVICE_EXECUTION_MANAGEMENT)
-    # Enable ccapi exchanges
-    add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE)
-    add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE_COIN_FUTURES)
-    add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE_US)
-    add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE_USDS_FUTURES)
-    add_compile_definitions(CCAPI_ENABLE_EXCHANGE_COINBASE)
-    add_compile_definitions(CCAPI_ENABLE_EXCHANGE_DERIBIT)
-    add_compile_definitions(CCAPI_ENABLE_EXCHANGE_OKX)
+  # This allows us to override hardcoded exchange-urls with ones read from
+  # config (i.e., resolved at runtime rather than compile-time)
+  add_compile_definitions(CCAPI_EXPOSE_INTERNAL)
 
-    set(CCAPI_LOG_LEVEL
-      "Warn"
-      CACHE STRING "<Fatal|Error|Warn|Info|Debug|Trace> Log level for ccapi")
-    string(TOUPPER "${CCAPI_LOG_LEVEL}" _ccapi_log_level_upper)
-    add_compile_definitions("CCAPI_ENABLE_LOG_${_ccapi_log_level_upper}")
+  # Enable ccapi services
+  add_compile_definitions(CCAPI_ENABLE_SERVICE_MARKET_DATA)
+  add_compile_definitions(CCAPI_ENABLE_SERVICE_EXECUTION_MANAGEMENT)
+  # Enable ccapi exchanges
+  add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE)
+  add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE_COIN_FUTURES)
+  add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE_US)
+  add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE_USDS_FUTURES)
+  add_compile_definitions(CCAPI_ENABLE_EXCHANGE_COINBASE)
+  add_compile_definitions(CCAPI_ENABLE_EXCHANGE_DERIBIT)
+  add_compile_definitions(CCAPI_ENABLE_EXCHANGE_OKX)
 
-    # ** Binance **
-    add_compile_definitions(CCAPI_BINANCE_URL_REST_BASE="https://api.binance.com")
-    add_compile_definitions(BINANCE_INSTRUMENTS_PATH="/api/v3/exchangeInfo")
-    add_compile_definitions(BINANCE_PRICE_STATS_PATH="/api/v3/ticker/24hr")
+  set(CCAPI_LOG_LEVEL
+    "Warn"
+    CACHE STRING "<Fatal|Error|Warn|Info|Debug|Trace> Log level for ccapi")
+  string(TOUPPER "${CCAPI_LOG_LEVEL}" _ccapi_log_level_upper)
+  add_compile_definitions("CCAPI_ENABLE_LOG_${_ccapi_log_level_upper}")
 
-    # ** Binance US **
-    add_compile_definitions(BINANCE_US_INSTRUMENTS_PATH="/api/v3/exchangeInfo")
-    add_compile_definitions(BINANCE_US_PRICE_STATS_PATH="/api/v3/ticker/24hr")
+  # ** Binance **
+  add_compile_definitions(CCAPI_BINANCE_URL_REST_BASE="https://api.binance.com")
+  add_compile_definitions(BINANCE_INSTRUMENTS_PATH="/api/v3/exchangeInfo")
+  add_compile_definitions(BINANCE_PRICE_STATS_PATH="/api/v3/ticker/24hr")
 
-    # ** Binance USDS Futures **
-    add_compile_definitions(CCAPI_BINANCE_USDS_FUTURES_URL_REST_BASE="https://dapi.binance.com")
-    add_compile_definitions(BINANCE_USDS_FUTURES_INSTRUMENTS_PATH="/dapi/v1/exchangeInfo")
-    add_compile_definitions(BINANCE_USDS_FUTURES_PRICE_STATS_PATH="/dapi/v1/ticker/24hr")
+  # ** Binance US **
+  add_compile_definitions(BINANCE_US_INSTRUMENTS_PATH="/api/v3/exchangeInfo")
+  add_compile_definitions(BINANCE_US_PRICE_STATS_PATH="/api/v3/ticker/24hr")
 
-    # ** Deribit **
-    add_compile_definitions(DERIBIT_INSTRUMENT_PATH="/api/v2/public/get_instrument")
-    add_compile_definitions(DERIBIT_INSTRUMENTS_PATH="/api/v2/public/get_instruments")
-    add_compile_definitions(DERIBIT_COMBOS_PATH="/api/v2/public/get_combos")
-    add_compile_definitions(DERIBIT_TICKER_PATH="/api/v2/public/ticker")
-    add_compile_definitions(DERIBIT_CURRENCIES_METHOD="public/get_currencies")
-    add_compile_definitions(DERIBIT_INSTRUMENT_METHOD="public/get_instrument")
-    add_compile_definitions(DERIBIT_INSTRUMENTS_METHOD="public/get_instruments")
-    add_compile_definitions(DERIBIT_COMBO_DETAILS_METHOD="public/get_combo_details")
-    add_compile_definitions(DERIBIT_COMBOS_METHOD="public/get_combos")
+  # ** Binance USDS Futures **
+  add_compile_definitions(CCAPI_BINANCE_USDS_FUTURES_URL_REST_BASE="https://dapi.binance.com")
+  add_compile_definitions(BINANCE_USDS_FUTURES_INSTRUMENTS_PATH="/dapi/v1/exchangeInfo")
+  add_compile_definitions(BINANCE_USDS_FUTURES_PRICE_STATS_PATH="/dapi/v1/ticker/24hr")
 
-    # ** Okx **
-    add_compile_definitions(OKX_INSTRUMENTS_PATH="/api/v5/public/instruments")
-    add_compile_definitions(OKX_UNDERLYING_PATH="/api/v5/public/underlying")
-  endif()
+  # ** Deribit **
+  add_compile_definitions(DERIBIT_INSTRUMENT_PATH="/api/v2/public/get_instrument")
+  add_compile_definitions(DERIBIT_INSTRUMENTS_PATH="/api/v2/public/get_instruments")
+  add_compile_definitions(DERIBIT_COMBOS_PATH="/api/v2/public/get_combos")
+  add_compile_definitions(DERIBIT_TICKER_PATH="/api/v2/public/ticker")
+  add_compile_definitions(DERIBIT_CURRENCIES_METHOD="public/get_currencies")
+  add_compile_definitions(DERIBIT_INSTRUMENT_METHOD="public/get_instrument")
+  add_compile_definitions(DERIBIT_INSTRUMENTS_METHOD="public/get_instruments")
+  add_compile_definitions(DERIBIT_COMBO_DETAILS_METHOD="public/get_combo_details")
+  add_compile_definitions(DERIBIT_COMBOS_METHOD="public/get_combos")
 
-  # subdirs
-  if (BUILD_SPARKD)
-    add_subdirectory(src/lib_system)
-    add_subdirectory(src/lib_http)
-    add_subdirectory(src/lib_domain)
-    add_subdirectory(src/lib_python_md)
-    add_subdirectory(src/app_sparkd)
-    add_subdirectory(src/data_files)
-  endif()
-  add_subdirectory(src/lib_spark)
-  if (BUILD_ADMIN_WEB_APP)
-    add_subdirectory(src/Spark.Admin.WebApp)
-  endif()
-  if(BUILD_DOCS)
-    add_subdirectory(doc/user)
-  endif()
-  if(BUILD_SDK)
-    add_subdirectory(src/sdk)
-  endif()
+  # ** Okx **
+  add_compile_definitions(OKX_INSTRUMENTS_PATH="/api/v5/public/instruments")
+  add_compile_definitions(OKX_UNDERLYING_PATH="/api/v5/public/underlying")
+endif()
+
+# subdirs
+add_subdirectory(src/lib_system)
+add_subdirectory(src/lib_domain)
+add_subdirectory(src/lib_http)
+if (BUILD_SPARKD)
+  add_subdirectory(src/lib_python_md)
+  add_subdirectory(src/app_sparkd)
+  add_subdirectory(src/data_files)
+endif()
+add_subdirectory(src/lib_spark)
+if (BUILD_ADMIN_WEB_APP)
+  add_subdirectory(src/Spark.Admin.WebApp)
+endif()
+if(BUILD_DOCS)
+  add_subdirectory(doc/user)
+endif()
+if(BUILD_SDK)
+  add_subdirectory(src/sdk)
 endif()

CMakePresets.json

@@ -28,7 +28,7 @@
         "CMAKE_SHARED_LINKER_FLAGS_RELEASE": "/OPT:REF /OPT:ICF /LTCG",
         "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
         "EXTERNALS_ROOT": "${sourceDir}/externals/windows",
-        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
+        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf",
         "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0;${sourceDir}/build/$<CONFIG>/bin/net7.0-windows",
         "EXPORT_MD_CONSUMER": {
           "type": "BOOL",
@@ -168,8 +168,8 @@
           "type": "BOOL",
           "value": "ON"
         },
-        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/Spark.Api.Server/$<CONFIG>/bin/net7.0",
+        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf",
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
         "EXPORT_MD_CONSUMER": {
           "type": "BOOL",
           "value": "ON"
@@ -207,7 +207,7 @@
       "description": "Data Stream utils ported over from Edge",
       "binaryDir": "${sourceDir}/build/${presetName}",
       "cacheVariables": {
-        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf",
+        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf;spark/config/profiles/data_stream/*.conf",
         "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0;${sourceDir}/build/$<CONFIG>/bin/net7.0-windows"
       }
     },
@@ -240,8 +240,7 @@
         "CMAKE_CXX_COMPILER": "clang++",
         "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
         "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
-        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf",
-        "CONFIG_PROFILES": "default;data_stream",
+        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf;spark/config/profiles/data_stream/*.conf",
         "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0;${sourceDir}/build/$<CONFIG>/bin/net8.0",
         "EXPORT_MD_CONSUMER": {
           "type": "BOOL",

Spark.Api.Server.sln

@@ -14,6 +14,36 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "spark_test", "src\client\sp
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Spark.Console", "src\client\Spark.Console\Spark.Console.csproj", "{89D666E6-B500-498F-B026-4D167F68ADFC}"
 EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{267F81EF-A8AB-49FD-9F56-D7EC3C1F7C7C}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "DataStreamSuite", "DataStreamSuite", "{C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamCore", "src\DataStreamSuite\DataStreamCore\DataStreamCore.csproj", "{EF26A475-6101-4C9C-A717-D60BB92A2302}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SharedStreams", "src\DataStreamSuite\SharedStreams\SharedStreams.csproj", "{911B08A3-515A-441D-84C8-8A43740B5FC1}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PubSub", "src\DataStreamSuite\PubSub\PubSub.csproj", "{14140310-2233-430A-821B-F3058A25234B}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BaseUtilities", "src\DataStreamSuite\BaseUtilities\BaseUtilities.csproj", "{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ExpressionCore", "src\DataStreamSuite\ExpressionCore\ExpressionCore.csproj", "{6F22DB96-6EAB-4034-99D9-287021C36F01}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ArithmeticExpressions", "src\DataStreamSuite\ArithmeticExpressions\ArithmeticExpressions.csproj", "{A5C33956-A41F-49B3-A41D-1082CBA59D02}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamExpressions", "src\DataStreamSuite\DataStreamExpressions\DataStreamExpressions.csproj", "{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnSortExpressions", "src\DataStreamSuite\Bts.ColumnSortExpressions\Bts.ColumnSortExpressions.csproj", "{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ColumnMapExpressions", "src\DataStreamSuite\ColumnMapExpressions\ColumnMapExpressions.csproj", "{B945A183-2174-4ACB-B289-8A5C546656FE}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BooleanExpressions", "src\DataStreamSuite\BooleanExpressions\BooleanExpressions.csproj", "{9F6D7DE8-185B-4C1D-B809-3314D092DF28}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnExpressions", "src\DataStreamSuite\Bts.ColumnExpressions\Bts.ColumnExpressions.csproj", "{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStream", "src\DataStreamSuite\DataStream\DataStream.csproj", "{39AEA075-6457-46C0-8821-1158BE6EFA57}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Net", "src\DataStreamSuite\Bts.Net\Bts.Net.csproj", "{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|x64 = Debug|x64
@@ -70,6 +100,110 @@ Global
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.Release|x64.Build.0 = Release|x64
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|x64.ActiveCfg = Debug|x64
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|x64.Build.0 = Debug|x64
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Debug|x64.Build.0 = Debug|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Release|x64.ActiveCfg = Release|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Release|x64.Build.0 = Release|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Debug|x64.Build.0 = Debug|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Release|x64.ActiveCfg = Release|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Release|x64.Build.0 = Release|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.Debug|x64.Build.0 = Debug|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.Release|x64.ActiveCfg = Release|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.Release|x64.Build.0 = Release|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Debug|x64.Build.0 = Debug|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Release|x64.ActiveCfg = Release|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Release|x64.Build.0 = Release|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Debug|x64.Build.0 = Debug|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Release|x64.ActiveCfg = Release|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Release|x64.Build.0 = Release|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Debug|x64.Build.0 = Debug|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Release|x64.ActiveCfg = Release|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Release|x64.Build.0 = Release|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Debug|x64.Build.0 = Debug|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Release|x64.ActiveCfg = Release|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Release|x64.Build.0 = Release|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Debug|x64.Build.0 = Debug|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Release|x64.ActiveCfg = Release|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Release|x64.Build.0 = Release|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.Debug|x64.Build.0 = Debug|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.Release|x64.ActiveCfg = Release|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.Release|x64.Build.0 = Release|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Debug|x64.Build.0 = Debug|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Release|x64.ActiveCfg = Release|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Release|x64.Build.0 = Release|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Debug|x64.Build.0 = Debug|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Release|x64.ActiveCfg = Release|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Release|x64.Build.0 = Release|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Debug|x64.Build.0 = Debug|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Release|x64.ActiveCfg = Release|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Release|x64.Build.0 = Release|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.Debug|x64.Build.0 = Debug|Any CPU
+		{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.Release|x64.ActiveCfg = Release|Any CPU
+		{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.Release|x64.Build.0 = Release|Any CPU
+		{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -80,4 +214,19 @@ Global
 	GlobalSection(Performance) = preSolution
 		HasPerformanceSessions = true
 	EndGlobalSection
+	GlobalSection(NestedProjects) = preSolution
+		{C09ED521-D2DF-40BC-A76F-EB3CB870B2F6} = {267F81EF-A8AB-49FD-9F56-D7EC3C1F7C7C}
+		{EF26A475-6101-4C9C-A717-D60BB92A2302} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{911B08A3-515A-441D-84C8-8A43740B5FC1} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{14140310-2233-430A-821B-F3058A25234B} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{6F22DB96-6EAB-4034-99D9-287021C36F01} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{B945A183-2174-4ACB-B289-8A5C546656FE} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{39AEA075-6457-46C0-8821-1158BE6EFA57} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+	EndGlobalSection
 EndGlobal

cmake/InitLibrary.cmake

@@ -72,6 +72,14 @@ function(init_library)
   else()
     add_library(${_name} ${${_name}_SOURCES} ${${_name}_CXX_PROTO_SOURCES})
   endif()
+  target_include_directories(
+    ${_name}
+    SYSTEM
+    BEFORE
+    PUBLIC
+      $<BUILD_INTERFACE:${EXTERNALS_ROOT}/include>
+      $<BUILD_INTERFACE:${EXTERNALS_ROOT}/include/zlib>
+  )
   target_include_directories(
     ${_name}
     PUBLIC
@@ -90,6 +98,7 @@ function(init_library)
     ${_name}
     PUBLIC
       $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
+      $<BUILD_INTERFACE:${EXTERNALS_ROOT}/lib/$<CONFIG>>
       $<INSTALL_INTERFACE:${CMAKE_INSTALL_LIBDIR}>
   )
   # Install main lib
@@ -103,7 +112,7 @@ function(init_library)
   )
 
   # Add tests executable (if there are any)
-  if (${_name}_TEST_SOURCES)
+  if (${_name}_TEST_SOURCES AND NOT WIN32)
     add_executable(${_name}-tests ${${_name}_TEST_SOURCES})
 
     get_target_property(_lib_type ${_name} TYPE)

cmake/ProtobufGenerateCpp.cmake

@@ -15,7 +15,11 @@ function(protobuf_generate_cpp)
   cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
 
   if (NOT DEFINED _ARGS_PROTOC)
-    set(_ARGS_PROTOC "${EXTERNALS_BIN}/protoc")
+    if (WIN32)
+      set(_ARGS_PROTOC "${EXTERNALS_BIN}/protoc.exe")
+    else()
+      set(_ARGS_PROTOC "${EXTERNALS_BIN}/protoc")
+    endif()
   endif()
   set(${_ARGS_CXX_SOURCES})
   set(${_ARGS_CXX_HEADERS})

cmake/ProtobufGenerateCppConfig.cmake

@@ -33,7 +33,7 @@ function(protobuf_generate_cpp_config)
 
   # Generate static defaults. Ideally, this would be a single string, but we use an array because
   # the compiler complains if the string is longer than 65k.
-  set(_cfg_defaults_str "const char * config_static_defaults[] = { ")
+  set(_cfg_defaults_str "inline const char * config_static_defaults[] = { ")
 
   foreach(_cfg_file_path ${_ARGS_PROTO_DEFAULTS})
     file(READ ${_cfg_file_path} _cfg_file_content)

cmake/SymLink.cmake

@@ -20,16 +20,14 @@ endfunction()
 
 function(sym_link_file)
   set(_options)
-  set(_one_value_args TARGET_NAME SOURCE DEST INSTALL_DEST)
+  set(_one_value_args SOURCE DEST INSTALL_DEST)
   set(_multi_value_args)
   cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-  add_custom_command(
-    OUTPUT ${_ARGS_DEST}
-    COMMAND ln ARGS -s ${_ARGS_SOURCE} ${_ARGS_DEST}
-  )
-  add_custom_target(
-    ${_ARGS_TARGET_NAME} ALL
-    DEPENDS ${_ARGS_DEST}
+  file(
+    CREATE_LINK
+      ${_ARGS_SOURCE}
+      ${_ARGS_DEST}
+    SYMBOLIC
   )
   if (DEFINED _ARGS_INSTALL_DEST)
     install(

cmake/vs_filters.cmake

@@ -5,7 +5,7 @@ cmake_minimum_required(VERSION 3.5)
 cmake_policy(SET CMP0076 NEW)
 
 # add protos to the library to facilitate source_group
-target_sources(libspark PUBLIC ${Config_Protos} ${NonConfig_Protos} ${Service_Protos})
+target_sources(spark PUBLIC ${Config_Protos} ${NonConfig_Protos} ${Service_Protos})
 
 function(assign_source_group)
   foreach(_source IN ITEMS ${ARGN})

compose.yaml

@@ -1,52 +1,59 @@
 services:
   sparkd:
     build:
-      dockerfile: src/server/sparkd/Dockerfile
+      dockerfile: src/app_sparkd/Dockerfile
+      args:
+        - IMAGE=${SPARKD__IMAGE}
+        - SOURCE_DIR=${SPARKD__SOURCE_DIR}
+        - CONF_FILE=${SPARKD__CONF_FILE}
+        - MD_RECORDING_DIR=${SPARKD__MD_RECORDING_DIR}
     command: [ "./sparkd", "run" ]
-    ports: [ "8000:8000", "8001:8001", "8002:8002" ]
+    ports:
+      - "8000:8000"
+      - "8001:8001"
+      - "8002:8002"
 
   pubsub:
     environment:
-      - DOTNET_ENVIRONMENT=Staging
+      - DOTNET_ENVIRONMENT=${PUBSUB__DOTNET_ENVIRONMENT}
     build:
       dockerfile: src/DataStreamSuite/PubSubServer/Dockerfile
+      args:
+        - CS_SOURCE_DIR=${PUBSUB__CS_SOURCE_DIR}
+        - CPP_SOURCE_DIR=${PUBSUB__CPP_SOURCE_DIR}
+        - CONF_FILE=${PUBSUB__CONF_FILE}
     ports:
-      - 22019:22019
+      - "22019:22019"
     command: [ "./PubSubServer" ]
 
   md-publisher:
     build:
       dockerfile: src/DataStreamSuite/DataStreamPublisher/Dockerfile
+      args:
+        - SOURCE_DIR=${MD_PUBLISHER__SOURCE_DIR}
+        - CONF_FILE=${MD_PUBLISHER__CONF_FILE}
     depends_on:
       - pubsub
       - sparkd
-    command:
-      [
-        "dockerize",
-        "-wait",
-        "tcp://pubsub:22019",
-        "-wait",
-        "tcp://sparkd:8000",
-        "-timeout",
-        "10s",
-        "./DataStreamPublisher"
-      ]
+    command: [ "dockerize", "-wait", "tcp://pubsub:22019", "-wait", "tcp://sparkd:8000", "-timeout", "10s", "./DataStreamPublisher" ]
     environment:
-      - DOTNET_ENVIRONMENT=MarketData
+      - DOTNET_ENVIRONMENT=${MD_PUBLISHER__DOTNET_ENVIRONMENT}
 
   webapp:
     ports:
-      - 8080:8080
-      - 8443:8443
+      - "8080:8080"
+      - "8443:8443"
     build:
       dockerfile: src/DataStreamSuite/Bts.DataStreamWebApp.Server/Dockerfile
+      args:
+        - SOURCE_DIR=${WEBAPP__SOURCE_DIR}
+        - WEB_ROOT=${WEBAPP__WEB_ROOT}
+        - CONF_FILE=${WEBAPP__CONF_FILE}
     command: [ "./Bts.DataStreamWebApp.Server" ]
     environment:
-      - ASPNETCORE_ENVIRONMENT=Staging
+      - ASPNETCORE_ENVIRONMENT=${WEBAPP__ASPNETCORE_ENVIRONMENT}
 
   api-server:
-    profiles:
-      - api-server
     depends_on:
       - sparkd
     ports:
@@ -54,9 +61,9 @@ services:
       - 9443:9443
     build:
       dockerfile: src/client/spark_api_server/Dockerfile
-    command: [ "./spark_api_server", "run" ]
+    command: [ "dockerize", "-wait", "tcp://sparkd:8000", "-timeout", "10s", "./spark_api_server", "run" ]
     environment:
-      - ASPNETCORE_ENVIRONMENT=Staging
+      - ASPNETCORE_ENVIRONMENT=${API_SERVER__ASPNETCORE_ENVIRONMENT}
 
   sparkd-ping-test:
     depends_on:

src/DataStreamSuite/Bts.DataStreamWebApp.Client/.env.development

@@ -1 +1 @@
-VITE_DATASTREAM_WEBSOCKET_URL=wss://localhost:443
+VITE_DATASTREAM_WEBSOCKET_URL=wss://localhost:8443/ws

src/DataStreamSuite/Bts.DataStreamWebApp.Client/.env.production

@@ -1 +1 @@
-VITE_DATASTREAM_WEBSOCKET_URL=wss://localhost:443
+VITE_DATASTREAM_WEBSOCKET_URL=wss://localhost:8443/ws

src/DataStreamSuite/Bts.DataStreamWebApp.Client/.eslintrc.cjs

@@ -1,32 +0,0 @@
-module.exports = {
-  root: true,
-  env: { browser: true, es2020: true },
-  extends: [
-    'eslint:recommended',
-    'plugin:react/recommended',
-    'plugin:react/jsx-runtime',
-    'plugin:react-hooks/recommended',
-  ],
-  ignorePatterns: ['dist', '.eslintrc.cjs'],
-  parserOptions: { ecmaVersion: 'latest', sourceType: 'module' },
-  settings: { react: { version: '18.3.1' } },
-  plugins: ['react-refresh'],
-  rules: {
-    "import/order": [
-      "error",
-      {
-        "groups": [
-          "builtin",
-          "external",
-          "internal",
-          ["parent", "sibling", "index"]
-        ],
-        "newlines-between": "always"
-      }
-    ],
-    'react-refresh/only-export-components': [
-      'warn',
-      { allowConstantExport: true },
-    ],
-  },
-}

src/DataStreamSuite/Bts.DataStreamWebApp.Client/eslint.config.js

@@ -0,0 +1,36 @@
+import importPlugin from "eslint-plugin-import";
+import reactPkg from "eslint-plugin-react";
+const { configs: reactConfigs } = reactPkg;
+import js from "@eslint/js";
+import defineConfig from "./vite.config.js";
+
+export default [
+  js.configs.recommended,
+  importPlugin.flatConfigs.recommended,
+  reactConfigs.flat.recommended,
+  reactConfigs.flat["jsx-runtime"],
+  {
+    files: ["**/*.{js,jsx}"],
+    languageOptions: {
+      ecmaVersion: "latest",
+      sourceType: "module",
+      globals: {
+        console: "readonly"
+      }
+    },
+    settings: {
+      "import/resolver": {
+        vite: {
+          viteConfig: defineConfig({ mode: "development" })
+        }
+      }
+    },
+    rules: {
+      semi: "error",
+      "prefer-const": "error",
+      "no-unused-vars": "warn",
+      "import/no-dynamic-require": "warn",
+      "import/no-nodejs-modules": "warn",
+    }
+  }
+];

src/DataStreamSuite/Bts.DataStreamWebApp.Client/package.json

@@ -10,35 +10,37 @@
     "preview": "vite preview"
   },
   "dependencies": {
-    "@angular/material": "^18.0.4",
-    "@emotion/react": "^11.11.4",
-    "@emotion/styled": "^11.11.5",
-    "@fontsource/noto-sans": "^5.0.22",
-    "@material/data-table": "^15.0.0-canary.7f224ddd4.0",
-    "@mui/icons-material": "^5.15.20",
-    "@mui/material": "^5.15.20",
-    "@mui/x-data-grid": "^7.7.1",
-    "bulma": "^1.0.1",
+    "@angular/material": "^18.2.8",
+    "@emotion/react": "^11.13.3",
+    "@emotion/styled": "^11.13.0",
+    "@fontsource/noto-sans": "^5.1.0",
+    "@material/data-table": "^15.0.0-canary.423edc3dc.0",
+    "@mui/icons-material": "^6.1.4",
+    "@mui/material": "^6.1.4",
+    "@mui/x-data-grid": "^7.20.0",
+    "bulma": "^1.0.2",
     "classnames": "^2.5.1",
     "react": "^18.3.1",
-    "react-data-grid": "^7.0.0-beta.44",
+    "react-data-grid": "^7.0.0-beta.47",
     "react-data-table-component": "^7.6.2",
     "react-dom": "^18.3.1",
-    "react-json-view-lite": "^1.4.0",
+    "react-json-view-lite": "^1.5.0",
     "react-pro-sidebar": "^1.1.0",
-    "react-use-websocket": "^4.8.1",
-    "rpc-websockets": "^7.11.2"
+    "react-use-websocket": "^4.9.0",
+    "rpc-websockets": "^9.0.4"
   },
   "devDependencies": {
-    "@types/react": "^18.3.3",
-    "@types/react-dom": "^18.3.0",
-    "@vitejs/plugin-react": "^4.3.1",
-    "eslint": "^8.57.0",
-    "eslint-plugin-import": "^2.29.1",
-    "eslint-plugin-react": "^7.34.2",
-    "eslint-plugin-react-hooks": "^4.6.2",
-    "eslint-plugin-react-refresh": "^0.4.7",
-    "react-mui-sidebar": "^1.3.5",
-    "vite": "^5.2.13"
+    "@types/react": "^18.3.11",
+    "@types/react-dom": "^18.3.1",
+    "@vitejs/plugin-react": "^4.3.2",
+    "eslint": "^9.12.0",
+    "@eslint/js": "^9.12.0",
+    "eslint-plugin-import": "^2.31.0",
+    "eslint-plugin-react": "^7.37.1",
+    "eslint-plugin-react-hooks": "^5.0.0",
+    "eslint-plugin-react-refresh": "^0.4.12",
+    "eslint-import-resolver-vite": "^2.0.1",
+    "react-mui-sidebar": "^1.3.8",
+    "vite": "^5.4.9"
   }
 }

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/DataTable.jsx

@@ -1,7 +1,7 @@
 import PropTypes from 'prop-types';
 import { useTheme } from '@mui/material/styles';
 import { DataGrid } from '@mui/x-data-grid';
-import './DataGrid.css'
+import './DataGrid.css';
 import { useMemo, useState } from 'react';
 
 export default function DataTable({ columns, rows }) {
@@ -23,7 +23,7 @@ export default function DataTable({ columns, rows }) {
         headerName: col.name,
         width: columnWidths[col.name],
       };
-    }
+    };
   }, [columnWidths]);
 
   return (

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/QueryStatusView.jsx

@@ -12,7 +12,7 @@ const QueryStatusView = ({ queryState, details, error }) => {
 
   let header = queryState;
   let headerContent = null;
-  let detailElements = [];
+  const detailElements = [];
   let severity = queryState == "Subscribed" ? "success" : "info";
   let cursor = details == null ? "default" : "pointer";
   if (error != null) {
@@ -20,14 +20,14 @@ const QueryStatusView = ({ queryState, details, error }) => {
     cursor = "pointer";
     header = `Error Code: ${error.code}`;
     headerContent = error.message;
-    detailElements.push(<Typography variant="body1">{error.data.error_message}</Typography>);
-    detailElements.push(<Typography variant="subtitle1">Stack Trace:</Typography>);
-    (error.data.stack_trace ?? []).forEach(line => detailElements.push(
-      <Typography variant="body2" component="pre" sx={{ whiteSpace: 'pre-wrap' }}>
+    detailElements.push(<Typography key="error-msg" variant="body1">{error.data.error_message}</Typography>);
+    detailElements.push(<Typography key="error-details" variant="subtitle1">Stack Trace:</Typography>);
+    (error.data.stack_trace ?? []).forEach((line, idx) => detailElements.push(
+      <Typography key={`stacktrace-${idx}`} variant="body2" component="pre" sx={{ whiteSpace: 'pre-wrap' }}>
         {line}
       </Typography>));
   } else if (details != null) {
-    detailElements.push(<Typography variant="body1">{details}</Typography>);
+    detailElements.push(<Typography key="status-msg" variant="body1">{details}</Typography>);
   }
   return (
     <Box sx={{ margin: theme.spacing(2, 0, 0, 0) }}>

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/ReactDataGrid.jsx

@@ -13,7 +13,7 @@ function getGridColDef(col) {
 export default function ReactDataGrid({columns, rows}) {
   return (
     <DataGrid className={"data-grid"} rows={rows} columns={columns.map(getGridColDef)} />
-  )
+  );
 }
 
 ReactDataGrid.propTypes = {

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/SideBar.jsx

@@ -3,7 +3,7 @@ import Button from '@mui/material/Button';
 import { useTheme } from '@mui/material/styles';
 import PropTypes from 'prop-types';
 import Tooltip from '@mui/material/Tooltip';
-import ChevronLeftRounded from "@mui/icons-material/ChevronLeftRounded"
+import ChevronLeftRounded from "@mui/icons-material/ChevronLeftRounded";
 import ChevronRightRounded from '@mui/icons-material/ChevronRightRounded';
 import FileCopyIcon from '@mui/icons-material/FileCopy';
 import './SideBar.css';
@@ -51,7 +51,7 @@ const Sidebar = ({ onSelection, onOpened, onClosed }) => {
       } catch (error) {
         console.error('Error fetching WebSocket config:', error);
       }
-    }
+    };
     fetchCannedQueries();
   }, []);
 

src/DataStreamSuite/Bts.DataStreamWebApp.Client/vite.config.js

@@ -1,5 +1,5 @@
 import { fileURLToPath, URL } from 'node:url';
-import { defineConfig, loadEnv } from 'vite';
+import { defineConfig } from 'vite';
 import plugin from '@vitejs/plugin-react';
 import fs from 'fs';
 import path from 'path';
@@ -51,11 +51,26 @@ export default defineConfig(({ $$command, mode, $$isSsrBuild, $$isPreview }) =>
       proxy: {
         '^/api': {
           target,
-          secure: false
+          secure: false,
+          configure: (proxy) => {
+            proxy.on('error', (err) => {
+              console.log('proxy error', err);
+            });
+            proxy.on('proxyReq', (proxyReq, req) => {
+              console.log(`Forwarding ${proxyReq.method} ${proxyReq.protocol}//${proxyReq.host}${proxyReq.path} to ${req.socket.remoteAddress}${req.url}`);
+            });
+            proxy.on('proxyRes', (proxyRes, req, res) => {
+              console.log(`Forwarding response ${req.socket.localAddress} to ${res.socket.remoteAddress}. Status=${proxyRes.statusCode}`);
+            });
+          },
         },
         '^/ws': {
           target,
           secure: false
+        },
+        '^/browse': {
+          target,
+          secure: false
         }
       },
       port: 5173,

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Controllers/DataStreamSubscriptionController.cs

@@ -1,14 +1,21 @@
 namespace Bts.DataStreamWebApp.Server.Controllers;
 
+using Bts.DataStream;
+
 using Microsoft.AspNetCore.Mvc;
 
 [Route("/ws")]
 public class DataStreamSubscriptionController : ControllerBase {
    readonly ILogger logger;
+   readonly IDataStreamFactory streamFactory;
 
-   public DataStreamSubscriptionController(ILogger<DataStreamSubscriptionController> logger) {
+   public DataStreamSubscriptionController(ILogger<DataStreamSubscriptionController> logger, IEnumerable<IDataStreamFactory> streamFactories) {
       this.logger = logger;
       logger.LogInformation("Creating {ControllerType}", GetType().Name);
+      var streamFactoryList = streamFactories.ToList();
+      streamFactory = streamFactoryList.Count == 1
+         ? streamFactoryList[0]
+         : new CompositeDataStreamFactory(streamFactoryList);
    }
 
    [Route("/ws")]
@@ -21,7 +28,7 @@ public class DataStreamSubscriptionController : ControllerBase {
       try {
          using var webSocket = await HttpContext.WebSockets.AcceptWebSocketAsync();
          logger.LogInformation("Websocket connection established.");
-         using var server = new DataStreamWebSocketServer(webSocket, logger);
+         using var server = new DataStreamWebSocketServer(webSocket, streamFactory, logger);
          await server.Run(cancellationToken);
       } catch (Exception ex) {
          logger.LogError("DataStreamSubscriptionController.Run threw exception: {Exception}", ex);

src/DataStreamSuite/Bts.DataStreamWebApp.Server/DataStreamWebSocketServer.cs

@@ -2,6 +2,7 @@ namespace Bts.DataStreamWebApp.Server.Controllers;
 
 using bts.websocket_core.json_rpc;
 
+using Bts.DataStream;
 using Bts.DataStreamWebApp.Server.Messages;
 
 using System.Net.WebSockets;
@@ -13,9 +14,11 @@ public class DataStreamWebSocketServer : IDisposable, IAsyncDisposable {
    readonly ILogger logger;
    readonly WebSocket webSocket;
    readonly Dictionary<string, DataStreamSubscription> subscriptions = new();
+   readonly IDataStreamFactory streamFactory;
 
-   public DataStreamWebSocketServer(WebSocket webSocket, ILogger logger) {
+   public DataStreamWebSocketServer(WebSocket webSocket, IDataStreamFactory streamFactory, ILogger logger) {
       this.webSocket = webSocket;
+      this.streamFactory = streamFactory;
       this.logger = logger;
    }
 
@@ -67,7 +70,7 @@ public class DataStreamWebSocketServer : IDisposable, IAsyncDisposable {
       }
       var subId = subReq.SubscriptionId ?? throw new InvalidOperationException("SubscriptionId is null (parse false)");
       disposeSubscription(subId);
-      if (!subReq.ParseQuery(out var vs, out parseErr, out errCode)) {
+      if (!subReq.ParseQuery(streamFactory, out var vs, out parseErr, out errCode)) {
          logger.LogError("Invalid 'subscribe' request: {Reason}", parseErr);
          var rpcError = JsonRpcResponse.CreateError<SubscriptionError>(
             req.Id, new() { Code = (long)errCode, Message = "Failed to parse subscription request", Data = parseErr });

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Dockerfile

@@ -1,11 +1,13 @@
 FROM bluetradingsystems/linux_dotnet7
 
-ARG SOURCE_DIR="build/Release/bin/net7.0/linux-x64/publish"
+ARG CS_SOURCE_DIR="build/Release/bin/net7.0/linux-x64/publish"
+ARG CPP_SOURCE_DIR="install/data-stream-suite-linux"
 ARG WEB_ROOT="build/Bts.DataStreamWebApp.Client/production/dist"
 ARG CONF_FILE="src/DataStreamSuite/Bts.DataStreamWebApp.Server/spark.conf"
 
 RUN dotnet dev-certs https --trust
-COPY --chown=bts:bts "${SOURCE_DIR}" "bin"
+COPY --chown=bts:bts "${CPP_SOURCE_DIR}/lib" "bin"
+COPY --chown=bts:bts "${CS_SOURCE_DIR}" "bin"
 COPY --chown=bts:bts "${WEB_ROOT}" "bin/wwwroot"
 COPY --chown=bts:bts "${CONF_FILE}" ".local/share/bts/Bts.DataStreamWebApp.Server/conf/spark.conf"
 WORKDIR "/home/bts/bin"

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Messages/SubscribeRequest.cs

@@ -4,6 +4,7 @@ using BCG;
 
 using bts.websocket_core.json_rpc;
 
+using Bts.DataStream;
 using Bts.DataStreamExpressions;
 
 using System.Diagnostics.CodeAnalysis;
@@ -37,7 +38,12 @@ public class SubscribeRequest {
       return true;
    }
 
-   internal bool ParseQuery([NotNullWhen(true)] out ViewStream? stream, [NotNullWhen(false)] out SubscriptionError? error, out ErrorCode errorCode) {
+   internal bool ParseQuery(
+      IDataStreamFactory streamFactory,
+      [NotNullWhen(true)] out ViewStream? stream,
+      [NotNullWhen(false)] out SubscriptionError? error,
+      out ErrorCode errorCode
+   ) {
       stream = null;
       errorCode = ErrorCode.None;
       error = null;
@@ -54,7 +60,7 @@ public class SubscribeRequest {
       }
       Exception? exception = null;
       try {
-         stream = exp.getView(new Dictionary<string, StatementExpression>(0));
+         stream = exp.getView(new Dictionary<string, StatementExpression>(0), streamFactory);
       } catch (Exception ex) {
          exception = ex;
       }

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Program.cs

@@ -1,5 +1,6 @@
 using Bts.DataStreamCore.Services;
 using Bts.PubSub.Services;
+using Bts.SharedStreams.Services;
 
 using Spark.Api.Services;
 using Spark.Core.Services;
@@ -78,5 +79,9 @@ static void ConfigureServices(IServiceCollection services, IConfiguration config
       .AddSparkApi(config.GetSection("SparkApi"))
       .AddMasterWorker(config.GetSection("MasterWorker"))
       .AddViewStreamDatabase()
+      .AddPrimitiveDataStreamFactory(config.GetSection("PrimitiveStreams"))
+      .AddOhlcvStreams(config.GetSection("StreamDatabase"))
+      .AddNaturalLanguageStreams(config.GetSection("NaturalLanguageStream"))
+      .AddCompositeDataStreamFactory()
       .AddPubSubClient(config.GetSection("PubSub"));
 }

src/DataStreamSuite/Bts.DataStreamWebApp.Server/appsettings.Development.json

@@ -1,14 +1,4 @@
 {
-  "Kestrel": {
-    "Endpoints": {
-      "Http": {
-        "Url": "http://localhost:8080"
-      },
-      "Https": {
-        "Url": "https://localhost:8443"
-      }
-    }
-  },
   "PubSub": {
     "Routing": {
       "PublisherRoutes": [

src/DataStreamSuite/Bts.DataStreamWebApp.Server/appsettings.NaturalLanguageCustom.json

@@ -0,0 +1,45 @@
+{
+  "NaturalLanguagePrompt": {
+    "prompts": [
+      {
+        "category": "Impact of financial events on the market",
+        "description": "Show how trading metrics (e.g., price movement, volume) are affected by events related to finance (rate announcements, earnings reports, etc.).",
+        "prompt": "Show me the price movement and volume of Crude Oil futures during the 5-minute window following the last 5 OPEC meetings",
+        "examples": [
+          {
+            "natural_language": "Show me the price movement and volume of Crude Oil futures during the 5-minute window following the last 5 OPEC meetings",
+            "dsl": [
+              "time_idx=EventTimeIndex.OpecMeetings(which: \"last 5\", duration: \"+5min\")",
+              "contract_idx=ActiveFutureContracts(predicate: \"product==CL\")",
+              "Bars(time_index: time_idx, contract_index: contract_idx, fields: [\"price_movement\", \"volume\"])"
+            ]
+          },
+          {
+            "natural_language": "How do OPEC meetings affect the price of Crude futures? Examine the 10 minutes after each meeting this year.",
+            "dsl": [
+              "time_idx=EventTimeIndex.OpecMeetings(duration: \"+10min\", predicate: \"timestamp.year==2024\")",
+              "contract_idx=ActiveFutureContracts(predicate: \"product==CL\")",
+              "Bars(time_index: time_idx, contract_index: contract_idx, fields: [\"price_movement\", \"volume\"])"
+            ]
+          },
+          {
+            "natural_language": "Show me how the SP 500 price moves after FOMC rate announcements that day",
+            "dsl": [
+              "time_idx=EventTimeIndex.FomcRateAnnouncements(which: \"timestamp.year==2024\", duration: \"+10min\")",
+              "contract_idx=ActiveFutureContracts(predicate: \"product==ES\")",
+              "Bars(time_index: time_idx, contract_index: contract_idx, fields: [\"price_movement\", \"volume\"])"
+            ]
+          },
+          {
+            "natural_language": "Analyze the impact of the FED's interest rate hikes in 2022 on the 10-year Treasury yield, focusing on the 2-week period following each hike.",
+            "dsl": [
+              "time_idx=FedInterestRateAnnouncements(predicate: \"timestamp.year==2022 && change_pct>0\", duration: \"+2 weeks\")",
+              "contract_idx=ActiveFutureContracts(predicate: \"product==ZN\")",
+              "Bars(time_index: time_idx, contract_index: contract_idx)"
+            ]
+          }
+        ]
+      }
+    ]
+  }
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/appsettings.Staging.json

@@ -30,5 +30,15 @@
         "benzinga_news"
       ]
     }
-  ]
+  ],
+  "Spark": {
+    "ConfigServer": "http://sparkd:8000/config"
+  },
+  "SparkApi": {
+    "IsEnabled": true,
+    "Credentials": {
+      "User": "bts",
+      "Password": ""
+    }
+  }
 }

src/DataStreamSuite/Bts.DataStreamWebApp.Server/appsettings.json

@@ -11,8 +11,14 @@
       "Http": {
         "Url": "http://0.0.0.0:8080"
       },
+      "HttpIpv6": {
+        "Url": "http://[::1]:8080"
+      },
       "Https": {
         "Url": "https://0.0.0.0:8443"
+      },
+      "HttpsIpv6": {
+        "Url": "https://[::1]:8443"
       }
     }
   },
@@ -22,6 +28,358 @@
   "ViewStreamDatabase": {
     "Interval": "00:00:00.1"
   },
+  "PrimitiveStream": {
+    "Assemblies": [
+      "DataStream.dll",
+      "SharedStreams.dll"
+    ],
+    "ExcludedTypes": [
+      "SettlementPriceStream",
+      "OHLCVStream"
+    ]
+  },
+  "StreamDatabase": {
+    "Path": "/home/bts/.local/share/bts/silver_pine/store.db",
+    "Query": "select * from metadata"
+  },
+  "NaturalLanguageStream": {
+    "prompts": [
+      {
+        "category": "Settlement",
+        "MulitLineDescription": [
+          "This is a few shot learning example for an NL to DSL. The DSL function in question is Settlement (symbol_or_contract, datetime1). ",
+          "The first argument is a future (like CL, for crude oil) or a specific future contract like crude oil Dec 24 future (CLZ4). ",
+          "The second argument is the date at which the settlement is to be fetched. There is an overloaded function, Settlement ",
+          "(symbol_or_contract, begin_dt, end_dt) which gets the settlements for all dates from begin_dt to end_dt. This will also have ",
+          "datetimes expressed as Today, Yesterday, Last Friday, First Friday of the month, Last Thursday of August etc. You will need to know ",
+          "what Today is, since there will be referecnes to last week, yesterday etc. These examples are assuming Today is 10/11/2024, however ",
+          "you should use the Today's real date"
+        ],
+        "prompt": "What was the settlement for SP 500 Future on June 8th?",
+        "examples": [
+          {
+            "natural_language": "Find Settlement price of the SP500 future on 04/22/2023.",
+            "dsl": "Settlement(\"ES\", 4/22/2023)"
+          },
+          {
+            "natural_language": "Find Settlement price of the Silver Dec 24 Future on 04/22/2023.",
+            "dsl": "Settlement(\"SIZ4\", 4/22/2023)"
+          },
+          {
+            "natural_language": "Find Yesterday's Settlement of the Crude Oil Future.",
+            "dsl": "Settlement(\"CL\", 10/10/2024)"
+          },
+          {
+            "natural_language": "Find today's settlement of the 10 year treasury future.",
+            "dsl": "Settlement(\"ZN\", 10/11/2024)"
+          },
+          {
+            "natural_language": "Find the Settlement of Natural Gas Dec 24 future on the first Friday of Sept.",
+            "dsl": "Settlement(\"NGZ4\", 9/6/2024)"
+          },
+          {
+            "natural_language": "Find the Settlement of WTI Jan 25 future in the first Week of Sept.",
+            "dsl": "Settlement(\"CLF5\", 9/2/2024, 9/6/2024)"
+          },
+          {
+            "natural_language": "Find the Settlement price for the Sp 500 Dec 24 Mar 25  spread for the first two weeks of September.",
+            "dsl": "Settlement(\"ESZ4-ESH5\", 9/14/2024, 9/1/2024)"
+          },
+          {
+            "natural_language": "Find the Settlement of Natural Gas future on the first Friday of Sept.",
+            "dsl": "Settlement(\"NG\", 9/6/2024)"
+          }
+        ]
+      },
+      {
+        "category": "BBO",
+        "MulitLineDescription": [
+          "This is a few shot learning example for an NL to DSL. The DSL function in question is Bbo(args...). ",
+          "It accepts the following parameters: ",
+          "symbol (a product code such as ZN for 10 year T-Note futures), ",
+          "contract (a specific contract, such as ZNZ4 for the 10 Year T-Note future Dec 2024 exp), ",
+          "venue (name of the exchange, such as CME, NASDAQ, NYSE, NYMEX, DERIBIT etc.), ",
+          "interval (sample frequency of the data), ",
+          "begin and end (inclusive beginning and end of the time window to query, respectively), ",
+          "Arguments are specified by name, for example: Bbo(symbol: \"ZN\", venue: \"CME\", interval: \"PT1M\", begin: \"2024-10-10\", end: \"2024-10-11\"). ",
+          "Depending on the wording of the request, some arguments may be omitted. ",
+          "The begin, end, and interval args should be expressed in iso 8601 format. ",
+          "Interpret dates according to the trading schedule rather than calendar date whenever possible. ",
+          "In some prompts, the time window will be expressed in relative terms (e.g., \"last Monday\", or \"yesterday\") ",
+          "which must be translated to exact dates. To correctly interpret the dsl in the examples, assume the current time is 2024-10-11T12:00-05:00."
+        ],
+        "prompt": "Show me bbo for SP 500 futures on Sep 23rd of this year.",
+        "examples": [
+          {
+            "natural_language": "Get SP500 future bbo from Wednesday.",
+            "dsl": "Bbo(symbol: \"ES\", venue: \"CME\", begin: \"2024-10-08T17:00:00-06:00\", end: \"2024-10-09T16:00:00-06:00\")"
+          },
+          {
+            "natural_language": "Get bbo for 10 year T-Notes in the first minute after the market opened yesterday.",
+            "dsl": "Bbo(symbol: \"ES\", venue: \"CME\", begin: \"2024-10-10T17:00:00-06:00\", end: \"2024-10-10T17:01:00-06:00\")"
+          },
+          {
+            "natural_language": "Retrieve NVIDIA L1 books from Oct 3",
+            "dsl": "Bbo(symbol: \"NVDA\", venue: \"NASDAQ\", begin: \"2024-10-02TT18:00:00-05:00\", end: \"2024-10-03T17:00:00-05:00\")"
+          },
+          {
+            "natural_language": "Show me L1 data for Crude oil futures this week",
+            "dsl": "Bbo(symbol: \"CL\", venue: \"CME\", begin: \"2024-10-06:T17:00:00-06:00\")"
+          },
+          {
+            "natural_language": "Fetch hourly bbo for Gold futures this week",
+            "dsl": "Bbo(symbol: \"GC\", venue: \"CME\", begin: \"2024-10-06:T17:00:00-06:00\", interval: \"PT1H\")"
+          },
+          {
+            "natural_language": "What was the minute by minute bbo for Tesla yesterday?",
+            "dsl": "Bbo(symbol: \"TSLA\", venue: \"NASDAQ\", begin: \"2024-10-09T18:00:00-05:00\", end: \"2024-10-10T17:00:00-05:00\", interval: \"PT1M\")"
+          }
+        ]
+      },
+      {
+        "category": "Order flow imbalance",
+        "MulitLineDescription": [
+          "This is a few shot learning example for an NL to DSL. The DSL function in question is OrderImbalance(args...). ",
+          "It accepts the following parameters: ",
+          "symbol (a product code such as ZN for 10 year T-Note futures), ",
+          "contract (a specific contract, such as ZNZ4 for the 10 Year T-Note future Dec 2024 exp), ",
+          "venue (name of the exchange, such as CME, NASDAQ, NYSE, NYMEX, DERIBIT etc.), ",
+          "interval (sample frequency of the data), ",
+          "levels (the number of price levels to examine)",
+          "begin and end (inclusive beginning and end of the time window to query, respectively), ",
+          "Arguments are specified by name, for example: OrderImbalance(symbol: \"ZN\", venue: \"CME\", interval: \"PT1M\", begin: \"2024-10-10\", end: \"2024-10-11\"). ",
+          "Depending on the wording of the request, some arguments may be omitted. ",
+          "The begin, end, and interval args should be expressed in iso 8601 format. ",
+          "If the prompt doesn't contain an explicit interval, please choose a reasonable value depending on the duration between begin and end.",
+          "Interpret dates according to the trading schedule rather than calendar date whenever possible. ",
+          "In some prompts, the time window will be expressed in relative terms (e.g., \"last Monday\", or \"yesterday\") ",
+          "which must be translated to exact dates. To correctly interpret the dsl in the examples, assume the current time is 2024-10-11T12:00-05:00."
+        ],
+        "examples": [
+          {
+            "natural_language": "Get SP500 order imbalance from Wednesday.",
+            "dsl": "OrderImbalance(symbol: \"ES\", venue: \"CME\", begin: \"2024-10-08T17:00:00-06:00\", end: \"2024-10-09T16:00:00-06:00\", \"PT1M\")"
+          },
+          {
+            "natural_language": "Get SP500 order imbalance from Wednesday for the top 5 price levels.",
+            "dsl": "OrderImbalance(symbol: \"ES\", venue: \"CME\", begin: \"2024-10-08T17:00:00-06:00\", end: \"2024-10-09T16:00:00-06:00\", levels: 5, interval: \"PT1M\")"
+          },
+          {
+            "natural_language": "Get orderflow imbalance for 10 year T-Notes in the first minute after the market opened yesterday.",
+            "dsl": "OrderImbalance(symbol: \"ES\", venue: \"CME\", begin: \"2024-10-10T17:00:00-06:00\", end: \"2024-10-10T17:01:00-06:00\", interval: \"PT1S\")"
+          },
+          {
+            "natural_language": "Retrieve NVIDIA order flow from Oct 3",
+            "dsl": "OrderImbalance(symbol: \"NVDA\", venue: \"NASDAQ\", begin: \"2024-10-02TT18:00:00-05:00\", end: \"2024-10-03T17:00:00-05:00\", interval: \"PT1S\")"
+          },
+          {
+            "natural_language": "Show me order imbalance for Crude oil futures this week",
+            "dsl": "OrderImbalance(symbol: \"CL\", venue: \"CME\", begin: \"2024-10-06:T17:00:00-06:00\", interval: \"PT1H\")"
+          },
+          {
+            "natural_language": "Fetch hourly orderflow for Gold futures this week",
+            "dsl": "OrderImbalance(symbol: \"GC\", venue: \"CME\", begin: \"2024-10-06:T17:00:00-06:00\", interval: \"PT1H\")"
+          },
+          {
+            "natural_language": "What was the minute by minute orderflow for Tesla yesterday?",
+            "dsl": "OrderImbalance(symbol: \"TSLA\", venue: \"NASDAQ\", begin: \"2024-10-09T18:00:00-05:00\", end: \"2024-10-10T17:00:00-05:00\", interval: \"PT1M\")"
+          }
+        ]
+      },
+      {
+        "category": "VWAP",
+        "description": "This is a few shot learning example for an NL to DSL. The DSL function in question is VWAP (symbol_or_contract, datetime1, datetime2). The first argument is a future (like CL, for crude oil, or ES for SP 500) or a specific future contract like crude oil Dec 24 future (CLZ4) or SP 500 Dec 2024 (ESZ4). The second argument is where the VWAP has to be calculated (also the beginning of the time window), and the third argument is datetime specifying the end of the time window. So, for example, a 20-day VWAP on 9/30/2023, the first argument will be 9/30/2023, and the third argument will be 9/10/2023 (20 days prior). This will also have datetimes expressed as Today, Yesterday, Last Friday, Last Sunday, etc. For the purpose of this few-shot prompts, let's assume today is 10/11/2024.",
+        "prompt": "What was the VWAP for SP 500 Future over the past 30 days?",
+        "examples": [
+          {
+            "natural_language": "Find the 20 day VWAP of the SP future on 04/22/2023.",
+            "dsl": "VWAP(\"ES\", 4/22/2023, 4/02/2023)"
+          },
+          {
+            "natural_language": "Find Yesterday's 10 day VWAP of the Crude Oil Future.",
+            "dsl": "VWAP(\"CL\", 10/10/2024, 10/1/2024)"
+          },
+          {
+            "natural_language": "Find today's 30-day VWAP of the 10 year treasury future.",
+            "dsl": "VWAP(\"ZN\", 10/11/2024, 9/11/2024)"
+          },
+          {
+            "natural_language": "Find the 10 hour VWAP of the Natural Gas future from yesterday at 5:00 PM.",
+            "dsl": "VWAP(\"NG\", 9/27/2024 17:00:00, 9/27/2024 7:00:00)"
+          }
+        ]
+      },
+      {
+        "category": "Price Diff",
+        "description": "This is a few shot learning example for an NL to DSL. The DSL function in question is OHLCVPriceDiff (symbol_or_contract, datetime1, datetime2). The first argument is a future (like CL, for crude oil) or a specific future contract like crude oil Dec 24 future (CLZ4). The second and third arguments are dates, that the function finds the difference of prices at. Assume today is 10/15/2024",
+        "prompt": "What is the difference in  SP 500 Future from July 10th to July 14th?",
+        "examples": [
+          {
+            "natural_language": "What is the difference in ZN price from July 10th to July 14th?",
+            "dsl": "OHLCVPriceDiff(\"ZN\", 7/14/2024, 7/10/2024)"
+          },
+          {
+            "natural_language": "What is the price difference in SP Future price from Sept 10th and Sept 5th?",
+            "dsl": "OHLCVPriceDiff(\"ES\", 9/10/2024, 9/5/2024)"
+          },
+          {
+            "natural_language": "What is the price difference in Natural Gas Dec Future price from Aug 30th and Aug 20th?",
+            "dsl": "OHLCVPriceDiff(\"NGZ4\", 8/30/2024, 8/20/2024)"
+          }
+        ]
+      },
+      {
+        "category": "Absolute Volume",
+        "description": "this is a few shot learning example for an NL to DSL. The DSL function in question is AbsVolume(symbol, ContractType, datetime1, datetime2), which computes the highest volume in the date range, specified by datetime1, and datetime2. The first argument is a future symbol (like CL for crude oil, ES for SP 500 etc).  The second argument is the ContractType, which is \"Future\" for futures and \"FutureSpread\" for future spreads. the 3rd argument is the begin date and the 4th argument is the end date, between which the max volume is computed. If no future is specified use \"*\" from the first arugment. For this prompt, assume today is 10/11/2024 ",
+        "prompt": "Which  SP 500 Future traded the most between 7/14/2024 and 7/10/2024",
+        "examples": [
+          {
+            "natural_language": "Which SP Future traded the most between 7/14/2024 and 7/10/2024",
+            "dsl": "AbsVolume(\"ES\", \"Future\", 7/14/2024, 7/10/2024)"
+          },
+          {
+            "natural_language": "Which Gold Spread traded the most between Today and Sept 20th",
+            "dsl": "AbsVolume(\"GC\", \"FutureSpread\", 10/1/2024, 9/20/2024)"
+          },
+          {
+            "natural_language": "Which 10 year Spread traded the highest volume last week?",
+            "dsl": "AbsVolume(\"ZN\", \"FutureSpread\", 9/27/2024, 9/23/2024)"
+          },
+          {
+            "natural_language": "Which Corn Spread traded the most today?",
+            "dsl": "AbsVolume(\"ZC\", \"FutureSpread\", 10/1/2024, 10/1/2024)"
+          },
+          {
+            "natural_language": "Which Gold Future traded the most today?",
+            "dsl": "AbsVolume(\"GC\", \"Future\", 9/30/2024, 9/30/2024)"
+          }
+        ]
+      },
+      {
+        "category": "Roll",
+        "description": "This is a few shot learning example for an NL to DSL. The DSL function in question is FuturesRoll(symbol) which computes the rollover date for a future, ie when the highest volume swtches from one expiration to the next one. ",
+        "prompt": "When does the roll typically occur In  SP 500?",
+        "examples": [
+          {
+            "natural_language": "When does the roll typically occur In CL?",
+            "dsl": "FuturesRoll(\"CL\")"
+          },
+          {
+            "natural_language": "When does the roll typically occur for 10 year treasury?",
+            "dsl": "FuturesRoll(\"ZN\")"
+          },
+          {
+            "natural_language": "When does the roll usually occur for Gold Future",
+            "dsl": "FuturesRoll(\"GC\")"
+          },
+          {
+            "natural_language": "At what date does the future rollover happens for SP500?",
+            "dsl": "FuturesRoll(\"ES\")"
+          },
+          {
+            "natural_language": "When does the future rollover happens for the Corn Future?",
+            "dsl": "FuturesRoll(\"ZC\")"
+          }
+        ]
+      },
+      {
+        "category": "Daily Move",
+        "description": "This is a few shot learning example for an NL to DSL. ",
+        "prompt": "How often did Sp 500 Future settle + or - 10% over the previous day?",
+        "examples": [
+          {
+            "natural_language": "List all the days when the SP 500 closed + or -5% from the previous day's close",
+            "dsl": "ColDiff(filteredOHLCV(\"ES\"),\"ts_event\", -1, \"close\", \"close\", \"close_diff\", 5)"
+          },
+          {
+            "natural_language": "List all the days when the Crude Oil closed + or -2.5% from the previous day's close",
+            "dsl": "ColDiff(filteredOHLCV(\"CL\"),\"ts_event\", -1, \"close\", \"close\", \"close_diff\", 2.5)"
+          },
+          {
+            "natural_language": "Show me all the days when the Ten year Treaury Future opened + or -1% from the previous day's open",
+            "dsl": "ColDiff(filteredOHLCV(\"ZN\"),\"ts_event\", -1, \"open\", \"open\", \"open_diff\", 1.0)"
+          },
+          {
+            "natural_language": "List all the trading days when the Silver Future opened + or -1% from the previous day's close",
+            "dsl": "ColDiff(filteredOHLCV(\"SI\"),\"ts_event\", -1, \"open\", \"open\", \"open_diff\", 1.0)"
+          }
+        ]
+      },
+      {
+        "category": "Average Volume",
+        "description": "This is a few shot learning example for an NL to DSL. For the purpose of this few shot prompts, lets assume today is 10/3/2024 ",
+        "prompt": "When does the most open interest typically shift from one contract to another in SP 500 Future?",
+        "examples": [
+          {
+            "natural_language": "When does the most open interest typically shift from one contract to another in e-minis? ",
+            "dsl": "FuturesRoll(\"ES\")"
+          },
+          {
+            "natural_language": "Usually when does the open interest move from one contract to another in WTI?",
+            "dsl": "FuturesRoll(\"CL\")"
+          }
+        ]
+      },
+      {
+        "category": "Events",
+        "description": "This is a few shot learning example for an NL to DSL. In the NL prompt the user is asking to show the price of a future (treasury future, or SP 500 future etc) around an event type. The event type could be \"FOMC Announcement\" or \"Storage numbers\" etc. The DSL is as follows: EventsPriceAction(future_name, event_type, pre_event_window, post_event_window, number_of_events) first arg is the name of the future, (ex ES or ZN etc), the second argument is the event type, for example, \"FOMC announcement\", the 3rd argument is a time window for how much pre-event data to include in the end result, the 4th argument is how much post-event data to include in the end result. Both are strings of the form \"5d\" for 5 days, \"5h\" for 5 hours and \"120m\" for 120 minutes and so on. Usually, only one of the event periods is provided. Set the other to be half of what the user provided. The last argument is the number of events to consider. So if it’s 7, that means the most recent 7 events. If the user does not specify that, pass a large number.",
+        "prompt": "What did the volume and price of SP500 Future do following the release of the last 10 FOMC rates?",
+        "examples": [
+          {
+            "natural_language": "How did SP 500 Future do in the 5 days following the FOMC announcements?",
+            "dsl": "EventsPriceAction(\"ES\", \"FOMC Announcement\", \"2d\", \"5d\", 10)"
+          },
+          {
+            "natural_language": "How did the price of Treasury Future evolve in the 2 days following the Job Reports?",
+            "dsl": "EventsPriceAction(\"ZN\", \"Job Reports\", \"1d\", \"2d\", 10)"
+          },
+          {
+            "natural_language": "Show me the price movement and volume of Crude Oil futures during the 5-minute window following the last 5 OPEC meetings.",
+            "dsl": "EventsPriceAction(\"CL\", \"OPEC Meeting\", \"2m\", \"5m\", 5)"
+          },
+          {
+            "natural_language": "Show me the price movement and volume of SP 500 futures preceding the 10-minute window of the last 5 OPEC meetings.",
+            "dsl": "EventsPriceAction(\"CL\", \"OPEC Meeting\", \"10m\", \"5m\", 5)"
+          }
+        ]
+      },
+      {
+        "category": "Indicator, Standard Deviation",
+        "MulitLineDescription": [
+          "This is a few shot learning example for an NL to DSL. In the NL prompt the user is asking to show all the days when ",
+          "the price of a future (treasury future, or SP 500 future etc) moved more than m times the N days standard deviation, ",
+          "where both m and N are provided. For example, the DSL for \"Using a 90d standard deviation, give me the dates and ",
+          "amounts of the last 4+ standard deviation moves in WTI futures\" would be: ",
+          "r=standardDeviation(\"ES\", \"90d\")\n",
+          "r=math(r, \"abs_daily_change = Abs(close - open) comp = std_dev * 4\")\n",
+          "r=filter(r,\"abs_daily_change >= comp\"). ",
+          "The first line, creates a stream for StandardDeviation, based on the number of days, for example 90d standard deviation, ",
+          "and assigned it to r, the second line takes r from the first line and passes it to a math stream that creates 2 new columns, ",
+          "abs_daily_change and comp. comp is the factor (4, from 4+ in the prompt) times standard deviation. The third line filters ",
+          "(like a sql where) of rows where abs_daily_change >= comp"
+        ],
+        "prompt": "Using a 90d standard deviation, give me the dates and amounts of the last 1+ standard deviation moves in SP  futures ?",
+        "examples": [
+          {
+            "natural_language": "Using a 90d standard deviation, give me the dates and amounts of the last 1+ standard deviation moves in WTI futures ?",
+            "dsl": "r=standardDeviation(\"CL\", \"90d\")\nr=math(r, \"abs_daily_change = Abs(close - open) comp = std_dev * 1.0\")\nr=filter(r,\"abs_daily_change >= comp\")"
+          },
+          {
+            "natural_language": "Show me the times the SP Futures moved twice the 60 day standard deviation",
+            "dsl": "r=standardDeviation(\"ES\", \"60d\")\nr=math(r, \"abs_daily_change = Abs(close - open) comp = std_dev * 2.0\")\nr=filter(r,\"abs_daily_change >= comp\")"
+          },
+          {
+            "natural_language": "List all the days the Gold Futures moved 1.5 times the 45 day standard deviation",
+            "dsl": "r=standardDeviation(\"GC\", \"45d\")\nr=math(r, \"abs_daily_change = Abs(close - open) comp = std_dev * 1.5\")\nr=filter(r,\"abs_daily_change >= comp\")"
+          },
+          {
+            "natural_language": "List all the days the 10 year treasury future moved more than twice the 1 month standard deviation",
+            "dsl": "r=standardDeviation(\"ZN\", \"30d\")\nr=math(r, \"abs_daily_change = Abs(close - open) comp = std_dev * 2.0\")\nr=filter(r,\"abs_daily_change >= comp\")"
+          }
+        ]
+      }
+    ]
+  },
   "SparkApi": {
     "IsEnabled": false,
     "Credentials": {
@@ -77,7 +435,7 @@
         {
           "TagType": ".*",
           "Assembly": "DataStreamCore",
-         "Class": "bts.PubsubServerGenericDataHandler"
+          "Class": "bts.PubsubServerGenericDataHandler"
         }
       ]
     }

src/DataStreamSuite/DataStream/DataStream.csproj

@@ -18,8 +18,11 @@
 
   <ItemGroup>
     <PackageReference Include="HtmlAgilityPack" Version="1.11.57" />
+    <PackageReference Include="Microsoft.Data.Analysis" Version="0.21.1" />
     <PackageReference Include="MongoDB.Bson" Version="2.23.1" />
     <PackageReference Include="MongoDB.Driver" Version="2.23.1" />
+    <PackageReference Include="Parquet.Net" Version="5.0.1" />
+    <PackageReference Include="System.Data.SQLite" Version="1.0.118" />
   </ItemGroup>
 
 </Project>

src/DataStreamSuite/DataStream/DataStreamExtensions.cs

@@ -2,15 +2,27 @@ namespace Bts.DataStream;
 
 using BCG;
 
+using bts.api.core;
+
+using Bts.DataStreamCore;
+
+using Microsoft.Extensions.Logging;
+
 using Spark.Core.Disposables;
 
 using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Linq;
 using System.Reactive.Linq;
 
 public static class DataStreamExtensions {
    public static ViewStream AsViewStream(this IObservable<DataPacket> source) =>
       source as ViewStream ?? new ObservableViewStream(source);
 
+   public static ViewStream AsViewStream<T>(this IObservable<IDataPacket<T>> source, string? name = null, ILogger? logger = null) where T : IBlueRow =>
+      source as ViewStream ?? new ObservableViewStream(source.Select(dp => new DataPacket(dp.newRows, dp.addedRows, dp.deletedRows)), name, logger);
+
    public static IObservable<DataPacket> AsObservable(this ViewStream source) {
       return source as IObservable<DataPacket> ?? Observable.Create<DataPacket>(Subscribe);
 
@@ -23,4 +35,100 @@ public static class DataStreamExtensions {
                $"Expected DataPacket, got {newValue?.GetType()}", nameof(newValue)));
       }
    }
+
+   public static IObservable<IDataPacket<T>> SelectDataPacket<T>(this IObservable<IDataPacket> source, Func<IBlueRow, T> selector) where T : IBlueRow =>
+      source.Select(dp => {
+         dp.get_new_added_deleted(out var newRows, out var addedRows, out var deletedRows);
+         return new DataPacket<T>(CastRows(newRows), CastRows(addedRows), CastRows(deletedRows));
+
+         IBlueRowCollection<T> CastRows(ICollection rows) =>
+            new BlueTableSelectorWrapper<T>(
+               rows as IBlueRowCollection ?? new BlueRowCollection(rows, dp.Schema), selector);
+      });
+
+   public static IObservable<TableUpdate<IBlueRow, int>> AsTables(this IObservable<IDataPacket> source) =>
+      source.AsTables<IBlueRow, int>(r => r, r => r.Id());
+
+   public static IObservable<TableUpdate<T, int>> AsTables<T>(this IObservable<IDataPacket> source, Func<IBlueRow, T> valueSelector) where T : IBlueRow =>
+      source.AsTables<T, int>(valueSelector, r => r.Id());
+
+   public static IObservable<TableUpdate<T, int>> AsTables<T>(
+      this IObservable<IDataPacket> source, Func<IBlueRow, T> valueSelector, Func<T, int> keySelector) =>
+      source.AsTables<T, int>(valueSelector, keySelector);
+
+   public static IObservable<TableUpdate<T, TKey>> AsTables<T, TKey>(
+      this IObservable<IDataPacket> source, Func<IBlueRow, T> valueSelector, Func<T, TKey> keySelector, Func<IBlueRow, TKey>? rowKeySelector = null) where TKey : IEquatable<TKey> {
+      rowKeySelector ??= r => keySelector(valueSelector(r));
+      return source.Scan(TableUpdate<T, TKey>.mk_empty(keySelector), (res, dp) => {
+         var bd = res.to_builder();
+         dp.get_new_added_deleted(out var newRows, out var addedRows, out var deletedRows);
+         foreach (IBlueRow r in addedRows) {
+            bd.add_or_update(valueSelector(r));
+         }
+         foreach (IBlueRow r in deletedRows) {
+            bd.remove(rowKeySelector(r));
+         }
+         return bd.to_table();
+      });
+   }
+
+   public static IObservable<IDataPacket<T>> AsDataPackets<T>(this IObservable<TableUpdate<T, int>> source, IBlueTableSchema schema) where T : IBlueRow =>
+      source.Scan(
+         (Table: new BlueTableImmutable<T>(schema), Dp: (IDataPacket<T>)DataPacket<T>.Empty(schema)),
+         (res, table) => {
+            var bd = res.Table.ToDataPacketBuilder();
+            foreach (var r in table.removed) {
+               bd.Remove(r.Id());
+            }
+            foreach (var r in table.added) {
+               bd.AddOrUpdate(r);
+            }
+            var resDp = bd.ToDataPacketT(out var newRows, out var addedRows, out var deletedRows);
+            return (newRows, resDp);
+         })
+         .Where(t => t.Dp.addedRows.Any() || t.Dp.deletedRows.Any())
+         .Select(t => t.Dp);
+
+   public static ViewStream AsViewStream<T>(this IObservable<TableUpdate<T, int>> source, IBlueTableSchema schema) where T : IBlueRow =>
+      source.AsDataPackets(schema).AsViewStream();
+
+   public static int ResolveIndices(this IBlueTableSchema schema, string name) {
+      var indices = ResolveIndices(schema, new[] { name });
+      return indices[0];
+   }
+
+   public static (int, int) ResolveIndices(this IBlueTableSchema schema, string name1, string name2) {
+      var indices = ResolveIndices(schema, new[] { name1, name2 });
+      return (indices[0], indices[1]);
+   }
+
+   public static (int, int, int) ResolveIndices(this IBlueTableSchema schema, string name1, string name2, string name3) {
+      var indices = ResolveIndices(schema, new[] { name1, name2, name3 });
+      return (indices[0], indices[1], indices[2]);
+   }
+
+   public static (int, int, int, int) ResolveIndices(this IBlueTableSchema schema, string name1, string name2, string name3, string name4) {
+      var indices = ResolveIndices(schema, new[] { name1, name2, name3, name4 });
+      return (indices[0], indices[1], indices[2], indices[3]);
+   }
+
+   public static int[] ResolveIndices(this IBlueTableSchema schema, string[] columnNames) {
+      var columns = schema.columnInfo;
+      var indices = new int[columnNames.Length];
+      int nResolved = 0;
+      for (int i = 0; i < columnNames.Length; i++) {
+         int origNResolved = nResolved;
+         for (int j = 0; j < columns.Length; j++) {
+            if (columns[j].colName == columnNames[i]) {
+               indices[i] = j;
+               nResolved++;
+               break;
+            }
+         }
+         if (nResolved == origNResolved) {
+            throw new ArgumentException($"No column named '{columnNames[i]}'", nameof(schema));
+         }
+      }
+      return indices;
+   }
 }

src/DataStreamSuite/DataStream/Misc/DSLStream.cs

@@ -0,0 +1,33 @@
+namespace bts.SilverPine {
+   using System;
+   using BCG;
+   using System.Collections.Generic;
+   using Bts.DataStreamExpressions;
+   using Bts.DataStream;
+
+   // All this does is take in a dsl expression, parse it and fire the data that come out of running it.
+   // So for example, if an expresison is passed 
+   [DataStreamUserRespresentation("dsl")]
+   public class DSLStream : ViewStream {
+      private readonly string _dsl;
+      public DSLStream(string dsl, IDataStreamFactory streamFactory) {
+         Log.Gen.info($"DSLStream.cons dsl: {dsl}");
+         _dsl = dsl;
+         var rs = build_stream(streamFactory, dsl);
+         rs = rs ?? new LiteralStream("error", $"msg:invalid dsl");
+         mySetRealSource(rs);
+      }
+
+      static ViewStream? build_stream(IDataStreamFactory streamFactory, string stream_expression) {
+         try {
+            var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression);
+            var rv = exp.getView(new Dictionary<string, StatementExpression>(0), streamFactory);
+            return rv;
+
+         } catch (Exception ex) {
+            Log.Gen.error($"Could not build stream: {stream_expression} ex: {ex}");
+            return null;
+         }
+      }
+   }
+}

src/DataStreamSuite/DataStream/ObservableViewStream.cs

@@ -4,14 +4,21 @@ using BCG;
 
 using bts;
 
+using Microsoft.Extensions.Logging;
+
 using System;
 
 public class ObservableViewStream : ViewStream, IObservable<DataPacket>, IDisposable {
    readonly IObservable<DataPacket> source;
+   readonly string? name;
+   readonly ILogger? logger;
    IDisposable? subscription;
 
-   public ObservableViewStream(IObservable<DataPacket> source) =>
+   public ObservableViewStream(IObservable<DataPacket> source, string? name = null, ILogger? logger = null) {
       this.source = source;
+      this.name = name;
+      this.logger = logger;
+   }
 
    public void Dispose() {
       GC.SuppressFinalize(this);
@@ -23,7 +30,13 @@ public class ObservableViewStream : ViewStream, IObservable<DataPacket>, IDispos
 
    protected override void myOnFirstSubscriber() {
       Log.isnull(subscription);
-      subscription = source.Subscribe(myFireEvent);
+      subscription = source.Subscribe(myFireEvent, OnError, OnCompleted);
+
+      void OnError(Exception exception) =>
+         logger?.LogError("Error in ObservableViewStream. name: {Name}, error: {Exception}", name, exception);
+
+      void OnCompleted() =>
+         logger?.LogInformation("ObservableViewStream completed. name: {Name}", name);
    }
 
    protected override void myOnLastSubscriber() {

src/DataStreamSuite/DataStream/Offloading/RemoteQueryManager.cs

@@ -34,12 +34,12 @@ namespace BCG {
             view_publisher = view_publisher_;
          }
       }
-      static ViewStream? build_stream(string stream_expression)
+      static ViewStream? build_stream(IDataStreamFactory streamFactory, string stream_expression)
       {
          try
          {
             var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression);
-            var rv = exp.getView(new Dictionary<string, StatementExpression>(0));
+            var rv = exp.getView(new Dictionary<string, StatementExpression>(0), streamFactory);
             return rv;
 
          } catch (Exception ex) {
@@ -51,15 +51,17 @@ namespace BCG {
          _stream_pub_map = new Dictionary<string, StreamPubUnit>();
       readonly IPubTableBroker _broker;
       readonly IManagedSubscriber _sub;
+      readonly IDataStreamFactory streamFactory;
       readonly object syncRoot = new();
       IDisposable? cloudStreamsSub;
       IDisposable? remoteRequestsSub;
       bool isDisposed;
 
 
-      public RemoteQueryManager(IManagedSubscriber managedSubscriber, IPubTableBroker pubTableBroker) {
+      public RemoteQueryManager(IManagedSubscriber managedSubscriber, IPubTableBroker pubTableBroker, IDataStreamFactory streamFactory) {
          _sub = managedSubscriber;
          _broker = pubTableBroker;
+         this.streamFactory = streamFactory;
       }
 
       public Task StartAsync(CancellationToken cancellationToken) {
@@ -102,7 +104,11 @@ namespace BCG {
 
       private void _spawn_remote_stream(string name, string stream_expression, bool persist) {
          bts.Log.Gen.info($"CloudStreamManager.assign name: {name} stream_expression: {stream_expression}");
-         ViewStream stream = build_stream(stream_expression);
+         var stream = build_stream(streamFactory, stream_expression);
+         if (stream is null) {
+            Log.DataFlow.error($"Failed to spawn remote stream. name: {name}; expression: {stream_expression}");
+            return;
+         }
          _expose_to_ws(name, stream);
          if (persist) {
             stream = new PersistorStream(stream, name, "C:\\Temp\\", 1000000);   // 10MBs

src/DataStreamSuite/DataStream/Offloading/RemoteQueryStream.cs

@@ -25,12 +25,12 @@ namespace BCG {
          byte[] rem_req_bytes = rem_req.serialize();
          _publisher_core.publish("System", "RemoteRequest", rem_req_bytes);
          _source_str = new TableStream(_name);
-         _source_str.onChange += new ChangeDelegate(_handle_data);
+         _source_str.onChange += _handle_data;
       }
 
       protected override void myOnLastSubscriber() {
          //maybe there should be a way to turn remote requests off ?
-         _source_str.onChange -= new ChangeDelegate(_handle_data);
+         _source_str.onChange -= _handle_data;
       }
 
       static string get_expression(params string[] lines) {

src/DataStreamSuite/DataStream/Primitives/Count/CountStream.cs

@@ -21,13 +21,11 @@ namespace BCG
          : this (new TableStream (tableName))
       {}
 
-      protected override void myOnFirstSubscriber () {
-         myVS.onChange += new ChangeDelegate(myHandleChanges);
-      }
-      protected override void myOnLastSubscriber () {
-         myVS.onChange -= new ChangeDelegate(myHandleChanges);
-      }
+      protected override void myOnFirstSubscriber () =>
+         myVS.onChange += myHandleChanges;
 
+      protected override void myOnLastSubscriber () =>
+         myVS.onChange -= myHandleChanges;
 
       private void myHandleChanges(object newValue)
       {

src/DataStreamSuite/DataStream/Primitives/Crushing/CrushingCollectorStream.cs

@@ -30,7 +30,7 @@ namespace BCG
 
         protected override void myOnFirstSubscriber()
         {
-            myToCrush.onChange += new ChangeDelegate(myToCrush_onChange);
+            myToCrush.onChange += myToCrush_onChange;
         }
 
         protected override void myOnLastSubscriber()

src/DataStreamSuite/DataStream/Primitives/Crushing/CrushingStream.cs

@@ -15,11 +15,11 @@ namespace BCG
             
             //set up an anonymous delegate for each of the streams
             //we do this so that we can unsubscribe
-            myDelegates = new ChangeDelegate[myToCrush.Length];
+            myDelegates = new DataStreamChangeDelegate[myToCrush.Length];
             for (int i = 0; i < myToCrush.Length; ++i)
             {
                 int tempI = i;
-                myDelegates[tempI] = new ChangeDelegate((o) => { myHandleChange(tempI, ((DataPacket)o).newRows); });
+                myDelegates[tempI] = new((o) => { myHandleChange(tempI, ((DataPacket)o).newRows); });
             }
         }
 
@@ -27,7 +27,7 @@ namespace BCG
         private ViewStream[] myToCrush;
         private ITableCrusher myCrusher;
 
-        private ChangeDelegate[] myDelegates;
+        private DataStreamChangeDelegate[] myDelegates;
 
         protected override void myOnFirstSubscriber()
         {

src/DataStreamSuite/DataStream/Primitives/Derivative/DerivativeStream.cs

@@ -28,7 +28,7 @@ namespace BCG
 
       protected override void myOnFirstSubscriber()
       {
-         myDataSource.onChange += new ChangeDelegate(myDataSource_onChange);
+         myDataSource.onChange += myDataSource_onChange;
       }
 
       protected override void myOnLastSubscriber()

src/DataStreamSuite/DataStream/Primitives/FileStreams/SaverStream.cs

@@ -18,7 +18,8 @@ namespace bts {
          _filename = filename;
       }
 
-      protected override void myOnFirstSubscriber() { _stream.onChange += new ChangeDelegate(_handle_changes); }
+      protected override void myOnFirstSubscriber() =>
+         _stream.onChange += _handle_changes;
 
       void _handle_changes(object dp_obj) {
          DataPacket dp = dp_obj as DataPacket;
@@ -26,7 +27,8 @@ namespace bts {
          myFireEvent(dp_obj);
       }
 
-      protected override void myOnLastSubscriber() { _stream.onChange -= new ChangeDelegate(_handle_changes); }
+      protected override void myOnLastSubscriber() =>
+         _stream.onChange -= _handle_changes;
 
       protected void _save(DataPacket dp) {
          StreamIOMessage.CreateMessage cr_msg = new StreamIOMessage.CreateMessage(dp.newRows);

src/DataStreamSuite/DataStream/Primitives/Filter/FilterStream.cs

@@ -37,12 +37,12 @@ namespace BCG.StreamWorld.Addition
 
       protected override void myOnFirstSubscriber()
       {
-         myBaseStream.onChange += new ChangeDelegate(myHandleChanges);
+         myBaseStream.onChange += myHandleChanges;
       }
 
       protected override void myOnLastSubscriber()
       {
-         myBaseStream.onChange -= new ChangeDelegate(myHandleChanges);
+         myBaseStream.onChange -= myHandleChanges;
          myBaseTable.clear();
          myStateTable.clear();
       }
@@ -228,13 +228,13 @@ namespace BCG
       #endregion
 
       #region overridden methods
-      protected override void myOnFirstSubscriber()
-      { myParentStream.onChange += new ChangeDelegate(myHandleCallbacks); }
+      protected override void myOnFirstSubscriber() =>
+         myParentStream.onChange += myHandleCallbacks;
 
       protected override void myOnLastSubscriber()
       {
          myLastFiredRows = null;
-         myParentStream.onChange -= new ChangeDelegate(myHandleCallbacks);
+         myParentStream.onChange -= myHandleCallbacks;
       }
 
       protected override void myFireEvent(object o)

src/DataStreamSuite/DataStream/Primitives/Filter/NullFilterStream.cs

@@ -0,0 +1,24 @@
+namespace bts.SilverPine {
+   using BCG;
+
+   using System;
+
+   public class NullFilterStream : ViewStream {
+      public NullFilterStream(ViewStream parent_str, string col_name)
+      : this(parent_str, col_name, true) { }
+      public NullFilterStream(ViewStream parent_str, string col_name, bool filter_out_doublenans = false) {
+         ViewStream rs = new FilterStream(parent_str,
+         r => {
+            bool is_dbnull = r.is_null(col_name);
+            if (is_dbnull) {
+               return false;
+            }
+            if (r.schema[col_name].type_code() == TypeCode.Double) {
+               return !Double.IsNaN(r.get<double>(col_name));
+            }
+            return true;
+         });
+         mySetRealSource(rs);
+      }
+   }
+}

src/DataStreamSuite/DataStream/Primitives/Freeze/FreezeDataStream.cs

@@ -36,11 +36,11 @@ namespace BCG
          get{return myIsFrozen;}
       }
 
-      protected override void myOnFirstSubscriber ()
-      {myStreamToFreeze.onChange += new DataStream.ChangeDelegate (myCallBack);}
+      protected override void myOnFirstSubscriber () =>
+         myStreamToFreeze.onChange += myCallBack;
 
-      protected override void myOnLastSubscriber ()
-      {myStreamToFreeze.onChange -= new DataStream.ChangeDelegate (myCallBack);}
+      protected override void myOnLastSubscriber () =>
+         myStreamToFreeze.onChange -= myCallBack;
 
       internal void myCallBack(object o)
       {

src/DataStreamSuite/DataStream/Primitives/Fuzzy/FuzzyJoinOverTimeStream.cs

@@ -96,15 +96,15 @@ namespace BCG
 
       protected override void myOnFirstSubscriber()
       {
-         this.myLeftSource.onChange += new ChangeDelegate(myLeft_onChange);
-         this.myRightSource.onChange += new ChangeDelegate(myRight_onChange);
+         myLeftSource.onChange += myLeft_onChange;
+         myRightSource.onChange += myRight_onChange;
          this.myVsd.addWorker(this);
       }
 
       protected override void myOnLastSubscriber()
       {
-         this.myLeftSource.onChange -= new ChangeDelegate(myLeft_onChange);
-         this.myRightSource.onChange -= new ChangeDelegate(myRight_onChange);
+         myLeftSource.onChange -= myLeft_onChange;
+         myRightSource.onChange -= myRight_onChange;
          this.myVsd.removeWorker(this);
       }
 

src/DataStreamSuite/DataStream/Primitives/GroupBy/AverageGroupsStream.cs

@@ -1,102 +1,102 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
+// using System;
+// using System.Collections.Generic;
+// using System.Linq;
+// using System.Text;
 
-namespace BCG
-{
+// namespace BCG
+// {
    
-    public class AverageGroupsStream : ViewStream
-    {
-        public AverageGroupsStream(ViewStream input, string colToAverage, string keyColumn)
-        {
+//     public class AverageGroupsStream : ViewStream
+//     {
+//         public AverageGroupsStream(ViewStream input, string colToAverage, string keyColumn)
+//         {
             
-            myColToAverage = colToAverage;
-            myInput = input;
-            myKeyColumn = keyColumn;
-        }
+//             myColToAverage = colToAverage;
+//             myInput = input;
+//             myKeyColumn = keyColumn;
+//         }
 
-        private ViewStream myInput;
-        private string myColToAverage;
-        private BlueTableSchema mySchema;
-        private string myKeyColumn;
+//         private ViewStream myInput;
+//         private string myColToAverage;
+//         private BlueTableSchema mySchema;
+//         private string myKeyColumn;
 
-        protected override void myOnFirstSubscriber()
-        {
-            myInput.onChange += new ChangeDelegate(myInput_onChange);
-        }
-        protected override void myOnLastSubscriber()
-        {
-            myInput.onChange -= myInput_onChange;
-        }
-        private void myInput_onChange(object newValue)
-        {
-            if (mySchema == null)
-                myBuildSchema(((DataPacket)newValue).schema);
+//         protected override void myOnFirstSubscriber()
+//         {
+//             myInput.onChange += new ChangeDelegate(myInput_onChange);
+//         }
+//         protected override void myOnLastSubscriber()
+//         {
+//             myInput.onChange -= myInput_onChange;
+//         }
+//         private void myInput_onChange(object newValue)
+//         {
+//             if (mySchema == null)
+//                 myBuildSchema(((DataPacket)newValue).schema);
 
-            IBlueRowCollection newRows = myEvaluate((DataPacket)newValue);
+//             IBlueRowCollection newRows = myEvaluate((DataPacket)newValue);
 
-            if (myLastValue == null)
-                myFireEvent(new DataPacket(newRows, newRows, new BlueTable(mySchema)));
-            else
-            {
-                IBlueRowCollection added, deleted;
-                BlueDataUtility.computeDifferences(newRows, ((DataPacket) myLastValue).newRows, out deleted, out added);
-                myFireEvent(new DataPacket(newRows, added, deleted));
-            }
-        }
+//             if (myLastValue == null)
+//                 myFireEvent(new DataPacket(newRows, newRows, new BlueTable(mySchema)));
+//             else
+//             {
+//                 IBlueRowCollection added, deleted;
+//                 BlueDataUtility.computeDifferences(newRows, ((DataPacket) myLastValue).newRows, out deleted, out added);
+//                 myFireEvent(new DataPacket(newRows, added, deleted));
+//             }
+//         }
 
-        private void myBuildSchema(IBlueTableSchema schema)
-        {
-            List<ColumnInfo> columns = new List<ColumnInfo>();
+//         private void myBuildSchema(IBlueTableSchema schema)
+//         {
+//             List<ColumnInfo> columns = new List<ColumnInfo>();
 
-            foreach (ColumnInfo ci in schema.columnInfo)
-                columns.Add(ci);
+//             foreach (ColumnInfo ci in schema.columnInfo)
+//                 columns.Add(ci);
 
-            columns.Add(new ColumnInfo("average_"+myColToAverage, columns.Count, typeof(double)));
-            mySchema = new BlueTableSchema("average_of_" + schema.tableName, columns.ToArray());
-        }
+//             columns.Add(new ColumnInfo("average_"+myColToAverage, columns.Count, typeof(double)));
+//             mySchema = new BlueTableSchema("average_of_" + schema.tableName, columns.ToArray());
+//         }
 
 
 
-        private IBlueRowCollection myEvaluate(DataPacket packet)
-        {
+//         private IBlueRowCollection myEvaluate(DataPacket packet)
+//         {
          
-            IDictionary<string, double> colToRunningTotal = new Dictionary<string, double>();
-            IDictionary<string, int> colToCount = new Dictionary<string, int>();
-            IDictionary<string, IBlueRow> colToLastRow = new Dictionary<string, IBlueRow>();
+//             IDictionary<string, double> colToRunningTotal = new Dictionary<string, double>();
+//             IDictionary<string, int> colToCount = new Dictionary<string, int>();
+//             IDictionary<string, IBlueRow> colToLastRow = new Dictionary<string, IBlueRow>();
 
-            foreach (IBlueRow r in packet.newRows)
-            {
-                double thisVal = Convert.ToDouble(r[myColToAverage]);
+//             foreach (IBlueRow r in packet.newRows)
+//             {
+//                 double thisVal = Convert.ToDouble(r[myColToAverage]);
 
-                string keyVal = r[myKeyColumn].ToString();
-                if (!colToCount.ContainsKey(keyVal))
-                {
-                    colToRunningTotal[keyVal] = thisVal;
-                    colToCount[keyVal] = 1;
-                }
-                else
-                {
-                    colToRunningTotal[keyVal] += thisVal;
-                    colToCount[keyVal] += 1;
-                }
-                colToLastRow[keyVal] = r;
-            }
+//                 string keyVal = r[myKeyColumn].ToString();
+//                 if (!colToCount.ContainsKey(keyVal))
+//                 {
+//                     colToRunningTotal[keyVal] = thisVal;
+//                     colToCount[keyVal] = 1;
+//                 }
+//                 else
+//                 {
+//                     colToRunningTotal[keyVal] += thisVal;
+//                     colToCount[keyVal] += 1;
+//                 }
+//                 colToLastRow[keyVal] = r;
+//             }
 
-            BlueTable result = new BlueTable(mySchema);
-            foreach (string key in colToCount.Keys)
-            {
-                object[] values = new object[mySchema.columnInfo.Length];
+//             BlueTable result = new BlueTable(mySchema);
+//             foreach (string key in colToCount.Keys)
+//             {
+//                 object[] values = new object[mySchema.columnInfo.Length];
 
-                colToLastRow[key].CopyTo(values, 0);
-                values[values.Length - 1] = packet.newRows.Count > 0 ? colToRunningTotal[key] / packet.newRows.Count : 0.0;
-                result.addRow(new BlueRow(mySchema, values));
-            }
+//                 colToLastRow[key].CopyTo(values, 0);
+//                 values[values.Length - 1] = packet.newRows.Count > 0 ? colToRunningTotal[key] / packet.newRows.Count : 0.0;
+//                 result.addRow(new BlueRow(mySchema, values));
+//             }
 
-            return result;
-        }
+//             return result;
+//         }
 
       
-    }
-}
+//     }
+// }

src/DataStreamSuite/DataStream/Primitives/GroupBy/AverageNonIncGroupsStream.cs

@@ -0,0 +1,31 @@
+namespace bts.SilverPine {
+   using BCG;
+
+   using System;
+   using System.Collections.Generic;
+   using System.Linq;
+
+   [DataStreamUserRespresentation("Average")]
+   public class AverageNonIncGroupsStream : ViewStream {
+      public AverageNonIncGroupsStream(ViewStream parent_stream, string inp_col_name, params string[] group_by_cols) {
+         NonIncGroupByStream rs = new NonIncGroupByStream(parent_stream, () => group_by_cols,
+                                                         (g_rows) => _find_mean(inp_col_name, g_rows));
+         mySetRealSource(rs);
+      }
+
+      private List<IBlueRow> _find_mean(string inp_col_name, List<IBlueRow> g_rows) {
+         IBlueRow inp_row = g_rows.ElementAt(0);
+         IEnumerable<double> inp_vals = BlueDataUtility.get_colvalues<double>(g_rows, inp_col_name);
+         double sum = 0.0;
+         inp_vals.ToList().ForEach(d => sum += d);
+         string out_col_name = $"average_{inp_col_name}";
+         double average = sum / inp_vals.Count();
+         IBlueTableSchema schema_out = inp_row.schema.add_column(TypeCode.Double, out_col_name);
+         MutableBlueRow out_row = new MutableBlueRow(schema_out, inp_row);
+         out_row.setValue(out_col_name, average);
+         List<IBlueRow> rv = new List<IBlueRow>();
+         rv.Add(out_row);
+         return rv;
+      }
+   }
+}

src/DataStreamSuite/DataStream/Primitives/GroupBy/GroupByBundle.cs

@@ -0,0 +1,91 @@
+namespace bts.SilverPine {
+   using BCG;
+
+   using System;
+   using System.Collections.Generic;
+   using System.Linq;
+
+   using ZstdSharp.Unsafe;
+
+   public enum GroupByBundleOp {
+      Mean,
+      Median,
+      Mode,
+      StdDev,
+      Max,
+      Min,
+      First,
+      Last,
+      Sum
+   }
+   public class GroupByBundle {
+      public readonly List<Tuple<string, GroupByBundleOp>> operations;
+
+      public static GroupByBundle parse(string text) {
+         // text is inp_col: op, inp_col: op, inp_col:op
+         List<Tuple<string, GroupByBundleOp>> rv = new List<Tuple<string, GroupByBundleOp>>();
+         string[] calcs = text.Split(',');
+         foreach (string c in calcs) {
+            string[] toks = c.Split(':');
+            string inp_col = toks[0].Trim();
+            string op_txt = toks[1].Trim();
+            GroupByBundleOp op = Enum.Parse<GroupByBundleOp>(op_txt, true);
+            rv.Add(Tuple.Create(inp_col, op));
+         }
+         return new GroupByBundle(rv);
+      }
+
+      public GroupByBundle(List<Tuple<string, GroupByBundleOp>> operations_) {
+         operations = operations_;
+      }
+
+      public static IBlueRow group_by(List<IBlueRow> rows, GroupByBundle parameters) {
+         if (rows.Count() == 0) {
+            return null;
+         }
+
+         IBlueRow ir = rows.ElementAt(0);
+         IBlueTableSchema in_schema = ir.schema;
+         IBlueTableSchema out_schema = in_schema;
+         IDictionary<string, Tuple<string, GroupByBundleOp>> out_col_map =
+            new Dictionary<string, Tuple<string, GroupByBundleOp>>();
+         foreach (var t in parameters.operations) {
+            string out_col_name = $"{t.Item1}_{t.Item2}".ToLower();
+            out_schema = out_schema.add_column(TypeCode.Double, out_col_name);
+            out_col_map[out_col_name] = t;
+         }
+         MutableBlueRow or = new MutableBlueRow(out_schema, ir);
+         foreach (var kv in out_col_map) {
+            string out_col_name = kv.Key;
+            string in_col_name = kv.Value.Item1;
+            GroupByBundleOp op = kv.Value.Item2;
+            IEnumerable<double> col_vals = BlueDataUtility.get_colvalues<double>(rows, in_col_name);
+            double comp_res = compute_op(col_vals, op);
+            or.setValue(out_col_name, comp_res);
+         }
+         return or;
+      }
+
+      public static double compute_op(IEnumerable<double> col_vals, GroupByBundleOp op) {
+         switch (op) {
+            case GroupByBundleOp.Max:
+               return col_vals.Max<double>();
+            case GroupByBundleOp.Min:
+               return col_vals.Min<double>();
+            case GroupByBundleOp.Mean:
+               return col_vals.Sum() / col_vals.Count();
+            case GroupByBundleOp.Sum:
+               return col_vals.Sum();
+            case GroupByBundleOp.First:
+               return col_vals.First();
+            case GroupByBundleOp.Last:
+               return col_vals.Last();
+            default:
+               // TODO
+               Log.assert(false);
+               return Double.NaN;
+         }
+      }
+
+   }
+}

src/DataStreamSuite/DataStream/Primitives/GroupBy/GroupByStream.cs

@@ -175,7 +175,7 @@ namespace BCG {
          //gets resubscribed, we start the process of building data again, and treat the callback as the first 
          //callback.
          myInit();
-         myStream.onChange -= new DataStream.ChangeDelegate(myChangeHandler);
+         myStream.onChange -= myChangeHandler;
       }
 
       protected override void myInit() {

src/DataStreamSuite/DataStream/Primitives/GroupBy/MaxStream.cs

@@ -10,6 +10,6 @@ namespace BCG {
         }
 
         public MaxStream(ViewStream stream, string outputCol)
-           : this(new AddColumnsStream(stream, "fakeCol=0"), outputCol, new string[] { "fakeCol" }) { }
+           : this(new AddColumnsStream(stream, "fake_col: 0"), outputCol, new string[] { "fake_col" }) { }
     }
 }

src/DataStreamSuite/DataStream/Primitives/GroupBy/MethodDrivenStream.cs

@@ -82,12 +82,12 @@ namespace BCG
 
       protected override void myOnFirstSubscriber() 
       {
-         myStream.onChange += new DataStream.ChangeDelegate (myChangeHandler);
+         myStream.onChange += myChangeHandler;
       }
 
       protected override void myOnLastSubscriber () 
       {
-         myStream.onChange -= new DataStream.ChangeDelegate (myChangeHandler);
+         myStream.onChange -= myChangeHandler;
       }
    }
 }

src/DataStreamSuite/DataStream/Primitives/GroupBy/NonIncremental/NonIncGroupByStream.cs

@@ -1,13 +1,40 @@
 using System;
 using System.Collections.Generic;
 using System.Diagnostics;
+using System.Linq;
+using System.Text.Json;
 
 using bts;
+using bts.SilverPine;
 
-namespace BCG
-{
-   public class NonIncGroupByStream : ViewStream
-   {
+namespace BCG {
+   [DataStreamUserRespresentation("aggregate")]
+   public class NonIncGroupByStream : ViewStream {
+      private class DelNonIncGroupByEvaluator : INonIncGroupByEvaluator {
+         Func<string[]> _groupby_cols_func;
+         Func<List<IBlueRow>, List<IBlueRow>> _evaluate_func;
+         public DelNonIncGroupByEvaluator(Func<string[]> groupby_cols_func,
+                                          Func<List<IBlueRow>, List<IBlueRow>> evaluate_func) {
+            _groupby_cols_func = groupby_cols_func;
+            _evaluate_func = evaluate_func;
+         }
+         public IBlueRowCollection evaluate(IDictionary<int, List<IBlueRow>> grouped_rows) {
+            BlueTable rv = null;
+            foreach (var kv in grouped_rows) {
+               List<IBlueRow> g_rows = kv.Value;
+               List<IBlueRow> out_rows = _evaluate_func.Invoke(g_rows);
+               if (out_rows != null && out_rows.Count > 0) {
+                  rv = rv ?? new BlueTable(out_rows.ElementAt(0).schema);
+               }
+               rv.addRows(out_rows);
+            }
+            return rv;
+         }
+         public string[] get_group_by_cols() {
+            return _groupby_cols_func();
+         }
+
+      }
       private ViewStream _main_stream;
       private INonIncGroupByEvaluator _groupby_func;
       private BlueRowHasher _row_hasher;
@@ -15,24 +42,47 @@ namespace BCG
       private Stopwatch _timer = new();
       private int _n_total_fires = 0, _n_expensive_fires = 0;
 
-      public NonIncGroupByStream(ViewStream main_stream, INonIncGroupByEvaluator groupby_func)
-      {
+      public NonIncGroupByStream(ViewStream main_stream, INonIncGroupByEvaluator groupby_func) {
          _main_stream = main_stream;
          _groupby_func = groupby_func;
          _groupby_cols = _groupby_func.get_group_by_cols();
-         _row_hasher = new BlueRowHasher(_groupby_cols.Length);
+         _row_hasher = new BlueRowHasher(_groupby_cols == null ? 0 : _groupby_cols.Length);
       }
 
+      public NonIncGroupByStream(ViewStream main_stream, Func<string[]> groupby_cols_func,
+                                 Func<List<IBlueRow>, List<IBlueRow>> evaluate_func)
+                                 : this(main_stream, new DelNonIncGroupByEvaluator(groupby_cols_func, evaluate_func)) { }
+
+      public NonIncGroupByStream(ViewStream parent_str, string groupby_col, string standard_group_by_text, string optional_sort_col = null) {
+         GroupByBundle std_group_by = GroupByBundle.parse(standard_group_by_text);
+         Func<string[]> groupby_cols_func = () => {
+            return String.IsNullOrEmpty(groupby_col) ? null : new string[] { groupby_col };
+         };
+         Func<List<IBlueRow>, List<IBlueRow>> evaluate_func = (group_rows) => {
+            if (!String.IsNullOrEmpty(optional_sort_col))
+               group_rows.Sort((r1, r2) => BlueDataUtility.compare_rows(r2, r1, optional_sort_col));
+            IBlueRow rv = GroupByBundle.group_by(group_rows, std_group_by);
+            return new IBlueRow[] { rv }.ToList();
+         };
+         ViewStream rs = new NonIncGroupByStream(parent_str, groupby_cols_func, evaluate_func);
+         mySetRealSource(rs);
+      }
+      public NonIncGroupByStream(ViewStream parent_str, string standard_group_by_text) :
+      this(parent_str, String.Empty, standard_group_by_text) {
+      }
+
+
       protected override void myOnFirstSubscriber() { _main_stream.onChange += _main_stream_onchange; }
 
       protected override void myOnLastSubscriber() { _main_stream.onChange -= _main_stream_onchange; }
 
-      private void _main_stream_onchange(object nv)
-      {
+      private void _main_stream_onchange(object nv) {
          DataPacket dp = nv as DataPacket;
          ///grouping together
          _timer.Restart();
-         IDictionary<int, List<IBlueRow>> grouped_rows = BlueDataUtility.group_together (dp.newRows, _groupby_cols, ref _row_hasher);
+         IDictionary<int, List<IBlueRow>> grouped_rows = (_groupby_cols == null || _groupby_cols.Length == 0) ?
+         new Dictionary<int, List<IBlueRow>>() { { 0, dp.newRows.to_list() } } :
+         BlueDataUtility.group_together(dp.newRows, _groupby_cols, ref _row_hasher);
          _timer.Stop();
          var groupElapsedMs = _timer.ElapsedMilliseconds;
          if (groupElapsedMs >= 50) {
@@ -48,14 +98,15 @@ namespace BCG
             return;
          }
          var applyElapsedMs = _timer.ElapsedMilliseconds;
-         _n_total_fires++;         
-         if (applyElapsedMs >= 50)
-         {
+         _n_total_fires++;
+         if (applyElapsedMs >= 50) {
             _n_expensive_fires++;
             Log.DataFlow.info(
                $"Time to evaluate {grouped_rows.Count} groups {_timer.ElapsedMilliseconds}ms, ratio {_n_expensive_fires}/{_n_total_fires}");
-         }         
-         myFireEvent(new DataPacket(total_state, total_state, myLastValue == null ? new BlueTable(total_state.schema) : (myLastValue as DataPacket).newRows));             
+         }
+         // TODO,
+         if (total_state != null)
+            myFireEvent(new DataPacket(total_state, total_state, myLastValue == null ? new BlueTable(total_state.schema) : (myLastValue as DataPacket).newRows));
       }
    }
 }

src/DataStreamSuite/DataStream/Primitives/Join/InternalJoinStream.cs

@@ -31,15 +31,15 @@ namespace BCG
       {
          myLeftOperand.master = this;
          myRightOperand.master = this;
-         myLeftStream.onChange += new DataStream.ChangeDelegate (myLeftOperand.changeHandler);
-         myRightStream.onChange += new DataStream.ChangeDelegate (myRightOperand.changeHandler);
+         myLeftStream.onChange += myLeftOperand.changeHandler;
+         myRightStream.onChange += myRightOperand.changeHandler;
       }
       
       protected override void myOnLastSubscriber () 
       {
          myInit ();
-         myLeftStream.onChange -= new DataStream.ChangeDelegate (myLeftOperand.changeHandler);
-         myRightStream.onChange -= new DataStream.ChangeDelegate (myRightOperand.changeHandler);
+         myLeftStream.onChange -= myLeftOperand.changeHandler;
+         myRightStream.onChange -= myRightOperand.changeHandler;
       }
 
       public JoinOperand getOtherOperand (JoinOperand jo)

src/DataStreamSuite/DataStream/Primitives/Join/JoinStream.cs

@@ -116,7 +116,7 @@ namespace BCG
                IDictionary  map = resolver.getMap ();
                map["id"] = "id";
                myPs = new ProjectStream (myIjs, map);
-               myPs.onChange += new DataStream.ChangeDelegate (myProjectChangeHandler);
+               myPs.onChange += myProjectChangeHandler;
                return;
             }
             else
@@ -129,26 +129,26 @@ namespace BCG
       private void myProjectChangeHandler (object o)
       {
          if (myIjs.myOnChangeDelegate.GetInvocationList().Length > 1)
-            myIjs.onChange -= new DataStream.ChangeDelegate (myInternalJoinChangeHandler);
+            myIjs.onChange -= myInternalJoinChangeHandler;
          myFireEvent ((o as ICloneable).Clone());
          
       }
 
       protected override void myOnFirstSubscriber ()
       {
-         myIjs.onChange += new DataStream.ChangeDelegate (myInternalJoinChangeHandler);
+         myIjs.onChange += myInternalJoinChangeHandler;
       }
 
       protected override void myOnLastSubscriber ()
       {
          // how does this relate to the -= in myProjectChangeHandler?
-         myIjs.onChange -= new DataStream.ChangeDelegate (myInternalJoinChangeHandler);
+         myIjs.onChange -= myInternalJoinChangeHandler;
 
          // if there is a  project after the ijs we need to 
          // unsubscribe to it otherwise references will be kept and 
          // a memory leak will occur
          if (myPs != null)
-            myPs.onChange -= new DataStream.ChangeDelegate (myProjectChangeHandler);
+            myPs.onChange -= myProjectChangeHandler;
          myPs = null;
       }
 

src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinBookkeeper.cs

@@ -1,15 +1,16 @@
 using bts;
+using bts.collections;
 
 using mbts;
 
 using System;
 using System.Collections.Generic;
 using System.Diagnostics;
+using System.Linq;
+using System.Security.Cryptography;
 
-namespace BCG
-{
-   public class NonIncJoinBookkeeper : IDisposable
-   {
+namespace BCG {
+   public class NonIncJoinBookkeeper : IDisposable {
       public Action on_new_data;
       private readonly NonIncJoinSpec _join_spec;
       private readonly int _join_position;
@@ -26,41 +27,71 @@ namespace BCG
 
       private int[] _join_col_indices = null,
                      _data_col_indices = null;
-      
-      public NonIncJoinBookkeeper(NonIncJoinSpec the_join_spec, int join_position, int start_index_in_output, BlueRowHasher row_hasher)
-      {
+
+      public NonIncJoinBookkeeper(NonIncJoinSpec the_join_spec, int join_position, int start_index_in_output, BlueRowHasher row_hasher) {
          _join_spec = the_join_spec;
          _join_position = join_position;
          _start_index_in_output = start_index_in_output;
          _row_hasher = row_hasher;
          _is_left_most = _join_position == 0;
          _is_tag_along = _join_spec.join_col_names.Length == 0;
+      }
+
+      private bool _filled_missing_data = false;
+      private void _fill_missing_data(DataPacket dp) {
+         if (_filled_missing_data)
+            return;
+         Func<string[], bool> null_or_empty = (str_arr) => {
+            if (str_arr == null || str_arr.Length == 0)
+               return true;
+            foreach (string s in str_arr) {
+               if (!String.IsNullOrEmpty(s)) {
+                  return false;
+               }
+            }
+            return true;
+         };
+         Func<string[], string[]> remove_bad_cols = (col_names) => {
+            HashSet<string> schema_col_map = new HashSet<string>(dp.schema.columnNames);
+            List<string> cols_list = new List<string>(col_names);
+            cols_list.RemoveWhere(c => !schema_col_map.Contains(c));
+            return cols_list.ToArray();
+         };
+         if (null_or_empty(_join_spec.data_col_names)) {
+            _join_spec.data_col_names = dp.schema.columnNames;
+         }
+         _join_spec.data_col_names = remove_bad_cols(_join_spec.data_col_names);
+         if (null_or_empty(_join_spec.data_col_names)) {
+            _join_spec.data_col_names = dp.schema.columnNames;
+         }
+         if (null_or_empty(_join_spec.projected_col_names)) {
+            _join_spec.projected_col_names = _join_spec.data_col_names;
+         }
+      }
+
+      public void init() {
          _join_spec.stream.onChange += _on_stream_cb;
       }
 
-      public void _on_stream_cb(object dp_obj)
-      {
+      public void _on_stream_cb(object dp_obj) {
+         Log.Gen.info($"NonIncJoinBookkeeper._on_stream_cb dp_obj: {dp_obj}");
          var timer = Stopwatch.StartNew();
          DataPacket dp = (DataPacket)dp_obj;
-         if (_schema == null)
-         {
+         _fill_missing_data(dp);
+         if (_schema == null) {
             _schema = dp.schema;
             _join_col_indices = BlueDataUtility.get_column_indices(_schema, _join_spec.join_col_names);
             _data_col_indices = BlueDataUtility.get_column_indices(_schema, _join_spec.data_col_names);
          }
 
-         if (_is_left_most)
-         {
+         if (_is_left_most) {
             _row_list.Clear();
             foreach (IBlueRow r in dp.newRows)
-               _row_list.Add(new Pair<int, IBlueRow>(_row_hasher.get_hash(r, _join_col_indices), r));            
-         }
-         else
-         {
+               _row_list.Add(new Pair<int, IBlueRow>(_row_hasher.get_hash(r, _join_col_indices), r));
+         } else {
             if (_is_tag_along)
                _solitary_row = BlueDataUtility.extractRow(dp.newRows, 0);
-            else
-            {
+            else {
                _row_map.Clear();
                foreach (IBlueRow r in dp.newRows)
                   _row_map[_row_hasher.get_hash(r, _join_col_indices)] = r;
@@ -79,31 +110,25 @@ namespace BCG
 
       public bool is_ready { get { return _schema != null; } }
 
-      public void constribute_cols_to_schema(List<ColumnInfo> cis)
-      {
+      public void constribute_cols_to_schema(List<ColumnInfo> cis) {
          for (int i = 0; i < _join_spec.data_col_names.Length; i++)
             cis.Add(new ColumnInfo(_join_spec.projected_col_names[i], cis.Count, _schema[_join_spec.data_col_names[i]].colType));
       }
 
       public List<Pair<int, IBlueRow>> get_list() { return _row_list; }
 
-      public void copy_master_row_to_output_row(int row_index, object[] output_row)
-      {
+      public void copy_master_row_to_output_row(int row_index, object[] output_row) {
          IBlueRow this_row = _row_list[row_index].second;
          BlueDataUtility.copy_to_row(this_row, _data_col_indices, output_row, _start_index_in_output);
       }
 
-      public void copy_matching_row_to_output_row(int row_hash, object[] output_row)
-      {
-         if (_is_tag_along)
-         {
+      public void copy_matching_row_to_output_row(int row_hash, object[] output_row) {
+         if (_is_tag_along) {
             if (_solitary_row != null)
                BlueDataUtility.copy_to_row(_solitary_row, _data_col_indices, output_row, _start_index_in_output);
             else
                copy_dbnull_to_output_row(output_row);
-         }
-         else
-         {
+         } else {
             IBlueRow this_row = null;
             if (_row_map.TryGetValue(row_hash, out this_row))
                BlueDataUtility.copy_to_row(this_row, _data_col_indices, output_row, _start_index_in_output);
@@ -112,8 +137,7 @@ namespace BCG
          }
       }
 
-      private void copy_dbnull_to_output_row(object[] output_row)
-      {
+      private void copy_dbnull_to_output_row(object[] output_row) {
          for (int i = 0; i < _data_col_indices.Length; i++)
             output_row[i + _start_index_in_output] = DBNull.Value;
       }

src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinSpec.cs

@@ -3,63 +3,63 @@ using System.Collections.Generic;
 using System.Linq;
 using System.Text;
 using System.Threading.Tasks;
+
 using mbts;
 
-namespace BCG
-{
-   public class NonIncJoinSpec
-   {
+namespace BCG {
+   // this a collection of data, theres no joining logic in here
+   public class NonIncJoinSpec {
       public ViewStream stream;
+      // join col names are the names on which the streams are joined (for equality),
+      // data col names are the names in the original stream
+      // projected col names are their new names, ie if its x:l_x, then x is in the 
+      // data_col_names and l_x in projected cols
       public string[] join_col_names, data_col_names, projected_col_names;
 
       public NonIncJoinSpec() { }
 
       public NonIncJoinSpec(ViewStream the_stream, string[] the_join_col_names, string[] the_data_col_names)
-         : this (the_stream, the_join_col_names, the_data_col_names, the_data_col_names)
-      { }
+         : this(the_stream, the_join_col_names, the_data_col_names, the_data_col_names) { }
 
-      public NonIncJoinSpec(ViewStream the_stream, string[] the_join_col_names, string[] the_data_col_names, string[] the_projected_col_names)
-      {
+      public NonIncJoinSpec(ViewStream the_stream, string[] the_join_col_names, string[] the_data_col_names, string[] the_projected_col_names) {
          stream = the_stream;
          join_col_names = the_join_col_names;
          data_col_names = the_data_col_names;
          projected_col_names = the_projected_col_names;
       }
 
-      public static NonIncJoinSpec[] from_triplet(params Triplet<ViewStream, string, string>[] join_specs)
-      {
+      public static NonIncJoinSpec[] from_triplet(params Triplet<ViewStream, string, string>[] join_specs) {
          List<NonIncJoinSpec> rv = new List<NonIncJoinSpec>();
-         foreach (Triplet<ViewStream, string, string> r in join_specs)
-         {
-            string[] keys = String.IsNullOrEmpty (r.second) ? new string[] {} : r.second.Split(' ');
+         foreach (Triplet<ViewStream, string, string> r in join_specs) {
+            string[] keys = String.IsNullOrEmpty(r.second) ? new string[] { } : r.second.Split(' ');
             string[] visible_cols = r.third.Split(' ');
             List<string> data_col_names = new List<string>(), mapped_col_names = new List<string>();
-            foreach (string s in visible_cols)
-            {
+            foreach (string s in visible_cols) {
                string[] toks = s.Split(':');
-               if (toks.Length == 1)
-               {
+               if (toks.Length == 1) {
                   data_col_names.Add(toks[0]);
                   mapped_col_names.Add(toks[0]);
-               }
-               else if (toks.Length == 2)
-               {
+               } else if (toks.Length == 2) {
                   data_col_names.Add(toks[0]);
                   mapped_col_names.Add(toks[1]);
                }
             }
-            rv.Add (new NonIncJoinSpec (r.first, keys, data_col_names.ToArray (), mapped_col_names.ToArray()));
+            rv.Add(new NonIncJoinSpec(r.first, keys, data_col_names.ToArray(), mapped_col_names.ToArray()));
          }
-         return rv.ToArray ();
+         return rv.ToArray();
+      }
+
+      public override string ToString() {
+         Func<string[], string> arr_txt = (a) => {
+            string rv = "";
+            foreach (string s in a) {
+               rv += "{s} ";
+            }
+            rv = rv.Trim();
+            return rv;
+         };
+         string rv = $"stream: {stream} join_col_names: {arr_txt(join_col_names)} data_col_names: {arr_txt(data_col_names)} projected_col_names: {arr_txt(projected_col_names)}";
+         return rv;
       }
    }
-
-   //[DataStreamUserRespresentation("ninctest")]
-   //public class NonIncJoinTestStream : ViewStream
-   //{
-   //   public NonIncJoinTestStream (string key_cols, string data_cols)
-   //   {
-   //      NonIncJoinSpec.from_triplet(new Triplet<ViewStream, string, string>(null, key_cols, data_cols));
-   //   }
-   //}
 }

src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinStream.cs

@@ -6,11 +6,9 @@ using System;
 using System.Collections.Generic;
 using System.Diagnostics;
 
-namespace BCG
-{
+namespace BCG {
    [DataStreamUserRespresentation("fastJoin")]
-   public class NonIncJoinStream : ViewStream, BCG.Utilities.General.IWorker
-   {
+   public class NonIncJoinStream : ViewStream, BCG.Utilities.General.IWorker {
       private readonly NonIncJoinSpec[] _join_specs;
       private readonly INonIncJoinInlineMapper _output_mapper;
       private readonly BlueRowHasher _row_hasher;
@@ -28,24 +26,19 @@ namespace BCG
                {
                   new Triplet<ViewStream, string, string> (left, left_join_cols, left_proj),
                   new Triplet<ViewStream, string, string> (right, right_join_cols, right_proj)
-               })
-      { }
+               }) { }
 
       public NonIncJoinStream(params Triplet<ViewStream, string, string>[] join_specs)
-         : this(NonIncJoinSpec.from_triplet(join_specs), null, 0)
-      { }
+         : this(NonIncJoinSpec.from_triplet(join_specs), null, 0) { }
 
       public NonIncJoinStream(ViewStreamDatabase vsd, INonIncJoinInlineMapper output_mapper, int throttles_ms,
                               params Triplet<ViewStream, string, string>[] join_specs)
-         : this(vsd, NonIncJoinSpec.from_triplet(join_specs), output_mapper, throttles_ms)
-      { }
+         : this(vsd, NonIncJoinSpec.from_triplet(join_specs), output_mapper, throttles_ms) { }
 
       public NonIncJoinStream(NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper output_mapper, int throttles_ms = 0)
-         : this(ViewStreamDatabase.instance, join_specs, output_mapper, throttles_ms)
-      { }
+         : this(ViewStreamDatabase.instance, join_specs, output_mapper, throttles_ms) { }
 
-      public NonIncJoinStream(ViewStreamDatabase vsd, NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper output_mapper, int throttles_ms = 0)
-      {
+      public NonIncJoinStream(ViewStreamDatabase vsd, NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper output_mapper, int throttles_ms = 0) {
          _join_specs = join_specs;
          _output_mapper = output_mapper;
          _row_hasher = new BlueRowHasher(_join_specs[0].join_col_names.Length);
@@ -53,12 +46,11 @@ namespace BCG
          _vsd = vsd;
       }
 
-      protected override void myOnFirstSubscriber()
-      {
+      protected override void myOnFirstSubscriber() {
          int data_col_no = 1;
-         for (int jp = 0; jp < _join_specs.Length; jp++)
-         {
+         for (int jp = 0; jp < _join_specs.Length; jp++) {
             NonIncJoinBookkeeper s = new NonIncJoinBookkeeper(_join_specs[jp], jp, data_col_no, _row_hasher);
+            Log.Gen.info($"NonIncJoinStream.myOnFirstSubscriber jp: {jp} spec: {_join_specs[jp]}");
             data_col_no += _join_specs[jp].data_col_names.Length;
             s.on_new_data += _stream_callback;
             _book_keeping_data.Add(s);
@@ -66,10 +58,11 @@ namespace BCG
          if (_output_mapper != null)
             _output_mapper.onMapperChange += _on_mapper_change;
          _vsd.addWorker(this);
+         foreach (var bkd in _book_keeping_data)
+            bkd.init();
       }
 
-      protected override void myOnLastSubscriber()
-      {
+      protected override void myOnLastSubscriber() {
          foreach (NonIncJoinBookkeeper d in _book_keeping_data)
             d.Dispose();
          if (_output_mapper != null)
@@ -77,30 +70,26 @@ namespace BCG
          _vsd.removeWorker(this);
       }
 
-      private void _stream_callback()
-      {
+      private void _stream_callback() {
+         Log.Gen.info("NonIncJoinStream._stream_callback received callback");
          _is_dirty = true;
          doWork();
       }
 
-      private void _on_mapper_change()
-      {
+      private void _on_mapper_change() {
          _is_dirty = true;
          doWork();
       }
 
-      private bool _is_ready()
-      {
+      private bool _is_ready() {
          bool rv = true;
          foreach (NonIncJoinBookkeeper d in _book_keeping_data)
             rv &= d.is_ready;
          return rv;
       }
 
-      private void _construct_schema()
-      {
-         if (_output_schema == null)
-         {
+      private void _construct_schema() {
+         if (_output_schema == null) {
             List<ColumnInfo> cis = new List<ColumnInfo>(new ColumnInfo[] { new ColumnInfo("id", 0, typeof(int)) });
             foreach (NonIncJoinBookkeeper d in _book_keeping_data)
                d.constribute_cols_to_schema(cis);
@@ -109,27 +98,23 @@ namespace BCG
       }
 
       private int _n_total_fires = 0, _n_expensive_fires = 0;
-      private void _form_and_fire()
-      {
-         if (_is_ready())
-         {
+      private void _form_and_fire() {
+         Log.Gen.info($"NonIncJoinStream._form_and_fire _is_ready(): {_is_ready()}");
+         if (_is_ready()) {
             _construct_schema();
             var timer = Stopwatch.StartNew();
             IList<Pair<int, IBlueRow>> left_row_list = _book_keeping_data[0].get_list();
             List<object[]> output_rows = new List<object[]>();
 
-            for (int i = 0; i < left_row_list.Count; i++)
-            {
+            for (int i = 0; i < left_row_list.Count; i++) {
                object[] or = new object[_output_schema.columnInfo.Length];
                or[0] = i;
                output_rows.Add(or);
             }
 
-            for (int ji = 0; ji < _book_keeping_data.Count; ji++)
-            {
+            for (int ji = 0; ji < _book_keeping_data.Count; ji++) {
                NonIncJoinBookkeeper bk_data = _book_keeping_data[ji];
-               for (int i = 0; i < left_row_list.Count; i++)
-               {
+               for (int i = 0; i < left_row_list.Count; i++) {
                   if (ji == 0)
                      bk_data.copy_master_row_to_output_row(i, output_rows[i]);
                   else
@@ -139,19 +124,20 @@ namespace BCG
             IBlueRowCollection dp_output_rows = _apply_map(new SequentialIDBlueTable(_output_schema, output_rows));
             timer.Stop();
             _n_total_fires++;
-            if (timer.ElapsedMilliseconds >= 250) {
-               _n_expensive_fires++;
-               string log_msg = $"Join and map time for {_book_keeping_data.Count} "
-                  + $"tables {timer.Elapsed} ratio {_n_expensive_fires}/{_n_total_fires}";
-            }
-            myFireEvent(new DataPacket(dp_output_rows, dp_output_rows, myLastValue == null ? new BlueTable(_output_schema) : (myLastValue as DataPacket).newRows));
+            //if (timer.ElapsedMilliseconds >= 250) {
+            _n_expensive_fires++;
+            string log_msg = $"Join and map time for {_book_keeping_data.Count} "
+               + $"tables {timer.Elapsed.TotalMilliseconds} msecs, ratio {_n_expensive_fires}/{_n_total_fires}";
+            Log.Gen.info(log_msg);
+            //}
+            var dp_out = BlueDataUtility.build_dp_inc(dp_output_rows, myLastValue as DataPacket);
+            myFireEvent(dp_out);
+            //myFireEvent(new DataPacket(dp_output_rows, dp_output_rows, myLastValue == null ? new BlueTable(_output_schema) : (myLastValue as DataPacket).newRows));
          }
       }
 
-      private IBlueRowCollection _apply_map(IBlueRowCollection unmapped_rows)
-      {
-         if (_output_mapper != null)
-         {
+      private IBlueRowCollection _apply_map(IBlueRowCollection unmapped_rows) {
+         if (_output_mapper != null) {
             IBlueRowCollection output_rows;
             var timer = Stopwatch.StartNew();
             output_rows = _output_mapper.map_rows(unmapped_rows);
@@ -160,17 +146,14 @@ namespace BCG
                Log.Gen.info($"Time to map {unmapped_rows.Count} rows into {output_rows.Count} rows {timer.Elapsed}");
             }
             return output_rows;
-         }
-         else
-         {
+         } else {
             return unmapped_rows;
          }
       }
 
-      public void doWork()
-      {
-         if (DateTime.Now - _last_fire_time >= _throttle_period && _is_dirty)
-         {
+      public void doWork() {
+         Log.Gen.info($"NonIncJoinStream.doWork _last_fire_time: {_last_fire_time} _is_dirty: {_is_dirty}");
+         if (DateTime.Now - _last_fire_time >= _throttle_period && _is_dirty) {
             _is_dirty = false;
             _last_fire_time = DateTime.Now;
             _form_and_fire();

src/DataStreamSuite/DataStream/Primitives/Join/ParallelJoinTest.cs

@@ -181,8 +181,8 @@ namespace BCG
 
        protected override void myOnFirstSubscriber()
        {
-           myVLeft.onChange += new ChangeDelegate(myVLeft_onChange);
-           myVRight.onChange += new ChangeDelegate(myVRight_onChange);
+           myVLeft.onChange += myVLeft_onChange;
+           myVRight.onChange += myVRight_onChange;
        }
 
        void myVRight_onChange(object newValue)

src/DataStreamSuite/DataStream/Primitives/Join/Proximity/ProximityJoinStream.cs

@@ -0,0 +1,68 @@
+namespace bts {
+   using BCG;
+   using BCG.Utilities.General;
+
+   using System;
+   using System.Diagnostics;
+
+   using ZstdSharp.Unsafe;
+
+   // functon that joins the 2 rows and returns the outputput row. The function
+   // can return a null which means the 2 rows dont need to be joined
+   using JoinFunc = System.Func<BCG.IBlueRow, BCG.IBlueRow, BCG.IBlueRow>;
+   public class ProximityJoinStream : ViewStream, IWorker {
+      private ViewStream _lvs, _rvs;
+      private JoinFunc _join_func;
+      private bool _dirty = false;
+      private Stopwatch _stopwatch = new Stopwatch();
+      private ViewStreamDatabase _vsd = ViewStreamDatabase.instance;
+      public ProximityJoinStream(ViewStream lvs, ViewStream rvs, JoinFunc join_func) {
+         _lvs = lvs;
+         _rvs = rvs;
+         _join_func = join_func;
+      }
+
+      protected override void myOnFirstSubscriber() {
+         _lvs.onChange += _handle_data;
+         _rvs.onChange += _handle_data;
+         _vsd.addWorker(this);
+      }
+
+      protected override void myOnLastSubscriber() {
+         _lvs.onChange -= _handle_data;
+         _rvs.onChange -= _handle_data;
+         _vsd.removeWorker(this);
+      }
+
+      private void _handle_data(object dp_obj) {
+         _dirty = true;
+         doWork();
+      }
+
+      public void doWork() {
+         Func<bool> all_data_revcd = () => {
+            return _lvs.rows != null && _rvs.rows != null;
+         };
+         _stopwatch.Start();
+         if (_dirty && all_data_revcd()) {
+            _dirty = false;
+            BlueTable rv = null;
+            foreach (IBlueRow lr in _lvs.rows) {
+               foreach (IBlueRow rr in _rvs.rows) {
+                  IBlueRow resr = _join_func(lr, rr);
+                  if (resr == null)
+                     continue;
+                  rv = rv ?? new BlueTable(resr.schema);
+                  rv.addRow(resr);
+               }
+            }
+            _stopwatch.Stop();
+            Log.Gen.info($"ProximityJoinStream left_count: {_lvs.rows.Count} right_count: {_rvs.rows.Count} result_count: {rv.Count} time: {_stopwatch.ElapsedMilliseconds} msecs");
+            var dp = BlueDataUtility.build_dp_inc(rv, myLastValue as DataPacket);
+            myFireEvent(dp);
+         }
+
+      }
+
+   }
+}

src/DataStreamSuite/DataStream/Primitives/Join/Proximity/TimeProximityJoinStream.cs

@@ -0,0 +1,47 @@
+namespace bts.SilverPine {
+   using BCG;
+
+   using System;
+   using System.CodeDom;
+   using System.Configuration;
+   using System.Reactive.Disposables;
+
+   public class TimeProximityJoinStream : ViewStream {
+      private int _id = 0;
+
+      public TimeProximityJoinStream(ViewStream ls, string l_time_col, ViewStream rs, string r_time_col,
+                                       string pre_join_period, string post_join_period) :
+            this(ls, l_time_col, rs, r_time_col, TimeBasedFilter.parse_duration(pre_join_period),
+            TimeBasedFilter.parse_duration(post_join_period)) { }
+
+
+      public TimeProximityJoinStream(ViewStream ls, string l_time_col, ViewStream rs, string r_time_col,
+                                       TimeSpan pre_join_period, TimeSpan post_join_period) {
+         Func<IBlueRow, IBlueRow, IBlueRow> create_row = (lr, rr) => {
+            var id = 0;
+            lock (this) {
+               id = _id++;
+            }
+            IBlueRow rv = BlueDataUtility.create_composite_row(lr, rr, id);
+            return rv;
+         };
+         Func<IBlueRow, IBlueRow, IBlueRow> join_func = (lr, rr) => {
+            DateTime ldt = lr.get_datetime(l_time_col);
+            DateTime rdt = rr.get_datetime(r_time_col);
+            TimeSpan diff = ldt - rdt;
+            if (ldt < rdt) {
+               if (diff.Duration() > pre_join_period) {
+                  return null;
+               }
+            } else {
+               if (diff > post_join_period)
+                  return null;
+            }
+            return create_row(lr, rr);
+         };
+
+         ViewStream rstr = new ProximityJoinStream(ls, rs, join_func);
+         mySetRealSource(rstr);
+      }
+   }
+}

src/DataStreamSuite/DataStream/Primitives/Join/SuperJoinStream.cs

@@ -327,14 +327,14 @@ namespace BCG
 
       protected override void myOnFirstSubscriber()
       {
-         myLeftStream.onChange += new ChangeDelegate(myHandleLeftStreamChanges);
-         myRightStream.onChange += new ChangeDelegate(myHandleRightStreamChanges);
+         myLeftStream.onChange += myHandleLeftStreamChanges;
+         myRightStream.onChange += myHandleRightStreamChanges;
       }
 
       protected override void myOnLastSubscriber()
       {
-         myLeftStream.onChange -= new ChangeDelegate(myHandleLeftStreamChanges);
-         myRightStream.onChange -= new ChangeDelegate(myHandleRightStreamChanges);
+         myLeftStream.onChange -= myHandleLeftStreamChanges;
+         myRightStream.onChange -= myHandleRightStreamChanges;
          myLeftKeyToLeftRows.Clear();
          myRightKeyToRightRows.Clear();
          myLeftIDToResultRows.Clear();

src/DataStreamSuite/DataStream/Primitives/LeftJoin/LeftInternalJoinStream.cs

@@ -23,15 +23,15 @@ namespace BCG
          myRightOperand.master = this;
          //the events are not hooked onto the operands delegate, but rather a level of indirection
          //added in the form on myLeftChangeHandler
-         myLeftStream.onChange += new DataStream.ChangeDelegate (myLeftChangeHandler);
-         myRightStream.onChange += new DataStream.ChangeDelegate (myRightChangeHandler);
+         myLeftStream.onChange += myLeftChangeHandler;
+         myRightStream.onChange += myRightChangeHandler;
       }
        
       protected override void myOnLastSubscriber () 
       {
          myInit ();
-         myLeftStream.onChange -= new DataStream.ChangeDelegate (myLeftChangeHandler);
-         myRightStream.onChange -= new DataStream.ChangeDelegate (myRightChangeHandler);
+         myLeftStream.onChange -= myLeftChangeHandler;
+         myRightStream.onChange -= myRightChangeHandler;
       }
 
       protected override void myInit()

src/DataStreamSuite/DataStream/Primitives/Literal/LiteralStream.cs

@@ -2,9 +2,13 @@ using System;
 using System.Collections;
 using System.Collections.Generic;
 
+using Amazon.Runtime.Internal.Util;
+
 using BCG;
 using BCG.Utilities;
 
+using bts;
+
 namespace BCG {
    public class LiteralStream : ViewStream {
 
@@ -22,6 +26,34 @@ namespace BCG {
          return rv;
       }
 
+      //literal("table1", "x:123, y:1.001, z:abc;x:123, y:1.001, z:abc")
+      public LiteralStream(string table_name, string rows) {
+         Func<string, int, IBlueRow> get_row = (json_row, id) => {
+            Dictionary<string, object> map = new Dictionary<string, object>();
+            string[] toks = json_row.Split(',', StringSplitOptions.TrimEntries);
+            //Logger.assert(col_vals.Length == 2);
+            foreach (string t in toks) {
+               var col_vals = t.Split(':', StringSplitOptions.TrimEntries);
+               string col = col_vals[0];
+               string val = col_vals[1];
+               map.Add(col, val);
+            }
+            map["id"] = id;
+            return new BlueRow(map);
+         };
+         string[] row_arr = rows.Split(';', StringSplitOptions.TrimEntries);
+         BlueTable table = null;
+         foreach (string json_string in row_arr) {
+            IBlueRow r = //BlueDataUtility.convert_json_to_row(json_string, table == null ? 0 : table.Count);
+            get_row(json_string, table == null ? 0 : table.Count);
+            table = table ?? new BlueTable(r.schema);
+            table.addRow(r);
+         }
+         ViewStream str = new BlueTableStream(table);
+         str = new SchemaGuesserStream(str);
+         mySetRealSource(str);
+      }
+
       public LiteralStream(string tableName, string schema, string value) {
          string[] columns = schema.Split(' ');
 
@@ -55,7 +87,7 @@ namespace BCG {
          mySetRealSource(new BlueRowCollectionStaticStream(t));
       }
 
-      public LiteralStream(string schema, string value) : this("literal", schema, value) { }
+      // public LiteralStream(string schema, string value) : this("literal", schema, value) { }
 
       // public LiteralStream(string value) {
 

src/DataStreamSuite/DataStream/Primitives/Map/GlobalMapStream.cs

@@ -0,0 +1,72 @@
+using BCG;
+
+using MongoDB.Driver.Core.WireProtocol.Messages;
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+
+// the first arg is all the rows, the second arg is the index of the row that needs mapping, and 
+// the func returns the mapped row (ie mapped version of the index row)
+using GlobalMap = System.Func<System.Collections.Generic.List<BCG.IBlueRow>, uint, BCG.IBlueRow>;
+
+namespace bts {
+   // In GlobalMap, each row is mapped with the full view of all rows, ie the map functions for a row
+   // can use the state fo all rows.
+   public class GlobalMapStream : ViewStream {
+      private ViewStream _base_stream;
+      private readonly string _sort_col_name, _group_by_col;
+      private GlobalMap _map_func;
+      public GlobalMapStream(ViewStream base_stream, string sort_col_name, string group_by_col, GlobalMap map_func) {
+         _base_stream = base_stream;
+         _sort_col_name = sort_col_name;
+         _group_by_col = group_by_col;
+         _map_func = map_func;
+      }
+
+      public GlobalMapStream(ViewStream base_stream, string sort_col_name, GlobalMap map_func) :
+      this(base_stream, sort_col_name, null, map_func) {
+      }
+
+      protected override void myOnFirstSubscriber() {
+         _base_stream.onChange += _handle_stream_update;
+      }
+
+      protected override void myOnLastSubscriber() {
+         _base_stream.onChange -= _handle_stream_update;
+      }
+
+      void _handle_stream_update(object dp_obj) {
+         DataPacket dp = dp_obj as DataPacket;
+         IDictionary<int, List<IBlueRow>> groups = !String.IsNullOrEmpty(_group_by_col) ?
+         BlueDataUtility.group_together(dp.newRows, new string[] { _group_by_col })
+         :
+         new Dictionary<int, List<IBlueRow>>() { { 0, dp.newRows.to_list() } };
+         BlueTable out_table = null;
+         foreach (var kv in groups) {
+            List<IBlueRow> group_rows = kv.Value;
+            group_rows.Sort((r1, r2) => BlueDataUtility.compare_rows(r2, r1, _sort_col_name));
+            uint i = 0;
+            foreach (IBlueRow r in group_rows) {
+               IBlueRow nr = _map_func(group_rows, i++);
+               out_table = out_table ?? new BlueTable(nr.schema);
+               out_table.addRow(nr);
+            }
+         }
+
+
+         // List<IBlueRow> rows = dp.newRows.to_ienum().ToList();
+         // rows.Sort((r1, r2) => BlueDataUtility.compare_rows(r1, r2, _sort_col_name));
+         // uint i = 0;
+
+         // BlueTable out_table = null;
+         // foreach (IBlueRow r in rows) {
+         //    IBlueRow nr = _map_func(rows, i++);
+         //    out_table = out_table ?? new BlueTable(nr.schema);
+         //    out_table.addRow(nr);
+         // }
+         DataPacket dp_out = BlueDataUtility.build_dp_inc(out_table, myLastValue as DataPacket);
+         myFireEvent(dp_out);
+      }
+   }
+}

src/DataStreamSuite/DataStream/Primitives/Map/MapStream.cs

@@ -119,13 +119,13 @@ namespace BCG {
 
 
       protected override void myOnFirstSubscriber() {
-         myStream.onChange += new DataStream.ChangeDelegate(myChangeHandler);
+         myStream.onChange += myChangeHandler;
       }
 
       protected override void myOnLastSubscriber() {
          myInit();
          if (myStream != null)
-            myStream.onChange -= new DataStream.ChangeDelegate(myChangeHandler);
+            myStream.onChange -= myChangeHandler;
       }
 
       protected override void myOnTableClear() {

src/DataStreamSuite/DataStream/Primitives/Map/SuperMapStream.cs

@@ -83,10 +83,11 @@ namespace BCG {
          myMapper.onMapperChange += new MapperChangeDelegate(myHandleMapChange);
       }
 
-      protected override void myOnFirstSubscriber() { myStream.onChange += new ChangeDelegate(myHandleData); }
+      protected override void myOnFirstSubscriber() =>
+         myStream.onChange += myHandleData;
 
       protected override void myOnLastSubscriber() {
-         myStream.onChange -= new ChangeDelegate(myHandleData);
+         myStream.onChange -= myHandleData;
          myRows = null;
          myBaseRows = null;
          myMap.Clear();

src/DataStreamSuite/DataStream/Primitives/Misc/DebugStream.cs

@@ -16,10 +16,10 @@ namespace BCG {
       }
 
       protected override void myOnFirstSubscriber() {
-         _stream.onChange += new ChangeDelegate(_onstream_change);
+         _stream.onChange += _onstream_change;
       }
       protected override void myOnLastSubscriber() {
-         _stream.onChange -= new ChangeDelegate(_onstream_change);
+         _stream.onChange -= _onstream_change;
       }
 
       private void _onstream_change(object o) {
@@ -45,4 +45,4 @@ namespace BCG {
 
       }
    }
-}
+}

src/DataStreamSuite/DataStream/Primitives/Misc/TimeSeriesStream.cs

@@ -1,9 +1,8 @@
-using Google.Protobuf.WellKnownTypes;
-
 using System;
-using BCG.Utilities.General;
 
-namespace BCG {
+using BCG;
+
+namespace bts.SilverPine {
    // Keeps adding changing rows to the stream, hence preserving the history
    // and creating a time series, for example this can be used to make bbo updates
    // into a time series of prices/qty 
@@ -17,10 +16,10 @@ namespace BCG {
       }
 
       protected override void myOnFirstSubscriber() {
-         _stream.onChange += new ChangeDelegate(_onstream_change);
+         _stream.onChange += _onstream_change;
       }
       protected override void myOnLastSubscriber() {
-         _stream.onChange -= new ChangeDelegate(_onstream_change);
+         _stream.onChange -= _onstream_change;
       }
 
 
@@ -41,7 +40,6 @@ namespace BCG {
          }
          _table.addRows(added_rows);
          myFireEvent(new DataPacket(_table, added_rows, new BlueTable(_table.schema)));
-
       }
    }
 

src/DataStreamSuite/DataStream/Primitives/Project/New/NewProjectStream.cs

@@ -137,10 +137,24 @@ namespace bts {
          HashSet<string> removed_cols = new HashSet<string>();
          Dictionary<string, string> mapped_to_orig_map = new Dictionary<string, string>();
          Dictionary<string, string> orig_to_mapped = new Dictionary<string, string>();
+         // by default every col will show up, so if we have the projection as (a:a1)
+         // it will convert a to a1, 
+         // examples: if a schema has cols a b c
+         // project(parent_str, "a:aa ") -> aa b c
+         // project(parent_str, "a:aa !c") -> aa b
+         // project(parent_str, "a:aa !") -> aa
+         // project(parent_str, "a:aa b = (aa*aa)!") -> aa b
+         bool hide_all = false;
          foreach (string p_tok in project_toks) {
             if (p_tok.StartsWith("!")) {
-               string removed_col = p_tok.Substring(1);
-               removed_cols.Add(removed_col);
+               if (p_tok.Length == 1) {
+                  // if it is a "blanket "!"=> hide all cols except the one that
+                  // are stated explicitly
+                  hide_all = true;
+               } else {
+                  string removed_col = p_tok.Substring(1);
+                  removed_cols.Add(removed_col);
+               }
             } else if (p_tok.Contains(":")) {
                string[] map_toks = p_tok.Split(":");
                string orig_col = map_toks[0];
@@ -155,12 +169,11 @@ namespace bts {
                }
             }
          }
+
          // make the new schema
          List<ColumnInfo> new_cols = new List<ColumnInfo>();
          foreach (ColumnInfo ci in input_schema.columnInfo) {
             if (ci.colName == "id") {
-               // if there's a hidden col, other columns will show by default, but if cols are being
-               // projected epxlicitely, than others wil be hidden
                ColumnInfo nc = new ColumnInfo(ci.colName, new_cols.Count, ci.colType);
                new_cols.Add(nc);
                continue;
@@ -172,12 +185,13 @@ namespace bts {
                ColumnInfo nc = new ColumnInfo(mapped_name, new_cols.Count, ci.colType);
                new_cols.Add(nc);
             } else {
-               if (ci.colName == "id" || removed_cols.Count > 0) {
-                  // if there's a hidden col, other columns will show by default, but if cols are being
-                  // projected epxlicitely, than others wil be hidden
-                  ColumnInfo nc = new ColumnInfo(ci.colName, new_cols.Count, ci.colType);
-                  new_cols.Add(nc);
-               }
+               if (hide_all)
+                  continue;
+               //if (ci.colName == "id" || removed_cols.Count > 0) {
+               // unless columns are  explicitely hidden, all columns show up
+               ColumnInfo nc = new ColumnInfo(ci.colName, new_cols.Count, ci.colType);
+               new_cols.Add(nc);
+               //}
             }
          }
          IBlueTableSchema new_schema = new BlueTableSchema(input_schema.tableName, new_cols.ToArray());

src/DataStreamSuite/DataStream/Primitives/Project/ProjectStream.cs

@@ -54,12 +54,12 @@ namespace BCG {
 
       protected override void myOnFirstSubscriber()
       {
-         myVs.onChange += new BCG.DataStream.ChangeDelegate (myHandleChange);
+         myVs.onChange += myHandleChange;
       }
 
       protected override void myOnLastSubscriber()
       {
-         myVs.onChange -= new BCG.DataStream.ChangeDelegate (myHandleChange);;
+         myVs.onChange -= myHandleChange;
       }
 
       internal void myHandleChange (object newValue)

src/DataStreamSuite/DataStream/Primitives/SQL/SQLite/SQLiteEnumerator.cs

@@ -0,0 +1,54 @@
+namespace Bts.DataStream;
+
+using BCG;
+
+using System;
+using System.Collections.Generic;
+using System.Data;
+using System.Data.SQLite;
+using System.Linq;
+using System.Threading;
+
+public class SQLiteEnumerable : IAsyncEnumerable<IBlueRowCollection> {
+   readonly string location;
+   readonly string statement;
+   readonly int pageSize;
+
+   public SQLiteEnumerable(string location, string statement, int pageSize = 65536) {
+      this.location = location;
+      this.statement = statement;
+      this.pageSize = pageSize;
+   }
+
+   public async IAsyncEnumerator<IBlueRowCollection> GetAsyncEnumerator(CancellationToken cancellationToken = default) {
+      var dataTable = new DataTable();
+      await using var conn = new SQLiteConnection($"Data Source={location};Version=3");
+      await conn.OpenAsync(cancellationToken);
+      await using var cmd = new SQLiteCommand(statement, conn);
+      await using var reader = cmd.ExecuteReader();
+      var schema = await reader.GetColumnSchemaAsync(cancellationToken);
+      var columns = schema.OrderBy(c => c.ColumnOrdinal ?? int.MaxValue)
+         .Select((c, i) => new ColumnInfo(c.ColumnName, c.ColumnOrdinal ?? i, c.DataType))
+         .Append(new ColumnInfo("id", schema.Count, typeof(int)))
+         .ToArray();
+      var blueSchema = new BlueTableSchema("anon", columns);
+      BlueTable? table = null;
+      int rowId = 1;
+      while (await reader.ReadAsync(cancellationToken)) {
+         table ??= new BlueTable(blueSchema);
+         var rowVals = new object[blueSchema.columnInfo.Length];
+         if (reader.GetValues(rowVals) != rowVals.Length - 1) {
+            throw new InvalidOperationException("Sqlite reader value count for single row not equal to column count");
+         }
+         rowVals[^1] = rowId++;
+         table.addRow(new BlueRow(blueSchema, rowVals));
+         if (table.Count == pageSize) {
+            yield return table;
+            table = null;
+         }
+      }
+      if (table is not null) {
+         yield return table;
+      }
+   }
+}

src/DataStreamSuite/DataStream/Primitives/SQL/SQLite/SQLiteHelper.cs

@@ -0,0 +1,122 @@
+using System;
+using System.Data;
+using System.Data.SQLite;
+using System.Diagnostics;
+using System.Collections;
+
+
+using BCG;
+using System.Linq;
+using System.Data.Entity;
+using System.Collections.Generic;
+using System.Data.SQLite;
+
+
+namespace bts.SilverPine {
+
+   public static class SQLiteHelper {
+      public static string location = "C:\\Users\\m_tah\\development\\data\\";
+      public static string db_name = "stream_data.db";
+
+      public static DataTable execute_select_query(string connection_string, string query) {
+         // Create a new DataTable to store the results
+         DataTable dataTable = new DataTable();
+
+         // Create and open a SQLite connection using the provided connection string
+         using (SQLiteConnection connection = new SQLiteConnection(connection_string)) {
+            try {
+               connection.Open();
+
+               // Create a SQLite command with the provided query
+               using (SQLiteCommand command = new SQLiteCommand(query, connection)) {
+                  // Use SQLiteDataAdapter to fill the DataTable
+                  using (SQLiteDataAdapter adapter = new SQLiteDataAdapter(command)) {
+                     adapter.Fill(dataTable);
+                  }
+               }
+            } catch (Exception ex) {
+               Console.WriteLine("An error occurred: " + ex.Message);
+            }
+         }
+         // Return the populated DataTable
+         return dataTable;
+      }
+
+      public static List<Dictionary<string, object>> ExecuteBulkRead(string connectionString, string query) {
+         var result = new List<Dictionary<string, object>>();
+
+         using (var connection = new SQLiteConnection(connectionString)) {
+            connection.Open();
+
+            // Begin a transaction to optimize read speed
+            using (var transaction = connection.BeginTransaction())
+            using (var command = new SQLiteCommand(query, connection, transaction))
+            using (var reader = command.ExecuteReader()) {
+               while (reader.Read()) {
+                  var row = new Dictionary<string, object>();
+
+                  for (int i = 0; i < reader.FieldCount; i++) {
+                     string columnName = reader.GetName(i);
+                     object columnValue = reader.IsDBNull(i) ? null : reader.GetValue(i);
+                     row[columnName] = columnValue;
+                  }
+
+                  result.Add(row);
+               }
+
+               // Commit the transaction
+               transaction.Commit();
+            }
+         }
+
+         return result;
+      }
+
+      public static List<Dictionary<string, object>> get_table(string connectionString, string query) {
+         var result = new List<Dictionary<string, object>>();
+
+         using (var connection = new SQLiteConnection(connectionString)) {
+            try {
+               connection.Open();
+
+               using (var command = new SQLiteCommand(query, connection))
+               using (var reader = command.ExecuteReader()) {
+                  while (reader.Read()) {
+                     var row = new Dictionary<string, object>();
+
+                     for (int i = 0; i < reader.FieldCount; i++) {
+                        string columnName = reader.GetName(i);
+                        object columnValue = reader.IsDBNull(i) ? null : reader.GetValue(i);
+                        row[columnName] = columnValue;
+                     }
+
+                     result.Add(row);
+                  }
+               }
+            } catch (Exception ex) {
+               Console.WriteLine("An error occurred: " + ex.Message);
+            }
+         }
+         return result;
+      }
+
+      public static IEnumerable<Dictionary<string, object>> get_table_2(string connectionString, string query) {
+         using (var connection = new SQLiteConnection(connectionString)) {
+            connection.Open();
+            using (var command = new SQLiteCommand(query, connection))
+            using (var reader = command.ExecuteReader()) {
+               while (reader.Read()) {
+                  var row = new Dictionary<string, object>();
+                  for (int i = 0; i < reader.FieldCount; i++) {
+                     string columnName = reader.GetName(i);
+                     object columnValue = reader.IsDBNull(i) ? null : reader.GetValue(i);
+                     row[columnName] = columnValue;
+                  }
+                  yield return row;
+               }
+            }
+         }
+      }
+   }
+}
+

src/DataStreamSuite/DataStream/Primitives/SQL/SQLite/SqliteStream.cs

@@ -0,0 +1,73 @@
+namespace bts.SilverPine {
+   using System;
+   using System.Data;
+   using System.Data.SQLite;
+   using System.Diagnostics;
+   using System.Collections;
+
+
+   using BCG;
+
+   [DataStreamUserRespresentation("SQLite")]
+   public class SQLiteStream : ViewStream {
+      private readonly string _connection_string, _query;
+      private Stopwatch _stopwatch = new Stopwatch();
+
+      private SQLiteStream(string connection_string, string query, bool noop) {
+         _connection_string = connection_string;
+         _query = query;
+      }
+
+      public SQLiteStream(string db_name, string query)
+      :
+      this($"Data Source={SQLiteHelper.location}{db_name};Version=3", query, true) { }
+
+      public SQLiteStream(string query)
+      :
+      this(SQLiteHelper.db_name, query) {
+      }
+
+
+      private void _read_from_sqlite_and_fire() {
+         _stopwatch.Start();
+         DataTable dt = SQLiteHelper.execute_select_query(_connection_string, _query);
+         _stopwatch.Stop();
+         Log.Gen.info($"SQLiteStream.ExecuteSelectQuery time to read db: {_stopwatch.ElapsedMilliseconds} msecs");
+         _stopwatch.Restart();
+         #region perf measures
+         // SQLiteHelper.get_table(_connection_string, _query);
+         // _stopwatch.Stop();
+         // Log.Gen.info($"SQLiteStream.get_table time to read db: {_stopwatch.ElapsedMilliseconds} msecs");
+         // _stopwatch.Restart();
+         // SQLiteHelper.ExecuteBulkRead(_connection_string, _query);
+         // _stopwatch.Stop();
+         // Log.Gen.info($"SQLiteStream.ExecuteBulkRead time to read db: {_stopwatch.ElapsedMilliseconds} msecs");
+         // _stopwatch.Restart();
+         // SQLiteHelper.get_table_2(_connection_string, _query);
+         // _stopwatch.Stop();
+         // Log.Gen.info($"SQLiteStream.get_table_2 time to read db: {_stopwatch.ElapsedMilliseconds} msecs");
+         // _stopwatch.Restart();
+         #endregion
+         IBlueRowCollection rows = BCG.BlueDataUtility.toIBlueRowCollection(dt);
+         //IBlueRowCollection rows = new DataTableWrapperTable(dt);
+         _stopwatch.Stop();
+         Log.Gen.info($"SQLiteStream._read_from_sqlite_and_fire time to load into IBlueRowCollection: {_stopwatch.ElapsedMilliseconds} msecs");
+         DataPacket dp = BlueDataUtility.build_dp_inc(rows, myLastValue as DataPacket);
+         myFireEvent(dp);
+      }
+
+      protected override void myOnFirstSubscriber() {
+         _read_from_sqlite_and_fire();
+      }
+
+      protected override void myOnLastSubscriber() {
+         // this value will be cleared by the base class
+         // but in this particular stream this behaviour
+         // is not usefull since any new subscribers 
+         // would force the query to be recreated
+         //myLastDP = myLastValue;
+      }
+
+   }
+
+}

src/DataStreamSuite/DataStream/Primitives/SideEffectStreams/CSVSaverStream.cs

@@ -3,12 +3,15 @@ using System.Collections.Generic;
 using System.Text;
 using System.IO;
 using BCG.Utilities;
+using System.Diagnostics;
+using Microsoft.Data.Analysis;
+using Parquet;
+using Parquet.Data;
 
-namespace BCG
-{
+
+namespace BCG {
    [DataStreamUserRespresentation("CSVSaver")]
-   public class CSVSaverStream : ViewStream, ISideEffectStream
-   {
+   public class CSVSaverStream : ViewStream, ISideEffectStream {
       private IBlueTableSchema myResultSchema = new BlueTableSchema("CSVResult", new ColumnInfo("id", 0, typeof(int)),
          new ColumnInfo("fileName", 1, typeof(string)),
          new ColumnInfo("nWrites", 2, typeof(int)),
@@ -20,11 +23,9 @@ namespace BCG
       private int myNWrites = 0, myNMaxWrites = Int32.MaxValue;
 
       public CSVSaverStream(ViewStream vs, string fullyQualifiedFileName)
-         : this(vs, fullyQualifiedFileName, true, Int32.MaxValue, false)
-      { }
+         : this(vs, fullyQualifiedFileName, true, Int32.MaxValue, false) { }
 
-      public CSVSaverStream(ViewStream vs, string fullyQualifiedFileName, bool overwrite, int nMaxWrites, bool fireResultSnapshot)
-      {
+      public CSVSaverStream(ViewStream vs, string fullyQualifiedFileName, bool overwrite, int nMaxWrites, bool fireResultSnapshot) {
          myStream = vs;
          myFullyQualifiedFileName = fullyQualifiedFileName;
          myOverwrite = overwrite;
@@ -32,22 +33,20 @@ namespace BCG
          myFireResultSnapshot = fireResultSnapshot;
       }
 
-      protected override void myOnFirstSubscriber() { myStream.onChange += new ChangeDelegate(myHandleChanges); }
+      protected override void myOnFirstSubscriber() =>
+         myStream.onChange += myHandleChanges;
 
-      protected override void myOnLastSubscriber() { myStream.onChange -= new ChangeDelegate(myHandleChanges); }
+      protected override void myOnLastSubscriber() =>
+         myStream.onChange -= myHandleChanges;
 
-      private void myHandleChanges(object newValue)
-      {
+      private void myHandleChanges(object newValue) {
          IBlueRowCollection totalState = (newValue as DataPacket).newRows;
-         if (totalState.Count > 0 && myNWrites < myNMaxWrites)
-         {
+         if (totalState.Count > 0 && myNWrites < myNMaxWrites) {
             myNWrites++;
             int i = 0;
             StreamWriter writer = new StreamWriter(new FileStream(myGetNewFileName(), FileMode.Create, FileAccess.Write));
-            foreach (IBlueRow r in totalState)
-            {
-               if (i++ == 0)
-               {
+            foreach (IBlueRow r in totalState) {
+               if (i++ == 0) {
                   string header = String.Empty;
                   foreach (string col in r.schema.columnNames)
                      if (col != "id")
@@ -59,20 +58,17 @@ namespace BCG
                writer.Flush();
             }
             writer.Close();
-            if (myFireResultSnapshot)
-            {
+            if (myFireResultSnapshot) {
                IBlueRow nr = new BlueRow(myResultSchema, 0, myFullyQualifiedFileName, myNWrites, totalState.Count);
                BlueTable nRows = new BlueTable(myResultSchema);
                nRows.addRow(nr);
                myFireEvent(new DataPacket(nRows, nRows, new BlueTable(myLastValue == null ? new BlueTable(myResultSchema) : (myLastValue as DataPacket).newRows)));
-            }
-            else
+            } else
                myFireEvent(newValue);
          }
       }
 
-      private string myGetNewFileName()
-      {
+      private string myGetNewFileName() {
          string[] tags = myFullyQualifiedFileName.Split('.');
          if (myOverwrite)
             return tags[0] + ".csv";
@@ -80,14 +76,11 @@ namespace BCG
             return tags[0] + "_" + myFormateDateTime(DateTime.Now) + ".csv";
       }
 
-      private static string myGetCSVRow(IBlueRow r)
-      {
+      private static string myGetCSVRow(IBlueRow r) {
          string rv = String.Empty;
          int i = 0;
-         foreach (string col in r.schema.columnNames)
-         {
-            if (col != "id")
-            {
+         foreach (string col in r.schema.columnNames) {
+            if (col != "id") {
                string colText = String.Empty;
                if (r[col] != null && r[col] != DBNull.Value)
                   colText = r[col].ToString();
@@ -99,8 +92,7 @@ namespace BCG
          return rv;
       }
 
-      private string myFormateDateTime(DateTime dt)
-      {
+      private string myFormateDateTime(DateTime dt) {
          return
          string.Format("{0}-{1}-{2}-{3}-{4}-{5}-{6}",
                         dt.Year.ToString("0000"),
@@ -114,52 +106,49 @@ namespace BCG
    }
 
    [DataStreamUserRespresentation("CSVReader")]
-   public class CSVReaderStream : ViewStream
-   {
-      private IBlueRowCollection myRows;
-
-      public CSVReaderStream(string fileAddress)
-      {
-         myRows = BlueDataUtility.createRowsFromCSVFileWithoutTypes(fileAddress);
+   public class CSVReaderStream : ViewStream {
+      private readonly string _file_address;
+      Stopwatch _stopwatch = new Stopwatch();
+      public CSVReaderStream(string fileAddress) {
+         _file_address = fileAddress;
       }
 
-      protected override void myOnFirstSubscriber()
-      {
-         myFireEvent(new DataPacket(myRows, myRows, new BlueTable(myRows.schema)));
+      protected override void myOnFirstSubscriber() {
+         _stopwatch.Start();
+         DataFrame df = DataFrame.LoadCsv(_file_address);
+         _stopwatch.Stop();
+         bts.Log.Gen.info($"CSVReaderStream.myOnFirstSubscriber stream csv load col_count: {df.Columns.Count} rows, time: {_stopwatch.ElapsedMilliseconds} msecs");
+         _stopwatch.Start();
+         IBlueRowCollection rows = BlueDataUtility.createRowsFromCSVFileWithoutTypes(_file_address);
+         _stopwatch.Stop();
+         bts.Log.Gen.info($"CSVReaderStream.myOnFirstSubscriber stream csv load count: {rows.Count} rows, time: {_stopwatch.ElapsedMilliseconds} msecs");
+         DataPacket dp_out = BlueDataUtility.build_dp_inc(rows, myLastValue as DataPacket);
+         myFireEvent(dp_out);
       }
    }
 
-   [DataStreamUserRespresentation ("CSVReaderStreamMapper")]
-   public class CSVReaderStreamMapperStream : ViewStream
-   {
-      public CSVReaderStreamMapperStream(ViewStream stream, string output_schema)
-      {
+   [DataStreamUserRespresentation("CSVReaderStreamMapper")]
+   public class CSVReaderStreamMapperStream : ViewStream {
+      public CSVReaderStreamMapperStream(ViewStream stream, string output_schema) {
          mySetRealSource(new MapStream(stream, new CSVReaderStreamMapper(output_schema)));
       }
    }
 
-   public class CSVReaderStreamMapper : IStreamMapper
-   {
+   public class CSVReaderStreamMapper : IStreamMapper {
       private IBlueTableSchema mySchema;
       private readonly string[] myOutputSchemaStringArray;
 
-      public CSVReaderStreamMapper(string outputSchema)
-      {
+      public CSVReaderStreamMapper(string outputSchema) {
          myOutputSchemaStringArray = outputSchema.Split(' ');
       }
 
-      public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema)
-      {
-         if (mySchema == null)
-         {
+      public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema) {
+         if (mySchema == null) {
             ColumnInfo[] ci = inputSchema.columnInfo;
-            foreach (string col in myOutputSchemaStringArray)
-            {
+            foreach (string col in myOutputSchemaStringArray) {
                string colName = col.Substring(0, col.IndexOf('\\')).ToLower();
-               for (int i = 0; i < ci.Length; i++)
-               {
-                  if (ci[i].colName.ToLower().Equals(colName))
-                  {
+               for (int i = 0; i < ci.Length; i++) {
+                  if (ci[i].colName.ToLower().Equals(colName)) {
                      Type t = myGetType(col.Substring(col.IndexOf('\\') + 1));
                      ci[i] = new ColumnInfo(ci[i].colName, i, t);
                      break;
@@ -171,74 +160,52 @@ namespace BCG
          return mySchema;
       }
 
-      public IBlueRow map(IBlueRow row)
-      {
+      public IBlueRow map(IBlueRow row) {
          object[] vals = new object[row.schema.columnInfo.Length];
          row.Values.CopyTo(vals, 0);
-         for (int i = 0; i < mySchema.columnInfo.Length; i++)
-         {
+         for (int i = 0; i < mySchema.columnInfo.Length; i++) {
             Type t = mySchema.columnInfo[i].colType;
-            try
-            {
+            try {
                vals[i] = System.Convert.ChangeType(vals[i], mySchema.columnInfo[i].colType, System.Globalization.CultureInfo.InvariantCulture);
-            }
-            catch
-            {
-               if (t.Equals(typeof(Int16)))
-               {
+            } catch {
+               if (t.Equals(typeof(Int16))) {
                   Int16 result;
-                  if (Int16.TryParse((string)vals[i], out result))
-                  {
+                  if (Int16.TryParse((string)vals[i], out result)) {
                      vals[i] = result;
                      continue;
                   }
                   double r2;
-                  if (Double.TryParse((string)vals[i], out r2))
-                  {
-                     try
-                     {
+                  if (Double.TryParse((string)vals[i], out r2)) {
+                     try {
                         vals[i] = (Int16)r2;
                         continue;
-                     }
-                     catch { }
+                     } catch { }
                   }
-               }
-               else if (t.Equals(typeof(Int32)))
-               {
+               } else if (t.Equals(typeof(Int32))) {
                   Int32 result;
-                  if (Int32.TryParse((string)vals[i], out result))
-                  {
+                  if (Int32.TryParse((string)vals[i], out result)) {
                      vals[i] = result;
                      continue;
                   }
                   double r2;
-                  if (Double.TryParse((string)vals[i], out r2))
-                  {
-                     try
-                     {
+                  if (Double.TryParse((string)vals[i], out r2)) {
+                     try {
                         vals[i] = (Int32)r2;
                         continue;
-                     }
-                     catch { }
+                     } catch { }
                   }
-               }
-               else if (t.Equals(typeof(Int64)))
-               {
+               } else if (t.Equals(typeof(Int64))) {
                   Int64 result;
-                  if (Int64.TryParse((string)vals[i], out result))
-                  {
+                  if (Int64.TryParse((string)vals[i], out result)) {
                      vals[i] = result;
                      continue;
                   }
                   double r2;
-                  if (Double.TryParse((string)vals[i], out r2))
-                  {
-                     try
-                     {
+                  if (Double.TryParse((string)vals[i], out r2)) {
+                     try {
                         vals[i] = (Int64)r2;
                         continue;
-                     }
-                     catch { }
+                     } catch { }
                   }
                }
                vals[i] = DBNull.Value;
@@ -247,8 +214,7 @@ namespace BCG
          return new BlueRow(mySchema, vals);
       }
 
-      private Type myGetType(string type)
-      {
+      private Type myGetType(string type) {
          type = type.ToLower();
          if (type.Contains("int"))
             return typeof(int);

src/DataStreamSuite/DataStream/Primitives/StaticCollectionStream/BlueTableStream.cs

@@ -1,31 +1,41 @@
 using System;
 using System.Collections;
 using System.Reflection;
+
 using BCG.Utilities;
 
-namespace BCG
-{
-   [DataStreamUserRespresentation (false)]
-   public class BlueRowCollectionStaticStream : ViewStream
-   {
-      public BlueRowCollectionStaticStream () {}
-
-      public BlueRowCollectionStaticStream (IBlueRowCollection rows) {loadCollection (rows);}
-
-      public void loadCollection (IBlueRowCollection rows)
-      {
-         myFireEvent (new DataPacket (rows, rows, myLastValue != null ?
-                                                   new BlueTable((myLastValue as DataPacket).newRows) : new BlueTable (rows.schema)));
+namespace BCG {
+   // if a table needs to be fired, it can be wrapped around this
+   public class BlueTableStream : ViewStream {
+      private IBlueRowCollection _rows;
+      public BlueTableStream(IBlueRowCollection rows) {
+         _rows = rows;
       }
 
-      public void crush_with_single_row(IBlueRow r)
-      {
+      protected override void myOnFirstSubscriber() {
+         var dp_out = BlueDataUtility.build_dp_inc(_rows, myLastValue as DataPacket);
+         myFireEvent(dp_out);
+      }
+   }
+
+   [DataStreamUserRespresentation(false)]
+   public class BlueRowCollectionStaticStream : ViewStream {
+      public BlueRowCollectionStaticStream() { }
+
+      public BlueRowCollectionStaticStream(IBlueRowCollection rows) { loadCollection(rows); }
+
+      public void loadCollection(IBlueRowCollection rows) {
+         myFireEvent(new DataPacket(rows, rows, myLastValue != null ?
+                                                   new BlueTable((myLastValue as DataPacket).newRows) : new BlueTable(rows.schema)));
+      }
+
+      public void crush_with_single_row(IBlueRow r) {
          BlueTable t = new BlueTable(r.schema);
          t.addRow(r);
          loadCollection(t);
       }
 
-      public void loadPacket (DataPacket dp) {myFireEvent (dp);}
+      public void loadPacket(DataPacket dp) { myFireEvent(dp); }
 
       /// <summary>
       /// Will work with streams that take ViewStream as first argument ex project, filter, distinct etc
@@ -35,17 +45,16 @@ namespace BCG
       /// <param name="streamType"></param>
       /// <param name="args"></param>
       /// <returns></returns>
-      public static IBlueRowCollection statelessOp (IBlueRowCollection rows, Type streamType, params object[] args)
-      {
-         ArrayList totArgs = new ArrayList (args);
-         BlueRowCollectionStaticStream v1 = new BlueRowCollectionStaticStream ();
-         totArgs.Insert (0, v1);
-         ViewStream retVal = Activator.CreateInstance (streamType, totArgs.ToArray()) as ViewStream;
-         v1.loadCollection (rows);
+      public static IBlueRowCollection statelessOp(IBlueRowCollection rows, Type streamType, params object[] args) {
+         ArrayList totArgs = new ArrayList(args);
+         BlueRowCollectionStaticStream v1 = new BlueRowCollectionStaticStream();
+         totArgs.Insert(0, v1);
+         ViewStream retVal = Activator.CreateInstance(streamType, totArgs.ToArray()) as ViewStream;
+         v1.loadCollection(rows);
          return retVal.rows;
       }
 
    }
 
-   
+
 }

src/DataStreamSuite/DataStream/Primitives/TextFileStreams/TextWriterStream.cs

@@ -35,14 +35,14 @@ namespace BCG
       #region ViewStream
       protected override void myOnFirstSubscriber()
       {
-         myStream.onChange +=new ChangeDelegate(myStreamChange);
+         myStream.onChange += myStreamChange;
          myVsd.addWorker(this);
       }
 
       protected override void  myOnLastSubscriber()
       {
          myVsd.removeWorker(this);
-         myStream.onChange -= new ChangeDelegate(myStreamChange);
+         myStream.onChange -= myStreamChange;
       }
       #endregion
 

src/DataStreamSuite/DataStream/Primitives/ThreadSafe/ThreadSafeStream.cs

@@ -28,7 +28,7 @@ namespace BCG
 
       protected override void myOnFirstSubscriber()
       {
-         myParentStream.onChange += new ChangeDelegate(myParentStreamChangeHandler);
+         myParentStream.onChange += myParentStreamChangeHandler;
          myVSD.addWorker(this);
       }
 
@@ -36,7 +36,7 @@ namespace BCG
 
       protected override void myOnLastSubscriber()
       {
-         myParentStream.onChange -= new ChangeDelegate(myParentStreamChangeHandler);
+         myParentStream.onChange -= myParentStreamChangeHandler;
          myVSD.removeWorker(this);
       }
 

src/DataStreamSuite/DataStream/Primitives/ThreadSafe/VSDBMarshallerStream.cs

@@ -53,11 +53,11 @@ namespace BCG
             if (r.is_subscription)
             {
                _current_source_subscription = r.subscription_id;
-               _source_stream.onChange += new ChangeDelegate(_source_stream_onChange);
+               _source_stream.onChange += _source_stream_onChange;
             }
             else
             {
-               _source_stream.onChange -= new ChangeDelegate(_source_stream_onChange);
+               _source_stream.onChange -= _source_stream_onChange;
             }
          }
       }

src/DataStreamSuite/DataStream/Primitives/Time/DateStandardizationStream.cs

@@ -80,7 +80,7 @@ namespace BCG {
 
       public IBlueRow map(IBlueRow row) {
          MutableBlueRow rv = new MutableBlueRow(getOutputSchema(row.schema), row);
-         rv.setValue(myConversionColName, mapDateTime(row[myConversionColName], myConversion));
+         rv.setValue(myConversionColName, mapDateTime(row.get_datetime(myConversionColName), myConversion));
          return rv;
          //return BlueDataUtility.getNewReplacedRow(row, mapDateTime(row[myConversionColName], myConversion), myConversionColName);
       }

src/DataStreamSuite/DataStream/Primitives/Time/DateTimeExpanstionStream.cs

@@ -0,0 +1,38 @@
+namespace bts.SilverPine {
+   using bts;
+
+   using BCG;
+
+   using System;
+
+   public class DateTimeExpanstionStream : ViewStream {
+      public DateTimeExpanstionStream(ViewStream parent_stream, string inp_col) {
+         Func<IBlueTableSchema, IBlueTableSchema> get_schema = (s) => {
+            IBlueTableSchema rv = s.add_column(System.TypeCode.DateTime, $"{inp_col}_date");
+            rv = rv.add_column(System.TypeCode.String, $"{inp_col}_day");
+            rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_day_of_month");
+            rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_month");
+            rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_hour");
+            rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_min");
+            rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_seconds_of_day");
+            return rv;
+         };
+         ViewStream rs = new MapStream(parent_stream,
+         s => get_schema(s),
+         r => {
+            DateTime dt = r.get_datetime(inp_col);
+            IBlueTableSchema s = get_schema(r.schema);
+            MutableBlueRow rv = new MutableBlueRow(s, r);
+            rv.setValue($"{inp_col}_date", dt.Date);
+            rv.setValue($"{inp_col}_day", dt.DayOfWeek.ToString());
+            rv.setValue($"{inp_col}_day_of_month", dt.Day);
+            rv.setValue($"{inp_col}_month", dt.Date.Month);
+            rv.setValue($"{inp_col}_hour", dt.Hour);
+            rv.setValue($"{inp_col}_min", dt.Minute);
+            rv.setValue($"{inp_col}_seconds_of_day", (dt - dt.Date).TotalSeconds);
+            return rv;
+         });
+         mySetRealSource(rs);
+      }
+   }
+}

src/DataStreamSuite/DataStream/Primitives/TimeAverage/TimeAverageStream.cs

@@ -24,7 +24,7 @@ namespace BCG
         protected override void myOnFirstSubscriber()
         {
             myKeyToTimeToValue = new Dictionary<string, SortedList<DateTime, double>>();
-            myToAverage.onChange += new ChangeDelegate(myToAverage_onChange);
+            myToAverage.onChange += myToAverage_onChange;
         }
 
 

src/DataStreamSuite/DataStream/Primitives/Timeseries/ColumnDifferenceStream.cs

@@ -0,0 +1,124 @@
+namespace bts.SilverPine {
+   using BCG;
+
+   using bts;
+   using bts.collections;
+
+   using System;
+   using System.CodeDom;
+   using System.Collections.Generic;
+   using System.Linq;
+
+   // Given 2 filters, and 2 columns, finds the different in the value of the 2 columns from the 2 rows
+   // so:
+   // f1(base_stream.rows)[col1] - f1(base_stream.rows)[col2]
+   [DataStreamUserRespresentation("ColDiff")]
+   public class ColumnDifferenceStream : ViewStream {
+      private readonly ViewStream _base_stream;
+      private readonly FilterStream.OnFilter _f1, _f2;
+      private readonly string _c1, _c2;
+
+      public ColumnDifferenceStream(ViewStream base_stream, FilterStream.OnFilter f1, FilterStream.OnFilter f2, string col1, string col2) {
+         _base_stream = base_stream;
+         _f1 = f1;
+         _f2 = f2;
+         _c1 = col1;
+         _c2 = col2;
+      }
+
+      public ColumnDifferenceStream(ViewStream base_stream, string f1, string f2, string col1, string col2) {
+         FilterStream.OnFilter f1_del = r => {
+            BooleanExpressionFilter rv = new BooleanExpressionFilter(f1);
+            return rv.filter(r);
+         };
+         FilterStream.OnFilter f2_del = r => {
+            BooleanExpressionFilter rv = new BooleanExpressionFilter(f2);
+            return rv.filter(r);
+         };
+         ViewStream rs = new ColumnDifferenceStream(base_stream, f1_del, f2_del, col1, col2);
+         mySetRealSource(rs);
+      }
+
+      // for each row, does a row[new_col_name] = row[col1] - rows[row_ffset][col2]
+      // we should generlize this for any math operation
+      public ColumnDifferenceStream(ViewStream parent_stream, string sort_col_name, int row_offset, string col1, string col2, string diff_col_name, double optional_perc_diff_filter = 0.0) {
+         // use GlobalMapStream
+         string diff_col_perc_name = $"{diff_col_name}_percentage";
+         System.Func<System.Collections.Generic.List<BCG.IBlueRow>, uint, BCG.IBlueRow> g_map_func = (rows, idx) => {
+            int minuend_idx = Convert.ToInt32(idx);
+            var inp_schema = rows[0].schema;
+            var out_schema = inp_schema.add_column(TypeCode.Double, diff_col_name);
+            out_schema = out_schema.add_column(TypeCode.Double, diff_col_perc_name);
+            // minuend - subtrahend
+            IBlueRow minuend_row = rows.ElementAt(minuend_idx);
+            MutableBlueRow rv = new MutableBlueRow(out_schema, minuend_row);
+            int subtrahend_index = minuend_idx + row_offset;
+            double diff_val = Double.NaN, diff_val_perc = Double.NaN;
+            if (subtrahend_index >= 0) {
+               try {
+                  IBlueRow subtrahend_row = rows.ElementAt(subtrahend_index);
+                  diff_val = minuend_row.get<double>(col1) - subtrahend_row.get<double>(col2);
+                  diff_val_perc = 100.0 * diff_val / subtrahend_row.get<double>(col1);
+               } catch (Exception ex) {
+                  Log.assert(false, ex.Message);
+               }
+            }
+            rv.setValue(diff_col_name, diff_val);
+            rv.setValue(diff_col_perc_name, diff_val_perc);
+            return rv;
+         };
+         ViewStream rs = new GlobalMapStream(parent_stream, sort_col_name, g_map_func);
+         if (optional_perc_diff_filter != 0) {
+            rs = new FilterStream(rs, r => {
+               double diff_val_perc = r.get<double>(diff_col_perc_name);
+               double abs_diff_val_perc = Math.Abs(diff_val_perc);
+               return abs_diff_val_perc >= optional_perc_diff_filter;
+            });
+         }
+         mySetRealSource(rs);
+      }
+
+      protected override void myOnFirstSubscriber() {
+         _base_stream.onChange += _handle_base_stream_update;
+      }
+
+      protected override void myOnLastSubscriber() {
+         _base_stream.onChange -= _handle_base_stream_update;
+      }
+
+      private void _handle_base_stream_update(object dp_obj) {
+         // could have been done with filter and cartesian prod, and then math
+         DataPacket dp = dp_obj as DataPacket;
+         List<IBlueRow> rows = dp.newRows.to_list();
+         var f1_rows = rows.Where(r => _f1(r));
+         //Log.assert(f1_rows.Count() == 1);
+         var f2_rows = rows.Where(r => _f2(r));
+         //Log.assert(f2_rows.Count() == 1);
+         if (f1_rows != null && f1_rows.Count() > 0 && f2_rows != null && f2_rows.Count() > 0) {
+            IBlueRow f1_r = f1_rows.ElementAt(0);
+            IBlueRow f2_r = f2_rows.ElementAt(0);
+            double diff = f1_r.get<double>(_c1) - f2_r.get<double>(_c2);
+            IBlueTableSchema out_schema = f1_r.schema.add_columns(Tuple.Create(typeof(double), "diff"));
+            BlueTable out_table = new BlueTable(out_schema);
+            MutableBlueRow out_row = new MutableBlueRow(out_schema, f1_r);
+            out_row.setValue("diff", diff);
+            out_table.addRow(out_row);
+            DataPacket dp_out = BlueDataUtility.build_dp(out_table, myLastValue as DataPacket);
+            myFireEvent(dp_out);
+         } else {
+            IBlueRow f1_r = rows.ElementAt(0);
+            IBlueRow f2_r = rows.ElementAt(0);
+            double diff = Double.NaN;
+            IBlueTableSchema out_schema = f1_r.schema.add_columns(Tuple.Create(typeof(double), "diff"));
+            BlueTable out_table = new BlueTable(out_schema);
+            MutableBlueRow out_row = new MutableBlueRow(out_schema, f1_r);
+            out_row.setValue("diff", diff);
+            out_table.addRow(out_row);
+            DataPacket dp_out = BlueDataUtility.build_dp(out_table, myLastValue as DataPacket);
+            myFireEvent(dp_out);
+         }
+
+      }
+   }
+
+}

src/DataStreamSuite/DataStream/Primitives/Timeseries/ColumnEvolutionStream.cs

@@ -0,0 +1,51 @@
+namespace bts.SilverPine {
+   using BCG;
+
+   using bts;
+
+   using System.Collections.Generic;
+   using System.Linq;
+
+   // Given a sort order, an optional group by col, it observes the values of a column 
+   // within that group (or the whole table, if grouping not specified) and returns all rows
+   // where the value changes, ie row[current][col] != row[current-1][col]
+   // TODO: a more general way to express a filter where row[current][col] != row[previous][col],
+   // or even expand it to row[current][col_a] - row[previous][col_b] < 0.5
+   [DataStreamUserRespresentation("ColEvol")]
+   public class ColumnEvolutionStream : ViewStream {
+      private readonly ViewStream _base_stream;
+      private readonly string _sort_col;
+      private readonly string _evol_col;
+      public ColumnEvolutionStream(ViewStream base_stream, string sort_col, string evolution_col) {
+         _base_stream = base_stream;
+         _sort_col = sort_col;
+         _evol_col = evolution_col;
+      }
+
+      protected override void myOnFirstSubscriber() {
+         _base_stream.onChange += _handle_base_stream_update;
+      }
+
+      protected override void myOnLastSubscriber() {
+         base.myOnLastSubscriber();
+      }
+
+      private void _handle_base_stream_update(object dp_obj) {
+         DataPacket dp = dp_obj as DataPacket;
+         List<IBlueRow> rows = dp.newRows.to_list();
+         rows.Sort((r1, r2) => BlueDataUtility.compare_rows(r2, r1, _sort_col));
+         BlueTable rv = new BlueTable(dp.schema);
+         IBlueRow prev_row = rows.ElementAt(0);
+         foreach (IBlueRow r in rows) {
+            var cr = BlueDataUtility.compare_rows(r, prev_row, _evol_col);
+            if (cr != 0) {
+               rv.addRow(r);
+            }
+            prev_row = r;
+         }
+         DataPacket dp_out = BlueDataUtility.build_dp(rv, myLastValue as DataPacket);
+         myFireEvent(dp_out);
+      }
+   }
+
+}

src/DataStreamSuite/DataStream/Primitives/Union/AlphabetizeColumnsStream.cs

@@ -12,10 +12,10 @@ namespace BCG {
          mySrc = src;
       }
       protected override void myOnFirstSubscriber() {
-         mySrc.onChange += new ChangeDelegate(mySrc_onChange);
+         mySrc.onChange += mySrc_onChange;
       }
       protected override void myOnLastSubscriber() {
-         mySrc.onChange -= new ChangeDelegate(mySrc_onChange);
+         mySrc.onChange -= mySrc_onChange;
       }
       private void mySrc_onChange(object newValue) {
          DataPacket dp = (DataPacket)newValue;

src/DataStreamSuite/DataStream/Primitives/Union/AppendStreamModified.cs

@@ -28,14 +28,14 @@ namespace BCG
 
       protected override void myOnFirstSubscriber()
       {
-         myLeftStream.onChange += new ChangeDelegate(myHandleLeftChanges);
-         myRightstream.onChange += new ChangeDelegate(myHandleRightChanges);
+         myLeftStream.onChange += myHandleLeftChanges;
+         myRightstream.onChange += myHandleRightChanges;
       }
 
       protected override void myOnLastSubscriber()
       {
-         myLeftStream.onChange -= new ChangeDelegate(myHandleLeftChanges);
-         myRightstream.onChange -= new ChangeDelegate(myHandleRightChanges);
+         myLeftStream.onChange -= myHandleLeftChanges;
+         myRightstream.onChange -= myHandleRightChanges;
       }
 
       private void myHandleLeftChanges (object newValue)

src/DataStreamSuite/DataStream/Primitives/Union/MutableAppendStream.cs

@@ -222,7 +222,7 @@ namespace BCG
             pj = new ProjectStream(pj, "!");
          myStreamIndex.Add(myMovingIndex, new Pair<ViewStream, ViewStream>(vs, pj));
          if (mySubscriptionActive)
-            pj.onChange += new ChangeDelegate(myHandleStreamChanges);
+            pj.onChange += myHandleStreamChanges;
       }
 
       public void removeStream(ViewStream vs)
@@ -235,7 +235,7 @@ namespace BCG
                index = p.Key;
                if (mySubscriptionActive)
                {
-                  p.Value.second.onChange -= new ChangeDelegate(myHandleStreamChanges);
+                  p.Value.second.onChange -= myHandleStreamChanges;
                   if (myTable != null)
                   {
                      IBlueRowCollection delRows = myTable.removeTable(p.Key);
@@ -254,14 +254,14 @@ namespace BCG
       {
          mySubscriptionActive = true;
          foreach (KeyValuePair<int, Pair<ViewStream, ViewStream>> p in myStreamIndex)
-            p.Value.second.onChange += new ChangeDelegate(myHandleStreamChanges);
+            p.Value.second.onChange += myHandleStreamChanges;
       }
 
       protected override void myOnLastSubscriber()
       {
          mySubscriptionActive = false;
          foreach (KeyValuePair<int, Pair<ViewStream, ViewStream>> p in myStreamIndex)
-            p.Value.second.onChange -= new ChangeDelegate(myHandleStreamChanges);
+            p.Value.second.onChange -= myHandleStreamChanges;
          if (myTable != null)
             myTable.clear();
       }

src/DataStreamSuite/DataStream/Primitives/Union/RenameStream.cs

@@ -14,11 +14,11 @@ namespace BCG {
       }
       protected override void myOnFirstSubscriber()
       {
-         mySrc.onChange += new ChangeDelegate(mySrc_onChange);
+         mySrc.onChange += mySrc_onChange;
       }
       protected override void myOnLastSubscriber () 
       {
-         mySrc.onChange -= new ChangeDelegate (mySrc_onChange);
+         mySrc.onChange -= mySrc_onChange;
       }
       private void mySrc_onChange (object newValue) {
          DataPacket dp = (DataPacket)newValue;

src/DataStreamSuite/DataStream/Primitives/Union/UnionStream.cs

@@ -21,15 +21,15 @@ namespace BCG
       {
          myLeftOp = new UnionOperand (true, this);
          myRightOp = new UnionOperand (false, this);
-         myLeftVs.onChange += new DataStream.ChangeDelegate (myLeftOp.myChangeHandler);
-         myRightVs.onChange += new DataStream.ChangeDelegate (myRightOp.myChangeHandler);
+         myLeftVs.onChange += myLeftOp.myChangeHandler;
+         myRightVs.onChange += myRightOp.myChangeHandler;
       }
 
       protected override void myOnLastSubscriber () 
       {
          myInit ();
-         myLeftVs.onChange -= new DataStream.ChangeDelegate (myLeftOp.myChangeHandler);
-         myRightVs.onChange -= new DataStream.ChangeDelegate (myRightOp.myChangeHandler);
+         myLeftVs.onChange -= myLeftOp.myChangeHandler;
+         myRightVs.onChange -= myRightOp.myChangeHandler;
       }
 
       public void startSession ()
@@ -116,4 +116,4 @@ namespace BCG
           receiveStream(dp, isLeft);
       }
    }
-}
+}

src/DataStreamSuite/DataStream/Publish/PublisherStream.cs

@@ -14,11 +14,11 @@ namespace BCG {
 
       protected override void myOnFirstSubscriber() {
          _vs_publisher = new ViewPublisher(_name, _broker, _main_stream);
-         _main_stream.onChange += new DataStream.ChangeDelegate(_handle_update);
+         _main_stream.onChange += _handle_update;
       }
 
       protected override void myOnLastSubscriber() {
-         _main_stream.onChange -= new DataStream.ChangeDelegate(_handle_update);
+         _main_stream.onChange -= _handle_update;
          _vs_publisher.Dispose();
       }
 
@@ -28,4 +28,4 @@ namespace BCG {
 
 
    }
-}
+}

src/DataStreamSuite/DataStream/Publish/ViewPublisher.cs

@@ -20,11 +20,11 @@ namespace BCG {
       public ViewPublisher(string name, IPubTableBroker broker, ViewStream vs) {
          _stream = String.IsNullOrEmpty(name) ? vs : new ChangeTableNameStream(vs, name);
          _broker = broker;
-         _stream.onChange += new DataStream.ChangeDelegate(_handle_update);
+         _stream.onChange += _handle_update;
       }
 
       public void Dispose() {
-         _stream.onChange -= new DataStream.ChangeDelegate(_handle_update);
+         _stream.onChange -= _handle_update;
       }
 
       private void _handle_update(object o) {

src/DataStreamSuite/DataStream/TableUpdateDataPacket.cs

@@ -0,0 +1,73 @@
+namespace Bts.DataStream;
+
+using BCG;
+
+using bts.api.core;
+
+using Bts.DataStreamCore;
+
+using System;
+using System.Collections;
+using System.Collections.Generic;
+
+
+public class TableUpdateDataPacket<T> : IDataPacket<T> where T : IBlueRow {
+   readonly TableUpdate<T, int> table;
+   readonly IBlueTableSchema schema;
+
+   public TableUpdateDataPacket(TableUpdate<T, int> table, IBlueTableSchema schema) {
+      this.table = table;
+      this.schema = schema;
+   }
+
+   public IBlueRowCollection<T> newRows {
+      get {
+         var rv = new BlueTableImmutable<T>(schema);
+         return rv;
+      }
+   }
+
+   public IBlueRowCollection<T> addedRows => throw new NotImplementedException();
+
+   public IBlueRowCollection<T> deletedRows => throw new NotImplementedException();
+
+   class MyRowCollection : IBlueRowCollection<T> {
+      readonly TableUpdate<T, int> table;
+      readonly IEnumerable<T> items;
+      Lazy<SortedList<int, T>> itemList;
+
+      public MyRowCollection(IEnumerable<T> items, IBlueTableSchema schema) {
+         this.items = items;
+         this.schema = schema;
+         itemList = new(() => {
+            var rv = new SortedList<int, T>();
+            foreach (var item in this.items) {
+               rv.Add(item.Id(), item);
+            }
+            return rv;
+         });
+      }
+
+      public int Count => itemList.Value.Count;
+      public bool IsSynchronized => false;
+      public object SyncRoot => this;
+
+      public T this[int id] => itemList.Value[id];
+
+      public IBlueTableSchema schema { get; }
+
+      public bool containsID(int id) =>
+         itemList.Value.ContainsKey(id);
+
+      public void CopyTo(Array array, int index) {
+      }
+
+      public IEnumerator<T> GetEnumerator() {
+         throw new NotImplementedException();
+      }
+
+      IEnumerator IEnumerable.GetEnumerator() {
+         throw new NotImplementedException();
+      }
+   }
+}

src/DataStreamSuite/DataStreamCore/BlueData/BlueColumnTable.cs

@@ -0,0 +1,127 @@
+namespace Bts.DataStreamCore;
+
+using BCG;
+
+using bts.utils;
+
+using mbts.utils;
+
+using Spark.Core;
+
+using System;
+
+using System.Collections;
+using System.Collections.Generic;
+using System.Linq;
+
+public class BlueColumnTable : IBlueRowCollection {
+   readonly Variant[][] columns;
+   readonly List<int> rowIds;
+   readonly int idColIndex;
+
+   public BlueColumnTable(IBlueTableSchema schema, int capacity = 4) {
+      ExceptionAssert.Argument.GreaterThanEqualTo(capacity, 0);
+      rowIds = new(capacity);
+      columns = new Variant[schema.columnInfo.Length][];
+      if (columns.Length == 0) {
+         throw new ArgumentException("Schema contains no columns", nameof(schema));
+      }
+      if (!schema.columnInfo.Select(c => c.colName).TryFindIndex("id", out idColIndex)) {
+         throw new ArgumentException("Schema contains no 'id' column", nameof(schema));
+      }
+      this.schema = schema;
+      for (int i = 0; i < columns.Length; i++) {
+         columns[schema.columnInfo[i].colNo] = new Variant[capacity];
+      }
+   }
+
+   public int Count => rowIds.Count;
+   public IBlueTableSchema schema { get; }
+
+   public IBlueRow this[int id] {
+      get {
+         var idx = rowIds.BinarySearch(id);
+         if (idx < 0) {
+            throw new KeyNotFoundException($"No row with id {id}");
+         }
+         return new MyRow(this, idx);
+      }
+   }
+
+   public void Add(Variant[] values) {
+      var id = values[idColIndex].VInt32;
+      var idx = rowIds.BinarySearch(id);
+      if (idx >= 0) {
+         throw new ArgumentException($"Row with id {id} already exists.", nameof(values));
+      }
+      idx = ~idx;
+      for (int i = 0; i < columns.Length; i++) {
+         int len = rowIds.Count;
+         ArrayHelper.Insert(ref columns[i], idx, values[i], ref len);
+      }
+      rowIds.Insert(idx, id);
+   }
+
+   public void AddOrUpdate(Variant[] values) {
+      var id = values[idColIndex].VInt32;
+      var idx = rowIds.BinarySearch(id);
+      if (idx >= 0) {
+         rowIds[idx] = id;
+         for (int i = 0; i < columns.Length; i++) {
+            columns[idx][i] = values[i];
+         }
+         return;
+      }
+      idx = ~idx;
+      int len;
+      for (int i = 0; i < columns.Length; i++) {
+         len = rowIds.Count;
+         ArrayHelper.Insert(ref columns[i], idx, values[i], ref len);
+      }
+      rowIds.Insert(idx, id);
+   }
+
+   public void Clear() =>
+      rowIds.Clear();
+
+   public bool Remove(int id) {
+      var idx = rowIds.BinarySearch(id);
+      if (idx < 0) {
+         return false;
+      }
+      rowIds.RemoveAt(idx);
+      for (int i = 0; i < columns.Length; i++) {
+         ArrayHelper.remove_at(ref columns[i], idx);
+      }
+      return true;
+   }
+
+   public void Trim() {
+      for (int i = 0; i < columns.Length; i++) {
+         ArrayHelper.trim(ref columns[i], rowIds.Count);
+      }
+   }
+
+   public bool containsID(int id) =>
+      rowIds.BinarySearch(id) >= 0;
+
+   public IEnumerator GetEnumerator() {
+      var count = rowIds.Count;
+      for (int i = 0; i < count; i++) {
+         yield return new MyRow(this, i);
+      }
+   }
+
+   class MyRow : IBlueRow {
+      readonly BlueColumnTable table;
+      readonly int rowIdx;
+
+      public MyRow(BlueColumnTable table, int rowIdx) {
+         this.table = table;
+         this.rowIdx = rowIdx;
+      }
+
+      public object this[int colNo] => table.columns[colNo][rowIdx].GetValue();
+      public IBlueTableSchema schema => table.schema;
+   }
+}

src/DataStreamSuite/DataStreamCore/BlueData/BlueDataUtility.cs

@@ -15,16 +15,209 @@ using mbts;
 using BCG.Extensions;
 using System.Text.Json;
 using System.Text.Json.Nodes;
+using Newtonsoft.Json;
+using BCG;
+using System.Runtime.CompilerServices;
+using bts;
+using Spark.Core;
+using System.Numerics;
+
+namespace bts {
+
+   // public class CompositeRow : IBlueRow {
+   //    private IBlueRow _lrow, _rrow;
+   //    private BlueRow _out_row;
+   //    public CompositeRow(IBlueRow lrow, IBlueRow rrow) {
+   //       _lrow = lrow;
+   //       _rrow = rrow;
+   //       IBlueTableSchema schema = _lrow.schema;
+   //       List<object> vals = new List<object>();
+   //       foreach (var o in _lrow) {
+   //          vals.Add(o);
+   //       }
+   //       foreach (ColumnInfo ci in _rrow.schema.columnInfo) {
+   //          if (ci.colName == "id") {
+   //             continue;
+   //          }
+   //          vals.Add(_rrow[ci.colNo]);
+   //          schema = schema.add_column(ci.type_code(), ci.colName);
+   //       }
+   //       _out_row = new BlueRow(schema, vals.ToArray());
+   //    }
+   //    public IBlueTableSchema schema {
+   //       get { return _out_row.schema; }
+   //    }
+   //    public object? this[int index] {
+   //       get { return _out_row[index]; }
+   //    }
+   //    public object? this[string colName] {
+   //       get {
+   //          return _out_row[colName];
+   //       }
+   //    }
+
+   // }
+}
 
 namespace BCG {
    public static class BlueDataUtility {
+      public static IBlueRow create_composite_row(IBlueRow lr, IBlueRow rr, int id) {
+         IBlueTableSchema schema = lr.schema;
+         List<object> vals = new List<object>();
+         foreach (ColumnInfo ci in lr.schema.columnInfo) {
+            if (ci.colName == "id") {
+               vals.Add(id);
+            }
+            vals.Add(lr[ci.colName]);
+         }
+         foreach (ColumnInfo ci in rr.schema.columnInfo) {
+            if (ci.colName == "id") {
+               continue;
+            }
+            schema = schema.add_column(ci.type_code(), ci.colName);
+            vals.Add(rr[ci.colName]);
+         }
+         IBlueRow r = new BlueRow(schema, vals.ToArray());
+         return r;
+      }
       #region Extensions
+      public static bool is_null(this IBlueRow r, string col_name) {
+         object col_val = r[col_name];
+         return col_val == null || DBNull.Value.Equals(col_val);
+      }
+
       public static DateTime get_datetime(this IBlueRow r, string col_name, DateTime err_value = default(DateTime)) {
          return BlueDataUtility.get_row_datetime_value(r, col_name, err_value);
       }
       public static DateTime get_datetime(this IBlueRow r, int col_no, DateTime err_value = default(DateTime)) {
          return BlueDataUtility.get_row_datetime_value(r, col_no, err_value);
       }
+
+      public static double? GetFloat64(this IBlueRow r, int colNo) =>
+         GetFloat64(r[colNo]);
+
+      public static double? GetFloat64(this IBlueRow r, string colName) =>
+         GetFloat64(r[colName]);
+
+      public static double? GetFloat64(object obj) {
+         if (obj is double d) {
+            return d;
+         }
+         if (obj is int i32) {
+            return i32;
+         }
+         if (obj is long i64) {
+            return i64;
+         }
+         if (obj is string s) {
+            return double.TryParse(s, out d) ? d : null;
+         }
+         if (obj is uint u32) {
+            return u32;
+         }
+         if (obj is ulong u64) {
+            return u64;
+         }
+         if (obj is short i16) {
+            return i16;
+         }
+         if (obj is ushort u16) {
+            return u16;
+         }
+         return null;
+      }
+
+      public static int? GetInt32(this IBlueRow r, int colNo) =>
+         GetInt32(r[colNo]);
+
+      public static int? GetInt32(this IBlueRow r, string colName) =>
+         GetInt32(r[colName]);
+
+      public static int? GetInt32(object obj) {
+         checked {
+            if (obj is double d) {
+               return (int)d;
+            }
+            if (obj is int i32) {
+               return i32;
+            }
+            if (obj is long i64) {
+               return (int)i64;
+            }
+            if (obj is string s) {
+               return double.TryParse(s, out d) ? (int)d : null;
+            }
+            if (obj is uint u32) {
+               return (int)u32;
+            }
+            if (obj is ulong u64) {
+               return (int)u64;
+            }
+            if (obj is short i16) {
+               return i16;
+            }
+            if (obj is ushort u16) {
+               return u16;
+            }
+         }
+         return null;
+      }
+
+      public static long? GetInt64(this IBlueRow r, int colNo) =>
+         GetInt64(r[colNo]);
+
+      public static long? GetInt64(this IBlueRow r, string colName) =>
+         GetInt64(r[colName]);
+
+      public static long? GetInt64(object obj) {
+         checked {
+            if (obj is double d) {
+               return (long)d;
+            }
+            if (obj is int i32) {
+               return i32;
+            }
+            if (obj is long i64) {
+               return i64;
+            }
+            if (obj is string s) {
+               return double.TryParse(s, out d) ? (long)d : null;
+            }
+            if (obj is uint u32) {
+               return u32;
+            }
+            if (obj is ulong u64) {
+               return (long)u64;
+            }
+            if (obj is short i16) {
+               return i16;
+            }
+            if (obj is ushort u16) {
+               return u16;
+            }
+         }
+         return null;
+      }
+
+      public static DateTime? GetUnixEpochDateTime(this IBlueRow row, string colName) =>
+         GetUnixEpochDateTime(row[colName]);
+
+      public static DateTime? GetUnixEpochDateTime(this IBlueRow row, int colNo) =>
+         GetUnixEpochDateTime(row[colNo]);
+
+      public static DateTime? GetUnixEpochDateTime(object obj) {
+         var tsNanos = GetInt64(obj);
+         return tsNanos.HasValue ? DateTime.UnixEpoch.Add(new TimeSpan(tsNanos.Value / 100)) : null;
+      }
+
+      public static DateTime? GetDateTime(object obj) {
+         var res = _get_datetime_value(obj, DateTime.MinValue);
+         if (res == DateTime.MinValue) {
+            return null;
+         }
+         return res;
+      }
+
       public static T get_enum<T>(this IBlueRow r, string col_name, T err_value = default(T)) where T : struct {
          return BlueDataUtility.get_row_enum_value<T>(r, col_name, err_value);
       }
@@ -49,6 +242,17 @@ namespace BCG {
             rv.Add(r.get<T>(col));
          return rv;
       }
+      public static IEnumerable<T> get_colvalues<T>(IEnumerable<IBlueRow> rows, string col, int start_index, int count) {
+         List<T> rv = new List<T>();
+         int i = 0;
+         int n_copied = 0;
+         foreach (IBlueRow r in rows)
+            if (i++ >= start_index && n_copied < count) {
+               rv.Add(r.get<T>(col));
+               n_copied++;
+            }
+         return rv;
+      }
 
       public static IDictionary<T, IEnumerable<IBlueRow>> group_by<T>(IBlueRowCollection rows, string gp_col) {
          Dictionary<T, List<IBlueRow>> rv = new Dictionary<T, List<IBlueRow>>();
@@ -112,13 +316,17 @@ namespace BCG {
          }
       }
 
-      public static IEnumerable<IBlueRow> to_ienum(this IBlueRowCollection rows) {
+      public static List<IBlueRow> to_list(this IBlueRowCollection rows) {
          List<IBlueRow> rv = new List<IBlueRow>();
          foreach (IBlueRow r in rows)
             rv.Add(r);
          return rv;
       }
 
+      public static IEnumerable<IBlueRow> to_ienum(this IBlueRowCollection rows) {
+         return rows.to_list();
+      }
+
       public static IDictionary<int, IBlueRow> to_map(this IBlueRowCollection rows) {
          Dictionary<int, IBlueRow> rv = new Dictionary<int, IBlueRow>();
          foreach (IBlueRow r in rows) {
@@ -128,6 +336,10 @@ namespace BCG {
          return rv;
       }
 
+      public static bool contains_column(this IBlueTableSchema input_schema, string col) {
+         return input_schema.columnNames.Contains(col);
+      }
+
       public static IBlueTableSchema add_columns(this IBlueTableSchema input_schema, params Tuple<Type, string>[] cols) {
          List<ColumnInfo> cis = new List<ColumnInfo>(input_schema.columnInfo);
          foreach (Tuple<Type, string> c in cols)
@@ -135,6 +347,11 @@ namespace BCG {
          return new BlueTableSchema(input_schema.tableName, cis.ToArray());
       }
 
+      public static IBlueTableSchema add_column(this IBlueTableSchema input_schema, TypeCode type_arg, string col_name) {
+         Tuple<Type, string> tup = new Tuple<Type, string>(_typecode_to_type[type_arg], col_name);
+         return add_columns(input_schema, tup);
+      }
+
       public static IBlueRowCollection to_collection(this IBlueRow r) {
          if (r == null)
             return null;
@@ -191,6 +408,18 @@ namespace BCG {
 
       #region json
 
+      public static Dictionary<string, string> convert_json_to_dictionary(string json_string) {
+         Dictionary<string, string> dictionary = JsonConvert.DeserializeObject<Dictionary<string, string>>(json_string);
+         return dictionary;
+      }
+      public static IBlueRow convert_json_to_row(string json_string, int id = 0) {
+         Dictionary<string, object> dictionary = JsonConvert.DeserializeObject<Dictionary<string, object>>(json_string);
+         if (!dictionary.ContainsKey("id")) {
+            dictionary.Add("id", id);
+         }
+         return new BlueRow(dictionary);
+      }
+
       public static List<JsonNode> json_to_json_node_list(string jsonString) {
          var jsonNodeList = new List<JsonNode>();
          bool noe = String.IsNullOrEmpty(jsonString);
@@ -209,7 +438,7 @@ namespace BCG {
             foreach (JsonNode jsonNode in jsonArray) {
                jsonNodeList.Add(jsonNode);
             }
-         } catch (JsonException ex) {
+         } catch (System.Text.Json.JsonException ex) {
             Console.WriteLine($"Invalid JSON: {ex.Message}");
          }
 
@@ -486,7 +715,16 @@ namespace BCG {
          ColumnInfo[] cis = new ColumnInfo[table.Columns.Count];
          foreach (DataColumn col in table.Columns)
             cis[col.Ordinal] = new ColumnInfo(col.ColumnName, col.Ordinal, col.DataType);
-         return new BlueTableSchema(cis, table.TableName);
+         IBlueTableSchema rv = new BlueTableSchema(cis, table.TableName);
+         rv = rv.add_id_col_if_missing();
+         return rv;
+      }
+      public static IBlueTableSchema add_id_col_if_missing(this IBlueTableSchema schema) {
+         IBlueTableSchema rv = schema;
+         if (!schema.columnNames.Contains("id")) {
+            rv = schema.add_columns(Tuple.Create(typeof(int), "id"));
+         }
+         return rv;
       }
 
       /// <summary>
@@ -744,10 +982,16 @@ namespace BCG {
       }
 
       public static IBlueRowCollection toIBlueRowCollection(DataTable dt) {
-         BlueTable retVal = new BlueTable(getSchema(dt));
-         foreach (DataRow dr in dt.Rows)
-            retVal.addRow(dr);
-         return retVal;
+         IBlueTableSchema schema = getSchema(dt);
+         BlueTable rv = new BlueTable(schema);
+         foreach (DataRow dr in dt.Rows) {
+            MutableBlueRow nr = new MutableBlueRow(schema);
+            foreach (DataColumn col in dr.Table.Columns)
+               nr.setValue(col.ColumnName, dr[col.ColumnName]);
+            nr.setValue("id", rv.Count);
+            rv.addRow(nr);
+         }
+         return rv;
       }
 
       public static IBlueRowCollection toIBlueRowCollection(DataRow[] rows) {
@@ -819,20 +1063,20 @@ namespace BCG {
       /// </summary>
       /// <param name="r1"></param>
       /// <param name="r2"></param>
-      /// <param name="colNames"></param>
+      /// <param name="col_names"></param>
       /// <returns></returns>
       public static int compare_rows(IBlueRow r1,
          IBlueRow r2,
-         params string[] colNames) {
-         int retVal = 0;
-         foreach (string colName in colNames) {
-            object val1 = r1[colName];
-            object val2 = r2[colName];
-            retVal = _compare_row_cols(val1, val2);
-            if (retVal != 0)
+         params string[] col_names) {
+         int comp_res = 0;
+         foreach (string cn in col_names) {
+            object val1 = r1[cn];
+            object val2 = r2[cn];
+            comp_res = _compare_row_cols(val1, val2);
+            if (comp_res != 0)
                break;
          }
-         return retVal;
+         return comp_res;
       }
 
       private static int _compare_row_cols(object v1, object v2) {
@@ -1608,8 +1852,10 @@ namespace BCG {
 
       public static int[] get_column_indices(IBlueTableSchema schema, string[] col_names) {
          List<int> rv = new List<int>();
-         foreach (string c in col_names)
+         foreach (string c in col_names) {
+            Log.assert(schema.contains_column(c), $"Could not find col: {c} in table: {schema.tableName}");
             rv.Add(schema[c].colNo);
+         }
          return rv.ToArray();
       }
 
@@ -2096,23 +2342,57 @@ namespace BCG {
          try {
             if (v == null || v == DBNull.Value)
                return error_val;
-            else {
-               if (v is DateTime)
-                  return (DateTime)v;
-               else if (v is string) {
-                  DateTime rv_dt = error_val;
-                  DateTime.TryParse((string)v, out rv_dt);
+
+            if (v is DateTime) {
+               return (DateTime)v;
+            } else if (v is string) {
+               DateTime rv_dt;
+
+               // Try parsing the string as a DateTime
+               if (DateTime.TryParse((string)v,
+                                     System.Globalization.CultureInfo.InvariantCulture,
+                                     System.Globalization.DateTimeStyles.RoundtripKind,
+                                     out rv_dt)) {
                   return rv_dt;
                } else {
-                  double oa_date = (double)Convert.ChangeType(v, typeof(Double));
-                  return DateTime.FromOADate(oa_date);
+                  return error_val; // Return the error value if parsing fails
                }
+            } else {
+               // Assume the input can be converted to a double and use FromOADate
+               double oa_date = (double)Convert.ChangeType(v, typeof(Double));
+               return DateTime.FromOADate(oa_date);
             }
          } catch {
             return error_val;
          }
       }
 
+      public static TimeSpan? get_timespan(this IBlueRow r, int col_no) =>
+         get_timespan(r[col_no]);
+
+      public static TimeSpan? get_timespan(this IBlueRow r, string col_name) =>
+         get_timespan(r[col_name]);
+
+      public static TimeSpan? get_timespan(object obj) {
+         if (obj is TimeSpan ts) {
+            return ts;
+         }
+         var str = obj as string ?? obj?.ToString();
+         if (str is null) {
+            return null;
+         }
+         if (TimeSpan.TryParse(str, out ts)) {
+            return ts;
+         }
+         if (str.StartsWith("PT")) {
+            try {
+               // Parse ISO 8601 duration
+               return System.Xml.XmlConvert.ToTimeSpan(str);
+            } catch {
+            }
+         }
+         return null;
+      }
       #endregion
 
       #region CSV methods

src/DataStreamSuite/DataStreamCore/BlueData/BlueRowCollection.cs

@@ -0,0 +1,56 @@
+#nullable enable
+namespace Bts.DataStreamCore;
+
+using BCG;
+
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Linq;
+
+public class BlueRowCollection : IBlueRowCollection {
+   readonly ICollection source;
+   readonly Func<int, IBlueRow> findRow;
+   readonly Func<int, bool> containsKey;
+
+   public BlueRowCollection(ICollection source, IBlueTableSchema schema, Func<int, IBlueRow>? findRow = null, Func<int, bool>? containsKey = null) {
+      this.source = source;
+      this.schema = schema;
+      if (findRow is null || containsKey is null) {
+         if (source is IBlueRowCollection brc) {
+            findRow ??= id => brc[id];
+            containsKey ??= brc.containsID;
+         } else if (source is IDictionary dict) {
+            findRow ??= id => dict[id] as IBlueRow ?? throw new KeyNotFoundException();
+            containsKey ??= id => dict.Contains(id);
+         } else if (source is IDictionary<int, IBlueRow> genDict) {
+            findRow ??= id => genDict[id];
+            containsKey ??= genDict.ContainsKey;
+         } else if (source is IReadOnlyDictionary<int, IBlueRow> roGenDict) {
+            findRow ??= id => roGenDict[id];
+            containsKey = roGenDict.ContainsKey;
+         } else {
+            var cachedRows = new Lazy<Dictionary<int, IBlueRow>>(() =>
+               source.Cast<IBlueRow>().ToDictionary(r => r.Id()));
+            findRow ??= id => cachedRows.Value[id];
+            // Don't remove this lambda expression. Intellisense may suggest it, but that would
+            // instantiate 'cachedRows' immediately and defeat the purpose of using Lazy<T>.
+            containsKey ??= id => cachedRows.Value.ContainsKey(id);
+         }
+      }
+      this.findRow = findRow;
+      this.containsKey = containsKey;
+   }
+
+   public IBlueRow this[int id] => findRow(id);
+
+   public IBlueTableSchema schema { get; }
+
+   public int Count => source.Count;
+
+   public bool containsID(int id) =>
+      containsKey(id);
+
+   public IEnumerator GetEnumerator() =>
+      source.GetEnumerator();
+}

src/DataStreamSuite/DataStreamCore/BlueData/BlueRowColumnMask.cs

@@ -0,0 +1,64 @@
+#nullable enable
+namespace Bts.DataStreamCore;
+
+using BCG;
+
+using bts.collections;
+using bts.utils;
+
+using System;
+using System.Collections.Generic;
+
+
+public class BlueRowColumnMask : IBlueRow {
+   readonly IBlueRow row;
+   readonly int[] hiddenColumns;
+
+   public BlueRowColumnMask(IBlueRow row, params int[] hiddenColumns) {
+      this.row = row;
+      this.hiddenColumns = hiddenColumns;
+      Array.Sort(hiddenColumns);
+   }
+
+   /// <summary>
+   /// Provides a way to explicitly skip sorting hiddenColumns. This is done at the callers risk,
+   /// as the index (int) operator performs a binary search on hiddenColumns.
+   /// </summary>
+   /// <param name="row">The underlying row whose columns to hide</param>
+   /// <param name="sortHiddenColumns">Whether or not to sort <paramref name="hiddenColumns"/> in
+   /// the constructor. They are sorted by default to ensure correct behavior in the index (int)
+   /// operator, which performs a binary search on this array.</param>
+   /// <param name="hiddenColumns">The columns (indices) to hide</param>
+   public BlueRowColumnMask(IBlueRow row, bool sortHiddenColumns, params int[] hiddenColumns) {
+      this.row = row;
+      this.hiddenColumns = hiddenColumns;
+      if (sortHiddenColumns) {
+         Array.Sort(hiddenColumns);
+      }
+   }
+
+   public object? this[string colName] {
+      get {
+         for (int i = 0; i < hiddenColumns.Length; i++) {
+            if (row.schema.columnInfo[i].colName == colName) {
+               throw new KeyNotFoundException();
+            }
+         }
+         return row[colName];
+      }
+   }
+   public object? this[int index] {
+      get {
+         if (index > row.schema.columnInfo.Length - hiddenColumns.Length) {
+            throw new IndexOutOfRangeException();
+         }
+         int pos = hiddenColumns.BinarySearch(index);
+         if (pos < 0) {
+            pos = ~pos;
+         }
+         return row[index + pos];
+      }
+   }
+
+   public IBlueTableSchema schema => row.schema;
+}

src/DataStreamSuite/DataStreamCore/BlueData/BlueRowEnumerator.cs

@@ -7,7 +7,7 @@ namespace BCG
    /// Used for enumerating an IEnumerable (ex ICollection). Furthermore, this is the 
    /// default Enueramtor used when enumerating asn IBlueRow
    /// </summary>
-   public class BlueRowEnumerator : IEnumerator
+   public class BlueRowEnumerator : IDictionaryEnumerator
       {
          protected int myPos = -1;
          protected int myRowSize = -1;
@@ -18,6 +18,11 @@ namespace BCG
             myRow = row;
             myRowSize = row.schema.columnNames.Length;
          }
+
+         public DictionaryEntry Entry =>
+            new(myRow.schema.columnInfo[myPos].colName, myRow[myPos]);
+         public object Key => myRow.schema.columnInfo[myPos].colName;
+         public object Value => myRow[myPos];
          
          #region IEnumerator members
          public void Reset()  {myPos = 0;}

src/DataStreamSuite/DataStreamCore/BlueData/BlueRowKeyCollection.cs

@@ -0,0 +1,30 @@
+#nullable enable
+namespace BCG;
+
+using System;
+using System.Collections;
+
+public class BlueRowKeyCollection : ICollection {
+   readonly ColumnInfo[] columns;
+
+   public BlueRowKeyCollection(ColumnInfo[] columns) =>
+      this.columns = columns;
+
+   public int Count => columns.Length;
+
+   public bool IsSynchronized => false;
+
+   public object SyncRoot => this;
+
+   public void CopyTo(Array array, int index) {
+      for (int i = 0; i < columns.Length; i++) {
+         array.SetValue(columns[i].colName, i + index);
+      }
+   }
+
+   public IEnumerator GetEnumerator() {
+      for (int i = 0; i < columns.Length; i++) {
+         yield return columns[i].colName;
+      }
+   }
+}