check pt: Combine lib_py_market_data and provider py scripts into one project

I'm not sure why, but my sparkd build (linux) suddenly began failing to
generate cpp headers for protos. The error message indicated it was due
to an unexpected carriage return (\r) in plugin.py:

/usr/bin/env: ‘python\r’: No such file or directory

.config/dotnet-tools.json

@@ -25,12 +25,6 @@
       "commands": [
         "fake"
       ]
-    },
-    "dotnet-t4": {
-      "version": "3.0.0",
-      "commands": [
-        "t4"
-      ]
     }
   }
-}
+}

.devcontainer/Dockerfile

@@ -87,10 +87,6 @@ RUN dpkg -i /emumba/emumba-deribit-connector-3.3.1-Linux.LICENSE_DISABLED.deb \
 # Pcap
 RUN apt-get update && apt-get install -y libpcap0.8-dev
 
-# Databento (deps)
-RUN apt-get update && apt-get -y install libzstd-dev
-
-# Docker in Docker
 RUN apt-get update && apt-get install -y \
     docker.io \
     docker-compose \

.env

@@ -1,81 +0,0 @@
-# This file defines the global default configuration for docker deployments.
-# It is tracked in source control. For temporary modifications, please create
-# a separate file called ".env.local" (it's in .gitignore, so won't be tracked).
-# If a variable is defined in both files, the value in .env.local will be used.
-#
-# About the deployment procedure:
-#   We use Docker Compose to build images and launch containers for various services
-#   that make up our software suite:
-#     - sparkd
-#     - pubsub
-#     - md-publisher
-#     - webapp (DSQA on the web)
-#     - api-server (websocket based)
-#   From inside a devcontainer, you can launch all services in separate containers
-#   on your machine by running:
-#
-#     tools/deploy.sh
-#
-#   This script builds release versions for each service, packaged (generally) in
-#   self-contained directories. Next, it runs a few "docker compose" commands to
-#   build docker images loaded with the self-contained binaries from the previous
-#   step, and start containers with those images to run each service. To view the
-#   commands that deploy.sh runs internally (without actually running them), run:
-#
-#     tools/deploy.sh --dry-run
-#
-#   For additional information/help, run:
-#
-#     tools/deploy.sh --help
-#
-
-### Sparkd ###
-# The internal image for sparkd to use. For more info, view:
-#   src/app_sparkd/Dockerfile
-SPARKD__IMAGE=sparkd
-# The 'spark.conf' for this service to use
-SPARKD__CONF_FILE=src/app_sparkd/spark.conf
-# To run replay md, put recorded md files in SPARKD__MD_RECORDING_DIR, and use:
-#   SPARKD__IMAGE=sparkd_md_replay
-#   SPARKD__CONF_FILE=src/app_sparkd/spark.local.conf
-# (configure which md to replay in the new SPARKD__CONF_FILE)
-SPARKD__MD_RECORDING_DIR=.md_recording
-SPARKD__SOURCE_DIR=install/sparkd-release-gcc
-SPARKD__CONFIG_PORT=8000
-SPARKD__QUICKBUS_PORT=8001
-SPARKD__SIM_FIX_PORT=8002
-
-### Pubsub ###
-# The 'spark.conf' for this service to use
-PUBSUB__CONF_FILE=src/DataStreamSuite/PubSubServer/spark.conf
-PUBSUB__CS_SOURCE_DIR=build/Release/bin/net7.0/linux-x64/publish
-PUBSUB__CPP_SOURCE_DIR=install/data-stream-suite-linux
-# The appsettings to load at runtime. Expects a file named:
-#   src/DataStreamSuite/PubSub/appsettings.${PUBSUB__DOTNET_ENVIRONMENT}.json
-PUBSUB__DOTNET_ENVIRONMENT=Staging
-
-### Md Publisher ###
-# The 'spark.conf' for this service to use
-MD_PUBLISHER__CONF_FILE=src/DataStreamSuite/DataStreamPublisher/spark.conf
-MD_PUBLISHER__SOURCE_DIR=build/Release/bin/net7.0/linux-x64/publish
-# Determines which appsettings to load at runtime. Expects a file named:
-#   src/DataStreamSuite/DataStreamPublisher/appsettings.${MD_PUBLISHER__DOTNET_ENVIRONMENT}.json
-MD_PUBLISHER__DOTNET_ENVIRONMENT=MarketData
-
-### Webapp (DSQA on the web) ###
-# The 'spark.conf' for this service to use
-WEBAPP__CONF_FILE=src/client/spark_api_server/spark.conf
-WEBAPP__SOURCE_DIR=build/Release/bin/net7.0/linux-x64/publish
-WEBAPP__WEB_ROOT=build/Bts.DataStreamWebApp.Client/production/dist
-# Determines which appsettings to load at runtime. Expects a file named:
-#   src/DataStreamSuite/Bts.DataStreamWebApp.Server/appsettings.${WEBAPP__ASPNETCORE_ENVIRONMENT}.json
-WEBAPP__ASPNETCORE_ENVIRONMENT=Staging
-
-### Api Server ###
-# The 'spark.conf' for this service to use
-API_SERVER__CONF_FILE=src/client/spark_api_server/spark.conf
-API_SERVER__CS_SOURCE_DIR=build/Release/bin/net7.0/linux-x64/publish
-API_SERVER__CPP_SOURCE_DIR=install/spark-api-server-linux
-# Determines which appsettings to load at runtime. Expects a file named:
-#   src/client/spark_api_server/appsettings.${API_SERVER__ASPNETCORE_ENVIRONMENT}.json
-API_SERVER__ASPNETCORE_ENVIRONMENT=Staging

.gitignore

@@ -59,6 +59,3 @@ trace_data/
 .devcontainer/
 *.Generated.cs
 *.lst
-src/app_sparkd/spark.local.conf
-.env.local
-.md_recording

.vscode/launch.json

@@ -16,20 +16,11 @@
         "program": "spark_api_server.exe",
         "args": [
           "run",
+          "--sparkd",
+          "http://127.0.0.1:8000/config"
         ],
         "cwd": "build/Debug/bin/net7.0",
       },
-      "linux": {
-        "preLaunchTask": "Build Api Server (Debug)",
-        "program": "spark_api_server",
-        "args": [
-          "run",
-        ],
-        "cwd": "build/Debug/bin/net7.0",
-      },
-      "env": {
-        "ASPNETCORE_ENVIRONMENT": "Development",
-      },
       "stopAtEntry": false,
       "console": "integratedTerminal"
     },
@@ -42,19 +33,11 @@
         "program": "spark_api_server.exe",
         "args": [
           "run",
+          "--sparkd",
+          "http://127.0.0.1:8000/config"
         ],
         "cwd": "build/Debug/bin/net7.0",
       },
-      "linux": {
-        "program": "spark_api_server",
-        "args": [
-          "run",
-        ],
-        "cwd": "build/Debug/bin/net7.0",
-      },
-      "env": {
-        "ASPNETCORE_ENVIRONMENT": "Development",
-      },
       "stopAtEntry": false,
       "console": "integratedTerminal"
     },
@@ -62,7 +45,7 @@
       "name": "Bts.DataStreamWebApp.Server",
       "type": "coreclr",
       "request": "launch",
-      "preLaunchTask": "Build DataStream (Debug)",
+      "preLaunchTask": "Build DataStream WebApp (Debug)",
       "program": "${workspaceFolder}/build/Debug/bin/net7.0/Bts.DataStreamWebApp.Server.dll",
       "cwd": "${workspaceFolder}/build/Debug/bin/net7.0",
       "env": {
@@ -122,18 +105,6 @@
       "stopAtEntry": false,
       "console": "integratedTerminal"
     },
-    {
-      "name": "(gdb) Launch",
-      "type": "cppdbg",
-      "request": "launch",
-      "program": "/home/bts/spark/build/sparkd-debug-clang/server/sparkd",
-      "args": ["run"],
-      "stopAtEntry": false,
-      "cwd": "/home/bts/spark/build/sparkd-debug-clang/server",
-      "environment": [],
-      "externalConsole": false,
-      "MIMode": "gdb",
-    },
     {
       "name": "DataStreamQueryAnalyzer",
       "type": "coreclr",
@@ -232,12 +203,12 @@
       "preLaunchTask": "Build Sparkd (Debug | Clang)",
       "program": "",
       "linux": {
-        "program": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd/sparkd",
+        "program": "${workspaceFolder}/build/sparkd-debug-clang/server/sparkd",
       },
       "args": [
         "run"
       ],
-      "cwd": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd",
+      "cwd": "${workspaceFolder}/build/sparkd-debug-clang/server",
       "stopAtEntry": false,
       "externalConsole": false,
       "MIMode": "gdb",
@@ -260,12 +231,12 @@
       "request": "launch",
       "program": "",
       "linux": {
-        "program": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd/sparkd",
+        "program": "${workspaceFolder}/build/sparkd-debug-clang/server/sparkd",
       },
       "args": [
         "run"
       ],
-      "cwd": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd",
+      "cwd": "${workspaceFolder}/build/sparkd-debug-clang/server",
       "stopAtEntry": false,
       "externalConsole": false,
       "MIMode": "gdb",
@@ -281,35 +252,7 @@
           "ignoreFailures": true
         }
       ]
-    },
-    {
-      "name": "Sparkd Web (Skip Build)",
-      "type": "cppdbg",
-      "request": "launch",
-      "program": "",
-      "linux": {
-        "program": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd/sparkd",
-      },
-      "args": [
-        "web"
-      ],
-      "cwd": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd",
-      "stopAtEntry": false,
-      "externalConsole": false,
-      "MIMode": "gdb",
-      "setupCommands": [
-        {
-          "description": "Enable pretty-printing for gdb",
-          "text": "-enable-pretty-printing",
-          "ignoreFailures": true
-        },
-        {
-          "description": "Set Disassembly Flavor to Intel",
-          "text": "-gdb-set disassembly-flavor intel",
-          "ignoreFailures": true
-        }
-      ]
-    },
+    }
   ],
   "inputs": [
     {

CMakeLists.txt

@@ -18,7 +18,7 @@ set(
 # SHARED_MEMORY_PREFIX serves as a kind of namespace so multiple applications
 # can run simultaneously without fighting over the same shm files (e.g., sparkd,
 # api server, etc.).
-set(SHARED_MEMORY_PREFIX "\${BTS_SHARED_MEMORY_PREFIX}" CACHE STRING "Prefix to use when creating shared memory files")
+set(SHARED_MEMORY_PREFIX "bts" CACHE STRING "Prefix to use when creating shared memory files")
 set(EXTERNALS_BIN "${EXTERNALS_ROOT}/bin")
 set(EXTERNALS_INCLUDE "${EXTERNALS_ROOT}/include")
 set(EXTERNALS_LIB "${EXTERNALS_ROOT}/lib/$<CONFIG>")
@@ -33,85 +33,95 @@ else()
   set(BUILD_ADMIN_WEB_APP ON CACHE BOOL "Build the Admin WebApp")
 endif()
 
+include_directories(BEFORE SYSTEM ${EXTERNALS_INCLUDE} ${EXTERNALS_ROOT}/include/zlib)
+link_directories(${EXTERNALS_LIB})
+
 add_compile_definitions(SHARED_MEMORY_PREFIX="${SHARED_MEMORY_PREFIX}")
 if(DEFINED APP_DATA_DIR)
   add_compile_definitions(APP_DATA_DIR="${APP_DATA_DIR}")
 endif()
 add_compile_definitions(EXPORT_MD_CONSUMER="${EXPORT_MD_CONSUMER}")
+if(WIN32)
+  # subdirs
+  add_subdirectory(src/lib_spark)
+else()
+  # make sparkd look for libspark.so in its directory
+  # set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE)
+  # set(CMAKE_INSTALL_RPATH ".")
+  # set(CMAKE_INSTALL_PREFIX "${CMAKE_SOURCE_DIR}/install/$<CONFIG>")
 
-if(BUILD_SPARKD)
-  # Note: All macros that begin w/ 'CCAPI' are ccapi's macros (defined and/or
-  # read by their code). We override some below.
-  add_compile_definitions(SPARK_ENABLE_CCAPI)
+  if(BUILD_SPARKD)
+    # Note: All macros that begin w/ 'CCAPI' are ccapi's macros (defined and/or
+    # read by their code). We override some below.
+    add_compile_definitions(SPARK_ENABLE_CCAPI)
 
-  # This allows us to override hardcoded exchange-urls with ones read from
-  # config (i.e., resolved at runtime rather than compile-time)
-  add_compile_definitions(CCAPI_EXPOSE_INTERNAL)
+    # This allows us to override hardcoded exchange-urls with ones read from
+    # config (i.e., resolved at runtime rather than compile-time)
+    add_compile_definitions(CCAPI_EXPOSE_INTERNAL)
 
-  # Enable ccapi services
-  add_compile_definitions(CCAPI_ENABLE_SERVICE_MARKET_DATA)
-  add_compile_definitions(CCAPI_ENABLE_SERVICE_EXECUTION_MANAGEMENT)
-  # Enable ccapi exchanges
-  add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE)
-  add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE_COIN_FUTURES)
-  add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE_US)
-  add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE_USDS_FUTURES)
-  add_compile_definitions(CCAPI_ENABLE_EXCHANGE_COINBASE)
-  add_compile_definitions(CCAPI_ENABLE_EXCHANGE_DERIBIT)
-  add_compile_definitions(CCAPI_ENABLE_EXCHANGE_OKX)
+    # Enable ccapi services
+    add_compile_definitions(CCAPI_ENABLE_SERVICE_MARKET_DATA)
+    add_compile_definitions(CCAPI_ENABLE_SERVICE_EXECUTION_MANAGEMENT)
+    # Enable ccapi exchanges
+    add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE)
+    add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE_COIN_FUTURES)
+    add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE_US)
+    add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE_USDS_FUTURES)
+    add_compile_definitions(CCAPI_ENABLE_EXCHANGE_COINBASE)
+    add_compile_definitions(CCAPI_ENABLE_EXCHANGE_DERIBIT)
+    add_compile_definitions(CCAPI_ENABLE_EXCHANGE_OKX)
 
-  set(CCAPI_LOG_LEVEL
-    "Warn"
-    CACHE STRING "<Fatal|Error|Warn|Info|Debug|Trace> Log level for ccapi")
-  string(TOUPPER "${CCAPI_LOG_LEVEL}" _ccapi_log_level_upper)
-  add_compile_definitions("CCAPI_ENABLE_LOG_${_ccapi_log_level_upper}")
+    set(CCAPI_LOG_LEVEL
+      "Warn"
+      CACHE STRING "<Fatal|Error|Warn|Info|Debug|Trace> Log level for ccapi")
+    string(TOUPPER "${CCAPI_LOG_LEVEL}" _ccapi_log_level_upper)
+    add_compile_definitions("CCAPI_ENABLE_LOG_${_ccapi_log_level_upper}")
 
-  # ** Binance **
-  add_compile_definitions(CCAPI_BINANCE_URL_REST_BASE="https://api.binance.com")
-  add_compile_definitions(BINANCE_INSTRUMENTS_PATH="/api/v3/exchangeInfo")
-  add_compile_definitions(BINANCE_PRICE_STATS_PATH="/api/v3/ticker/24hr")
+    # ** Binance **
+    add_compile_definitions(CCAPI_BINANCE_URL_REST_BASE="https://api.binance.com")
+    add_compile_definitions(BINANCE_INSTRUMENTS_PATH="/api/v3/exchangeInfo")
+    add_compile_definitions(BINANCE_PRICE_STATS_PATH="/api/v3/ticker/24hr")
 
-  # ** Binance US **
-  add_compile_definitions(BINANCE_US_INSTRUMENTS_PATH="/api/v3/exchangeInfo")
-  add_compile_definitions(BINANCE_US_PRICE_STATS_PATH="/api/v3/ticker/24hr")
+    # ** Binance US **
+    add_compile_definitions(BINANCE_US_INSTRUMENTS_PATH="/api/v3/exchangeInfo")
+    add_compile_definitions(BINANCE_US_PRICE_STATS_PATH="/api/v3/ticker/24hr")
 
-  # ** Binance USDS Futures **
-  add_compile_definitions(CCAPI_BINANCE_USDS_FUTURES_URL_REST_BASE="https://dapi.binance.com")
-  add_compile_definitions(BINANCE_USDS_FUTURES_INSTRUMENTS_PATH="/dapi/v1/exchangeInfo")
-  add_compile_definitions(BINANCE_USDS_FUTURES_PRICE_STATS_PATH="/dapi/v1/ticker/24hr")
+    # ** Binance USDS Futures **
+    add_compile_definitions(CCAPI_BINANCE_USDS_FUTURES_URL_REST_BASE="https://dapi.binance.com")
+    add_compile_definitions(BINANCE_USDS_FUTURES_INSTRUMENTS_PATH="/dapi/v1/exchangeInfo")
+    add_compile_definitions(BINANCE_USDS_FUTURES_PRICE_STATS_PATH="/dapi/v1/ticker/24hr")
 
-  # ** Deribit **
-  add_compile_definitions(DERIBIT_INSTRUMENT_PATH="/api/v2/public/get_instrument")
-  add_compile_definitions(DERIBIT_INSTRUMENTS_PATH="/api/v2/public/get_instruments")
-  add_compile_definitions(DERIBIT_COMBOS_PATH="/api/v2/public/get_combos")
-  add_compile_definitions(DERIBIT_TICKER_PATH="/api/v2/public/ticker")
-  add_compile_definitions(DERIBIT_CURRENCIES_METHOD="public/get_currencies")
-  add_compile_definitions(DERIBIT_INSTRUMENT_METHOD="public/get_instrument")
-  add_compile_definitions(DERIBIT_INSTRUMENTS_METHOD="public/get_instruments")
-  add_compile_definitions(DERIBIT_COMBO_DETAILS_METHOD="public/get_combo_details")
-  add_compile_definitions(DERIBIT_COMBOS_METHOD="public/get_combos")
+    # ** Deribit **
+    add_compile_definitions(DERIBIT_INSTRUMENT_PATH="/api/v2/public/get_instrument")
+    add_compile_definitions(DERIBIT_INSTRUMENTS_PATH="/api/v2/public/get_instruments")
+    add_compile_definitions(DERIBIT_COMBOS_PATH="/api/v2/public/get_combos")
+    add_compile_definitions(DERIBIT_TICKER_PATH="/api/v2/public/ticker")
+    add_compile_definitions(DERIBIT_CURRENCIES_METHOD="public/get_currencies")
+    add_compile_definitions(DERIBIT_INSTRUMENT_METHOD="public/get_instrument")
+    add_compile_definitions(DERIBIT_INSTRUMENTS_METHOD="public/get_instruments")
+    add_compile_definitions(DERIBIT_COMBO_DETAILS_METHOD="public/get_combo_details")
+    add_compile_definitions(DERIBIT_COMBOS_METHOD="public/get_combos")
 
-  # ** Okx **
-  add_compile_definitions(OKX_INSTRUMENTS_PATH="/api/v5/public/instruments")
-  add_compile_definitions(OKX_UNDERLYING_PATH="/api/v5/public/underlying")
-endif()
+    # ** Okx **
+    add_compile_definitions(OKX_INSTRUMENTS_PATH="/api/v5/public/instruments")
+    add_compile_definitions(OKX_UNDERLYING_PATH="/api/v5/public/underlying")
+  endif()
 
-# subdirs
-add_subdirectory(src/lib_system)
-add_subdirectory(src/lib_domain)
-add_subdirectory(src/lib_http)
-if (BUILD_SPARKD)
-  add_subdirectory(src/lib_python_md)
-  add_subdirectory(src/app_sparkd)
-  add_subdirectory(src/data_files)
-endif()
-add_subdirectory(src/lib_spark)
-if (BUILD_ADMIN_WEB_APP)
-  add_subdirectory(src/Spark.Admin.WebApp)
-endif()
-if(BUILD_DOCS)
-  add_subdirectory(doc/user)
-endif()
-if(BUILD_SDK)
-  add_subdirectory(src/sdk)
+  # subdirs
+  if (BUILD_SPARKD)
+    add_subdirectory(src/app_sparkd)
+    add_subdirectory(src/data_files)
+    add_subdirectory(src/lib_py_market_data)
+    # add_subdirectory(src/py_market_data_providers)
+  endif()
+  add_subdirectory(src/lib_spark)
+  if (BUILD_ADMIN_WEB_APP)
+    add_subdirectory(src/Spark.Admin.WebApp)
+  endif()
+  if(BUILD_DOCS)
+    add_subdirectory(doc/user)
+  endif()
+  if(BUILD_SDK)
+    add_subdirectory(src/sdk)
+  endif()
 endif()

CMakePresets.json

@@ -28,7 +28,7 @@
         "CMAKE_SHARED_LINKER_FLAGS_RELEASE": "/OPT:REF /OPT:ICF /LTCG",
         "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
         "EXTERNALS_ROOT": "${sourceDir}/externals/windows",
-        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf",
+        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
         "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0;${sourceDir}/build/$<CONFIG>/bin/net7.0-windows",
         "EXPORT_MD_CONSUMER": {
           "type": "BOOL",
@@ -53,7 +53,6 @@
         "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
         "CMAKE_PREFIX_PATH": "${sourceDir}/cmake/;${sourceDir}/.venv/lib/python3.10/site-packages/pybind11/share/cmake/pybind11",
         "CMAKE_EXE_LINKER_FLAGS": "-Wno-undef",
-        "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
         "EXTERNALS_ROOT": "${sourceDir}/externals/linux",
         "BUILD_SDK": {
           "type": "BOOL",
@@ -77,7 +76,7 @@
           "value": "ON"
         },
         "CCAPI_LOG_LEVEL": "Info",
-        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/md_dev/*.conf;spark/config/profiles/default/*.conf"
+        "CONFIG_DEFAULTS_PATH": "config/profiles/md_dev/*.conf;config/profiles/default/*.conf"
       }
     },
 
@@ -88,7 +87,7 @@
       "cacheVariables": {
         "CMAKE_BUILD_TYPE": "Release",
         "CCAPI_LOG_LEVEL": "Warn",
-        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/md_prod/*.conf;spark/config/profiles/default/*.conf"
+        "CONFIG_DEFAULTS_PATH": "config/profiles/md_prod/*.conf;config/profiles/default/*.conf"
       }
     },
 
@@ -168,8 +167,8 @@
           "type": "BOOL",
           "value": "ON"
         },
-        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
+        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/Spark.Api.Server/$<CONFIG>/bin/net7.0",
         "EXPORT_MD_CONSUMER": {
           "type": "BOOL",
           "value": "ON"
@@ -207,7 +206,7 @@
       "description": "Data Stream utils ported over from Edge",
       "binaryDir": "${sourceDir}/build/${presetName}",
       "cacheVariables": {
-        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf;spark/config/profiles/data_stream/*.conf",
+        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf",
         "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0;${sourceDir}/build/$<CONFIG>/bin/net7.0-windows"
       }
     },
@@ -240,7 +239,8 @@
         "CMAKE_CXX_COMPILER": "clang++",
         "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
         "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
-        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf;spark/config/profiles/data_stream/*.conf",
+        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf",
+        "CONFIG_PROFILES": "default;data_stream",
         "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0;${sourceDir}/build/$<CONFIG>/bin/net8.0",
         "EXPORT_MD_CONSUMER": {
           "type": "BOOL",
@@ -294,14 +294,6 @@
       "inherits": "sparkd"
     },
 
-    {
-      "name": "sparkd-debug-clang-j6",
-      "configurePreset": "sparkd-debug-clang",
-      "displayName": "Sparkd (Debug | Clang | jobs=6)",
-      "inherits": "sparkd",
-      "jobs": 6
-    },
-
     {
       "name": "sparkd-debug-gcc",
       "configurePreset": "sparkd-debug-gcc",

Spark.Api.Server.sln

@@ -14,36 +14,6 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "spark_test", "src\client\sp
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Spark.Console", "src\client\Spark.Console\Spark.Console.csproj", "{89D666E6-B500-498F-B026-4D167F68ADFC}"
 EndProject
-Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{267F81EF-A8AB-49FD-9F56-D7EC3C1F7C7C}"
-EndProject
-Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "DataStreamSuite", "DataStreamSuite", "{C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamCore", "src\DataStreamSuite\DataStreamCore\DataStreamCore.csproj", "{EF26A475-6101-4C9C-A717-D60BB92A2302}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SharedStreams", "src\DataStreamSuite\SharedStreams\SharedStreams.csproj", "{911B08A3-515A-441D-84C8-8A43740B5FC1}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PubSub", "src\DataStreamSuite\PubSub\PubSub.csproj", "{14140310-2233-430A-821B-F3058A25234B}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BaseUtilities", "src\DataStreamSuite\BaseUtilities\BaseUtilities.csproj", "{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ExpressionCore", "src\DataStreamSuite\ExpressionCore\ExpressionCore.csproj", "{6F22DB96-6EAB-4034-99D9-287021C36F01}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ArithmeticExpressions", "src\DataStreamSuite\ArithmeticExpressions\ArithmeticExpressions.csproj", "{A5C33956-A41F-49B3-A41D-1082CBA59D02}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamExpressions", "src\DataStreamSuite\DataStreamExpressions\DataStreamExpressions.csproj", "{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnSortExpressions", "src\DataStreamSuite\Bts.ColumnSortExpressions\Bts.ColumnSortExpressions.csproj", "{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ColumnMapExpressions", "src\DataStreamSuite\ColumnMapExpressions\ColumnMapExpressions.csproj", "{B945A183-2174-4ACB-B289-8A5C546656FE}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BooleanExpressions", "src\DataStreamSuite\BooleanExpressions\BooleanExpressions.csproj", "{9F6D7DE8-185B-4C1D-B809-3314D092DF28}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnExpressions", "src\DataStreamSuite\Bts.ColumnExpressions\Bts.ColumnExpressions.csproj", "{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStream", "src\DataStreamSuite\DataStream\DataStream.csproj", "{39AEA075-6457-46C0-8821-1158BE6EFA57}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Net", "src\DataStreamSuite\Bts.Net\Bts.Net.csproj", "{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}"
-EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|x64 = Debug|x64
@@ -100,110 +70,6 @@ Global
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.Release|x64.Build.0 = Release|x64
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|x64.ActiveCfg = Debug|x64
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|x64.Build.0 = Debug|x64
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Debug|x64.Build.0 = Debug|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Release|x64.ActiveCfg = Release|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Release|x64.Build.0 = Release|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Debug|x64.Build.0 = Debug|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Release|x64.ActiveCfg = Release|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Release|x64.Build.0 = Release|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.Debug|x64.Build.0 = Debug|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.Release|x64.ActiveCfg = Release|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.Release|x64.Build.0 = Release|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Debug|x64.Build.0 = Debug|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Release|x64.ActiveCfg = Release|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Release|x64.Build.0 = Release|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Debug|x64.Build.0 = Debug|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Release|x64.ActiveCfg = Release|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Release|x64.Build.0 = Release|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Debug|x64.Build.0 = Debug|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Release|x64.ActiveCfg = Release|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Release|x64.Build.0 = Release|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Debug|x64.Build.0 = Debug|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Release|x64.ActiveCfg = Release|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Release|x64.Build.0 = Release|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Debug|x64.Build.0 = Debug|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Release|x64.ActiveCfg = Release|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Release|x64.Build.0 = Release|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.Debug|x64.Build.0 = Debug|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.Release|x64.ActiveCfg = Release|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.Release|x64.Build.0 = Release|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Debug|x64.Build.0 = Debug|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Release|x64.ActiveCfg = Release|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Release|x64.Build.0 = Release|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Debug|x64.Build.0 = Debug|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Release|x64.ActiveCfg = Release|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Release|x64.Build.0 = Release|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Debug|x64.Build.0 = Debug|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Release|x64.ActiveCfg = Release|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Release|x64.Build.0 = Release|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.Debug|x64.Build.0 = Debug|Any CPU
-		{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.Release|x64.ActiveCfg = Release|Any CPU
-		{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.Release|x64.Build.0 = Release|Any CPU
-		{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -214,19 +80,4 @@ Global
 	GlobalSection(Performance) = preSolution
 		HasPerformanceSessions = true
 	EndGlobalSection
-	GlobalSection(NestedProjects) = preSolution
-		{C09ED521-D2DF-40BC-A76F-EB3CB870B2F6} = {267F81EF-A8AB-49FD-9F56-D7EC3C1F7C7C}
-		{EF26A475-6101-4C9C-A717-D60BB92A2302} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-		{911B08A3-515A-441D-84C8-8A43740B5FC1} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-		{14140310-2233-430A-821B-F3058A25234B} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-		{6F22DB96-6EAB-4034-99D9-287021C36F01} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-		{B945A183-2174-4ACB-B289-8A5C546656FE} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-		{39AEA075-6457-46C0-8821-1158BE6EFA57} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-	EndGlobalSection
 EndGlobal

cmake/ExportToParent.cmake

@@ -1,7 +0,0 @@
-include_guard(GLOBAL)
-
-macro(export_to_parent)
-  foreach(var IN ITEMS ${ARGN})
-    set(${var} ${${var}} PARENT_SCOPE)
-  endforeach()
-endmacro()

cmake/InitExecutable.cmake

@@ -0,0 +1,66 @@
+function(init_executable)
+  set(source_dir)
+  if (${ARGC} EQUAL 0)
+    set(source_dir ${PROJECT_NAME})
+  elseif(${ARGC} GREATER 1)
+    message(FATAL_ERROR "init_shared_library received invalid arguments. Usage: init_shared_library(<OPTIONAL:source_dir>)")
+  else()
+    set(source_dir ${ARGV})
+  endif()
+
+  set(_options)
+  set(_one_value_args CXX_SOURCES CXX_HEADERS CXX_OUT_DIR)
+  set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+  if (${_ARGS_PROTO_SOURCES} EQUAL "GLOB_RECURSIVE")
+    file(
+      GLOB_RECURSE ${PROJECT_NAME}_PROTO_SOURCES
+      RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
+      *.proto
+    )
+  endif()
+
+  set(${PROJECT_NAME}_SOURCES PARENT_SCOPE)
+  file(
+    GLOB_RECURSE ${PROJECT_NAME}_SOURCES
+    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
+    ${source_dir}/*.cpp
+  )
+  set(${PROJECT_NAME}_SOURCES ${${PROJECT_NAME}_SOURCES} PARENT_SCOPE)
+  # set(${PROJECT_NAME}_Sources ${${PROJECT_NAME}_Sources} PARENT_SCOPE)
+  # Note: cpp files inside 'tests' dir(s) are considered tests.
+  # Remove tests from main sources, and add them to test sources
+  set(${PROJECT_NAME}_TEST_SOURCES)
+  foreach(item ${${PROJECT_NAME}_SOURCES})
+    if(${item} MATCHES "tests/*.cpp")
+      list(REMOVE_ITEM ${PROJECT_NAME}_SOURCES ${item})
+      list(APPEND ${PROJECT_NAME}_TEST_SOURCES ${item})
+    endif()
+  set(${PROJECT_NAME}_TEST_SOURCES ${${PROJECT_NAME}_TEST_SOURCES} PARENT_SCOPE)
+  endforeach()
+
+  # Add executable
+  add_executable(${PROJECT_NAME} ${${PROJECT_NAME}_SOURCES})
+  target_include_directories(
+    ${PROJECT_NAME}
+    PUBLIC
+      $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
+      $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>
+  )
+
+  target_link_directories(
+    ${PROJECT_NAME}
+    PUBLIC
+      $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
+      $<INSTALL_INTERFACE:${CMAKE_INSTALL_LIBDIR}>
+  )
+
+  # Add tests executable (if there are any)
+  if (${PROJECT_NAME}_TEST_SOURCES)
+    add_executable(${PROJECT_NAME}-tests ${${PROJECT_NAME}_TEST_SOURCEs})
+    target_link_libraries(${PROJECT_NAME}-tests PRIVATE ${PROJECT_NAME})
+    # Catch2
+    find_package(Catch2 CONFIG REQUIRED COMPONENTS Catch2WithMain)
+    target_link_libraries(${PROJECT_NAME}-tests PRIVATE Catch2::Catch2WithMain)
+  endif()
+endfunction()

cmake/InitLibrary.cmake

@@ -1,142 +0,0 @@
-include_guard(DIRECTORY)
-include(${CMAKE_SOURCE_DIR}/cmake/ExportToParent.cmake)
-include(${CMAKE_SOURCE_DIR}/cmake/InstallBasic.cmake)
-include(${CMAKE_SOURCE_DIR}/cmake/ProtobufGenerateCpp.cmake)
-include(${CMAKE_SOURCE_DIR}/cmake/SetFirstDefined.cmake)
-include(${CMAKE_SOURCE_DIR}/cmake/safe_dep_glob.cmake)
-
-function(init_library)
-  set(_options SHARED STATIC)
-  set(_one_value_args
-    NAME
-    SOURCE_DIR
-    INCLUDE_DIR
-  )
-  set(_multi_value_args)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-  if (_ARGS_SHARED AND _ARGS_STATIC)
-    message(FATAL_ERROR "Invalid args passed to init_library: cannot specify STATIC and SHARED")
-  endif()
-  set_first_defined(_name _ARGS_NAME PROJECT_NAME)
-  set_first_defined(_source_dir _ARGS_SOURCE_DIR _name)
-  set_first_defined(_include_dir _ARGS_INCLUDE_DIR _source_dir)
-
-  # Cpp sources
-  file(
-    GLOB_RECURSE ${_name}_SOURCES
-    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
-    ${_source_dir}/*.cpp
-  )
-  # Note: cpp files inside 'tests' dir(s) are considered tests.
-  # Remove tests from main sources, and add them to test sources
-  foreach(item ${${_name}_SOURCES})
-    if(${item} MATCHES "tests/.*\.cpp$")
-      list(APPEND ${_name}_TEST_SOURCES ${item})
-    endif()
-  endforeach()
-  foreach(item ${${_name}_TEST_SOURCES})
-    list(REMOVE_ITEM ${_name}_SOURCES ${item})
-  endforeach()
-
-  # Proto sources 
-  file(
-    GLOB_RECURSE ${_name}_PROTO_SOURCES
-    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
-    ${_source_dir}/protobufs/*.proto
-  )
-  update_deps_file("${${_name}_PROTO_SOURCES};")
-  protobuf_generate_cpp(
-    CXX_SOURCES ${_name}_CXX_PROTO_SOURCES
-    CXX_HEADERS ${_name}_CXX_PROTO_HEADERS
-    CXX_OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}
-    PROTO_INCLUDE_DIRS
-      ${EXTERNALS_INCLUDE}
-      ${CMAKE_CURRENT_SOURCE_DIR}
-    PROTO_SOURCES
-      ${${_name}_PROTO_SOURCES}
-  )
-
-  export_to_parent(
-    ${_name}_SOURCES
-    ${_name}_TEST_SOURCES
-    ${_name}_PROTO_SOURCES
-    ${_name}_CXX_PROTO_SOURCES
-    ${_name}_CXX_PROTO_HEADERS
-  )
-
-  # Add main lib
-  if (_ARGS_STATIC)
-    add_library(${_name} STATIC ${${_name}_SOURCES} ${${_name}_CXX_PROTO_SOURCES})
-  elseif(_ARGS_SHARED)
-    add_library(${_name} SHARED ${${_name}_SOURCES} ${${_name}_CXX_PROTO_SOURCES})
-  else()
-    add_library(${_name} ${${_name}_SOURCES} ${${_name}_CXX_PROTO_SOURCES})
-  endif()
-  target_include_directories(
-    ${_name}
-    SYSTEM
-    BEFORE
-    PUBLIC
-      $<BUILD_INTERFACE:${EXTERNALS_ROOT}/include>
-      $<BUILD_INTERFACE:${EXTERNALS_ROOT}/include/zlib>
-  )
-  target_include_directories(
-    ${_name}
-    PUBLIC
-      $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
-      $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>
-  )
-  # Include protos (if any)
-  if (${_name}_PROTO_SOURCES)
-    target_include_directories(
-      ${_name}
-      PUBLIC
-        $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
-    )
-  endif()
-  target_link_directories(
-    ${_name}
-    PUBLIC
-      $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
-      $<BUILD_INTERFACE:${EXTERNALS_ROOT}/lib/$<CONFIG>>
-      $<INSTALL_INTERFACE:${CMAKE_INSTALL_LIBDIR}>
-  )
-  # Install main lib
-  foreach(_hdr ${${_name}_CXX_PROTO_HEADERS})
-    file(RELATIVE_PATH _relative_path ${CMAKE_CURRENT_BINARY_DIR} ${_hdr})
-    install(FILES ${_hdr} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/${_relative_path})
-  endforeach()
-  install_basic(
-    NAME ${_name}
-    INCLUDE_DIR ${_include_dir}
-  )
-
-  # Add tests executable (if there are any)
-  if (${_name}_TEST_SOURCES AND NOT WIN32)
-    add_executable(${_name}-tests ${${_name}_TEST_SOURCES})
-
-    get_target_property(_lib_type ${_name} TYPE)
-    if (_lib_type STREQUAL "SHARED_LIBRARY")
-      # Set RPATH so the tests exe knows where to find the shared lib.
-      set_target_properties(
-        ${_name}-tests
-        PROPERTIES
-          BUILD_RPATH "$ORIGIN"
-          INSTALL_RPATH "$ORIGIN/../${CMAKE_INSTALL_LIBDIR}"
-      )
-    endif()
-
-    # Link main lib
-    target_link_directories(
-      ${_name}-tests
-      PRIVATE
-        $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
-        $<INSTALL_INTERFACE:${CMAKE_INSTALL_LIBDIR}>
-    )
-    target_link_libraries(${_name}-tests PRIVATE ${_name})
-
-    # Catch2
-    find_package(Catch2 CONFIG REQUIRED COMPONENTS Catch2WithMain)
-    target_link_libraries(${_name}-tests PRIVATE Catch2::Catch2WithMain)
-  endif()
-endfunction()

cmake/InitSharedLibrary.cmake

@@ -0,0 +1,52 @@
+function(init_shared_library)
+  set(source_dir)
+  if (${ARGC} EQUAL 0)
+    set(source_dir ${PROJECT_NAME})
+  elseif(${ARGC} GREATER 1)
+    message(FATAL_ERROR "init_shared_library received invalid arguments. Usage: init_shared_library(<OPTIONAL:source_dir>)")
+  else()
+    set(source_dir ${ARGV})
+  endif()
+  set(${PROJECT_NAME}_Sources PARENT_SCOPE)
+  file(
+    GLOB_RECURSE ${PROJECT_NAME}_Sources
+    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
+    ${source_dir}/*.cpp)
+  set(${PROJECT_NAME}_Sources ${${PROJECT_NAME}_Sources} PARENT_SCOPE)
+  # set(${PROJECT_NAME}_Sources ${${PROJECT_NAME}_Sources} PARENT_SCOPE)
+  # Note: cpp files inside 'tests' dir(s) are considered tests.
+  # Remove tests from main sources, and add them to test sources
+  set(${PROJECT_NAME}_Test_Sources)
+  foreach(item ${${PROJECT_NAME}_Sources})
+    if(${item} MATCHES "tests/*.cpp")
+      list(REMOVE_ITEM ${PROJECT_NAME}_Sources ${item})
+      list(APPEND ${PROJECT_NAME}_Test_Sources ${item})
+    endif()
+  set(${PROJECT_NAME}_Test_Sources ${${PROJECT_NAME}_Test_Sources} PARENT_SCOPE)
+  endforeach()
+
+  # Add main library
+  add_library(${PROJECT_NAME} SHARED ${${PROJECT_NAME}_Sources})
+  target_include_directories(
+    ${PROJECT_NAME}
+    PUBLIC
+      $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
+      $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>
+  )
+
+  target_link_directories(
+    ${PROJECT_NAME}
+    PUBLIC
+      $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
+      $<INSTALL_INTERFACE:${CMAKE_INSTALL_LIBDIR}>
+  )
+
+  # Add tests executable (if there are any)
+  if (${PROJECT_NAME}_Test_Sources)
+    add_executable(${PROJECT_NAME}-tests ${${PROJECT_NAME}_Test_Sources})
+    target_link_libraries(${PROJECT_NAME}-tests PRIVATE ${PROJECT_NAME})
+    # Catch2
+    find_package(Catch2 CONFIG REQUIRED COMPONENTS Catch2WithMain)
+    target_link_libraries(${PROJECT_NAME}-tests PRIVATE Catch2::Catch2WithMain)
+  endif()
+endfunction(init_shared_library)

cmake/InstallBasic.cmake

@@ -1,39 +0,0 @@
-include_guard(DIRECTORY)
-include(GNUInstallDirs)
-include(${CMAKE_SOURCE_DIR}/cmake/PackBasic.cmake)
-include(${CMAKE_SOURCE_DIR}/cmake/SetFirstDefined.cmake)
-
-function(install_basic)
-  set(_options)
-  set(_one_value_args NAME INCLUDE_DIR)
-  set(_multi_value_args)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-  set_first_defined(_name _ARGS_NAME PROJECT_NAME)
-  set_first_defined(_include_dir _ARGS_INCLUDE_DIR _name)
-
-  file(
-    GLOB_RECURSE ${_name}_HEADERS
-    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
-    ${_include_dir}/*.h)
-
-  target_sources(${_name}
-    PUBLIC
-      FILE_SET HEADERS FILES ${${_name}_HEADERS})
-
-  install(
-    TARGETS ${_name}
-    EXPORT ${_name}Targets
-    RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
-    LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
-    ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
-    FILE_SET HEADERS DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}
-  )
-
-  install(
-    EXPORT ${_name}Targets
-    FILE ${_name}Targets.cmake
-    NAMESPACE ${_name}::
-    DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name})
-
-  pack_basic()
-endfunction()

cmake/InstallBasicConfig.cmake

@@ -0,0 +1,29 @@
+include(${CMAKE_SOURCE_DIR}/cmake/PackBasic.cmake)
+
+function(install_basic_config)
+  file(
+    GLOB_RECURSE ${PROJECT_NAME}_HEADERS
+    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
+    *.h)
+
+  target_sources(${PROJECT_NAME}
+    PUBLIC
+      FILE_SET HEADERS FILES ${${PROJECT_NAME}_HEADERS})
+
+  install(
+    TARGETS ${PROJECT_NAME}
+    EXPORT ${PROJECT_NAME}Targets
+    RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
+    LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
+    ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
+    FILE_SET HEADERS DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}
+  )
+
+  install(
+    EXPORT ${PROJECT_NAME}Targets
+    FILE ${PROJECT_NAME}Targets.cmake
+    NAMESPACE ${PROJECT_NAME}::
+    DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME})
+
+  pack_basic()
+endfunction()

cmake/PackBasic.cmake

@@ -1,44 +1,21 @@
-include_guard(DIRECTORY)
 include(CMakePackageConfigHelpers)
-include(${CMAKE_SOURCE_DIR}/cmake/SetFirstDefined.cmake)
 
 function(pack_basic)
-  set(_options)
-  set(_one_value_args NAME)
-  set(_multi_value_args)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-  set_first_defined(_name _ARGS_NAME PROJECT_NAME)
-
-  find_package(Git QUIET)
-
-  if(GIT_FOUND)
-    execute_process(
-      COMMAND ${GIT_EXECUTABLE} describe --tags --always --dirty
-      WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
-      OUTPUT_VARIABLE PROJECT_VERSION
-      OUTPUT_STRIP_TRAILING_WHITESPACE
-    )
-  endif()
-
-  if(NOT GIT_FOUND OR PROJECT_VERSION STREQUAL "")
-    set(PROJECT_VERSION "0.0.0")
-  endif()
-
   write_basic_package_version_file(
-    "${CMAKE_CURRENT_BINARY_DIR}/${_name}ConfigVersion.cmake"
+    "${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}ConfigVersion.cmake"
     VERSION ${PROJECT_VERSION}
     COMPATIBILITY AnyNewerVersion
   )
 
   configure_package_config_file(
     "${CMAKE_CURRENT_SOURCE_DIR}/Config.cmake.in"
-    "${CMAKE_CURRENT_BINARY_DIR}/${_name}Config.cmake"
-    INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name}
+    "${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}Config.cmake"
+    INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}
   )
 
   install(
     FILES
-      "${CMAKE_CURRENT_BINARY_DIR}/${_name}Config.cmake"
-      "${CMAKE_CURRENT_BINARY_DIR}/${_name}ConfigVersion.cmake"
-    DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name})
+      "${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}Config.cmake"
+      "${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}ConfigVersion.cmake"
+    DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME})
 endfunction()

cmake/Protobuf.cmake

@@ -0,0 +1,167 @@
+set(Protobuf_PROTOC_EXECUTABLE "${EXTERNALS_BIN}/protoc")
+set(PROTOBUF_INCLUDE_DIRS "${EXTERNALS_INCLUDE}")
+set(PROTOBUF_STATIC_LIB "libprotobuf")
+set(_protobuf_src_dir ${CMAKE_SOURCE_DIR}/src/lib_spark)
+set(_protobuf_bin_dir ${CMAKE_BINARY_DIR}/src/lib_spark)
+
+function(Protobuf_Generate_Cpp)
+set(_options)
+  set(_one_value_args CXX_SOURCES CXX_HEADERS CXX_OUT_DIR)
+  set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+  set(${_ARGS_CXX_SOURCES})
+  set(${_ARGS_CXX_HEADERS})
+
+  set(_proto_include_cli_args)
+  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
+    list(APPEND _proto_include_cli_args -I ${_dir})
+  endforeach()
+
+  foreach(FIL ${_ARGS_PROTO_SOURCES})
+    get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
+    get_filename_component(FIL_WE ${FIL} NAME_WE)
+    get_filename_component(FIL_DIR ${FIL} DIRECTORY)
+    if(FIL_DIR)
+      set(FIL_WE "${FIL_DIR}/${FIL_WE}")
+    endif()
+
+    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc")
+    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h")
+
+    add_custom_command(
+      OUTPUT
+        "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
+        "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
+      COMMAND
+        ${Protobuf_PROTOC_EXECUTABLE}
+      ARGS
+        --cpp_out=${_ARGS_CXX_OUT_DIR} ${_proto_include_cli_args} ${ABS_FIL}
+      DEPENDS ${ABS_FIL}
+      COMMENT "Running C++ protocol buffer compiler on ${FIL}"
+      VERBATIM)
+  endforeach()
+
+  set_source_files_properties(${${_ARGS_CXX_SOURCES}} ${${_ARGS_CXX_HEADERS}} PROPERTIES GENERATED TRUE)
+
+  # remove some warnings from protobuf generated files
+  if(WIN32)
+    set_source_files_properties(${${_ARGS_CXX_SOURCES}} PROPERTIES COMPILE_FLAGS
+                                                      "/wd4309 /wd4838")
+  else()
+    set_source_files_properties(
+      ${${_ARGS_CXX_SOURCES}}
+      PROPERTIES
+        COMPILE_FLAGS
+        "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
+    )
+  endif()
+
+  set(${_ARGS_CXX_SOURCES}
+      ${${_ARGS_CXX_SOURCES}}
+      PARENT_SCOPE)
+  set(${_ARGS_CXX_HEADERS}
+      ${${_ARGS_CXX_HEADERS}}
+      PARENT_SCOPE)
+endfunction()
+
+function(Service_Compiler_Generate_Cpp)
+  set(_options)
+  set(_one_value_args CXX_SOURCES CXX_HEADERS CXX_OUT_DIR)
+  set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+  set(${_ARGS_CXX_SOURCES})
+  set(${_ARGS_CXX_HEADERS})
+
+  set(_proto_include_cli_args)
+  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
+    list(APPEND _proto_include_cli_args -I ${_dir})
+  endforeach()
+
+  foreach(FIL ${_ARGS_PROTO_SOURCES})
+    get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
+    get_filename_component(FIL_WE ${FIL} NAME_WE)
+    get_filename_component(FIL_DIR ${FIL} DIRECTORY)
+    if(FIL_DIR)
+      set(FIL_WE "${FIL_DIR}/${FIL_WE}")
+    endif()
+
+    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc")
+    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h")
+
+    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc")
+    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h")
+
+    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc")
+    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h")
+
+    set(TEMPLATES
+        ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/client.pb.cc
+        ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/client.pb.h
+        ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/server.pb.cc
+        ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/server.pb.h
+    )
+
+    set(_plugin_py ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/plugin.py)
+    if(WIN32)
+      set(_plugin_cmd ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/plugin.cmd)
+      add_custom_command(
+        OUTPUT "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h"
+        COMMAND SET PYTHONPATH=${SPARK_PYTHON}
+        COMMAND ${CMAKE_SOURCE_DIR}/.venv/Scripts/activate.bat
+        COMMAND
+          ${Protobuf_PROTOC_EXECUTABLE} ARGS --cpp_out=${_ARGS_CXX_OUT_DIR}
+          --plugin=protoc-gen-service=${_plugin_cmd}
+          --service_out=lang=cpp:${_ARGS_CXX_OUT_DIR} -I
+          ${PROTOBUF_INCLUDE_DIRS} -I ${_protobuf_src_dir} ${ABS_FIL}
+        DEPENDS ${ABS_FIL} ${TEMPLATES} ${_plugin_cmd} ${_plugin_py}
+        COMMENT "Running C++ service compiler on ${FIL}"
+        VERBATIM)
+    else()
+      add_custom_command(
+        OUTPUT "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h"
+        COMMAND ${CMAKE_SOURCE_DIR}/tools/venv_exec.sh
+        ARGS
+          ${Protobuf_PROTOC_EXECUTABLE}
+          --cpp_out=${_ARGS_CXX_OUT_DIR}
+          --plugin=protoc-gen-service=${_plugin_py}
+          --service_out=lang=cpp,make_file_name=${ABS_FIL}:${_ARGS_CXX_OUT_DIR}
+          ${_proto_include_cli_args} ${ABS_FIL}
+        DEPENDS ${ABS_FIL} ${TEMPLATES} ${_plugin_py}
+        COMMENT "Running C++ service compiler on ${FIL}"
+        VERBATIM)
+    endif()
+
+  endforeach()
+
+  # remove some warnings from protobuf generated files
+  if(WIN32)
+    set_source_files_properties(${${_ARGS_CXX_SOURCES}} PROPERTIES COMPILE_FLAGS
+                                                      "/wd4309 /wd4838")
+  else()
+    set_source_files_properties(
+      ${${_ARGS_CXX_SOURCES}}
+      PROPERTIES
+        COMPILE_FLAGS
+        "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
+    )
+  endif()
+
+  set(${_ARGS_CXX_SOURCES}
+      ${${_ARGS_CXX_SOURCES}}
+      PARENT_SCOPE)
+  set(${_ARGS_CXX_HEADERS}
+      ${${_ARGS_CXX_HEADERS}}
+      PARENT_SCOPE)
+
+endfunction()
+

cmake/ProtobufGenerateCpp.cmake

@@ -1,80 +0,0 @@
-include_guard(GLOBAL)
-
-function(protobuf_generate_cpp)
-  set(_options)
-  set(_one_value_args
-    CXX_SOURCES
-    CXX_HEADERS
-    CXX_OUT_DIR
-    PROTOC
-  )
-  set(_multi_value_args
-    PROTO_INCLUDE_DIRS
-    PROTO_SOURCES
-  )
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-
-  if (NOT DEFINED _ARGS_PROTOC)
-    if (WIN32)
-      set(_ARGS_PROTOC "${EXTERNALS_BIN}/protoc.exe")
-    else()
-      set(_ARGS_PROTOC "${EXTERNALS_BIN}/protoc")
-    endif()
-  endif()
-  set(${_ARGS_CXX_SOURCES})
-  set(${_ARGS_CXX_HEADERS})
-
-  set(_proto_include_cli_args)
-  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
-    list(APPEND _proto_include_cli_args -I ${_dir})
-  endforeach()
-
-  foreach(_fil ${_ARGS_PROTO_SOURCES})
-    get_filename_component(_abs_fil ${_fil} ABSOLUTE)
-    get_filename_component(_fil_we ${_fil} NAME_WE)
-    get_filename_component(_fil_dir ${_fil} DIRECTORY)
-    if(_fil_dir)
-      set(_fil_we "${_fil_dir}/${_fil_we}")
-    endif()
-    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.cc")
-    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.h")
-    message(STATUS "add_custom_command for ${_fil} -> ${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.h")
-    add_custom_command(
-      OUTPUT
-        "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.cc"
-        "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.h"
-      COMMAND
-        ${_ARGS_PROTOC}
-      ARGS
-        --cpp_out=${_ARGS_CXX_OUT_DIR} ${_proto_include_cli_args} ${_fil}
-      DEPENDS ${_abs_fil}
-      COMMENT "Running C++ protocol buffer compiler on ${_fil}"
-      VERBATIM
-    )
-  endforeach()
-
-  set_source_files_properties(
-    ${${_ARGS_CXX_SOURCES}}
-    ${${_ARGS_CXX_HEADERS}}
-    PROPERTIES
-      GENERATED TRUE
-  )
-
-  # remove some warnings from protobuf generated files
-  if(WIN32)
-    set_source_files_properties(
-      ${${_ARGS_CXX_SOURCES}}
-      PROPERTIES
-        COMPILE_FLAGS "/wd4309 /wd4838"
-    )
-  else()
-    set_source_files_properties(
-      ${${_ARGS_CXX_SOURCES}}
-      PROPERTIES
-        COMPILE_FLAGS "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
-    )
-  endif()
-
-  set(${_ARGS_CXX_SOURCES} ${${_ARGS_CXX_SOURCES}} PARENT_SCOPE)
-  set(${_ARGS_CXX_HEADERS} ${${_ARGS_CXX_HEADERS}} PARENT_SCOPE)
-endfunction()

cmake/ProtobufGenerateCppConfig.cmake

@@ -1,58 +0,0 @@
-function(protobuf_generate_cpp_config)
-  set(_options)
-  set(_one_value_args
-    PROTO_SOURCES_HEADER
-    PROTO_DEFAULTS_HEADER
-  )
-  set(_multi_value_args
-    PROTO_SOURCES
-    PROTO_DEFAULTS
-  )
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-
-  # Configure the 'config_proto_files' header. It provides an array of strings populated with the
-  # files in PROTO_SOURCES. At runtime, we use this array to generate a special 'config' proto
-  # message with fields for each message defined in said files. A global instance of this 'config'
-  # proto is then constructed, and populated with [compile-time] defaults, and [runtime] overrides
-  # (i.e., in spark.conf). Calls to `bts::config::get<T>()` retrieve the value from the field
-  # associated with type `T` in this global instance.
-  message(STATUS "PROTO_SOURCES: ${_ARGS_PROTO_SOURCES}")
-  message(STATUS "PROTO_SOURCES_HEADER: ${_ARGS_PROTO_SOURCES_HEADER}")
-  message(STATUS "PROTO_DEFAULTS_HEADER: ${_ARGS_PROTO_DEFAULTS_HEADER}")
-  if (_ARGS_PROTO_SOURCES_HEADER)
-    set(PROTO_SOURCES)
-    set(PROTO_SOURCES_CXX_ARRAY_ITEMS "")
-    foreach(_proto_path ${_ARGS_PROTO_SOURCES})
-      list(APPEND PROTO_SOURCES ${_proto_path})
-      string(APPEND PROTO_SOURCES_CXX_ARRAY_ITEMS "\"${_proto_path}\",\n")
-    endforeach()
-    list(LENGTH PROTO_SOURCES PROTO_SOURCES_LENGTH)
-    message(STATUS "CONFIGURING PROTO_SOURCES_HEADER): ${_ARGS_PROTO_SOURCES_HEADER}")
-    configure_file(${CMAKE_SOURCE_DIR}/cmake/config_proto_files.h.in ${_ARGS_PROTO_SOURCES_HEADER})
-  endif()
-
-  # Generate static defaults. Ideally, this would be a single string, but we use an array because
-  # the compiler complains if the string is longer than 65k.
-  set(_cfg_defaults_str "inline const char * config_static_defaults[] = { ")
-
-  foreach(_cfg_file_path ${_ARGS_PROTO_DEFAULTS})
-    file(READ ${_cfg_file_path} _cfg_file_content)
-    string(REPLACE "\"" "\\\"" _cfg_file_content_escaped ${_cfg_file_content})
-    string(REPLACE "\n" ";" _cfg_lines ${_cfg_file_content_escaped})
-
-    foreach(_line ${_cfg_lines})
-      string(APPEND _cfg_defaults_str "\n  \"${_line}\",")
-    endforeach(_line)
-  endforeach(_cfg_file_path)
-
-  string(APPEND _cfg_defaults_str "};")
-  set(_old_cfg_defaults_str "")
-
-  if(EXISTS ${_ARGS_PROTO_DEFAULTS_HEADER})
-    file(READ ${_ARGS_PROTO_DEFAULTS_HEADER} _old_cfg_defaults_str)
-  endif()
-
-  if(NOT _cfg_defaults_str STREQUAL _old_cfg_defaults_str)
-    file(WRITE "${_ARGS_PROTO_DEFAULTS_HEADER}" "${_cfg_defaults_str}")
-  endif()
-endfunction()

cmake/ProtobufGenerateFs.cmake

@@ -1,49 +0,0 @@
-include_guard(DIRECTORY)
-
-function(protobuf_generate_fs)
-  set(_options)
-  set(_one_value_args TARGET_NAME OUT_DIR PROTOC)
-  set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-
-  if (NOT DEFINED _ARGS_TARGET_NAME)
-    set(_ARGS_TARGET_NAME "${PROJECT_NAME}_generate_fs_protos")
-  endif()
-  if (NOT DEFINED _ARGS_PROTOC)
-    set(_ARGS_PROTOC "${EXTERNALS_BIN}/protoc")
-  endif()
-
-  set(_proto_include_cli_args)
-  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
-    list(APPEND _proto_include_cli_args -I ${_dir})
-  endforeach()
-
-  set(_outputs)
-  foreach(_proto_source ${_ARGS_PROTO_SOURCES})
-    # get_filename_component(_abs_proto_source ${_proto_source} ABSOLUTE)
-    get_filename_component(_proto_source_we ${_proto_source} NAME_WE)
-    # get_filename_component(_proto_source_dir ${_proto_source} DIRECTORY)
-    list(APPEND _outputs ${_ARGS_OUT_DIR}/${_proto_source_we}.fable.fs)
-  endforeach()
-
-  set(_venv_exec ${CMAKE_SOURCE_DIR}/tools/venv_exec.sh)
-  set(_plugin_py ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/plugin.py)
-  set(_template_fs ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/fs/fable.fs)
-  add_custom_command(
-    OUTPUT ${_outputs}
-    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
-    COMMAND ${_venv_exec}
-      ${_ARGS_PROTOC}
-      --plugin=protoc-gen-service=${_plugin_py}
-      --service_out=lang=fs,strip_out_path=1:${_ARGS_OUT_DIR}
-      ${_proto_include_cli_args}
-      ${_ARGS_PROTO_SOURCES}
-    DEPENDS
-      ${_venv_exec}
-      ${_plugin_py}
-      ${_template_fs}
-      ${CMAKE_CURRENT_SOURCE_DIR}/package-lock.json
-      ${_ARGS_PROTO_SOURCES}
-  )
-  add_custom_target(${_ARGS_TARGET_NAME} ALL DEPENDS "${_outputs}")
-endfunction()

cmake/ProtobufGenerateJs.cmake

@@ -1,33 +0,0 @@
-include_guard(DIRECTORY)
-
-function(protobuf_generate_js)
-  set(_options)
-  set(_one_value_args TARGET_NAME OUTPUT)
-  set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-
-  if (NOT DEFINED _ARGS_TARGET_NAME)
-    set(_ARGS_TARGET_NAME "${PROJECT_NAME}_generate_js_protos")
-  endif()
-
-  set(_proto_include_cli_args)
-  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
-    list(APPEND _proto_include_cli_args -p ${_dir})
-  endforeach()
-
-  add_custom_command(
-    OUTPUT ${_ARGS_OUTPUT}
-    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
-    COMMAND npx
-      --no-install pbjs
-      -t json-module
-      -w commonjs
-      -o ${_ARGS_OUTPUT}
-      ${_proto_include_cli_args}
-      ${_ARGS_PROTO_SOURCES}
-    DEPENDS
-      ${CMAKE_CURRENT_SOURCE_DIR}/package-lock.json
-      ${_ARGS_PROTO_SOURCES}
-  )
-  add_custom_target(${_ARGS_TARGET_NAME} ALL DEPENDS ${_ARGS_OUTPUT})
-endfunction()

cmake/SetFirstDefined.cmake

@@ -1,9 +1,7 @@
-include_guard(GLOBAL)
-
-macro(set_first_defined RESULT)
-  foreach(_arg ${ARGN})
-    if (DEFINED ${_arg})
-      set(${RESULT} ${${_arg}})
+macro(set_first_defined RESULT CANDIDATE_VARS)
+  foreach(_candidate_var ${CANDIDATE_VARS})
+    if (DEFINED _candidate_var)
+      set(${RESULT} ${_candidate_var})
       break()
     endif()
   endforeach()

cmake/SymLink.cmake

@@ -1,33 +1,26 @@
-include_guard(DIRECTORY)
-include(${CMAKE_SOURCE_DIR}/cmake/TernaryOperator.cmake)
-
-function(sym_link_dir)
-  set(_options)
-  set(_one_value_args TARGET_NAME SOURCE DEST COMMENT)
-  set(_multi_value_args)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+function(sym_link_dir TARGET_NAME SOURCE DEST)
   add_custom_command(
-    OUTPUT ${_ARGS_DEST}
-    COMMAND ln ARGS -sf ${_ARGS_SOURCE} ${_ARGS_DEST}
+    OUTPUT ${DEST}
+    COMMAND ln ARGS -sf ${SOURCE} ${DEST}
   )
-  set_ternary(_comment _ARGS_COMMENT "${_ARGS_COMMENT}" "linking dir ${_ARGS_SOURCE} -> ${_ARGS_DEST}")
   add_custom_target(
-    ${_ARGS_TARGET_NAME} ALL
-    DEPENDS ${_ARGS_DEST}
-    COMMENT "${_comment}"
+    ${TARGET_NAME} ALL
+    DEPENDS ${DEST}
   )
 endfunction()
 
 function(sym_link_file)
   set(_options)
-  set(_one_value_args SOURCE DEST INSTALL_DEST)
+  set(_one_value_args TARGET_NAME SOURCE DEST INSTALL_DEST)
   set(_multi_value_args)
   cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-  file(
-    CREATE_LINK
-      ${_ARGS_SOURCE}
-      ${_ARGS_DEST}
-    SYMBOLIC
+  add_custom_command(
+    OUTPUT ${_ARGS_DEST}
+    COMMAND ln ARGS -s ${_ARGS_SOURCE} ${_ARGS_DEST}
+  )
+  add_custom_target(
+    ${_ARGS_TARGET_NAME} ALL
+    DEPENDS ${_ARGS_DEST}
   )
   if (DEFINED _ARGS_INSTALL_DEST)
     install(

cmake/TernaryOperator.cmake

@@ -1,5 +1,3 @@
-include_guard(GLOBAL)
-
 macro(set_ternary RESULT CONDITION TRUE_VALUE FALSE_VALUE)
   if(${CONDITION})
     set(${RESULT} ${TRUE_VALUE})

cmake/config_proto_files.h.in

@@ -1,11 +0,0 @@
-#pragma once
-
-#include <array>
-
-namespace bts::@PROJECT_NAME@::protobufs {
-
-inline std::array<const char *, @PROTO_SOURCES_LENGTH@> config_proto_files = {
-@PROTO_SOURCES_CXX_ARRAY_ITEMS@
-};
-
-} // namespace bts::@PROJECT_NAME@::protobufs

cmake/vs_filters.cmake

@@ -5,7 +5,7 @@ cmake_minimum_required(VERSION 3.5)
 cmake_policy(SET CMP0076 NEW)
 
 # add protos to the library to facilitate source_group
-target_sources(spark PUBLIC ${Config_Protos} ${NonConfig_Protos} ${Service_Protos})
+target_sources(libspark PUBLIC ${Config_Protos} ${NonConfig_Protos} ${Service_Protos})
 
 function(assign_source_group)
   foreach(_source IN ITEMS ${ARGN})

compose.yaml

@@ -1,59 +1,52 @@
 services:
   sparkd:
     build:
-      dockerfile: src/app_sparkd/Dockerfile
-      args:
-        - IMAGE=${SPARKD__IMAGE}
-        - SOURCE_DIR=${SPARKD__SOURCE_DIR}
-        - CONF_FILE=${SPARKD__CONF_FILE}
-        - MD_RECORDING_DIR=${SPARKD__MD_RECORDING_DIR}
+      dockerfile: src/server/sparkd/Dockerfile
     command: [ "./sparkd", "run" ]
-    ports:
-      - "8000:8000"
-      - "8001:8001"
-      - "8002:8002"
+    ports: [ "8000:8000", "8001:8001", "8002:8002" ]
 
   pubsub:
     environment:
-      - DOTNET_ENVIRONMENT=${PUBSUB__DOTNET_ENVIRONMENT}
+      - DOTNET_ENVIRONMENT=Staging
     build:
       dockerfile: src/DataStreamSuite/PubSubServer/Dockerfile
-      args:
-        - CS_SOURCE_DIR=${PUBSUB__CS_SOURCE_DIR}
-        - CPP_SOURCE_DIR=${PUBSUB__CPP_SOURCE_DIR}
-        - CONF_FILE=${PUBSUB__CONF_FILE}
     ports:
-      - "22019:22019"
+      - 22019:22019
     command: [ "./PubSubServer" ]
 
   md-publisher:
     build:
       dockerfile: src/DataStreamSuite/DataStreamPublisher/Dockerfile
-      args:
-        - SOURCE_DIR=${MD_PUBLISHER__SOURCE_DIR}
-        - CONF_FILE=${MD_PUBLISHER__CONF_FILE}
     depends_on:
       - pubsub
       - sparkd
-    command: [ "dockerize", "-wait", "tcp://pubsub:22019", "-wait", "tcp://sparkd:8000", "-timeout", "10s", "./DataStreamPublisher" ]
+    command:
+      [
+        "dockerize",
+        "-wait",
+        "tcp://pubsub:22019",
+        "-wait",
+        "tcp://sparkd:8000",
+        "-timeout",
+        "10s",
+        "./DataStreamPublisher"
+      ]
     environment:
-      - DOTNET_ENVIRONMENT=${MD_PUBLISHER__DOTNET_ENVIRONMENT}
+      - DOTNET_ENVIRONMENT=MarketData
 
   webapp:
     ports:
-      - "8080:8080"
-      - "8443:8443"
+      - 8080:8080
+      - 8443:8443
     build:
       dockerfile: src/DataStreamSuite/Bts.DataStreamWebApp.Server/Dockerfile
-      args:
-        - SOURCE_DIR=${WEBAPP__SOURCE_DIR}
-        - WEB_ROOT=${WEBAPP__WEB_ROOT}
-        - CONF_FILE=${WEBAPP__CONF_FILE}
     command: [ "./Bts.DataStreamWebApp.Server" ]
     environment:
-      - ASPNETCORE_ENVIRONMENT=${WEBAPP__ASPNETCORE_ENVIRONMENT}
+      - ASPNETCORE_ENVIRONMENT=Staging
 
   api-server:
+    profiles:
+      - api-server
     depends_on:
       - sparkd
     ports:
@@ -61,9 +54,9 @@ services:
       - 9443:9443
     build:
       dockerfile: src/client/spark_api_server/Dockerfile
-    command: [ "dockerize", "-wait", "tcp://sparkd:8000", "-timeout", "10s", "./spark_api_server", "run" ]
+    command: [ "./spark_api_server", "run" ]
     environment:
-      - ASPNETCORE_ENVIRONMENT=${API_SERVER__ASPNETCORE_ENVIRONMENT}
+      - ASPNETCORE_ENVIRONMENT=Staging
 
   sparkd-ping-test:
     depends_on:

doc/api/content/server/ref/subscriptions.md

@@ -53,7 +53,7 @@ The status callback is called when the market for this security opens, closes, o
 // Subscribe to security trading status
 void IAlgoServices::subscribe_status(SecurityKey);
 // You implement
-virtual void on_status(SecurityKey, domain::protobufs::TradingStatus) override;
+virtual void on_status(SecurityKey, co2::TradingStatus) override;
 ```
 
 # Child Orders

docker/ubuntu-22.04/dotnet-7/Dockerfile

@@ -24,6 +24,7 @@ RUN locale-gen en_US.UTF-8
 ARG docker_user="bts"
 ARG docker_uid="1000"
 ARG docker_pass="bts"
+# ARG entrypoint
 
 RUN \
   groupadd -g "$docker_uid" "$docker_user" \

src/DataStreamSuite/Bts.DataStreamWebApp.Client/.env.development

@@ -1 +1 @@
-VITE_DATASTREAM_WEBSOCKET_URL=wss://localhost:8443/ws
+VITE_DATASTREAM_WEBSOCKET_URL=wss://localhost:443

src/DataStreamSuite/Bts.DataStreamWebApp.Client/.env.production

@@ -1 +1 @@
-VITE_DATASTREAM_WEBSOCKET_URL=wss://localhost:8443/ws
+VITE_DATASTREAM_WEBSOCKET_URL=wss://localhost:443

src/DataStreamSuite/Bts.DataStreamWebApp.Client/.eslintrc.cjs

@@ -0,0 +1,32 @@
+module.exports = {
+  root: true,
+  env: { browser: true, es2020: true },
+  extends: [
+    'eslint:recommended',
+    'plugin:react/recommended',
+    'plugin:react/jsx-runtime',
+    'plugin:react-hooks/recommended',
+  ],
+  ignorePatterns: ['dist', '.eslintrc.cjs'],
+  parserOptions: { ecmaVersion: 'latest', sourceType: 'module' },
+  settings: { react: { version: '18.3.1' } },
+  plugins: ['react-refresh'],
+  rules: {
+    "import/order": [
+      "error",
+      {
+        "groups": [
+          "builtin",
+          "external",
+          "internal",
+          ["parent", "sibling", "index"]
+        ],
+        "newlines-between": "always"
+      }
+    ],
+    'react-refresh/only-export-components': [
+      'warn',
+      { allowConstantExport: true },
+    ],
+  },
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Client/eslint.config.js

@@ -1,36 +0,0 @@
-import importPlugin from "eslint-plugin-import";
-import reactPkg from "eslint-plugin-react";
-const { configs: reactConfigs } = reactPkg;
-import js from "@eslint/js";
-import defineConfig from "./vite.config.js";
-
-export default [
-  js.configs.recommended,
-  importPlugin.flatConfigs.recommended,
-  reactConfigs.flat.recommended,
-  reactConfigs.flat["jsx-runtime"],
-  {
-    files: ["**/*.{js,jsx}"],
-    languageOptions: {
-      ecmaVersion: "latest",
-      sourceType: "module",
-      globals: {
-        console: "readonly"
-      }
-    },
-    settings: {
-      "import/resolver": {
-        vite: {
-          viteConfig: defineConfig({ mode: "development" })
-        }
-      }
-    },
-    rules: {
-      semi: "error",
-      "prefer-const": "error",
-      "no-unused-vars": "warn",
-      "import/no-dynamic-require": "warn",
-      "import/no-nodejs-modules": "warn",
-    }
-  }
-];

src/DataStreamSuite/Bts.DataStreamWebApp.Client/package.json

@@ -10,37 +10,35 @@
     "preview": "vite preview"
   },
   "dependencies": {
-    "@angular/material": "^18.2.8",
-    "@emotion/react": "^11.13.3",
-    "@emotion/styled": "^11.13.0",
-    "@fontsource/noto-sans": "^5.1.0",
-    "@material/data-table": "^15.0.0-canary.423edc3dc.0",
-    "@mui/icons-material": "^6.1.4",
-    "@mui/material": "^6.1.4",
-    "@mui/x-data-grid": "^7.20.0",
-    "bulma": "^1.0.2",
+    "@angular/material": "^18.0.4",
+    "@emotion/react": "^11.11.4",
+    "@emotion/styled": "^11.11.5",
+    "@fontsource/noto-sans": "^5.0.22",
+    "@material/data-table": "^15.0.0-canary.7f224ddd4.0",
+    "@mui/icons-material": "^5.15.20",
+    "@mui/material": "^5.15.20",
+    "@mui/x-data-grid": "^7.7.1",
+    "bulma": "^1.0.1",
     "classnames": "^2.5.1",
     "react": "^18.3.1",
-    "react-data-grid": "^7.0.0-beta.47",
+    "react-data-grid": "^7.0.0-beta.44",
     "react-data-table-component": "^7.6.2",
     "react-dom": "^18.3.1",
-    "react-json-view-lite": "^1.5.0",
+    "react-json-view-lite": "^1.4.0",
     "react-pro-sidebar": "^1.1.0",
-    "react-use-websocket": "^4.9.0",
-    "rpc-websockets": "^9.0.4"
+    "react-use-websocket": "^4.8.1",
+    "rpc-websockets": "^7.11.2"
   },
   "devDependencies": {
-    "@types/react": "^18.3.11",
-    "@types/react-dom": "^18.3.1",
-    "@vitejs/plugin-react": "^4.3.2",
-    "eslint": "^9.12.0",
-    "@eslint/js": "^9.12.0",
-    "eslint-plugin-import": "^2.31.0",
-    "eslint-plugin-react": "^7.37.1",
-    "eslint-plugin-react-hooks": "^5.0.0",
-    "eslint-plugin-react-refresh": "^0.4.12",
-    "eslint-import-resolver-vite": "^2.0.1",
-    "react-mui-sidebar": "^1.3.8",
-    "vite": "^5.4.9"
+    "@types/react": "^18.3.3",
+    "@types/react-dom": "^18.3.0",
+    "@vitejs/plugin-react": "^4.3.1",
+    "eslint": "^8.57.0",
+    "eslint-plugin-import": "^2.29.1",
+    "eslint-plugin-react": "^7.34.2",
+    "eslint-plugin-react-hooks": "^4.6.2",
+    "eslint-plugin-react-refresh": "^0.4.7",
+    "react-mui-sidebar": "^1.3.5",
+    "vite": "^5.2.13"
   }
 }

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/DataTable.jsx

@@ -1,7 +1,7 @@
 import PropTypes from 'prop-types';
 import { useTheme } from '@mui/material/styles';
 import { DataGrid } from '@mui/x-data-grid';
-import './DataGrid.css';
+import './DataGrid.css'
 import { useMemo, useState } from 'react';
 
 export default function DataTable({ columns, rows }) {
@@ -23,7 +23,7 @@ export default function DataTable({ columns, rows }) {
         headerName: col.name,
         width: columnWidths[col.name],
       };
-    };
+    }
   }, [columnWidths]);
 
   return (

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/QueryStatusView.jsx

@@ -12,7 +12,7 @@ const QueryStatusView = ({ queryState, details, error }) => {
 
   let header = queryState;
   let headerContent = null;
-  const detailElements = [];
+  let detailElements = [];
   let severity = queryState == "Subscribed" ? "success" : "info";
   let cursor = details == null ? "default" : "pointer";
   if (error != null) {
@@ -20,14 +20,14 @@ const QueryStatusView = ({ queryState, details, error }) => {
     cursor = "pointer";
     header = `Error Code: ${error.code}`;
     headerContent = error.message;
-    detailElements.push(<Typography key="error-msg" variant="body1">{error.data.error_message}</Typography>);
-    detailElements.push(<Typography key="error-details" variant="subtitle1">Stack Trace:</Typography>);
-    (error.data.stack_trace ?? []).forEach((line, idx) => detailElements.push(
-      <Typography key={`stacktrace-${idx}`} variant="body2" component="pre" sx={{ whiteSpace: 'pre-wrap' }}>
+    detailElements.push(<Typography variant="body1">{error.data.error_message}</Typography>);
+    detailElements.push(<Typography variant="subtitle1">Stack Trace:</Typography>);
+    (error.data.stack_trace ?? []).forEach(line => detailElements.push(
+      <Typography variant="body2" component="pre" sx={{ whiteSpace: 'pre-wrap' }}>
         {line}
       </Typography>));
   } else if (details != null) {
-    detailElements.push(<Typography key="status-msg" variant="body1">{details}</Typography>);
+    detailElements.push(<Typography variant="body1">{details}</Typography>);
   }
   return (
     <Box sx={{ margin: theme.spacing(2, 0, 0, 0) }}>

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/ReactDataGrid.jsx

@@ -13,7 +13,7 @@ function getGridColDef(col) {
 export default function ReactDataGrid({columns, rows}) {
   return (
     <DataGrid className={"data-grid"} rows={rows} columns={columns.map(getGridColDef)} />
-  );
+  )
 }
 
 ReactDataGrid.propTypes = {

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/SideBar.jsx

@@ -3,7 +3,7 @@ import Button from '@mui/material/Button';
 import { useTheme } from '@mui/material/styles';
 import PropTypes from 'prop-types';
 import Tooltip from '@mui/material/Tooltip';
-import ChevronLeftRounded from "@mui/icons-material/ChevronLeftRounded";
+import ChevronLeftRounded from "@mui/icons-material/ChevronLeftRounded"
 import ChevronRightRounded from '@mui/icons-material/ChevronRightRounded';
 import FileCopyIcon from '@mui/icons-material/FileCopy';
 import './SideBar.css';
@@ -51,7 +51,7 @@ const Sidebar = ({ onSelection, onOpened, onClosed }) => {
       } catch (error) {
         console.error('Error fetching WebSocket config:', error);
       }
-    };
+    }
     fetchCannedQueries();
   }, []);
 

src/DataStreamSuite/Bts.DataStreamWebApp.Client/vite.config.js

@@ -1,5 +1,5 @@
 import { fileURLToPath, URL } from 'node:url';
-import { defineConfig } from 'vite';
+import { defineConfig, loadEnv } from 'vite';
 import plugin from '@vitejs/plugin-react';
 import fs from 'fs';
 import path from 'path';
@@ -50,25 +50,10 @@ export default defineConfig(({ $$command, mode, $$isSsrBuild, $$isPreview }) =>
     server: {
       proxy: {
         '^/api': {
-          target,
-          secure: false,
-          configure: (proxy) => {
-            proxy.on('error', (err) => {
-              console.log('proxy error', err);
-            });
-            proxy.on('proxyReq', (proxyReq, req) => {
-              console.log(`Forwarding ${proxyReq.method} ${proxyReq.protocol}//${proxyReq.host}${proxyReq.path} to ${req.socket.remoteAddress}${req.url}`);
-            });
-            proxy.on('proxyRes', (proxyRes, req, res) => {
-              console.log(`Forwarding response ${req.socket.localAddress} to ${res.socket.remoteAddress}. Status=${proxyRes.statusCode}`);
-            });
-          },
-        },
-        '^/ws': {
           target,
           secure: false
         },
-        '^/browse': {
+        '^/ws': {
           target,
           secure: false
         }

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Controllers/DataStreamSubscriptionController.cs

@@ -1,21 +1,14 @@
 namespace Bts.DataStreamWebApp.Server.Controllers;
 
-using Bts.DataStream;
-
 using Microsoft.AspNetCore.Mvc;
 
 [Route("/ws")]
 public class DataStreamSubscriptionController : ControllerBase {
    readonly ILogger logger;
-   readonly IDataStreamFactory streamFactory;
 
-   public DataStreamSubscriptionController(ILogger<DataStreamSubscriptionController> logger, IEnumerable<IDataStreamFactory> streamFactories) {
+   public DataStreamSubscriptionController(ILogger<DataStreamSubscriptionController> logger) {
       this.logger = logger;
       logger.LogInformation("Creating {ControllerType}", GetType().Name);
-      var streamFactoryList = streamFactories.ToList();
-      streamFactory = streamFactoryList.Count == 1
-         ? streamFactoryList[0]
-         : new CompositeDataStreamFactory(streamFactoryList);
    }
 
    [Route("/ws")]
@@ -28,7 +21,7 @@ public class DataStreamSubscriptionController : ControllerBase {
       try {
          using var webSocket = await HttpContext.WebSockets.AcceptWebSocketAsync();
          logger.LogInformation("Websocket connection established.");
-         using var server = new DataStreamWebSocketServer(webSocket, streamFactory, logger);
+         using var server = new DataStreamWebSocketServer(webSocket, logger);
          await server.Run(cancellationToken);
       } catch (Exception ex) {
          logger.LogError("DataStreamSubscriptionController.Run threw exception: {Exception}", ex);

src/DataStreamSuite/Bts.DataStreamWebApp.Server/DataStreamWebSocketServer.cs

@@ -2,7 +2,6 @@ namespace Bts.DataStreamWebApp.Server.Controllers;
 
 using bts.websocket_core.json_rpc;
 
-using Bts.DataStream;
 using Bts.DataStreamWebApp.Server.Messages;
 
 using System.Net.WebSockets;
@@ -14,11 +13,9 @@ public class DataStreamWebSocketServer : IDisposable, IAsyncDisposable {
    readonly ILogger logger;
    readonly WebSocket webSocket;
    readonly Dictionary<string, DataStreamSubscription> subscriptions = new();
-   readonly IDataStreamFactory streamFactory;
 
-   public DataStreamWebSocketServer(WebSocket webSocket, IDataStreamFactory streamFactory, ILogger logger) {
+   public DataStreamWebSocketServer(WebSocket webSocket, ILogger logger) {
       this.webSocket = webSocket;
-      this.streamFactory = streamFactory;
       this.logger = logger;
    }
 
@@ -70,7 +67,7 @@ public class DataStreamWebSocketServer : IDisposable, IAsyncDisposable {
       }
       var subId = subReq.SubscriptionId ?? throw new InvalidOperationException("SubscriptionId is null (parse false)");
       disposeSubscription(subId);
-      if (!subReq.ParseQuery(streamFactory, out var vs, out parseErr, out errCode)) {
+      if (!subReq.ParseQuery(out var vs, out parseErr, out errCode)) {
          logger.LogError("Invalid 'subscribe' request: {Reason}", parseErr);
          var rpcError = JsonRpcResponse.CreateError<SubscriptionError>(
             req.Id, new() { Code = (long)errCode, Message = "Failed to parse subscription request", Data = parseErr });

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Dockerfile

@@ -1,13 +1,7 @@
 FROM bluetradingsystems/linux_dotnet7
 
-ARG CS_SOURCE_DIR="build/Release/bin/net7.0/linux-x64/publish"
-ARG CPP_SOURCE_DIR="install/data-stream-suite-linux"
-ARG WEB_ROOT="build/Bts.DataStreamWebApp.Client/production/dist"
-ARG CONF_FILE="src/DataStreamSuite/Bts.DataStreamWebApp.Server/spark.conf"
-
 RUN dotnet dev-certs https --trust
-COPY --chown=bts:bts "${CPP_SOURCE_DIR}/lib" "bin"
-COPY --chown=bts:bts "${CS_SOURCE_DIR}" "bin"
-COPY --chown=bts:bts "${WEB_ROOT}" "bin/wwwroot"
-COPY --chown=bts:bts "${CONF_FILE}" ".local/share/bts/Bts.DataStreamWebApp.Server/conf/spark.conf"
+COPY --chown=bts:bts "build/Release/bin/net7.0/linux-x64/publish" "bin"
+COPY --chown=bts:bts "build/Bts.DataStreamWebApp.Client/production/dist" "bin/wwwroot"
+COPY --chown=bts:bts "src/DataStreamSuite/Bts.DataStreamWebApp.Server/spark.conf" ".local/share/bts/Bts.DataStreamWebApp.Server/conf/spark.conf"
 WORKDIR "/home/bts/bin"

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Messages/SubscribeRequest.cs

@@ -4,7 +4,6 @@ using BCG;
 
 using bts.websocket_core.json_rpc;
 
-using Bts.DataStream;
 using Bts.DataStreamExpressions;
 
 using System.Diagnostics.CodeAnalysis;
@@ -38,12 +37,7 @@ public class SubscribeRequest {
       return true;
    }
 
-   internal bool ParseQuery(
-      IDataStreamFactory streamFactory,
-      [NotNullWhen(true)] out ViewStream? stream,
-      [NotNullWhen(false)] out SubscriptionError? error,
-      out ErrorCode errorCode
-   ) {
+   internal bool ParseQuery([NotNullWhen(true)] out ViewStream? stream, [NotNullWhen(false)] out SubscriptionError? error, out ErrorCode errorCode) {
       stream = null;
       errorCode = ErrorCode.None;
       error = null;
@@ -60,7 +54,7 @@ public class SubscribeRequest {
       }
       Exception? exception = null;
       try {
-         stream = exp.getView(new Dictionary<string, StatementExpression>(0), streamFactory);
+         stream = exp.getView(new Dictionary<string, StatementExpression>(0));
       } catch (Exception ex) {
          exception = ex;
       }

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Program.cs

@@ -1,6 +1,5 @@
 using Bts.DataStreamCore.Services;
 using Bts.PubSub.Services;
-using Bts.SharedStreams.Services;
 
 using Spark.Api.Services;
 using Spark.Core.Services;
@@ -79,9 +78,5 @@ static void ConfigureServices(IServiceCollection services, IConfiguration config
       .AddSparkApi(config.GetSection("SparkApi"))
       .AddMasterWorker(config.GetSection("MasterWorker"))
       .AddViewStreamDatabase()
-      .AddPrimitiveDataStreamFactory(config.GetSection("PrimitiveStreams"))
-      .AddOhlcvStreams(config.GetSection("StreamDatabase"))
-      .AddNaturalLanguageStreams(config.GetSection("NaturalLanguageStream"))
-      .AddCompositeDataStreamFactory()
       .AddPubSubClient(config.GetSection("PubSub"));
 }

src/DataStreamSuite/Bts.DataStreamWebApp.Server/appsettings.Development.json

@@ -1,4 +1,14 @@
 {
+  "Kestrel": {
+    "Endpoints": {
+      "Http": {
+        "Url": "http://localhost:8080"
+      },
+      "Https": {
+        "Url": "https://localhost:8443"
+      }
+    }
+  },
   "PubSub": {
     "Routing": {
       "PublisherRoutes": [

src/DataStreamSuite/Bts.DataStreamWebApp.Server/appsettings.NaturalLanguageCustom.json

@@ -1,45 +0,0 @@
-{
-  "NaturalLanguagePrompt": {
-    "prompts": [
-      {
-        "category": "Impact of financial events on the market",
-        "description": "Show how trading metrics (e.g., price movement, volume) are affected by events related to finance (rate announcements, earnings reports, etc.).",
-        "prompt": "Show me the price movement and volume of Crude Oil futures during the 5-minute window following the last 5 OPEC meetings",
-        "examples": [
-          {
-            "natural_language": "Show me the price movement and volume of Crude Oil futures during the 5-minute window following the last 5 OPEC meetings",
-            "dsl": [
-              "time_idx=EventTimeIndex.OpecMeetings(which: \"last 5\", duration: \"+5min\")",
-              "contract_idx=ActiveFutureContracts(predicate: \"product==CL\")",
-              "Bars(time_index: time_idx, contract_index: contract_idx, fields: [\"price_movement\", \"volume\"])"
-            ]
-          },
-          {
-            "natural_language": "How do OPEC meetings affect the price of Crude futures? Examine the 10 minutes after each meeting this year.",
-            "dsl": [
-              "time_idx=EventTimeIndex.OpecMeetings(duration: \"+10min\", predicate: \"timestamp.year==2024\")",
-              "contract_idx=ActiveFutureContracts(predicate: \"product==CL\")",
-              "Bars(time_index: time_idx, contract_index: contract_idx, fields: [\"price_movement\", \"volume\"])"
-            ]
-          },
-          {
-            "natural_language": "Show me how the SP 500 price moves after FOMC rate announcements that day",
-            "dsl": [
-              "time_idx=EventTimeIndex.FomcRateAnnouncements(which: \"timestamp.year==2024\", duration: \"+10min\")",
-              "contract_idx=ActiveFutureContracts(predicate: \"product==ES\")",
-              "Bars(time_index: time_idx, contract_index: contract_idx, fields: [\"price_movement\", \"volume\"])"
-            ]
-          },
-          {
-            "natural_language": "Analyze the impact of the FED's interest rate hikes in 2022 on the 10-year Treasury yield, focusing on the 2-week period following each hike.",
-            "dsl": [
-              "time_idx=FedInterestRateAnnouncements(predicate: \"timestamp.year==2022 && change_pct>0\", duration: \"+2 weeks\")",
-              "contract_idx=ActiveFutureContracts(predicate: \"product==ZN\")",
-              "Bars(time_index: time_idx, contract_index: contract_idx)"
-            ]
-          }
-        ]
-      }
-    ]
-  }
-}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/appsettings.Staging.json

@@ -30,15 +30,5 @@
         "benzinga_news"
       ]
     }
-  ],
-  "Spark": {
-    "ConfigServer": "http://sparkd:8000/config"
-  },
-  "SparkApi": {
-    "IsEnabled": true,
-    "Credentials": {
-      "User": "bts",
-      "Password": ""
-    }
-  }
+  ]
 }

src/DataStreamSuite/Bts.DataStreamWebApp.Server/appsettings.json

@@ -11,14 +11,8 @@
       "Http": {
         "Url": "http://0.0.0.0:8080"
       },
-      "HttpIpv6": {
-        "Url": "http://[::1]:8080"
-      },
       "Https": {
         "Url": "https://0.0.0.0:8443"
-      },
-      "HttpsIpv6": {
-        "Url": "https://[::1]:8443"
       }
     }
   },
@@ -28,358 +22,6 @@
   "ViewStreamDatabase": {
     "Interval": "00:00:00.1"
   },
-  "PrimitiveStream": {
-    "Assemblies": [
-      "DataStream.dll",
-      "SharedStreams.dll"
-    ],
-    "ExcludedTypes": [
-      "SettlementPriceStream",
-      "OHLCVStream"
-    ]
-  },
-  "StreamDatabase": {
-    "Path": "/home/bts/.local/share/bts/silver_pine/store.db",
-    "Query": "select * from metadata"
-  },
-  "NaturalLanguageStream": {
-    "prompts": [
-      {
-        "category": "Settlement",
-        "MulitLineDescription": [
-          "This is a few shot learning example for an NL to DSL. The DSL function in question is Settlement (symbol_or_contract, datetime1). ",
-          "The first argument is a future (like CL, for crude oil) or a specific future contract like crude oil Dec 24 future (CLZ4). ",
-          "The second argument is the date at which the settlement is to be fetched. There is an overloaded function, Settlement ",
-          "(symbol_or_contract, begin_dt, end_dt) which gets the settlements for all dates from begin_dt to end_dt. This will also have ",
-          "datetimes expressed as Today, Yesterday, Last Friday, First Friday of the month, Last Thursday of August etc. You will need to know ",
-          "what Today is, since there will be referecnes to last week, yesterday etc. These examples are assuming Today is 10/11/2024, however ",
-          "you should use the Today's real date"
-        ],
-        "prompt": "What was the settlement for SP 500 Future on June 8th?",
-        "examples": [
-          {
-            "natural_language": "Find Settlement price of the SP500 future on 04/22/2023.",
-            "dsl": "Settlement(\"ES\", 4/22/2023)"
-          },
-          {
-            "natural_language": "Find Settlement price of the Silver Dec 24 Future on 04/22/2023.",
-            "dsl": "Settlement(\"SIZ4\", 4/22/2023)"
-          },
-          {
-            "natural_language": "Find Yesterday's Settlement of the Crude Oil Future.",
-            "dsl": "Settlement(\"CL\", 10/10/2024)"
-          },
-          {
-            "natural_language": "Find today's settlement of the 10 year treasury future.",
-            "dsl": "Settlement(\"ZN\", 10/11/2024)"
-          },
-          {
-            "natural_language": "Find the Settlement of Natural Gas Dec 24 future on the first Friday of Sept.",
-            "dsl": "Settlement(\"NGZ4\", 9/6/2024)"
-          },
-          {
-            "natural_language": "Find the Settlement of WTI Jan 25 future in the first Week of Sept.",
-            "dsl": "Settlement(\"CLF5\", 9/2/2024, 9/6/2024)"
-          },
-          {
-            "natural_language": "Find the Settlement price for the Sp 500 Dec 24 Mar 25  spread for the first two weeks of September.",
-            "dsl": "Settlement(\"ESZ4-ESH5\", 9/14/2024, 9/1/2024)"
-          },
-          {
-            "natural_language": "Find the Settlement of Natural Gas future on the first Friday of Sept.",
-            "dsl": "Settlement(\"NG\", 9/6/2024)"
-          }
-        ]
-      },
-      {
-        "category": "BBO",
-        "MulitLineDescription": [
-          "This is a few shot learning example for an NL to DSL. The DSL function in question is Bbo(args...). ",
-          "It accepts the following parameters: ",
-          "symbol (a product code such as ZN for 10 year T-Note futures), ",
-          "contract (a specific contract, such as ZNZ4 for the 10 Year T-Note future Dec 2024 exp), ",
-          "venue (name of the exchange, such as CME, NASDAQ, NYSE, NYMEX, DERIBIT etc.), ",
-          "interval (sample frequency of the data), ",
-          "begin and end (inclusive beginning and end of the time window to query, respectively), ",
-          "Arguments are specified by name, for example: Bbo(symbol: \"ZN\", venue: \"CME\", interval: \"PT1M\", begin: \"2024-10-10\", end: \"2024-10-11\"). ",
-          "Depending on the wording of the request, some arguments may be omitted. ",
-          "The begin, end, and interval args should be expressed in iso 8601 format. ",
-          "Interpret dates according to the trading schedule rather than calendar date whenever possible. ",
-          "In some prompts, the time window will be expressed in relative terms (e.g., \"last Monday\", or \"yesterday\") ",
-          "which must be translated to exact dates. To correctly interpret the dsl in the examples, assume the current time is 2024-10-11T12:00-05:00."
-        ],
-        "prompt": "Show me bbo for SP 500 futures on Sep 23rd of this year.",
-        "examples": [
-          {
-            "natural_language": "Get SP500 future bbo from Wednesday.",
-            "dsl": "Bbo(symbol: \"ES\", venue: \"CME\", begin: \"2024-10-08T17:00:00-06:00\", end: \"2024-10-09T16:00:00-06:00\")"
-          },
-          {
-            "natural_language": "Get bbo for 10 year T-Notes in the first minute after the market opened yesterday.",
-            "dsl": "Bbo(symbol: \"ES\", venue: \"CME\", begin: \"2024-10-10T17:00:00-06:00\", end: \"2024-10-10T17:01:00-06:00\")"
-          },
-          {
-            "natural_language": "Retrieve NVIDIA L1 books from Oct 3",
-            "dsl": "Bbo(symbol: \"NVDA\", venue: \"NASDAQ\", begin: \"2024-10-02TT18:00:00-05:00\", end: \"2024-10-03T17:00:00-05:00\")"
-          },
-          {
-            "natural_language": "Show me L1 data for Crude oil futures this week",
-            "dsl": "Bbo(symbol: \"CL\", venue: \"CME\", begin: \"2024-10-06:T17:00:00-06:00\")"
-          },
-          {
-            "natural_language": "Fetch hourly bbo for Gold futures this week",
-            "dsl": "Bbo(symbol: \"GC\", venue: \"CME\", begin: \"2024-10-06:T17:00:00-06:00\", interval: \"PT1H\")"
-          },
-          {
-            "natural_language": "What was the minute by minute bbo for Tesla yesterday?",
-            "dsl": "Bbo(symbol: \"TSLA\", venue: \"NASDAQ\", begin: \"2024-10-09T18:00:00-05:00\", end: \"2024-10-10T17:00:00-05:00\", interval: \"PT1M\")"
-          }
-        ]
-      },
-      {
-        "category": "Order flow imbalance",
-        "MulitLineDescription": [
-          "This is a few shot learning example for an NL to DSL. The DSL function in question is OrderImbalance(args...). ",
-          "It accepts the following parameters: ",
-          "symbol (a product code such as ZN for 10 year T-Note futures), ",
-          "contract (a specific contract, such as ZNZ4 for the 10 Year T-Note future Dec 2024 exp), ",
-          "venue (name of the exchange, such as CME, NASDAQ, NYSE, NYMEX, DERIBIT etc.), ",
-          "interval (sample frequency of the data), ",
-          "levels (the number of price levels to examine)",
-          "begin and end (inclusive beginning and end of the time window to query, respectively), ",
-          "Arguments are specified by name, for example: OrderImbalance(symbol: \"ZN\", venue: \"CME\", interval: \"PT1M\", begin: \"2024-10-10\", end: \"2024-10-11\"). ",
-          "Depending on the wording of the request, some arguments may be omitted. ",
-          "The begin, end, and interval args should be expressed in iso 8601 format. ",
-          "If the prompt doesn't contain an explicit interval, please choose a reasonable value depending on the duration between begin and end.",
-          "Interpret dates according to the trading schedule rather than calendar date whenever possible. ",
-          "In some prompts, the time window will be expressed in relative terms (e.g., \"last Monday\", or \"yesterday\") ",
-          "which must be translated to exact dates. To correctly interpret the dsl in the examples, assume the current time is 2024-10-11T12:00-05:00."
-        ],
-        "examples": [
-          {
-            "natural_language": "Get SP500 order imbalance from Wednesday.",
-            "dsl": "OrderImbalance(symbol: \"ES\", venue: \"CME\", begin: \"2024-10-08T17:00:00-06:00\", end: \"2024-10-09T16:00:00-06:00\", \"PT1M\")"
-          },
-          {
-            "natural_language": "Get SP500 order imbalance from Wednesday for the top 5 price levels.",
-            "dsl": "OrderImbalance(symbol: \"ES\", venue: \"CME\", begin: \"2024-10-08T17:00:00-06:00\", end: \"2024-10-09T16:00:00-06:00\", levels: 5, interval: \"PT1M\")"
-          },
-          {
-            "natural_language": "Get orderflow imbalance for 10 year T-Notes in the first minute after the market opened yesterday.",
-            "dsl": "OrderImbalance(symbol: \"ES\", venue: \"CME\", begin: \"2024-10-10T17:00:00-06:00\", end: \"2024-10-10T17:01:00-06:00\", interval: \"PT1S\")"
-          },
-          {
-            "natural_language": "Retrieve NVIDIA order flow from Oct 3",
-            "dsl": "OrderImbalance(symbol: \"NVDA\", venue: \"NASDAQ\", begin: \"2024-10-02TT18:00:00-05:00\", end: \"2024-10-03T17:00:00-05:00\", interval: \"PT1S\")"
-          },
-          {
-            "natural_language": "Show me order imbalance for Crude oil futures this week",
-            "dsl": "OrderImbalance(symbol: \"CL\", venue: \"CME\", begin: \"2024-10-06:T17:00:00-06:00\", interval: \"PT1H\")"
-          },
-          {
-            "natural_language": "Fetch hourly orderflow for Gold futures this week",
-            "dsl": "OrderImbalance(symbol: \"GC\", venue: \"CME\", begin: \"2024-10-06:T17:00:00-06:00\", interval: \"PT1H\")"
-          },
-          {
-            "natural_language": "What was the minute by minute orderflow for Tesla yesterday?",
-            "dsl": "OrderImbalance(symbol: \"TSLA\", venue: \"NASDAQ\", begin: \"2024-10-09T18:00:00-05:00\", end: \"2024-10-10T17:00:00-05:00\", interval: \"PT1M\")"
-          }
-        ]
-      },
-      {
-        "category": "VWAP",
-        "description": "This is a few shot learning example for an NL to DSL. The DSL function in question is VWAP (symbol_or_contract, datetime1, datetime2). The first argument is a future (like CL, for crude oil, or ES for SP 500) or a specific future contract like crude oil Dec 24 future (CLZ4) or SP 500 Dec 2024 (ESZ4). The second argument is where the VWAP has to be calculated (also the beginning of the time window), and the third argument is datetime specifying the end of the time window. So, for example, a 20-day VWAP on 9/30/2023, the first argument will be 9/30/2023, and the third argument will be 9/10/2023 (20 days prior). This will also have datetimes expressed as Today, Yesterday, Last Friday, Last Sunday, etc. For the purpose of this few-shot prompts, let's assume today is 10/11/2024.",
-        "prompt": "What was the VWAP for SP 500 Future over the past 30 days?",
-        "examples": [
-          {
-            "natural_language": "Find the 20 day VWAP of the SP future on 04/22/2023.",
-            "dsl": "VWAP(\"ES\", 4/22/2023, 4/02/2023)"
-          },
-          {
-            "natural_language": "Find Yesterday's 10 day VWAP of the Crude Oil Future.",
-            "dsl": "VWAP(\"CL\", 10/10/2024, 10/1/2024)"
-          },
-          {
-            "natural_language": "Find today's 30-day VWAP of the 10 year treasury future.",
-            "dsl": "VWAP(\"ZN\", 10/11/2024, 9/11/2024)"
-          },
-          {
-            "natural_language": "Find the 10 hour VWAP of the Natural Gas future from yesterday at 5:00 PM.",
-            "dsl": "VWAP(\"NG\", 9/27/2024 17:00:00, 9/27/2024 7:00:00)"
-          }
-        ]
-      },
-      {
-        "category": "Price Diff",
-        "description": "This is a few shot learning example for an NL to DSL. The DSL function in question is OHLCVPriceDiff (symbol_or_contract, datetime1, datetime2). The first argument is a future (like CL, for crude oil) or a specific future contract like crude oil Dec 24 future (CLZ4). The second and third arguments are dates, that the function finds the difference of prices at. Assume today is 10/15/2024",
-        "prompt": "What is the difference in  SP 500 Future from July 10th to July 14th?",
-        "examples": [
-          {
-            "natural_language": "What is the difference in ZN price from July 10th to July 14th?",
-            "dsl": "OHLCVPriceDiff(\"ZN\", 7/14/2024, 7/10/2024)"
-          },
-          {
-            "natural_language": "What is the price difference in SP Future price from Sept 10th and Sept 5th?",
-            "dsl": "OHLCVPriceDiff(\"ES\", 9/10/2024, 9/5/2024)"
-          },
-          {
-            "natural_language": "What is the price difference in Natural Gas Dec Future price from Aug 30th and Aug 20th?",
-            "dsl": "OHLCVPriceDiff(\"NGZ4\", 8/30/2024, 8/20/2024)"
-          }
-        ]
-      },
-      {
-        "category": "Absolute Volume",
-        "description": "this is a few shot learning example for an NL to DSL. The DSL function in question is AbsVolume(symbol, ContractType, datetime1, datetime2), which computes the highest volume in the date range, specified by datetime1, and datetime2. The first argument is a future symbol (like CL for crude oil, ES for SP 500 etc).  The second argument is the ContractType, which is \"Future\" for futures and \"FutureSpread\" for future spreads. the 3rd argument is the begin date and the 4th argument is the end date, between which the max volume is computed. If no future is specified use \"*\" from the first arugment. For this prompt, assume today is 10/11/2024 ",
-        "prompt": "Which  SP 500 Future traded the most between 7/14/2024 and 7/10/2024",
-        "examples": [
-          {
-            "natural_language": "Which SP Future traded the most between 7/14/2024 and 7/10/2024",
-            "dsl": "AbsVolume(\"ES\", \"Future\", 7/14/2024, 7/10/2024)"
-          },
-          {
-            "natural_language": "Which Gold Spread traded the most between Today and Sept 20th",
-            "dsl": "AbsVolume(\"GC\", \"FutureSpread\", 10/1/2024, 9/20/2024)"
-          },
-          {
-            "natural_language": "Which 10 year Spread traded the highest volume last week?",
-            "dsl": "AbsVolume(\"ZN\", \"FutureSpread\", 9/27/2024, 9/23/2024)"
-          },
-          {
-            "natural_language": "Which Corn Spread traded the most today?",
-            "dsl": "AbsVolume(\"ZC\", \"FutureSpread\", 10/1/2024, 10/1/2024)"
-          },
-          {
-            "natural_language": "Which Gold Future traded the most today?",
-            "dsl": "AbsVolume(\"GC\", \"Future\", 9/30/2024, 9/30/2024)"
-          }
-        ]
-      },
-      {
-        "category": "Roll",
-        "description": "This is a few shot learning example for an NL to DSL. The DSL function in question is FuturesRoll(symbol) which computes the rollover date for a future, ie when the highest volume swtches from one expiration to the next one. ",
-        "prompt": "When does the roll typically occur In  SP 500?",
-        "examples": [
-          {
-            "natural_language": "When does the roll typically occur In CL?",
-            "dsl": "FuturesRoll(\"CL\")"
-          },
-          {
-            "natural_language": "When does the roll typically occur for 10 year treasury?",
-            "dsl": "FuturesRoll(\"ZN\")"
-          },
-          {
-            "natural_language": "When does the roll usually occur for Gold Future",
-            "dsl": "FuturesRoll(\"GC\")"
-          },
-          {
-            "natural_language": "At what date does the future rollover happens for SP500?",
-            "dsl": "FuturesRoll(\"ES\")"
-          },
-          {
-            "natural_language": "When does the future rollover happens for the Corn Future?",
-            "dsl": "FuturesRoll(\"ZC\")"
-          }
-        ]
-      },
-      {
-        "category": "Daily Move",
-        "description": "This is a few shot learning example for an NL to DSL. ",
-        "prompt": "How often did Sp 500 Future settle + or - 10% over the previous day?",
-        "examples": [
-          {
-            "natural_language": "List all the days when the SP 500 closed + or -5% from the previous day's close",
-            "dsl": "ColDiff(filteredOHLCV(\"ES\"),\"ts_event\", -1, \"close\", \"close\", \"close_diff\", 5)"
-          },
-          {
-            "natural_language": "List all the days when the Crude Oil closed + or -2.5% from the previous day's close",
-            "dsl": "ColDiff(filteredOHLCV(\"CL\"),\"ts_event\", -1, \"close\", \"close\", \"close_diff\", 2.5)"
-          },
-          {
-            "natural_language": "Show me all the days when the Ten year Treaury Future opened + or -1% from the previous day's open",
-            "dsl": "ColDiff(filteredOHLCV(\"ZN\"),\"ts_event\", -1, \"open\", \"open\", \"open_diff\", 1.0)"
-          },
-          {
-            "natural_language": "List all the trading days when the Silver Future opened + or -1% from the previous day's close",
-            "dsl": "ColDiff(filteredOHLCV(\"SI\"),\"ts_event\", -1, \"open\", \"open\", \"open_diff\", 1.0)"
-          }
-        ]
-      },
-      {
-        "category": "Average Volume",
-        "description": "This is a few shot learning example for an NL to DSL. For the purpose of this few shot prompts, lets assume today is 10/3/2024 ",
-        "prompt": "When does the most open interest typically shift from one contract to another in SP 500 Future?",
-        "examples": [
-          {
-            "natural_language": "When does the most open interest typically shift from one contract to another in e-minis? ",
-            "dsl": "FuturesRoll(\"ES\")"
-          },
-          {
-            "natural_language": "Usually when does the open interest move from one contract to another in WTI?",
-            "dsl": "FuturesRoll(\"CL\")"
-          }
-        ]
-      },
-      {
-        "category": "Events",
-        "description": "This is a few shot learning example for an NL to DSL. In the NL prompt the user is asking to show the price of a future (treasury future, or SP 500 future etc) around an event type. The event type could be \"FOMC Announcement\" or \"Storage numbers\" etc. The DSL is as follows: EventsPriceAction(future_name, event_type, pre_event_window, post_event_window, number_of_events) first arg is the name of the future, (ex ES or ZN etc), the second argument is the event type, for example, \"FOMC announcement\", the 3rd argument is a time window for how much pre-event data to include in the end result, the 4th argument is how much post-event data to include in the end result. Both are strings of the form \"5d\" for 5 days, \"5h\" for 5 hours and \"120m\" for 120 minutes and so on. Usually, only one of the event periods is provided. Set the other to be half of what the user provided. The last argument is the number of events to consider. So if it’s 7, that means the most recent 7 events. If the user does not specify that, pass a large number.",
-        "prompt": "What did the volume and price of SP500 Future do following the release of the last 10 FOMC rates?",
-        "examples": [
-          {
-            "natural_language": "How did SP 500 Future do in the 5 days following the FOMC announcements?",
-            "dsl": "EventsPriceAction(\"ES\", \"FOMC Announcement\", \"2d\", \"5d\", 10)"
-          },
-          {
-            "natural_language": "How did the price of Treasury Future evolve in the 2 days following the Job Reports?",
-            "dsl": "EventsPriceAction(\"ZN\", \"Job Reports\", \"1d\", \"2d\", 10)"
-          },
-          {
-            "natural_language": "Show me the price movement and volume of Crude Oil futures during the 5-minute window following the last 5 OPEC meetings.",
-            "dsl": "EventsPriceAction(\"CL\", \"OPEC Meeting\", \"2m\", \"5m\", 5)"
-          },
-          {
-            "natural_language": "Show me the price movement and volume of SP 500 futures preceding the 10-minute window of the last 5 OPEC meetings.",
-            "dsl": "EventsPriceAction(\"CL\", \"OPEC Meeting\", \"10m\", \"5m\", 5)"
-          }
-        ]
-      },
-      {
-        "category": "Indicator, Standard Deviation",
-        "MulitLineDescription": [
-          "This is a few shot learning example for an NL to DSL. In the NL prompt the user is asking to show all the days when ",
-          "the price of a future (treasury future, or SP 500 future etc) moved more than m times the N days standard deviation, ",
-          "where both m and N are provided. For example, the DSL for \"Using a 90d standard deviation, give me the dates and ",
-          "amounts of the last 4+ standard deviation moves in WTI futures\" would be: ",
-          "r=standardDeviation(\"ES\", \"90d\")\n",
-          "r=math(r, \"abs_daily_change = Abs(close - open) comp = std_dev * 4\")\n",
-          "r=filter(r,\"abs_daily_change >= comp\"). ",
-          "The first line, creates a stream for StandardDeviation, based on the number of days, for example 90d standard deviation, ",
-          "and assigned it to r, the second line takes r from the first line and passes it to a math stream that creates 2 new columns, ",
-          "abs_daily_change and comp. comp is the factor (4, from 4+ in the prompt) times standard deviation. The third line filters ",
-          "(like a sql where) of rows where abs_daily_change >= comp"
-        ],
-        "prompt": "Using a 90d standard deviation, give me the dates and amounts of the last 1+ standard deviation moves in SP  futures ?",
-        "examples": [
-          {
-            "natural_language": "Using a 90d standard deviation, give me the dates and amounts of the last 1+ standard deviation moves in WTI futures ?",
-            "dsl": "r=standardDeviation(\"CL\", \"90d\")\nr=math(r, \"abs_daily_change = Abs(close - open) comp = std_dev * 1.0\")\nr=filter(r,\"abs_daily_change >= comp\")"
-          },
-          {
-            "natural_language": "Show me the times the SP Futures moved twice the 60 day standard deviation",
-            "dsl": "r=standardDeviation(\"ES\", \"60d\")\nr=math(r, \"abs_daily_change = Abs(close - open) comp = std_dev * 2.0\")\nr=filter(r,\"abs_daily_change >= comp\")"
-          },
-          {
-            "natural_language": "List all the days the Gold Futures moved 1.5 times the 45 day standard deviation",
-            "dsl": "r=standardDeviation(\"GC\", \"45d\")\nr=math(r, \"abs_daily_change = Abs(close - open) comp = std_dev * 1.5\")\nr=filter(r,\"abs_daily_change >= comp\")"
-          },
-          {
-            "natural_language": "List all the days the 10 year treasury future moved more than twice the 1 month standard deviation",
-            "dsl": "r=standardDeviation(\"ZN\", \"30d\")\nr=math(r, \"abs_daily_change = Abs(close - open) comp = std_dev * 2.0\")\nr=filter(r,\"abs_daily_change >= comp\")"
-          }
-        ]
-      }
-    ]
-  },
   "SparkApi": {
     "IsEnabled": false,
     "Credentials": {
@@ -435,7 +77,7 @@
         {
           "TagType": ".*",
           "Assembly": "DataStreamCore",
-          "Class": "bts.PubsubServerGenericDataHandler"
+         "Class": "bts.PubsubServerGenericDataHandler"
         }
       ]
     }

src/DataStreamSuite/Bts.DataStreamWebApp.Server/production.Dockerfile

@@ -0,0 +1,6 @@
+FROM bluetradingsystems/linux_dotnet7
+
+RUN dotnet dev-certs https --trust
+COPY --chown=bts:bts "bin" "bin"
+COPY --chown=bts:bts "webapp.spark.conf" ".local/share/DataStream/conf/spark.conf"
+WORKDIR "/home/bts/bin"

src/DataStreamSuite/DataStream/DataStream.csproj

@@ -18,11 +18,8 @@
 
   <ItemGroup>
     <PackageReference Include="HtmlAgilityPack" Version="1.11.57" />
-    <PackageReference Include="Microsoft.Data.Analysis" Version="0.21.1" />
     <PackageReference Include="MongoDB.Bson" Version="2.23.1" />
     <PackageReference Include="MongoDB.Driver" Version="2.23.1" />
-    <PackageReference Include="Parquet.Net" Version="5.0.1" />
-    <PackageReference Include="System.Data.SQLite" Version="1.0.118" />
   </ItemGroup>
 
 </Project>

src/DataStreamSuite/DataStream/DataStreamExtensions.cs

@@ -2,27 +2,15 @@ namespace Bts.DataStream;
 
 using BCG;
 
-using bts.api.core;
-
-using Bts.DataStreamCore;
-
-using Microsoft.Extensions.Logging;
-
 using Spark.Core.Disposables;
 
 using System;
-using System.Collections;
-using System.Collections.Generic;
-using System.Linq;
 using System.Reactive.Linq;
 
 public static class DataStreamExtensions {
    public static ViewStream AsViewStream(this IObservable<DataPacket> source) =>
       source as ViewStream ?? new ObservableViewStream(source);
 
-   public static ViewStream AsViewStream<T>(this IObservable<IDataPacket<T>> source, string? name = null, ILogger? logger = null) where T : IBlueRow =>
-      source as ViewStream ?? new ObservableViewStream(source.Select(dp => new DataPacket(dp.newRows, dp.addedRows, dp.deletedRows)), name, logger);
-
    public static IObservable<DataPacket> AsObservable(this ViewStream source) {
       return source as IObservable<DataPacket> ?? Observable.Create<DataPacket>(Subscribe);
 
@@ -35,100 +23,4 @@ public static class DataStreamExtensions {
                $"Expected DataPacket, got {newValue?.GetType()}", nameof(newValue)));
       }
    }
-
-   public static IObservable<IDataPacket<T>> SelectDataPacket<T>(this IObservable<IDataPacket> source, Func<IBlueRow, T> selector) where T : IBlueRow =>
-      source.Select(dp => {
-         dp.get_new_added_deleted(out var newRows, out var addedRows, out var deletedRows);
-         return new DataPacket<T>(CastRows(newRows), CastRows(addedRows), CastRows(deletedRows));
-
-         IBlueRowCollection<T> CastRows(ICollection rows) =>
-            new BlueTableSelectorWrapper<T>(
-               rows as IBlueRowCollection ?? new BlueRowCollection(rows, dp.Schema), selector);
-      });
-
-   public static IObservable<TableUpdate<IBlueRow, int>> AsTables(this IObservable<IDataPacket> source) =>
-      source.AsTables<IBlueRow, int>(r => r, r => r.Id());
-
-   public static IObservable<TableUpdate<T, int>> AsTables<T>(this IObservable<IDataPacket> source, Func<IBlueRow, T> valueSelector) where T : IBlueRow =>
-      source.AsTables<T, int>(valueSelector, r => r.Id());
-
-   public static IObservable<TableUpdate<T, int>> AsTables<T>(
-      this IObservable<IDataPacket> source, Func<IBlueRow, T> valueSelector, Func<T, int> keySelector) =>
-      source.AsTables<T, int>(valueSelector, keySelector);
-
-   public static IObservable<TableUpdate<T, TKey>> AsTables<T, TKey>(
-      this IObservable<IDataPacket> source, Func<IBlueRow, T> valueSelector, Func<T, TKey> keySelector, Func<IBlueRow, TKey>? rowKeySelector = null) where TKey : IEquatable<TKey> {
-      rowKeySelector ??= r => keySelector(valueSelector(r));
-      return source.Scan(TableUpdate<T, TKey>.mk_empty(keySelector), (res, dp) => {
-         var bd = res.to_builder();
-         dp.get_new_added_deleted(out var newRows, out var addedRows, out var deletedRows);
-         foreach (IBlueRow r in addedRows) {
-            bd.add_or_update(valueSelector(r));
-         }
-         foreach (IBlueRow r in deletedRows) {
-            bd.remove(rowKeySelector(r));
-         }
-         return bd.to_table();
-      });
-   }
-
-   public static IObservable<IDataPacket<T>> AsDataPackets<T>(this IObservable<TableUpdate<T, int>> source, IBlueTableSchema schema) where T : IBlueRow =>
-      source.Scan(
-         (Table: new BlueTableImmutable<T>(schema), Dp: (IDataPacket<T>)DataPacket<T>.Empty(schema)),
-         (res, table) => {
-            var bd = res.Table.ToDataPacketBuilder();
-            foreach (var r in table.removed) {
-               bd.Remove(r.Id());
-            }
-            foreach (var r in table.added) {
-               bd.AddOrUpdate(r);
-            }
-            var resDp = bd.ToDataPacketT(out var newRows, out var addedRows, out var deletedRows);
-            return (newRows, resDp);
-         })
-         .Where(t => t.Dp.addedRows.Any() || t.Dp.deletedRows.Any())
-         .Select(t => t.Dp);
-
-   public static ViewStream AsViewStream<T>(this IObservable<TableUpdate<T, int>> source, IBlueTableSchema schema) where T : IBlueRow =>
-      source.AsDataPackets(schema).AsViewStream();
-
-   public static int ResolveIndices(this IBlueTableSchema schema, string name) {
-      var indices = ResolveIndices(schema, new[] { name });
-      return indices[0];
-   }
-
-   public static (int, int) ResolveIndices(this IBlueTableSchema schema, string name1, string name2) {
-      var indices = ResolveIndices(schema, new[] { name1, name2 });
-      return (indices[0], indices[1]);
-   }
-
-   public static (int, int, int) ResolveIndices(this IBlueTableSchema schema, string name1, string name2, string name3) {
-      var indices = ResolveIndices(schema, new[] { name1, name2, name3 });
-      return (indices[0], indices[1], indices[2]);
-   }
-
-   public static (int, int, int, int) ResolveIndices(this IBlueTableSchema schema, string name1, string name2, string name3, string name4) {
-      var indices = ResolveIndices(schema, new[] { name1, name2, name3, name4 });
-      return (indices[0], indices[1], indices[2], indices[3]);
-   }
-
-   public static int[] ResolveIndices(this IBlueTableSchema schema, string[] columnNames) {
-      var columns = schema.columnInfo;
-      var indices = new int[columnNames.Length];
-      int nResolved = 0;
-      for (int i = 0; i < columnNames.Length; i++) {
-         int origNResolved = nResolved;
-         for (int j = 0; j < columns.Length; j++) {
-            if (columns[j].colName == columnNames[i]) {
-               indices[i] = j;
-               nResolved++;
-               break;
-            }
-         }
-         if (nResolved == origNResolved) {
-            throw new ArgumentException($"No column named '{columnNames[i]}'", nameof(schema));
-         }
-      }
-      return indices;
-   }
 }

src/DataStreamSuite/DataStream/Misc/DSLStream.cs

@@ -1,33 +0,0 @@
-namespace bts.SilverPine {
-   using System;
-   using BCG;
-   using System.Collections.Generic;
-   using Bts.DataStreamExpressions;
-   using Bts.DataStream;
-
-   // All this does is take in a dsl expression, parse it and fire the data that come out of running it.
-   // So for example, if an expresison is passed 
-   [DataStreamUserRespresentation("dsl")]
-   public class DSLStream : ViewStream {
-      private readonly string _dsl;
-      public DSLStream(string dsl, IDataStreamFactory streamFactory) {
-         Log.Gen.info($"DSLStream.cons dsl: {dsl}");
-         _dsl = dsl;
-         var rs = build_stream(streamFactory, dsl);
-         rs = rs ?? new LiteralStream("error", $"msg:invalid dsl");
-         mySetRealSource(rs);
-      }
-
-      static ViewStream? build_stream(IDataStreamFactory streamFactory, string stream_expression) {
-         try {
-            var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression);
-            var rv = exp.getView(new Dictionary<string, StatementExpression>(0), streamFactory);
-            return rv;
-
-         } catch (Exception ex) {
-            Log.Gen.error($"Could not build stream: {stream_expression} ex: {ex}");
-            return null;
-         }
-      }
-   }
-}

src/DataStreamSuite/DataStream/ObservableViewStream.cs

@@ -4,21 +4,14 @@ using BCG;
 
 using bts;
 
-using Microsoft.Extensions.Logging;
-
 using System;
 
 public class ObservableViewStream : ViewStream, IObservable<DataPacket>, IDisposable {
    readonly IObservable<DataPacket> source;
-   readonly string? name;
-   readonly ILogger? logger;
    IDisposable? subscription;
 
-   public ObservableViewStream(IObservable<DataPacket> source, string? name = null, ILogger? logger = null) {
+   public ObservableViewStream(IObservable<DataPacket> source) =>
       this.source = source;
-      this.name = name;
-      this.logger = logger;
-   }
 
    public void Dispose() {
       GC.SuppressFinalize(this);
@@ -30,13 +23,7 @@ public class ObservableViewStream : ViewStream, IObservable<DataPacket>, IDispos
 
    protected override void myOnFirstSubscriber() {
       Log.isnull(subscription);
-      subscription = source.Subscribe(myFireEvent, OnError, OnCompleted);
-
-      void OnError(Exception exception) =>
-         logger?.LogError("Error in ObservableViewStream. name: {Name}, error: {Exception}", name, exception);
-
-      void OnCompleted() =>
-         logger?.LogInformation("ObservableViewStream completed. name: {Name}", name);
+      subscription = source.Subscribe(myFireEvent);
    }
 
    protected override void myOnLastSubscriber() {

src/DataStreamSuite/DataStream/Offloading/RemoteQueryManager.cs

@@ -34,12 +34,12 @@ namespace BCG {
             view_publisher = view_publisher_;
          }
       }
-      static ViewStream? build_stream(IDataStreamFactory streamFactory, string stream_expression)
+      static ViewStream? build_stream(string stream_expression)
       {
          try
          {
             var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression);
-            var rv = exp.getView(new Dictionary<string, StatementExpression>(0), streamFactory);
+            var rv = exp.getView(new Dictionary<string, StatementExpression>(0));
             return rv;
 
          } catch (Exception ex) {
@@ -51,17 +51,15 @@ namespace BCG {
          _stream_pub_map = new Dictionary<string, StreamPubUnit>();
       readonly IPubTableBroker _broker;
       readonly IManagedSubscriber _sub;
-      readonly IDataStreamFactory streamFactory;
       readonly object syncRoot = new();
       IDisposable? cloudStreamsSub;
       IDisposable? remoteRequestsSub;
       bool isDisposed;
 
 
-      public RemoteQueryManager(IManagedSubscriber managedSubscriber, IPubTableBroker pubTableBroker, IDataStreamFactory streamFactory) {
+      public RemoteQueryManager(IManagedSubscriber managedSubscriber, IPubTableBroker pubTableBroker) {
          _sub = managedSubscriber;
          _broker = pubTableBroker;
-         this.streamFactory = streamFactory;
       }
 
       public Task StartAsync(CancellationToken cancellationToken) {
@@ -104,11 +102,7 @@ namespace BCG {
 
       private void _spawn_remote_stream(string name, string stream_expression, bool persist) {
          bts.Log.Gen.info($"CloudStreamManager.assign name: {name} stream_expression: {stream_expression}");
-         var stream = build_stream(streamFactory, stream_expression);
-         if (stream is null) {
-            Log.DataFlow.error($"Failed to spawn remote stream. name: {name}; expression: {stream_expression}");
-            return;
-         }
+         ViewStream stream = build_stream(stream_expression);
          _expose_to_ws(name, stream);
          if (persist) {
             stream = new PersistorStream(stream, name, "C:\\Temp\\", 1000000);   // 10MBs

src/DataStreamSuite/DataStream/Offloading/RemoteQueryStream.cs

@@ -25,12 +25,12 @@ namespace BCG {
          byte[] rem_req_bytes = rem_req.serialize();
          _publisher_core.publish("System", "RemoteRequest", rem_req_bytes);
          _source_str = new TableStream(_name);
-         _source_str.onChange += _handle_data;
+         _source_str.onChange += new ChangeDelegate(_handle_data);
       }
 
       protected override void myOnLastSubscriber() {
          //maybe there should be a way to turn remote requests off ?
-         _source_str.onChange -= _handle_data;
+         _source_str.onChange -= new ChangeDelegate(_handle_data);
       }
 
       static string get_expression(params string[] lines) {

src/DataStreamSuite/DataStream/Primitives/Count/CountStream.cs

@@ -21,11 +21,13 @@ namespace BCG
          : this (new TableStream (tableName))
       {}
 
-      protected override void myOnFirstSubscriber () =>
-         myVS.onChange += myHandleChanges;
+      protected override void myOnFirstSubscriber () {
+         myVS.onChange += new ChangeDelegate(myHandleChanges);
+      }
+      protected override void myOnLastSubscriber () {
+         myVS.onChange -= new ChangeDelegate(myHandleChanges);
+      }
 
-      protected override void myOnLastSubscriber () =>
-         myVS.onChange -= myHandleChanges;
 
       private void myHandleChanges(object newValue)
       {

src/DataStreamSuite/DataStream/Primitives/Crushing/CrushingCollectorStream.cs

@@ -30,7 +30,7 @@ namespace BCG
 
         protected override void myOnFirstSubscriber()
         {
-            myToCrush.onChange += myToCrush_onChange;
+            myToCrush.onChange += new ChangeDelegate(myToCrush_onChange);
         }
 
         protected override void myOnLastSubscriber()

src/DataStreamSuite/DataStream/Primitives/Crushing/CrushingStream.cs

@@ -15,11 +15,11 @@ namespace BCG
             
             //set up an anonymous delegate for each of the streams
             //we do this so that we can unsubscribe
-            myDelegates = new DataStreamChangeDelegate[myToCrush.Length];
+            myDelegates = new ChangeDelegate[myToCrush.Length];
             for (int i = 0; i < myToCrush.Length; ++i)
             {
                 int tempI = i;
-                myDelegates[tempI] = new((o) => { myHandleChange(tempI, ((DataPacket)o).newRows); });
+                myDelegates[tempI] = new ChangeDelegate((o) => { myHandleChange(tempI, ((DataPacket)o).newRows); });
             }
         }
 
@@ -27,7 +27,7 @@ namespace BCG
         private ViewStream[] myToCrush;
         private ITableCrusher myCrusher;
 
-        private DataStreamChangeDelegate[] myDelegates;
+        private ChangeDelegate[] myDelegates;
 
         protected override void myOnFirstSubscriber()
         {

src/DataStreamSuite/DataStream/Primitives/Derivative/DerivativeStream.cs

@@ -28,7 +28,7 @@ namespace BCG
 
       protected override void myOnFirstSubscriber()
       {
-         myDataSource.onChange += myDataSource_onChange;
+         myDataSource.onChange += new ChangeDelegate(myDataSource_onChange);
       }
 
       protected override void myOnLastSubscriber()

src/DataStreamSuite/DataStream/Primitives/FileStreams/DumpLoaderStream.cs

@@ -0,0 +1,49 @@
+// using System;
+// using System.Collections.Generic;
+// using System.Text;
+// using System.IO;
+
+// using System.Runtime.Serialization.Formatters.Binary;
+
+// namespace BCG
+// {
+//    public class DumpLoaderStream : ViewStream, ISideEffectStream
+//    {
+//       //wtf
+//       //public class Exception : System.Exception
+//       //{public Exception(string msg) : base(msg) { }}
+
+//       public DumpLoaderStream(string fileName, int secondsBetweenPolls)
+//       {
+//          mySetRealSource(
+//             new PollStream(() => new LoaderStream(fileName).rows, secondsBetweenPolls, 
+//                ViewStreamDatabase.instance));
+//       }
+
+//       public DumpLoaderStream(string fileName)
+//       {
+//         if (fileName == null)
+//            throw new LoaderStream.Exception("File name cannot be null");
+
+//         for (int i = 0; i < 5; i++)
+//            try
+//            {
+//               using (FileStream fs = new FileStream(fileName, FileMode.Open, FileAccess.Read))
+//               {
+//                  IClientDeserializer deser = new ClientUniversalDeserializer();
+//                  byte[] the_ops;
+//                  Array the_rows;
+//                  deser.deserialize(new Stream[] { fs }, out the_ops, out the_rows);
+
+//                  mySetRealSource(new BlueRowCollectionStaticStream((IBlueRowCollection)the_rows.GetValue(0)));
+//               }
+//               break;
+//            }
+//            catch (IOException)
+//            {
+//               System.Threading.Thread.Sleep(50);
+//            }
+
+//       }
+//    }
+// }

src/DataStreamSuite/DataStream/Primitives/FileStreams/LoaderStream.cs

@@ -1,7 +1,6 @@
 using BCG;
 
 namespace bts {
-   // This is the other side of the SaverStream, its "loads" the stream and fires once
    [DataStreamUserRespresentation("loader")]
    public class LoaderStream : ViewStream {
 

src/DataStreamSuite/DataStream/Primitives/FileStreams/PersistorStream.cs

@@ -5,11 +5,10 @@ using System.Text;
 using System.IO;
 using Google.Protobuf;
 using System.Globalization;
-using BCG;
+using bts;
 
-namespace bts {
-   // Writes all the udpates to a stream in a file or multiples files dependong on the rollover size (as opposed to Saver
-   // which just writes one update, as a create)
+namespace BCG {
+   // Writes all the udpates to a stream in a file.
    // File naming:
    // token_epoch_yyyyMMdd_HHmmss
    // File location: directory(passed)\token\
@@ -95,10 +94,10 @@ namespace bts {
       void _on_change(object new_value) {
          DataPacket dp = new_value as DataPacket;
          byte[] serialized_update = null;
-         bool first_write = _serializer == null || _n_updates_in_current_file == 0;
+         bool _write_create = _serializer == null || _n_updates_in_current_file == 0;
          _serializer = _serializer ?? new UniversalStreamIOSerializer();
-         if (first_write) {
-            // first file always starts with create
+         if (_write_create) {
+            // new files always starts with create
             serialized_update = _serializer.serialize(new bts.StreamIOMessage(new bts.StreamIOMessage.CreateMessage(dp.newRows)));
          } else
             serialized_update = _serializer.serialize(new bts.StreamIOMessage(new bts.StreamIOMessage.MultipleRemoveAddMessage(dp.deletedRows, dp.addedRows)));
@@ -106,8 +105,8 @@ namespace bts {
          long utc_epoch = epoch_now();
 
          //time, length of data, data
-         //bw.Write(utc_epoch);
-         //bw.Write(serialized_update.Length);
+         bw.Write(utc_epoch);
+         bw.Write(serialized_update.Length);
          bw.Write(serialized_update);
          _out_fs.Flush();
          myFireEvent(dp);

src/DataStreamSuite/DataStream/Primitives/FileStreams/PlaybackStream.cs

@@ -0,0 +1,190 @@
+// using System;
+// using System.Collections.Generic;
+// using System.Linq;
+// using System.Text;
+// using System.IO;
+// using System.Runtime.Serialization.Formatters.Binary;
+// using BCG.Utilities.General;
+// using bts.collections;
+// using System.Reflection.Metadata;
+
+
+// namespace BCG {
+
+//    public class PlaybackStream : ViewStream, IWorker {
+//       private readonly PlaybackType _type;
+//       private BlueTable _current_table;
+//       private IClientDeserializer _deserilializer = new ClientUniversalDeserializer();
+//       // these are the packets already downloaded from the current file
+//       private List<PlaybackPacket> _packets = new List<PlaybackPacket>();
+//       //private IEnumerable<FileStream> _file_streams = new List<FileStream>();
+//       private IEnumerable<string> _file_names;
+//       private Tuple<int, FileStream> _current_fs = new Tuple<int, FileStream>(-1, null);
+//       private DateTime _playback_t0, _playback_t1;
+//       private bool _all_files_read = false;
+//       SnapshotData _snapshot_data = null; // only initialized when in snapshot mode
+
+//       public PlaybackStream(string datetime_start, string datetime_end, string directory_name, string token) {
+//          _type = PlaybackType.Streaming;
+//          _playback_t0 = PersistorStream.parse_datetime(datetime_start);
+//          _playback_t1 = PersistorStream.parse_datetime(datetime_end);
+//          string token_directory_name = $"{directory_name}\\{token}";
+//          bts.Log.Gen.info($"PlaybackStream t0: {_playback_t0} t1: {_playback_t1} dir: {token_directory_name}");
+//          List<string> files_in_range = new List<string>();
+//          if (Directory.Exists(token_directory_name)) {
+//             _file_names =
+//          Directory.GetFiles(token_directory_name).Where(f => {
+//             DateTime file_time = PersistorStream.parse_datetime_from_filename(f);
+//             return file_time > _playback_t0 && file_time < _playback_t1;
+//          });
+//          }
+//       }
+
+//       protected override void myOnFirstSubscriber() {
+//          FileStream fs = new FileStream(_snapshot_data.snapshot_file, FileMode.Open, FileAccess.Read);
+//          _snapshot_data.packets = StreamPersistorUtils.load_packets_from_file(fs);
+//          PlaybackPacket snapshot_packet = StreamPersistorUtils.find_packet_at_time(_snapshot_data.packets, _snapshot_data.snapshot_time);
+//          if (snapshot_packet != null) {
+//             foreach (PlaybackPacket p in _snapshot_data.packets) {
+//                byte[] ops;
+//                Array row_array;
+//                _deserilializer.deserialize(new Stream[] { new MemoryStream(p.data) }, out ops, out row_array);
+//                for (int i = 0; i < ops.Length; i++) {
+//                   IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
+//                   byte op = ops[i];
+//                   if (op == StreamOpCode.create) {
+//                      IBlueRowCollection full_table = rows_args[0];
+//                      _current_table = new BlueTable(full_table);
+//                   } else if (op == StreamOpCode.multipleRemoveAdd) {
+//                      //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
+//                      IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
+//                      IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
+//                      BlueDataUtility.absorbChanges(_current_table, dels, adds);
+//                   } else {
+//                      bts.Log.assert(false, $"Oprator: {op} is not expected");
+//                   }
+//                   if (p.idx == snapshot_packet.idx) {
+//                      myFireEvent(new DataPacket(_current_table, _current_table, new BlueTable(_current_table.schema)));
+//                      break;
+//                   }
+//                }
+//             }
+//          }
+//       }
+
+
+//       private void _load_packets_from_current_file() {
+//          int curr_idx = _current_fs.Item1;
+//          FileStream curr_fs = _current_fs.Item2;
+//          _packets = new List<PlaybackPacket>();
+//          _packets.AddRange(StreamPersistorUtils.load_packets_from_file(curr_fs));
+//       }
+
+//       private bool current_file_all_replayed {
+//          get {
+//             bool rv = true;
+//             _packets.ForEach(p => rv = rv && p.replayed);
+//             return rv;
+//          }
+//       }
+
+//       protected override void myOnLastSubscriber() {
+//          var fs = _current_fs.Item2;
+//          if (fs != null) {
+//             fs.Close();
+//          }
+//          ViewStreamDatabase.instance.removeWorker(this);
+//       }
+
+//       // public event PlaybackStreamProgress onProgress;
+//       // public event Action onFileExhausted;
+//       private PlaybackPacket _locate_packet_to_replay() {
+//          foreach (var p in _packets) {
+//             // returns the first non replayed packet
+//             if (!p.replayed)
+//                return p;
+//          }
+//          //else
+//          {
+//             double secondsElapsed = new BinaryReader(myFileReader).ReadDouble();
+//             myNextPacketTime = myPlaybackStart + new TimeSpan(0, 0, 0, 0, (int)(1000 * secondsElapsed / myPlaybackSpeed));
+//             if (onProgress != null)
+//                onProgress(myFileReader.Position, myFileReader.Length);
+//          }
+//       }
+
+//       #region IWorker Members
+
+//       public void doWork() {
+//          if (_all_files_read)
+//             return;
+//          if (current_file_all_replayed) {
+//             _load_next_file();
+//          }
+//          PlaybackPacket curr_packet = _locate_packet_to_replay();
+//          if (curr_packet.idx == 0) { // first packet in the file, fire it rightaway 
+//             byte[] ops;
+//             Array row_array;
+//             // what a stupid interface!
+//             _deserilializer.deserialize(new Stream[] { new MemoryStream(curr_packet.data) }, out ops, out row_array);
+//             _fire_packet(ops, row_array);
+//             curr_packet.replay_timestamp = PersistorStream.epoch_now();
+//          } else {
+//             PlaybackPacket prev_packet = _get_previous_packet(curr_packet);
+//             // this is the time we need to fire (replay) this packet
+//             long time_to_fire = prev_packet.replay_timestamp + curr_packet.offset_from_previous;
+//             long now_epoch = PersistorStream.epoch_now();
+//             // fire when we are past the 
+//             if (now_epoch >= time_to_fire) {
+//                byte[] ops;
+//                Array row_array;
+//                _deserilializer.deserialize(new Stream[] { new MemoryStream(curr_packet.data) }, out ops, out row_array);
+//                _fire_packet(ops, row_array);
+//                curr_packet.replay_timestamp = PersistorStream.epoch_now();
+//             }
+//          }
+//       }
+
+//       private void _fire_packet(byte[] ops, Array row_array) {
+//          for (int i = 0; i < ops.Length; i++) {
+//             IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
+//             byte op = ops[i];
+//             if (op == StreamOpCode.create) {
+//                IBlueRowCollection full_table = rows_args[0];
+//                _current_table = new BlueTable(full_table);
+//                //_current_table.addRows(full_table);
+//                myFireEvent(new DataPacket(full_table, full_table, myLastValue == null ? new BlueTable(full_table.schema) :
+//                                           (myLastValue as DataPacket).newRows));
+//             } else if (op == StreamOpCode.multipleRemoveAdd) {
+//                //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
+//                IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
+//                IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
+//                BlueDataUtility.absorbChanges(_current_table, dels, adds);
+//                myFireEvent(new DataPacket(_current_table, adds, dels));
+//             } else {
+//                bts.Log.assert(false, $"Oprator: {op} is not expected");
+//             }
+
+//          }
+//       }
+
+//       private void _load_next_file() {
+//          var new_idx = _current_fs.Item1 + 1;
+//          if (new_idx >= _file_names.Count()) {
+//             _all_files_read = true;
+//             bts.Log.Gen.info("All files read");
+//             return;
+//          }
+//          var prev_fs = _current_fs.Item2;
+//          if (prev_fs != null)
+//             prev_fs.Close();
+//          var new_filename = _file_names.ElementAt(new_idx);
+//          bts.Log.Gen.info("PlaybackStream_load_next_file new_filename=: {new_filename}");
+//          var new_fs = new FileStream(new_filename, FileMode.Open, FileAccess.Read);
+//          _current_fs = new Tuple<int, FileStream>(new_idx, new_fs);
+//          _load_packets_from_current_file();
+//       }
+
+//       #endregion
+//    }
+// }

src/DataStreamSuite/DataStream/Primitives/FileStreams/RecorderStream.cs

@@ -0,0 +1,68 @@
+// using System;
+// using System.Collections.Generic;
+// using System.Linq;
+// using System.Text;
+// using System.IO;
+
+// namespace BCG
+// {
+//    public class RecorderStream : ViewStream
+//    {
+//       private ViewStream mySource;
+//       private DateTime myStartTime;
+//       private FileStream myOutFile;
+//       private string myFileName;
+//       private IBlueDataSerializer mySerializer;
+
+//       public RecorderStream(ViewStream src, string recordFile)
+//       {
+//          mySource = src;
+//          myFileName = recordFile;
+
+//          myFileName = myFileName.Replace("%DATE%", DateTime.Now.ToString("yyyyMMdd_HHmmss")); 
+//       }
+
+//       protected override void myOnFirstSubscriber()
+//       {
+//          myStartTime = DateTime.Now;         
+//          mySource.onChange += new ChangeDelegate(mySource_onChange);
+//          myOutFile = new FileStream(myFileName, FileMode.OpenOrCreate);
+//       }
+
+//       protected override void myOnLastSubscriber()
+//       {
+//          mySource.onChange -= mySource_onChange;
+//          myOutFile.Close();
+//       }
+
+//       void mySource_onChange(object newValue)
+//       {
+//          DataPacket dp = (DataPacket)newValue;         
+
+//          BlueTable toWrite = new BlueTable(dp.schema);
+
+//          if (mySerializer == null)
+//          {
+//             mySerializer = new UniversalSerializer(dp.schema);
+//          }
+//          foreach (IBlueRow r in dp.newRows)
+//             toWrite.addRow(r);
+
+//          byte[] result;
+
+//          if (myLastValue == null)
+//             mySerializer.serialize(StreamOpCode.create, new IBlueRowCollection[] { dp.newRows }, out result);
+//          else
+//             mySerializer.serialize(StreamOpCode.multipleRemoveAdd, new IBlueRowCollection[] { dp.deletedRows, dp.addedRows }, out result);
+
+//          BinaryWriter bw = new BinaryWriter(myOutFile);
+//          //write down what time this happened
+//          bw.Write((DateTime.Now - myStartTime).TotalSeconds);
+//          bw.Write(result.Length);
+//          bw.Write(result);         
+//          myOutFile.Flush();
+
+//          myFireEvent(dp);
+//       }
+//    }
+// }

src/DataStreamSuite/DataStream/Primitives/FileStreams/SaverStream.cs

@@ -5,8 +5,7 @@ using System.IO;
 using BCG;
 
 namespace bts {
-   // this saves the updates as a Create on the same file, each update calls -save, which
-   // reqrites the whole data to the packet. The other side of this stream is the Loader
+   // this saves the total state in the filr provided in the cons
    [DataStreamUserRespresentation("saver")]
    public class SaverStream : ViewStream {
       private ViewStream _stream;
@@ -18,8 +17,7 @@ namespace bts {
          _filename = filename;
       }
 
-      protected override void myOnFirstSubscriber() =>
-         _stream.onChange += _handle_changes;
+      protected override void myOnFirstSubscriber() { _stream.onChange += new ChangeDelegate(_handle_changes); }
 
       void _handle_changes(object dp_obj) {
          DataPacket dp = dp_obj as DataPacket;
@@ -27,13 +25,13 @@ namespace bts {
          myFireEvent(dp_obj);
       }
 
-      protected override void myOnLastSubscriber() =>
-         _stream.onChange -= _handle_changes;
+      protected override void myOnLastSubscriber() { _stream.onChange -= new ChangeDelegate(_handle_changes); }
 
       protected void _save(DataPacket dp) {
          StreamIOMessage.CreateMessage cr_msg = new StreamIOMessage.CreateMessage(dp.newRows);
          byte[] str_bytes = _serializer.serialize(new StreamIOMessage(cr_msg));
          StreamSerializationUtils.write_to_file(str_bytes, _filename);
+
       }
    }
 }

src/DataStreamSuite/DataStream/Primitives/FileStreams/StreamPersistorUtils.cs

@@ -3,7 +3,6 @@ using System.Collections.Generic;
 using System.Linq;
 using System.Globalization;
 using System.IO;
-using bts;
 
 
 namespace BCG {

src/DataStreamSuite/DataStream/Primitives/Filter/FilterStream.cs

@@ -37,12 +37,12 @@ namespace BCG.StreamWorld.Addition
 
       protected override void myOnFirstSubscriber()
       {
-         myBaseStream.onChange += myHandleChanges;
+         myBaseStream.onChange += new ChangeDelegate(myHandleChanges);
       }
 
       protected override void myOnLastSubscriber()
       {
-         myBaseStream.onChange -= myHandleChanges;
+         myBaseStream.onChange -= new ChangeDelegate(myHandleChanges);
          myBaseTable.clear();
          myStateTable.clear();
       }
@@ -228,13 +228,13 @@ namespace BCG
       #endregion
 
       #region overridden methods
-      protected override void myOnFirstSubscriber() =>
-         myParentStream.onChange += myHandleCallbacks;
+      protected override void myOnFirstSubscriber()
+      { myParentStream.onChange += new ChangeDelegate(myHandleCallbacks); }
 
       protected override void myOnLastSubscriber()
       {
          myLastFiredRows = null;
-         myParentStream.onChange -= myHandleCallbacks;
+         myParentStream.onChange -= new ChangeDelegate(myHandleCallbacks);
       }
 
       protected override void myFireEvent(object o)

src/DataStreamSuite/DataStream/Primitives/Filter/NullFilterStream.cs

@@ -1,24 +0,0 @@
-namespace bts.SilverPine {
-   using BCG;
-
-   using System;
-
-   public class NullFilterStream : ViewStream {
-      public NullFilterStream(ViewStream parent_str, string col_name)
-      : this(parent_str, col_name, true) { }
-      public NullFilterStream(ViewStream parent_str, string col_name, bool filter_out_doublenans = false) {
-         ViewStream rs = new FilterStream(parent_str,
-         r => {
-            bool is_dbnull = r.is_null(col_name);
-            if (is_dbnull) {
-               return false;
-            }
-            if (r.schema[col_name].type_code() == TypeCode.Double) {
-               return !Double.IsNaN(r.get<double>(col_name));
-            }
-            return true;
-         });
-         mySetRealSource(rs);
-      }
-   }
-}

src/DataStreamSuite/DataStream/Primitives/Freeze/FreezeDataStream.cs

@@ -36,11 +36,11 @@ namespace BCG
          get{return myIsFrozen;}
       }
 
-      protected override void myOnFirstSubscriber () =>
-         myStreamToFreeze.onChange += myCallBack;
+      protected override void myOnFirstSubscriber ()
+      {myStreamToFreeze.onChange += new DataStream.ChangeDelegate (myCallBack);}
 
-      protected override void myOnLastSubscriber () =>
-         myStreamToFreeze.onChange -= myCallBack;
+      protected override void myOnLastSubscriber ()
+      {myStreamToFreeze.onChange -= new DataStream.ChangeDelegate (myCallBack);}
 
       internal void myCallBack(object o)
       {

src/DataStreamSuite/DataStream/Primitives/Fuzzy/FuzzyJoinOverTimeStream.cs

@@ -96,15 +96,15 @@ namespace BCG
 
       protected override void myOnFirstSubscriber()
       {
-         myLeftSource.onChange += myLeft_onChange;
-         myRightSource.onChange += myRight_onChange;
+         this.myLeftSource.onChange += new ChangeDelegate(myLeft_onChange);
+         this.myRightSource.onChange += new ChangeDelegate(myRight_onChange);
          this.myVsd.addWorker(this);
       }
 
       protected override void myOnLastSubscriber()
       {
-         myLeftSource.onChange -= myLeft_onChange;
-         myRightSource.onChange -= myRight_onChange;
+         this.myLeftSource.onChange -= new ChangeDelegate(myLeft_onChange);
+         this.myRightSource.onChange -= new ChangeDelegate(myRight_onChange);
          this.myVsd.removeWorker(this);
       }
 

src/DataStreamSuite/DataStream/Primitives/GroupBy/AverageGroupsStream.cs

@@ -1,102 +1,102 @@
-// using System;
-// using System.Collections.Generic;
-// using System.Linq;
-// using System.Text;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
 
-// namespace BCG
-// {
+namespace BCG
+{
    
-//     public class AverageGroupsStream : ViewStream
-//     {
-//         public AverageGroupsStream(ViewStream input, string colToAverage, string keyColumn)
-//         {
+    public class AverageGroupsStream : ViewStream
+    {
+        public AverageGroupsStream(ViewStream input, string colToAverage, string keyColumn)
+        {
             
-//             myColToAverage = colToAverage;
-//             myInput = input;
-//             myKeyColumn = keyColumn;
-//         }
+            myColToAverage = colToAverage;
+            myInput = input;
+            myKeyColumn = keyColumn;
+        }
 
-//         private ViewStream myInput;
-//         private string myColToAverage;
-//         private BlueTableSchema mySchema;
-//         private string myKeyColumn;
+        private ViewStream myInput;
+        private string myColToAverage;
+        private BlueTableSchema mySchema;
+        private string myKeyColumn;
 
-//         protected override void myOnFirstSubscriber()
-//         {
-//             myInput.onChange += new ChangeDelegate(myInput_onChange);
-//         }
-//         protected override void myOnLastSubscriber()
-//         {
-//             myInput.onChange -= myInput_onChange;
-//         }
-//         private void myInput_onChange(object newValue)
-//         {
-//             if (mySchema == null)
-//                 myBuildSchema(((DataPacket)newValue).schema);
+        protected override void myOnFirstSubscriber()
+        {
+            myInput.onChange += new ChangeDelegate(myInput_onChange);
+        }
+        protected override void myOnLastSubscriber()
+        {
+            myInput.onChange -= myInput_onChange;
+        }
+        private void myInput_onChange(object newValue)
+        {
+            if (mySchema == null)
+                myBuildSchema(((DataPacket)newValue).schema);
 
-//             IBlueRowCollection newRows = myEvaluate((DataPacket)newValue);
+            IBlueRowCollection newRows = myEvaluate((DataPacket)newValue);
 
-//             if (myLastValue == null)
-//                 myFireEvent(new DataPacket(newRows, newRows, new BlueTable(mySchema)));
-//             else
-//             {
-//                 IBlueRowCollection added, deleted;
-//                 BlueDataUtility.computeDifferences(newRows, ((DataPacket) myLastValue).newRows, out deleted, out added);
-//                 myFireEvent(new DataPacket(newRows, added, deleted));
-//             }
-//         }
+            if (myLastValue == null)
+                myFireEvent(new DataPacket(newRows, newRows, new BlueTable(mySchema)));
+            else
+            {
+                IBlueRowCollection added, deleted;
+                BlueDataUtility.computeDifferences(newRows, ((DataPacket) myLastValue).newRows, out deleted, out added);
+                myFireEvent(new DataPacket(newRows, added, deleted));
+            }
+        }
 
-//         private void myBuildSchema(IBlueTableSchema schema)
-//         {
-//             List<ColumnInfo> columns = new List<ColumnInfo>();
+        private void myBuildSchema(IBlueTableSchema schema)
+        {
+            List<ColumnInfo> columns = new List<ColumnInfo>();
 
-//             foreach (ColumnInfo ci in schema.columnInfo)
-//                 columns.Add(ci);
+            foreach (ColumnInfo ci in schema.columnInfo)
+                columns.Add(ci);
 
-//             columns.Add(new ColumnInfo("average_"+myColToAverage, columns.Count, typeof(double)));
-//             mySchema = new BlueTableSchema("average_of_" + schema.tableName, columns.ToArray());
-//         }
+            columns.Add(new ColumnInfo("average_"+myColToAverage, columns.Count, typeof(double)));
+            mySchema = new BlueTableSchema("average_of_" + schema.tableName, columns.ToArray());
+        }
 
 
 
-//         private IBlueRowCollection myEvaluate(DataPacket packet)
-//         {
+        private IBlueRowCollection myEvaluate(DataPacket packet)
+        {
          
-//             IDictionary<string, double> colToRunningTotal = new Dictionary<string, double>();
-//             IDictionary<string, int> colToCount = new Dictionary<string, int>();
-//             IDictionary<string, IBlueRow> colToLastRow = new Dictionary<string, IBlueRow>();
+            IDictionary<string, double> colToRunningTotal = new Dictionary<string, double>();
+            IDictionary<string, int> colToCount = new Dictionary<string, int>();
+            IDictionary<string, IBlueRow> colToLastRow = new Dictionary<string, IBlueRow>();
 
-//             foreach (IBlueRow r in packet.newRows)
-//             {
-//                 double thisVal = Convert.ToDouble(r[myColToAverage]);
+            foreach (IBlueRow r in packet.newRows)
+            {
+                double thisVal = Convert.ToDouble(r[myColToAverage]);
 
-//                 string keyVal = r[myKeyColumn].ToString();
-//                 if (!colToCount.ContainsKey(keyVal))
-//                 {
-//                     colToRunningTotal[keyVal] = thisVal;
-//                     colToCount[keyVal] = 1;
-//                 }
-//                 else
-//                 {
-//                     colToRunningTotal[keyVal] += thisVal;
-//                     colToCount[keyVal] += 1;
-//                 }
-//                 colToLastRow[keyVal] = r;
-//             }
+                string keyVal = r[myKeyColumn].ToString();
+                if (!colToCount.ContainsKey(keyVal))
+                {
+                    colToRunningTotal[keyVal] = thisVal;
+                    colToCount[keyVal] = 1;
+                }
+                else
+                {
+                    colToRunningTotal[keyVal] += thisVal;
+                    colToCount[keyVal] += 1;
+                }
+                colToLastRow[keyVal] = r;
+            }
 
-//             BlueTable result = new BlueTable(mySchema);
-//             foreach (string key in colToCount.Keys)
-//             {
-//                 object[] values = new object[mySchema.columnInfo.Length];
+            BlueTable result = new BlueTable(mySchema);
+            foreach (string key in colToCount.Keys)
+            {
+                object[] values = new object[mySchema.columnInfo.Length];
 
-//                 colToLastRow[key].CopyTo(values, 0);
-//                 values[values.Length - 1] = packet.newRows.Count > 0 ? colToRunningTotal[key] / packet.newRows.Count : 0.0;
-//                 result.addRow(new BlueRow(mySchema, values));
-//             }
+                colToLastRow[key].CopyTo(values, 0);
+                values[values.Length - 1] = packet.newRows.Count > 0 ? colToRunningTotal[key] / packet.newRows.Count : 0.0;
+                result.addRow(new BlueRow(mySchema, values));
+            }
 
-//             return result;
-//         }
+            return result;
+        }
 
       
-//     }
-// }
+    }
+}

src/DataStreamSuite/DataStream/Primitives/GroupBy/AverageNonIncGroupsStream.cs

@@ -1,31 +0,0 @@
-namespace bts.SilverPine {
-   using BCG;
-
-   using System;
-   using System.Collections.Generic;
-   using System.Linq;
-
-   [DataStreamUserRespresentation("Average")]
-   public class AverageNonIncGroupsStream : ViewStream {
-      public AverageNonIncGroupsStream(ViewStream parent_stream, string inp_col_name, params string[] group_by_cols) {
-         NonIncGroupByStream rs = new NonIncGroupByStream(parent_stream, () => group_by_cols,
-                                                         (g_rows) => _find_mean(inp_col_name, g_rows));
-         mySetRealSource(rs);
-      }
-
-      private List<IBlueRow> _find_mean(string inp_col_name, List<IBlueRow> g_rows) {
-         IBlueRow inp_row = g_rows.ElementAt(0);
-         IEnumerable<double> inp_vals = BlueDataUtility.get_colvalues<double>(g_rows, inp_col_name);
-         double sum = 0.0;
-         inp_vals.ToList().ForEach(d => sum += d);
-         string out_col_name = $"average_{inp_col_name}";
-         double average = sum / inp_vals.Count();
-         IBlueTableSchema schema_out = inp_row.schema.add_column(TypeCode.Double, out_col_name);
-         MutableBlueRow out_row = new MutableBlueRow(schema_out, inp_row);
-         out_row.setValue(out_col_name, average);
-         List<IBlueRow> rv = new List<IBlueRow>();
-         rv.Add(out_row);
-         return rv;
-      }
-   }
-}

src/DataStreamSuite/DataStream/Primitives/GroupBy/GroupByBundle.cs

@@ -1,91 +0,0 @@
-namespace bts.SilverPine {
-   using BCG;
-
-   using System;
-   using System.Collections.Generic;
-   using System.Linq;
-
-   using ZstdSharp.Unsafe;
-
-   public enum GroupByBundleOp {
-      Mean,
-      Median,
-      Mode,
-      StdDev,
-      Max,
-      Min,
-      First,
-      Last,
-      Sum
-   }
-   public class GroupByBundle {
-      public readonly List<Tuple<string, GroupByBundleOp>> operations;
-
-      public static GroupByBundle parse(string text) {
-         // text is inp_col: op, inp_col: op, inp_col:op
-         List<Tuple<string, GroupByBundleOp>> rv = new List<Tuple<string, GroupByBundleOp>>();
-         string[] calcs = text.Split(',');
-         foreach (string c in calcs) {
-            string[] toks = c.Split(':');
-            string inp_col = toks[0].Trim();
-            string op_txt = toks[1].Trim();
-            GroupByBundleOp op = Enum.Parse<GroupByBundleOp>(op_txt, true);
-            rv.Add(Tuple.Create(inp_col, op));
-         }
-         return new GroupByBundle(rv);
-      }
-
-      public GroupByBundle(List<Tuple<string, GroupByBundleOp>> operations_) {
-         operations = operations_;
-      }
-
-      public static IBlueRow group_by(List<IBlueRow> rows, GroupByBundle parameters) {
-         if (rows.Count() == 0) {
-            return null;
-         }
-
-         IBlueRow ir = rows.ElementAt(0);
-         IBlueTableSchema in_schema = ir.schema;
-         IBlueTableSchema out_schema = in_schema;
-         IDictionary<string, Tuple<string, GroupByBundleOp>> out_col_map =
-            new Dictionary<string, Tuple<string, GroupByBundleOp>>();
-         foreach (var t in parameters.operations) {
-            string out_col_name = $"{t.Item1}_{t.Item2}".ToLower();
-            out_schema = out_schema.add_column(TypeCode.Double, out_col_name);
-            out_col_map[out_col_name] = t;
-         }
-         MutableBlueRow or = new MutableBlueRow(out_schema, ir);
-         foreach (var kv in out_col_map) {
-            string out_col_name = kv.Key;
-            string in_col_name = kv.Value.Item1;
-            GroupByBundleOp op = kv.Value.Item2;
-            IEnumerable<double> col_vals = BlueDataUtility.get_colvalues<double>(rows, in_col_name);
-            double comp_res = compute_op(col_vals, op);
-            or.setValue(out_col_name, comp_res);
-         }
-         return or;
-      }
-
-      public static double compute_op(IEnumerable<double> col_vals, GroupByBundleOp op) {
-         switch (op) {
-            case GroupByBundleOp.Max:
-               return col_vals.Max<double>();
-            case GroupByBundleOp.Min:
-               return col_vals.Min<double>();
-            case GroupByBundleOp.Mean:
-               return col_vals.Sum() / col_vals.Count();
-            case GroupByBundleOp.Sum:
-               return col_vals.Sum();
-            case GroupByBundleOp.First:
-               return col_vals.First();
-            case GroupByBundleOp.Last:
-               return col_vals.Last();
-            default:
-               // TODO
-               Log.assert(false);
-               return Double.NaN;
-         }
-      }
-
-   }
-}

src/DataStreamSuite/DataStream/Primitives/GroupBy/GroupByStream.cs

@@ -175,7 +175,7 @@ namespace BCG {
          //gets resubscribed, we start the process of building data again, and treat the callback as the first 
          //callback.
          myInit();
-         myStream.onChange -= myChangeHandler;
+         myStream.onChange -= new DataStream.ChangeDelegate(myChangeHandler);
       }
 
       protected override void myInit() {

src/DataStreamSuite/DataStream/Primitives/GroupBy/MaxStream.cs

@@ -10,6 +10,6 @@ namespace BCG {
         }
 
         public MaxStream(ViewStream stream, string outputCol)
-           : this(new AddColumnsStream(stream, "fake_col: 0"), outputCol, new string[] { "fake_col" }) { }
+           : this(new AddColumnsStream(stream, "fakeCol=0"), outputCol, new string[] { "fakeCol" }) { }
     }
 }

src/DataStreamSuite/DataStream/Primitives/GroupBy/MethodDrivenStream.cs

@@ -82,12 +82,12 @@ namespace BCG
 
       protected override void myOnFirstSubscriber() 
       {
-         myStream.onChange += myChangeHandler;
+         myStream.onChange += new DataStream.ChangeDelegate (myChangeHandler);
       }
 
       protected override void myOnLastSubscriber () 
       {
-         myStream.onChange -= myChangeHandler;
+         myStream.onChange -= new DataStream.ChangeDelegate (myChangeHandler);
       }
    }
 }

src/DataStreamSuite/DataStream/Primitives/GroupBy/NonIncremental/NonIncGroupByStream.cs

@@ -1,40 +1,13 @@
 using System;
 using System.Collections.Generic;
 using System.Diagnostics;
-using System.Linq;
-using System.Text.Json;
 
 using bts;
-using bts.SilverPine;
 
-namespace BCG {
-   [DataStreamUserRespresentation("aggregate")]
-   public class NonIncGroupByStream : ViewStream {
-      private class DelNonIncGroupByEvaluator : INonIncGroupByEvaluator {
-         Func<string[]> _groupby_cols_func;
-         Func<List<IBlueRow>, List<IBlueRow>> _evaluate_func;
-         public DelNonIncGroupByEvaluator(Func<string[]> groupby_cols_func,
-                                          Func<List<IBlueRow>, List<IBlueRow>> evaluate_func) {
-            _groupby_cols_func = groupby_cols_func;
-            _evaluate_func = evaluate_func;
-         }
-         public IBlueRowCollection evaluate(IDictionary<int, List<IBlueRow>> grouped_rows) {
-            BlueTable rv = null;
-            foreach (var kv in grouped_rows) {
-               List<IBlueRow> g_rows = kv.Value;
-               List<IBlueRow> out_rows = _evaluate_func.Invoke(g_rows);
-               if (out_rows != null && out_rows.Count > 0) {
-                  rv = rv ?? new BlueTable(out_rows.ElementAt(0).schema);
-               }
-               rv.addRows(out_rows);
-            }
-            return rv;
-         }
-         public string[] get_group_by_cols() {
-            return _groupby_cols_func();
-         }
-
-      }
+namespace BCG
+{
+   public class NonIncGroupByStream : ViewStream
+   {
       private ViewStream _main_stream;
       private INonIncGroupByEvaluator _groupby_func;
       private BlueRowHasher _row_hasher;
@@ -42,47 +15,24 @@ namespace BCG {
       private Stopwatch _timer = new();
       private int _n_total_fires = 0, _n_expensive_fires = 0;
 
-      public NonIncGroupByStream(ViewStream main_stream, INonIncGroupByEvaluator groupby_func) {
+      public NonIncGroupByStream(ViewStream main_stream, INonIncGroupByEvaluator groupby_func)
+      {
          _main_stream = main_stream;
          _groupby_func = groupby_func;
          _groupby_cols = _groupby_func.get_group_by_cols();
-         _row_hasher = new BlueRowHasher(_groupby_cols == null ? 0 : _groupby_cols.Length);
+         _row_hasher = new BlueRowHasher(_groupby_cols.Length);
       }
 
-      public NonIncGroupByStream(ViewStream main_stream, Func<string[]> groupby_cols_func,
-                                 Func<List<IBlueRow>, List<IBlueRow>> evaluate_func)
-                                 : this(main_stream, new DelNonIncGroupByEvaluator(groupby_cols_func, evaluate_func)) { }
-
-      public NonIncGroupByStream(ViewStream parent_str, string groupby_col, string standard_group_by_text, string optional_sort_col = null) {
-         GroupByBundle std_group_by = GroupByBundle.parse(standard_group_by_text);
-         Func<string[]> groupby_cols_func = () => {
-            return String.IsNullOrEmpty(groupby_col) ? null : new string[] { groupby_col };
-         };
-         Func<List<IBlueRow>, List<IBlueRow>> evaluate_func = (group_rows) => {
-            if (!String.IsNullOrEmpty(optional_sort_col))
-               group_rows.Sort((r1, r2) => BlueDataUtility.compare_rows(r2, r1, optional_sort_col));
-            IBlueRow rv = GroupByBundle.group_by(group_rows, std_group_by);
-            return new IBlueRow[] { rv }.ToList();
-         };
-         ViewStream rs = new NonIncGroupByStream(parent_str, groupby_cols_func, evaluate_func);
-         mySetRealSource(rs);
-      }
-      public NonIncGroupByStream(ViewStream parent_str, string standard_group_by_text) :
-      this(parent_str, String.Empty, standard_group_by_text) {
-      }
-
-
       protected override void myOnFirstSubscriber() { _main_stream.onChange += _main_stream_onchange; }
 
       protected override void myOnLastSubscriber() { _main_stream.onChange -= _main_stream_onchange; }
 
-      private void _main_stream_onchange(object nv) {
+      private void _main_stream_onchange(object nv)
+      {
          DataPacket dp = nv as DataPacket;
          ///grouping together
          _timer.Restart();
-         IDictionary<int, List<IBlueRow>> grouped_rows = (_groupby_cols == null || _groupby_cols.Length == 0) ?
-         new Dictionary<int, List<IBlueRow>>() { { 0, dp.newRows.to_list() } } :
-         BlueDataUtility.group_together(dp.newRows, _groupby_cols, ref _row_hasher);
+         IDictionary<int, List<IBlueRow>> grouped_rows = BlueDataUtility.group_together (dp.newRows, _groupby_cols, ref _row_hasher);
          _timer.Stop();
          var groupElapsedMs = _timer.ElapsedMilliseconds;
          if (groupElapsedMs >= 50) {
@@ -98,15 +48,14 @@ namespace BCG {
             return;
          }
          var applyElapsedMs = _timer.ElapsedMilliseconds;
-         _n_total_fires++;
-         if (applyElapsedMs >= 50) {
+         _n_total_fires++;         
+         if (applyElapsedMs >= 50)
+         {
             _n_expensive_fires++;
             Log.DataFlow.info(
                $"Time to evaluate {grouped_rows.Count} groups {_timer.ElapsedMilliseconds}ms, ratio {_n_expensive_fires}/{_n_total_fires}");
-         }
-         // TODO,
-         if (total_state != null)
-            myFireEvent(new DataPacket(total_state, total_state, myLastValue == null ? new BlueTable(total_state.schema) : (myLastValue as DataPacket).newRows));
+         }         
+         myFireEvent(new DataPacket(total_state, total_state, myLastValue == null ? new BlueTable(total_state.schema) : (myLastValue as DataPacket).newRows));             
       }
    }
 }

src/DataStreamSuite/DataStream/Primitives/Join/InternalJoinStream.cs

@@ -31,15 +31,15 @@ namespace BCG
       {
          myLeftOperand.master = this;
          myRightOperand.master = this;
-         myLeftStream.onChange += myLeftOperand.changeHandler;
-         myRightStream.onChange += myRightOperand.changeHandler;
+         myLeftStream.onChange += new DataStream.ChangeDelegate (myLeftOperand.changeHandler);
+         myRightStream.onChange += new DataStream.ChangeDelegate (myRightOperand.changeHandler);
       }
       
       protected override void myOnLastSubscriber () 
       {
          myInit ();
-         myLeftStream.onChange -= myLeftOperand.changeHandler;
-         myRightStream.onChange -= myRightOperand.changeHandler;
+         myLeftStream.onChange -= new DataStream.ChangeDelegate (myLeftOperand.changeHandler);
+         myRightStream.onChange -= new DataStream.ChangeDelegate (myRightOperand.changeHandler);
       }
 
       public JoinOperand getOtherOperand (JoinOperand jo)

src/DataStreamSuite/DataStream/Primitives/Join/JoinStream.cs

@@ -116,7 +116,7 @@ namespace BCG
                IDictionary  map = resolver.getMap ();
                map["id"] = "id";
                myPs = new ProjectStream (myIjs, map);
-               myPs.onChange += myProjectChangeHandler;
+               myPs.onChange += new DataStream.ChangeDelegate (myProjectChangeHandler);
                return;
             }
             else
@@ -129,26 +129,26 @@ namespace BCG
       private void myProjectChangeHandler (object o)
       {
          if (myIjs.myOnChangeDelegate.GetInvocationList().Length > 1)
-            myIjs.onChange -= myInternalJoinChangeHandler;
+            myIjs.onChange -= new DataStream.ChangeDelegate (myInternalJoinChangeHandler);
          myFireEvent ((o as ICloneable).Clone());
          
       }
 
       protected override void myOnFirstSubscriber ()
       {
-         myIjs.onChange += myInternalJoinChangeHandler;
+         myIjs.onChange += new DataStream.ChangeDelegate (myInternalJoinChangeHandler);
       }
 
       protected override void myOnLastSubscriber ()
       {
          // how does this relate to the -= in myProjectChangeHandler?
-         myIjs.onChange -= myInternalJoinChangeHandler;
+         myIjs.onChange -= new DataStream.ChangeDelegate (myInternalJoinChangeHandler);
 
          // if there is a  project after the ijs we need to 
          // unsubscribe to it otherwise references will be kept and 
          // a memory leak will occur
          if (myPs != null)
-            myPs.onChange -= myProjectChangeHandler;
+            myPs.onChange -= new DataStream.ChangeDelegate (myProjectChangeHandler);
          myPs = null;
       }
 

src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinBookkeeper.cs

@@ -1,16 +1,15 @@
 using bts;
-using bts.collections;
 
 using mbts;
 
 using System;
 using System.Collections.Generic;
 using System.Diagnostics;
-using System.Linq;
-using System.Security.Cryptography;
 
-namespace BCG {
-   public class NonIncJoinBookkeeper : IDisposable {
+namespace BCG
+{
+   public class NonIncJoinBookkeeper : IDisposable
+   {
       public Action on_new_data;
       private readonly NonIncJoinSpec _join_spec;
       private readonly int _join_position;
@@ -27,71 +26,41 @@ namespace BCG {
 
       private int[] _join_col_indices = null,
                      _data_col_indices = null;
-
-      public NonIncJoinBookkeeper(NonIncJoinSpec the_join_spec, int join_position, int start_index_in_output, BlueRowHasher row_hasher) {
+      
+      public NonIncJoinBookkeeper(NonIncJoinSpec the_join_spec, int join_position, int start_index_in_output, BlueRowHasher row_hasher)
+      {
          _join_spec = the_join_spec;
          _join_position = join_position;
          _start_index_in_output = start_index_in_output;
          _row_hasher = row_hasher;
          _is_left_most = _join_position == 0;
          _is_tag_along = _join_spec.join_col_names.Length == 0;
-      }
-
-      private bool _filled_missing_data = false;
-      private void _fill_missing_data(DataPacket dp) {
-         if (_filled_missing_data)
-            return;
-         Func<string[], bool> null_or_empty = (str_arr) => {
-            if (str_arr == null || str_arr.Length == 0)
-               return true;
-            foreach (string s in str_arr) {
-               if (!String.IsNullOrEmpty(s)) {
-                  return false;
-               }
-            }
-            return true;
-         };
-         Func<string[], string[]> remove_bad_cols = (col_names) => {
-            HashSet<string> schema_col_map = new HashSet<string>(dp.schema.columnNames);
-            List<string> cols_list = new List<string>(col_names);
-            cols_list.RemoveWhere(c => !schema_col_map.Contains(c));
-            return cols_list.ToArray();
-         };
-         if (null_or_empty(_join_spec.data_col_names)) {
-            _join_spec.data_col_names = dp.schema.columnNames;
-         }
-         _join_spec.data_col_names = remove_bad_cols(_join_spec.data_col_names);
-         if (null_or_empty(_join_spec.data_col_names)) {
-            _join_spec.data_col_names = dp.schema.columnNames;
-         }
-         if (null_or_empty(_join_spec.projected_col_names)) {
-            _join_spec.projected_col_names = _join_spec.data_col_names;
-         }
-      }
-
-      public void init() {
          _join_spec.stream.onChange += _on_stream_cb;
       }
 
-      public void _on_stream_cb(object dp_obj) {
-         Log.Gen.info($"NonIncJoinBookkeeper._on_stream_cb dp_obj: {dp_obj}");
+      public void _on_stream_cb(object dp_obj)
+      {
          var timer = Stopwatch.StartNew();
          DataPacket dp = (DataPacket)dp_obj;
-         _fill_missing_data(dp);
-         if (_schema == null) {
+         if (_schema == null)
+         {
             _schema = dp.schema;
             _join_col_indices = BlueDataUtility.get_column_indices(_schema, _join_spec.join_col_names);
             _data_col_indices = BlueDataUtility.get_column_indices(_schema, _join_spec.data_col_names);
          }
 
-         if (_is_left_most) {
+         if (_is_left_most)
+         {
             _row_list.Clear();
             foreach (IBlueRow r in dp.newRows)
-               _row_list.Add(new Pair<int, IBlueRow>(_row_hasher.get_hash(r, _join_col_indices), r));
-         } else {
+               _row_list.Add(new Pair<int, IBlueRow>(_row_hasher.get_hash(r, _join_col_indices), r));            
+         }
+         else
+         {
             if (_is_tag_along)
                _solitary_row = BlueDataUtility.extractRow(dp.newRows, 0);
-            else {
+            else
+            {
                _row_map.Clear();
                foreach (IBlueRow r in dp.newRows)
                   _row_map[_row_hasher.get_hash(r, _join_col_indices)] = r;
@@ -110,25 +79,31 @@ namespace BCG {
 
       public bool is_ready { get { return _schema != null; } }
 
-      public void constribute_cols_to_schema(List<ColumnInfo> cis) {
+      public void constribute_cols_to_schema(List<ColumnInfo> cis)
+      {
          for (int i = 0; i < _join_spec.data_col_names.Length; i++)
             cis.Add(new ColumnInfo(_join_spec.projected_col_names[i], cis.Count, _schema[_join_spec.data_col_names[i]].colType));
       }
 
       public List<Pair<int, IBlueRow>> get_list() { return _row_list; }
 
-      public void copy_master_row_to_output_row(int row_index, object[] output_row) {
+      public void copy_master_row_to_output_row(int row_index, object[] output_row)
+      {
          IBlueRow this_row = _row_list[row_index].second;
          BlueDataUtility.copy_to_row(this_row, _data_col_indices, output_row, _start_index_in_output);
       }
 
-      public void copy_matching_row_to_output_row(int row_hash, object[] output_row) {
-         if (_is_tag_along) {
+      public void copy_matching_row_to_output_row(int row_hash, object[] output_row)
+      {
+         if (_is_tag_along)
+         {
             if (_solitary_row != null)
                BlueDataUtility.copy_to_row(_solitary_row, _data_col_indices, output_row, _start_index_in_output);
             else
                copy_dbnull_to_output_row(output_row);
-         } else {
+         }
+         else
+         {
             IBlueRow this_row = null;
             if (_row_map.TryGetValue(row_hash, out this_row))
                BlueDataUtility.copy_to_row(this_row, _data_col_indices, output_row, _start_index_in_output);
@@ -137,7 +112,8 @@ namespace BCG {
          }
       }
 
-      private void copy_dbnull_to_output_row(object[] output_row) {
+      private void copy_dbnull_to_output_row(object[] output_row)
+      {
          for (int i = 0; i < _data_col_indices.Length; i++)
             output_row[i + _start_index_in_output] = DBNull.Value;
       }

src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinSpec.cs

@@ -3,63 +3,63 @@ using System.Collections.Generic;
 using System.Linq;
 using System.Text;
 using System.Threading.Tasks;
-
 using mbts;
 
-namespace BCG {
-   // this a collection of data, theres no joining logic in here
-   public class NonIncJoinSpec {
+namespace BCG
+{
+   public class NonIncJoinSpec
+   {
       public ViewStream stream;
-      // join col names are the names on which the streams are joined (for equality),
-      // data col names are the names in the original stream
-      // projected col names are their new names, ie if its x:l_x, then x is in the 
-      // data_col_names and l_x in projected cols
       public string[] join_col_names, data_col_names, projected_col_names;
 
       public NonIncJoinSpec() { }
 
       public NonIncJoinSpec(ViewStream the_stream, string[] the_join_col_names, string[] the_data_col_names)
-         : this(the_stream, the_join_col_names, the_data_col_names, the_data_col_names) { }
+         : this (the_stream, the_join_col_names, the_data_col_names, the_data_col_names)
+      { }
 
-      public NonIncJoinSpec(ViewStream the_stream, string[] the_join_col_names, string[] the_data_col_names, string[] the_projected_col_names) {
+      public NonIncJoinSpec(ViewStream the_stream, string[] the_join_col_names, string[] the_data_col_names, string[] the_projected_col_names)
+      {
          stream = the_stream;
          join_col_names = the_join_col_names;
          data_col_names = the_data_col_names;
          projected_col_names = the_projected_col_names;
       }
 
-      public static NonIncJoinSpec[] from_triplet(params Triplet<ViewStream, string, string>[] join_specs) {
+      public static NonIncJoinSpec[] from_triplet(params Triplet<ViewStream, string, string>[] join_specs)
+      {
          List<NonIncJoinSpec> rv = new List<NonIncJoinSpec>();
-         foreach (Triplet<ViewStream, string, string> r in join_specs) {
-            string[] keys = String.IsNullOrEmpty(r.second) ? new string[] { } : r.second.Split(' ');
+         foreach (Triplet<ViewStream, string, string> r in join_specs)
+         {
+            string[] keys = String.IsNullOrEmpty (r.second) ? new string[] {} : r.second.Split(' ');
             string[] visible_cols = r.third.Split(' ');
             List<string> data_col_names = new List<string>(), mapped_col_names = new List<string>();
-            foreach (string s in visible_cols) {
+            foreach (string s in visible_cols)
+            {
                string[] toks = s.Split(':');
-               if (toks.Length == 1) {
+               if (toks.Length == 1)
+               {
                   data_col_names.Add(toks[0]);
                   mapped_col_names.Add(toks[0]);
-               } else if (toks.Length == 2) {
+               }
+               else if (toks.Length == 2)
+               {
                   data_col_names.Add(toks[0]);
                   mapped_col_names.Add(toks[1]);
                }
             }
-            rv.Add(new NonIncJoinSpec(r.first, keys, data_col_names.ToArray(), mapped_col_names.ToArray()));
+            rv.Add (new NonIncJoinSpec (r.first, keys, data_col_names.ToArray (), mapped_col_names.ToArray()));
          }
-         return rv.ToArray();
-      }
-
-      public override string ToString() {
-         Func<string[], string> arr_txt = (a) => {
-            string rv = "";
-            foreach (string s in a) {
-               rv += "{s} ";
-            }
-            rv = rv.Trim();
-            return rv;
-         };
-         string rv = $"stream: {stream} join_col_names: {arr_txt(join_col_names)} data_col_names: {arr_txt(data_col_names)} projected_col_names: {arr_txt(projected_col_names)}";
-         return rv;
+         return rv.ToArray ();
       }
    }
+
+   //[DataStreamUserRespresentation("ninctest")]
+   //public class NonIncJoinTestStream : ViewStream
+   //{
+   //   public NonIncJoinTestStream (string key_cols, string data_cols)
+   //   {
+   //      NonIncJoinSpec.from_triplet(new Triplet<ViewStream, string, string>(null, key_cols, data_cols));
+   //   }
+   //}
 }

src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinStream.cs

@@ -6,9 +6,11 @@ using System;
 using System.Collections.Generic;
 using System.Diagnostics;
 
-namespace BCG {
+namespace BCG
+{
    [DataStreamUserRespresentation("fastJoin")]
-   public class NonIncJoinStream : ViewStream, BCG.Utilities.General.IWorker {
+   public class NonIncJoinStream : ViewStream, BCG.Utilities.General.IWorker
+   {
       private readonly NonIncJoinSpec[] _join_specs;
       private readonly INonIncJoinInlineMapper _output_mapper;
       private readonly BlueRowHasher _row_hasher;
@@ -26,19 +28,24 @@ namespace BCG {
                {
                   new Triplet<ViewStream, string, string> (left, left_join_cols, left_proj),
                   new Triplet<ViewStream, string, string> (right, right_join_cols, right_proj)
-               }) { }
+               })
+      { }
 
       public NonIncJoinStream(params Triplet<ViewStream, string, string>[] join_specs)
-         : this(NonIncJoinSpec.from_triplet(join_specs), null, 0) { }
+         : this(NonIncJoinSpec.from_triplet(join_specs), null, 0)
+      { }
 
       public NonIncJoinStream(ViewStreamDatabase vsd, INonIncJoinInlineMapper output_mapper, int throttles_ms,
                               params Triplet<ViewStream, string, string>[] join_specs)
-         : this(vsd, NonIncJoinSpec.from_triplet(join_specs), output_mapper, throttles_ms) { }
+         : this(vsd, NonIncJoinSpec.from_triplet(join_specs), output_mapper, throttles_ms)
+      { }
 
       public NonIncJoinStream(NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper output_mapper, int throttles_ms = 0)
-         : this(ViewStreamDatabase.instance, join_specs, output_mapper, throttles_ms) { }
+         : this(ViewStreamDatabase.instance, join_specs, output_mapper, throttles_ms)
+      { }
 
-      public NonIncJoinStream(ViewStreamDatabase vsd, NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper output_mapper, int throttles_ms = 0) {
+      public NonIncJoinStream(ViewStreamDatabase vsd, NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper output_mapper, int throttles_ms = 0)
+      {
          _join_specs = join_specs;
          _output_mapper = output_mapper;
          _row_hasher = new BlueRowHasher(_join_specs[0].join_col_names.Length);
@@ -46,11 +53,12 @@ namespace BCG {
          _vsd = vsd;
       }
 
-      protected override void myOnFirstSubscriber() {
+      protected override void myOnFirstSubscriber()
+      {
          int data_col_no = 1;
-         for (int jp = 0; jp < _join_specs.Length; jp++) {
+         for (int jp = 0; jp < _join_specs.Length; jp++)
+         {
             NonIncJoinBookkeeper s = new NonIncJoinBookkeeper(_join_specs[jp], jp, data_col_no, _row_hasher);
-            Log.Gen.info($"NonIncJoinStream.myOnFirstSubscriber jp: {jp} spec: {_join_specs[jp]}");
             data_col_no += _join_specs[jp].data_col_names.Length;
             s.on_new_data += _stream_callback;
             _book_keeping_data.Add(s);
@@ -58,11 +66,10 @@ namespace BCG {
          if (_output_mapper != null)
             _output_mapper.onMapperChange += _on_mapper_change;
          _vsd.addWorker(this);
-         foreach (var bkd in _book_keeping_data)
-            bkd.init();
       }
 
-      protected override void myOnLastSubscriber() {
+      protected override void myOnLastSubscriber()
+      {
          foreach (NonIncJoinBookkeeper d in _book_keeping_data)
             d.Dispose();
          if (_output_mapper != null)
@@ -70,26 +77,30 @@ namespace BCG {
          _vsd.removeWorker(this);
       }
 
-      private void _stream_callback() {
-         Log.Gen.info("NonIncJoinStream._stream_callback received callback");
+      private void _stream_callback()
+      {
          _is_dirty = true;
          doWork();
       }
 
-      private void _on_mapper_change() {
+      private void _on_mapper_change()
+      {
          _is_dirty = true;
          doWork();
       }
 
-      private bool _is_ready() {
+      private bool _is_ready()
+      {
          bool rv = true;
          foreach (NonIncJoinBookkeeper d in _book_keeping_data)
             rv &= d.is_ready;
          return rv;
       }
 
-      private void _construct_schema() {
-         if (_output_schema == null) {
+      private void _construct_schema()
+      {
+         if (_output_schema == null)
+         {
             List<ColumnInfo> cis = new List<ColumnInfo>(new ColumnInfo[] { new ColumnInfo("id", 0, typeof(int)) });
             foreach (NonIncJoinBookkeeper d in _book_keeping_data)
                d.constribute_cols_to_schema(cis);
@@ -98,23 +109,27 @@ namespace BCG {
       }
 
       private int _n_total_fires = 0, _n_expensive_fires = 0;
-      private void _form_and_fire() {
-         Log.Gen.info($"NonIncJoinStream._form_and_fire _is_ready(): {_is_ready()}");
-         if (_is_ready()) {
+      private void _form_and_fire()
+      {
+         if (_is_ready())
+         {
             _construct_schema();
             var timer = Stopwatch.StartNew();
             IList<Pair<int, IBlueRow>> left_row_list = _book_keeping_data[0].get_list();
             List<object[]> output_rows = new List<object[]>();
 
-            for (int i = 0; i < left_row_list.Count; i++) {
+            for (int i = 0; i < left_row_list.Count; i++)
+            {
                object[] or = new object[_output_schema.columnInfo.Length];
                or[0] = i;
                output_rows.Add(or);
             }
 
-            for (int ji = 0; ji < _book_keeping_data.Count; ji++) {
+            for (int ji = 0; ji < _book_keeping_data.Count; ji++)
+            {
                NonIncJoinBookkeeper bk_data = _book_keeping_data[ji];
-               for (int i = 0; i < left_row_list.Count; i++) {
+               for (int i = 0; i < left_row_list.Count; i++)
+               {
                   if (ji == 0)
                      bk_data.copy_master_row_to_output_row(i, output_rows[i]);
                   else
@@ -124,20 +139,19 @@ namespace BCG {
             IBlueRowCollection dp_output_rows = _apply_map(new SequentialIDBlueTable(_output_schema, output_rows));
             timer.Stop();
             _n_total_fires++;
-            //if (timer.ElapsedMilliseconds >= 250) {
-            _n_expensive_fires++;
-            string log_msg = $"Join and map time for {_book_keeping_data.Count} "
-               + $"tables {timer.Elapsed.TotalMilliseconds} msecs, ratio {_n_expensive_fires}/{_n_total_fires}";
-            Log.Gen.info(log_msg);
-            //}
-            var dp_out = BlueDataUtility.build_dp_inc(dp_output_rows, myLastValue as DataPacket);
-            myFireEvent(dp_out);
-            //myFireEvent(new DataPacket(dp_output_rows, dp_output_rows, myLastValue == null ? new BlueTable(_output_schema) : (myLastValue as DataPacket).newRows));
+            if (timer.ElapsedMilliseconds >= 250) {
+               _n_expensive_fires++;
+               string log_msg = $"Join and map time for {_book_keeping_data.Count} "
+                  + $"tables {timer.Elapsed} ratio {_n_expensive_fires}/{_n_total_fires}";
+            }
+            myFireEvent(new DataPacket(dp_output_rows, dp_output_rows, myLastValue == null ? new BlueTable(_output_schema) : (myLastValue as DataPacket).newRows));
          }
       }
 
-      private IBlueRowCollection _apply_map(IBlueRowCollection unmapped_rows) {
-         if (_output_mapper != null) {
+      private IBlueRowCollection _apply_map(IBlueRowCollection unmapped_rows)
+      {
+         if (_output_mapper != null)
+         {
             IBlueRowCollection output_rows;
             var timer = Stopwatch.StartNew();
             output_rows = _output_mapper.map_rows(unmapped_rows);
@@ -146,14 +160,17 @@ namespace BCG {
                Log.Gen.info($"Time to map {unmapped_rows.Count} rows into {output_rows.Count} rows {timer.Elapsed}");
             }
             return output_rows;
-         } else {
+         }
+         else
+         {
             return unmapped_rows;
          }
       }
 
-      public void doWork() {
-         Log.Gen.info($"NonIncJoinStream.doWork _last_fire_time: {_last_fire_time} _is_dirty: {_is_dirty}");
-         if (DateTime.Now - _last_fire_time >= _throttle_period && _is_dirty) {
+      public void doWork()
+      {
+         if (DateTime.Now - _last_fire_time >= _throttle_period && _is_dirty)
+         {
             _is_dirty = false;
             _last_fire_time = DateTime.Now;
             _form_and_fire();

src/DataStreamSuite/DataStream/Primitives/Join/ParallelJoinTest.cs

@@ -181,8 +181,8 @@ namespace BCG
 
        protected override void myOnFirstSubscriber()
        {
-           myVLeft.onChange += myVLeft_onChange;
-           myVRight.onChange += myVRight_onChange;
+           myVLeft.onChange += new ChangeDelegate(myVLeft_onChange);
+           myVRight.onChange += new ChangeDelegate(myVRight_onChange);
        }
 
        void myVRight_onChange(object newValue)

src/DataStreamSuite/DataStream/Primitives/Join/Proximity/ProximityJoinStream.cs

@@ -1,68 +0,0 @@
-namespace bts {
-   using BCG;
-   using BCG.Utilities.General;
-
-   using System;
-   using System.Diagnostics;
-
-   using ZstdSharp.Unsafe;
-
-   // functon that joins the 2 rows and returns the outputput row. The function
-   // can return a null which means the 2 rows dont need to be joined
-   using JoinFunc = System.Func<BCG.IBlueRow, BCG.IBlueRow, BCG.IBlueRow>;
-   public class ProximityJoinStream : ViewStream, IWorker {
-      private ViewStream _lvs, _rvs;
-      private JoinFunc _join_func;
-      private bool _dirty = false;
-      private Stopwatch _stopwatch = new Stopwatch();
-      private ViewStreamDatabase _vsd = ViewStreamDatabase.instance;
-      public ProximityJoinStream(ViewStream lvs, ViewStream rvs, JoinFunc join_func) {
-         _lvs = lvs;
-         _rvs = rvs;
-         _join_func = join_func;
-      }
-
-      protected override void myOnFirstSubscriber() {
-         _lvs.onChange += _handle_data;
-         _rvs.onChange += _handle_data;
-         _vsd.addWorker(this);
-      }
-
-      protected override void myOnLastSubscriber() {
-         _lvs.onChange -= _handle_data;
-         _rvs.onChange -= _handle_data;
-         _vsd.removeWorker(this);
-      }
-
-      private void _handle_data(object dp_obj) {
-         _dirty = true;
-         doWork();
-      }
-
-      public void doWork() {
-         Func<bool> all_data_revcd = () => {
-            return _lvs.rows != null && _rvs.rows != null;
-         };
-         _stopwatch.Start();
-         if (_dirty && all_data_revcd()) {
-            _dirty = false;
-            BlueTable rv = null;
-            foreach (IBlueRow lr in _lvs.rows) {
-               foreach (IBlueRow rr in _rvs.rows) {
-                  IBlueRow resr = _join_func(lr, rr);
-                  if (resr == null)
-                     continue;
-                  rv = rv ?? new BlueTable(resr.schema);
-                  rv.addRow(resr);
-               }
-            }
-            _stopwatch.Stop();
-            Log.Gen.info($"ProximityJoinStream left_count: {_lvs.rows.Count} right_count: {_rvs.rows.Count} result_count: {rv.Count} time: {_stopwatch.ElapsedMilliseconds} msecs");
-            var dp = BlueDataUtility.build_dp_inc(rv, myLastValue as DataPacket);
-            myFireEvent(dp);
-         }
-
-      }
-
-   }
-}

src/DataStreamSuite/DataStream/Primitives/Join/Proximity/TimeProximityJoinStream.cs

@@ -1,47 +0,0 @@
-namespace bts.SilverPine {
-   using BCG;
-
-   using System;
-   using System.CodeDom;
-   using System.Configuration;
-   using System.Reactive.Disposables;
-
-   public class TimeProximityJoinStream : ViewStream {
-      private int _id = 0;
-
-      public TimeProximityJoinStream(ViewStream ls, string l_time_col, ViewStream rs, string r_time_col,
-                                       string pre_join_period, string post_join_period) :
-            this(ls, l_time_col, rs, r_time_col, TimeBasedFilter.parse_duration(pre_join_period),
-            TimeBasedFilter.parse_duration(post_join_period)) { }
-
-
-      public TimeProximityJoinStream(ViewStream ls, string l_time_col, ViewStream rs, string r_time_col,
-                                       TimeSpan pre_join_period, TimeSpan post_join_period) {
-         Func<IBlueRow, IBlueRow, IBlueRow> create_row = (lr, rr) => {
-            var id = 0;
-            lock (this) {
-               id = _id++;
-            }
-            IBlueRow rv = BlueDataUtility.create_composite_row(lr, rr, id);
-            return rv;
-         };
-         Func<IBlueRow, IBlueRow, IBlueRow> join_func = (lr, rr) => {
-            DateTime ldt = lr.get_datetime(l_time_col);
-            DateTime rdt = rr.get_datetime(r_time_col);
-            TimeSpan diff = ldt - rdt;
-            if (ldt < rdt) {
-               if (diff.Duration() > pre_join_period) {
-                  return null;
-               }
-            } else {
-               if (diff > post_join_period)
-                  return null;
-            }
-            return create_row(lr, rr);
-         };
-
-         ViewStream rstr = new ProximityJoinStream(ls, rs, join_func);
-         mySetRealSource(rstr);
-      }
-   }
-}

src/DataStreamSuite/DataStream/Primitives/Join/SuperJoinStream.cs

@@ -327,14 +327,14 @@ namespace BCG
 
       protected override void myOnFirstSubscriber()
       {
-         myLeftStream.onChange += myHandleLeftStreamChanges;
-         myRightStream.onChange += myHandleRightStreamChanges;
+         myLeftStream.onChange += new ChangeDelegate(myHandleLeftStreamChanges);
+         myRightStream.onChange += new ChangeDelegate(myHandleRightStreamChanges);
       }
 
       protected override void myOnLastSubscriber()
       {
-         myLeftStream.onChange -= myHandleLeftStreamChanges;
-         myRightStream.onChange -= myHandleRightStreamChanges;
+         myLeftStream.onChange -= new ChangeDelegate(myHandleLeftStreamChanges);
+         myRightStream.onChange -= new ChangeDelegate(myHandleRightStreamChanges);
          myLeftKeyToLeftRows.Clear();
          myRightKeyToRightRows.Clear();
          myLeftIDToResultRows.Clear();

src/DataStreamSuite/DataStream/Primitives/LeftJoin/LeftInternalJoinStream.cs

@@ -23,15 +23,15 @@ namespace BCG
          myRightOperand.master = this;
          //the events are not hooked onto the operands delegate, but rather a level of indirection
          //added in the form on myLeftChangeHandler
-         myLeftStream.onChange += myLeftChangeHandler;
-         myRightStream.onChange += myRightChangeHandler;
+         myLeftStream.onChange += new DataStream.ChangeDelegate (myLeftChangeHandler);
+         myRightStream.onChange += new DataStream.ChangeDelegate (myRightChangeHandler);
       }
        
       protected override void myOnLastSubscriber () 
       {
          myInit ();
-         myLeftStream.onChange -= myLeftChangeHandler;
-         myRightStream.onChange -= myRightChangeHandler;
+         myLeftStream.onChange -= new DataStream.ChangeDelegate (myLeftChangeHandler);
+         myRightStream.onChange -= new DataStream.ChangeDelegate (myRightChangeHandler);
       }
 
       protected override void myInit()

src/DataStreamSuite/DataStream/Primitives/Literal/LiteralStream.cs

@@ -2,13 +2,9 @@ using System;
 using System.Collections;
 using System.Collections.Generic;
 
-using Amazon.Runtime.Internal.Util;
-
 using BCG;
 using BCG.Utilities;
 
-using bts;
-
 namespace BCG {
    public class LiteralStream : ViewStream {
 
@@ -26,34 +22,6 @@ namespace BCG {
          return rv;
       }
 
-      //literal("table1", "x:123, y:1.001, z:abc;x:123, y:1.001, z:abc")
-      public LiteralStream(string table_name, string rows) {
-         Func<string, int, IBlueRow> get_row = (json_row, id) => {
-            Dictionary<string, object> map = new Dictionary<string, object>();
-            string[] toks = json_row.Split(',', StringSplitOptions.TrimEntries);
-            //Logger.assert(col_vals.Length == 2);
-            foreach (string t in toks) {
-               var col_vals = t.Split(':', StringSplitOptions.TrimEntries);
-               string col = col_vals[0];
-               string val = col_vals[1];
-               map.Add(col, val);
-            }
-            map["id"] = id;
-            return new BlueRow(map);
-         };
-         string[] row_arr = rows.Split(';', StringSplitOptions.TrimEntries);
-         BlueTable table = null;
-         foreach (string json_string in row_arr) {
-            IBlueRow r = //BlueDataUtility.convert_json_to_row(json_string, table == null ? 0 : table.Count);
-            get_row(json_string, table == null ? 0 : table.Count);
-            table = table ?? new BlueTable(r.schema);
-            table.addRow(r);
-         }
-         ViewStream str = new BlueTableStream(table);
-         str = new SchemaGuesserStream(str);
-         mySetRealSource(str);
-      }
-
       public LiteralStream(string tableName, string schema, string value) {
          string[] columns = schema.Split(' ');
 
@@ -87,7 +55,7 @@ namespace BCG {
          mySetRealSource(new BlueRowCollectionStaticStream(t));
       }
 
-      // public LiteralStream(string schema, string value) : this("literal", schema, value) { }
+      public LiteralStream(string schema, string value) : this("literal", schema, value) { }
 
       // public LiteralStream(string value) {
 

src/DataStreamSuite/DataStream/Primitives/Map/GlobalMapStream.cs

@@ -1,72 +0,0 @@
-using BCG;
-
-using MongoDB.Driver.Core.WireProtocol.Messages;
-
-using System;
-using System.Collections.Generic;
-using System.Linq;
-
-// the first arg is all the rows, the second arg is the index of the row that needs mapping, and 
-// the func returns the mapped row (ie mapped version of the index row)
-using GlobalMap = System.Func<System.Collections.Generic.List<BCG.IBlueRow>, uint, BCG.IBlueRow>;
-
-namespace bts {
-   // In GlobalMap, each row is mapped with the full view of all rows, ie the map functions for a row
-   // can use the state fo all rows.
-   public class GlobalMapStream : ViewStream {
-      private ViewStream _base_stream;
-      private readonly string _sort_col_name, _group_by_col;
-      private GlobalMap _map_func;
-      public GlobalMapStream(ViewStream base_stream, string sort_col_name, string group_by_col, GlobalMap map_func) {
-         _base_stream = base_stream;
-         _sort_col_name = sort_col_name;
-         _group_by_col = group_by_col;
-         _map_func = map_func;
-      }
-
-      public GlobalMapStream(ViewStream base_stream, string sort_col_name, GlobalMap map_func) :
-      this(base_stream, sort_col_name, null, map_func) {
-      }
-
-      protected override void myOnFirstSubscriber() {
-         _base_stream.onChange += _handle_stream_update;
-      }
-
-      protected override void myOnLastSubscriber() {
-         _base_stream.onChange -= _handle_stream_update;
-      }
-
-      void _handle_stream_update(object dp_obj) {
-         DataPacket dp = dp_obj as DataPacket;
-         IDictionary<int, List<IBlueRow>> groups = !String.IsNullOrEmpty(_group_by_col) ?
-         BlueDataUtility.group_together(dp.newRows, new string[] { _group_by_col })
-         :
-         new Dictionary<int, List<IBlueRow>>() { { 0, dp.newRows.to_list() } };
-         BlueTable out_table = null;
-         foreach (var kv in groups) {
-            List<IBlueRow> group_rows = kv.Value;
-            group_rows.Sort((r1, r2) => BlueDataUtility.compare_rows(r2, r1, _sort_col_name));
-            uint i = 0;
-            foreach (IBlueRow r in group_rows) {
-               IBlueRow nr = _map_func(group_rows, i++);
-               out_table = out_table ?? new BlueTable(nr.schema);
-               out_table.addRow(nr);
-            }
-         }
-
-
-         // List<IBlueRow> rows = dp.newRows.to_ienum().ToList();
-         // rows.Sort((r1, r2) => BlueDataUtility.compare_rows(r1, r2, _sort_col_name));
-         // uint i = 0;
-
-         // BlueTable out_table = null;
-         // foreach (IBlueRow r in rows) {
-         //    IBlueRow nr = _map_func(rows, i++);
-         //    out_table = out_table ?? new BlueTable(nr.schema);
-         //    out_table.addRow(nr);
-         // }
-         DataPacket dp_out = BlueDataUtility.build_dp_inc(out_table, myLastValue as DataPacket);
-         myFireEvent(dp_out);
-      }
-   }
-}

src/DataStreamSuite/DataStream/Primitives/Map/MapStream.cs

@@ -119,13 +119,13 @@ namespace BCG {
 
 
       protected override void myOnFirstSubscriber() {
-         myStream.onChange += myChangeHandler;
+         myStream.onChange += new DataStream.ChangeDelegate(myChangeHandler);
       }
 
       protected override void myOnLastSubscriber() {
          myInit();
          if (myStream != null)
-            myStream.onChange -= myChangeHandler;
+            myStream.onChange -= new DataStream.ChangeDelegate(myChangeHandler);
       }
 
       protected override void myOnTableClear() {

src/DataStreamSuite/DataStream/Primitives/Map/SuperMapStream.cs

@@ -83,11 +83,10 @@ namespace BCG {
          myMapper.onMapperChange += new MapperChangeDelegate(myHandleMapChange);
       }
 
-      protected override void myOnFirstSubscriber() =>
-         myStream.onChange += myHandleData;
+      protected override void myOnFirstSubscriber() { myStream.onChange += new ChangeDelegate(myHandleData); }
 
       protected override void myOnLastSubscriber() {
-         myStream.onChange -= myHandleData;
+         myStream.onChange -= new ChangeDelegate(myHandleData);
          myRows = null;
          myBaseRows = null;
          myMap.Clear();

src/DataStreamSuite/DataStream/Primitives/Misc/DebugStream.cs

@@ -16,10 +16,10 @@ namespace BCG {
       }
 
       protected override void myOnFirstSubscriber() {
-         _stream.onChange += _onstream_change;
+         _stream.onChange += new ChangeDelegate(_onstream_change);
       }
       protected override void myOnLastSubscriber() {
-         _stream.onChange -= _onstream_change;
+         _stream.onChange -= new ChangeDelegate(_onstream_change);
       }
 
       private void _onstream_change(object o) {
@@ -45,4 +45,4 @@ namespace BCG {
 
       }
    }
-}
+}

src/DataStreamSuite/DataStream/Primitives/Misc/TimeSeriesStream.cs

@@ -1,8 +1,9 @@
+using Google.Protobuf.WellKnownTypes;
+
 using System;
+using BCG.Utilities.General;
 
-using BCG;
-
-namespace bts.SilverPine {
+namespace BCG {
    // Keeps adding changing rows to the stream, hence preserving the history
    // and creating a time series, for example this can be used to make bbo updates
    // into a time series of prices/qty 
@@ -16,10 +17,10 @@ namespace bts.SilverPine {
       }
 
       protected override void myOnFirstSubscriber() {
-         _stream.onChange += _onstream_change;
+         _stream.onChange += new ChangeDelegate(_onstream_change);
       }
       protected override void myOnLastSubscriber() {
-         _stream.onChange -= _onstream_change;
+         _stream.onChange -= new ChangeDelegate(_onstream_change);
       }
 
 
@@ -40,6 +41,7 @@ namespace bts.SilverPine {
          }
          _table.addRows(added_rows);
          myFireEvent(new DataPacket(_table, added_rows, new BlueTable(_table.schema)));
+
       }
    }
 

src/DataStreamSuite/DataStream/Primitives/Project/New/NewProjectStream.cs

@@ -137,24 +137,10 @@ namespace bts {
          HashSet<string> removed_cols = new HashSet<string>();
          Dictionary<string, string> mapped_to_orig_map = new Dictionary<string, string>();
          Dictionary<string, string> orig_to_mapped = new Dictionary<string, string>();
-         // by default every col will show up, so if we have the projection as (a:a1)
-         // it will convert a to a1, 
-         // examples: if a schema has cols a b c
-         // project(parent_str, "a:aa ") -> aa b c
-         // project(parent_str, "a:aa !c") -> aa b
-         // project(parent_str, "a:aa !") -> aa
-         // project(parent_str, "a:aa b = (aa*aa)!") -> aa b
-         bool hide_all = false;
          foreach (string p_tok in project_toks) {
             if (p_tok.StartsWith("!")) {
-               if (p_tok.Length == 1) {
-                  // if it is a "blanket "!"=> hide all cols except the one that
-                  // are stated explicitly
-                  hide_all = true;
-               } else {
-                  string removed_col = p_tok.Substring(1);
-                  removed_cols.Add(removed_col);
-               }
+               string removed_col = p_tok.Substring(1);
+               removed_cols.Add(removed_col);
             } else if (p_tok.Contains(":")) {
                string[] map_toks = p_tok.Split(":");
                string orig_col = map_toks[0];
@@ -169,11 +155,12 @@ namespace bts {
                }
             }
          }
-
          // make the new schema
          List<ColumnInfo> new_cols = new List<ColumnInfo>();
          foreach (ColumnInfo ci in input_schema.columnInfo) {
             if (ci.colName == "id") {
+               // if there's a hidden col, other columns will show by default, but if cols are being
+               // projected epxlicitely, than others wil be hidden
                ColumnInfo nc = new ColumnInfo(ci.colName, new_cols.Count, ci.colType);
                new_cols.Add(nc);
                continue;
@@ -185,13 +172,12 @@ namespace bts {
                ColumnInfo nc = new ColumnInfo(mapped_name, new_cols.Count, ci.colType);
                new_cols.Add(nc);
             } else {
-               if (hide_all)
-                  continue;
-               //if (ci.colName == "id" || removed_cols.Count > 0) {
-               // unless columns are  explicitely hidden, all columns show up
-               ColumnInfo nc = new ColumnInfo(ci.colName, new_cols.Count, ci.colType);
-               new_cols.Add(nc);
-               //}
+               if (ci.colName == "id" || removed_cols.Count > 0) {
+                  // if there's a hidden col, other columns will show by default, but if cols are being
+                  // projected epxlicitely, than others wil be hidden
+                  ColumnInfo nc = new ColumnInfo(ci.colName, new_cols.Count, ci.colType);
+                  new_cols.Add(nc);
+               }
             }
          }
          IBlueTableSchema new_schema = new BlueTableSchema(input_schema.tableName, new_cols.ToArray());

src/DataStreamSuite/DataStream/Primitives/Project/ProjectStream.cs

@@ -54,12 +54,12 @@ namespace BCG {
 
       protected override void myOnFirstSubscriber()
       {
-         myVs.onChange += myHandleChange;
+         myVs.onChange += new BCG.DataStream.ChangeDelegate (myHandleChange);
       }
 
       protected override void myOnLastSubscriber()
       {
-         myVs.onChange -= myHandleChange;
+         myVs.onChange -= new BCG.DataStream.ChangeDelegate (myHandleChange);;
       }
 
       internal void myHandleChange (object newValue)

src/DataStreamSuite/DataStream/Primitives/SQL/SQLite/SQLiteEnumerator.cs

@@ -1,54 +0,0 @@
-namespace Bts.DataStream;
-
-using BCG;
-
-using System;
-using System.Collections.Generic;
-using System.Data;
-using System.Data.SQLite;
-using System.Linq;
-using System.Threading;
-
-public class SQLiteEnumerable : IAsyncEnumerable<IBlueRowCollection> {
-   readonly string location;
-   readonly string statement;
-   readonly int pageSize;
-
-   public SQLiteEnumerable(string location, string statement, int pageSize = 65536) {
-      this.location = location;
-      this.statement = statement;
-      this.pageSize = pageSize;
-   }
-
-   public async IAsyncEnumerator<IBlueRowCollection> GetAsyncEnumerator(CancellationToken cancellationToken = default) {
-      var dataTable = new DataTable();
-      await using var conn = new SQLiteConnection($"Data Source={location};Version=3");
-      await conn.OpenAsync(cancellationToken);
-      await using var cmd = new SQLiteCommand(statement, conn);
-      await using var reader = cmd.ExecuteReader();
-      var schema = await reader.GetColumnSchemaAsync(cancellationToken);
-      var columns = schema.OrderBy(c => c.ColumnOrdinal ?? int.MaxValue)
-         .Select((c, i) => new ColumnInfo(c.ColumnName, c.ColumnOrdinal ?? i, c.DataType))
-         .Append(new ColumnInfo("id", schema.Count, typeof(int)))
-         .ToArray();
-      var blueSchema = new BlueTableSchema("anon", columns);
-      BlueTable? table = null;
-      int rowId = 1;
-      while (await reader.ReadAsync(cancellationToken)) {
-         table ??= new BlueTable(blueSchema);
-         var rowVals = new object[blueSchema.columnInfo.Length];
-         if (reader.GetValues(rowVals) != rowVals.Length - 1) {
-            throw new InvalidOperationException("Sqlite reader value count for single row not equal to column count");
-         }
-         rowVals[^1] = rowId++;
-         table.addRow(new BlueRow(blueSchema, rowVals));
-         if (table.Count == pageSize) {
-            yield return table;
-            table = null;
-         }
-      }
-      if (table is not null) {
-         yield return table;
-      }
-   }
-}

src/DataStreamSuite/DataStream/Primitives/SQL/SQLite/SQLiteHelper.cs

@@ -1,122 +0,0 @@
-using System;
-using System.Data;
-using System.Data.SQLite;
-using System.Diagnostics;
-using System.Collections;
-
-
-using BCG;
-using System.Linq;
-using System.Data.Entity;
-using System.Collections.Generic;
-using System.Data.SQLite;
-
-
-namespace bts.SilverPine {
-
-   public static class SQLiteHelper {
-      public static string location = "C:\\Users\\m_tah\\development\\data\\";
-      public static string db_name = "stream_data.db";
-
-      public static DataTable execute_select_query(string connection_string, string query) {
-         // Create a new DataTable to store the results
-         DataTable dataTable = new DataTable();
-
-         // Create and open a SQLite connection using the provided connection string
-         using (SQLiteConnection connection = new SQLiteConnection(connection_string)) {
-            try {
-               connection.Open();
-
-               // Create a SQLite command with the provided query
-               using (SQLiteCommand command = new SQLiteCommand(query, connection)) {
-                  // Use SQLiteDataAdapter to fill the DataTable
-                  using (SQLiteDataAdapter adapter = new SQLiteDataAdapter(command)) {
-                     adapter.Fill(dataTable);
-                  }
-               }
-            } catch (Exception ex) {
-               Console.WriteLine("An error occurred: " + ex.Message);
-            }
-         }
-         // Return the populated DataTable
-         return dataTable;
-      }
-
-      public static List<Dictionary<string, object>> ExecuteBulkRead(string connectionString, string query) {
-         var result = new List<Dictionary<string, object>>();
-
-         using (var connection = new SQLiteConnection(connectionString)) {
-            connection.Open();
-
-            // Begin a transaction to optimize read speed
-            using (var transaction = connection.BeginTransaction())
-            using (var command = new SQLiteCommand(query, connection, transaction))
-            using (var reader = command.ExecuteReader()) {
-               while (reader.Read()) {
-                  var row = new Dictionary<string, object>();
-
-                  for (int i = 0; i < reader.FieldCount; i++) {
-                     string columnName = reader.GetName(i);
-                     object columnValue = reader.IsDBNull(i) ? null : reader.GetValue(i);
-                     row[columnName] = columnValue;
-                  }
-
-                  result.Add(row);
-               }
-
-               // Commit the transaction
-               transaction.Commit();
-            }
-         }
-
-         return result;
-      }
-
-      public static List<Dictionary<string, object>> get_table(string connectionString, string query) {
-         var result = new List<Dictionary<string, object>>();
-
-         using (var connection = new SQLiteConnection(connectionString)) {
-            try {
-               connection.Open();
-
-               using (var command = new SQLiteCommand(query, connection))
-               using (var reader = command.ExecuteReader()) {
-                  while (reader.Read()) {
-                     var row = new Dictionary<string, object>();
-
-                     for (int i = 0; i < reader.FieldCount; i++) {
-                        string columnName = reader.GetName(i);
-                        object columnValue = reader.IsDBNull(i) ? null : reader.GetValue(i);
-                        row[columnName] = columnValue;
-                     }
-
-                     result.Add(row);
-                  }
-               }
-            } catch (Exception ex) {
-               Console.WriteLine("An error occurred: " + ex.Message);
-            }
-         }
-         return result;
-      }
-
-      public static IEnumerable<Dictionary<string, object>> get_table_2(string connectionString, string query) {
-         using (var connection = new SQLiteConnection(connectionString)) {
-            connection.Open();
-            using (var command = new SQLiteCommand(query, connection))
-            using (var reader = command.ExecuteReader()) {
-               while (reader.Read()) {
-                  var row = new Dictionary<string, object>();
-                  for (int i = 0; i < reader.FieldCount; i++) {
-                     string columnName = reader.GetName(i);
-                     object columnValue = reader.IsDBNull(i) ? null : reader.GetValue(i);
-                     row[columnName] = columnValue;
-                  }
-                  yield return row;
-               }
-            }
-         }
-      }
-   }
-}
-

src/DataStreamSuite/DataStream/Primitives/SQL/SQLite/SqliteStream.cs

@@ -1,73 +0,0 @@
-namespace bts.SilverPine {
-   using System;
-   using System.Data;
-   using System.Data.SQLite;
-   using System.Diagnostics;
-   using System.Collections;
-
-
-   using BCG;
-
-   [DataStreamUserRespresentation("SQLite")]
-   public class SQLiteStream : ViewStream {
-      private readonly string _connection_string, _query;
-      private Stopwatch _stopwatch = new Stopwatch();
-
-      private SQLiteStream(string connection_string, string query, bool noop) {
-         _connection_string = connection_string;
-         _query = query;
-      }
-
-      public SQLiteStream(string db_name, string query)
-      :
-      this($"Data Source={SQLiteHelper.location}{db_name};Version=3", query, true) { }
-
-      public SQLiteStream(string query)
-      :
-      this(SQLiteHelper.db_name, query) {
-      }
-
-
-      private void _read_from_sqlite_and_fire() {
-         _stopwatch.Start();
-         DataTable dt = SQLiteHelper.execute_select_query(_connection_string, _query);
-         _stopwatch.Stop();
-         Log.Gen.info($"SQLiteStream.ExecuteSelectQuery time to read db: {_stopwatch.ElapsedMilliseconds} msecs");
-         _stopwatch.Restart();
-         #region perf measures
-         // SQLiteHelper.get_table(_connection_string, _query);
-         // _stopwatch.Stop();
-         // Log.Gen.info($"SQLiteStream.get_table time to read db: {_stopwatch.ElapsedMilliseconds} msecs");
-         // _stopwatch.Restart();
-         // SQLiteHelper.ExecuteBulkRead(_connection_string, _query);
-         // _stopwatch.Stop();
-         // Log.Gen.info($"SQLiteStream.ExecuteBulkRead time to read db: {_stopwatch.ElapsedMilliseconds} msecs");
-         // _stopwatch.Restart();
-         // SQLiteHelper.get_table_2(_connection_string, _query);
-         // _stopwatch.Stop();
-         // Log.Gen.info($"SQLiteStream.get_table_2 time to read db: {_stopwatch.ElapsedMilliseconds} msecs");
-         // _stopwatch.Restart();
-         #endregion
-         IBlueRowCollection rows = BCG.BlueDataUtility.toIBlueRowCollection(dt);
-         //IBlueRowCollection rows = new DataTableWrapperTable(dt);
-         _stopwatch.Stop();
-         Log.Gen.info($"SQLiteStream._read_from_sqlite_and_fire time to load into IBlueRowCollection: {_stopwatch.ElapsedMilliseconds} msecs");
-         DataPacket dp = BlueDataUtility.build_dp_inc(rows, myLastValue as DataPacket);
-         myFireEvent(dp);
-      }
-
-      protected override void myOnFirstSubscriber() {
-         _read_from_sqlite_and_fire();
-      }
-
-      protected override void myOnLastSubscriber() {
-         // this value will be cleared by the base class
-         // but in this particular stream this behaviour
-         // is not usefull since any new subscribers 
-         // would force the query to be recreated
-         //myLastDP = myLastValue;
-      }
-
-   }
-
-}

src/DataStreamSuite/DataStream/Primitives/SideEffectStreams/CSVSaverStream.cs

@@ -3,15 +3,12 @@ using System.Collections.Generic;
 using System.Text;
 using System.IO;
 using BCG.Utilities;
-using System.Diagnostics;
-using Microsoft.Data.Analysis;
-using Parquet;
-using Parquet.Data;
 
-
-namespace BCG {
+namespace BCG
+{
    [DataStreamUserRespresentation("CSVSaver")]
-   public class CSVSaverStream : ViewStream, ISideEffectStream {
+   public class CSVSaverStream : ViewStream, ISideEffectStream
+   {
       private IBlueTableSchema myResultSchema = new BlueTableSchema("CSVResult", new ColumnInfo("id", 0, typeof(int)),
          new ColumnInfo("fileName", 1, typeof(string)),
          new ColumnInfo("nWrites", 2, typeof(int)),
@@ -23,9 +20,11 @@ namespace BCG {
       private int myNWrites = 0, myNMaxWrites = Int32.MaxValue;
 
       public CSVSaverStream(ViewStream vs, string fullyQualifiedFileName)
-         : this(vs, fullyQualifiedFileName, true, Int32.MaxValue, false) { }
+         : this(vs, fullyQualifiedFileName, true, Int32.MaxValue, false)
+      { }
 
-      public CSVSaverStream(ViewStream vs, string fullyQualifiedFileName, bool overwrite, int nMaxWrites, bool fireResultSnapshot) {
+      public CSVSaverStream(ViewStream vs, string fullyQualifiedFileName, bool overwrite, int nMaxWrites, bool fireResultSnapshot)
+      {
          myStream = vs;
          myFullyQualifiedFileName = fullyQualifiedFileName;
          myOverwrite = overwrite;
@@ -33,20 +32,22 @@ namespace BCG {
          myFireResultSnapshot = fireResultSnapshot;
       }
 
-      protected override void myOnFirstSubscriber() =>
-         myStream.onChange += myHandleChanges;
+      protected override void myOnFirstSubscriber() { myStream.onChange += new ChangeDelegate(myHandleChanges); }
 
-      protected override void myOnLastSubscriber() =>
-         myStream.onChange -= myHandleChanges;
+      protected override void myOnLastSubscriber() { myStream.onChange -= new ChangeDelegate(myHandleChanges); }
 
-      private void myHandleChanges(object newValue) {
+      private void myHandleChanges(object newValue)
+      {
          IBlueRowCollection totalState = (newValue as DataPacket).newRows;
-         if (totalState.Count > 0 && myNWrites < myNMaxWrites) {
+         if (totalState.Count > 0 && myNWrites < myNMaxWrites)
+         {
             myNWrites++;
             int i = 0;
             StreamWriter writer = new StreamWriter(new FileStream(myGetNewFileName(), FileMode.Create, FileAccess.Write));
-            foreach (IBlueRow r in totalState) {
-               if (i++ == 0) {
+            foreach (IBlueRow r in totalState)
+            {
+               if (i++ == 0)
+               {
                   string header = String.Empty;
                   foreach (string col in r.schema.columnNames)
                      if (col != "id")
@@ -58,17 +59,20 @@ namespace BCG {
                writer.Flush();
             }
             writer.Close();
-            if (myFireResultSnapshot) {
+            if (myFireResultSnapshot)
+            {
                IBlueRow nr = new BlueRow(myResultSchema, 0, myFullyQualifiedFileName, myNWrites, totalState.Count);
                BlueTable nRows = new BlueTable(myResultSchema);
                nRows.addRow(nr);
                myFireEvent(new DataPacket(nRows, nRows, new BlueTable(myLastValue == null ? new BlueTable(myResultSchema) : (myLastValue as DataPacket).newRows)));
-            } else
+            }
+            else
                myFireEvent(newValue);
          }
       }
 
-      private string myGetNewFileName() {
+      private string myGetNewFileName()
+      {
          string[] tags = myFullyQualifiedFileName.Split('.');
          if (myOverwrite)
             return tags[0] + ".csv";
@@ -76,11 +80,14 @@ namespace BCG {
             return tags[0] + "_" + myFormateDateTime(DateTime.Now) + ".csv";
       }
 
-      private static string myGetCSVRow(IBlueRow r) {
+      private static string myGetCSVRow(IBlueRow r)
+      {
          string rv = String.Empty;
          int i = 0;
-         foreach (string col in r.schema.columnNames) {
-            if (col != "id") {
+         foreach (string col in r.schema.columnNames)
+         {
+            if (col != "id")
+            {
                string colText = String.Empty;
                if (r[col] != null && r[col] != DBNull.Value)
                   colText = r[col].ToString();
@@ -92,7 +99,8 @@ namespace BCG {
          return rv;
       }
 
-      private string myFormateDateTime(DateTime dt) {
+      private string myFormateDateTime(DateTime dt)
+      {
          return
          string.Format("{0}-{1}-{2}-{3}-{4}-{5}-{6}",
                         dt.Year.ToString("0000"),
@@ -106,49 +114,52 @@ namespace BCG {
    }
 
    [DataStreamUserRespresentation("CSVReader")]
-   public class CSVReaderStream : ViewStream {
-      private readonly string _file_address;
-      Stopwatch _stopwatch = new Stopwatch();
-      public CSVReaderStream(string fileAddress) {
-         _file_address = fileAddress;
+   public class CSVReaderStream : ViewStream
+   {
+      private IBlueRowCollection myRows;
+
+      public CSVReaderStream(string fileAddress)
+      {
+         myRows = BlueDataUtility.createRowsFromCSVFileWithoutTypes(fileAddress);
       }
 
-      protected override void myOnFirstSubscriber() {
-         _stopwatch.Start();
-         DataFrame df = DataFrame.LoadCsv(_file_address);
-         _stopwatch.Stop();
-         bts.Log.Gen.info($"CSVReaderStream.myOnFirstSubscriber stream csv load col_count: {df.Columns.Count} rows, time: {_stopwatch.ElapsedMilliseconds} msecs");
-         _stopwatch.Start();
-         IBlueRowCollection rows = BlueDataUtility.createRowsFromCSVFileWithoutTypes(_file_address);
-         _stopwatch.Stop();
-         bts.Log.Gen.info($"CSVReaderStream.myOnFirstSubscriber stream csv load count: {rows.Count} rows, time: {_stopwatch.ElapsedMilliseconds} msecs");
-         DataPacket dp_out = BlueDataUtility.build_dp_inc(rows, myLastValue as DataPacket);
-         myFireEvent(dp_out);
+      protected override void myOnFirstSubscriber()
+      {
+         myFireEvent(new DataPacket(myRows, myRows, new BlueTable(myRows.schema)));
       }
    }
 
-   [DataStreamUserRespresentation("CSVReaderStreamMapper")]
-   public class CSVReaderStreamMapperStream : ViewStream {
-      public CSVReaderStreamMapperStream(ViewStream stream, string output_schema) {
+   [DataStreamUserRespresentation ("CSVReaderStreamMapper")]
+   public class CSVReaderStreamMapperStream : ViewStream
+   {
+      public CSVReaderStreamMapperStream(ViewStream stream, string output_schema)
+      {
          mySetRealSource(new MapStream(stream, new CSVReaderStreamMapper(output_schema)));
       }
    }
 
-   public class CSVReaderStreamMapper : IStreamMapper {
+   public class CSVReaderStreamMapper : IStreamMapper
+   {
       private IBlueTableSchema mySchema;
       private readonly string[] myOutputSchemaStringArray;
 
-      public CSVReaderStreamMapper(string outputSchema) {
+      public CSVReaderStreamMapper(string outputSchema)
+      {
          myOutputSchemaStringArray = outputSchema.Split(' ');
       }
 
-      public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema) {
-         if (mySchema == null) {
+      public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema)
+      {
+         if (mySchema == null)
+         {
             ColumnInfo[] ci = inputSchema.columnInfo;
-            foreach (string col in myOutputSchemaStringArray) {
+            foreach (string col in myOutputSchemaStringArray)
+            {
                string colName = col.Substring(0, col.IndexOf('\\')).ToLower();
-               for (int i = 0; i < ci.Length; i++) {
-                  if (ci[i].colName.ToLower().Equals(colName)) {
+               for (int i = 0; i < ci.Length; i++)
+               {
+                  if (ci[i].colName.ToLower().Equals(colName))
+                  {
                      Type t = myGetType(col.Substring(col.IndexOf('\\') + 1));
                      ci[i] = new ColumnInfo(ci[i].colName, i, t);
                      break;
@@ -160,52 +171,74 @@ namespace BCG {
          return mySchema;
       }
 
-      public IBlueRow map(IBlueRow row) {
+      public IBlueRow map(IBlueRow row)
+      {
          object[] vals = new object[row.schema.columnInfo.Length];
          row.Values.CopyTo(vals, 0);
-         for (int i = 0; i < mySchema.columnInfo.Length; i++) {
+         for (int i = 0; i < mySchema.columnInfo.Length; i++)
+         {
             Type t = mySchema.columnInfo[i].colType;
-            try {
+            try
+            {
                vals[i] = System.Convert.ChangeType(vals[i], mySchema.columnInfo[i].colType, System.Globalization.CultureInfo.InvariantCulture);
-            } catch {
-               if (t.Equals(typeof(Int16))) {
+            }
+            catch
+            {
+               if (t.Equals(typeof(Int16)))
+               {
                   Int16 result;
-                  if (Int16.TryParse((string)vals[i], out result)) {
+                  if (Int16.TryParse((string)vals[i], out result))
+                  {
                      vals[i] = result;
                      continue;
                   }
                   double r2;
-                  if (Double.TryParse((string)vals[i], out r2)) {
-                     try {
+                  if (Double.TryParse((string)vals[i], out r2))
+                  {
+                     try
+                     {
                         vals[i] = (Int16)r2;
                         continue;
-                     } catch { }
+                     }
+                     catch { }
                   }
-               } else if (t.Equals(typeof(Int32))) {
+               }
+               else if (t.Equals(typeof(Int32)))
+               {
                   Int32 result;
-                  if (Int32.TryParse((string)vals[i], out result)) {
+                  if (Int32.TryParse((string)vals[i], out result))
+                  {
                      vals[i] = result;
                      continue;
                   }
                   double r2;
-                  if (Double.TryParse((string)vals[i], out r2)) {
-                     try {
+                  if (Double.TryParse((string)vals[i], out r2))
+                  {
+                     try
+                     {
                         vals[i] = (Int32)r2;
                         continue;
-                     } catch { }
+                     }
+                     catch { }
                   }
-               } else if (t.Equals(typeof(Int64))) {
+               }
+               else if (t.Equals(typeof(Int64)))
+               {
                   Int64 result;
-                  if (Int64.TryParse((string)vals[i], out result)) {
+                  if (Int64.TryParse((string)vals[i], out result))
+                  {
                      vals[i] = result;
                      continue;
                   }
                   double r2;
-                  if (Double.TryParse((string)vals[i], out r2)) {
-                     try {
+                  if (Double.TryParse((string)vals[i], out r2))
+                  {
+                     try
+                     {
                         vals[i] = (Int64)r2;
                         continue;
-                     } catch { }
+                     }
+                     catch { }
                   }
                }
                vals[i] = DBNull.Value;
@@ -214,7 +247,8 @@ namespace BCG {
          return new BlueRow(mySchema, vals);
       }
 
-      private Type myGetType(string type) {
+      private Type myGetType(string type)
+      {
          type = type.ToLower();
          if (type.Contains("int"))
             return typeof(int);