Reversehedging fixed

Previously, it was black and hard to read since the background is dark.

.config/dotnet-tools.json

@@ -9,7 +9,7 @@
       ]
     },
     "fable": {
-      "version": "4.0.6",
+      "version": "4.24.0",
       "commands": [
         "fable"
       ]
@@ -27,4 +27,4 @@
       ]
     }
   }
-}
+}

.devcontainer/Dockerfile

@@ -87,6 +87,15 @@ RUN dpkg -i /emumba/emumba-deribit-connector-3.3.1-Linux.LICENSE_DISABLED.deb \
 # Pcap
 RUN apt-get update && apt-get install -y libpcap0.8-dev
 
+# Databento (deps)
+RUN apt-get update && apt-get -y install libzstd-dev
+
+# Docker in Docker
+RUN apt-get update && apt-get install -y \
+    docker.io \
+    docker-compose \
+    docker-compose-v2
+
 # Set up default executables (update-alternatives)
 RUN \
    update-alternatives --install /usr/bin/clang clang /usr/bin/clang-$clang_major 100; \

.devcontainer/devcontainer.json

@@ -1,15 +1,10 @@
 {
   "dockerFile": "Dockerfile",
-  // Creates local dir to mount to HOME in docker 
-  "initializeCommand": "mkdir -p ${localEnv:HOME}/.bts/devcontainer-mounts/${devcontainerId}",
   "hostRequirements": {
-    "memory": "4gb",
-    "storage": "6gb"
+    "memory": "8gb",
+    "storage": "10gb"
   },
-  "postCreateCommand": "/home/bts/spark/docker/sparkd/dev/devcontainer_post_create.sh",
   "containerUser": "bts",
-  "workspaceMount": "source=${localWorkspaceFolder},target=/home/bts/spark,type=bind",
-  "workspaceFolder": "/home/bts/spark",
   "updateRemoteUserUID": true,
   "shutdownAction": "none",
   // Note: If running multiple containers on the same host, each must use a
@@ -17,34 +12,20 @@
   // example, if 8000-8003 is taken, go with 9000-9003, then 10000-10003, etc.
   "runArgs": [
     "--shm-size",
-    "2gb",
-    "-p",
-    "8000:8000",
-    "-p",
-    "8001:8001",
-    "-p",
-    "8002:8002",
-    "-p",
-    "8003:8003",
-    "-p",
-    "12345:12345",
+    "8gb"
   ],
   "mounts": [
-    "source=${localEnv:HOME}/.bts/devcontainer-mounts/${devcontainerId},target=/home/bts,type=bind"
+    "source=/var/run/docker.sock,target=/var/run/docker.sock,type=bind"
   ],
   "customizations": {
     "vscode": {
-      "settings": {
-        "clangd.arguments": [
-          "--compile-commands-dir=${workspaceFolder}/build/clang/debug"
-        ]
-      },
       "extensions": [
         "eamodio.gitlens",
         "llvm-vs-code-extensions.vscode-clangd",
         "ms-vscode.cmake-tools",
-        "ms-vscode.cpptools"
+        "ms-vscode.cpptools",
+        "xaver.clang-format"
       ]
     }
   }
-}
+}

.env

@@ -0,0 +1,48 @@
+# This file defines the global default configuration for docker deployments.
+# It is tracked in source control. For temporary modifications, please create
+# a separate file called ".env.local" (it's in .gitignore, so won't be tracked).
+# If a variable is defined in both files, the value in .env.local will be used.
+#
+# About the deployment procedure:
+#   We use Docker Compose to build images and launch containers for various services
+#   that make up our software suite:
+#     - sparkd
+#     - TODO: pubsub
+#     - TODO: md-publisher
+#     - TODO: webapp (DSQA on the web)
+#     - TODO: api-server (websocket based)
+#   From inside a devcontainer, you can launch all services in separate containers
+#   on your machine by running:
+#
+#     tools/deploy.sh
+#
+#   This script builds release versions for each service, packaged (generally) in
+#   self-contained directories. Next, it runs a few "docker compose" commands to
+#   build docker images loaded with the self-contained binaries from the previous
+#   step, and start containers with those images to run each service. To view the
+#   commands that deploy.sh runs internally (without actually running them), run:
+#
+#     tools/deploy.sh --dry-run
+#
+#   For additional information/help, run:
+#
+#     tools/deploy.sh --help
+#
+
+### Sparkd ###
+# The internal image for sparkd to use. For more info, view:
+#   src/server/sparkd/Dockerfile
+SPARKD__IMAGE=sparkd
+# The config dir for this service to use
+SPARKD__CONFIG_DIR=src/server/sparkd/configurations/dev
+# Mount a specific volume or dir to "~/sparkdata/data" inside the docker.
+# Defaults to a volume named "sparkd_<config_port>" (e.g., sparkd_8000).
+# SPARKD__DATA_DIR=/home/bts/sparkdata/data
+SPARKD__SOURCE_DIR=install/sparkd-release-gcc
+SPARKD__RUNTIME_CONF_DIR=/home/ubuntu/8000/sparkdata/conf/
+SPARKD__CONFIG_PORT=8000
+SPARKD__QUICKBUS_PORT=8001
+SPARKD__SIM_FIX_PORT=8002
+SPARKD__SHM_SIZE=4gb
+
+# TODO: add config for other services (pubsub, md-publisher, webapp, api-server, etc.)

.gitattributes

@@ -1,3 +1,3 @@
-*.cmd eol=crlf
-*.cs eol=crlf
-*.sh eol=lf
+*.cmd text=auto eol=crlf
+*.cs text=auto eol=crlf
+*.sh text=auto eol=lf

.gitignore

@@ -59,3 +59,4 @@ trace_data/
 .devcontainer/
 *.Generated.cs
 *.lst
+.env.local

.gitmodules

@@ -1,6 +1,6 @@
 [submodule "externals"]
 	path = externals
-	url = git@bitbucket.org:bluetradesys/externals.git
+	url = https://bitbucket.org/bluetradesys/externals.git
 [submodule "vcpkg"]
 	path = vcpkg
 	url = https://github.com/microsoft/vcpkg.git

.vscode/launch.json

@@ -26,11 +26,55 @@
       "stopAtEntry": false,
       "console": "integratedTerminal"
     },
+    {
+      "name": "(gdb) Launch",
+      "type": "cppdbg",
+      "request": "launch",
+      "program": "/home/bts/spark/build/sparkd-debug-clang/server/sparkd",
+      "args": [
+        "run"
+      ],
+      "stopAtEntry": false,
+      "cwd": "/home/bts/spark/build/sparkd-debug-clang/server",
+      "environment": [],
+      "externalConsole": false,
+      "MIMode": "gdb",
+    },
+    {
+      "name": "spreader tests2",
+      "type": "cppdbg",
+      "request": "launch",
+      "program": "/home/bts/spark/build/sparkd-debug-clang/server/sparkd",
+      "args": [
+        "test",
+        "[spreader2]"
+      ],
+      "stopAtEntry": false,
+      "cwd": "/home/bts/spark/build/sparkd-debug-clang/server",
+      "environment": [],
+      "externalConsole": false,
+      "MIMode": "gdb",
+    },
+    {
+      "name": "spreader tests",
+      "type": "cppdbg",
+      "request": "launch",
+      "program": "/home/bts/spark/build/sparkd-debug-clang/server/sparkd",
+      "args": [
+        "test",
+        "[spreader]"
+      ],
+      "stopAtEntry": false,
+      "cwd": "/home/bts/spark/build/sparkd-debug-clang/server",
+      "environment": [],
+      "externalConsole": false,
+      "MIMode": "gdb",
+    },
     {
       "name": "DataStreamQueryAnalyzer",
       "type": "coreclr",
       "request": "launch",
-      "preLaunchTask": "Build DataStream (Debug)",
+      //"preLaunchTask": "Build DataStreamQueryAnalyzer (Debug)",
       "program": "",
       "windows": {
         "program": "DataStreamQueryAnalyzer.exe",
@@ -79,6 +123,18 @@
       "stopAtEntry": false,
       "console": "integratedTerminal"
     },
+    {
+      "name": "Spark (Skip Build)",
+      "type": "coreclr",
+      "request": "launch",
+      "program": "",
+      "windows": {
+        "program": "Spark.exe",
+        "cwd": "build/Debug/bin/net7.0-windows",
+      },
+      "stopAtEntry": false,
+      "console": "integratedTerminal"
+    },
     {
       "name": "Spark (Parameterized)",
       "type": "coreclr",
@@ -97,21 +153,138 @@
       "console": "integratedTerminal"
     },
     {
-      "name": "Sparkd",
+      "name": "Spark Test",
       "type": "coreclr",
       "request": "launch",
-      "preLaunchTask": "Build Sparkd (Debug | Clang)",
       "program": "",
-      "linux": {
-        "program": "sparkd",
+      "windows": {
+        "preLaunchTask": "Build Spark (Debug)",
+        "program": "Spark.Console.exe",
+        "cwd": "build/Debug/bin/net7.0",
+        "args": [
+          "test",
+          "${input:sparkTest}"
+        ]
       },
-      "args": [
-        "run"
-      ],
-      "cwd": "build/sparkd-debug-clang/server",
       "stopAtEntry": false,
       "console": "integratedTerminal"
-    }
+    },
+    {
+      "name": "Spark Test (Skip Build)",
+      "type": "coreclr",
+      "request": "launch",
+      "program": "",
+      "windows": {
+        "program": "Spark.Console.exe",
+        "cwd": "build/Debug/bin/net7.0",
+        "args": [
+          "test",
+          "${input:sparkTest}"
+        ]
+      },
+      "stopAtEntry": false,
+      "console": "integratedTerminal"
+    },
+    {
+      "name": "Sparkd",
+      "type": "cppdbg",
+      "request": "launch",
+      "program": "",
+      "linux": {
+        "preLaunchTask": "Build Sparkd (Debug)",
+        "program": "${workspaceFolder}/build/sparkd-debug-clang/server/sparkd",
+        "args": [
+          "run"
+        ],
+        "stopAtEntry": false,
+        "cwd": "${workspaceFolder}/build/sparkd-debug-clang/server",
+        "environment": [],
+        "externalConsole": false,
+        "MIMode": "gdb",
+        "setupCommands": [
+          {
+            "description": "Enable pretty-printing for gdb",
+            "text": "-enable-pretty-printing",
+            "ignoreFailures": true
+          }
+        ]
+      }
+    },
+    {
+      "name": "Sparkd (Skip Build)",
+      "type": "cppdbg",
+      "request": "launch",
+      "program": "",
+      "linux": {
+        "program": "${workspaceFolder}/build/sparkd-debug-clang/server/sparkd",
+        "args": [
+          "run"
+        ],
+        "stopAtEntry": false,
+        "cwd": "${workspaceFolder}/build/sparkd-debug-clang/server",
+        "environment": [],
+        "externalConsole": false,
+        "MIMode": "gdb",
+        "setupCommands": [
+          {
+            "description": "Enable pretty-printing for gdb",
+            "text": "-enable-pretty-printing",
+            "ignoreFailures": true
+          }
+        ]
+      }
+    },
+    {
+      "name": "Sparkd Test",
+      "type": "cppdbg",
+      "request": "launch",
+      "program": "",
+      "linux": {
+        "preLaunchTask": "Build Sparkd (Debug)",
+        "program": "${workspaceFolder}/build/sparkd-debug-clang/server/sparkd",
+        "args": [
+          "test",
+          "${input:sparkdTest}",
+        ],
+        "stopAtEntry": false,
+        "cwd": "${workspaceFolder}/build/sparkd-debug-clang/server",
+        "environment": [],
+        "externalConsole": false,
+        "MIMode": "gdb",
+        "setupCommands": [
+          {
+            "description": "Enable pretty-printing for gdb",
+            "text": "-enable-pretty-printing",
+            "ignoreFailures": true
+          }
+        ]
+      }
+    },
+    {
+      "name": "Sparkd Test (Skip Build)",
+      "type": "cppdbg",
+      "request": "launch",
+      "program": "",
+      "linux": {
+        "program": "${workspaceFolder}/build/sparkd-debug-clang/server/sparkd",
+        "args": [
+          "test",
+          "${input:sparkdTest}",
+        ],
+        "stopAtEntry": false,
+        "cwd": "${workspaceFolder}/build/sparkd-debug-clang/server",
+        "environment": [],
+        "externalConsole": false,
+        "MIMode": "gdb",
+        "setupCommands": [
+          {
+            "description": "Enable pretty-printing for gdb",
+            "text": "-enable-pretty-printing",
+            "ignoreFailures": true
+          }
+        ]
+      }
+    },
   ],
   "inputs": [
     {
@@ -142,5 +315,15 @@
       "description": "Config Server URL",
       "default": "http://localhost:8000/config",
     },
+    {
+      "id": "sparkTest",
+      "type": "promptString",
+      "description": "Test to run",
+    },
+    {
+      "id": "sparkdTest",
+      "type": "promptString",
+      "description": "Test to run",
+    },
   ]
-}
+}

.vscode/tasks.json

@@ -526,6 +526,28 @@
         "color": "terminal.ansiGreen",
       },
     },
+    {
+      "label": "Run PubSubServer (Debug | Skip Build)",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "PubSubServer",
+      },
+      "windows": {
+        "command": "PubSubServer.exe",
+      },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
+      ],
+      "options": {
+        "cwd": "${workspaceFolder}/build/Debug/bin/net7.0"
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      },
+    },
     {
       "label": "Run PubSubServer (Release)",
       "dependsOn": "Build DataStream (Release)",
@@ -549,6 +571,28 @@
         "color": "terminal.ansiGreen",
       },
     },
+    {
+      "label": "Run PubSubServer (Release | Skip Build)",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "PubSubServer",
+      },
+      "windows": {
+        "command": "PubSubServer.exe",
+      },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
+      ],
+      "options": {
+        "cwd": "${workspaceFolder}/build/Release/bin/net7.0"
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      },
+    },
     {
       "label": "Run Spark",
       "dependsOn": "Build Spark",
@@ -597,6 +641,21 @@
         "color": "terminal.ansiGreen",
       }
     },
+    {
+      "label": "Run Spark (Skip Build)",
+      "type": "process",
+      "command": "",
+      "windows": {
+        "command": "Spark.exe",
+        "options": {
+          "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0-windows"
+        },
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      }
+    },
     {
       "label": "Run Spark Config Query",
       "dependsOn": "Build Spark",
@@ -640,6 +699,24 @@
         "color": "terminal.ansiGreen",
       },
     },
+    {
+      "label": "Run Sparkd (Skip Build)",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "sparkd",
+      },
+      "args": [
+        "run"
+      ],
+      "options": {
+        "cwd": "${workspaceFolder}/build/${input:sparkdCmakePreset}/server"
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      },
+    },
     {
       "label": "Run Sparkd Config Query",
       "dependsOn": "Build Sparkd",
@@ -661,6 +738,65 @@
         "color": "terminal.ansiGreen",
       },
     },
+    {
+      "label": "Test Spark (Debug)",
+      "dependsOn": "Build Spark (Debug)",
+      "type": "process",
+      "icon": {
+        "id": "coverage",
+        "color": "terminal.ansiYellow",
+      },
+      "command": "",
+      "windows": {
+        "command": "Spark.Console.exe",
+        "args": [
+          "test",
+          "${input:sparkTest}"
+        ],
+        "options": {
+          "cwd": "${workspaceFolder}/build/Debug/bin/net7.0",
+        }
+      },
+    },
+    {
+      "label": "Test Sparkd",
+      "dependsOn": "Build Sparkd",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "sparkd",
+      },
+      "args": [
+        "test",
+        "${input:sparkdTest}",
+      ],
+      "options": {
+        "cwd": "${workspaceFolder}/build/${input:sparkdCmakePreset}/server"
+      },
+      "icon": {
+        "id": "coverage",
+        "color": "terminal.ansiYellow",
+      },
+    },
+    {
+      "label": "Test Sparkd (Skip Build)",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "sparkd",
+      },
+      "args": [
+        "test",
+        "${input:sparkdTest}",
+      ],
+      "options": {
+        "cwd": "${workspaceFolder}/build/${input:sparkdCmakePreset}/server"
+      },
+      "icon": {
+        "id": "coverage",
+        "color": "terminal.ansiYellow",
+      },
+    },
     {
       "label": "View DataStreamQueryAnalyzer Logs (Last Run)",
       "type": "process",
@@ -684,6 +820,36 @@
         "id": "wrench"
       },
     },
+    {
+      "label": "View PubSubServer Logs (Last Run)",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "tools/logging/show_last_run.sh",
+        "args": [
+          "${env:HOME}/.local/share/DataStream/data/logging.PubSubServer",
+          "Starting PubSubServer"
+        ]
+      },
+      "windows": {
+        "command": "tools/logging/show_last_run.cmd",
+        "args": [
+          "DataStream",
+          "PubSubServer.exe",
+          "Starting PubSubServer"
+        ]
+      },
+      "presentation": {
+        "reveal": "always",
+        "focus": true,
+        "panel": "shared",
+        "showReuseMessage": false,
+        "close": true,
+      },
+      "icon": {
+        "id": "wrench"
+      },
+    },
   ],
   "inputs": [
     {
@@ -749,6 +915,16 @@
       "id": "sparkdConfigQuery",
       "type": "promptString",
       "description": "Config Type(s)"
-    }
+    },
+    {
+      "id": "sparkTest",
+      "type": "promptString",
+      "description": "Test to run",
+    },
+    {
+      "id": "sparkdTest",
+      "type": "promptString",
+      "description": "Test to run",
+    },
   ]
 }

CMakeLists.txt

@@ -1,7 +1,10 @@
 cmake_minimum_required(VERSION 3.12)
 project(spark)
+include(GNUInstallDirs)
+include(${CMAKE_SOURCE_DIR}/cmake/GitVersion.cmake)
 set(CMAKE_CXX_STANDARD 20)
 
+get_version_from_git()
 if(WIN32)
   set(platform "windows")
 else()
@@ -49,8 +52,6 @@ else()
   # make sparkd look for libspark.so in its directory
   set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE)
   set(CMAKE_INSTALL_RPATH ".")
-  set(CMAKE_INSTALL_PREFIX "${CMAKE_SOURCE_DIR}/install")
-  set(SPARK_INSTALL_PREFIX "${CMAKE_INSTALL_PREFIX}/$<CONFIG>")
 
   if(BUILD_SPARKD)
     # Note: All macros that begin w/ 'CCAPI' are ccapi's macros (defined and/or
@@ -89,9 +90,15 @@ else()
     add_compile_definitions(BINANCE_US_PRICE_STATS_PATH="/api/v3/ticker/24hr")
 
     # ** Binance USDS Futures **
-    add_compile_definitions(CCAPI_BINANCE_USDS_FUTURES_URL_REST_BASE="https://dapi.binance.com")
-    add_compile_definitions(BINANCE_USDS_FUTURES_INSTRUMENTS_PATH="/dapi/v1/exchangeInfo")
-    add_compile_definitions(BINANCE_USDS_FUTURES_PRICE_STATS_PATH="/dapi/v1/ticker/24hr")
+    add_compile_definitions(CCAPI_BINANCE_USDS_FUTURES_URL_REST_BASE="https://fapi.binance.com")
+    add_compile_definitions(BINANCE_USDS_FUTURES_INSTRUMENTS_PATH="/fapi/v1/exchangeInfo")
+    add_compile_definitions(BINANCE_USDS_FUTURES_PRICE_STATS_PATH="/fapi/v1/ticker/24hr")
+    add_compile_definitions(BINANCE_USDS_FUTURES_USER_DATA_PATH="/api/v3/userDataStream")
+
+    # ** Binance Coin Futures **
+    add_compile_definitions(CCAPI_BINANCE_COIN_FUTURES_URL_REST_BASE="https://dapi.binance.com")
+    add_compile_definitions(BINANCE_COIN_FUTURES_INSTRUMENTS_PATH="/dapi/v1/exchangeInfo")
+    add_compile_definitions(BINANCE_COIN_FUTURES_PRICE_STATS_PATH="/dapi/v1/ticker/24hr")
 
     # ** Deribit **
     add_compile_definitions(DERIBIT_INSTRUMENT_PATH="/api/v2/public/get_instrument")
@@ -111,6 +118,7 @@ else()
 
   # subdirs
   if (BUILD_SPARKD)
+    add_subdirectory(src/lib_binance_execution)
     add_subdirectory(src/server/sparkd)
     add_subdirectory(src/data_files)
   endif()
@@ -132,6 +140,8 @@ else()
       COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/src/server/sparkd/sparkd ${CMAKE_BINARY_DIR}/server
       COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/data_files ${CMAKE_BINARY_DIR}/server
       COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/src/lib/libspark.so ${CMAKE_BINARY_DIR}/server
+      COMMAND ln ARGS -sf ${EXTERNALS_ROOT}/lib/${CMAKE_BUILD_TYPE}/libonixs-fix-engine.so.4 ${CMAKE_BINARY_DIR}/server
+      
       COMMAND
         ln ARGS -sf
         ${EXTERNALS_ROOT}/lib/Release/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.16.0.0
@@ -145,11 +155,14 @@ else()
 
     install(
       DIRECTORY "${CMAKE_BINARY_DIR}/server/plugins"
-      DESTINATION "${SPARK_INSTALL_PREFIX}/bin")
+      DESTINATION "${CMAKE_INSTALL_BINDIR}")
     install(
       FILES
       "${SPARK_LIB}/libACE-5.7.5.so"
       "${SPARK_LIB}/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.16.0.0"
-      DESTINATION "${SPARK_INSTALL_PREFIX}/bin")
+      "${SPARK_LIB}/libonixs-fix-engine.so.4.10.1"
+      "${SPARK_LIB}/libonixs-fix-engine.so.4"
+      "${SPARK_LIB}/libonixs-fix-engine.so"
+      DESTINATION "${CMAKE_INSTALL_BINDIR}")
   endif()
 endif()

CMakePresets.json

@@ -26,6 +26,7 @@
         "CMAKE_CXX_FLAGS_DEBUG": "$env{SPARK_CMAKE_CXX_FLAGS} /MTd",
         "CMAKE_CXX_FLAGS_RELEASE": "$env{SPARK_CMAKE_CXX_FLAGS} /MT",
         "CMAKE_CXX_STANDARD": "20",
+        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
         "CMAKE_SHARED_LINKER_FLAGS_DEBUG": "/DEBUG",
         "CMAKE_SHARED_LINKER_FLAGS_RELEASE": "/OPT:REF /OPT:ICF /LTCG",
         "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
@@ -61,7 +62,7 @@
         },
         "CMAKE_CXX_STANDARD": "20",
         "CMAKE_INSTALL_RPATH": ".",
-        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}/sparkd",
+        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
         "CMAKE_EXE_LINKER_FLAGS": "-Wno-undef",
         "EXTERNALS_ROOT": "${sourceDir}/externals/linux",
         "BUILD_SDK": {
@@ -302,16 +303,7 @@
         "lhs": "${hostSystemName}",
         "rhs": "Linux"
       },
-      "hidden": true,
-      "targets": [
-        "data_files_dir",
-        "admin_web_app_bundle",
-        "install",
-        "libspark",
-        "rapidjson_project",
-        "server_layout",
-        "sparkd"
-      ]
+      "hidden": true
     },
 
     {

Spark.Api.Server.sln

@@ -14,6 +14,34 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "spark_test", "src\client\sp
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Spark.Console", "src\client\Spark.Console\Spark.Console.csproj", "{89D666E6-B500-498F-B026-4D167F68ADFC}"
 EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{267F81EF-A8AB-49FD-9F56-D7EC3C1F7C7C}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "DataStreamSuite", "DataStreamSuite", "{C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamCore", "src\DataStreamSuite\DataStreamCore\DataStreamCore.csproj", "{EF26A475-6101-4C9C-A717-D60BB92A2302}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SharedStreams", "src\DataStreamSuite\SharedStreams\SharedStreams.csproj", "{911B08A3-515A-441D-84C8-8A43740B5FC1}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PubSub", "src\DataStreamSuite\PubSub\PubSub.csproj", "{14140310-2233-430A-821B-F3058A25234B}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BaseUtilities", "src\DataStreamSuite\BaseUtilities\BaseUtilities.csproj", "{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ExpressionCore", "src\DataStreamSuite\ExpressionCore\ExpressionCore.csproj", "{6F22DB96-6EAB-4034-99D9-287021C36F01}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ArithmeticExpressions", "src\DataStreamSuite\ArithmeticExpressions\ArithmeticExpressions.csproj", "{A5C33956-A41F-49B3-A41D-1082CBA59D02}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamExpressions", "src\DataStreamSuite\DataStreamExpressions\DataStreamExpressions.csproj", "{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnSortExpressions", "src\DataStreamSuite\Bts.ColumnSortExpressions\Bts.ColumnSortExpressions.csproj", "{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ColumnMapExpressions", "src\DataStreamSuite\ColumnMapExpressions\ColumnMapExpressions.csproj", "{B945A183-2174-4ACB-B289-8A5C546656FE}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BooleanExpressions", "src\DataStreamSuite\BooleanExpressions\BooleanExpressions.csproj", "{9F6D7DE8-185B-4C1D-B809-3314D092DF28}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnExpressions", "src\DataStreamSuite\Bts.ColumnExpressions\Bts.ColumnExpressions.csproj", "{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStream", "src\DataStreamSuite\DataStream\DataStream.csproj", "{39AEA075-6457-46C0-8821-1158BE6EFA57}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|x64 = Debug|x64
@@ -70,6 +98,102 @@ Global
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.Release|x64.Build.0 = Release|x64
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|x64.ActiveCfg = Debug|x64
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|x64.Build.0 = Debug|x64
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Debug|x64.Build.0 = Debug|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Release|x64.ActiveCfg = Release|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Release|x64.Build.0 = Release|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Debug|x64.Build.0 = Debug|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Release|x64.ActiveCfg = Release|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Release|x64.Build.0 = Release|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.Debug|x64.Build.0 = Debug|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.Release|x64.ActiveCfg = Release|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.Release|x64.Build.0 = Release|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Debug|x64.Build.0 = Debug|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Release|x64.ActiveCfg = Release|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Release|x64.Build.0 = Release|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Debug|x64.Build.0 = Debug|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Release|x64.ActiveCfg = Release|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Release|x64.Build.0 = Release|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Debug|x64.Build.0 = Debug|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Release|x64.ActiveCfg = Release|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Release|x64.Build.0 = Release|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Debug|x64.Build.0 = Debug|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Release|x64.ActiveCfg = Release|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Release|x64.Build.0 = Release|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Debug|x64.Build.0 = Debug|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Release|x64.ActiveCfg = Release|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Release|x64.Build.0 = Release|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.Debug|x64.Build.0 = Debug|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.Release|x64.ActiveCfg = Release|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.Release|x64.Build.0 = Release|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Debug|x64.Build.0 = Debug|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Release|x64.ActiveCfg = Release|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Release|x64.Build.0 = Release|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Debug|x64.Build.0 = Debug|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Release|x64.ActiveCfg = Release|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Release|x64.Build.0 = Release|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Debug|x64.Build.0 = Debug|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Release|x64.ActiveCfg = Release|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Release|x64.Build.0 = Release|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -80,4 +204,19 @@ Global
 	GlobalSection(Performance) = preSolution
 		HasPerformanceSessions = true
 	EndGlobalSection
+	GlobalSection(NestedProjects) = preSolution
+		{C09ED521-D2DF-40BC-A76F-EB3CB870B2F6} = {267F81EF-A8AB-49FD-9F56-D7EC3C1F7C7C}
+		{EF26A475-6101-4C9C-A717-D60BB92A2302} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{911B08A3-515A-441D-84C8-8A43740B5FC1} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{14140310-2233-430A-821B-F3058A25234B} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{6F22DB96-6EAB-4034-99D9-287021C36F01} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{B945A183-2174-4ACB-B289-8A5C546656FE} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{39AEA075-6457-46C0-8821-1158BE6EFA57} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+	EndGlobalSection
 EndGlobal

cmake/GitVersion.cmake

@@ -0,0 +1,48 @@
+include_guard(GLOBAL)
+
+function(get_version_from_git)
+  find_package(Git QUIET)
+  if(NOT Git_FOUND)
+    message(WARNING "Git not found")
+    return()
+  endif()
+
+  execute_process(
+    COMMAND ${GIT_EXECUTABLE} describe --tags --always
+    WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
+    OUTPUT_VARIABLE GIT_TAG
+    OUTPUT_STRIP_TRAILING_WHITESPACE
+    RESULT_VARIABLE GIT_RESULT
+  )
+
+  if(NOT GIT_RESULT EQUAL 0)
+    message(WARNING "Failed to get git tag")
+    return()
+  endif()
+
+  execute_process(
+    COMMAND ${GIT_EXECUTABLE} rev-parse --short=7 HEAD
+    WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
+    OUTPUT_VARIABLE GIT_COMMIT_SHORT_HASH
+    OUTPUT_STRIP_TRAILING_WHITESPACE
+  )
+  set(PROJECT_VERSION_BUILD_METADATA ${GIT_COMMIT_SHORT_HASH})
+  set(PROJECT_VERSION_BUILD_METADATA ${GIT_COMMIT_SHORT_HASH} PARENT_SCOPE)
+
+  string(REGEX REPLACE "^v" "" CLEAN_TAG "${GIT_TAG}")
+  if(CLEAN_TAG MATCHES "^([0-9]+)\\.([0-9]+)\\.([0-9]+)(-.*)?$")
+    set(PROJECT_VERSION_MAJOR ${CMAKE_MATCH_1})
+    set(PROJECT_VERSION_MAJOR ${CMAKE_MATCH_1} PARENT_SCOPE)
+    set(PROJECT_VERSION_MINOR ${CMAKE_MATCH_2})
+    set(PROJECT_VERSION_MINOR ${CMAKE_MATCH_2} PARENT_SCOPE)
+    set(PROJECT_VERSION_PATCH ${CMAKE_MATCH_3})
+    set(PROJECT_VERSION_PATCH ${CMAKE_MATCH_3} PARENT_SCOPE)
+
+    set(PROJECT_VERSION "${CMAKE_MATCH_1}.${CMAKE_MATCH_2}.${CMAKE_MATCH_3}") 
+    set(PROJECT_VERSION "${CMAKE_MATCH_1}.${CMAKE_MATCH_2}.${CMAKE_MATCH_3}" PARENT_SCOPE)
+    set(PROJECT_VERSION_FULL "${CMAKE_MATCH_1}.${CMAKE_MATCH_2}.${CMAKE_MATCH_3}+${GIT_COMMIT_SHORT_HASH}")
+    set(PROJECT_VERSION_FULL "${CMAKE_MATCH_1}.${CMAKE_MATCH_2}.${CMAKE_MATCH_3}+${GIT_COMMIT_SHORT_HASH}" PARENT_SCOPE)
+  else()
+    message(WARNING "Tag '${CLEAN_TAG}' does not match semver format")
+  endif()
+endfunction()

cmake/ProtobufGenerateFs.cmake

@@ -0,0 +1,49 @@
+include_guard(DIRECTORY)
+
+function(protobuf_generate_fs)
+  set(_options)
+  set(_one_value_args TARGET_NAME OUT_DIR PROTOC)
+  set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+
+  if (NOT DEFINED _ARGS_TARGET_NAME)
+    set(_ARGS_TARGET_NAME "${PROJECT_NAME}_generate_fs_protos")
+  endif()
+  if (NOT DEFINED _ARGS_PROTOC)
+    set(_ARGS_PROTOC "${SPARK_BIN}/protoc")
+  endif()
+
+  set(_proto_include_cli_args)
+  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
+    list(APPEND _proto_include_cli_args -I ${_dir})
+  endforeach()
+
+  set(_outputs)
+  foreach(_proto_source ${_ARGS_PROTO_SOURCES})
+    # get_filename_component(_abs_proto_source ${_proto_source} ABSOLUTE)
+    get_filename_component(_proto_source_we ${_proto_source} NAME_WE)
+    # get_filename_component(_proto_source_dir ${_proto_source} DIRECTORY)
+    list(APPEND _outputs ${_ARGS_OUT_DIR}/${_proto_source_we}.fable.fs)
+  endforeach()
+
+  set(_venv_exec ${CMAKE_SOURCE_DIR}/tools/venv_exec.sh)
+  set(_plugin_py ${CMAKE_SOURCE_DIR}/src/lib/api/service_compiler/plugin.py)
+  set(_template_fs ${CMAKE_SOURCE_DIR}/src/lib/api/service_compiler/templates/fs/fable.fs)
+  add_custom_command(
+    OUTPUT ${_outputs}
+    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+    COMMAND ${_venv_exec}
+      ${_ARGS_PROTOC}
+      --plugin=protoc-gen-service=${_plugin_py}
+      --service_out=lang=fs,strip_out_path=1:${_ARGS_OUT_DIR}
+      ${_proto_include_cli_args}
+      ${_ARGS_PROTO_SOURCES}
+    DEPENDS
+      ${_venv_exec}
+      ${_plugin_py}
+      ${_template_fs}
+      ${CMAKE_CURRENT_SOURCE_DIR}/package-lock.json
+      ${_ARGS_PROTO_SOURCES}
+  )
+  add_custom_target(${_ARGS_TARGET_NAME} ALL DEPENDS "${_outputs}")
+endfunction()

cmake/ProtobufGenerateJs.cmake

@@ -0,0 +1,33 @@
+include_guard(DIRECTORY)
+
+function(protobuf_generate_js)
+  set(_options)
+  set(_one_value_args TARGET_NAME OUTPUT)
+  set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+
+  if (NOT DEFINED _ARGS_TARGET_NAME)
+    set(_ARGS_TARGET_NAME "${PROJECT_NAME}_generate_js_protos")
+  endif()
+
+  set(_proto_include_cli_args)
+  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
+    list(APPEND _proto_include_cli_args -p ${_dir})
+  endforeach()
+
+  add_custom_command(
+    OUTPUT ${_ARGS_OUTPUT}
+    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+    COMMAND npx
+      --no-install pbjs
+      -t json-module
+      -w commonjs
+      -o ${_ARGS_OUTPUT}
+      ${_proto_include_cli_args}
+      ${_ARGS_PROTO_SOURCES}
+    DEPENDS
+      ${CMAKE_CURRENT_SOURCE_DIR}/package-lock.json
+      ${_ARGS_PROTO_SOURCES}
+  )
+  add_custom_target(${_ARGS_TARGET_NAME} ALL DEPENDS ${_ARGS_OUTPUT})
+endfunction()

cmake/SymLink.cmake

@@ -0,0 +1,49 @@
+include_guard(DIRECTORY)
+include(${CMAKE_SOURCE_DIR}/cmake/TernaryOperator.cmake)
+
+function(sym_link_dir)
+  set(_options)
+  set(_one_value_args TARGET_NAME SOURCE DEST COMMENT)
+  set(_multi_value_args)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+  add_custom_command(
+    OUTPUT ${_ARGS_DEST}
+    COMMAND ln ARGS -sf ${_ARGS_SOURCE} ${_ARGS_DEST}
+  )
+  set_ternary(_comment _ARGS_COMMENT "${_ARGS_COMMENT}" "linking dir ${_ARGS_SOURCE} -> ${_ARGS_DEST}")
+  add_custom_target(
+    ${_ARGS_TARGET_NAME} ALL
+    DEPENDS ${_ARGS_DEST}
+    COMMENT "${_comment}"
+  )
+endfunction()
+
+function(sym_link_file)
+  set(_options)
+  set(_one_value_args SOURCE DEST INSTALL_DEST)
+  set(_multi_value_args)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+  file(
+    CREATE_LINK
+      ${_ARGS_SOURCE}
+      ${_ARGS_DEST}
+    SYMBOLIC
+  )
+  if (DEFINED _ARGS_INSTALL_DEST)
+    install(
+      FILES
+      ${_ARGS_SOURCE}
+      DESTINATION ${_ARGS_INSTALL_DEST}
+    )
+  endif()
+endfunction()
+
+function(sym_link_local_lib LIB_NAME)
+  set(_lib_file_name "${CMAKE_SHARED_LIBRARY_PREFIX}${LIB_NAME}.so")
+  file(
+    CREATE_LINK
+      "${CMAKE_BINARY_DIR}/src/lib_${LIB_NAME}/${_lib_file_name}"
+      "${CMAKE_CURRENT_BINARY_DIR}/${_lib_file_name}"
+    SYMBOLIC
+  )
+endfunction()

cmake/TernaryOperator.cmake

@@ -0,0 +1,9 @@
+include_guard(GLOBAL)
+
+macro(set_ternary RESULT CONDITION TRUE_VALUE FALSE_VALUE)
+  if(${CONDITION})
+    set(${RESULT} ${TRUE_VALUE})
+  else()
+    set(${RESULT} ${FALSE_VALUE})
+  endif()
+endmacro()

cmake/version.h.in

@@ -0,0 +1,11 @@
+#ifndef BTS_SPARK_VERSION_H
+#define BTS_SPARK_VERSION_H
+
+#define BTS_SPARK_VERSION_MAJOR @PROJECT_VERSION_MAJOR@
+#define BTS_SPARK_VERSION_MINOR @PROJECT_VERSION_MINOR@
+#define BTS_SPARK_VERSION_PATCH @PROJECT_VERSION_PATCH@
+#define BTS_SPARK_VERSION_BUILD_METADATA @PROJECT_VERSION_BUILD_METADATA@
+#define BTS_SPARK_VERSION "@PROJECT_VERSION@"
+#define BTS_SPARK_VERSION_FULL "@PROJECT_VERSION_FULL@"
+
+#endif

compose.yaml

@@ -0,0 +1,100 @@
+services:
+  sparkd:
+    volumes:
+      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/sparkdata/data
+    build:
+      dockerfile: src/server/sparkd/Dockerfile
+      args:
+        - IMAGE=${SPARKD__IMAGE}
+        - SOURCE_DIR=${SPARKD__SOURCE_DIR}
+        - CONFIG_DIR=${SPARKD__CONFIG_DIR}
+    shm_size: "${SPARKD__SHM_SIZE:-512mb}"
+    command: >
+      sh -c "
+        echo bts | sudo -S chown -R bts:bts /home/bts/sparkdata/data
+        if [ -e "/home/bts/sparkdata/data/logging.sparkd/log" ]; then
+          mv "/home/bts/sparkdata/data/logging.sparkd/log" "/home/bts/sparkdata/data/logging.sparkd/log_$(date '+%Y%m%d-%H%M')"
+        fi
+        if [ -e "/home/bts/sparkdata/conf/.env.local" ]; then
+          echo 'Sourcing /home/bts/sparkdata/conf/.env.local'
+          set -a
+          . /home/bts/sparkdata/conf/.env.local
+          set +a
+        fi
+        ./sparkd run"
+    env_file:
+      - .env
+      - ${SPARKD__CONFIG_DIR}/.env
+    ports:
+      - "${SPARKD__CONFIG_PORT}:${SPARKD__CONFIG_PORT}"
+      - "${SPARKD__QUICKBUS_PORT}:${SPARKD__QUICKBUS_PORT}"
+      - "${SPARKD__SIM_FIX_PORT}:${SPARKD__SIM_FIX_PORT}"
+    deploy:
+      resources:
+        limits:
+          memory: "${SPARKD__MEMORY:-4gb}"
+  sparkd-symbols:
+    volumes:
+      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/sparkdata/data
+      - ${SPARKD__RUNTIME_CONF_DIR:-sparkd_conf}:/home/bts/sparkdata/conf
+    build:
+      dockerfile: src/server/sparkd/Dockerfile
+      args:
+        - IMAGE=${SPARKD__IMAGE}
+        - SOURCE_DIR=${SPARKD__SOURCE_DIR}
+        - CONFIG_DIR=${SPARKD__CONFIG_DIR}
+    shm_size: "${SPARKD__SHM_SIZE:-512mb}"
+    command: >
+      sh -c "
+        echo bts | sudo -S chown -R bts:bts /home/bts/sparkdata/data
+        if [ -e "/home/bts/sparkdata/data/logging.sparkd/log" ]; then
+          mv "/home/bts/sparkdata/data/logging.sparkd/log" "/home/bts/sparkdata/data/logging.sparkd/log_$(date '+%Y%m%d-%H%M')"
+        fi
+        ls -alh /home/bts/sparkdata/conf/
+        cat /home/bts/sparkdata/conf/.env.local
+        if [ -e "/home/bts/sparkdata/conf/.env.local" ]; then
+          echo 'Sourcing /home/bts/sparkdata/conf/.env.local'
+          set -a
+          . /home/bts/sparkdata/conf/.env.local
+          set +a
+        fi
+        ./sparkd run"
+    env_file:
+      - .env
+      - ${SPARKD__CONFIG_DIR}/.env
+    ports:
+      - "${SPARKD__CONFIG_PORT}:${SPARKD__CONFIG_PORT}"
+      - "${SPARKD__QUICKBUS_PORT}:${SPARKD__QUICKBUS_PORT}"
+      - "${SPARKD__SIM_FIX_PORT}:${SPARKD__SIM_FIX_PORT}"
+    deploy:
+      resources:
+        limits:
+          memory: "${SPARKD__MEMORY:-4gb}"
+
+  sparkd-tail:
+    image: bluetradingsystems/linux_dotnet7
+    volumes:
+      - type: volume
+        read_only: true
+        source: sparkd_data
+        target: /home/bts/sparkdata/data/logging.sparkd
+        volume:
+          subpath: logging.sparkd
+    command: ["tail", "-f", "/home/bts/sparkdata/data/logging.sparkd/log"]
+
+  sparkd-ping-test:
+    depends_on:
+      - sparkd
+    profiles:
+      - ping-test
+    build:
+      context: .
+      dockerfile_inline: |
+        FROM bluetradingsystems/linux_dotnet7
+        USER "root"
+        RUN apt-get update && apt-get -y install iputils-ping
+    command: [ "ping", "sparkd" ]
+
+volumes:
+  sparkd_data:
+    name: "sparkd_${SPARKD__CONFIG_PORT:-8000}"

docker/images/bluetradingsystems/linux_dotnet7/Dockerfile

@@ -0,0 +1,38 @@
+FROM ubuntu:22.04
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update && apt-get install -y pkg-config
+RUN apt-get update && apt-get install -y locales
+RUN apt-get update && apt-get install -y ca-certificates
+RUN apt-get update && apt-get install -y libssh2-1-dev
+RUN apt-get update && apt-get install -y software-properties-common
+RUN apt-get update && apt-get install -y sudo
+
+# GCC
+ARG gcc_major=13
+RUN add-apt-repository ppa:ubuntu-toolchain-r/test \
+  && apt-get update \
+  && apt-get install -y g++-$gcc_major
+
+RUN apt-get update \
+  && apt-get install -y dotnet7
+
+RUN locale-gen en_US.UTF-8
+
+ARG docker_user="bts"
+ARG docker_uid="1000"
+ARG docker_pass="bts"
+
+RUN \
+  groupadd -g "$docker_uid" "$docker_user" \
+  && useradd \
+  -m \
+  -u "$docker_uid" \
+  -g "$docker_uid" \
+  -G sudo \
+  --password "$(openssl passwd -1 "$docker_pass")" \
+  "$docker_user"
+
+USER bts
+WORKDIR /home/bts

requirements.txt

@@ -1,11 +1,20 @@
 astroid==2.7.3
 autopep8==1.5.7
+blinker==1.9.0
+boto3==1.37.9
+botocore==1.37.9
 certifi==2023.5.7
 charset-normalizer==3.1.0
+click==8.1.8
 colorama==0.4.4
+Flask==3.1.0
 idna==3.4
 isort==5.9.3
+itsdangerous==2.2.0
+Jinja2==3.1.6
+jmespath==1.0.1
 lazy-object-proxy==1.6.0
+MarkupSafe==3.0.2
 mccabe==0.6.1
 platformdirs==2.3.0
 protobuf==4.21.12
@@ -14,9 +23,14 @@ pycodestyle==2.7.0
 pycparser==2.20
 pylint==2.10.2
 PyMeta3==0.5.1
+python-dateutil==2.9.0.post0
+python-dotenv==1.0.1
 requests==2.31.0
+s3transfer==0.11.4
+six==1.17.0
 termcolor==1.1.0
 toml==0.10.2
 urllib3==2.0.3
+Werkzeug==3.1.3
 wrapt==1.12.1
 yapf==0.31.0

src/DataStreamSuite/DataStream/DataStream.csproj

@@ -18,8 +18,11 @@
 
   <ItemGroup>
     <PackageReference Include="HtmlAgilityPack" Version="1.11.57" />
+    <PackageReference Include="Microsoft.Data.Analysis" Version="0.21.1" />
     <PackageReference Include="MongoDB.Bson" Version="2.23.1" />
     <PackageReference Include="MongoDB.Driver" Version="2.23.1" />
+    <PackageReference Include="Parquet.Net" Version="5.0.1" />
+    <PackageReference Include="System.Data.SQLite" Version="1.0.118" />
   </ItemGroup>
 
 </Project>

src/DataStreamSuite/DataStream/Misc/DSLStream.cs

@@ -0,0 +1,32 @@
+namespace bts.SilverPine {
+   using System;
+   using BCG;
+   using System.Collections.Generic;
+   using Bts.DataStreamExpressions;
+
+   // All this does is take in a dsl expression, parse it and fire the data that come out of running it.
+   // So for example, if an expresison is passed 
+   [DataStreamUserRespresentation("dsl")]
+   public class DSLStream : ViewStream {
+      private readonly string _dsl;
+      public DSLStream(string dsl) {
+         Log.Gen.info($"DSLStream.cons dsl: {dsl}");
+         _dsl = dsl;
+         ViewStream rs = build_stream(dsl);
+         rs = rs ?? new LiteralStream("error", $"msg:invalid dsl \"{dsl}\"");
+         mySetRealSource(rs);
+      }
+
+      static ViewStream? build_stream(string stream_expression) {
+         try {
+            var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression);
+            var rv = exp.getView(new Dictionary<string, StatementExpression>(0));
+            return rv;
+
+         } catch (Exception ex) {
+            Log.Gen.error($"Could not build stream: {stream_expression} ex: {ex}");
+            return null;
+         }
+      }
+   }
+}

src/DataStreamSuite/DataStream/Primitives/Distinct/DistinctRowStream.cs

@@ -1,23 +1,24 @@
+using bts;
+
 using System;
 using System.Collections;
 
-namespace BCG
-{
+namespace BCG {
 	/// <summary>
 	/// Provides a stream that with distinct rows with respect to the columns
 	/// the user specifies in the argument
 	/// </summary>
-	[DataStreamUserRespresentation ("distinct")]
-	public class DistinctRowStream : ViewStream
-	{
-		public DistinctRowStream (ViewStream stream, params string[] cols)
-		{
-         ArrayList groupByCols = new ArrayList ();
-         foreach (string col in cols)
-            foreach (string c in col.Split (' '))
-               groupByCols.Add (c);
-         mySetRealSource (new GroupByStream (stream, 
-            new DistinctRowEvaluator (groupByCols.ToArray (typeof(string)) as string[]), false));
+	[DataStreamUserRespresentation("distinct")]
+	public class DistinctRowStream : ViewStream {
+		public DistinctRowStream(ViewStream stream, params string[] cols) {
+			var g_by_eval = new DistinctRowEvaluator(cols);
+			ViewStream r = new GroupByStream(stream, g_by_eval);
+			var proj_exp = String.Empty;
+			foreach (var c in cols)
+				proj_exp += $"{c} ";
+			proj_exp = proj_exp.Trim();
+			r = new NewProjectStream(r, proj_exp);
+			mySetRealSource(r);
 		}
 	}
 }

src/DataStreamSuite/DataStream/Primitives/FileStreams/DumpLoaderStream.cs

@@ -1,49 +0,0 @@
-// using System;
-// using System.Collections.Generic;
-// using System.Text;
-// using System.IO;
-
-// using System.Runtime.Serialization.Formatters.Binary;
-
-// namespace BCG
-// {
-//    public class DumpLoaderStream : ViewStream, ISideEffectStream
-//    {
-//       //wtf
-//       //public class Exception : System.Exception
-//       //{public Exception(string msg) : base(msg) { }}
-
-//       public DumpLoaderStream(string fileName, int secondsBetweenPolls)
-//       {
-//          mySetRealSource(
-//             new PollStream(() => new LoaderStream(fileName).rows, secondsBetweenPolls, 
-//                ViewStreamDatabase.instance));
-//       }
-
-//       public DumpLoaderStream(string fileName)
-//       {
-//         if (fileName == null)
-//            throw new LoaderStream.Exception("File name cannot be null");
-
-//         for (int i = 0; i < 5; i++)
-//            try
-//            {
-//               using (FileStream fs = new FileStream(fileName, FileMode.Open, FileAccess.Read))
-//               {
-//                  IClientDeserializer deser = new ClientUniversalDeserializer();
-//                  byte[] the_ops;
-//                  Array the_rows;
-//                  deser.deserialize(new Stream[] { fs }, out the_ops, out the_rows);
-
-//                  mySetRealSource(new BlueRowCollectionStaticStream((IBlueRowCollection)the_rows.GetValue(0)));
-//               }
-//               break;
-//            }
-//            catch (IOException)
-//            {
-//               System.Threading.Thread.Sleep(50);
-//            }
-
-//       }
-//    }
-// }

src/DataStreamSuite/DataStream/Primitives/FileStreams/LoaderStream.cs

@@ -1,6 +1,7 @@
 using BCG;
 
 namespace bts {
+   // This is the other side of the SaverStream, its "loads" the stream and fires once
    [DataStreamUserRespresentation("loader")]
    public class LoaderStream : ViewStream {
 

src/DataStreamSuite/DataStream/Primitives/FileStreams/PersistorStream.cs

@@ -5,10 +5,11 @@ using System.Text;
 using System.IO;
 using Google.Protobuf;
 using System.Globalization;
-using bts;
+using BCG;
 
-namespace BCG {
-   // Writes all the udpates to a stream in a file.
+namespace bts {
+   // Writes all the udpates to a stream in a file or multiples files dependong on the rollover size (as opposed to Saver
+   // which just writes one update, as a create)
    // File naming:
    // token_epoch_yyyyMMdd_HHmmss
    // File location: directory(passed)\token\
@@ -94,10 +95,10 @@ namespace BCG {
       void _on_change(object new_value) {
          DataPacket dp = new_value as DataPacket;
          byte[] serialized_update = null;
-         bool _write_create = _serializer == null || _n_updates_in_current_file == 0;
+         bool first_write = _serializer == null || _n_updates_in_current_file == 0;
          _serializer = _serializer ?? new UniversalStreamIOSerializer();
-         if (_write_create) {
-            // new files always starts with create
+         if (first_write) {
+            // first file always starts with create
             serialized_update = _serializer.serialize(new bts.StreamIOMessage(new bts.StreamIOMessage.CreateMessage(dp.newRows)));
          } else
             serialized_update = _serializer.serialize(new bts.StreamIOMessage(new bts.StreamIOMessage.MultipleRemoveAddMessage(dp.deletedRows, dp.addedRows)));
@@ -105,8 +106,8 @@ namespace BCG {
          long utc_epoch = epoch_now();
 
          //time, length of data, data
-         bw.Write(utc_epoch);
-         bw.Write(serialized_update.Length);
+         //bw.Write(utc_epoch);
+         //bw.Write(serialized_update.Length);
          bw.Write(serialized_update);
          _out_fs.Flush();
          myFireEvent(dp);

src/DataStreamSuite/DataStream/Primitives/FileStreams/PlaybackStream.cs

@@ -1,190 +0,0 @@
-// using System;
-// using System.Collections.Generic;
-// using System.Linq;
-// using System.Text;
-// using System.IO;
-// using System.Runtime.Serialization.Formatters.Binary;
-// using BCG.Utilities.General;
-// using bts.collections;
-// using System.Reflection.Metadata;
-
-
-// namespace BCG {
-
-//    public class PlaybackStream : ViewStream, IWorker {
-//       private readonly PlaybackType _type;
-//       private BlueTable _current_table;
-//       private IClientDeserializer _deserilializer = new ClientUniversalDeserializer();
-//       // these are the packets already downloaded from the current file
-//       private List<PlaybackPacket> _packets = new List<PlaybackPacket>();
-//       //private IEnumerable<FileStream> _file_streams = new List<FileStream>();
-//       private IEnumerable<string> _file_names;
-//       private Tuple<int, FileStream> _current_fs = new Tuple<int, FileStream>(-1, null);
-//       private DateTime _playback_t0, _playback_t1;
-//       private bool _all_files_read = false;
-//       SnapshotData _snapshot_data = null; // only initialized when in snapshot mode
-
-//       public PlaybackStream(string datetime_start, string datetime_end, string directory_name, string token) {
-//          _type = PlaybackType.Streaming;
-//          _playback_t0 = PersistorStream.parse_datetime(datetime_start);
-//          _playback_t1 = PersistorStream.parse_datetime(datetime_end);
-//          string token_directory_name = $"{directory_name}\\{token}";
-//          bts.Log.Gen.info($"PlaybackStream t0: {_playback_t0} t1: {_playback_t1} dir: {token_directory_name}");
-//          List<string> files_in_range = new List<string>();
-//          if (Directory.Exists(token_directory_name)) {
-//             _file_names =
-//          Directory.GetFiles(token_directory_name).Where(f => {
-//             DateTime file_time = PersistorStream.parse_datetime_from_filename(f);
-//             return file_time > _playback_t0 && file_time < _playback_t1;
-//          });
-//          }
-//       }
-
-//       protected override void myOnFirstSubscriber() {
-//          FileStream fs = new FileStream(_snapshot_data.snapshot_file, FileMode.Open, FileAccess.Read);
-//          _snapshot_data.packets = StreamPersistorUtils.load_packets_from_file(fs);
-//          PlaybackPacket snapshot_packet = StreamPersistorUtils.find_packet_at_time(_snapshot_data.packets, _snapshot_data.snapshot_time);
-//          if (snapshot_packet != null) {
-//             foreach (PlaybackPacket p in _snapshot_data.packets) {
-//                byte[] ops;
-//                Array row_array;
-//                _deserilializer.deserialize(new Stream[] { new MemoryStream(p.data) }, out ops, out row_array);
-//                for (int i = 0; i < ops.Length; i++) {
-//                   IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
-//                   byte op = ops[i];
-//                   if (op == StreamOpCode.create) {
-//                      IBlueRowCollection full_table = rows_args[0];
-//                      _current_table = new BlueTable(full_table);
-//                   } else if (op == StreamOpCode.multipleRemoveAdd) {
-//                      //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
-//                      IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
-//                      IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
-//                      BlueDataUtility.absorbChanges(_current_table, dels, adds);
-//                   } else {
-//                      bts.Log.assert(false, $"Oprator: {op} is not expected");
-//                   }
-//                   if (p.idx == snapshot_packet.idx) {
-//                      myFireEvent(new DataPacket(_current_table, _current_table, new BlueTable(_current_table.schema)));
-//                      break;
-//                   }
-//                }
-//             }
-//          }
-//       }
-
-
-//       private void _load_packets_from_current_file() {
-//          int curr_idx = _current_fs.Item1;
-//          FileStream curr_fs = _current_fs.Item2;
-//          _packets = new List<PlaybackPacket>();
-//          _packets.AddRange(StreamPersistorUtils.load_packets_from_file(curr_fs));
-//       }
-
-//       private bool current_file_all_replayed {
-//          get {
-//             bool rv = true;
-//             _packets.ForEach(p => rv = rv && p.replayed);
-//             return rv;
-//          }
-//       }
-
-//       protected override void myOnLastSubscriber() {
-//          var fs = _current_fs.Item2;
-//          if (fs != null) {
-//             fs.Close();
-//          }
-//          ViewStreamDatabase.instance.removeWorker(this);
-//       }
-
-//       // public event PlaybackStreamProgress onProgress;
-//       // public event Action onFileExhausted;
-//       private PlaybackPacket _locate_packet_to_replay() {
-//          foreach (var p in _packets) {
-//             // returns the first non replayed packet
-//             if (!p.replayed)
-//                return p;
-//          }
-//          //else
-//          {
-//             double secondsElapsed = new BinaryReader(myFileReader).ReadDouble();
-//             myNextPacketTime = myPlaybackStart + new TimeSpan(0, 0, 0, 0, (int)(1000 * secondsElapsed / myPlaybackSpeed));
-//             if (onProgress != null)
-//                onProgress(myFileReader.Position, myFileReader.Length);
-//          }
-//       }
-
-//       #region IWorker Members
-
-//       public void doWork() {
-//          if (_all_files_read)
-//             return;
-//          if (current_file_all_replayed) {
-//             _load_next_file();
-//          }
-//          PlaybackPacket curr_packet = _locate_packet_to_replay();
-//          if (curr_packet.idx == 0) { // first packet in the file, fire it rightaway 
-//             byte[] ops;
-//             Array row_array;
-//             // what a stupid interface!
-//             _deserilializer.deserialize(new Stream[] { new MemoryStream(curr_packet.data) }, out ops, out row_array);
-//             _fire_packet(ops, row_array);
-//             curr_packet.replay_timestamp = PersistorStream.epoch_now();
-//          } else {
-//             PlaybackPacket prev_packet = _get_previous_packet(curr_packet);
-//             // this is the time we need to fire (replay) this packet
-//             long time_to_fire = prev_packet.replay_timestamp + curr_packet.offset_from_previous;
-//             long now_epoch = PersistorStream.epoch_now();
-//             // fire when we are past the 
-//             if (now_epoch >= time_to_fire) {
-//                byte[] ops;
-//                Array row_array;
-//                _deserilializer.deserialize(new Stream[] { new MemoryStream(curr_packet.data) }, out ops, out row_array);
-//                _fire_packet(ops, row_array);
-//                curr_packet.replay_timestamp = PersistorStream.epoch_now();
-//             }
-//          }
-//       }
-
-//       private void _fire_packet(byte[] ops, Array row_array) {
-//          for (int i = 0; i < ops.Length; i++) {
-//             IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
-//             byte op = ops[i];
-//             if (op == StreamOpCode.create) {
-//                IBlueRowCollection full_table = rows_args[0];
-//                _current_table = new BlueTable(full_table);
-//                //_current_table.addRows(full_table);
-//                myFireEvent(new DataPacket(full_table, full_table, myLastValue == null ? new BlueTable(full_table.schema) :
-//                                           (myLastValue as DataPacket).newRows));
-//             } else if (op == StreamOpCode.multipleRemoveAdd) {
-//                //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
-//                IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
-//                IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
-//                BlueDataUtility.absorbChanges(_current_table, dels, adds);
-//                myFireEvent(new DataPacket(_current_table, adds, dels));
-//             } else {
-//                bts.Log.assert(false, $"Oprator: {op} is not expected");
-//             }
-
-//          }
-//       }
-
-//       private void _load_next_file() {
-//          var new_idx = _current_fs.Item1 + 1;
-//          if (new_idx >= _file_names.Count()) {
-//             _all_files_read = true;
-//             bts.Log.Gen.info("All files read");
-//             return;
-//          }
-//          var prev_fs = _current_fs.Item2;
-//          if (prev_fs != null)
-//             prev_fs.Close();
-//          var new_filename = _file_names.ElementAt(new_idx);
-//          bts.Log.Gen.info("PlaybackStream_load_next_file new_filename=: {new_filename}");
-//          var new_fs = new FileStream(new_filename, FileMode.Open, FileAccess.Read);
-//          _current_fs = new Tuple<int, FileStream>(new_idx, new_fs);
-//          _load_packets_from_current_file();
-//       }
-
-//       #endregion
-//    }
-// }

src/DataStreamSuite/DataStream/Primitives/FileStreams/RecorderStream.cs

@@ -1,68 +0,0 @@
-// using System;
-// using System.Collections.Generic;
-// using System.Linq;
-// using System.Text;
-// using System.IO;
-
-// namespace BCG
-// {
-//    public class RecorderStream : ViewStream
-//    {
-//       private ViewStream mySource;
-//       private DateTime myStartTime;
-//       private FileStream myOutFile;
-//       private string myFileName;
-//       private IBlueDataSerializer mySerializer;
-
-//       public RecorderStream(ViewStream src, string recordFile)
-//       {
-//          mySource = src;
-//          myFileName = recordFile;
-
-//          myFileName = myFileName.Replace("%DATE%", DateTime.Now.ToString("yyyyMMdd_HHmmss")); 
-//       }
-
-//       protected override void myOnFirstSubscriber()
-//       {
-//          myStartTime = DateTime.Now;         
-//          mySource.onChange += new ChangeDelegate(mySource_onChange);
-//          myOutFile = new FileStream(myFileName, FileMode.OpenOrCreate);
-//       }
-
-//       protected override void myOnLastSubscriber()
-//       {
-//          mySource.onChange -= mySource_onChange;
-//          myOutFile.Close();
-//       }
-
-//       void mySource_onChange(object newValue)
-//       {
-//          DataPacket dp = (DataPacket)newValue;         
-
-//          BlueTable toWrite = new BlueTable(dp.schema);
-
-//          if (mySerializer == null)
-//          {
-//             mySerializer = new UniversalSerializer(dp.schema);
-//          }
-//          foreach (IBlueRow r in dp.newRows)
-//             toWrite.addRow(r);
-
-//          byte[] result;
-
-//          if (myLastValue == null)
-//             mySerializer.serialize(StreamOpCode.create, new IBlueRowCollection[] { dp.newRows }, out result);
-//          else
-//             mySerializer.serialize(StreamOpCode.multipleRemoveAdd, new IBlueRowCollection[] { dp.deletedRows, dp.addedRows }, out result);
-
-//          BinaryWriter bw = new BinaryWriter(myOutFile);
-//          //write down what time this happened
-//          bw.Write((DateTime.Now - myStartTime).TotalSeconds);
-//          bw.Write(result.Length);
-//          bw.Write(result);         
-//          myOutFile.Flush();
-
-//          myFireEvent(dp);
-//       }
-//    }
-// }

src/DataStreamSuite/DataStream/Primitives/FileStreams/SaverStream.cs

@@ -5,7 +5,8 @@ using System.IO;
 using BCG;
 
 namespace bts {
-   // this saves the total state in the filr provided in the cons
+   // this saves the updates as a Create on the same file, each update calls -save, which
+   // reqrites the whole data to the packet. The other side of this stream is the Loader
    [DataStreamUserRespresentation("saver")]
    public class SaverStream : ViewStream {
       private ViewStream _stream;
@@ -31,7 +32,6 @@ namespace bts {
          StreamIOMessage.CreateMessage cr_msg = new StreamIOMessage.CreateMessage(dp.newRows);
          byte[] str_bytes = _serializer.serialize(new StreamIOMessage(cr_msg));
          StreamSerializationUtils.write_to_file(str_bytes, _filename);
-
       }
    }
 }

src/DataStreamSuite/DataStream/Primitives/FileStreams/StreamPersistorUtils.cs

@@ -3,6 +3,7 @@ using System.Collections.Generic;
 using System.Linq;
 using System.Globalization;
 using System.IO;
+using bts;
 
 
 namespace BCG {

src/DataStreamSuite/DataStream/Primitives/Filter/NullFilterStream.cs

@@ -0,0 +1,24 @@
+namespace bts.SilverPine {
+   using BCG;
+
+   using System;
+
+   public class NullFilterStream : ViewStream {
+      public NullFilterStream(ViewStream parent_str, string col_name)
+      : this(parent_str, col_name, true) { }
+      public NullFilterStream(ViewStream parent_str, string col_name, bool filter_out_doublenans = false) {
+         ViewStream rs = new FilterStream(parent_str,
+         r => {
+            bool is_dbnull = r.is_null(col_name);
+            if (is_dbnull) {
+               return false;
+            }
+            if (r.schema[col_name].type_code() == TypeCode.Double) {
+               return !Double.IsNaN(r.get<double>(col_name));
+            }
+            return true;
+         });
+         mySetRealSource(rs);
+      }
+   }
+}

src/DataStreamSuite/DataStream/Primitives/GroupBy/AverageGroupsStream.cs

@@ -1,102 +1,102 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
+// using System;
+// using System.Collections.Generic;
+// using System.Linq;
+// using System.Text;
 
-namespace BCG
-{
+// namespace BCG
+// {
    
-    public class AverageGroupsStream : ViewStream
-    {
-        public AverageGroupsStream(ViewStream input, string colToAverage, string keyColumn)
-        {
+//     public class AverageGroupsStream : ViewStream
+//     {
+//         public AverageGroupsStream(ViewStream input, string colToAverage, string keyColumn)
+//         {
             
-            myColToAverage = colToAverage;
-            myInput = input;
-            myKeyColumn = keyColumn;
-        }
+//             myColToAverage = colToAverage;
+//             myInput = input;
+//             myKeyColumn = keyColumn;
+//         }
 
-        private ViewStream myInput;
-        private string myColToAverage;
-        private BlueTableSchema mySchema;
-        private string myKeyColumn;
+//         private ViewStream myInput;
+//         private string myColToAverage;
+//         private BlueTableSchema mySchema;
+//         private string myKeyColumn;
 
-        protected override void myOnFirstSubscriber()
-        {
-            myInput.onChange += new ChangeDelegate(myInput_onChange);
-        }
-        protected override void myOnLastSubscriber()
-        {
-            myInput.onChange -= myInput_onChange;
-        }
-        private void myInput_onChange(object newValue)
-        {
-            if (mySchema == null)
-                myBuildSchema(((DataPacket)newValue).schema);
+//         protected override void myOnFirstSubscriber()
+//         {
+//             myInput.onChange += new ChangeDelegate(myInput_onChange);
+//         }
+//         protected override void myOnLastSubscriber()
+//         {
+//             myInput.onChange -= myInput_onChange;
+//         }
+//         private void myInput_onChange(object newValue)
+//         {
+//             if (mySchema == null)
+//                 myBuildSchema(((DataPacket)newValue).schema);
 
-            IBlueRowCollection newRows = myEvaluate((DataPacket)newValue);
+//             IBlueRowCollection newRows = myEvaluate((DataPacket)newValue);
 
-            if (myLastValue == null)
-                myFireEvent(new DataPacket(newRows, newRows, new BlueTable(mySchema)));
-            else
-            {
-                IBlueRowCollection added, deleted;
-                BlueDataUtility.computeDifferences(newRows, ((DataPacket) myLastValue).newRows, out deleted, out added);
-                myFireEvent(new DataPacket(newRows, added, deleted));
-            }
-        }
+//             if (myLastValue == null)
+//                 myFireEvent(new DataPacket(newRows, newRows, new BlueTable(mySchema)));
+//             else
+//             {
+//                 IBlueRowCollection added, deleted;
+//                 BlueDataUtility.computeDifferences(newRows, ((DataPacket) myLastValue).newRows, out deleted, out added);
+//                 myFireEvent(new DataPacket(newRows, added, deleted));
+//             }
+//         }
 
-        private void myBuildSchema(IBlueTableSchema schema)
-        {
-            List<ColumnInfo> columns = new List<ColumnInfo>();
+//         private void myBuildSchema(IBlueTableSchema schema)
+//         {
+//             List<ColumnInfo> columns = new List<ColumnInfo>();
 
-            foreach (ColumnInfo ci in schema.columnInfo)
-                columns.Add(ci);
+//             foreach (ColumnInfo ci in schema.columnInfo)
+//                 columns.Add(ci);
 
-            columns.Add(new ColumnInfo("average_"+myColToAverage, columns.Count, typeof(double)));
-            mySchema = new BlueTableSchema("average_of_" + schema.tableName, columns.ToArray());
-        }
+//             columns.Add(new ColumnInfo("average_"+myColToAverage, columns.Count, typeof(double)));
+//             mySchema = new BlueTableSchema("average_of_" + schema.tableName, columns.ToArray());
+//         }
 
 
 
-        private IBlueRowCollection myEvaluate(DataPacket packet)
-        {
+//         private IBlueRowCollection myEvaluate(DataPacket packet)
+//         {
          
-            IDictionary<string, double> colToRunningTotal = new Dictionary<string, double>();
-            IDictionary<string, int> colToCount = new Dictionary<string, int>();
-            IDictionary<string, IBlueRow> colToLastRow = new Dictionary<string, IBlueRow>();
+//             IDictionary<string, double> colToRunningTotal = new Dictionary<string, double>();
+//             IDictionary<string, int> colToCount = new Dictionary<string, int>();
+//             IDictionary<string, IBlueRow> colToLastRow = new Dictionary<string, IBlueRow>();
 
-            foreach (IBlueRow r in packet.newRows)
-            {
-                double thisVal = Convert.ToDouble(r[myColToAverage]);
+//             foreach (IBlueRow r in packet.newRows)
+//             {
+//                 double thisVal = Convert.ToDouble(r[myColToAverage]);
 
-                string keyVal = r[myKeyColumn].ToString();
-                if (!colToCount.ContainsKey(keyVal))
-                {
-                    colToRunningTotal[keyVal] = thisVal;
-                    colToCount[keyVal] = 1;
-                }
-                else
-                {
-                    colToRunningTotal[keyVal] += thisVal;
-                    colToCount[keyVal] += 1;
-                }
-                colToLastRow[keyVal] = r;
-            }
+//                 string keyVal = r[myKeyColumn].ToString();
+//                 if (!colToCount.ContainsKey(keyVal))
+//                 {
+//                     colToRunningTotal[keyVal] = thisVal;
+//                     colToCount[keyVal] = 1;
+//                 }
+//                 else
+//                 {
+//                     colToRunningTotal[keyVal] += thisVal;
+//                     colToCount[keyVal] += 1;
+//                 }
+//                 colToLastRow[keyVal] = r;
+//             }
 
-            BlueTable result = new BlueTable(mySchema);
-            foreach (string key in colToCount.Keys)
-            {
-                object[] values = new object[mySchema.columnInfo.Length];
+//             BlueTable result = new BlueTable(mySchema);
+//             foreach (string key in colToCount.Keys)
+//             {
+//                 object[] values = new object[mySchema.columnInfo.Length];
 
-                colToLastRow[key].CopyTo(values, 0);
-                values[values.Length - 1] = packet.newRows.Count > 0 ? colToRunningTotal[key] / packet.newRows.Count : 0.0;
-                result.addRow(new BlueRow(mySchema, values));
-            }
+//                 colToLastRow[key].CopyTo(values, 0);
+//                 values[values.Length - 1] = packet.newRows.Count > 0 ? colToRunningTotal[key] / packet.newRows.Count : 0.0;
+//                 result.addRow(new BlueRow(mySchema, values));
+//             }
 
-            return result;
-        }
+//             return result;
+//         }
 
       
-    }
-}
+//     }
+// }

src/DataStreamSuite/DataStream/Primitives/GroupBy/AverageNonIncGroupsStream.cs

@@ -0,0 +1,31 @@
+namespace bts.SilverPine {
+   using BCG;
+
+   using System;
+   using System.Collections.Generic;
+   using System.Linq;
+
+   [DataStreamUserRespresentation("Average")]
+   public class AverageNonIncGroupsStream : ViewStream {
+      public AverageNonIncGroupsStream(ViewStream parent_stream, string inp_col_name, params string[] group_by_cols) {
+         NonIncGroupByStream rs = new NonIncGroupByStream(parent_stream, () => group_by_cols,
+                                                         (g_rows) => _find_mean(inp_col_name, g_rows));
+         mySetRealSource(rs);
+      }
+
+      private List<IBlueRow> _find_mean(string inp_col_name, List<IBlueRow> g_rows) {
+         IBlueRow inp_row = g_rows.ElementAt(0);
+         IEnumerable<double> inp_vals = BlueDataUtility.get_colvalues<double>(g_rows, inp_col_name);
+         double sum = 0.0;
+         inp_vals.ToList().ForEach(d => sum += d);
+         string out_col_name = $"average_{inp_col_name}";
+         double average = sum / inp_vals.Count();
+         IBlueTableSchema schema_out = inp_row.schema.add_column(TypeCode.Double, out_col_name);
+         MutableBlueRow out_row = new MutableBlueRow(schema_out, inp_row);
+         out_row.setValue(out_col_name, average);
+         List<IBlueRow> rv = new List<IBlueRow>();
+         rv.Add(out_row);
+         return rv;
+      }
+   }
+}

src/DataStreamSuite/DataStream/Primitives/GroupBy/GroupByBundle.cs

@@ -0,0 +1,91 @@
+namespace bts.SilverPine {
+   using BCG;
+
+   using System;
+   using System.Collections.Generic;
+   using System.Linq;
+
+   using ZstdSharp.Unsafe;
+
+   public enum GroupByBundleOp {
+      Mean,
+      Median,
+      Mode,
+      StdDev,
+      Max,
+      Min,
+      First,
+      Last,
+      Sum
+   }
+   public class GroupByBundle {
+      public readonly List<Tuple<string, GroupByBundleOp>> operations;
+
+      public static GroupByBundle parse(string text) {
+         // text is inp_col: op, inp_col: op, inp_col:op
+         List<Tuple<string, GroupByBundleOp>> rv = new List<Tuple<string, GroupByBundleOp>>();
+         string[] calcs = text.Split(',');
+         foreach (string c in calcs) {
+            string[] toks = c.Split(':');
+            string inp_col = toks[0].Trim();
+            string op_txt = toks[1].Trim();
+            GroupByBundleOp op = Enum.Parse<GroupByBundleOp>(op_txt, true);
+            rv.Add(Tuple.Create(inp_col, op));
+         }
+         return new GroupByBundle(rv);
+      }
+
+      public GroupByBundle(List<Tuple<string, GroupByBundleOp>> operations_) {
+         operations = operations_;
+      }
+
+      public static IBlueRow group_by(List<IBlueRow> rows, GroupByBundle parameters) {
+         if (rows.Count() == 0) {
+            return null;
+         }
+
+         IBlueRow ir = rows.ElementAt(0);
+         IBlueTableSchema in_schema = ir.schema;
+         IBlueTableSchema out_schema = in_schema;
+         IDictionary<string, Tuple<string, GroupByBundleOp>> out_col_map =
+            new Dictionary<string, Tuple<string, GroupByBundleOp>>();
+         foreach (var t in parameters.operations) {
+            string out_col_name = $"{t.Item1}_{t.Item2}".ToLower();
+            out_schema = out_schema.add_column(TypeCode.Double, out_col_name);
+            out_col_map[out_col_name] = t;
+         }
+         MutableBlueRow or = new MutableBlueRow(out_schema, ir);
+         foreach (var kv in out_col_map) {
+            string out_col_name = kv.Key;
+            string in_col_name = kv.Value.Item1;
+            GroupByBundleOp op = kv.Value.Item2;
+            IEnumerable<double> col_vals = BlueDataUtility.get_colvalues<double>(rows, in_col_name);
+            double comp_res = compute_op(col_vals, op);
+            or.setValue(out_col_name, comp_res);
+         }
+         return or;
+      }
+
+      public static double compute_op(IEnumerable<double> col_vals, GroupByBundleOp op) {
+         switch (op) {
+            case GroupByBundleOp.Max:
+               return col_vals.Max<double>();
+            case GroupByBundleOp.Min:
+               return col_vals.Min<double>();
+            case GroupByBundleOp.Mean:
+               return col_vals.Sum() / col_vals.Count();
+            case GroupByBundleOp.Sum:
+               return col_vals.Sum();
+            case GroupByBundleOp.First:
+               return col_vals.First();
+            case GroupByBundleOp.Last:
+               return col_vals.Last();
+            default:
+               // TODO
+               Log.assert(false);
+               return Double.NaN;
+         }
+      }
+
+   }
+}

src/DataStreamSuite/DataStream/Primitives/GroupBy/MaxStream.cs

@@ -10,6 +10,6 @@ namespace BCG {
         }
 
         public MaxStream(ViewStream stream, string outputCol)
-           : this(new AddColumnsStream(stream, "fakeCol=0"), outputCol, new string[] { "fakeCol" }) { }
+           : this(new AddColumnsStream(stream, "fake_col: 0"), outputCol, new string[] { "fake_col" }) { }
     }
 }

src/DataStreamSuite/DataStream/Primitives/GroupBy/NonIncremental/NonIncGroupByStream.cs

@@ -1,13 +1,40 @@
 using System;
 using System.Collections.Generic;
 using System.Diagnostics;
+using System.Linq;
+using System.Text.Json;
 
 using bts;
+using bts.SilverPine;
 
-namespace BCG
-{
-   public class NonIncGroupByStream : ViewStream
-   {
+namespace BCG {
+   [DataStreamUserRespresentation("aggregate")]
+   public class NonIncGroupByStream : ViewStream {
+      private class DelNonIncGroupByEvaluator : INonIncGroupByEvaluator {
+         Func<string[]> _groupby_cols_func;
+         Func<List<IBlueRow>, List<IBlueRow>> _evaluate_func;
+         public DelNonIncGroupByEvaluator(Func<string[]> groupby_cols_func,
+                                          Func<List<IBlueRow>, List<IBlueRow>> evaluate_func) {
+            _groupby_cols_func = groupby_cols_func;
+            _evaluate_func = evaluate_func;
+         }
+         public IBlueRowCollection evaluate(IDictionary<int, List<IBlueRow>> grouped_rows) {
+            BlueTable rv = null;
+            foreach (var kv in grouped_rows) {
+               List<IBlueRow> g_rows = kv.Value;
+               List<IBlueRow> out_rows = _evaluate_func.Invoke(g_rows);
+               if (out_rows != null && out_rows.Count > 0) {
+                  rv = rv ?? new BlueTable(out_rows.ElementAt(0).schema);
+               }
+               rv.addRows(out_rows);
+            }
+            return rv;
+         }
+         public string[] get_group_by_cols() {
+            return _groupby_cols_func();
+         }
+
+      }
       private ViewStream _main_stream;
       private INonIncGroupByEvaluator _groupby_func;
       private BlueRowHasher _row_hasher;
@@ -15,24 +42,47 @@ namespace BCG
       private Stopwatch _timer = new();
       private int _n_total_fires = 0, _n_expensive_fires = 0;
 
-      public NonIncGroupByStream(ViewStream main_stream, INonIncGroupByEvaluator groupby_func)
-      {
+      public NonIncGroupByStream(ViewStream main_stream, INonIncGroupByEvaluator groupby_func) {
          _main_stream = main_stream;
          _groupby_func = groupby_func;
          _groupby_cols = _groupby_func.get_group_by_cols();
-         _row_hasher = new BlueRowHasher(_groupby_cols.Length);
+         _row_hasher = new BlueRowHasher(_groupby_cols == null ? 0 : _groupby_cols.Length);
       }
 
+      public NonIncGroupByStream(ViewStream main_stream, Func<string[]> groupby_cols_func,
+                                 Func<List<IBlueRow>, List<IBlueRow>> evaluate_func)
+                                 : this(main_stream, new DelNonIncGroupByEvaluator(groupby_cols_func, evaluate_func)) { }
+
+      public NonIncGroupByStream(ViewStream parent_str, string groupby_col, string standard_group_by_text, string optional_sort_col = null) {
+         GroupByBundle std_group_by = GroupByBundle.parse(standard_group_by_text);
+         Func<string[]> groupby_cols_func = () => {
+            return String.IsNullOrEmpty(groupby_col) ? null : new string[] { groupby_col };
+         };
+         Func<List<IBlueRow>, List<IBlueRow>> evaluate_func = (group_rows) => {
+            if (!String.IsNullOrEmpty(optional_sort_col))
+               group_rows.Sort((r1, r2) => BlueDataUtility.compare_rows(r2, r1, optional_sort_col));
+            IBlueRow rv = GroupByBundle.group_by(group_rows, std_group_by);
+            return new IBlueRow[] { rv }.ToList();
+         };
+         ViewStream rs = new NonIncGroupByStream(parent_str, groupby_cols_func, evaluate_func);
+         mySetRealSource(rs);
+      }
+      public NonIncGroupByStream(ViewStream parent_str, string standard_group_by_text) :
+      this(parent_str, String.Empty, standard_group_by_text) {
+      }
+
+
       protected override void myOnFirstSubscriber() { _main_stream.onChange += _main_stream_onchange; }
 
       protected override void myOnLastSubscriber() { _main_stream.onChange -= _main_stream_onchange; }
 
-      private void _main_stream_onchange(object nv)
-      {
+      private void _main_stream_onchange(object nv) {
          DataPacket dp = nv as DataPacket;
          ///grouping together
          _timer.Restart();
-         IDictionary<int, List<IBlueRow>> grouped_rows = BlueDataUtility.group_together (dp.newRows, _groupby_cols, ref _row_hasher);
+         IDictionary<int, List<IBlueRow>> grouped_rows = (_groupby_cols == null || _groupby_cols.Length == 0) ?
+         new Dictionary<int, List<IBlueRow>>() { { 0, dp.newRows.to_list() } } :
+         BlueDataUtility.group_together(dp.newRows, _groupby_cols, ref _row_hasher);
          _timer.Stop();
          var groupElapsedMs = _timer.ElapsedMilliseconds;
          if (groupElapsedMs >= 50) {
@@ -44,14 +94,15 @@ namespace BCG
          IBlueRowCollection total_state = _groupby_func.evaluate(grouped_rows);
          _timer.Stop();
          var applyElapsedMs = _timer.ElapsedMilliseconds;
-         _n_total_fires++;         
-         if (applyElapsedMs >= 50)
-         {
+         _n_total_fires++;
+         if (applyElapsedMs >= 50) {
             _n_expensive_fires++;
             Log.DataFlow.info(
                $"Time to evaluate {grouped_rows.Count} groups {_timer.ElapsedMilliseconds}ms, ratio {_n_expensive_fires}/{_n_total_fires}");
-         }         
-         myFireEvent(new DataPacket(total_state, total_state, myLastValue == null ? new BlueTable(total_state.schema) : (myLastValue as DataPacket).newRows));             
+         }
+         // TODO,
+         if (total_state != null)
+            myFireEvent(new DataPacket(total_state, total_state, myLastValue == null ? new BlueTable(total_state.schema) : (myLastValue as DataPacket).newRows));
       }
    }
 }

src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinBookkeeper.cs

@@ -1,15 +1,16 @@
 using bts;
+using bts.collections;
 
 using mbts;
 
 using System;
 using System.Collections.Generic;
 using System.Diagnostics;
+using System.Linq;
+using System.Security.Cryptography;
 
-namespace BCG
-{
-   public class NonIncJoinBookkeeper : IDisposable
-   {
+namespace BCG {
+   public class NonIncJoinBookkeeper : IDisposable {
       public Action on_new_data;
       private readonly NonIncJoinSpec _join_spec;
       private readonly int _join_position;
@@ -26,41 +27,71 @@ namespace BCG
 
       private int[] _join_col_indices = null,
                      _data_col_indices = null;
-      
-      public NonIncJoinBookkeeper(NonIncJoinSpec the_join_spec, int join_position, int start_index_in_output, BlueRowHasher row_hasher)
-      {
+
+      public NonIncJoinBookkeeper(NonIncJoinSpec the_join_spec, int join_position, int start_index_in_output, BlueRowHasher row_hasher) {
          _join_spec = the_join_spec;
          _join_position = join_position;
          _start_index_in_output = start_index_in_output;
          _row_hasher = row_hasher;
          _is_left_most = _join_position == 0;
          _is_tag_along = _join_spec.join_col_names.Length == 0;
+      }
+
+      private bool _filled_missing_data = false;
+      private void _fill_missing_data(DataPacket dp) {
+         if (_filled_missing_data)
+            return;
+         Func<string[], bool> null_or_empty = (str_arr) => {
+            if (str_arr == null || str_arr.Length == 0)
+               return true;
+            foreach (string s in str_arr) {
+               if (!String.IsNullOrEmpty(s)) {
+                  return false;
+               }
+            }
+            return true;
+         };
+         Func<string[], string[]> remove_bad_cols = (col_names) => {
+            HashSet<string> schema_col_map = new HashSet<string>(dp.schema.columnNames);
+            List<string> cols_list = new List<string>(col_names);
+            cols_list.RemoveWhere(c => !schema_col_map.Contains(c));
+            return cols_list.ToArray();
+         };
+         if (null_or_empty(_join_spec.data_col_names)) {
+            _join_spec.data_col_names = dp.schema.columnNames;
+         }
+         _join_spec.data_col_names = remove_bad_cols(_join_spec.data_col_names);
+         if (null_or_empty(_join_spec.data_col_names)) {
+            _join_spec.data_col_names = dp.schema.columnNames;
+         }
+         if (null_or_empty(_join_spec.projected_col_names)) {
+            _join_spec.projected_col_names = _join_spec.data_col_names;
+         }
+      }
+
+      public void init() {
          _join_spec.stream.onChange += _on_stream_cb;
       }
 
-      public void _on_stream_cb(object dp_obj)
-      {
+      public void _on_stream_cb(object dp_obj) {
+         Log.Gen.info($"NonIncJoinBookkeeper._on_stream_cb dp_obj: {dp_obj}");
          var timer = Stopwatch.StartNew();
          DataPacket dp = (DataPacket)dp_obj;
-         if (_schema == null)
-         {
+         _fill_missing_data(dp);
+         if (_schema == null) {
             _schema = dp.schema;
             _join_col_indices = BlueDataUtility.get_column_indices(_schema, _join_spec.join_col_names);
             _data_col_indices = BlueDataUtility.get_column_indices(_schema, _join_spec.data_col_names);
          }
 
-         if (_is_left_most)
-         {
+         if (_is_left_most) {
             _row_list.Clear();
             foreach (IBlueRow r in dp.newRows)
-               _row_list.Add(new Pair<int, IBlueRow>(_row_hasher.get_hash(r, _join_col_indices), r));            
-         }
-         else
-         {
+               _row_list.Add(new Pair<int, IBlueRow>(_row_hasher.get_hash(r, _join_col_indices), r));
+         } else {
             if (_is_tag_along)
                _solitary_row = BlueDataUtility.extractRow(dp.newRows, 0);
-            else
-            {
+            else {
                _row_map.Clear();
                foreach (IBlueRow r in dp.newRows)
                   _row_map[_row_hasher.get_hash(r, _join_col_indices)] = r;
@@ -79,31 +110,25 @@ namespace BCG
 
       public bool is_ready { get { return _schema != null; } }
 
-      public void constribute_cols_to_schema(List<ColumnInfo> cis)
-      {
+      public void constribute_cols_to_schema(List<ColumnInfo> cis) {
          for (int i = 0; i < _join_spec.data_col_names.Length; i++)
             cis.Add(new ColumnInfo(_join_spec.projected_col_names[i], cis.Count, _schema[_join_spec.data_col_names[i]].colType));
       }
 
       public List<Pair<int, IBlueRow>> get_list() { return _row_list; }
 
-      public void copy_master_row_to_output_row(int row_index, object[] output_row)
-      {
+      public void copy_master_row_to_output_row(int row_index, object[] output_row) {
          IBlueRow this_row = _row_list[row_index].second;
          BlueDataUtility.copy_to_row(this_row, _data_col_indices, output_row, _start_index_in_output);
       }
 
-      public void copy_matching_row_to_output_row(int row_hash, object[] output_row)
-      {
-         if (_is_tag_along)
-         {
+      public void copy_matching_row_to_output_row(int row_hash, object[] output_row) {
+         if (_is_tag_along) {
             if (_solitary_row != null)
                BlueDataUtility.copy_to_row(_solitary_row, _data_col_indices, output_row, _start_index_in_output);
             else
                copy_dbnull_to_output_row(output_row);
-         }
-         else
-         {
+         } else {
             IBlueRow this_row = null;
             if (_row_map.TryGetValue(row_hash, out this_row))
                BlueDataUtility.copy_to_row(this_row, _data_col_indices, output_row, _start_index_in_output);
@@ -112,8 +137,7 @@ namespace BCG
          }
       }
 
-      private void copy_dbnull_to_output_row(object[] output_row)
-      {
+      private void copy_dbnull_to_output_row(object[] output_row) {
          for (int i = 0; i < _data_col_indices.Length; i++)
             output_row[i + _start_index_in_output] = DBNull.Value;
       }

src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinSpec.cs

@@ -3,63 +3,63 @@ using System.Collections.Generic;
 using System.Linq;
 using System.Text;
 using System.Threading.Tasks;
+
 using mbts;
 
-namespace BCG
-{
-   public class NonIncJoinSpec
-   {
+namespace BCG {
+   // this a collection of data, theres no joining logic in here
+   public class NonIncJoinSpec {
       public ViewStream stream;
+      // join col names are the names on which the streams are joined (for equality),
+      // data col names are the names in the original stream
+      // projected col names are their new names, ie if its x:l_x, then x is in the 
+      // data_col_names and l_x in projected cols
       public string[] join_col_names, data_col_names, projected_col_names;
 
       public NonIncJoinSpec() { }
 
       public NonIncJoinSpec(ViewStream the_stream, string[] the_join_col_names, string[] the_data_col_names)
-         : this (the_stream, the_join_col_names, the_data_col_names, the_data_col_names)
-      { }
+         : this(the_stream, the_join_col_names, the_data_col_names, the_data_col_names) { }
 
-      public NonIncJoinSpec(ViewStream the_stream, string[] the_join_col_names, string[] the_data_col_names, string[] the_projected_col_names)
-      {
+      public NonIncJoinSpec(ViewStream the_stream, string[] the_join_col_names, string[] the_data_col_names, string[] the_projected_col_names) {
          stream = the_stream;
          join_col_names = the_join_col_names;
          data_col_names = the_data_col_names;
          projected_col_names = the_projected_col_names;
       }
 
-      public static NonIncJoinSpec[] from_triplet(params Triplet<ViewStream, string, string>[] join_specs)
-      {
+      public static NonIncJoinSpec[] from_triplet(params Triplet<ViewStream, string, string>[] join_specs) {
          List<NonIncJoinSpec> rv = new List<NonIncJoinSpec>();
-         foreach (Triplet<ViewStream, string, string> r in join_specs)
-         {
-            string[] keys = String.IsNullOrEmpty (r.second) ? new string[] {} : r.second.Split(' ');
+         foreach (Triplet<ViewStream, string, string> r in join_specs) {
+            string[] keys = String.IsNullOrEmpty(r.second) ? new string[] { } : r.second.Split(' ');
             string[] visible_cols = r.third.Split(' ');
             List<string> data_col_names = new List<string>(), mapped_col_names = new List<string>();
-            foreach (string s in visible_cols)
-            {
+            foreach (string s in visible_cols) {
                string[] toks = s.Split(':');
-               if (toks.Length == 1)
-               {
+               if (toks.Length == 1) {
                   data_col_names.Add(toks[0]);
                   mapped_col_names.Add(toks[0]);
-               }
-               else if (toks.Length == 2)
-               {
+               } else if (toks.Length == 2) {
                   data_col_names.Add(toks[0]);
                   mapped_col_names.Add(toks[1]);
                }
             }
-            rv.Add (new NonIncJoinSpec (r.first, keys, data_col_names.ToArray (), mapped_col_names.ToArray()));
+            rv.Add(new NonIncJoinSpec(r.first, keys, data_col_names.ToArray(), mapped_col_names.ToArray()));
          }
-         return rv.ToArray ();
+         return rv.ToArray();
+      }
+
+      public override string ToString() {
+         Func<string[], string> arr_txt = (a) => {
+            string rv = "";
+            foreach (string s in a) {
+               rv += "{s} ";
+            }
+            rv = rv.Trim();
+            return rv;
+         };
+         string rv = $"stream: {stream} join_col_names: {arr_txt(join_col_names)} data_col_names: {arr_txt(data_col_names)} projected_col_names: {arr_txt(projected_col_names)}";
+         return rv;
       }
    }
-
-   //[DataStreamUserRespresentation("ninctest")]
-   //public class NonIncJoinTestStream : ViewStream
-   //{
-   //   public NonIncJoinTestStream (string key_cols, string data_cols)
-   //   {
-   //      NonIncJoinSpec.from_triplet(new Triplet<ViewStream, string, string>(null, key_cols, data_cols));
-   //   }
-   //}
 }

src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinStream.cs

@@ -6,11 +6,9 @@ using System;
 using System.Collections.Generic;
 using System.Diagnostics;
 
-namespace BCG
-{
+namespace BCG {
    [DataStreamUserRespresentation("fastJoin")]
-   public class NonIncJoinStream : ViewStream, BCG.Utilities.General.IWorker
-   {
+   public class NonIncJoinStream : ViewStream, BCG.Utilities.General.IWorker {
       private readonly NonIncJoinSpec[] _join_specs;
       private readonly INonIncJoinInlineMapper _output_mapper;
       private readonly BlueRowHasher _row_hasher;
@@ -28,24 +26,19 @@ namespace BCG
                {
                   new Triplet<ViewStream, string, string> (left, left_join_cols, left_proj),
                   new Triplet<ViewStream, string, string> (right, right_join_cols, right_proj)
-               })
-      { }
+               }) { }
 
       public NonIncJoinStream(params Triplet<ViewStream, string, string>[] join_specs)
-         : this(NonIncJoinSpec.from_triplet(join_specs), null, 0)
-      { }
+         : this(NonIncJoinSpec.from_triplet(join_specs), null, 0) { }
 
       public NonIncJoinStream(ViewStreamDatabase vsd, INonIncJoinInlineMapper output_mapper, int throttles_ms,
                               params Triplet<ViewStream, string, string>[] join_specs)
-         : this(vsd, NonIncJoinSpec.from_triplet(join_specs), output_mapper, throttles_ms)
-      { }
+         : this(vsd, NonIncJoinSpec.from_triplet(join_specs), output_mapper, throttles_ms) { }
 
       public NonIncJoinStream(NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper output_mapper, int throttles_ms = 0)
-         : this(ViewStreamDatabase.instance, join_specs, output_mapper, throttles_ms)
-      { }
+         : this(ViewStreamDatabase.instance, join_specs, output_mapper, throttles_ms) { }
 
-      public NonIncJoinStream(ViewStreamDatabase vsd, NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper output_mapper, int throttles_ms = 0)
-      {
+      public NonIncJoinStream(ViewStreamDatabase vsd, NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper output_mapper, int throttles_ms = 0) {
          _join_specs = join_specs;
          _output_mapper = output_mapper;
          _row_hasher = new BlueRowHasher(_join_specs[0].join_col_names.Length);
@@ -53,12 +46,11 @@ namespace BCG
          _vsd = vsd;
       }
 
-      protected override void myOnFirstSubscriber()
-      {
+      protected override void myOnFirstSubscriber() {
          int data_col_no = 1;
-         for (int jp = 0; jp < _join_specs.Length; jp++)
-         {
+         for (int jp = 0; jp < _join_specs.Length; jp++) {
             NonIncJoinBookkeeper s = new NonIncJoinBookkeeper(_join_specs[jp], jp, data_col_no, _row_hasher);
+            Log.Gen.info($"NonIncJoinStream.myOnFirstSubscriber jp: {jp} spec: {_join_specs[jp]}");
             data_col_no += _join_specs[jp].data_col_names.Length;
             s.on_new_data += _stream_callback;
             _book_keeping_data.Add(s);
@@ -66,10 +58,11 @@ namespace BCG
          if (_output_mapper != null)
             _output_mapper.onMapperChange += _on_mapper_change;
          _vsd.addWorker(this);
+         foreach (var bkd in _book_keeping_data)
+            bkd.init();
       }
 
-      protected override void myOnLastSubscriber()
-      {
+      protected override void myOnLastSubscriber() {
          foreach (NonIncJoinBookkeeper d in _book_keeping_data)
             d.Dispose();
          if (_output_mapper != null)
@@ -77,30 +70,26 @@ namespace BCG
          _vsd.removeWorker(this);
       }
 
-      private void _stream_callback()
-      {
+      private void _stream_callback() {
+         Log.Gen.info("NonIncJoinStream._stream_callback received callback");
          _is_dirty = true;
          doWork();
       }
 
-      private void _on_mapper_change()
-      {
+      private void _on_mapper_change() {
          _is_dirty = true;
          doWork();
       }
 
-      private bool _is_ready()
-      {
+      private bool _is_ready() {
          bool rv = true;
          foreach (NonIncJoinBookkeeper d in _book_keeping_data)
             rv &= d.is_ready;
          return rv;
       }
 
-      private void _construct_schema()
-      {
-         if (_output_schema == null)
-         {
+      private void _construct_schema() {
+         if (_output_schema == null) {
             List<ColumnInfo> cis = new List<ColumnInfo>(new ColumnInfo[] { new ColumnInfo("id", 0, typeof(int)) });
             foreach (NonIncJoinBookkeeper d in _book_keeping_data)
                d.constribute_cols_to_schema(cis);
@@ -109,27 +98,23 @@ namespace BCG
       }
 
       private int _n_total_fires = 0, _n_expensive_fires = 0;
-      private void _form_and_fire()
-      {
-         if (_is_ready())
-         {
+      private void _form_and_fire() {
+         Log.Gen.info($"NonIncJoinStream._form_and_fire _is_ready(): {_is_ready()}");
+         if (_is_ready()) {
             _construct_schema();
             var timer = Stopwatch.StartNew();
             IList<Pair<int, IBlueRow>> left_row_list = _book_keeping_data[0].get_list();
             List<object[]> output_rows = new List<object[]>();
 
-            for (int i = 0; i < left_row_list.Count; i++)
-            {
+            for (int i = 0; i < left_row_list.Count; i++) {
                object[] or = new object[_output_schema.columnInfo.Length];
                or[0] = i;
                output_rows.Add(or);
             }
 
-            for (int ji = 0; ji < _book_keeping_data.Count; ji++)
-            {
+            for (int ji = 0; ji < _book_keeping_data.Count; ji++) {
                NonIncJoinBookkeeper bk_data = _book_keeping_data[ji];
-               for (int i = 0; i < left_row_list.Count; i++)
-               {
+               for (int i = 0; i < left_row_list.Count; i++) {
                   if (ji == 0)
                      bk_data.copy_master_row_to_output_row(i, output_rows[i]);
                   else
@@ -139,19 +124,20 @@ namespace BCG
             IBlueRowCollection dp_output_rows = _apply_map(new SequentialIDBlueTable(_output_schema, output_rows));
             timer.Stop();
             _n_total_fires++;
-            if (timer.ElapsedMilliseconds >= 250) {
-               _n_expensive_fires++;
-               string log_msg = $"Join and map time for {_book_keeping_data.Count} "
-                  + $"tables {timer.Elapsed} ratio {_n_expensive_fires}/{_n_total_fires}";
-            }
-            myFireEvent(new DataPacket(dp_output_rows, dp_output_rows, myLastValue == null ? new BlueTable(_output_schema) : (myLastValue as DataPacket).newRows));
+            //if (timer.ElapsedMilliseconds >= 250) {
+            _n_expensive_fires++;
+            string log_msg = $"Join and map time for {_book_keeping_data.Count} "
+               + $"tables {timer.Elapsed.TotalMilliseconds} msecs, ratio {_n_expensive_fires}/{_n_total_fires}";
+            Log.Gen.info(log_msg);
+            //}
+            var dp_out = BlueDataUtility.build_dp_inc(dp_output_rows, myLastValue as DataPacket);
+            myFireEvent(dp_out);
+            //myFireEvent(new DataPacket(dp_output_rows, dp_output_rows, myLastValue == null ? new BlueTable(_output_schema) : (myLastValue as DataPacket).newRows));
          }
       }
 
-      private IBlueRowCollection _apply_map(IBlueRowCollection unmapped_rows)
-      {
-         if (_output_mapper != null)
-         {
+      private IBlueRowCollection _apply_map(IBlueRowCollection unmapped_rows) {
+         if (_output_mapper != null) {
             IBlueRowCollection output_rows;
             var timer = Stopwatch.StartNew();
             output_rows = _output_mapper.map_rows(unmapped_rows);
@@ -160,17 +146,14 @@ namespace BCG
                Log.Gen.info($"Time to map {unmapped_rows.Count} rows into {output_rows.Count} rows {timer.Elapsed}");
             }
             return output_rows;
-         }
-         else
-         {
+         } else {
             return unmapped_rows;
          }
       }
 
-      public void doWork()
-      {
-         if (DateTime.Now - _last_fire_time >= _throttle_period && _is_dirty)
-         {
+      public void doWork() {
+         Log.Gen.info($"NonIncJoinStream.doWork _last_fire_time: {_last_fire_time} _is_dirty: {_is_dirty}");
+         if (DateTime.Now - _last_fire_time >= _throttle_period && _is_dirty) {
             _is_dirty = false;
             _last_fire_time = DateTime.Now;
             _form_and_fire();

src/DataStreamSuite/DataStream/Primitives/Join/Proximity/ProximityJoinStream.cs

@@ -0,0 +1,68 @@
+namespace bts {
+   using BCG;
+   using BCG.Utilities.General;
+
+   using System;
+   using System.Diagnostics;
+
+   using ZstdSharp.Unsafe;
+
+   // functon that joins the 2 rows and returns the outputput row. The function
+   // can return a null which means the 2 rows dont need to be joined
+   using JoinFunc = System.Func<BCG.IBlueRow, BCG.IBlueRow, BCG.IBlueRow>;
+   public class ProximityJoinStream : ViewStream, IWorker {
+      private ViewStream _lvs, _rvs;
+      private JoinFunc _join_func;
+      private bool _dirty = false;
+      private Stopwatch _stopwatch = new Stopwatch();
+      private ViewStreamDatabase _vsd = ViewStreamDatabase.instance;
+      public ProximityJoinStream(ViewStream lvs, ViewStream rvs, JoinFunc join_func) {
+         _lvs = lvs;
+         _rvs = rvs;
+         _join_func = join_func;
+      }
+
+      protected override void myOnFirstSubscriber() {
+         _lvs.onChange += _handle_data;
+         _rvs.onChange += _handle_data;
+         _vsd.addWorker(this);
+      }
+
+      protected override void myOnLastSubscriber() {
+         _lvs.onChange -= _handle_data;
+         _rvs.onChange -= _handle_data;
+         _vsd.removeWorker(this);
+      }
+
+      private void _handle_data(object dp_obj) {
+         _dirty = true;
+         doWork();
+      }
+
+      public void doWork() {
+         Func<bool> all_data_revcd = () => {
+            return _lvs.rows != null && _rvs.rows != null;
+         };
+         _stopwatch.Start();
+         if (_dirty && all_data_revcd()) {
+            _dirty = false;
+            BlueTable rv = null;
+            foreach (IBlueRow lr in _lvs.rows) {
+               foreach (IBlueRow rr in _rvs.rows) {
+                  IBlueRow resr = _join_func(lr, rr);
+                  if (resr == null)
+                     continue;
+                  rv = rv ?? new BlueTable(resr.schema);
+                  rv.addRow(resr);
+               }
+            }
+            _stopwatch.Stop();
+            Log.Gen.info($"ProximityJoinStream left_count: {_lvs.rows.Count} right_count: {_rvs.rows.Count} result_count: {rv.Count} time: {_stopwatch.ElapsedMilliseconds} msecs");
+            var dp = BlueDataUtility.build_dp_inc(rv, myLastValue as DataPacket);
+            myFireEvent(dp);
+         }
+
+      }
+
+   }
+}

src/DataStreamSuite/DataStream/Primitives/Join/Proximity/TimeProximityJoinStream.cs

@@ -0,0 +1,47 @@
+namespace bts.SilverPine {
+   using BCG;
+
+   using System;
+   using System.CodeDom;
+   using System.Configuration;
+   using System.Reactive.Disposables;
+
+   public class TimeProximityJoinStream : ViewStream {
+      private int _id = 0;
+
+      public TimeProximityJoinStream(ViewStream ls, string l_time_col, ViewStream rs, string r_time_col,
+                                       string pre_join_period, string post_join_period) :
+            this(ls, l_time_col, rs, r_time_col, TimeBasedFilter.parse_duration(pre_join_period),
+            TimeBasedFilter.parse_duration(post_join_period)) { }
+
+
+      public TimeProximityJoinStream(ViewStream ls, string l_time_col, ViewStream rs, string r_time_col,
+                                       TimeSpan pre_join_period, TimeSpan post_join_period) {
+         Func<IBlueRow, IBlueRow, IBlueRow> create_row = (lr, rr) => {
+            var id = 0;
+            lock (this) {
+               id = _id++;
+            }
+            IBlueRow rv = BlueDataUtility.create_composite_row(lr, rr, id);
+            return rv;
+         };
+         Func<IBlueRow, IBlueRow, IBlueRow> join_func = (lr, rr) => {
+            DateTime ldt = lr.get_datetime(l_time_col);
+            DateTime rdt = rr.get_datetime(r_time_col);
+            TimeSpan diff = ldt - rdt;
+            if (ldt < rdt) {
+               if (diff.Duration() > pre_join_period) {
+                  return null;
+               }
+            } else {
+               if (diff > post_join_period)
+                  return null;
+            }
+            return create_row(lr, rr);
+         };
+
+         ViewStream rstr = new ProximityJoinStream(ls, rs, join_func);
+         mySetRealSource(rstr);
+      }
+   }
+}

src/DataStreamSuite/DataStream/Primitives/Literal/LiteralStream.cs

@@ -2,9 +2,13 @@ using System;
 using System.Collections;
 using System.Collections.Generic;
 
+using Amazon.Runtime.Internal.Util;
+
 using BCG;
 using BCG.Utilities;
 
+using bts;
+
 namespace BCG {
    public class LiteralStream : ViewStream {
 
@@ -22,6 +26,34 @@ namespace BCG {
          return rv;
       }
 
+      //literal("table1", "x:123, y:1.001, z:abc;x:123, y:1.001, z:abc")
+      public LiteralStream(string table_name, string rows) {
+         Func<string, int, IBlueRow> get_row = (json_row, id) => {
+            Dictionary<string, object> map = new Dictionary<string, object>();
+            string[] toks = json_row.Split(',', StringSplitOptions.TrimEntries);
+            //Logger.assert(col_vals.Length == 2);
+            foreach (string t in toks) {
+               var col_vals = t.Split(':', StringSplitOptions.TrimEntries);
+               string col = col_vals[0];
+               string val = col_vals[1];
+               map.Add(col, val);
+            }
+            map["id"] = id;
+            return new BlueRow(map);
+         };
+         string[] row_arr = rows.Split(';', StringSplitOptions.TrimEntries);
+         BlueTable table = null;
+         foreach (string json_string in row_arr) {
+            IBlueRow r = //BlueDataUtility.convert_json_to_row(json_string, table == null ? 0 : table.Count);
+            get_row(json_string, table == null ? 0 : table.Count);
+            table = table ?? new BlueTable(r.schema);
+            table.addRow(r);
+         }
+         ViewStream str = new BlueTableStream(table);
+         str = new SchemaGuesserStream(str);
+         mySetRealSource(str);
+      }
+
       public LiteralStream(string tableName, string schema, string value) {
          string[] columns = schema.Split(' ');
 
@@ -55,7 +87,7 @@ namespace BCG {
          mySetRealSource(new BlueRowCollectionStaticStream(t));
       }
 
-      public LiteralStream(string schema, string value) : this("literal", schema, value) { }
+      // public LiteralStream(string schema, string value) : this("literal", schema, value) { }
 
       // public LiteralStream(string value) {
 

src/DataStreamSuite/DataStream/Primitives/Map/GlobalMapStream.cs

@@ -0,0 +1,72 @@
+using BCG;
+
+using MongoDB.Driver.Core.WireProtocol.Messages;
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+
+// the first arg is all the rows, the second arg is the index of the row that needs mapping, and 
+// the func returns the mapped row (ie mapped version of the index row)
+using GlobalMap = System.Func<System.Collections.Generic.List<BCG.IBlueRow>, uint, BCG.IBlueRow>;
+
+namespace bts {
+   // In GlobalMap, each row is mapped with the full view of all rows, ie the map functions for a row
+   // can use the state fo all rows.
+   public class GlobalMapStream : ViewStream {
+      private ViewStream _base_stream;
+      private readonly string _sort_col_name, _group_by_col;
+      private GlobalMap _map_func;
+      public GlobalMapStream(ViewStream base_stream, string sort_col_name, string group_by_col, GlobalMap map_func) {
+         _base_stream = base_stream;
+         _sort_col_name = sort_col_name;
+         _group_by_col = group_by_col;
+         _map_func = map_func;
+      }
+
+      public GlobalMapStream(ViewStream base_stream, string sort_col_name, GlobalMap map_func) :
+      this(base_stream, sort_col_name, null, map_func) {
+      }
+
+      protected override void myOnFirstSubscriber() {
+         _base_stream.onChange += _handle_stream_update;
+      }
+
+      protected override void myOnLastSubscriber() {
+         _base_stream.onChange -= _handle_stream_update;
+      }
+
+      void _handle_stream_update(object dp_obj) {
+         DataPacket dp = dp_obj as DataPacket;
+         IDictionary<int, List<IBlueRow>> groups = !String.IsNullOrEmpty(_group_by_col) ?
+         BlueDataUtility.group_together(dp.newRows, new string[] { _group_by_col })
+         :
+         new Dictionary<int, List<IBlueRow>>() { { 0, dp.newRows.to_list() } };
+         BlueTable out_table = null;
+         foreach (var kv in groups) {
+            List<IBlueRow> group_rows = kv.Value;
+            group_rows.Sort((r1, r2) => BlueDataUtility.compare_rows(r2, r1, _sort_col_name));
+            uint i = 0;
+            foreach (IBlueRow r in group_rows) {
+               IBlueRow nr = _map_func(group_rows, i++);
+               out_table = out_table ?? new BlueTable(nr.schema);
+               out_table.addRow(nr);
+            }
+         }
+
+
+         // List<IBlueRow> rows = dp.newRows.to_ienum().ToList();
+         // rows.Sort((r1, r2) => BlueDataUtility.compare_rows(r1, r2, _sort_col_name));
+         // uint i = 0;
+
+         // BlueTable out_table = null;
+         // foreach (IBlueRow r in rows) {
+         //    IBlueRow nr = _map_func(rows, i++);
+         //    out_table = out_table ?? new BlueTable(nr.schema);
+         //    out_table.addRow(nr);
+         // }
+         DataPacket dp_out = BlueDataUtility.build_dp_inc(out_table, myLastValue as DataPacket);
+         myFireEvent(dp_out);
+      }
+   }
+}

src/DataStreamSuite/DataStream/Primitives/Misc/PubSubTestStream.cs

@@ -8,26 +8,33 @@ using System;
 using System.IO;
 using System.Linq;
 
-namespace bts {
-    public class PubSubTestingStream : ViewStream, IWorker {
-        public class PayloadMessage {
+namespace bts
+{
+    public class PubSubTestingStream : ViewStream, IWorker
+    {
+        public class PayloadMessage
+        {
             public int msg_id = 0;
             public string guid;
             public Int64 travel_epoch_ms = 0;
             public byte[] payload = null;
-            public PayloadMessage(int msg_id_args, Int64 send_epoch_args, int payload_size, byte payload_byte) {
+            public PayloadMessage(int msg_id_args, Int64 send_epoch_args, int payload_size, byte payload_byte)
+            {
                 msg_id = msg_id_args;
                 travel_epoch_ms = send_epoch_args;
                 payload = new byte[payload_size];
-                for (int i = 0; i < payload_size; i++) {
+                for (int i = 0; i < payload_size; i++)
+                {
                     payload[i] = payload_byte;
                 }
             }
             private PayloadMessage() { }
             public PayloadMessage(int msg_id, int payload_size, byte payload_byte) :
-            this(msg_id, PubSubTestingStream.epoch_now(), payload_size, payload_byte) { }
+            this(msg_id, PubSubTestingStream.epoch_now(), payload_size, payload_byte)
+            { }
 
-            public byte[] serialize() {
+            public byte[] serialize()
+            {
                 var str = new MemoryStream();
                 BinaryWriter bw = new BinaryWriter(str);
                 bw.Write(msg_id);
@@ -40,7 +47,8 @@ namespace bts {
                 return bytes;
             }
 
-            public static PayloadMessage deserialize(byte[] ser_bytes) {
+            public static PayloadMessage deserialize(byte[] ser_bytes)
+            {
                 var str = new MemoryStream(ser_bytes);
                 BinaryReader br = new BinaryReader(str);
                 int msg_id = br.ReadInt32();
@@ -48,7 +56,8 @@ namespace bts {
                 Int64 epoch = br.ReadInt64();
                 int len = br.ReadInt32();
                 var payload = br.ReadBytes(len);
-                return new PayloadMessage() {
+                return new PayloadMessage()
+                {
                     msg_id = msg_id,
                     guid = guid,
                     travel_epoch_ms = epoch,
@@ -73,7 +82,8 @@ namespace bts {
         private readonly int _payload_sz;
         private const byte _payload_byte = (byte)'A';
         private readonly string _tag_name;
-        public PubSubTestingStream(int publish_period_msecs, int max_pubs, int payload_size, string tag_name, ViewStreamDatabase vsd) {
+        public PubSubTestingStream(int publish_period_msecs, int max_pubs, int payload_size, string tag_name, ViewStreamDatabase vsd)
+        {
             _vsd = vsd;
             _payload_sz = payload_size;
             var schema = BlueDataUtility.form_schema("PubSubTest",
@@ -96,43 +106,53 @@ namespace bts {
         public PubSubTestingStream(int publish_period_seconds, int max_pubs, int msg_size, string tag_name)
         : this(publish_period_seconds, max_pubs, msg_size, tag_name, ViewStreamDatabase.instance) { }
 
-        protected override void myOnFirstSubscriber() {
+        protected override void myOnFirstSubscriber()
+        {
             _vsd.addWorker(this);
             System.Threading.Thread th = new System.Threading.Thread(_subscribe_thread_func);
             th.IsBackground = true;
             th.Start();
         }
 
-        protected override void myOnLastSubscriber() {
+        protected override void myOnLastSubscriber()
+        {
             _vsd.removeWorker(this);
             _continue_sub_thread = false;
         }
 
-        void _log_updates(IEnumerable<PubSubServerMessage> updates) {
+        void _log_updates(IEnumerable<PubSubServerMessage> updates)
+        {
             var idx = 0;
-            foreach (var msg in updates) {
+            foreach (var msg in updates)
+            {
                 bts.Log.Gen.info($"PubSubTesStream._subscribe_thread_func idx: {idx} msg: {msg}");
                 idx++;
             }
         }
 
-        private void _subscribe_thread_func() {
+        private void _subscribe_thread_func()
+        {
             // subscription is done in another thread
             SubscriberCore sub_core = new SubscriberCore();
             sub_core.subscribe(_tag_name);
-            while (_continue_sub_thread) {
+            while (_continue_sub_thread)
+            {
                 // TODO recv_id
                 var updates = sub_core.get_updates(TimeSpan.FromMilliseconds(250));
                 if (updates == null || updates.Count() == 0)
                     continue;
                 _log_updates(updates);
                 _n_updates += updates.Count();
-                if (updates.Count() > 0) {
+                if (updates.Count() > 0)
+                {
                     var server_msg = updates.Last();
-                    if (server_msg.data != null && server_msg.data.Length > 0) {
+                    if (server_msg.data != null && server_msg.data.Length > 0)
+                    {
                         _last_recvd_msg = PayloadMessage.deserialize(server_msg.data);
                         _last_recvd_msg.travel_epoch_ms = epoch_now();
-                    } else {
+                    }
+                    else
+                    {
                         // why are we getting empty or null data ? (server is sending data regardless?)
                     }
                     _fire_dirty = true;
@@ -140,29 +160,36 @@ namespace bts {
             }
         }
 
-        private void _publish_next_packet() {
-            if (_last_sent_msg == null) {
-                _last_sent_msg = new PayloadMessage(0, _payload_sz, _payload_byte) {
+        private void _publish_next_packet()
+        {
+            if (_last_sent_msg == null)
+            {
+                _last_sent_msg = new PayloadMessage(0, _payload_sz, _payload_byte)
+                {
                     guid = Guid.NewGuid().ToString()
                 };
-            } else {
+            }
+            else
+            {
                 _last_sent_msg.msg_id++;
                 _last_sent_msg.guid = Guid.NewGuid().ToString();
                 _last_sent_msg.travel_epoch_ms = epoch_now();
             }
             var ser_bytes = _last_sent_msg.serialize();
             _n_pubs++;
-            _pub_core.publish(String.Empty, _tag_name, ser_bytes);
+            _pub_core.publish("Blob", _tag_name, ser_bytes);
             _fire_dirty = true;
         }
 
-        private void _fire_dp() {
+        private void _fire_dp()
+        {
             object last_sent_id = _last_sent_msg == null ? (object)DBNull.Value : (object)_last_sent_msg.msg_id;
             object last_recv_id = _last_recvd_msg == null ? (object)DBNull.Value : (object)_last_recvd_msg.msg_id;
             object echo_time = DBNull.Value;
             object last_sent_guid = _last_sent_msg == null ? (object)DBNull.Value : (object)_last_sent_msg.guid;
             object last_recv_guid = _last_recvd_msg == null ? (object)DBNull.Value : (object)_last_recvd_msg.guid;
-            if (_last_sent_msg != null && _last_recvd_msg != null) {
+            if (_last_sent_msg != null && _last_recvd_msg != null)
+            {
                 echo_time = _last_recvd_msg.travel_epoch_ms - _last_sent_msg.travel_epoch_ms;
             }
             var match = last_sent_id.Equals(last_recv_id) && last_sent_guid.Equals(last_recv_guid);
@@ -174,21 +201,25 @@ namespace bts {
             myFireEvent(dp);
         }
 
-        public void doWork() {
+        public void doWork()
+        {
             if (DateTime.Now - _last_publish_time > _publish_period)
-                if (_n_pubs <= _max_pubs) {
+                if (_n_pubs <= _max_pubs)
+                {
                     _publish_next_packet();
                     _last_publish_time = DateTime.Now;
                 }
             // publish 
             // TODO: send id
-            if (_fire_dirty) {
+            if (_fire_dirty)
+            {
                 _fire_dirty = false;
                 _fire_dp();
             }
         }
 
-        public static long epoch_now() {
+        public static long epoch_now()
+        {
             return DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
         }
 

src/DataStreamSuite/DataStream/Primitives/Misc/PubsubEchoTestStream.cs

@@ -9,17 +9,22 @@ using System.Linq;
 using System.Runtime.CompilerServices;
 using bts.api;
 
-namespace bts {
+namespace bts
+{
    [DataStreamUserRespresentation("pss_echo")]
-   public class PubsubEchoTestStream : ViewStream, IWorker {
-      public class PayloadMessage {
+   public class PubsubEchoTestStream : ViewStream, IWorker
+   {
+      public class PayloadMessage
+      {
          public readonly string msg_id;
          public readonly Int64 timestamp = 0;
          public readonly byte[] payload = null;
 
-         public static byte[] create_data(int payload_size, byte payload_byte) {
+         public static byte[] create_data(int payload_size, byte payload_byte)
+         {
             var rv = new byte[payload_size];
-            for (int i = 0; i < payload_size; i++) {
+            for (int i = 0; i < payload_size; i++)
+            {
                rv[i] = payload_byte;
             }
             return rv;
@@ -35,14 +40,16 @@ namespace bts {
          //       payload[i] = payload_byte;
          //    }
          // }
-         public PayloadMessage(string guid_, Int64 timestamp_, byte[] payload_) {
+         public PayloadMessage(string guid_, Int64 timestamp_, byte[] payload_)
+         {
             msg_id = guid_;
             timestamp = timestamp_;
             payload = payload_;
          }
 
          private PayloadMessage() { }
-         public byte[] serialize() {
+         public byte[] serialize()
+         {
             var str = new MemoryStream();
             BinaryWriter bw = new BinaryWriter(str);
             bw.Write(msg_id);
@@ -54,7 +61,8 @@ namespace bts {
             return bytes;
          }
 
-         public static PayloadMessage deserialize(byte[] ser_bytes) {
+         public static PayloadMessage deserialize(byte[] ser_bytes)
+         {
             var str = new MemoryStream(ser_bytes);
             BinaryReader br = new BinaryReader(str);
             string guid = br.ReadString();
@@ -80,13 +88,15 @@ namespace bts {
       private readonly string _tag_name;
       private readonly int _payload_sz;
       private readonly IPubSubRouter _router;
-      private enum IOState {
+      private enum IOState
+      {
          SEND = 0,
          RECV = 1
       };
       IOState _state = IOState.SEND;
 
-      public PubsubEchoTestStream(string server_url, TimeSpan publish_period, int payload_size, string tag_name, ViewStreamDatabase vsd) {
+      public PubsubEchoTestStream(string server_url, TimeSpan publish_period, int payload_size, string tag_name, ViewStreamDatabase vsd)
+      {
          var route = new PubSubRoute(server_url, ".*");
          _router = new PubSubRouter(route);
          _vsd = vsd;
@@ -107,7 +117,8 @@ namespace bts {
       public PubsubEchoTestStream(string server_url, int publish_period_msecs, int payload_size, string tag_name)
       : this(server_url, TimeSpan.FromMilliseconds(publish_period_msecs), payload_size, tag_name, ViewStreamDatabase.instance) { }
 
-      protected override void myOnFirstSubscriber() {
+      protected override void myOnFirstSubscriber()
+      {
          _pub_core = new PublisherCore(_router);
          _sub_core = new SubscriberCore(_router);
          _sub_core.subscribe(_tag_name);
@@ -115,12 +126,14 @@ namespace bts {
 
       }
 
-      protected override void myOnLastSubscriber() {
+      protected override void myOnLastSubscriber()
+      {
          _vsd.removeWorker(this);
          _sub_core.unsubscribe(_tag_name);
       }
 
-      private void _fire_dp_after_send() {
+      private void _fire_dp_after_send()
+      {
          // as soon as the data is published, its reflected in the stream state
          Log.assert(_last_sent_msg != null);
          BlueRow r = new BlueRow(_table.schema, _table.Count, _last_sent_msg.msg_id, _last_sent_msg.timestamp, DBNull.Value, DBNull.Value,
@@ -130,7 +143,8 @@ namespace bts {
          myFireEvent(dp_out);
       }
 
-      private void _fire_dp_after_recv() {
+      private void _fire_dp_after_recv()
+      {
          Log.assert(_last_recvd_msg != null);
          // update the last publish row
          int id = _table.Count - 1;
@@ -144,11 +158,13 @@ namespace bts {
          myFireEvent(dp_out);
       }
 
-      private string _get_next_id() {
+      private string _get_next_id()
+      {
          return _table.Count.ToString();
       }
 
-      private void _publish_next_packet() {
+      private void _publish_next_packet()
+      {
          var bytes = PayloadMessage.create_data(_payload_sz, _payload_byte);
          _last_sent_msg = new PayloadMessage(_get_next_id(), epoch_now(), bytes);
          var ser_bytes = _last_sent_msg.serialize();
@@ -157,19 +173,25 @@ namespace bts {
          _last_publish_time = DateTime.Now;
       }
 
-      public void doWork() {
-         Func<IEnumerable<PubSubServerMessage>, PayloadMessage> find_packet = (updates) => {
-            foreach (var u in updates) {
+      public void doWork()
+      {
+         Func<IEnumerable<PubSubServerMessage>, PayloadMessage> find_packet = (updates) =>
+         {
+            foreach (var u in updates)
+            {
                var recv_msg = PayloadMessage.deserialize(u.data);
-               if (recv_msg.msg_id == _last_sent_msg.msg_id) {
+               if (recv_msg.msg_id == _last_sent_msg.msg_id)
+               {
                   return recv_msg;
                }
             }
             return null;
          };
-         switch (_state) {
+         switch (_state)
+         {
             case IOState.SEND:
-               if (DateTime.Now - _last_publish_time >= _publish_period) {
+               if (DateTime.Now - _last_publish_time >= _publish_period)
+               {
                   _publish_next_packet();
                   _last_publish_time = DateTime.Now;
                   _fire_dp_after_send();
@@ -181,7 +203,8 @@ namespace bts {
                var updates = _sub_core.get_updates(TimeSpan.Zero);
                updates = updates.Where(u => u.tag == _tag_name);
                var recv_msg = find_packet(updates);
-               if (recv_msg != null) {
+               if (recv_msg != null)
+               {
                   _last_recvd_msg = recv_msg;
                   _fire_dp_after_recv();
                   _state = IOState.SEND;
@@ -191,7 +214,8 @@ namespace bts {
          }
       }
 
-      public static long epoch_now() {
+      public static long epoch_now()
+      {
          return DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
       }
 

src/DataStreamSuite/DataStream/Primitives/Misc/TimeSeriesStream.cs

@@ -1,9 +1,8 @@
-using Google.Protobuf.WellKnownTypes;
-
 using System;
-using BCG.Utilities.General;
 
-namespace BCG {
+using BCG;
+
+namespace bts.SilverPine {
    // Keeps adding changing rows to the stream, hence preserving the history
    // and creating a time series, for example this can be used to make bbo updates
    // into a time series of prices/qty 
@@ -41,7 +40,6 @@ namespace BCG {
          }
          _table.addRows(added_rows);
          myFireEvent(new DataPacket(_table, added_rows, new BlueTable(_table.schema)));
-
       }
    }
 

src/DataStreamSuite/DataStream/Primitives/MongoDB/MongoDBStream.cs

@@ -7,122 +7,127 @@ using MongoDB.Driver;
 using System;
 using System.Collections.Generic;
 using System.Diagnostics;
+using System.Threading.Tasks;
+using System.Threading;
 
 namespace bts {
-    [DataStreamUserRespresentation("mongodb_sql")]
-    public class MongoDBSQLStream : ViewStream, IWorker {
-        private ViewStreamDatabase _vsd;
-        private readonly string _connection_string, _db_name, _collection_name;
-        private MongoClient _client;
-        // make this a sorted table
-        private BlueTable _table;
+    // [DataStreamUserRespresentation("mongodb_sql")]
+    // public class MongoDBSQLStream : ViewStream, IWorker {
+    //     private ViewStreamDatabase _vsd;
+    //     private readonly string _connection_string, _db_name, _collection_name;
+    //     private MongoClient _client;
+    //     // make this a sorted table
+    //     private BlueTable _table;
 
 
-        // Access the database
-        private IMongoDatabase _database;
-        private IDictionary<string, int> _id_map = new Dictionary<string, int>();
-        // mongodb://192.168.1.80:27017
+    //     // Access the database
+    //     private IMongoDatabase _database;
+    //     private IDictionary<string, int> _id_map = new Dictionary<string, int>();
+    //     // mongodb://192.168.1.80:27017
 
-        public MongoDBSQLStream(string connection_string, string db_name, string collection_name)
-        : this(connection_string, db_name, collection_name, ViewStreamDatabase.instance) {
+    //     public MongoDBSQLStream(string connection_string, string db_name, string collection_name)
+    //     : this(connection_string, db_name, collection_name, ViewStreamDatabase.instance) {
 
-        }
-        public MongoDBSQLStream(string connection_string, string db_name, string collection_name, ViewStreamDatabase vsd) {
-            bts.Log.Gen.info($"MongoDBStream constructed connection_string: {connection_string} " +
-                                    $"db_name: {db_name}");
-            _connection_string = connection_string;
-            _db_name = db_name;
-            _collection_name = collection_name;
-            _vsd = vsd;
-            _client = new MongoClient(_connection_string);
-            _database = _client.GetDatabase(_db_name);
-        }
+    //     }
+    //     public MongoDBSQLStream(string connection_string, string db_name, string collection_name, ViewStreamDatabase vsd) {
+    //         bts.Log.Gen.info($"MongoDBStream constructed connection_string: {connection_string} " +
+    //                                 $"db_name: {db_name}");
+    //         _connection_string = connection_string;
+    //         _db_name = db_name;
+    //         _collection_name = collection_name;
+    //         _vsd = vsd;
+    //         _client = new MongoClient(_connection_string);
+    //         _database = _client.GetDatabase(_db_name);
+    //     }
 
-        protected override void myOnFirstSubscriber() {
-            MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(10));
-        }
+    //     protected override void myOnFirstSubscriber() {
+    //         MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(10));
+    //     }
 
-        protected override void myOnLastSubscriber() {
-            MasterWorker.getInstance(false).removeWorker(this);
-        }
+    //     protected override void myOnLastSubscriber() {
+    //         MasterWorker.getInstance(false).removeWorker(this);
+    //     }
 
-        public void doWork() {
-            _read_mogodb();
-        }
+    //     public void doWork() {
+    //         _read_mogodb();
+    //     }
 
-        private void _read_mogodb() {
-            var timer = Stopwatch.StartNew();
-            // Access the collection
-            IMongoCollection<BsonDocument> collection = _database.GetCollection<BsonDocument>(name: _collection_name);
+    //     private void _read_mogodb() {
+    //         var timer = Stopwatch.StartNew();
+    //         // Access the collection
+    //         IMongoCollection<BsonDocument> collection = _database.GetCollection<BsonDocument>(name: _collection_name);
 
-            // Fetch all documents from the collection
-            var documents = collection.Find(Builders<BsonDocument>.Filter.Empty).ToCursor();
-            _table = _table == null ? null : new BlueTable(_table.schema);
-            // Iterate through the documents and print values
-            foreach (var document in documents.ToEnumerable()) {
-                bts.Log.Gen.info($"document: {document} json: {document.ToJson()}");
-                if (_table == null) {
-                    _init_table(document);
-                }
-                _add_row(document);
-            }
-            timer.Stop();
-            Log.Gen.info($"MongoDBStream time to read {_collection_name} {timer.ElapsedMilliseconds}ms");
-            timer.Restart();
-            var dp = BlueDataUtility.build_dp_inc(_table, myLastValue == null ? null : myLastValue as DataPacket, true);
-            timer.Stop();
-            Log.Gen.info($"MongoDBStream time to compare tables :{timer.ElapsedMilliseconds}ms");
-            if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
-                return;
-            myFireEvent(dp);
-        }
+    //         // Fetch all documents from the collection
+    //         var documents = collection.Find(Builders<BsonDocument>.Filter.Empty).ToCursor();
+    //         _table = _table == null ? null : new BlueTable(_table.schema);
+    //         // Iterate through the documents and print values
+    //         foreach (var document in documents.ToEnumerable()) {
+    //             bts.Log.Gen.info($"document: {document} json: {document.ToJson()}");
+    //             if (_table == null) {
+    //                 _init_table(document);
+    //             }
+    //             _add_row(document);
+    //         }
+    //         timer.Stop();
+    //         Log.Gen.info($"MongoDBStream time to read {_collection_name} {timer.ElapsedMilliseconds}ms");
+    //         timer.Restart();
+    //         var dp = BlueDataUtility.build_dp_inc(_table, myLastValue == null ? null : myLastValue as DataPacket, true);
+    //         timer.Stop();
+    //         Log.Gen.info($"MongoDBStream time to compare tables :{timer.ElapsedMilliseconds}ms");
+    //         if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
+    //             return;
+    //         myFireEvent(dp);
+    //     }
 
-        private void _init_table(BsonDocument doc_row) {
-            bts.Log.assert(_table == null);
-            List<ColumnInfo> cis = new List<ColumnInfo>();
-            cis.Add((new ColumnInfo("id", 0, typeof(Int32))));
-            foreach (var n in doc_row.Names) {
-                cis.Add(new ColumnInfo(n, cis.Count, typeof(string)));
-            }
-            IBlueTableSchema schema = new BlueTableSchema(cis.ToArray(), _collection_name);
-            _table = new BlueTable(schema);
-        }
+    //     private void _init_table(BsonDocument doc_row) {
+    //         bts.Log.assert(_table == null);
+    //         List<ColumnInfo> cis = new List<ColumnInfo>();
+    //         cis.Add((new ColumnInfo("id", 0, typeof(Int32))));
+    //         foreach (var n in doc_row.Names) {
+    //             cis.Add(new ColumnInfo(n, cis.Count, typeof(string)));
+    //         }
+    //         IBlueTableSchema schema = new BlueTableSchema(cis.ToArray(), _collection_name);
+    //         _table = new BlueTable(schema);
+    //     }
 
-        private void _add_row(BsonDocument doc) {
-            var key_val = doc["_id"];
-            var key = key_val.ToString();
-            int id;
-            if (!_id_map.TryGetValue(key, out id)) {
-                id = _id_map.Count;
-                _id_map.Add(key, id);
-            }
-            MutableBlueRow nr = new MutableBlueRow(_table.schema);
-            nr.setValue("id", id);
-            foreach (var n in doc.Names) {
-                var val = doc[n].ToString();
-                nr.setValue(n, val);
-            }
-            _table.addRow(nr);
-        }
-    }
+    //     private void _add_row(BsonDocument doc) {
+    //         var key_val = doc["_id"];
+    //         var key = key_val.ToString();
+    //         int id;
+    //         if (!_id_map.TryGetValue(key, out id)) {
+    //             id = _id_map.Count;
+    //             _id_map.Add(key, id);
+    //         }
+    //         MutableBlueRow nr = new MutableBlueRow(_table.schema);
+    //         nr.setValue("id", id);
+    //         foreach (var n in doc.Names) {
+    //             var val = doc[n].ToString();
+    //             nr.setValue(n, val);
+    //         }
+    //         _table.addRow(nr);
+    //     }
+    // }
 
     [DataStreamUserRespresentation("mongodb")]
     public class MongoDBStream : ViewStream, IWorker {
         private ViewStreamDatabase _vsd;
         private readonly string _connection_string, _db_name, _collection_name;
+        private readonly TimeSpan _poll_period;
         private MongoClient _client;
         private BlueTable _table;
+        private SyncQueue<DataPacket> _table_queue = new SyncQueue<DataPacket>();
+        private BlockingQueue _work_queue = new BlockingQueue();
 
         // Access the database
         private IMongoDatabase _database;
         private IDictionary<string, int> _id_map = new Dictionary<string, int>();
         // mongodb://192.168.1.80:27017
 
-        public MongoDBStream(string connection_string, string db_name, string collection_name)
-        : this(connection_string, db_name, collection_name, ViewStreamDatabase.instance) {
+        public MongoDBStream(string connection_string, string db_name, string collection_name, string poll_period)
+        : this(connection_string, db_name, collection_name, TimeBasedFilter.parse_duration(poll_period), ViewStreamDatabase.instance) {
 
         }
-        public MongoDBStream(string connection_string, string db_name, string collection_name, ViewStreamDatabase vsd) {
+        public MongoDBStream(string connection_string, string db_name, string collection_name, TimeSpan poll_period, ViewStreamDatabase vsd) {
             bts.Log.Gen.info($"MongoDBStream constructed connection_string: {connection_string} " +
                                     $"db_name: {db_name}");
             _connection_string = connection_string;
@@ -130,19 +135,30 @@ namespace bts {
             _collection_name = collection_name;
             _vsd = vsd;
             _client = new MongoClient(_connection_string);
+            _poll_period = poll_period;
             _database = _client.GetDatabase(_db_name);
         }
 
         protected override void myOnFirstSubscriber() {
-            MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(10));
+            _vsd.addWorker(this);
+            //MasterWorker.getInstance(false).addWorker(this, _poll_period);
         }
 
         protected override void myOnLastSubscriber() {
-            MasterWorker.getInstance(false).removeWorker(this);
+            _vsd.removeWorker(this);
+            //MasterWorker.getInstance(false).removeWorker(this);
         }
 
+        private DateTime _last_poll_time = DateTime.MinValue;
         public void doWork() {
-            _read_mogodb();
+            var now = DateTime.Now;
+            if (now - _last_poll_time > _poll_period) {
+                ThreadPool.QueueUserWorkItem(_ => _read_mogodb());
+                _last_poll_time = DateTime.Now;
+            }
+            var dp = _table_queue.dequeue(0);
+            if (dp != null)
+                myFireEvent(dp);
         }
 
         private void _read_mogodb() {
@@ -166,10 +182,11 @@ namespace bts {
             timer.Restart();
             var dp = BlueDataUtility.build_dp_inc(_table, myLastValue == null ? null : myLastValue as DataPacket, true);
             timer.Stop();
-            Log.Gen.info($"MongoDBStream time to compare tables: {timer.ElapsedMilliseconds}ms");;
+            Log.Gen.info($"MongoDBStream time to compare tables: {timer.ElapsedMilliseconds}ms"); ;
             if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
                 return;
-            myFireEvent(dp);
+            _table_queue.enqueue(dp);
+            //myFireEvent(dp);
         }
 
         private void _init_table(BsonDocument doc_row) {
@@ -208,46 +225,46 @@ namespace bts {
         }
     }
 
-    [DataStreamUserRespresentation("mongodb_flat")]
-    public class MongoDBFlatStream : ViewStream {
-        public MongoDBFlatStream(string connection_string, string db_name, string collection_name, string timestamp_col, string ttl_duration)
-        :
-        this(ViewStreamDatabase.instance, connection_string, db_name, collection_name, timestamp_col, TimeBasedFilter.parse_duration(ttl_duration)) { }
+    // [DataStreamUserRespresentation("mongodb_flat")]
+    // public class MongoDBFlatStream : ViewStream {
+    //     public MongoDBFlatStream(string connection_string, string db_name, string collection_name, string timestamp_col, string ttl_duration)
+    //     :
+    //     this(ViewStreamDatabase.instance, connection_string, db_name, collection_name, timestamp_col, TimeBasedFilter.parse_duration(ttl_duration)) { }
 
-        public MongoDBFlatStream(ViewStreamDatabase vsd, string connection_string, string db_name, string collection_name, string timestamp_col, TimeSpan ttl_duration) {
-            ViewStream rs = new MongoDBStream(connection_string, db_name, collection_name, vsd);
-            rs = new KeyValueToRelationalStream(rs, "doc_id", "key", "val");
-            rs = new DateStandardizationStream(rs, timestamp_col, DateConversion.ISO8601);
-            //rs = new TimeBasedFilterStream(vsd, rs, timestamp_col, ttl_duration);
-            mySetRealSource(rs);
-        }
-    }
+    //     public MongoDBFlatStream(ViewStreamDatabase vsd, string connection_string, string db_name, string collection_name, string timestamp_col, TimeSpan ttl_duration) {
+    //         ViewStream rs = new MongoDBStream(connection_string, db_name, collection_name, vsd);
+    //         rs = new KeyValueToRelationalStream(rs, "doc_id", "key", "val");
+    //         rs = new DateStandardizationStream(rs, timestamp_col, DateConversion.ISO8601);
+    //         //rs = new TimeBasedFilterStream(vsd, rs, timestamp_col, ttl_duration);
+    //         mySetRealSource(rs);
+    //     }
+    // }
 
 
 
-    [DataStreamUserRespresentation("newsreplay")]
-    public class NewsReplayStream : ViewStream {
-        public NewsReplayStream(string connection_string, string db_name, string collection_name,
-                                int buffer_sz, string fire_period) :
-        this(ViewStreamDatabase.instance, connection_string, db_name, collection_name, buffer_sz, TimeBasedFilter.parse_duration(fire_period)) { }
+    // [DataStreamUserRespresentation("newsreplay")]
+    // public class NewsReplayStream : ViewStream {
+    //     public NewsReplayStream(string connection_string, string db_name, string collection_name,
+    //                             int buffer_sz, string fire_period) :
+    //     this(ViewStreamDatabase.instance, connection_string, db_name, collection_name, buffer_sz, TimeBasedFilter.parse_duration(fire_period)) { }
 
-        public NewsReplayStream(ViewStreamDatabase vsd, string connection_string, string db_name, string collection_name,
-                                int buffer_sz, TimeSpan fire_period) {
-            ViewStream r;
+    //     public NewsReplayStream(ViewStreamDatabase vsd, string connection_string, string db_name, string collection_name,
+    //                             int buffer_sz, TimeSpan fire_period) {
+    //         ViewStream r;
 
-            r = new MongoDBStream(connection_string, db_name, collection_name, vsd);
-            r = new KeyValueToRelationalStream(r,
-                                                "doc_id",
-                                                "key",
-                                                "value");
-            r = new DateStandardizationStream(r,
-                                               "created_at",
-                                               "ISO8601");
-            r = new SnapshotReplayStream(vsd, r, "created_at", buffer_sz, fire_period);
+    //         r = new MongoDBStream(connection_string, db_name, collection_name, vsd);
+    //         r = new KeyValueToRelationalStream(r,
+    //                                             "doc_id",
+    //                                             "key",
+    //                                             "value");
+    //         r = new DateStandardizationStream(r,
+    //                                            "created_at",
+    //                                            "ISO8601");
+    //         r = new SnapshotReplayStream(vsd, r, "created_at", buffer_sz, fire_period);
 
-            mySetRealSource(r);
-        }
-    }
+    //         mySetRealSource(r);
+    //     }
+    // }
 
 
 }

src/DataStreamSuite/DataStream/Primitives/Project/New/NewProjectStream.cs

@@ -6,6 +6,7 @@ namespace bts {
    using System;
    using System.Collections;
    using System.Collections.Generic;
+   using System.Linq;
 
    public class ProjectData {
       // schema from the base stream
@@ -136,10 +137,24 @@ namespace bts {
          HashSet<string> removed_cols = new HashSet<string>();
          Dictionary<string, string> mapped_to_orig_map = new Dictionary<string, string>();
          Dictionary<string, string> orig_to_mapped = new Dictionary<string, string>();
+         // by default every col will show up, so if we have the projection as (a:a1)
+         // it will convert a to a1, 
+         // examples: if a schema has cols a b c
+         // project(parent_str, "a:aa ") -> aa b c
+         // project(parent_str, "a:aa !c") -> aa b
+         // project(parent_str, "a:aa !") -> aa
+         // project(parent_str, "a:aa b = (aa*aa)!") -> aa b
+         bool hide_all = false;
          foreach (string p_tok in project_toks) {
             if (p_tok.StartsWith("!")) {
-               string removed_col = p_tok.Substring(1);
-               removed_cols.Add(removed_col);
+               if (p_tok.Length == 1) {
+                  // if it is a "blanket "!"=> hide all cols except the one that
+                  // are stated explicitly
+                  hide_all = true;
+               } else {
+                  string removed_col = p_tok.Substring(1);
+                  removed_cols.Add(removed_col);
+               }
             } else if (p_tok.Contains(":")) {
                string[] map_toks = p_tok.Split(":");
                string orig_col = map_toks[0];
@@ -147,12 +162,22 @@ namespace bts {
                mapped_to_orig_map.Add(mapped_col, orig_col);
                orig_to_mapped.Add(orig_col, mapped_col);
             } else {
-               Log.Gen.error($"NewProjectStream unexpecetd token: {p_tok}");
+               if (input_schema.columnNames.Contains(p_tok)) {
+                  string orig_col = p_tok;
+                  string mapped_col = orig_col;
+                  orig_to_mapped.Add(orig_col, mapped_col);
+               }
             }
          }
+
          // make the new schema
          List<ColumnInfo> new_cols = new List<ColumnInfo>();
          foreach (ColumnInfo ci in input_schema.columnInfo) {
+            if (ci.colName == "id") {
+               ColumnInfo nc = new ColumnInfo(ci.colName, new_cols.Count, ci.colType);
+               new_cols.Add(nc);
+               continue;
+            }
             if (removed_cols.Contains(ci.colName)) {
                continue;
             } else if (orig_to_mapped.ContainsKey(ci.colName)) {
@@ -160,8 +185,13 @@ namespace bts {
                ColumnInfo nc = new ColumnInfo(mapped_name, new_cols.Count, ci.colType);
                new_cols.Add(nc);
             } else {
+               if (hide_all)
+                  continue;
+               //if (ci.colName == "id" || removed_cols.Count > 0) {
+               // unless columns are  explicitely hidden, all columns show up
                ColumnInfo nc = new ColumnInfo(ci.colName, new_cols.Count, ci.colType);
                new_cols.Add(nc);
+               //}
             }
          }
          IBlueTableSchema new_schema = new BlueTableSchema(input_schema.tableName, new_cols.ToArray());

src/DataStreamSuite/DataStream/Primitives/SQL/SQLite/SQLiteHelper.cs

@@ -0,0 +1,122 @@
+using System;
+using System.Data;
+using System.Data.SQLite;
+using System.Diagnostics;
+using System.Collections;
+
+
+using BCG;
+using System.Linq;
+using System.Data.Entity;
+using System.Collections.Generic;
+using System.Data.SQLite;
+
+
+namespace bts.SilverPine {
+
+   public static class SQLiteHelper {
+      public static string location = "C:\\Users\\m_tah\\development\\data\\";
+      public static string db_name = "stream_data.db";
+
+      public static DataTable execute_select_query(string connection_string, string query) {
+         // Create a new DataTable to store the results
+         DataTable dataTable = new DataTable();
+
+         // Create and open a SQLite connection using the provided connection string
+         using (SQLiteConnection connection = new SQLiteConnection(connection_string)) {
+            try {
+               connection.Open();
+
+               // Create a SQLite command with the provided query
+               using (SQLiteCommand command = new SQLiteCommand(query, connection)) {
+                  // Use SQLiteDataAdapter to fill the DataTable
+                  using (SQLiteDataAdapter adapter = new SQLiteDataAdapter(command)) {
+                     adapter.Fill(dataTable);
+                  }
+               }
+            } catch (Exception ex) {
+               Console.WriteLine("An error occurred: " + ex.Message);
+            }
+         }
+         // Return the populated DataTable
+         return dataTable;
+      }
+
+      public static List<Dictionary<string, object>> ExecuteBulkRead(string connectionString, string query) {
+         var result = new List<Dictionary<string, object>>();
+
+         using (var connection = new SQLiteConnection(connectionString)) {
+            connection.Open();
+
+            // Begin a transaction to optimize read speed
+            using (var transaction = connection.BeginTransaction())
+            using (var command = new SQLiteCommand(query, connection, transaction))
+            using (var reader = command.ExecuteReader()) {
+               while (reader.Read()) {
+                  var row = new Dictionary<string, object>();
+
+                  for (int i = 0; i < reader.FieldCount; i++) {
+                     string columnName = reader.GetName(i);
+                     object columnValue = reader.IsDBNull(i) ? null : reader.GetValue(i);
+                     row[columnName] = columnValue;
+                  }
+
+                  result.Add(row);
+               }
+
+               // Commit the transaction
+               transaction.Commit();
+            }
+         }
+
+         return result;
+      }
+
+      public static List<Dictionary<string, object>> get_table(string connectionString, string query) {
+         var result = new List<Dictionary<string, object>>();
+
+         using (var connection = new SQLiteConnection(connectionString)) {
+            try {
+               connection.Open();
+
+               using (var command = new SQLiteCommand(query, connection))
+               using (var reader = command.ExecuteReader()) {
+                  while (reader.Read()) {
+                     var row = new Dictionary<string, object>();
+
+                     for (int i = 0; i < reader.FieldCount; i++) {
+                        string columnName = reader.GetName(i);
+                        object columnValue = reader.IsDBNull(i) ? null : reader.GetValue(i);
+                        row[columnName] = columnValue;
+                     }
+
+                     result.Add(row);
+                  }
+               }
+            } catch (Exception ex) {
+               Console.WriteLine("An error occurred: " + ex.Message);
+            }
+         }
+         return result;
+      }
+
+      public static IEnumerable<Dictionary<string, object>> get_table_2(string connectionString, string query) {
+         using (var connection = new SQLiteConnection(connectionString)) {
+            connection.Open();
+            using (var command = new SQLiteCommand(query, connection))
+            using (var reader = command.ExecuteReader()) {
+               while (reader.Read()) {
+                  var row = new Dictionary<string, object>();
+                  for (int i = 0; i < reader.FieldCount; i++) {
+                     string columnName = reader.GetName(i);
+                     object columnValue = reader.IsDBNull(i) ? null : reader.GetValue(i);
+                     row[columnName] = columnValue;
+                  }
+                  yield return row;
+               }
+            }
+         }
+      }
+   }
+}
+

src/DataStreamSuite/DataStream/Primitives/SQL/SQLite/SqliteStream.cs

@@ -0,0 +1,73 @@
+namespace bts.SilverPine {
+   using System;
+   using System.Data;
+   using System.Data.SQLite;
+   using System.Diagnostics;
+   using System.Collections;
+
+
+   using BCG;
+
+   [DataStreamUserRespresentation("SQLite")]
+   public class SQLiteStream : ViewStream {
+      private readonly string _connection_string, _query;
+      private Stopwatch _stopwatch = new Stopwatch();
+
+      private SQLiteStream(string connection_string, string query, bool noop) {
+         _connection_string = connection_string;
+         _query = query;
+      }
+
+      public SQLiteStream(string db_name, string query)
+      :
+      this($"Data Source={SQLiteHelper.location}{db_name};Version=3", query, true) { }
+
+      public SQLiteStream(string query)
+      :
+      this(SQLiteHelper.db_name, query) {
+      }
+
+
+      private void _read_from_sqlite_and_fire() {
+         _stopwatch.Start();
+         DataTable dt = SQLiteHelper.execute_select_query(_connection_string, _query);
+         _stopwatch.Stop();
+         Log.Gen.info($"SQLiteStream.ExecuteSelectQuery time to read db: {_stopwatch.ElapsedMilliseconds} msecs");
+         _stopwatch.Restart();
+         #region perf measures
+         // SQLiteHelper.get_table(_connection_string, _query);
+         // _stopwatch.Stop();
+         // Log.Gen.info($"SQLiteStream.get_table time to read db: {_stopwatch.ElapsedMilliseconds} msecs");
+         // _stopwatch.Restart();
+         // SQLiteHelper.ExecuteBulkRead(_connection_string, _query);
+         // _stopwatch.Stop();
+         // Log.Gen.info($"SQLiteStream.ExecuteBulkRead time to read db: {_stopwatch.ElapsedMilliseconds} msecs");
+         // _stopwatch.Restart();
+         // SQLiteHelper.get_table_2(_connection_string, _query);
+         // _stopwatch.Stop();
+         // Log.Gen.info($"SQLiteStream.get_table_2 time to read db: {_stopwatch.ElapsedMilliseconds} msecs");
+         // _stopwatch.Restart();
+         #endregion
+         IBlueRowCollection rows = BCG.BlueDataUtility.toIBlueRowCollection(dt);
+         //IBlueRowCollection rows = new DataTableWrapperTable(dt);
+         _stopwatch.Stop();
+         Log.Gen.info($"SQLiteStream._read_from_sqlite_and_fire time to load into IBlueRowCollection: {_stopwatch.ElapsedMilliseconds} msecs");
+         DataPacket dp = BlueDataUtility.build_dp_inc(rows, myLastValue as DataPacket);
+         myFireEvent(dp);
+      }
+
+      protected override void myOnFirstSubscriber() {
+         _read_from_sqlite_and_fire();
+      }
+
+      protected override void myOnLastSubscriber() {
+         // this value will be cleared by the base class
+         // but in this particular stream this behaviour
+         // is not usefull since any new subscribers 
+         // would force the query to be recreated
+         //myLastDP = myLastValue;
+      }
+
+   }
+
+}

src/DataStreamSuite/DataStream/Primitives/SideEffectStreams/CSVSaverStream.cs

@@ -3,12 +3,15 @@ using System.Collections.Generic;
 using System.Text;
 using System.IO;
 using BCG.Utilities;
+using System.Diagnostics;
+using Microsoft.Data.Analysis;
+using Parquet;
+using Parquet.Data;
 
-namespace BCG
-{
+
+namespace BCG {
    [DataStreamUserRespresentation("CSVSaver")]
-   public class CSVSaverStream : ViewStream, ISideEffectStream
-   {
+   public class CSVSaverStream : ViewStream, ISideEffectStream {
       private IBlueTableSchema myResultSchema = new BlueTableSchema("CSVResult", new ColumnInfo("id", 0, typeof(int)),
          new ColumnInfo("fileName", 1, typeof(string)),
          new ColumnInfo("nWrites", 2, typeof(int)),
@@ -20,11 +23,9 @@ namespace BCG
       private int myNWrites = 0, myNMaxWrites = Int32.MaxValue;
 
       public CSVSaverStream(ViewStream vs, string fullyQualifiedFileName)
-         : this(vs, fullyQualifiedFileName, true, Int32.MaxValue, false)
-      { }
+         : this(vs, fullyQualifiedFileName, true, Int32.MaxValue, false) { }
 
-      public CSVSaverStream(ViewStream vs, string fullyQualifiedFileName, bool overwrite, int nMaxWrites, bool fireResultSnapshot)
-      {
+      public CSVSaverStream(ViewStream vs, string fullyQualifiedFileName, bool overwrite, int nMaxWrites, bool fireResultSnapshot) {
          myStream = vs;
          myFullyQualifiedFileName = fullyQualifiedFileName;
          myOverwrite = overwrite;
@@ -36,18 +37,14 @@ namespace BCG
 
       protected override void myOnLastSubscriber() { myStream.onChange -= new ChangeDelegate(myHandleChanges); }
 
-      private void myHandleChanges(object newValue)
-      {
+      private void myHandleChanges(object newValue) {
          IBlueRowCollection totalState = (newValue as DataPacket).newRows;
-         if (totalState.Count > 0 && myNWrites < myNMaxWrites)
-         {
+         if (totalState.Count > 0 && myNWrites < myNMaxWrites) {
             myNWrites++;
             int i = 0;
             StreamWriter writer = new StreamWriter(new FileStream(myGetNewFileName(), FileMode.Create, FileAccess.Write));
-            foreach (IBlueRow r in totalState)
-            {
-               if (i++ == 0)
-               {
+            foreach (IBlueRow r in totalState) {
+               if (i++ == 0) {
                   string header = String.Empty;
                   foreach (string col in r.schema.columnNames)
                      if (col != "id")
@@ -59,20 +56,17 @@ namespace BCG
                writer.Flush();
             }
             writer.Close();
-            if (myFireResultSnapshot)
-            {
+            if (myFireResultSnapshot) {
                IBlueRow nr = new BlueRow(myResultSchema, 0, myFullyQualifiedFileName, myNWrites, totalState.Count);
                BlueTable nRows = new BlueTable(myResultSchema);
                nRows.addRow(nr);
                myFireEvent(new DataPacket(nRows, nRows, new BlueTable(myLastValue == null ? new BlueTable(myResultSchema) : (myLastValue as DataPacket).newRows)));
-            }
-            else
+            } else
                myFireEvent(newValue);
          }
       }
 
-      private string myGetNewFileName()
-      {
+      private string myGetNewFileName() {
          string[] tags = myFullyQualifiedFileName.Split('.');
          if (myOverwrite)
             return tags[0] + ".csv";
@@ -80,14 +74,11 @@ namespace BCG
             return tags[0] + "_" + myFormateDateTime(DateTime.Now) + ".csv";
       }
 
-      private static string myGetCSVRow(IBlueRow r)
-      {
+      private static string myGetCSVRow(IBlueRow r) {
          string rv = String.Empty;
          int i = 0;
-         foreach (string col in r.schema.columnNames)
-         {
-            if (col != "id")
-            {
+         foreach (string col in r.schema.columnNames) {
+            if (col != "id") {
                string colText = String.Empty;
                if (r[col] != null && r[col] != DBNull.Value)
                   colText = r[col].ToString();
@@ -99,8 +90,7 @@ namespace BCG
          return rv;
       }
 
-      private string myFormateDateTime(DateTime dt)
-      {
+      private string myFormateDateTime(DateTime dt) {
          return
          string.Format("{0}-{1}-{2}-{3}-{4}-{5}-{6}",
                         dt.Year.ToString("0000"),
@@ -114,52 +104,49 @@ namespace BCG
    }
 
    [DataStreamUserRespresentation("CSVReader")]
-   public class CSVReaderStream : ViewStream
-   {
-      private IBlueRowCollection myRows;
-
-      public CSVReaderStream(string fileAddress)
-      {
-         myRows = BlueDataUtility.createRowsFromCSVFileWithoutTypes(fileAddress);
+   public class CSVReaderStream : ViewStream {
+      private readonly string _file_address;
+      Stopwatch _stopwatch = new Stopwatch();
+      public CSVReaderStream(string fileAddress) {
+         _file_address = fileAddress;
       }
 
-      protected override void myOnFirstSubscriber()
-      {
-         myFireEvent(new DataPacket(myRows, myRows, new BlueTable(myRows.schema)));
+      protected override void myOnFirstSubscriber() {
+         _stopwatch.Start();
+         DataFrame df = DataFrame.LoadCsv(_file_address);
+         _stopwatch.Stop();
+         bts.Log.Gen.info($"CSVReaderStream.myOnFirstSubscriber stream csv load col_count: {df.Columns.Count} rows, time: {_stopwatch.ElapsedMilliseconds} msecs");
+         _stopwatch.Start();
+         IBlueRowCollection rows = BlueDataUtility.createRowsFromCSVFileWithoutTypes(_file_address);
+         _stopwatch.Stop();
+         bts.Log.Gen.info($"CSVReaderStream.myOnFirstSubscriber stream csv load count: {rows.Count} rows, time: {_stopwatch.ElapsedMilliseconds} msecs");
+         DataPacket dp_out = BlueDataUtility.build_dp_inc(rows, myLastValue as DataPacket);
+         myFireEvent(dp_out);
       }
    }
 
-   [DataStreamUserRespresentation ("CSVReaderStreamMapper")]
-   public class CSVReaderStreamMapperStream : ViewStream
-   {
-      public CSVReaderStreamMapperStream(ViewStream stream, string output_schema)
-      {
+   [DataStreamUserRespresentation("CSVReaderStreamMapper")]
+   public class CSVReaderStreamMapperStream : ViewStream {
+      public CSVReaderStreamMapperStream(ViewStream stream, string output_schema) {
          mySetRealSource(new MapStream(stream, new CSVReaderStreamMapper(output_schema)));
       }
    }
 
-   public class CSVReaderStreamMapper : IStreamMapper
-   {
+   public class CSVReaderStreamMapper : IStreamMapper {
       private IBlueTableSchema mySchema;
       private readonly string[] myOutputSchemaStringArray;
 
-      public CSVReaderStreamMapper(string outputSchema)
-      {
+      public CSVReaderStreamMapper(string outputSchema) {
          myOutputSchemaStringArray = outputSchema.Split(' ');
       }
 
-      public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema)
-      {
-         if (mySchema == null)
-         {
+      public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema) {
+         if (mySchema == null) {
             ColumnInfo[] ci = inputSchema.columnInfo;
-            foreach (string col in myOutputSchemaStringArray)
-            {
+            foreach (string col in myOutputSchemaStringArray) {
                string colName = col.Substring(0, col.IndexOf('\\')).ToLower();
-               for (int i = 0; i < ci.Length; i++)
-               {
-                  if (ci[i].colName.ToLower().Equals(colName))
-                  {
+               for (int i = 0; i < ci.Length; i++) {
+                  if (ci[i].colName.ToLower().Equals(colName)) {
                      Type t = myGetType(col.Substring(col.IndexOf('\\') + 1));
                      ci[i] = new ColumnInfo(ci[i].colName, i, t);
                      break;
@@ -171,74 +158,52 @@ namespace BCG
          return mySchema;
       }
 
-      public IBlueRow map(IBlueRow row)
-      {
+      public IBlueRow map(IBlueRow row) {
          object[] vals = new object[row.schema.columnInfo.Length];
          row.Values.CopyTo(vals, 0);
-         for (int i = 0; i < mySchema.columnInfo.Length; i++)
-         {
+         for (int i = 0; i < mySchema.columnInfo.Length; i++) {
             Type t = mySchema.columnInfo[i].colType;
-            try
-            {
+            try {
                vals[i] = System.Convert.ChangeType(vals[i], mySchema.columnInfo[i].colType, System.Globalization.CultureInfo.InvariantCulture);
-            }
-            catch
-            {
-               if (t.Equals(typeof(Int16)))
-               {
+            } catch {
+               if (t.Equals(typeof(Int16))) {
                   Int16 result;
-                  if (Int16.TryParse((string)vals[i], out result))
-                  {
+                  if (Int16.TryParse((string)vals[i], out result)) {
                      vals[i] = result;
                      continue;
                   }
                   double r2;
-                  if (Double.TryParse((string)vals[i], out r2))
-                  {
-                     try
-                     {
+                  if (Double.TryParse((string)vals[i], out r2)) {
+                     try {
                         vals[i] = (Int16)r2;
                         continue;
-                     }
-                     catch { }
+                     } catch { }
                   }
-               }
-               else if (t.Equals(typeof(Int32)))
-               {
+               } else if (t.Equals(typeof(Int32))) {
                   Int32 result;
-                  if (Int32.TryParse((string)vals[i], out result))
-                  {
+                  if (Int32.TryParse((string)vals[i], out result)) {
                      vals[i] = result;
                      continue;
                   }
                   double r2;
-                  if (Double.TryParse((string)vals[i], out r2))
-                  {
-                     try
-                     {
+                  if (Double.TryParse((string)vals[i], out r2)) {
+                     try {
                         vals[i] = (Int32)r2;
                         continue;
-                     }
-                     catch { }
+                     } catch { }
                   }
-               }
-               else if (t.Equals(typeof(Int64)))
-               {
+               } else if (t.Equals(typeof(Int64))) {
                   Int64 result;
-                  if (Int64.TryParse((string)vals[i], out result))
-                  {
+                  if (Int64.TryParse((string)vals[i], out result)) {
                      vals[i] = result;
                      continue;
                   }
                   double r2;
-                  if (Double.TryParse((string)vals[i], out r2))
-                  {
-                     try
-                     {
+                  if (Double.TryParse((string)vals[i], out r2)) {
+                     try {
                         vals[i] = (Int64)r2;
                         continue;
-                     }
-                     catch { }
+                     } catch { }
                   }
                }
                vals[i] = DBNull.Value;
@@ -247,8 +212,7 @@ namespace BCG
          return new BlueRow(mySchema, vals);
       }
 
-      private Type myGetType(string type)
-      {
+      private Type myGetType(string type) {
          type = type.ToLower();
          if (type.Contains("int"))
             return typeof(int);

src/DataStreamSuite/DataStream/Primitives/StaticCollectionStream/BlueTableStream.cs

@@ -1,31 +1,41 @@
 using System;
 using System.Collections;
 using System.Reflection;
+
 using BCG.Utilities;
 
-namespace BCG
-{
-   [DataStreamUserRespresentation (false)]
-   public class BlueRowCollectionStaticStream : ViewStream
-   {
-      public BlueRowCollectionStaticStream () {}
-
-      public BlueRowCollectionStaticStream (IBlueRowCollection rows) {loadCollection (rows);}
-
-      public void loadCollection (IBlueRowCollection rows)
-      {
-         myFireEvent (new DataPacket (rows, rows, myLastValue != null ?
-                                                   new BlueTable((myLastValue as DataPacket).newRows) : new BlueTable (rows.schema)));
+namespace BCG {
+   // if a table needs to be fired, it can be wrapped around this
+   public class BlueTableStream : ViewStream {
+      private IBlueRowCollection _rows;
+      public BlueTableStream(IBlueRowCollection rows) {
+         _rows = rows;
       }
 
-      public void crush_with_single_row(IBlueRow r)
-      {
+      protected override void myOnFirstSubscriber() {
+         var dp_out = BlueDataUtility.build_dp_inc(_rows, myLastValue as DataPacket);
+         myFireEvent(dp_out);
+      }
+   }
+
+   [DataStreamUserRespresentation(false)]
+   public class BlueRowCollectionStaticStream : ViewStream {
+      public BlueRowCollectionStaticStream() { }
+
+      public BlueRowCollectionStaticStream(IBlueRowCollection rows) { loadCollection(rows); }
+
+      public void loadCollection(IBlueRowCollection rows) {
+         myFireEvent(new DataPacket(rows, rows, myLastValue != null ?
+                                                   new BlueTable((myLastValue as DataPacket).newRows) : new BlueTable(rows.schema)));
+      }
+
+      public void crush_with_single_row(IBlueRow r) {
          BlueTable t = new BlueTable(r.schema);
          t.addRow(r);
          loadCollection(t);
       }
 
-      public void loadPacket (DataPacket dp) {myFireEvent (dp);}
+      public void loadPacket(DataPacket dp) { myFireEvent(dp); }
 
       /// <summary>
       /// Will work with streams that take ViewStream as first argument ex project, filter, distinct etc
@@ -35,17 +45,16 @@ namespace BCG
       /// <param name="streamType"></param>
       /// <param name="args"></param>
       /// <returns></returns>
-      public static IBlueRowCollection statelessOp (IBlueRowCollection rows, Type streamType, params object[] args)
-      {
-         ArrayList totArgs = new ArrayList (args);
-         BlueRowCollectionStaticStream v1 = new BlueRowCollectionStaticStream ();
-         totArgs.Insert (0, v1);
-         ViewStream retVal = Activator.CreateInstance (streamType, totArgs.ToArray()) as ViewStream;
-         v1.loadCollection (rows);
+      public static IBlueRowCollection statelessOp(IBlueRowCollection rows, Type streamType, params object[] args) {
+         ArrayList totArgs = new ArrayList(args);
+         BlueRowCollectionStaticStream v1 = new BlueRowCollectionStaticStream();
+         totArgs.Insert(0, v1);
+         ViewStream retVal = Activator.CreateInstance(streamType, totArgs.ToArray()) as ViewStream;
+         v1.loadCollection(rows);
          return retVal.rows;
       }
 
    }
 
-   
+
 }

src/DataStreamSuite/DataStream/Primitives/Time/DateStandardizationStream.cs

@@ -80,7 +80,7 @@ namespace BCG {
 
       public IBlueRow map(IBlueRow row) {
          MutableBlueRow rv = new MutableBlueRow(getOutputSchema(row.schema), row);
-         rv.setValue(myConversionColName, mapDateTime(row[myConversionColName], myConversion));
+         rv.setValue(myConversionColName, mapDateTime(row.get_datetime(myConversionColName), myConversion));
          return rv;
          //return BlueDataUtility.getNewReplacedRow(row, mapDateTime(row[myConversionColName], myConversion), myConversionColName);
       }

src/DataStreamSuite/DataStream/Primitives/Time/DateTimeExpanstionStream.cs

@@ -0,0 +1,38 @@
+namespace bts.SilverPine {
+   using bts;
+
+   using BCG;
+
+   using System;
+   [DataStreamUserRespresentation("DateTimeExpanderStream")]
+   public class DateTimeExpanstionStream : ViewStream {
+      public DateTimeExpanstionStream(ViewStream parent_stream, string inp_col) {
+         Func<IBlueTableSchema, IBlueTableSchema> get_schema = (s) => {
+            IBlueTableSchema rv = s.add_column(System.TypeCode.DateTime, $"{inp_col}_date");
+            rv = rv.add_column(System.TypeCode.String, $"{inp_col}_day");
+            rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_day_of_month");
+            rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_month");
+            rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_hour");
+            rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_min");
+            rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_seconds_of_day");
+            return rv;
+         };
+         ViewStream rs = new MapStream(parent_stream,
+         s => get_schema(s),
+         r => {
+            DateTime dt = r.get_datetime(inp_col);
+            IBlueTableSchema s = get_schema(r.schema);
+            MutableBlueRow rv = new MutableBlueRow(s, r);
+            rv.setValue($"{inp_col}_date", dt.Date);
+            rv.setValue($"{inp_col}_day_of_week", dt.DayOfWeek.ToString());
+            rv.setValue($"{inp_col}_day_of_month", dt.Day);
+            rv.setValue($"{inp_col}_month", dt.Date.Month);
+            rv.setValue($"{inp_col}_hour", dt.Hour);
+            rv.setValue($"{inp_col}_minute", dt.Minute);
+            rv.setValue($"{inp_col}_seconds_of_day", (dt - dt.Date).TotalSeconds);
+            return rv;
+         });
+         mySetRealSource(rs);
+      }
+   }
+}

src/DataStreamSuite/DataStream/Primitives/Timeseries/ColumnDifferenceStream.cs

@@ -0,0 +1,124 @@
+namespace bts.SilverPine {
+   using BCG;
+
+   using bts;
+   using bts.collections;
+
+   using System;
+   using System.CodeDom;
+   using System.Collections.Generic;
+   using System.Linq;
+
+   // Given 2 filters, and 2 columns, finds the different in the value of the 2 columns from the 2 rows
+   // so:
+   // f1(base_stream.rows)[col1] - f1(base_stream.rows)[col2]
+   [DataStreamUserRespresentation("ColDiff")]
+   public class ColumnDifferenceStream : ViewStream {
+      private readonly ViewStream _base_stream;
+      private readonly FilterStream.OnFilter _f1, _f2;
+      private readonly string _c1, _c2;
+
+      public ColumnDifferenceStream(ViewStream base_stream, FilterStream.OnFilter f1, FilterStream.OnFilter f2, string col1, string col2) {
+         _base_stream = base_stream;
+         _f1 = f1;
+         _f2 = f2;
+         _c1 = col1;
+         _c2 = col2;
+      }
+
+      public ColumnDifferenceStream(ViewStream base_stream, string f1, string f2, string col1, string col2) {
+         FilterStream.OnFilter f1_del = r => {
+            BooleanExpressionFilter rv = new BooleanExpressionFilter(f1);
+            return rv.filter(r);
+         };
+         FilterStream.OnFilter f2_del = r => {
+            BooleanExpressionFilter rv = new BooleanExpressionFilter(f2);
+            return rv.filter(r);
+         };
+         ViewStream rs = new ColumnDifferenceStream(base_stream, f1_del, f2_del, col1, col2);
+         mySetRealSource(rs);
+      }
+
+      // for each row, does a row[new_col_name] = row[col1] - rows[row_ffset][col2]
+      // we should generlize this for any math operation
+      public ColumnDifferenceStream(ViewStream parent_stream, string sort_col_name, int row_offset, string col1, string col2, string diff_col_name, double optional_perc_diff_filter = 0.0) {
+         // use GlobalMapStream
+         string diff_col_perc_name = $"{diff_col_name}_percentage";
+         System.Func<System.Collections.Generic.List<BCG.IBlueRow>, uint, BCG.IBlueRow> g_map_func = (rows, idx) => {
+            int minuend_idx = Convert.ToInt32(idx);
+            var inp_schema = rows[0].schema;
+            var out_schema = inp_schema.add_column(TypeCode.Double, diff_col_name);
+            out_schema = out_schema.add_column(TypeCode.Double, diff_col_perc_name);
+            // minuend - subtrahend
+            IBlueRow minuend_row = rows.ElementAt(minuend_idx);
+            MutableBlueRow rv = new MutableBlueRow(out_schema, minuend_row);
+            int subtrahend_index = minuend_idx + row_offset;
+            double diff_val = Double.NaN, diff_val_perc = Double.NaN;
+            if (subtrahend_index >= 0) {
+               try {
+                  IBlueRow subtrahend_row = rows.ElementAt(subtrahend_index);
+                  diff_val = minuend_row.get<double>(col1) - subtrahend_row.get<double>(col2);
+                  diff_val_perc = 100.0 * diff_val / subtrahend_row.get<double>(col1);
+               } catch (Exception ex) {
+                  Log.assert(false, ex.Message);
+               }
+            }
+            rv.setValue(diff_col_name, diff_val);
+            rv.setValue(diff_col_perc_name, diff_val_perc);
+            return rv;
+         };
+         ViewStream rs = new GlobalMapStream(parent_stream, sort_col_name, g_map_func);
+         if (optional_perc_diff_filter != 0) {
+            rs = new FilterStream(rs, r => {
+               double diff_val_perc = r.get<double>(diff_col_perc_name);
+               double abs_diff_val_perc = Math.Abs(diff_val_perc);
+               return abs_diff_val_perc >= optional_perc_diff_filter;
+            });
+         }
+         mySetRealSource(rs);
+      }
+
+      protected override void myOnFirstSubscriber() {
+         _base_stream.onChange += _handle_base_stream_update;
+      }
+
+      protected override void myOnLastSubscriber() {
+         _base_stream.onChange -= _handle_base_stream_update;
+      }
+
+      private void _handle_base_stream_update(object dp_obj) {
+         // could have been done with filter and cartesian prod, and then math
+         DataPacket dp = dp_obj as DataPacket;
+         List<IBlueRow> rows = dp.newRows.to_list();
+         var f1_rows = rows.Where(r => _f1(r));
+         //Log.assert(f1_rows.Count() == 1);
+         var f2_rows = rows.Where(r => _f2(r));
+         //Log.assert(f2_rows.Count() == 1);
+         if (f1_rows != null && f1_rows.Count() > 0 && f2_rows != null && f2_rows.Count() > 0) {
+            IBlueRow f1_r = f1_rows.ElementAt(0);
+            IBlueRow f2_r = f2_rows.ElementAt(0);
+            double diff = f1_r.get<double>(_c1) - f2_r.get<double>(_c2);
+            IBlueTableSchema out_schema = f1_r.schema.add_columns(Tuple.Create(typeof(double), "diff"));
+            BlueTable out_table = new BlueTable(out_schema);
+            MutableBlueRow out_row = new MutableBlueRow(out_schema, f1_r);
+            out_row.setValue("diff", diff);
+            out_table.addRow(out_row);
+            DataPacket dp_out = BlueDataUtility.build_dp(out_table, myLastValue as DataPacket);
+            myFireEvent(dp_out);
+         } else {
+            IBlueRow f1_r = rows.ElementAt(0);
+            IBlueRow f2_r = rows.ElementAt(0);
+            double diff = Double.NaN;
+            IBlueTableSchema out_schema = f1_r.schema.add_columns(Tuple.Create(typeof(double), "diff"));
+            BlueTable out_table = new BlueTable(out_schema);
+            MutableBlueRow out_row = new MutableBlueRow(out_schema, f1_r);
+            out_row.setValue("diff", diff);
+            out_table.addRow(out_row);
+            DataPacket dp_out = BlueDataUtility.build_dp(out_table, myLastValue as DataPacket);
+            myFireEvent(dp_out);
+         }
+
+      }
+   }
+
+}

src/DataStreamSuite/DataStream/Primitives/Timeseries/ColumnEvolutionStream.cs

@@ -0,0 +1,51 @@
+namespace bts.SilverPine {
+   using BCG;
+
+   using bts;
+
+   using System.Collections.Generic;
+   using System.Linq;
+
+   // Given a sort order, an optional group by col, it observes the values of a column 
+   // within that group (or the whole table, if grouping not specified) and returns all rows
+   // where the value changes, ie row[current][col] != row[current-1][col]
+   // TODO: a more general way to express a filter where row[current][col] != row[previous][col],
+   // or even expand it to row[current][col_a] - row[previous][col_b] < 0.5
+   [DataStreamUserRespresentation("ColEvol")]
+   public class ColumnEvolutionStream : ViewStream {
+      private readonly ViewStream _base_stream;
+      private readonly string _sort_col;
+      private readonly string _evol_col;
+      public ColumnEvolutionStream(ViewStream base_stream, string sort_col, string evolution_col) {
+         _base_stream = base_stream;
+         _sort_col = sort_col;
+         _evol_col = evolution_col;
+      }
+
+      protected override void myOnFirstSubscriber() {
+         _base_stream.onChange += _handle_base_stream_update;
+      }
+
+      protected override void myOnLastSubscriber() {
+         base.myOnLastSubscriber();
+      }
+
+      private void _handle_base_stream_update(object dp_obj) {
+         DataPacket dp = dp_obj as DataPacket;
+         List<IBlueRow> rows = dp.newRows.to_list();
+         rows.Sort((r1, r2) => BlueDataUtility.compare_rows(r2, r1, _sort_col));
+         BlueTable rv = new BlueTable(dp.schema);
+         IBlueRow prev_row = rows.ElementAt(0);
+         foreach (IBlueRow r in rows) {
+            var cr = BlueDataUtility.compare_rows(r, prev_row, _evol_col);
+            if (cr != 0) {
+               rv.addRow(r);
+            }
+            prev_row = r;
+         }
+         DataPacket dp_out = BlueDataUtility.build_dp(rv, myLastValue as DataPacket);
+         myFireEvent(dp_out);
+      }
+   }
+
+}

src/DataStreamSuite/DataStreamCore/BlueData/BlueDataUtility.cs

@@ -15,10 +15,75 @@ using mbts;
 using BCG.Extensions;
 using System.Text.Json;
 using System.Text.Json.Nodes;
+using Newtonsoft.Json;
+using BCG;
+using System.Runtime.CompilerServices;
+using bts;
+
+namespace bts {
+
+   // public class CompositeRow : IBlueRow {
+   //    private IBlueRow _lrow, _rrow;
+   //    private BlueRow _out_row;
+   //    public CompositeRow(IBlueRow lrow, IBlueRow rrow) {
+   //       _lrow = lrow;
+   //       _rrow = rrow;
+   //       IBlueTableSchema schema = _lrow.schema;
+   //       List<object> vals = new List<object>();
+   //       foreach (var o in _lrow) {
+   //          vals.Add(o);
+   //       }
+   //       foreach (ColumnInfo ci in _rrow.schema.columnInfo) {
+   //          if (ci.colName == "id") {
+   //             continue;
+   //          }
+   //          vals.Add(_rrow[ci.colNo]);
+   //          schema = schema.add_column(ci.type_code(), ci.colName);
+   //       }
+   //       _out_row = new BlueRow(schema, vals.ToArray());
+   //    }
+   //    public IBlueTableSchema schema {
+   //       get { return _out_row.schema; }
+   //    }
+   //    public object? this[int index] {
+   //       get { return _out_row[index]; }
+   //    }
+   //    public object? this[string colName] {
+   //       get {
+   //          return _out_row[colName];
+   //       }
+   //    }
+
+   // }
+}
 
 namespace BCG {
    public static class BlueDataUtility {
+      public static IBlueRow create_composite_row(IBlueRow lr, IBlueRow rr, int id) {
+         IBlueTableSchema schema = lr.schema;
+         List<object> vals = new List<object>();
+         foreach (ColumnInfo ci in lr.schema.columnInfo) {
+            if (ci.colName == "id") {
+               vals.Add(id);
+            }
+            vals.Add(lr[ci.colName]);
+         }
+         foreach (ColumnInfo ci in rr.schema.columnInfo) {
+            if (ci.colName == "id") {
+               continue;
+            }
+            schema = schema.add_column(ci.type_code(), ci.colName);
+            vals.Add(rr[ci.colName]);
+         }
+         IBlueRow r = new BlueRow(schema, vals.ToArray());
+         return r;
+      }
       #region Extensions
+      public static bool is_null(this IBlueRow r, string col_name) {
+         object col_val = r[col_name];
+         return col_val == null || DBNull.Value.Equals(col_val);
+      }
+
       public static DateTime get_datetime(this IBlueRow r, string col_name, DateTime err_value = default(DateTime)) {
          return BlueDataUtility.get_row_datetime_value(r, col_name, err_value);
       }
@@ -49,6 +114,17 @@ namespace BCG {
             rv.Add(r.get<T>(col));
          return rv;
       }
+      public static IEnumerable<T> get_colvalues<T>(IEnumerable<IBlueRow> rows, string col, int start_index, int count) {
+         List<T> rv = new List<T>();
+         int i = 0;
+         int n_copied = 0;
+         foreach (IBlueRow r in rows)
+            if (i++ >= start_index && n_copied < count) {
+               rv.Add(r.get<T>(col));
+               n_copied++;
+            }
+         return rv;
+      }
 
       public static IDictionary<T, IEnumerable<IBlueRow>> group_by<T>(IBlueRowCollection rows, string gp_col) {
          Dictionary<T, List<IBlueRow>> rv = new Dictionary<T, List<IBlueRow>>();
@@ -112,13 +188,17 @@ namespace BCG {
          }
       }
 
-      public static IEnumerable<IBlueRow> to_ienum(this IBlueRowCollection rows) {
+      public static List<IBlueRow> to_list(this IBlueRowCollection rows) {
          List<IBlueRow> rv = new List<IBlueRow>();
          foreach (IBlueRow r in rows)
             rv.Add(r);
          return rv;
       }
 
+      public static IEnumerable<IBlueRow> to_ienum(this IBlueRowCollection rows) {
+         return rows.to_list();
+      }
+
       public static IDictionary<int, IBlueRow> to_map(this IBlueRowCollection rows) {
          Dictionary<int, IBlueRow> rv = new Dictionary<int, IBlueRow>();
          foreach (IBlueRow r in rows) {
@@ -128,6 +208,10 @@ namespace BCG {
          return rv;
       }
 
+      public static bool contains_column(this IBlueTableSchema input_schema, string col) {
+         return input_schema.columnNames.Contains(col);
+      }
+
       public static IBlueTableSchema add_columns(this IBlueTableSchema input_schema, params Tuple<Type, string>[] cols) {
          List<ColumnInfo> cis = new List<ColumnInfo>(input_schema.columnInfo);
          foreach (Tuple<Type, string> c in cols)
@@ -135,6 +219,11 @@ namespace BCG {
          return new BlueTableSchema(input_schema.tableName, cis.ToArray());
       }
 
+      public static IBlueTableSchema add_column(this IBlueTableSchema input_schema, TypeCode type_arg, string col_name) {
+         Tuple<Type, string> tup = new Tuple<Type, string>(_typecode_to_type[type_arg], col_name);
+         return add_columns(input_schema, tup);
+      }
+
       public static IBlueRowCollection to_collection(this IBlueRow r) {
          if (r == null)
             return null;
@@ -191,6 +280,18 @@ namespace BCG {
 
       #region json
 
+      public static Dictionary<string, string> convert_json_to_dictionary(string json_string) {
+         Dictionary<string, string> dictionary = JsonConvert.DeserializeObject<Dictionary<string, string>>(json_string);
+         return dictionary;
+      }
+      public static IBlueRow convert_json_to_row(string json_string, int id = 0) {
+         Dictionary<string, object> dictionary = JsonConvert.DeserializeObject<Dictionary<string, object>>(json_string);
+         if (!dictionary.ContainsKey("id")) {
+            dictionary.Add("id", id);
+         }
+         return new BlueRow(dictionary);
+      }
+
       public static List<JsonNode> json_to_json_node_list(string jsonString) {
          var jsonNodeList = new List<JsonNode>();
          bool noe = String.IsNullOrEmpty(jsonString);
@@ -209,7 +310,7 @@ namespace BCG {
             foreach (JsonNode jsonNode in jsonArray) {
                jsonNodeList.Add(jsonNode);
             }
-         } catch (JsonException ex) {
+         } catch (System.Text.Json.JsonException ex) {
             Console.WriteLine($"Invalid JSON: {ex.Message}");
          }
 
@@ -486,7 +587,16 @@ namespace BCG {
          ColumnInfo[] cis = new ColumnInfo[table.Columns.Count];
          foreach (DataColumn col in table.Columns)
             cis[col.Ordinal] = new ColumnInfo(col.ColumnName, col.Ordinal, col.DataType);
-         return new BlueTableSchema(cis, table.TableName);
+         IBlueTableSchema rv = new BlueTableSchema(cis, table.TableName);
+         rv = rv.add_id_col_if_missing();
+         return rv;
+      }
+      public static IBlueTableSchema add_id_col_if_missing(this IBlueTableSchema schema) {
+         IBlueTableSchema rv = schema;
+         if (!schema.columnNames.Contains("id")) {
+            rv = schema.add_columns(Tuple.Create(typeof(int), "id"));
+         }
+         return rv;
       }
 
       /// <summary>
@@ -744,10 +854,16 @@ namespace BCG {
       }
 
       public static IBlueRowCollection toIBlueRowCollection(DataTable dt) {
-         BlueTable retVal = new BlueTable(getSchema(dt));
-         foreach (DataRow dr in dt.Rows)
-            retVal.addRow(dr);
-         return retVal;
+         IBlueTableSchema schema = getSchema(dt);
+         BlueTable rv = new BlueTable(schema);
+         foreach (DataRow dr in dt.Rows) {
+            MutableBlueRow nr = new MutableBlueRow(schema);
+            foreach (DataColumn col in dr.Table.Columns)
+               nr.setValue(col.ColumnName, dr[col.ColumnName]);
+            nr.setValue("id", rv.Count);
+            rv.addRow(nr);
+         }
+         return rv;
       }
 
       public static IBlueRowCollection toIBlueRowCollection(DataRow[] rows) {
@@ -819,20 +935,20 @@ namespace BCG {
       /// </summary>
       /// <param name="r1"></param>
       /// <param name="r2"></param>
-      /// <param name="colNames"></param>
+      /// <param name="col_names"></param>
       /// <returns></returns>
       public static int compare_rows(IBlueRow r1,
          IBlueRow r2,
-         params string[] colNames) {
-         int retVal = 0;
-         foreach (string colName in colNames) {
-            object val1 = r1[colName];
-            object val2 = r2[colName];
-            retVal = _compare_row_cols(val1, val2);
-            if (retVal != 0)
+         params string[] col_names) {
+         int comp_res = 0;
+         foreach (string cn in col_names) {
+            object val1 = r1[cn];
+            object val2 = r2[cn];
+            comp_res = _compare_row_cols(val1, val2);
+            if (comp_res != 0)
                break;
          }
-         return retVal;
+         return comp_res;
       }
 
       private static int _compare_row_cols(object v1, object v2) {
@@ -1608,8 +1724,10 @@ namespace BCG {
 
       public static int[] get_column_indices(IBlueTableSchema schema, string[] col_names) {
          List<int> rv = new List<int>();
-         foreach (string c in col_names)
+         foreach (string c in col_names) {
+            Log.assert(schema.contains_column(c), $"Could not find col: {c} in table: {schema.tableName}");
             rv.Add(schema[c].colNo);
+         }
          return rv.ToArray();
       }
 
@@ -1783,6 +1901,18 @@ namespace BCG {
          #endregion
       }
 
+      public static IBlueRow find_max(IEnumerable<IBlueRow> rows, params string[] cols_to_compare) {
+         if (rows == null || rows.Count() == 0)
+            return null;
+         IBlueRow max_row = rows.ElementAt(0);
+         foreach (IBlueRow r in rows) {
+            int c = compare_rows(r, max_row, cols_to_compare);
+            if (c > 0)
+               max_row = r;
+         }
+         return max_row;
+      }
+
       public static int compare_rows(IBlueRow r1, IBlueRow r2, Dictionary<string, bool> sort_map) {
          foreach (KeyValuePair<string, bool> p in sort_map) {
             int comp = compare_send_dontcare_to_last(r1, p.Key, r2, p.Key, p.Value);
@@ -2084,23 +2214,53 @@ namespace BCG {
          try {
             if (v == null || v == DBNull.Value)
                return error_val;
-            else {
-               if (v is DateTime)
-                  return (DateTime)v;
-               else if (v is string) {
-                  DateTime rv_dt = error_val;
-                  DateTime.TryParse((string)v, out rv_dt);
+
+            if (v is DateTime) {
+               return (DateTime)v;
+            } else if (v is string) {
+               DateTime rv_dt;
+
+               // Try parsing the string as a DateTime
+               if (DateTime.TryParse((string)v,
+                                     System.Globalization.CultureInfo.InvariantCulture,
+                                     System.Globalization.DateTimeStyles.RoundtripKind,
+                                     out rv_dt)) {
                   return rv_dt;
                } else {
-                  double oa_date = (double)Convert.ChangeType(v, typeof(Double));
-                  return DateTime.FromOADate(oa_date);
+                  return error_val; // Return the error value if parsing fails
                }
+            } else {
+               // Assume the input can be converted to a double and use FromOADate
+               double oa_date = (double)Convert.ChangeType(v, typeof(Double));
+               return DateTime.FromOADate(oa_date);
             }
          } catch {
             return error_val;
          }
       }
 
+
+      // private static DateTime _get_datetime_value(object v, DateTime error_val) {
+      //    try {
+      //       if (v == null || v == DBNull.Value)
+      //          return error_val;
+      //       else {
+      //          if (v is DateTime)
+      //             return (DateTime)v;
+      //          else if (v is string) {
+      //             DateTime rv_dt = error_val;
+      //             DateTime.TryParse((string)v, out rv_dt);
+      //             return rv_dt;
+      //          } else {
+      //             double oa_date = (double)Convert.ChangeType(v, typeof(Double));
+      //             return DateTime.FromOADate(oa_date);
+      //          }
+      //       }
+      //    } catch {
+      //       return error_val;
+      //    }
+      // }
+
       #endregion
 
       #region CSV methods

src/DataStreamSuite/DataStreamCore/DataStreamCore.csproj

@@ -8,6 +8,9 @@
 
   <ItemGroup>
     <PackageReference Include="Google.Protobuf" Version="3.24.3" />
+    <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
+    <PackageReference Include="OpenAI" Version="2.0.0" />
+    <PackageReference Include="SimMetrics.Net" Version="1.0.5" />
   </ItemGroup>
 
   <ItemGroup>

src/DataStreamSuite/DataStreamCore/DynamicLoading/PrimitiveManager.cs

@@ -47,6 +47,11 @@ namespace BCG {
       /// </summary>
       /// <param name="assemblyName"></param>
       public void loadAssembly(string assemblyName) {
+         Func<Type, bool> is_silverpine_only = (t) => {
+            if (!assemblyName.Contains("SharedStreams.dll"))
+               return true;
+            return t.Namespace == "bts.SilverPine";
+         };
          bts.Log.Gen.info($"PrimitiveManager.loadAssembly assemblyName: {assemblyName}");
          if (myAssemblyNames.Contains(assemblyName))
             return;
@@ -56,7 +61,7 @@ namespace BCG {
             Assembly.LoadFrom(assemblyName);
 
          foreach (Type type in loadedAssembly.GetTypes())
-            if (!type.IsInterface
+            if (!type.IsInterface && is_silverpine_only(type)
                && !type.IsAbstract
                && type.IsSubclassOf(typeof(ViewStream))
                && type.IsPublic) {

src/DataStreamSuite/DataStreamExpressions/DataStreamExpressionTokens.lex

@@ -9,7 +9,15 @@
 Letter                   [a-zA-Z]
 BooleanConstant          'true'|'false'
 CSStringConstant         \"([^\"\\]*|\\.)*\"
-DateTimeConstant         ([1-9]|'10'|'11'|'12')\/([1-9]|([1-2][0-9])|'30'|'31')\/[0-9][0-9][0-9]*
+YearInt                  [0-9]{4}
+MonthInt                 [0-9]{1,2}
+DayInt                   [0-9]{1,2}
+/**
+ * DateTimeConstant supports American (mm/dd/yyyy) and European (yyyy/mm/dd)
+ * formats. Month and day can be 1 or 2 digits. It does not validate that the
+ * numbers represent an actual date.
+ */
+DateTimeConstant         ({MonthInt}\/{DayInt}\/{YearInt}|{YearInt}\/{MonthInt}\/{DayInt})
 IntegerConstant          -?[0-9]+
 NullConstant             'null'
 RealConstant             -?[0-9]+\.[0-9]+

src/DataStreamSuite/DataStreamPublisher/Configurations/MarketDataPublisher.json

@@ -1,6 +1,6 @@
 {
   "user": "bts",
-  "sparkd": "http://127.0.0.1:8000/config",
+  "sparkd": "http://192.168.1.80:8000/config",
   "pubsub": {
     "milliseconds_between_sub_info_updates": 1000,
     "milliseconds_between_time_slice_updates": 5000,
@@ -105,5 +105,25 @@
         }
       ]
     }
+    // {
+    //   "name": "news_data",
+    //   "streams": [
+    //     {
+    //       "name": "benzinga_news",
+    //       "expression": "benzingaNews(\"mongodb://localhost:27017\",\"local\",\"news_headlines\", \"30s\", \"5d\")",
+    //       "is_on_websocket": true
+    //     },
+    //     {
+    //       "name": "benzinga_news_symbols",
+    //       "expression": "benzingaNewsSymbols(benzinga_news)",
+    //       "is_on_websocket": true
+    //     },
+    //     {
+    //       "name": "benzinga_news_symbol_state",
+    //       "expression": "benzingaNewsSymbolState(benzinga_news)",
+    //       "is_on_websocket": true
+    //     }
+    //   ]
+    // }
   ]
 }

src/DataStreamSuite/DataStreamPublisher/Configurations/TestDataPublisher.json

@@ -9,6 +9,21 @@
           "expression": "milliseconds(1000,10)",
           "is_on_websocket": true
         },
+        {
+          "name": "benzinga_news",
+          "expression": "benzingaNews(\"mongodb://localhost:27017\",\"local\",\"news_headlines\", \"30s\", \"5d\")",
+          "is_on_websocket": true
+        },
+        {
+          "name": "benzinga_news_symbols",
+          "expression": "benzingaNewsSymbols(benzinga_news)",
+          "is_on_websocket": true
+        },
+        {
+          "name": "benzinga_news_symbol_state",
+          "expression": "benzingaNewsSymbolState(benzinga_news)",
+          "is_on_websocket": true
+        },
         {
           "name": "test_literal",
           "expression": "literal(\"x\", \"420.0\")",

src/DataStreamSuite/DataStreamPublisher/DataStreamPublisherConfig.cs

@@ -10,7 +10,8 @@ using System.IO;
 using System.Text.Json;
 using System.Text.Json.Serialization;
 
-public class DataStreamPublisherConfig {
+public class DataStreamPublisherConfig
+{
    /// <summary>
    /// The login user for sparkd
    /// </summary>
@@ -38,6 +39,18 @@ public class DataStreamPublisherConfig {
    [JsonPropertyName("publishers")]
    public List<PublisherNode>? Publishers { get; set; }
 
+   [JsonPropertyName("mongodb_url")]
+   public string MONGO_DB_URL { get; set; }
+
+   [JsonPropertyName("mongodb_db_name")]
+   public string MONGO_DB_NAME { get; set; }
+
+   [JsonPropertyName("mongodb_db_collection")]
+   public string MONGO_DB_COLLECTION { get; set; }
+
+   [JsonPropertyName("benzinga_news_ttl")]
+   public string BENZINGA_NEWS_TTL { get; set; }
+
    [JsonPropertyName("pubsub")]
    public PubSubConfig? PubSub { get; set; }
 
@@ -49,11 +62,13 @@ public class DataStreamPublisherConfig {
    public string? ConfigFile { get; set; }
 
    public static DataStreamPublisherConfig Default() =>
-      new() {
+      new()
+      {
          PubSub = PubSubConfig.Default(),
       };
 
-   public void MergeWith(DataStreamPublisherConfig other) {
+   public void MergeWith(DataStreamPublisherConfig other)
+   {
       User ??= other.User;
       Password ??= other.Password;
       Sparkd ??= other.Sparkd;
@@ -69,8 +84,10 @@ public class DataStreamPublisherConfig {
    /// <exception cref="InvalidOperationException">
    /// JsonSerializer.Deserialize returns null
    /// </exception>
-   public void MergeWithConfigFile() {
-      if (ConfigFile is null) {
+   public void MergeWithConfigFile()
+   {
+      if (ConfigFile is null)
+      {
          return;
       }
       var cfgString = File.ReadAllText(ConfigFile);

src/DataStreamSuite/DataStreamPublisher/Program.cs

@@ -23,8 +23,10 @@ using System.Threading.Tasks;
 await Parser.Default.ParseArguments<DataStreamPublisherConfig>(args)
    .MapResult(RunAsync, errs => Task.FromResult(1));
 
-async Task<int> RunAsync(DataStreamPublisherConfig cfg) {
-   try {
+async Task<int> RunAsync(DataStreamPublisherConfig cfg)
+{
+   try
+   {
       cfg.MergeWithConfigFile();
       cfg.MergeWith(DataStreamPublisherConfig.Default());
       PubSubConfig.SetGlobalConfig(Log.notnull(cfg.PubSub));
@@ -33,7 +35,9 @@ async Task<int> RunAsync(DataStreamPublisherConfig cfg) {
       var service_ctx = ServiceContext.create(cfg.Sparkd);
       await service_ctx.login(cfg.User ?? "bts", cfg.Password ?? string.Empty);
       _publish_streams(cfg.Publishers ?? new(0));
-   } catch (Exception ex) {
+   }
+   catch (Exception ex)
+   {
       // Write the error to console first, in case the logging system is compromised.
       var error_msg = $"DataStreamPublisher threw exception:\n{ex}";
       Console.Error.WriteLine(error_msg);
@@ -44,29 +48,39 @@ async Task<int> RunAsync(DataStreamPublisherConfig cfg) {
    return 0;
 }
 
-static ViewStream? build_stream(string? stream_expression) {
-   if (stream_expression is null) {
+static ViewStream? build_stream(string? stream_expression)
+{
+   // Console.WriteLine("Hit key to continue");
+   // Console.ReadLine();
+   if (stream_expression is null)
+   {
       return null;
    }
-   try {
+   try
+   {
       bts.Log.Gen.info($"build_stream stream_expression: {stream_expression}");
       var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression);
       var rv = exp.getView(new Dictionary<string, StatementExpression>(0));
       return rv;
 
-   } catch (Exception ex) {
+   }
+   catch (Exception ex)
+   {
       bts.Log.assert(false, $"Could not build stream: {stream_expression} ex: {ex}");
       return null;
    }
 }
 
-void _publish_streams(IEnumerable<PublisherNode> publishers) {
+void _publish_streams(IEnumerable<PublisherNode> publishers)
+{
    PubTableBroker pb_broker = new PubTableBroker();
    RemoteQueryManager csm = new RemoteQueryManager();
    var csvPublishers = new List<CSVPublisher>();
 
-   foreach (var publisher in publishers) {
-      if (Directory.Exists(publisher.CsvDir)) {
+   foreach (var publisher in publishers)
+   {
+      if (Directory.Exists(publisher.CsvDir))
+      {
          csvPublishers.Add(new(publisher.CsvDir, pb_broker));
       }
    }
@@ -75,7 +89,8 @@ void _publish_streams(IEnumerable<PublisherNode> publishers) {
       .SelectMany(p => p.Streams.empty_if_null().Select(createViewPublisher))
       .ToList();
 
-   while (true) {
+   while (true)
+   {
       System.Threading.Thread.Sleep(TimeSpan.FromSeconds(1));
       ViewStreamDatabase.instance.doWork();
       csm.doWork();
@@ -83,16 +98,19 @@ void _publish_streams(IEnumerable<PublisherNode> publishers) {
       ManagedSubscriber.instance.doWork();
    }
 
-   ViewPublisher? createViewPublisher(PublishedStream ps) {
+   ViewPublisher? createViewPublisher(PublishedStream ps)
+   {
       Log.Gen.info(
          $"Instantiating ViewPublisher for: {JsonSerializer.Serialize(ps)}");
-      if (string.IsNullOrEmpty(ps.Name)) {
+      if (string.IsNullOrEmpty(ps.Name))
+      {
          Log.Gen.warn(
             $"Failed to instantiate ViewPublisher. Null/empty 'name': {JsonSerializer.Serialize(ps)}");
          return null;
       }
       var vs = build_stream(ps.Expression);
-      if (vs is null) {
+      if (vs is null)
+      {
          Log.Gen.warn(
             $"Failed to instantiate ViewPublisher. Invalid 'expression': {JsonSerializer.Serialize(ps)}");
          return null;
@@ -103,31 +121,39 @@ void _publish_streams(IEnumerable<PublisherNode> publishers) {
    }
 }
 
-void on_change(object obj) {
+void on_change(object obj)
+{
    bts.Log.Gen.info($"DSPublisher on_change dp: {obj}");
 }
 
-public class CSVPublisher : IWorker {
+public class CSVPublisher : IWorker
+{
    private IDictionary<string, ViewPublisher> _pubs = new Dictionary<string, ViewPublisher>();
    private readonly string _csv_directory;
    private PubTableBroker _pub_table_broker;
 
-   public CSVPublisher(string csv_directory, PubTableBroker pub_table_broker) {
+   public CSVPublisher(string csv_directory, PubTableBroker pub_table_broker)
+   {
       _csv_directory = csv_directory;
       _pub_table_broker = pub_table_broker;
       MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(1));
    }
-   public void doWork() {
+   public void doWork()
+   {
       var file_names = Directory.GetFiles(_csv_directory);
-      foreach (var fn in file_names) {
-         if (!_pubs.ContainsKey(fn)) {
+      foreach (var fn in file_names)
+      {
+         if (!_pubs.ContainsKey(fn))
+         {
             _handle_new_file(fn);
          }
       }
    }
 
-   private void _handle_new_file(string file_name) {
-      if (file_name.EndsWith(".csv")) {
+   private void _handle_new_file(string file_name)
+   {
+      if (file_name.EndsWith(".csv"))
+      {
          ViewStream vs = new CSVReaderStream(file_name);
          vs = new SchemaGuesserStream(vs);
          var toks = file_name.Split('\\');

src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json

@@ -1,5 +1,6 @@
 {
   "user": "bts",
+  "sparkd": "http://192.168.1.80:8000/config",
   "pubsub": {
     "milliseconds_between_sub_info_updates": 1000,
     "milliseconds_between_time_slice_updates": 5000,
@@ -17,14 +18,14 @@
     "publisher_routes": [
       {
         "name": "default",
-        "server_address": "bluto://192.168.1.80:12345",
+        "server_address": "bts-cloud://192.168.1.80:12345",
         "tag": ".*"
       }
     ],
     "subscriber_routes": [
       {
         "name": "default",
-        "server_address": "bluto://192.168.1.80:12345",
+        "server_address": "bts-cloud://192.168.1.80:12345",
         "tag": ".*"
       }
     ],

src/DataStreamSuite/DataStreamQueryAnalyzer/MainForm.cs

@@ -169,8 +169,10 @@ namespace BCG
       }
       #endregion
 
+      private Stopwatch _stopwatch = new Stopwatch();
       private void myTryExpression(string expStr)
       {
+         _stopwatch.Restart();
          myFreezeView(false); // force unfreeze the view
          DataStreamExpression oldExp = myExp;
          try
@@ -344,6 +346,7 @@ namespace BCG
       // this is the stream callback
       private void myCallback(object o)
       {
+         _stopwatch.Stop();
          var timer = Stopwatch.StartNew();
          DataPacket dp = o as DataPacket;
          myCrono.Stop();
@@ -359,7 +362,8 @@ namespace BCG
          myToolStripStatusLabelRows.BackColor = Color.Black;
          myToolStripStatusLabelTime.ForeColor = Color.White;
          myToolStripStatusLabelRows.ForeColor = Color.White;
-         myToolStripStatusLabelTime.Text = "Last update: " + DateTime.Now.ToString("hh:mm:ss:fff tt");
+         string time_text = "Last update: " + DateTime.Now.ToString("hh:mm:ss:fff tt") + $" time: {_stopwatch.ElapsedMilliseconds} msecs\t\t";
+         myToolStripStatusLabelTime.Text = time_text;
          myToolStripStatusLabelRows.Text = "New = " + dp.newRows.Count + "  Added = " + dp.addedRows.Count + "  Deleted = " + dp.deletedRows.Count;
          Log.Ui.diag(Text);
          int hscrollval = myChangeStyle ? 0 : myDataGrid.HorizScrollBarValue;

src/DataStreamSuite/DataStreamSuite.sln

@@ -36,6 +36,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnExpressions", "Bt
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnSortExpressions", "Bts.ColumnSortExpressions\Bts.ColumnSortExpressions.csproj", "{B038CB72-4E81-4AF1-976F-ED8655C2C3D6}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamQueryAnalyzer", "DataStreamQueryAnalyzer\DataStreamQueryAnalyzer.csproj", "{BFA5BA5C-927A-41A9-A696-1E2F4B5FFD15}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -113,5 +115,9 @@ Global
 		{B038CB72-4E81-4AF1-976F-ED8655C2C3D6}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{B038CB72-4E81-4AF1-976F-ED8655C2C3D6}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{B038CB72-4E81-4AF1-976F-ED8655C2C3D6}.Release|Any CPU.Build.0 = Release|Any CPU
+		{BFA5BA5C-927A-41A9-A696-1E2F4B5FFD15}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{BFA5BA5C-927A-41A9-A696-1E2F4B5FFD15}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{BFA5BA5C-927A-41A9-A696-1E2F4B5FFD15}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{BFA5BA5C-927A-41A9-A696-1E2F4B5FFD15}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 EndGlobal

src/DataStreamSuite/PubSub/Config/PubSubConfig.cs

@@ -117,10 +117,10 @@ public class PubSubConfig {
          WaitPeriodAfterGc = 1000,
          SimpleEchoServer = false,
          PublisherRoutes = new() {
-            new("bts-cloud://127.0.0.1:12345", ".*")
+            new("bluto://127.0.0.1:12345", ".*")
          },
          SubscriberRoutes = new() {
-            new("bts-cloud://127.0.0.1:12345", ".*")
+            new("bluto://127.0.0.1:12345", ".*")
          }
       };
       return rv;

src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnection_APM.cs

@@ -36,14 +36,17 @@ using System.Linq;
 using Bts.PubSub;
 using Spark.Core;
 
-namespace bts {
-   public class AsyncTCPConnection_APM : ITCPConnection, IWorker, IDisposable {
+namespace bts
+{
+   public class AsyncTCPConnection_APM : ITCPConnection, IWorker, IDisposable
+   {
 
       // this class is used as the argument of the completion function for BeginSend/Receive   
       // serves as the base class for both Send and ReeceiveMessage classes.
       // since both TCP sends and receives happen in multiple transfers, this
       // provides a wrapper around copying the partial transfers into the complete buffer
-      public class AsyncMessageBase {
+      public class AsyncMessageBase
+      {
          readonly string msg_id = Guid.NewGuid().ToString();
          // every BeginSend or BeginReceive has a unique key associated with it
          public int key;
@@ -55,37 +58,45 @@ namespace bts {
          public int transfer_ptr;
          protected int _n_transfers = 0;
          public bool completed_sync = false;
-         public AsyncMessageBase(byte[] the_buffer, int the_offset) {
+         public AsyncMessageBase(byte[] the_buffer, int the_offset)
+         {
             buffer = the_buffer;
             transfer_ptr = the_offset;
          }
-         public int remaining_size {
-            get {
+         public int remaining_size
+         {
+            get
+            {
                return buffer.Length - transfer_ptr;
             }
          }
 
          public int original_size => buffer.Length;
 
-         public void advance_transfer_pointer(int advance_by) {
+         public void advance_transfer_pointer(int advance_by)
+         {
             transfer_ptr += advance_by;
             _n_transfers++;
-            if (transfer_ptr > original_size) {
+            if (transfer_ptr > original_size)
+            {
                bts.Log.Gen.error($"transfer went beyond initial size {ToString()}");
             }
             // TODO: log if it goes past the buffer
          }
 
-         public bool fully_transfered() {
+         public bool fully_transfered()
+         {
             return transfer_ptr >= original_size;
          }
 
-         public override string ToString() {
+         public override string ToString()
+         {
             return $"{GetType().ToString()} msg_id: {msg_id} key: {key} original_length: {original_size} transfer_ptr: {transfer_ptr} n_transfers: {_n_transfers} " +
                    $"fully_transfered: {fully_transfered()} completed_sync: {completed_sync}";
          }
       }
-      public class AsyncSendMessage : AsyncMessageBase {
+      public class AsyncSendMessage : AsyncMessageBase
+      {
          public string message_id;
          public SendConfirmationDelegate on_send_del;
          //private int _n_transfers = 0;
@@ -94,35 +105,43 @@ namespace bts {
             : this(the_buffer, the_offset, String.Empty, null) { }
 
          public AsyncSendMessage(byte[] the_buffer, int the_transfer_ptr, string the_message_id, SendConfirmationDelegate the_on_send_del)
-         : base(the_buffer, the_transfer_ptr) {
+         : base(the_buffer, the_transfer_ptr)
+         {
             message_id = the_message_id;
             on_send_del = the_on_send_del;
          }
-         public override string ToString() {
+         public override string ToString()
+         {
             return base.ToString();
          }
 
       }
 
-      public class AsyncReceiveMessage : AsyncMessageBase {
-         public enum ReadState {
+      public class AsyncReceiveMessage : AsyncMessageBase
+      {
+         public enum ReadState
+         {
             reading_size,
             reading_data
          };
          readonly ReadState read_state;
 
          public AsyncReceiveMessage(int size, ReadState the_read_state) :
-         this(new byte[size], 0, the_read_state) { }
+         this(new byte[size], 0, the_read_state)
+         { }
          public AsyncReceiveMessage(byte[] the_buffer, int the_transfer_ptr, ReadState the_read_state)
-         : base(the_buffer, the_transfer_ptr) {
+         : base(the_buffer, the_transfer_ptr)
+         {
             read_state = the_read_state;
          }
 
-         public int read_int32(int start_idx = 0) {
+         public int read_int32(int start_idx = 0)
+         {
             return BitConverter.ToInt32(buffer, start_idx);
          }
 
-         public byte[] read_bytes(int start_idx) {
+         public byte[] read_bytes(int start_idx)
+         {
             byte[] rv = new byte[buffer.Length - start_idx];
             Array.Copy(buffer, start_idx, rv, 0, rv.Length);
             return rv;
@@ -131,16 +150,19 @@ namespace bts {
          public bool is_reading_size() => read_state == ReadState.reading_size;
       }
 
-      public class ErrorMessage {
+      public class ErrorMessage
+      {
          public readonly SocketError? socket_error = null;
          public readonly Exception exception = null;
          public readonly bool is_exception = false;
          public readonly string context = String.Empty;
-         public ErrorMessage(SocketError socket_error_, string context_) {
+         public ErrorMessage(SocketError socket_error_, string context_)
+         {
             socket_error = socket_error_;
             context = context_;
          }
-         public ErrorMessage(Exception exception_, string context_) {
+         public ErrorMessage(Exception exception_, string context_)
+         {
             is_exception = true;
             exception = exception_;
             context = context_;
@@ -148,7 +170,8 @@ namespace bts {
 
          public bool oom => exception != null && exception is OutOfMemoryException;
 
-         public override string ToString() {
+         public override string ToString()
+         {
             if (is_exception)
                return $"ErrorMessage exception: {exception} context: {context}";
             return $"socket_error: {socket_error} context: {context}";
@@ -174,13 +197,15 @@ namespace bts {
       private bool _hb_received_yet = false, _data_received_yet = false, _hb_sent_once = false;
       private DateTime _last_hb_received_time = new DateTime(1900, 1, 1);
 
-      public AsyncTCPConnection_APM(Socket sock, TimeSpan hb_period, ITCPConnection.ConnectionSide connection_side) {
+      public AsyncTCPConnection_APM(Socket sock, TimeSpan hb_period, ITCPConnection.ConnectionSide connection_side)
+      {
          _sock = sock;
          _hb_period = hb_period;
          _connection_side = connection_side;
          _sock.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.DontRoute, true);
          #region testing options
-         switch (connection_side) {
+         switch (connection_side)
+         {
             case ITCPConnection.ConnectionSide.Server:
                TCPSocketUtils.set_server_socket_options(ref _sock);
                break;
@@ -195,10 +220,13 @@ namespace bts {
          _detect_connection_status_change();
       }
 
-      private void _detect_connection_status_change() {
-         Action testing_func = () => {
+      private void _detect_connection_status_change()
+      {
+         Action testing_func = () =>
+         {
             bool connected = _sock.Connected;
-            while (true) {
+            while (true)
+            {
                if (connected != _sock.Connected)
                   Log.Gen.error($"Connection status changed this: {ToString()} {connected} -> {_sock.Connected}");
                connected = _sock.Connected;
@@ -208,26 +236,30 @@ namespace bts {
          ThreadPool.QueueUserWorkItem(_ => testing_func());
       }
 
-      public override string ToString() {
+      public override string ToString()
+      {
          return $"AsyncTCPConnection_APM side: {_connection_side} peer: {_peer_conn_info}, self: {_self_conn_info}";
       }
 
       #region ITCPConnection Members
 
-      public void sendWithConfirmation(byte[] data, ref string conf_id, SendConfirmationDelegate del) {
+      public void sendWithConfirmation(byte[] data, ref string conf_id, SendConfirmationDelegate del)
+      {
          conf_id = Guid.NewGuid().ToString();
          byte[] msg = _build_data_message(data);
          _begin_or_resume_sending(new AsyncSendMessage(msg, 0, conf_id, del));
       }
 
-      public void send(byte[] data) {
-         Log.Gen.info($"AsyncTCPConnection_APM.send data: {AsyncTCPConnection_AsyncWrapper.to_hex(data, 0, data.Length)}");
+      public void send(byte[] data)
+      {
+         //Log.Gen.info($"AsyncTCPConnection_APM.send data: {AsyncTCPConnection_AsyncWrapper.to_hex(data, 0, data.Length)}");
          byte[] msg = _build_data_message(data);
-         Log.Gen.info($"AsyncTCPConnection_APM.send msg: {AsyncTCPConnection_AsyncWrapper.to_hex(msg, 0, msg.Length)}");
+         //Log.Gen.info($"AsyncTCPConnection_APM.send msg: {AsyncTCPConnection_AsyncWrapper.to_hex(msg, 0, msg.Length)}");
          _begin_or_resume_sending(new AsyncSendMessage(msg, 0));
       }
 
-      public void register(ITCPMessageHandler messageHandler) {
+      public void register(ITCPMessageHandler messageHandler)
+      {
          bts.Log.assert_d(_message_handler == null, "Message Handler is already assigned");
          //bts.Log.Gen.info($"AsyncTCPConnection_APM for peer = {_peer_conn_info} being registered");
          _message_handler = messageHandler;
@@ -236,15 +268,18 @@ namespace bts {
          _read_next_msg();
       }
 
-      public RemoteConnectionInfo get_peer_address() {
+      public RemoteConnectionInfo get_peer_address()
+      {
          return TCPSocketUtils.get_peer_address(_sock);
       }
 
-      public RemoteConnectionInfo get_self_address() {
+      public RemoteConnectionInfo get_self_address()
+      {
          return TCPSocketUtils.get_peer_address(_sock);
       }
 
-      public bool isDisposed {
+      public bool isDisposed
+      {
          get { return !_is_alive; }
       }
 
@@ -253,12 +288,14 @@ namespace bts {
       #region IWorker Members
 
       bool _last_connected = false;
-      public void doWork() {
+      public void doWork()
+      {
          if (!_is_alive)
             return;
          return;
          // disable hbs for now
-         if (!_hb_sent_once) {
+         if (!_hb_sent_once)
+         {
             bts.Log.Gen.info("Sending heartbeat to peer {_peer_conn_info} for the first time");
             _hb_sent_once = true;
          }
@@ -266,12 +303,14 @@ namespace bts {
          _begin_or_resume_sending(new AsyncSendMessage(msg, 0));
       }
 
-      private static int _tid() {
+      private static int _tid()
+      {
          var tid = System.Threading.Thread.CurrentThread.ManagedThreadId;
          return tid;
       }
 
-      private static string _cb_txt(IAsyncResult res) {
+      private static string _cb_txt(IAsyncResult res)
+      {
          return $"completed?: {res.IsCompleted} sync?: {res.CompletedSynchronously}";
       }
 
@@ -279,10 +318,13 @@ namespace bts {
 
       #region sending data
 
-      private void _begin_or_resume_sending(AsyncSendMessage message) {
+      private void _begin_or_resume_sending(AsyncSendMessage message)
+      {
          //bts.Log.Gen.info($"AsyncTCPConnection_APM._begin_or_resume_sending this: {ToString()} tid: {_tid()} message: {message} _is_alive: {_is_alive}");
-         if (_is_alive) {
-            try {
+         if (_is_alive)
+         {
+            try
+            {
                SocketError begin_send_err;
                // since we get message on on_send we dont need to keep the state around
                message.key = _send_key++;
@@ -290,35 +332,45 @@ namespace bts {
                                  out begin_send_err, new AsyncCallback(BeginSend_callback_function), message);
                if (begin_send_err != SocketError.Success)
                   _handle_error(new ErrorMessage(begin_send_err, "_begin_or_resume_sending"));
-            } catch (Exception ex) {
+            }
+            catch (Exception ex)
+            {
                _handle_error(new ErrorMessage(ex, "_begin_or_resume_sending"));
             }
          }
       }
 
-      private void BeginSend_callback_function(IAsyncResult res) {
-         if (res.CompletedSynchronously) {
+      private void BeginSend_callback_function(IAsyncResult res)
+      {
+         if (res.CompletedSynchronously)
+         {
             ThreadPool.QueueUserWorkItem(_ => _on_send_async(res));
-         } else
+         }
+         else
             _on_send_async(res);
       }
 
-      private void _on_send_async(IAsyncResult res) {
+      private void _on_send_async(IAsyncResult res)
+      {
          AsyncSendMessage message = res.AsyncState as AsyncSendMessage;
-         if (_send_key_map.Contains(message.key)) {
+         if (_send_key_map.Contains(message.key))
+         {
             bts.Log.Gen.warn($"AsyncTCPConnection_APM._on_send key: {message.key} already handled");
             return;
          }
          _send_key_map.Add(message.key);
          // bts.Log.Gen.info($"AsyncTCPConnection_APM._on_send: pre endSend this: {ToString()} tid: {_tid()} " +
          //                         $"res: {_cb_txt(res)} message: {res.AsyncState} _is_alive: {_is_alive}");
-         if (_is_alive) {
-            try {
+         if (_is_alive)
+         {
+            try
+            {
                SocketError send_err;
                int bytes_sent = _sock.EndSend(res, out send_err);
                //message.offset += bytesSent;
                message.advance_transfer_pointer(bytes_sent);
-               if (bytes_sent == 0) {
+               if (bytes_sent == 0)
+               {
                   bts.Log.Gen.error($"AsyncTCPConnection_APM._on_send this: {this} sent 0 bytes, throwing exception");
                   _handle_error(new ErrorMessage(new ApplicationException("Sent 0 bytes, peer socket is likely shutdown"),
                         "_on_send"));
@@ -326,22 +378,31 @@ namespace bts {
                }
                bts.Log.Gen.info($"AsyncTCPConnection_APM._on_send: post endSend this: {ToString()} tid: {_tid()} res: {_cb_txt(res)} " +
                $"message: {res.AsyncState} bytes_sent: {bytes_sent} _is_alive: {_is_alive} send_err: {send_err}");
-               if (send_err == SocketError.Success) {
+               if (send_err == SocketError.Success)
+               {
                   // successful send of some bytes
                   //if (message.offset == message.buffer.Length) {
-                  if (message.fully_transfered()) {
+                  if (message.fully_transfered())
+                  {
                      // fully sent, if confirmation needs to be sent out                     
-                     if (!String.IsNullOrEmpty(message.message_id)) {
+                     if (!String.IsNullOrEmpty(message.message_id))
+                     {
                         message.on_send_del(message.message_id, true);
                      }
-                  } else {
+                  }
+                  else
+                  {
                      // there is more to send
                      _begin_or_resume_sending(message);
                   }
-               } else {
+               }
+               else
+               {
                   _handle_error(new ErrorMessage(send_err, "_on_send"));
                }
-            } catch (Exception ex) {
+            }
+            catch (Exception ex)
+            {
                _handle_error(new ErrorMessage(ex, "_on_send"));
             }
          }
@@ -352,8 +413,10 @@ namespace bts {
       /// future references
       /// </summary>
       /// <returns></returns>
-      private byte[] _build_hb_message() {
-         if (_cached_hb_message == null) {
+      private byte[] _build_hb_message()
+      {
+         if (_cached_hb_message == null)
+         {
             BinaryWriter bw = new BinaryWriter(new MemoryStream());
             // the total msg =[msg_sz, msg_type] 
             int msg_sz = 4;
@@ -365,7 +428,8 @@ namespace bts {
       }
 
       // writes size prefixed msg
-      private byte[] _build_data_message(byte[] data) {
+      private byte[] _build_data_message(byte[] data)
+      {
          // TODO, avoid unecessary memory creation (write directly into byte[] using BitConverter)
          MemoryStream str = new MemoryStream();
          BinaryWriter bw = new BinaryWriter(str);
@@ -384,28 +448,34 @@ namespace bts {
 
       #region receiving data
 
-      private void _read_next_msg() {
+      private void _read_next_msg()
+      {
          _read_size();
       }
 
-      private void _read_size() {
+      private void _read_size()
+      {
          _begin_or_resume_reading(new AsyncReceiveMessage(4, AsyncReceiveMessage.ReadState.reading_size));
       }
 
-      private void _read_data(int data_size) {
+      private void _read_data(int data_size)
+      {
          // TODO, this could be a problem is the size is too big - maybe read it in chunks (later)
          _begin_or_resume_reading(new AsyncReceiveMessage(data_size, AsyncReceiveMessage.ReadState.reading_data));
       }
 
-      private static string IAsyncResult_txt(IAsyncResult res) {
+      private static string IAsyncResult_txt(IAsyncResult res)
+      {
          return $"IsCompleted: {res.IsCompleted} CompletedSynchronously: {res.CompletedSynchronously}";
       }
 
-      private void _begin_or_resume_reading(AsyncReceiveMessage rcv_msg) {
+      private void _begin_or_resume_reading(AsyncReceiveMessage rcv_msg)
+      {
          //bts.Log.Gen.info($"AsyncTCPConnection_APM._begin_or_resume_reading: this: {ToString()} tid: {_tid()} message: {rcv_msg} _is_alive: {_is_alive}");
          if (!_is_alive)
             return;
-         try {
+         try
+         {
             SocketError read_sock_err;
             rcv_msg.key = _receive_key++;
             var res = _sock.BeginReceive(rcv_msg.buffer, rcv_msg.transfer_ptr,
@@ -413,50 +483,61 @@ namespace bts {
                                  SocketFlags.None, out read_sock_err,
                                  new AsyncCallback(BeginReceive_callback_function),
                                  rcv_msg);
-            if (read_sock_err != SocketError.Success) {
+            if (read_sock_err != SocketError.Success)
+            {
                bts.Log.Gen.error($"_begin_or_resume_reading this: {this} error: {read_sock_err} rcv_msg: {rcv_msg}");
                _handle_error(new ErrorMessage(read_sock_err, "_beging_or_resume_reading"));
             }
-         } catch (Exception ex) {
+         }
+         catch (Exception ex)
+         {
             _handle_error(new ErrorMessage(ex, "_beging_or_resume_reading"));
          }
       }
 
-      private void _log_incoming_msg(IAsyncResult res) {
-         AsyncReceiveMessage rcv_message = res.AsyncState as AsyncReceiveMessage;
-         var buff_txt = AsyncTCPConnection_AsyncWrapper.to_hex(rcv_message.buffer, 0, rcv_message.buffer.Length);
-         Log.Gen.info($"BeginReceive_callback_function _log_incoming_msg buff: {buff_txt}");
+      private void _log_incoming_msg(IAsyncResult res)
+      {
+         // AsyncReceiveMessage rcv_message = res.AsyncState as AsyncReceiveMessage;
+         // var buff_txt = AsyncTCPConnection_AsyncWrapper.to_hex(rcv_message.buffer, 0, rcv_message.buffer.Length);
+         // Log.Gen.info($"BeginReceive_callback_function _log_incoming_msg buff: {buff_txt}");
 
       }
 
       // this is the function passed to BeginReceive, we 
-      private void BeginReceive_callback_function(IAsyncResult res) {
+      private void BeginReceive_callback_function(IAsyncResult res)
+      {
          _log_incoming_msg(res);
-         if (res.CompletedSynchronously) {
+         if (res.CompletedSynchronously)
+         {
             ThreadPool.QueueUserWorkItem(_ => _on_read_async(res));
-         } else
+         }
+         else
             _on_read_async(res);
       }
 
       // this will always be called on the non main-thread. the reason this is on its own thread is that this function ends up
       // calling _begin_or_resume_reading, which in tuns calls _on_read_async. So to not cause a stack overflow, this is broken
       // always called on a different thread than _begin_or_resume_reading
-      private void _on_read_async(IAsyncResult res) {
+      private void _on_read_async(IAsyncResult res)
+      {
          AsyncReceiveMessage rcv_message = res.AsyncState as AsyncReceiveMessage;
-         if (_read_key_map.Contains(rcv_message.key)) {
+         if (_read_key_map.Contains(rcv_message.key))
+         {
             bts.Log.Gen.warn($"AsyncTCPConnection_APM._on_read key: {rcv_message.key} already handled");
             return;
          }
          _read_key_map.Add(rcv_message.key);
          if (!_is_alive)
             return;
-         try {
+         try
+         {
             SocketError read_err;
             Log.Gen.info($"AsyncTCPConnection_APM._on_read pre-EndReceive rcv_message: {rcv_message} res: {IAsyncResult_txt(res)}");
             int bytes_read = _sock.EndReceive(res, out read_err);
             rcv_message.advance_transfer_pointer(bytes_read);
             Log.Gen.info($"AsyncTCPConnection_APM._on_read post-EndReceive rcv_message: {rcv_message} res: {IAsyncResult_txt(res)} bytes_read: {bytes_read}");
-            if (bytes_read == 0) {
+            if (bytes_read == 0)
+            {
                bts.Log.Gen.error($"AsyncTCPConnection_APM._on_read this: {this} read 0 bytes, throwing exception");
                //bts.Log.Gen.error($"AsyncTCPConnection_APM._on_read this: {this} read 0 bytes, sleeping and ignoring");
                //System.Threading.Thread.Sleep(50);
@@ -464,23 +545,32 @@ namespace bts {
                return;
             }
             //return;
-            if (read_err == SocketError.Success) {
-               if (rcv_message.fully_transfered()) {
-                  if (rcv_message.is_reading_size()) {
+            if (read_err == SocketError.Success)
+            {
+               if (rcv_message.fully_transfered())
+               {
+                  if (rcv_message.is_reading_size())
+                  {
                      var msg_size = _on_read_size_completely(rcv_message);
-                     if (msg_size != null && msg_size.HasValue) {
+                     if (msg_size != null && msg_size.HasValue)
+                     {
                         _read_data(msg_size.Value);
                      }
                      return;
-                  } else {
+                  }
+                  else
+                  {
                      // data is read
                      int msg_type = rcv_message.read_int32();
                      bts.Log.assert(msg_type == HB_MSG_SIGNAL || msg_type == DATA_MSG_SIGNAL);
-                     if (msg_type == DATA_MSG_SIGNAL) {
+                     if (msg_type == DATA_MSG_SIGNAL)
+                     {
                         _on_read_data_completely(rcv_message);
                         _read_next_msg();
                         return;
-                     } else if (msg_type == HB_MSG_SIGNAL) {
+                     }
+                     else if (msg_type == HB_MSG_SIGNAL)
+                     {
                         _on_read_hb_completely(rcv_message);
                         _read_next_msg();
                         return;
@@ -488,45 +578,57 @@ namespace bts {
                      // go back to reading size for a new message
                      //_read_size();
                   }
-               } else {
+               }
+               else
+               {
                   // msg has not been fully rd, we have already advanced the transfer ptr
                   _begin_or_resume_reading(rcv_msg: rcv_message);
                }
-            } else {
+            }
+            else
+            {
                _handle_error(new ErrorMessage(read_err, "_on_read_async"));
             }
-         } catch (Exception ex) {
+         }
+         catch (Exception ex)
+         {
             _handle_error(new ErrorMessage(ex, "_on_read_async"));
          }
       }
 
       #endregion
 
-      private int? _on_read_size_completely(AsyncReceiveMessage msg) {
+      private int? _on_read_size_completely(AsyncReceiveMessage msg)
+      {
          bts.Log.assert(msg.is_reading_size());
          bts.Log.assert(msg.fully_transfered());
          bts.Log.assert(msg.original_size == 4);
          var msg_sz = msg.read_int32();
-         var buff_txt = AsyncTCPConnection_AsyncWrapper.to_hex(msg.buffer, 0, msg.buffer.Length);
-         bts.Log.Gen.info($"AsyncTCPConnection_APM._on_read_size_completely buff: {buff_txt}");
+         //var buff_txt = AsyncTCPConnection_AsyncWrapper.to_hex(msg.buffer, 0, msg.buffer.Length);
+         //bts.Log.Gen.info($"AsyncTCPConnection_APM._on_read_size_completely buff: {buff_txt}");
          bts.Log.Gen.info($"AsyncTCPConnection_APM._on_read_size_completely msg: {msg} msg_sz: {msg_sz}");
-         if (msg_sz < 0 || msg_sz > _max_msg_size) {
+         if (msg_sz < 0 || msg_sz > _max_msg_size)
+         {
             string err_msg = $"Encountered malformed message, prefixed_size: {msg_sz}";
             _handle_error(new ErrorMessage(new ApplicationException(err_msg), "_on_read_size_completely"));
             return null;
-         } else {
+         }
+         else
+         {
             // sz is determined, now we read that many bytes
             //_read_data(msg_sz);
             return msg_sz;
          }
       }
 
-      private void _on_read_hb_completely(AsyncReceiveMessage msg) {
+      private void _on_read_hb_completely(AsyncReceiveMessage msg)
+      {
          bts.Log.assert(!msg.is_reading_size());
          bts.Log.assert(msg.fully_transfered());
          bts.Log.assert(msg.original_size == 4);
          bts.Log.assert(msg.read_int32(0) == HB_MSG_SIGNAL);
-         if (!_hb_received_yet) {
+         if (!_hb_received_yet)
+         {
             _hb_received_yet = true;
             //bts.Log.Gen.info($"AsyncTCPConnection_APM HB received for the first time from peer {_peer_conn_info}");
          }
@@ -534,7 +636,8 @@ namespace bts {
          _message_handler.onData(new RemoteMessage(null, RemoteMessageType.Heartbeat, _peer_conn_info));
       }
 
-      private void _on_read_data_completely(AsyncReceiveMessage msg) {
+      private void _on_read_data_completely(AsyncReceiveMessage msg)
+      {
          bts.Log.assert(!msg.is_reading_size());
          bts.Log.assert(msg.fully_transfered());
          var type = msg.read_int32(0);
@@ -543,7 +646,8 @@ namespace bts {
          byte[] data = msg.read_bytes(4);
          ///callbacks can be made through thread pool to avoid this
          ///thread from being blocked, by this callback
-         if (!_data_received_yet) {
+         if (!_data_received_yet)
+         {
             _data_received_yet = true;
             //bts.Log.Gen.info($"AsyncTCPConnection_APM data received for the first time from peer {_peer_conn_info}");
          }
@@ -553,18 +657,24 @@ namespace bts {
 
       #region IDisposable Members
 
-      public void Dispose() {
+      public void Dispose()
+      {
          // socket is closed and disposed here
-         lock (this) {
+         lock (this)
+         {
             bts.Log.Gen.info($"AsyncTCPConnection_APM.Dispose this: {this} Dispose called _dipose_impetus: {_dipose_impetus}");
-            if (_is_alive) {
+            if (_is_alive)
+            {
                _master_worker.removeWorker(this);
                _is_alive = false;
                bts.Log.Gen.info($"AsyncTCPConnection_APM.Dispose this: {this} being disposed, last HB was received at {_last_hb_received_time}");
-               try {
+               try
+               {
                   bts.Log.Gen.info($"AsyncTCPConnection_APM.Dispose this: {this} shutting down socket now");
                   _sock.Shutdown(SocketShutdown.Both);
-               } catch (Exception ex) {
+               }
+               catch (Exception ex)
+               {
                   String.Format("AsyncTCPConnection_APM.Dispose exception: {ex} thrown when closing socket");
                   if (ex is OutOfMemoryException)
                      throw ex;
@@ -577,8 +687,10 @@ namespace bts {
       #endregion
 
       private string _dipose_impetus = String.Empty;
-      private void _handle_error(ErrorMessage err_msg) {
-         lock (this) {
+      private void _handle_error(ErrorMessage err_msg)
+      {
+         lock (this)
+         {
             Log.Gen.error($"AsyncTCPConnection_APM._handle_error this: {this} error_msg: {err_msg}");
             if (err_msg.oom)
                throw err_msg.exception;

src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnection_AsyncWrapper.cs

@@ -1,4 +1,5 @@
-namespace bts {
+namespace bts
+{
    using BCG;
    using BCG.Utilities.General;
    using System;
@@ -11,15 +12,19 @@ namespace bts {
 
    // this ITCPConnection uses the TCPClient class instead of a raw socket (in the hope, that it would configure the socket correctly)
 
-   public class AsyncTCPConnection_AsyncWrapper : ITCPConnection, IWorker, IDisposable {
-      public class ConnectionID {
+   public class AsyncTCPConnection_AsyncWrapper : ITCPConnection, IWorker, IDisposable
+   {
+      public class ConnectionID
+      {
          public readonly RemoteConnectionInfo self;
          public readonly RemoteConnectionInfo peer;
          public readonly Type conn_type;
-         public override string ToString() {
+         public override string ToString()
+         {
             return $"{conn_type} self: {self} peer: {peer}";
          }
-         public ConnectionID(RemoteConnectionInfo self_, RemoteConnectionInfo peer_, Type conn_type_) {
+         public ConnectionID(RemoteConnectionInfo self_, RemoteConnectionInfo peer_, Type conn_type_)
+         {
             self = self_;
             peer = peer_;
             conn_type = conn_type_;
@@ -33,30 +38,34 @@ namespace bts {
       private bool _is_alive = true;
       ConnectionID _connection_id;
 
-      public static string to_hex(byte[] byte_arr, int start_pos, int end_pos) {
-         var len = end_pos - start_pos;
-         StringBuilder hex_builder = new StringBuilder(len * 2);
-         int i = 0;
+      // public static string to_hex(byte[] byte_arr, int start_pos, int end_pos) {
+      //    var len = end_pos - start_pos;
+      //    StringBuilder hex_builder = new StringBuilder(len * 2);
+      //    int i = 0;
 
-         foreach (byte b in byte_arr) {
-            if (i >= start_pos && i < end_pos)
-               hex_builder.Append($"{b:X2} ");
-            i++;
-         }
-         return hex_builder.ToString().Trim();
-      }
+      //    foreach (byte b in byte_arr) {
+      //       if (i >= start_pos && i < end_pos)
+      //          hex_builder.Append($"{b:X2} ");
+      //       i++;
+      //    }
+      //    return hex_builder.ToString().Trim();
+      // }
 
-      public AsyncTCPConnection_AsyncWrapper(TcpClient tcp_client) {
+      public AsyncTCPConnection_AsyncWrapper(TcpClient tcp_client)
+      {
          _tcp_client = tcp_client;
          _nw_stream = _tcp_client.GetStream();
          _connection_id = new ConnectionID(TCPSocketUtils.get_self_address(_tcp_client.Client), TCPSocketUtils.get_peer_address(_tcp_client.Client), GetType());
          Log.Gen.info("AsyncTCPConnection_AsyncWrapper.cons connection: {_connection_id}");
       }
 
-      private void _detect_connection_status_change() {
-         Action testing_func = () => {
+      private void _detect_connection_status_change()
+      {
+         Action testing_func = () =>
+         {
             bool connected = _tcp_client.Connected;
-            while (true) {
+            while (true)
+            {
                if (connected != _tcp_client.Connected)
                   Log.Gen.error($"Connection status changed this: {ToString()} {connected} -> {_tcp_client.Connected}");
                connected = _tcp_client.Connected;
@@ -69,14 +78,17 @@ namespace bts {
       }
 
       #region Sending 
-      public void _send_fixed_size_msg(byte[] buffer) {
-         //Log.Gen.info($"AsyncTCPConnection_AsyncWrapper._send_fixed_size_msg buffer: {to_hex(buffer, 0, buffer.Length)}");
+      public void _send_fixed_size_msg(byte[] buffer)
+      {
+         // string buff_txt = to_hex(buffer, 0, buffer.Length);
+         // Log.Gen.info($"AsyncTCPConnection_AsyncWrapper._send_fixed_size_msg buffer: {buff_txt}");
 
          int total_bytes_sent = 0;
          int n_bytes = buffer.Length;
 
          int n_transfers = 0;
-         while (_is_alive && total_bytes_sent < n_bytes) {
+         while (_is_alive && total_bytes_sent < n_bytes)
+         {
             int remaining_bytes = n_bytes - total_bytes_sent;
             _nw_stream.Write(buffer, total_bytes_sent, remaining_bytes);
             total_bytes_sent += remaining_bytes;
@@ -85,8 +97,10 @@ namespace bts {
          Log.Gen.info($"AsyncTCPConnection_AsyncWrapper._send_fixed_size_msg sent {buffer.Length} bytes in {n_transfers} transfers");
       }
 
-      private void _send_one_msg(byte[] msg) {
-         try {
+      private void _send_one_msg(byte[] msg)
+      {
+         try
+         {
             // the total msg = [msg_sz, msg_type, data] 
             int sz = msg.Length + 4;
             MemoryStream str = new MemoryStream();
@@ -97,15 +111,20 @@ namespace bts {
             str.Position = 0;
             byte[] buff = str.ToArray();
             _send_fixed_size_msg(buff);
-         } catch (Exception ex) {
+         }
+         catch (Exception ex)
+         {
             _handle_error(new ErrorMessage(ex, $"_send_one_msg call_stacK: {ex.StackTrace}"));
          }
       }
 
-      private void _send_forever() {
-         while (_is_alive) {
+      private void _send_forever()
+      {
+         while (_is_alive)
+         {
             byte[] data = _send_queue.dequeue();
-            if (data != null) {
+            if (data != null)
+            {
                _send_one_msg(data);
             }
          }
@@ -118,39 +137,47 @@ namespace bts {
       }
       #endregion
 
-      public void register(ITCPMessageHandler message_handler) {
+      public void register(ITCPMessageHandler message_handler)
+      {
          _msg_handler = message_handler;
          // starting thread pool threads to keep reading and sending
          ThreadPool.QueueUserWorkItem(_ => _read_msgs_forever());
          ThreadPool.QueueUserWorkItem(_ => _send_forever());
       }
 
-      public RemoteConnectionInfo get_peer_address() {
+      public RemoteConnectionInfo get_peer_address()
+      {
          return _connection_id.peer;
       }
 
-      public RemoteConnectionInfo get_self_address() {
+      public RemoteConnectionInfo get_self_address()
+      {
          return _connection_id.self;
       }
 
       public bool isDisposed { get { return !_is_alive; } }
 
-      public void doWork() {
+      public void doWork()
+      {
          // send hb
       }
 
       #region Reading
       // This makes the blocking IO call, the callstack is _read_msgs_forever -> _read_one_message -> _read_fixed_size_msg
-      private byte[] _read_fixed_size_msg(int n_bytes) {
-         if (n_bytes <= 0) {
+      private byte[] _read_fixed_size_msg(int n_bytes)
+      {
+         if (n_bytes <= 0)
+         {
             throw new ArgumentException("Number of bytes to read must be greater than zero.", nameof(n_bytes));
          }
          byte[] buffer = new byte[n_bytes];
          int total_bytes_read = 0;
          int n_transfers = 0;
-         while (_is_alive && total_bytes_read < n_bytes) {
+         while (_is_alive && total_bytes_read < n_bytes)
+         {
             int bytes_read = _nw_stream.Read(buffer, total_bytes_read, n_bytes - total_bytes_read);
-            if (bytes_read <= 0) {
+            if (bytes_read <= 0)
+            {
                throw new IOException("Read 0 bytes, the connection was closed before all data could be read.");
             }
             n_transfers++;
@@ -161,37 +188,44 @@ namespace bts {
          return buffer;
       }
 
-      private class OneRecvMessage {
+      private class OneRecvMessage
+      {
          public int msg_type;
          public byte[] data;
-         public OneRecvMessage(int msg_type_, byte[] data_) {
+         public OneRecvMessage(int msg_type_, byte[] data_)
+         {
             msg_type = msg_type_;
             data = data_;
          }
-         public OneRecvMessage(byte[] msg) {
+         public OneRecvMessage(byte[] msg)
+         {
             MemoryStream str = new MemoryStream(msg);
             BinaryReader br = new BinaryReader(str);
             int msg_type = br.ReadInt32();
-            if (msg_type == DATA_MSG_SIGNAL) {
+            if (msg_type == DATA_MSG_SIGNAL)
+            {
                // TODO, polish it, too much memory bneing allocated
                data = br.ReadBytes(msg.Length - 4);
             }
          }
 
-         public override string ToString() {
+         public override string ToString()
+         {
             string rv = $"OneMessage msg_type: {msg_type} data_len: {data.Length}";
             return rv;
          }
       }
 
-      private OneRecvMessage _read_one_message() {
+      private OneRecvMessage _read_one_message()
+      {
          // read size first
          byte[] sz_arr = _read_fixed_size_msg(4);
          BinaryReader br = new BinaryReader(new MemoryStream(sz_arr));
          br.BaseStream.Position = 0;
          int msg_sz = br.ReadInt32();
          Log.Gen.info($"AsyncTCPConnection_AsyncWrapper._read_one_message read msg_sz: {msg_sz}");
-         if (msg_sz < 0) {
+         if (msg_sz < 0)
+         {
             throw new IOException("Read 0 bytes, the connection was closed before all data could be read.");
          }
          byte[] msg = _read_fixed_size_msg(msg_sz);
@@ -201,19 +235,29 @@ namespace bts {
          return rv;
       }
 
-      private void _read_msgs_forever() {
-         while (_is_alive) {
-            try {
+      private void _read_msgs_forever()
+      {
+         while (_is_alive)
+         {
+            try
+            {
                OneRecvMessage one_msg = _read_one_message();
-               if (one_msg.msg_type == DATA_MSG_SIGNAL) {
+               if (one_msg.msg_type == DATA_MSG_SIGNAL)
+               {
                   _msg_handler.onData(new RemoteMessage(one_msg.data, RemoteMessageType.Data, _connection_id.peer));
-               } else if (one_msg.msg_type == HB_MSG_SIGNAL) {
+               }
+               else if (one_msg.msg_type == HB_MSG_SIGNAL)
+               {
                   // keep a last hb time an dudpate it here atomically
                   _msg_handler.onData(new RemoteMessage(null, RemoteMessageType.Heartbeat, _connection_id.peer));
-               } else {
+               }
+               else
+               {
                   throw new ApplicationException($"Unknown message type: {one_msg.msg_type}");
                }
-            } catch (Exception ex) {
+            }
+            catch (Exception ex)
+            {
                _handle_error(new ErrorMessage(ex, $"_read_msgs_forever, call_stack: {ex.StackTrace}"));
             }
          }
@@ -223,16 +267,19 @@ namespace bts {
 
       public bool is_disposed => !_is_alive;
 
-      public class ErrorMessage {
+      public class ErrorMessage
+      {
          public readonly SocketError? socket_error = null;
          public readonly Exception exception = null;
          public readonly bool is_exception = false;
          public readonly string context = String.Empty;
-         public ErrorMessage(SocketError socket_error_, string context_) {
+         public ErrorMessage(SocketError socket_error_, string context_)
+         {
             socket_error = socket_error_;
             context = context_;
          }
-         public ErrorMessage(Exception exception_, string context_) {
+         public ErrorMessage(Exception exception_, string context_)
+         {
             is_exception = true;
             exception = exception_;
             context = context_;
@@ -240,23 +287,30 @@ namespace bts {
 
          public bool oom => exception != null && exception is OutOfMemoryException;
 
-         public override string ToString() {
+         public override string ToString()
+         {
             if (is_exception)
                return $"ErrorMessage exception: {exception} context: {context}";
             return $"socket_error: {socket_error} context: {context}";
          }
       }
 
-      public void Dispose() {
+      public void Dispose()
+      {
          // socket is closed and disposed here
-         lock (this) {
-            if (_is_alive) {
+         lock (this)
+         {
+            if (_is_alive)
+            {
                //_master_worker.removeWorker(this);
                _is_alive = false;
-               try {
+               try
+               {
                   bts.Log.Gen.info($"AsyncTCPConnection_APM.Dispose this: {this} closing tcp_client nin an orderly fashion now");
                   _tcp_client.Close();
-               } catch (Exception ex) {
+               }
+               catch (Exception ex)
+               {
                   String.Format("AsyncTCPConnection_APM.Dispose exception: {ex} thrown when closing socket");
                   if (ex is OutOfMemoryException)
                      throw ex;
@@ -266,8 +320,10 @@ namespace bts {
          }
       }
 
-      private void _handle_error(ErrorMessage err_msg) {
-         lock (this) {
+      private void _handle_error(ErrorMessage err_msg)
+      {
+         lock (this)
+         {
             Log.Gen.error($"AsyncTCPConnection_APM._handle_error this: {this} error_msg: {err_msg}");
             if (err_msg.oom)
                throw err_msg.exception;
@@ -275,7 +331,8 @@ namespace bts {
          }
       }
 
-      public override string ToString() {
+      public override string ToString()
+      {
          return _connection_id.ToString();
       }
    }

src/DataStreamSuite/PubSub/Server/Communication/ManagedTCP/ManagedTCPMessageHub.cs

@@ -285,4 +285,4 @@ namespace BCG {
 
 
 
-}
+}

src/DataStreamSuite/PubSub/Server/PubSubSingleTagState.cs

@@ -1,126 +1,126 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
+// using System;
+// using System.Collections.Generic;
+// using System.Linq;
+// using System.Text;
 
-/*
- *  This class contains the recent channges of state of a single pubsub tag, with respect to who has subscribed, who has published, etc. 
- *  It has a list of new subscription events. 
- * 
- */
-namespace BCG {
-   public class PubSubSingleTagState {
-      private string myTag;
-      public string tag {
-         get { return myTag; }
-         set { myTag = value; }
+// /*
+//  *  This class contains the recent channges of state of a single pubsub tag, with respect to who has subscribed, who has published, etc. 
+//  *  It has a list of new subscription events. 
+//  * 
+//  */
+// namespace BCG {
+//    public class PubSubSingleTagState {
+//       private string myTag;
+//       public string tag {
+//          get { return myTag; }
+//          set { myTag = value; }
 
-      }
-      //we have this here as SortedList so that we can compare for equality
-      private SortedList<string, string> myPublishers;
-      private SortedList<string, string> mySubscribers;
-      private Dictionary<string, PubSubSingleTagStateEvent> myNewSubscribers;
-      private Dictionary<string, PubSubSingleTagStateEvent> myRemovedSubscribers;
-      private Dictionary<string, PubSubSingleTagStateEvent> myNewPublishers;
-      private Dictionary<string, PubSubSingleTagStateEvent> myRemovedPublishers;
+//       }
+//       //we have this here as SortedList so that we can compare for equality
+//       private SortedList<string, string> myPublishers;
+//       private SortedList<string, string> mySubscribers;
+//       private Dictionary<string, PubSubSingleTagStateEvent> myNewSubscribers;
+//       private Dictionary<string, PubSubSingleTagStateEvent> myRemovedSubscribers;
+//       private Dictionary<string, PubSubSingleTagStateEvent> myNewPublishers;
+//       private Dictionary<string, PubSubSingleTagStateEvent> myRemovedPublishers;
 
 
 
-      public PubSubSingleTagState() {
-         //the total state of the tag
-         myPublishers = new SortedList<string, string>();
-         mySubscribers = new SortedList<string, string>();
+//       public PubSubSingleTagState() {
+//          //the total state of the tag
+//          myPublishers = new SortedList<string, string>();
+//          mySubscribers = new SortedList<string, string>();
 
-         //how has the state changed recentlty
-         myNewSubscribers = new Dictionary<string, PubSubSingleTagStateEvent>();
-         myNewPublishers = new Dictionary<string, PubSubSingleTagStateEvent>();
-         myRemovedPublishers = new Dictionary<string, PubSubSingleTagStateEvent>();
-         myRemovedSubscribers = new Dictionary<string, PubSubSingleTagStateEvent>();
-      }
+//          //how has the state changed recentlty
+//          myNewSubscribers = new Dictionary<string, PubSubSingleTagStateEvent>();
+//          myNewPublishers = new Dictionary<string, PubSubSingleTagStateEvent>();
+//          myRemovedPublishers = new Dictionary<string, PubSubSingleTagStateEvent>();
+//          myRemovedSubscribers = new Dictionary<string, PubSubSingleTagStateEvent>();
+//       }
 
-      //returns a list of all the events that need to be published.
-      public PubSubSingleTagStateEvent[] getNewEventsForPublishing() {
-         PubSubSingleTagStateEvent[] newEvents = new PubSubSingleTagStateEvent[myNewSubscribers.Count + myNewPublishers.Count + myRemovedPublishers.Count + myRemovedSubscribers.Count];
+//       //returns a list of all the events that need to be published.
+//       public PubSubSingleTagStateEvent[] getNewEventsForPublishing() {
+//          PubSubSingleTagStateEvent[] newEvents = new PubSubSingleTagStateEvent[myNewSubscribers.Count + myNewPublishers.Count + myRemovedPublishers.Count + myRemovedSubscribers.Count];
 
-         myNewSubscribers.Values.CopyTo(newEvents, 0);
-         int index = myNewSubscribers.Count;
-         myNewPublishers.Values.CopyTo(newEvents, index);
-         index += myNewPublishers.Count;
-         myRemovedSubscribers.Values.CopyTo(newEvents, index);
-         index += myRemovedSubscribers.Count;
-         myRemovedPublishers.Values.CopyTo(newEvents, index);
+//          myNewSubscribers.Values.CopyTo(newEvents, 0);
+//          int index = myNewSubscribers.Count;
+//          myNewPublishers.Values.CopyTo(newEvents, index);
+//          index += myNewPublishers.Count;
+//          myRemovedSubscribers.Values.CopyTo(newEvents, index);
+//          index += myRemovedSubscribers.Count;
+//          myRemovedPublishers.Values.CopyTo(newEvents, index);
 
-         return newEvents;
+//          return newEvents;
 
-      }
+//       }
 
-      //call this after you have published the new events
-      //this is made public for code readability reasons 
-      //i.e. so that someone reading the SubInfoStateWatcher can understand what's going on more easily
-      public void resetNewEvents() {
-         myNewSubscribers = new Dictionary<string, PubSubSingleTagStateEvent>();
-         myNewPublishers = new Dictionary<string, PubSubSingleTagStateEvent>();
-         myRemovedSubscribers = new Dictionary<string, PubSubSingleTagStateEvent>();
-         myRemovedPublishers = new Dictionary<string, PubSubSingleTagStateEvent>();
-      }
+//       //call this after you have published the new events
+//       //this is made public for code readability reasons 
+//       //i.e. so that someone reading the SubInfoStateWatcher can understand what's going on more easily
+//       public void resetNewEvents() {
+//          myNewSubscribers = new Dictionary<string, PubSubSingleTagStateEvent>();
+//          myNewPublishers = new Dictionary<string, PubSubSingleTagStateEvent>();
+//          myRemovedSubscribers = new Dictionary<string, PubSubSingleTagStateEvent>();
+//          myRemovedPublishers = new Dictionary<string, PubSubSingleTagStateEvent>();
+//       }
 
-      public void newPublish(PubSubSingleTagStateEvent newPublisher) {
-         //don't do anything here if we already knew about the publishers because we were already dealing with this publishers
-         if (!myPublishers.ContainsKey(newPublisher.app)) {
-            myPublishers.Add(newPublisher.app, newPublisher.app);
+//       public void newPublish(PubSubSingleTagStateEvent newPublisher) {
+//          //don't do anything here if we already knew about the publishers because we were already dealing with this publishers
+//          if (!myPublishers.ContainsKey(newPublisher.app)) {
+//             myPublishers.Add(newPublisher.app, newPublisher.app);
 
-            //if we were going to delete it, that means it was already being published by this app
-            //that means we should no logner delete it, but we don't have to announce that it was published again
-            if (myRemovedPublishers.ContainsKey(newPublisher.app))
-               myRemovedPublishers.Remove(newPublisher.app);
-            else
-               myNewPublishers[newPublisher.app] = newPublisher;
+//             //if we were going to delete it, that means it was already being published by this app
+//             //that means we should no logner delete it, but we don't have to announce that it was published again
+//             if (myRemovedPublishers.ContainsKey(newPublisher.app))
+//                myRemovedPublishers.Remove(newPublisher.app);
+//             else
+//                myNewPublishers[newPublisher.app] = newPublisher;
 
 
-         }
-      }
+//          }
+//       }
 
-      public void removePublisher(PubSubSingleTagStateEvent removedPublisher) {
-         if (myPublishers.ContainsKey(removedPublisher.app)) {
-            myPublishers.Remove(removedPublisher.app);
+//       public void removePublisher(PubSubSingleTagStateEvent removedPublisher) {
+//          if (myPublishers.ContainsKey(removedPublisher.app)) {
+//             myPublishers.Remove(removedPublisher.app);
 
-            //same kind of logic here. If we were about to publish this new publisher, we no longer need to announce it
-            if (myNewPublishers.ContainsKey(removedPublisher.app))
-               myNewPublishers.Remove(removedPublisher.app);
-            else
-               myRemovedPublishers[removedPublisher.app] = removedPublisher;
-         }
+//             //same kind of logic here. If we were about to publish this new publisher, we no longer need to announce it
+//             if (myNewPublishers.ContainsKey(removedPublisher.app))
+//                myNewPublishers.Remove(removedPublisher.app);
+//             else
+//                myRemovedPublishers[removedPublisher.app] = removedPublisher;
+//          }
 
-      }
+//       }
 
-      public void newSubscription(PubSubSingleTagStateEvent newSubscriber) {
-         if (!mySubscribers.ContainsKey(newSubscriber.app)) {
-            mySubscribers.Add(newSubscriber.app, newSubscriber.app);
+//       public void newSubscription(PubSubSingleTagStateEvent newSubscriber) {
+//          if (!mySubscribers.ContainsKey(newSubscriber.app)) {
+//             mySubscribers.Add(newSubscriber.app, newSubscriber.app);
 
-            if (myRemovedSubscribers.ContainsKey(newSubscriber.app))
-               myRemovedSubscribers.Remove(newSubscriber.app);
-            else
-               myNewSubscribers[newSubscriber.app] = newSubscriber;
+//             if (myRemovedSubscribers.ContainsKey(newSubscriber.app))
+//                myRemovedSubscribers.Remove(newSubscriber.app);
+//             else
+//                myNewSubscribers[newSubscriber.app] = newSubscriber;
 
 
-         }
-      }
+//          }
+//       }
 
-      public void removeSubscriber(PubSubSingleTagStateEvent removedSubscriber) {
-         if (mySubscribers.ContainsKey(removedSubscriber.app)) {
-            mySubscribers.Remove(removedSubscriber.app);
-            //if we were going to add this subscriber, we no logner want to
-            //this means no event needs to be published
-            if (myNewSubscribers.ContainsKey(removedSubscriber.app))
-               myNewSubscribers.Remove(removedSubscriber.app);
-            else
-               //we have to remove this guy in this updated
-               myRemovedSubscribers[removedSubscriber.app] = removedSubscriber;
-         }
+//       public void removeSubscriber(PubSubSingleTagStateEvent removedSubscriber) {
+//          if (mySubscribers.ContainsKey(removedSubscriber.app)) {
+//             mySubscribers.Remove(removedSubscriber.app);
+//             //if we were going to add this subscriber, we no logner want to
+//             //this means no event needs to be published
+//             if (myNewSubscribers.ContainsKey(removedSubscriber.app))
+//                myNewSubscribers.Remove(removedSubscriber.app);
+//             else
+//                //we have to remove this guy in this updated
+//                myRemovedSubscribers[removedSubscriber.app] = removedSubscriber;
+//          }
 
 
-      }
+//       }
 
 
-   }
-}
+//    }
+// }

src/DataStreamSuite/PubSubServer/Configurations/PubsubServer_Linux.json

@@ -0,0 +1,61 @@
+{
+   "user": "bts",
+   "pubsub": {
+      "milliseconds_between_sub_info_updates": 1000,
+      "milliseconds_between_time_slice_updates": 5000,
+      "milliseconds_between_no_buffer_space_retries": 100,
+      "retries_on_no_buffer_space": 1,
+      "do_it_all_seconds_between_logs": 0,
+      "heartbeat_interval": 15000,
+      "missing_heartbeats_before_gc": 50,
+      "use_managed_sockets": true,
+      "state_update_period": 5000,
+      "statistics_update_period": 5000,
+      "disposed_connection_check_period": 1000,
+      "message_processing_warning_threshold": 1000,
+      "server_port": 12345,
+      "publisher_routes": [
+         {
+            "name": "default",
+            "server_address": "bts-cloud://192.168.1.80:12345",
+            "tag": ".*"
+         }
+      ],
+      "subscriber_routes": [
+         {
+            "name": "default",
+            "server_address": "bts-cloud://192.168.1.80:12345",
+            "tag": ".*"
+         }
+      ],
+      "max_packet_size": 500000000,
+      "blue_comm_must_die": true,
+      "crushable_tags": [],
+      "tag_type_to_assembly_map": [
+         {
+            "handler_class": "BCG.PubSubServerTagHandler",
+            "assembly": "DataStreamCore.dll"
+         }
+      ],
+      "server_queue": "PubSubQueue",
+      "port_range": {
+         "first": 30400,
+         "last": 31700
+      },
+      "bind_to_random_port": false,
+      "debug_tags": "^NOTAGS$",
+      "latency_report_threshold_ms": 1500,
+      "use_substations": false,
+      "use_dynamic_clients": false,
+      "is_substation": false,
+      "internally_publish_sub_info": true,
+      "use_managed_clients_in_native_code": false,
+      "compressed_tags": "^NOTAGS$",
+      "tags_to_report_compression_ratio": "^NOTAGS$",
+      "disable_msmq": false,
+      "use_prepublished_server_state": false,
+      "ip_translation": null,
+      "gc_activation_threshold": 0,
+      "wait_period_after_gc": 1000
+   }
+}

src/DataStreamSuite/SharedStreams/Business/BenzingaNews/BenzingaNewsStream.cs

@@ -7,26 +7,45 @@ namespace bts {
    using System.Text.Json;
    using System.Text.Json.Nodes;
    [DataStreamUserRespresentation("benzingaNews")]
-   public class BenzingaNewStream : ViewStream {
+   public class BenzingaNewsStream : ViewStream {
       private IBlueTableSchema _schema = null;
       private int _id = 0;
-      // constructor takes in the raw mongo db stream, and the time to live (ttl), "7d" for example
-      public BenzingaNewStream(ViewStream mongodb_raw_stream, string ttl) {
+
+
+      private static ViewStream _pre_process_raw_kvstream(ViewStream mongo_db_str) {
+         ViewStream r = new KeyValueToRelationalStream(mongo_db_str, "doc_id", "key", "value");
+         r = new DateStandardizationStream(r, "created_at", "ISO8601");
+         return r;
+      }
+
+      public BenzingaNewsStream(string connection_string, string db_name, string collection_name, string poll_period, string ttl)
+      : this(new MongoDBStream(connection_string, db_name, collection_name, poll_period), ttl) {
+      }
+
+      // public BenzingaNewsStream()
+      // : this(new MongoDBStream(DataStreamPub connection_string, db_name, collection_name, poll_period), ttl) {
+      // }
+
+      // assumes it gets a raw mongodb_raw_kv_stream with key value pairs
+      public BenzingaNewsStream(ViewStream mongodb_raw_kv_stream, string ttl) {
+         ViewStream output_stream = _pre_process_raw_kvstream(mongodb_raw_kv_stream);
          // Hiding body (with project, columns can be hidden, or renamed)
-         ViewStream output_stream = new ProjectStream(mongodb_raw_stream, "!body");
+         output_stream = new ProjectStream(output_stream, "!body");
          // add column news_source = 'Benzinga', so we can append other news sources
          output_stream = new AddColumnsStream(output_stream, @"news_source : Benzinga");
          // now SuperMapStream is used to map one row into multiple, one per symbol,
          // (SuperMapStream takes 2 lambdas, one to get the enw schema and another to 
          // return an array of rows, per one row)
-         output_stream = new SuperMapStream(mongodb_raw_stream,
+         output_stream = new SuperMapStream(output_stream,
 
         (IBlueTableSchema s) => {
            // this lambda returns the new schema (ie with extra cols symbol, exchange, is_primary_exchange)
            if (_schema == null) {
               var out_schema = s;
               out_schema = BlueDataUtility.remove_columns(out_schema, "securities");
-              out_schema = BlueDataUtility.add_column(s, "doc_id_internal", typeof(string));
+              out_schema = BlueDataUtility.remove_columns(out_schema, "body");
+              out_schema = BlueDataUtility.remove_columns(out_schema, "tags");
+              out_schema = BlueDataUtility.add_column(out_schema, "doc_id_internal", typeof(string));
               out_schema = BlueDataUtility.add_column(out_schema, "symbol", typeof(string));
               out_schema = BlueDataUtility.add_column(out_schema, "exchange", typeof(string));
               out_schema = BlueDataUtility.add_column(out_schema, "is_primary_exchange", typeof(bool));
@@ -44,11 +63,13 @@ namespace bts {
            }
            foreach (JsonNode jnode in json_objs) {
               if (jnode is JsonObject json_object) {
+                 string symbol = jnode["symbol"] == null ? String.Empty : jnode["symbol"].ToString();
+                 string exchange = jnode["exchange"] == null ? String.Empty : jnode["exchange"].ToString();
                  MutableBlueRow nr = new MutableBlueRow(_schema, r);
                  nr.setValue("id", _id++);
                  nr.setValue("doc_id_internal", $"{r["doc_id"]}_{jnode["symbol"]}");
-                 nr.setValue("symbol", jnode["symbol"]);
-                 nr.setValue("exchange", jnode["exchange"]);
+                 nr.setValue("symbol", symbol);
+                 nr.setValue("exchange", exchange);
                  nr.setValue("is_primary_exchange", Boolean.Parse(jnode["primary"].ToString()));
                  expanded_rows.Add(nr);
               }

src/DataStreamSuite/SharedStreams/Business/BenzingaNews/BenzingaNewsSymbolStateStream.cs

@@ -0,0 +1,33 @@
+namespace bts {
+   using BCG;
+
+   using System.Collections.Generic;
+   using System.Linq;
+   using System.Security.Cryptography;
+
+   //[DataStreamUserRespresentation("benzingaNewsSymbols")]
+   public class BenzingaNewsSymbolStateStream : ViewStream, INonIncGroupByEvaluator {
+      public BenzingaNewsSymbolStateStream(ViewStream news_str) {
+         ViewStream r = new NonIncGroupByStream(news_str, this);
+         mySetRealSource(r);
+      }
+      public BenzingaNewsSymbolStateStream(string connection_string, string db_name, string collection_name, string poll_period, string ttl) :
+      this(new BenzingaNewsStream(connection_string, db_name, collection_name, poll_period, ttl)) {
+      }
+
+      public IBlueRowCollection evaluate(IDictionary<int, List<IBlueRow>> grouped_rows) {
+         BlueTable rv = null;
+         foreach (var kv in grouped_rows) {
+            var symbol_rows = kv.Value;
+            IBlueRow state_row = BlueDataUtility.find_max(symbol_rows, "created_at");
+            rv = rv == null ? new BlueTable(state_row.schema) : rv;
+            rv.addRow(state_row);
+         }
+         return rv;
+      }
+
+      public string[] get_group_by_cols() {
+         return new string[] { "symbol" };
+      }
+   }
+}

src/DataStreamSuite/SharedStreams/Business/BenzingaNews/BenzingaNewsSymbolsStream.cs

@@ -0,0 +1,14 @@
+namespace bts {
+   using BCG;
+   [DataStreamUserRespresentation("benzingaNewsSymbols")]
+
+   public class BenzingaNewsSymbolsStream : ViewStream {
+      public BenzingaNewsSymbolsStream(ViewStream news_str) {
+         ViewStream rs = new DistinctRowStream(news_str, "symbol");
+         mySetRealSource(rs);
+      }
+      public BenzingaNewsSymbolsStream(string connection_string, string db_name, string collection_name, string poll_period, string ttl) :
+      this(new BenzingaNewsStream(connection_string, db_name, collection_name, poll_period, ttl)) {
+      }
+   }
+}

src/DataStreamSuite/SharedStreams/Business/ContractUtils.cs

@@ -0,0 +1,252 @@
+using bts.SilverPine;
+
+namespace bts.SilverPine {
+   using bts.api;
+   using bts.spark;
+
+   using System;
+
+   using ZstdSharp.Unsafe;
+
+   public static class ContractUtils {
+      public class FutureTradableContract {
+         public string symbol;
+         public DateTime expiration;
+         public Exchange exchange;
+         public string contract_text;
+      }
+      public class FutureSpreadContract {
+         public List<Tuple<double, bts.SilverPine.ContractUtils.FutureTradableContract>> legs;
+         public string symbol {
+            get {
+               return legs == null || legs.Count == 0 ? String.Empty : legs.ElementAt(0).Item2.symbol;
+            }
+         }
+         public string contract_text;
+      }
+      public class FutureContinousContract {
+         public string symbol;
+         public Exchange exchange;
+      }
+      // this is for all future conytracts belonging to a symbol
+      public class FutureWildcardContract {
+         public string symbol;
+         public Exchange exchange;
+      }
+      public class FutureContract {
+         public FutureWildcardContract wildcard_contract = null;
+         public FutureTradableContract future_tradable_contract = null;
+         public FutureSpreadContract spread_tradable_contract = null;
+         public FutureContinousContract continous_constract = null;
+
+         public FutureContract(string txt) {
+            wildcard_contract = parse_wildcard_contract(txt);
+            if (wildcard_contract != null)
+               return;
+            FutureTradableContract fc = parse_future_contract(txt);
+            if (fc != null) {
+               future_tradable_contract = fc;
+               return;
+            }
+            FutureSpreadContract sc = parse_future_spread_contract(txt);
+            if (sc != null) {
+               spread_tradable_contract = sc;
+               return;
+            }
+            // this is not totally true, since ParseFutureSpreadContract does not
+            // parse ICS spreads
+            continous_constract = new FutureContinousContract() { symbol = txt };
+         }
+         public bool is_continous_contract => continous_constract != null;
+         public string symbol() {
+            if (wildcard_contract != null)
+               return wildcard_contract.symbol;
+            if (continous_constract != null) {
+               return continous_constract.symbol;
+            } else if (future_tradable_contract != null) {
+               return future_tradable_contract.symbol;
+            } else if (spread_tradable_contract != null) {
+               return spread_tradable_contract.symbol;
+            }
+            Log.assert(false);
+            return String.Empty;
+         }
+         public string contract_text {
+            get {
+               if (future_tradable_contract != null)
+                  return future_tradable_contract.contract_text;
+               if (spread_tradable_contract != null)
+                  return spread_tradable_contract.contract_text;
+               return null;
+            }
+         }
+      }
+      #region Parsing
+      public static FutureWildcardContract parse_wildcard_contract(string contract_txt, Exchange exchange = Exchange.CME) {
+         if (contract_txt.EndsWith("*")) {
+            contract_txt = contract_txt.Replace("*", String.Empty);
+            return new FutureWildcardContract() {
+               symbol = contract_txt,
+               exchange = exchange
+            };
+         }
+         return null;
+      }
+      public static FutureTradableContract parse_future_contract(string contract_txt, Exchange exchange = Exchange.CME) {
+         // Validate the future string length
+         if (!contract_txt.All(c => Char.IsLetterOrDigit(c)))
+            return null;
+         if (contract_txt.Length < 4) {
+            return null;
+         }
+
+         // Extract the symbol and the expiration month/year codes
+         string symbol = contract_txt.Substring(0, contract_txt.Length - 2);
+         char monthCode = contract_txt[contract_txt.Length - 2];
+         char yearCode = contract_txt[contract_txt.Length - 1];
+
+         // Map month code to month, return null if the month code is invalid
+         int month = monthCode switch {
+            'F' => 1,  // January
+            'G' => 2,  // February
+            'H' => 3,  // March
+            'J' => 4,  // April
+            'K' => 5,  // May
+            'M' => 6,  // June
+            'N' => 7,  // July
+            'Q' => 8,  // August
+            'U' => 9,  // September
+            'V' => 10, // October
+            'X' => 11, // November
+            'Z' => 12, // December
+            _ => 0 // Return 0 for invalid month code
+         };
+
+         if (month == 0) {
+            return null; // Invalid month code
+         }
+
+         // Calculate the year based on the current decade
+         int currentYear = DateTime.Now.Year;
+         int decadeStart = currentYear / 10 * 10;
+         int year = decadeStart + (yearCode - '0');
+
+         // If the year is in the past, assume the next decade
+         if (year < currentYear) {
+            year += 10;
+         }
+
+         // Create the FutureContract instance and set its properties
+         DateTime expiration = new DateTime(year, month, 1);
+         var futureContract = new FutureTradableContract {
+            symbol = symbol,
+            expiration = expiration,
+            exchange = exchange,
+            contract_text = contract_txt
+         };
+
+         return futureContract;
+      }
+      // Method to parse a futures spread, e.g. "CLX4-CLZ4"
+      public static FutureSpreadContract parse_future_spread_contract(string contract_txt, Exchange exchange = Exchange.CME) {
+         // Split the string by '-' to handle each leg
+         var spreadParts = contract_txt.Split('-');
+         if (spreadParts.Length < 2) {
+            return null; // Not a valid spread format
+         }
+
+         var spreadContract = new FutureSpreadContract { legs = new List<Tuple<double, FutureTradableContract>>() };
+
+         foreach (var part in spreadParts) {
+            // Split each leg by '*', assuming the format is weight*future or just future
+            string[] legParts = part.Split('*');
+            double weight = 1.0;
+            string futurePart;
+
+            if (legParts.Length == 2) {
+               if (!double.TryParse(legParts[0], out weight)) {
+                  return null; // Invalid weight, parsing failed
+               }
+               futurePart = legParts[1];
+            } else if (legParts.Length == 1) {
+               futurePart = legParts[0];
+            } else {
+               return null; // Invalid format, too many '*' parts
+            }
+
+            // Parse the future contract
+            var futureContract = parse_future_contract(futurePart, exchange);
+            if (futureContract == null) {
+               return null; // Failed to parse the future part
+            }
+
+            // Add the leg to the spread contract
+            spreadContract.legs.Add(new Tuple<double, FutureTradableContract>(weight, futureContract));
+         }
+         return spreadContract;
+      }
+
+      // public static ContractType DetermineFutureOrSpread(string contract) {
+      //    if (contract.Contains("-")) {
+      //       return ContractType.FutureSpread;
+      //    } else {
+      //       return ContractType.Future;
+      //    }
+      // }
+      #endregion
+
+      public static ContractType get_type(string contract_txt) {
+         FutureTradableContract fc = parse_future_contract(contract_txt);
+         if (fc != null) {
+            return ContractType.Future;
+         }
+         FutureSpreadContract sc = parse_future_spread_contract(contract_txt);
+         if (sc != null) {
+            return ContractType.FutureSpread;
+         }
+         // tehre are some non calendar spreads we cant parse
+         //Log.assert(false);
+         return bts.SilverPine.ContractUtils.ContractType.FutureSpread;
+      }
+
+      public static ContractCategory get_category(string symbol) {
+         var tp = get_symbol_category(symbol);
+         return tp.Item1;
+      }
+
+      public static Tuple<ContractCategory, string> get_symbol_category(string symbol) {
+         ContractUtils.FutureTradableContract fc = parse_future_contract(symbol);
+         if (fc != null) {
+            return Tuple.Create(ContractCategory.FutureString, fc.symbol);
+         }
+         var sc = parse_future_spread_contract(symbol);
+         if (sc != null) {
+            return Tuple.Create(ContractCategory.FutureString, sc.symbol);
+         }
+         return Tuple.Create(ContractCategory.Symbol, symbol);
+      }
+
+
+
+      public enum ContractType {
+         Spot,
+         Option,
+         Future,
+         FutureSpread,
+         OptionSpread
+      }
+      public enum ContractCategory {
+         Symbol = 0,
+         FutureString,
+         SpreadString
+      }
+      public enum CallPut {
+         Call,
+         Put
+      }
+      public enum Exchange {
+         CME
+      }
+
+   }
+}

src/DataStreamSuite/SharedStreams/Business/EventsPriceActionStream.cs

@@ -0,0 +1,23 @@
+namespace bts.SilverPine {
+   using BCG;
+
+   // Finds the regions around specified events 
+   [DataStreamUserRespresentation("EventsPriceAction")]
+   public class EventsPriceActionStream : ViewStream {
+      public EventsPriceActionStream(string symbol, string event_type, string pre_event_period,
+            string post_event_period, int n_events) :
+         this(symbol, event_type, TimeBasedFilter.parse_duration(pre_event_period), TimeBasedFilter.parse_duration(post_event_period),
+         n_events, ViewStreamDatabase.instance) { }
+
+      public EventsPriceActionStream(string symbol, string event_type, TimeSpan pre_event_period,
+      TimeSpan post_event_period, int n_events, ViewStreamDatabase vsd) {
+         // TODO: figure out which bars to use, 1 min or 1 day or live, based on pre_event_period / post_event_period
+         FuturesActiveContractStream olhcv_str = new FuturesActiveContractStream(symbol);
+         ViewStream events_str = new EventsStream(vsd, event_type);
+         // todo
+         //events_str = new TopStream(events_str, n_events, "-ts");
+         ViewStream rs = new TimeProximityJoinStream(olhcv_str, "ts_event", events_str, "ts", pre_event_period, post_event_period);
+         mySetRealSource(rs);
+      }
+   }
+}

src/DataStreamSuite/SharedStreams/Business/Indicators/IndicatorStream.cs

@@ -0,0 +1,20 @@
+namespace bts.SilverPine {
+   using BCG;
+
+   public class IndicatorStream : ViewStream {
+      enum Indicator {
+         StandardDeviation = 0
+      }
+      public class IndicatorInfo {
+         Indicator indicator;
+         IDictionary<string, object> parameters;
+      }
+      public IndicatorStream(string symbol_name, string indicator_name, string indicator_params_json) {
+
+      }
+      public IndicatorStream(string symbol_name, IndicatorInfo info) {
+
+      }
+
+   }
+}

src/DataStreamSuite/SharedStreams/Business/MarketData/AbsoluteVolumeStream.cs

@@ -0,0 +1,57 @@
+namespace bts.SilverPine {
+   using BCG;
+
+   using ZstdSharp.Unsafe;
+
+   [DataStreamUserRespresentation("AbsVolume")]
+   // Calculates the volume within a time window (this does not do rates, like highest daily volume etc,
+   // but with some thought can perhaps be extended)
+   public class AbsoluteVolumeStream : ViewStream {
+      // TODO, abstract this out
+      // adds contract_type to OHLCV stream
+      private static ViewStream _get_ohlcv_steam(string symbol, DateTime d1, DateTime d2) {
+         var ohlcv_params = OHLCVStream.get_params(d1, d2);
+         ViewStream rs = new OHLCVStream(symbol, ohlcv_params);
+         Func<IBlueTableSchema, IBlueTableSchema> create_schema =
+         s => BlueDataUtility.add_columns(s, Tuple.Create(typeof(string), "contract_type"));
+         rs = new SuperMapStream(rs,
+         s => create_schema(s),
+         r => {
+            IBlueTableSchema out_schema = create_schema(r.schema);
+            string contract_txt = r.get<string>("contract");
+            var contr_type = ContractUtils.get_type(contract_txt);
+            MutableBlueRow out_row = new MutableBlueRow(out_schema, r);
+            out_row.setValue("contract_type", contr_type.ToString());
+            return new IBlueRow[] { out_row };
+         });
+         return rs;
+      }
+
+      public AbsoluteVolumeStream(string symbol, string ctype, DateTime d1)
+      : this(symbol, ctype, d1, d1) { }
+
+      public AbsoluteVolumeStream(string symbol, string ctype, DateTime d1, DateTime d2)
+      : this(symbol, Enum.Parse<ContractUtils.ContractType>(ctype), d1, d2) {
+      }
+
+      static void swap_if_necessary(ref DateTime dt1, ref DateTime dt2) {
+         if (dt1 < dt2) {
+            DateTime aux = dt1;
+            dt1 = dt2;
+            dt2 = aux;
+         }
+      }
+
+      private AbsoluteVolumeStream(string symbol, ContractUtils.ContractType ctype, DateTime d1, DateTime d2) {
+         swap_if_necessary(ref d1, ref d2);
+         ViewStream ohlcv_str = _get_ohlcv_steam(symbol, d1, d2);
+         ohlcv_str = new FilterStream(ohlcv_str,
+            r => r.get_datetime("ts_event") <= d1 && r.get_datetime("ts_event") >= d2);
+         ohlcv_str = new FilterStream(ohlcv_str, r => String.Compare(ctype.ToString(), r.get<string>("contract_type"), true) == 0);
+         ViewStream rs = new MaxStream(ohlcv_str, "volume");
+         rs = new NewProjectStream(rs, "volume:total_volume contract !");
+         mySetRealSource(rs);
+      }
+
+   }
+}

src/DataStreamSuite/SharedStreams/Business/MarketData/BBOStream.cs

@@ -1,35 +1,105 @@
-namespace BCG {
+namespace bts.SilverPine {
    using BCG.Utilities.General;
 
-   using bts.api;
-   using bts.api.domain;
-   using bts.api.synthetic;
    using bts.md;
-   using bts.spark;
-   using bts.utils;
-
-   using mbts;
-   using mbts.co2;
-   using mbts.config;
-   using mbts.spark;
-   using mbts.ui;
-
-   using Spark.Core.Data;
-   using Spark.Core.Disposables;
+   using BCG;
    using Spark.Core.Reactive;
 
    using System;
-   using System.Collections.Concurrent;
    using System.Collections.Generic;
-   using System.Collections.ObjectModel;
    using System.Linq;
    using System.Reactive.Linq;
    using bts.api.core;
    using Spark.Api.MarketData;
 
    using SecurityKey = System.UInt32;
-   using System.Data;
-   using System.Reflection.Metadata;
+   using System.Data.Entity.Infrastructure;
+
+   public class FakeBBOStream : ViewStream, IWorker {
+      private string _symbol;
+      public BlueTable _table = new BlueTable(BBOStream._bbo_schema);
+      private DateTime _last_fired_time = DateTime.MinValue;
+      private TimeSpan _period = TimeSpan.FromMilliseconds(333);
+      private double _start_price;
+      private double _inc;
+      private int _total_count = 5;
+      private Random _rnd = new Random(DateTime.Now.Millisecond);
+      private string[] _months = new string[]{
+         "F","G", "H", "J","K", "M", "N","Q","U","V","X","Z"
+      };
+      private ViewStreamDatabase _vsd;
+      public FakeBBOStream(string contract_type, string exchange, string symbol, double start_price, double inc)
+      :
+      this(contract_type, exchange, symbol, start_price, inc, ViewStreamDatabase.instance) { }
+
+      public FakeBBOStream(string contract_type, string exchange, string symbol, double start_price, double inc, ViewStreamDatabase vsd) {
+         _symbol = symbol;
+         _start_price = start_price;
+         _inc = inc;
+         _vsd = vsd;
+         Log.assert(contract_type.ToLower() == "future");
+      }
+
+      protected override void myOnFirstSubscriber() {
+         _vsd.addWorker(this);
+      }
+
+      protected override void myOnLastSubscriber() {
+         _vsd.removeWorker(this);
+      }
+
+      private bool _random_truefase() {
+         int r = _rnd.Next(0, 100);
+         return r % 2 == 0;
+      }
+
+      private IBlueRow _perturb_row(IBlueRow er) {
+         MutableBlueRow rv = new MutableBlueRow(er);
+         double bid_price = rv.get<double>("bid_price");
+         bid_price += _inc;
+         double ask_price = bid_price + _inc;
+         rv.setValue("bid_price", bid_price);
+         rv.setValue("ask_price", ask_price);
+         return rv;
+      }
+
+      public void doWork() {
+         if (DateTime.Now - _last_fired_time < _period)
+            return;
+         _last_fired_time = DateTime.Now;
+         if (_table.Count == 0) {
+            // first fire_symbol
+            string year_code = DateTime.Now.Year.ToString().Last().ToString();
+            for (int i = 0; i < _total_count; i++) {
+               string month_code = _months[i % _months.Length];
+               string sec = $"{_symbol}{month_code}{year_code}";
+               double bid_price = _start_price + _rnd.Next(-10, 10) * _inc;
+               double ask_price = bid_price + _inc;
+               BlueRow nr = new BlueRow(_table.schema, _table.Count, _table.Count, sec,
+                                       bid_price, _rnd.Next(1, 10),
+                                       ask_price, _rnd.Next(1, 10));
+               _table.addRow(nr);
+               var dp = BlueDataUtility.build_dp(new BlueTable(_table), myLastValue as DataPacket);
+               myFireEvent(dp);
+            }
+         } else {
+            BlueTable deleted_rows = new BlueTable(_table.schema);
+            BlueTable added_rows = new BlueTable(_table.schema);
+            // go through all rows, select some at random, add them to deleted, get
+            // the replaced one and add it to added, updates table state and fire
+            for (int i = 0; i < _table.Count; i++) {
+               if (_random_truefase()) {
+                  IBlueRow er = _table[i];
+                  IBlueRow nr = _perturb_row(er);
+                  _table.deleteRow(er);
+                  _table.addRow(nr);
+               }
+            }
+            var dp = BlueDataUtility.build_dp_inc(new BlueTable(_table), myLastValue as DataPacket, true);
+            myFireEvent(dp);
+         }
+      }
+   }
 
    [DataStreamUserRespresentation("bbo")]
    public class BBOStream : ViewStream, IWorker {
@@ -39,7 +109,7 @@ namespace BCG {
       private int _id = 0;
       IReadOnlyDictionary<SecurityKey, BookTuple> _cached_state = null;
       MarketDataClient _md_client;
-      private readonly IBlueTableSchema _bbo_schema = BlueDataUtility.form_schema("bbo",
+      internal static IBlueTableSchema _bbo_schema = BlueDataUtility.form_schema("bbo",
                                                 Tuple.Create("id", TypeCode.Int32),
                                                 Tuple.Create("sec_key", TypeCode.UInt32),
                                                 Tuple.Create("sec", TypeCode.String),
@@ -54,9 +124,11 @@ namespace BCG {
       }
 
       public BBOStream(ViewStreamDatabase vsd) {
-         bts.Log.Gen.info("BBOStream.cons");
-         _vsd = vsd;
-         _md_client = new MarketDataClient(new MDSource(TimeSpan.FromSeconds(0.5)));
+         // bts.Log.Gen.info("BBOStream.cons");
+         // _vsd = vsd;
+         // _md_client = new MarketDataClient(new MDSource(TimeSpan.FromSeconds(0.5)));
+         FakeBBOStream rs = new FakeBBOStream("Future", "CME", "ES", 5000, 0.25, vsd);
+         mySetRealSource(rs);
       }
 
       protected override void myOnFirstSubscriber() {

src/DataStreamSuite/SharedStreams/Business/MarketData/Bars/HistoricalBarsStream.cs

@@ -0,0 +1,249 @@
+namespace bts.SilverPine {
+   using BCG;
+
+   using System.Runtime.CompilerServices;
+
+   using ZstdSharp.Unsafe;
+
+   public class DateTimeRange {
+      public readonly DateTime range_beg;
+      public readonly DateTime range_end;
+      public DateTimeRange(DateTime range_beg_, DateTime range_end_) {
+         range_beg = range_beg_;
+         range_end = range_end_;
+      }
+      public static DateTimeRange parse(string range_beg_, string range_end_) {
+         DateTime range_beg = String.IsNullOrEmpty(range_beg_) ? DateTime.MinValue : DateTime.Parse(range_beg_);
+         DateTime range_end = String.IsNullOrEmpty(range_end_) ? DateTime.MaxValue : DateTime.Parse(range_end_);
+         return new DateTimeRange(range_beg, range_end);
+      }
+   }
+
+   public class FutureBarsLoader {
+      public static string get_db_table_name(BarSize bar_size, string symbol, bool is_continous) {
+         switch (bar_size) {
+            case BarSize.ONE_DAY: {
+                  if (is_continous) {
+                     return $"bars_future_{symbol}_continous_1day";
+                  } else
+                     return $"bars_future_{symbol}_all_1day";
+               }
+            case BarSize.ONE_MIN: {
+                  if (is_continous) {
+                     return $"bars_future_{symbol}_continous_1min";
+                  } else
+                     return $"bars_future_{symbol}_all_1min";
+               }
+         }
+         Log.assert(false);
+         return null;
+      }
+   }
+   public enum BarSize {
+      ONE_DAY = 0,
+      ONE_MIN
+   }
+
+   [DataStreamUserRespresentation("FuturesHistorical1DayBars")]
+   public class FuturesHistorical1DayBarsStream : ViewStream {
+
+      // this gets the raw bars for a symbol (perhaps make it into a raw bars stream?)
+      public FuturesHistorical1DayBarsStream(string symbol) :
+      this(symbol, DateTime.MinValue, DateTime.MaxValue) { }
+
+      // creates the non raw stream (ie for tradable contract, or continous contract)
+      public FuturesHistorical1DayBarsStream(string future_contract, DateTime range_beg, DateTime range_end)
+     : this(new ContractUtils.FutureContract(future_contract), new DateTimeRange(range_beg, range_end)) { }
+
+      // this creates a tradable contracts bars  streams or continous contracts stream. if get_all_rows is true, 
+      // it gets the raw table with all contracts for that symbol
+      public FuturesHistorical1DayBarsStream(ContractUtils.FutureContract future_contract,
+                                             DateTimeRange dt_range) {
+         DateTime beg = dt_range.range_beg;//.ToUniversalTime(),
+         DateTime end = dt_range.range_end;//.ToUniversalTime();
+         beg = beg.Date;
+         end = end.Date.Add(new TimeSpan(23, 59, 59));
+         // this returns rows from beg and end inclusive
+         Func<string, SQLiteStream> get_sqlite_str = (table_name) => {
+            string range_beg_iso8601 = beg.ToString("yyyy-MM-ddTHH:mm:ss"),
+               range_end_iso8601 = end.ToString("yyyy-MM-ddTHH:mm:ss");
+            string sql_str = $"select * from {table_name} where ts_event >= '{range_beg_iso8601}' AND ts_event <= '{range_end_iso8601}'";
+            return new SQLiteStream(SQLiteHelper.db_name, sql_str);
+         };
+         string symbol = future_contract.symbol();
+         if (future_contract.wildcard_contract != null) {
+            string table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_DAY, future_contract.symbol(), false);
+            ViewStream rs = get_sqlite_str(table_name);
+            //rs = new DateStandardizationStream(rs, "ts_event", DateConversion.ChopTime);
+            mySetRealSource(rs);
+            return;
+         }
+         if (future_contract.wildcard_contract != null) {
+            string table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_DAY, future_contract.symbol(), false);
+            ViewStream rs = get_sqlite_str(table_name);
+            //rs = new DateStandardizationStream(rs, "ts_event", DateConversion.ChopTime);
+            mySetRealSource(rs);
+            return;
+         }
+         if (future_contract.is_continous_contract) {
+            string table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_DAY, symbol, true);
+            ViewStream rs = get_sqlite_str(table_name);
+            //rs = new DateStandardizationStream(rs, "ts_event", DateConversion.ChopTime);
+            mySetRealSource(rs);
+            return;
+         } else {
+            string table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_DAY, future_contract.symbol(), false);
+            ViewStream rs = get_sqlite_str(table_name);
+            //rs = new DateStandardizationStream(rs, "ts_event", DateConversion.ChopTime);
+            //TODO: filter it 
+            rs = new FilterStream(rs, $"symbol = '{future_contract.contract_text}'");
+            mySetRealSource(rs);
+         }
+      }
+   }
+
+   [DataStreamUserRespresentation("FuturesHistorical1MinBars")]
+   public class FuturesHistorical1MinuteBarsStream : ViewStream {
+      // same as the above but for one min
+      // this gets the raw bars for a symbol
+      public FuturesHistorical1MinuteBarsStream(string symbol) :
+      this(symbol, DateTime.MinValue, DateTime.MaxValue) { }
+
+      // creates the non raw stream (ie for tradable contract, or continous contract)
+      public FuturesHistorical1MinuteBarsStream(string future_contract, DateTime range_beg, DateTime range_end)
+     : this(new ContractUtils.FutureContract(future_contract), new DateTimeRange(range_beg, range_end)) { }
+
+      // this creates a tradable contracts bars  streams or continous contracts stream. if get_all_rows is true, 
+      // it gets the raw table with all contracts for that symbol
+      public FuturesHistorical1MinuteBarsStream(ContractUtils.FutureContract future_contract,
+                                             DateTimeRange dt_range) {
+         Func<string, SQLiteStream> get_sqlite_str = (table_name) => {
+            string range_beg_iso8601 = dt_range.range_beg.ToString("yyyy-MM-ddTHH:mm:ss"),
+               range_end_iso8601 = dt_range.range_end.ToString("yyyy-MM-ddTHH:mm:ss");
+            //string sql_str = $"select * from {table_name} where DATE(ts_event) >= '{range_beg_iso8601}' AND DATE(ts_event) <= '{range_end_iso8601}'";
+            string sql_str = $"select * from {table_name} where ts_event >= '{range_beg_iso8601}' AND ts_event <= '{range_end_iso8601}'";
+            return new SQLiteStream(SQLiteHelper.db_name, sql_str);
+         };
+         string symbol = future_contract.symbol();
+         if (future_contract.wildcard_contract != null) {
+            string table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_MIN, future_contract.symbol(), false);
+            ViewStream rs = get_sqlite_str(table_name);
+            mySetRealSource(rs);
+            return;
+         }
+         if (future_contract.is_continous_contract) {
+            string table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_MIN, symbol, true);
+            ViewStream rs = get_sqlite_str(table_name);
+            mySetRealSource(rs);
+            return;
+         } else {
+            string table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_MIN, future_contract.symbol(), false);
+            ViewStream rs = get_sqlite_str(table_name);
+            // TODO: filter
+            rs = new FilterStream(rs, $"symbol = '{future_contract.contract_text}'");
+            mySetRealSource(rs);
+         }
+      }
+   }
+
+   [DataStreamUserRespresentation("FutureHistoricalBars")]
+   public class FutureHistoricalBarsStream : ViewStream {
+      public FutureHistoricalBarsStream(string fut_contract, string range_beg, string range_end) {
+         DateTime dt_range_beg = DateTime.Parse(range_beg);
+         DateTime dt_range_end = DateTime.Parse(range_end);
+         TimeSpan diff = dt_range_end - dt_range_beg;
+         BarSize bar_size = diff < TimeSpan.FromDays(1.0) ? BarSize.ONE_MIN : BarSize.ONE_DAY;
+         ViewStream rs = new FutureHistoricalBarsStream(fut_contract, bar_size.ToString(), dt_range_beg, dt_range_end);
+         mySetRealSource(rs);
+      }
+      public FutureHistoricalBarsStream(string fut_contract, string bar_size)
+      : this(fut_contract, bar_size, DateTime.MinValue, DateTime.MaxValue) { }
+
+      public FutureHistoricalBarsStream(string fut_contract, string bar_size,
+                                        string dt_range_beg, string dt_range_end)
+                                        : this(fut_contract, bar_size, DateTime.Parse(dt_range_beg),
+                                        DateTime.Parse(dt_range_end)) { }
+
+      public FutureHistoricalBarsStream(string fut_contract, string bar_size,
+                                        DateTime dt_range_beg, DateTime dt_range_end)
+                                        :
+      this(new ContractUtils.FutureContract(fut_contract), Enum.Parse<BarSize>(bar_size, true),
+         new DateTimeRange(dt_range_beg, dt_range_end)) { }
+
+      public FutureHistoricalBarsStream(ContractUtils.FutureContract fut_contract, BarSize bar_size,
+                                        DateTimeRange dt_range) {
+         switch (bar_size) {
+            case BarSize.ONE_DAY: {
+                  ViewStream rs = new FuturesHistorical1DayBarsStream(fut_contract, dt_range);
+                  mySetRealSource(rs);
+                  break;
+               }
+            case BarSize.ONE_MIN: {
+                  ViewStream rs = new FuturesHistorical1MinuteBarsStream(fut_contract, dt_range);
+                  mySetRealSource(rs);
+                  break;
+               }
+         }
+      }
+   }
+
+   [DataStreamUserRespresentation("FuturePriceDiff")]
+   public class FutureHistoricalBarsPriceDiffStream : ViewStream {
+
+      public FutureHistoricalBarsPriceDiffStream(string future_contract, string dt1, string dt2)
+      :
+      this(new ContractUtils.FutureContract(future_contract), DateTime.Parse(dt1), DateTime.Parse(dt2), "close", ViewStreamDatabase.instance) {
+      }
+
+
+      public FutureHistoricalBarsPriceDiffStream(string future_contract, DateTime dt1, DateTime dt2)
+      :
+      this(new ContractUtils.FutureContract(future_contract), dt1, dt2, "close", ViewStreamDatabase.instance) {
+      }
+
+      public FutureHistoricalBarsPriceDiffStream(string future_contract, DateTime dt1, DateTime dt2, string col)
+      :
+      this(new ContractUtils.FutureContract(future_contract), dt1, dt2, col, ViewStreamDatabase.instance) {
+      }
+
+
+      public FutureHistoricalBarsPriceDiffStream(ContractUtils.FutureContract fc, DateTime dt1, DateTime dt2, string col, ViewStreamDatabase vsd) {
+         BarSize bsize = dt2 - dt1 >= TimeSpan.FromDays(1.0) ? BarSize.ONE_DAY : BarSize.ONE_MIN;
+         ViewStream rs = new FutureHistoricalBarsStream(fc, BarSize.ONE_DAY, new DateTimeRange(dt1, dt2));
+
+         rs = new ColumnDifferenceStream(rs, r => {
+            DateTime r_dt = r.get_datetime("ts_event");
+            var rv = dt1.Equals(r_dt);
+            return rv;
+         },
+         //r => dt2.Equals(r.get_datetime("ts_event")), col, col);
+         r => {
+            DateTime r_dt = r.get_datetime("ts_event");
+            var rv = dt2.Equals(r_dt);
+            return rv;
+         }, col, col);
+         rs = new NewProjectStream(rs, "diff !");
+         mySetRealSource(rs);
+      }
+   }
+
+   // takes the contract name and expands it into multuiple columns corresponding to the contracts.
+   // the contract columns are a union, of all properties with some of them empty
+   public class ContractExpanderStream : ViewStream {
+      private static IBlueTableSchema map_schema(IBlueTableSchema schema) {
+         return schema.add_column(TypeCode.String, "contract_type");
+      }
+      public ContractExpanderStream(ViewStream parent_str, string contract_col_name) {
+         ViewStream rs = new MapStream(parent_str, (schema) => map_schema(schema),
+         r => {
+            string contract = r.get<string>(contract_col_name);
+            var ct = ContractUtils.get_type(contract);
+            MutableBlueRow rv = new MutableBlueRow(map_schema(r.schema), r);
+            rv.setValue("contract_type", ct.ToString());
+            return rv;
+         });
+         mySetRealSource(rs);
+      }
+   }
+
+}

src/DataStreamSuite/SharedStreams/Business/MarketData/Bars/VolumeAggregationStream.cs

@@ -0,0 +1,46 @@
+using BCG;
+
+namespace bts.SilverPine {
+   /*
+      ----------------------------------------------------------------------
+                                  *Expression*
+      ----------------------------------------------------------------------
+          //r=FutureHistoricalBars("ES*", "02/01/2024 09:30 AM", "02/01/2024 09:30 PM")
+          r=FutureHistoricalBars("ES*","One_Day", 02/01/2024, 02/01/2025)
+          r=contractExpander(r,"symbol")
+          r=filter(r,"contract_type='FutureSpread'")
+          r=aggregate(r,"symbol","volume:Sum","ts_event")
+          r=aggregate(r,"volume_sum:Max")
+          r=newProject(r,"volume_sum_max:volume symbol !")
+
+      ----------------------------------------------------------------------
+                                    *Schema*
+      ----------------------------------------------------------------------
+          symbol                          System.String
+          id                              System.Int32
+          volume                          System.Double
+   */
+
+   [DataStreamUserRespresentation("VolumeAgg")]
+   public class VolumeAggregationStream : ViewStream {
+      public VolumeAggregationStream(string future_contract, string contract_type, DateTime range_beg, DateTime range_end)
+      : this(future_contract, contract_type, range_beg, range_end, ViewStreamDatabase.instance) { }
+
+      public VolumeAggregationStream(string future_contract, string contract_type, string range_beg, string range_end)
+      :
+      this(future_contract, contract_type, DateTime.Parse(range_beg), DateTime.Parse(range_end), ViewStreamDatabase.instance) { }
+
+      public VolumeAggregationStream(string future_contract, string contract_type, DateTime range_beg, DateTime range_end, ViewStreamDatabase db) {
+         future_contract = future_contract.EndsWith("*") ? future_contract : future_contract + "*";
+         ViewStream r;
+         r = new FutureHistoricalBarsStream(future_contract,
+                                             range_beg.ToString(), range_end.ToString());
+         r = new ContractExpanderStream(r, "symbol");
+         r = new FilterStream(r, $"contract_type='{contract_type}'");
+         r = new NonIncGroupByStream(r, "symbol", "volume:Sum", "ts_event");
+         r = new NonIncGroupByStream(r, "volume_sum:Max");
+         r = new NewProjectStream(r, "volume_sum_max:volume symbol !");
+         mySetRealSource(r);
+      }
+   }
+}

src/DataStreamSuite/SharedStreams/Business/MarketData/Charting/ChartingHelper.cs

@@ -0,0 +1,152 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+
+using bts;
+
+namespace bts.SilverPine {
+   public static class IndicatorCalc {
+      public static double calculate_pearson_correlation(List<double> prices1, List<double> prices2) {
+         if (prices1 == null)
+            throw new ArgumentNullException(nameof(prices1), "First price list cannot be null.");
+         if (prices2 == null)
+            throw new ArgumentNullException(nameof(prices2), "Second price list cannot be null.");
+         if (prices1.Count != prices2.Count)
+            throw new ArgumentException("Price lists must have the same number of elements.");
+         if (prices1.Count < 2)
+            throw new ArgumentException("At least two data points are required to calculate correlation.");
+
+         int n = prices1.Count;
+         double sumX = 0;
+         double sumY = 0;
+         double sumX2 = 0;
+         double sumY2 = 0;
+         double sumXY = 0;
+
+         for (int i = 0; i < n; i++) {
+            double x = prices1[i];
+            double y = prices2[i];
+
+            sumX += x;
+            sumY += y;
+            sumX2 += x * x;
+            sumY2 += y * y;
+            sumXY += x * y;
+         }
+
+         double numerator = (n * sumXY) - (sumX * sumY);
+         double denominatorPart1 = (n * sumX2) - (sumX * sumX);
+         double denominatorPart2 = (n * sumY2) - (sumY * sumY);
+         double denominator = Math.Sqrt(denominatorPart1 * denominatorPart2);
+
+         if (denominator == 0)
+            throw new InvalidOperationException("Denominator is zero, correlation is undefined.");
+
+         return numerator / denominator;
+      }
+      public static double calculate_VWAP(List<Tuple<int, double>> volumePriceData) {
+         Log.Gen.info($"ChartingHelper.calculate_VWAP n_vals: {volumePriceData.Count}");
+         double totalVolume = 0;
+         double totalPriceVolume = 0;
+
+         foreach (var data in volumePriceData) {
+            int volume = data.Item1; // Volume
+            double price = data.Item2; // Closing price
+
+            // Accumulate total price-volume (price * volume) and total volume
+            totalPriceVolume += price * volume;
+            totalVolume += volume;
+         }
+
+         // VWAP is totalPriceVolume divided by totalVolume
+         return totalVolume > 0 ? totalPriceVolume / totalVolume : 0;
+      }
+
+      // Calculate Volatility (Standard Deviation of returns) over a given period
+      public static double calculate_volatility(IList<double> prices, int period = 252) {
+         if (prices == null || prices.Count < period + 1) {
+            throw new ArgumentException("Insufficient data to calculate volatility.");
+         }
+
+         // Calculate daily returns
+         var returns = new List<double>();
+         for (int i = 1; i < prices.Count; i++) {
+            double dailyReturn = (prices[i] - prices[i - 1]) / prices[i - 1];
+            returns.Add(dailyReturn);
+         }
+
+         // Get the returns over the desired period
+         var recentReturns = returns.Skip(returns.Count - period).Take(period).ToList();
+
+         // Calculate the mean of returns
+         double meanReturn = recentReturns.Average();
+
+         // Calculate the variance (sum of squared differences from the mean)
+         double variance = recentReturns.Sum(r => Math.Pow(r - meanReturn, 2)) / period;
+
+         // Volatility is the square root of variance (standard deviation)
+         double volatility = Math.Sqrt(variance);
+
+         // Annualize volatility by multiplying by the square root of 252 (number of trading days in a year)
+         return volatility * Math.Sqrt(252);
+      }
+
+      // Calculate SMA (Simple Moving Average) for a given period (e.g., 50, 100, or 200)
+      public static double calculate_SMA(IList<double> prices, int period = 50) {
+         if (prices == null || prices.Count < period) {
+            throw new ArgumentException("Insufficient data to calculate SMA.");
+         }
+
+         // Sum up the prices over the given period
+         double sum = prices.Skip(prices.Count - period).Take(period).Sum();
+
+         // Return the average
+         return sum / period;
+      }
+
+      public static double calculate_RSI(IList<double> prices, int period = 14) {
+         if (prices == null || prices.Count < period + 1) {
+            return Double.NaN;
+            //throw new ArgumentException("Insufficient data to calculate RSI.");
+         }
+
+         double gainSum = 0;
+         double lossSum = 0;
+
+         // Calculate the initial average gain and loss over the first 'period' days
+         for (int i = 1; i <= period; i++) {
+            double change = prices[i] - prices[i - 1];
+            if (change > 0) {
+               gainSum += change;
+            } else {
+               lossSum -= change; // Subtract since loss is negative
+            }
+         }
+
+         double avgGain = gainSum / period;
+         double avgLoss = lossSum / period;
+
+         // Calculate RSI using smoothed values
+         for (int i = period + 1; i < prices.Count; i++) {
+            double change = prices[i] - prices[i - 1];
+            if (change > 0) {
+               avgGain = (avgGain * (period - 1) + change) / period;
+               avgLoss = (avgLoss * (period - 1)) / period;
+            } else {
+               avgGain = (avgGain * (period - 1)) / period;
+               avgLoss = (avgLoss * (period - 1) - change) / period;
+            }
+         }
+
+         // Prevent division by zero in case of no losses
+         if (avgLoss == 0) {
+            return 100;
+         }
+
+         double rs = avgGain / avgLoss;
+         double rsi = 100 - (100 / (1 + rs));
+
+         return rsi;
+      }
+   }
+}

src/DataStreamSuite/SharedStreams/Business/MarketData/Charting/DailyChangeStream.cs

@@ -0,0 +1,31 @@
+namespace bts.SilverPine {
+   using BCG;
+
+   using bts;
+
+   [DataStreamUserRespresentation("DailyChange")]
+   // takes in the ohlcv stream and computes the closing vs closing difference
+   public class DailyChange : ViewStream {
+      public DailyChange(ViewStream ohlc_str) {
+         GlobalMapStream rs = new GlobalMapStream(ohlc_str, "ts_event", "contract",
+         (rows, idx) => {
+            double daily_change = Double.NaN;
+            double daily_change_perc = Double.NaN;
+            IBlueRow ir = rows.ElementAt((int)idx);
+            if (idx < rows.Count - 1) {
+               IBlueRow pr = rows.ElementAt((int)idx + 1);
+               double close = ir.get<double>("close"), prev_close = pr.get<double>("close");
+               daily_change = close - prev_close;
+               daily_change_perc = 100.0 * daily_change / prev_close;
+            }
+            IBlueTableSchema schema = ir.schema.add_columns(Tuple.Create(typeof(double), "daily_change"));
+            schema = schema.add_columns(Tuple.Create(typeof(double), "daily_change_perc"));
+            MutableBlueRow nr = new MutableBlueRow(schema, ir);
+            nr.setValue("daily_change", daily_change);
+            nr.setValue("daily_change_perc", daily_change_perc);
+            return nr;
+         });
+         mySetRealSource(rs);
+      }
+   }
+}

src/DataStreamSuite/SharedStreams/Business/MarketData/Charting/EMAStream.cs

@@ -1,54 +0,0 @@
-
-using Google.Protobuf.WellKnownTypes;
-
-using System;
-using BCG.Utilities.General;
-using System.Data;
-using bts.api.domain;
-
-namespace BCG {
-    // Stream computes the EMA of a column in any stream. It takes in the base stream, the column name and a smoothing factor
-    // as args, and create one row with an ema column, which is the running ema of the column provided 
-    [DataStreamUserRespresentation("ema")]
-    public class EMAStream : ViewStream {
-        private ViewStream _stream;
-        private readonly string _col_name;
-        private EMACalculator _calc;
-        private readonly IBlueTableSchema _schema = BlueDataUtility.form_schema(
-            "SMA", Tuple.Create("id", TypeCode.Int32),
-            Tuple.Create("ema", TypeCode.Double));
-
-        public EMAStream(ViewStream stream, string col_name, double smoothing_factor) {
-            _col_name = col_name;
-            _stream = stream;
-            _calc = new EMACalculator(smoothing_factor);
-        }
-
-        protected override void myOnFirstSubscriber() {
-            _stream.onChange += new ChangeDelegate(_onstream_change);
-        }
-        protected override void myOnLastSubscriber() {
-            _stream.onChange -= new ChangeDelegate(_onstream_change);
-        }
-
-
-        private void _onstream_change(object o) {
-
-            DataPacket dp = o as DataPacket;
-            foreach (IBlueRow r in dp.addedRows) {
-                double price = r.get<double>(_col_name);
-                if (!Double.IsNaN(price))
-                    _calc.add(price);
-            }
-            IBlueRow nr = new BlueRow(_schema, 0, _calc.get_EMA());
-            BlueTable state = new BlueTable(_schema);
-            state.addRow(nr);
-            DataPacket dp_out = BlueDataUtility.build_dp_inc(state, myLastValue as DataPacket);
-            myFireEvent(dp_out);
-        }
-    }
-
-}
-
-
-

src/DataStreamSuite/SharedStreams/Business/MarketData/Charting/RSIStream.cs

@@ -0,0 +1,26 @@
+namespace bts.SilverPine {
+   using BCG;
+
+   using bts;
+
+   [DataStreamUserRespresentation("RSI")]
+   public class RSIStream : ViewStream {
+      // public RSIStream(ViewStreamDatabase vsd, string symbol, int n_days) {
+
+      // }
+
+      public RSIStream(ViewStream instrument_ohlc_str, int period) {
+         GlobalMapStream rs = new GlobalMapStream(instrument_ohlc_str, "ts_event", "contract",
+         (rows, idx) => {
+            IBlueRow ir = rows.ElementAt((int)idx);
+            IEnumerable<double> closes = BlueDataUtility.get_colvalues<double>(rows, "close", (int)idx + 1, period + 2);
+            double rsi = IndicatorCalc.calculate_RSI(closes.ToList(), period);
+            IBlueTableSchema schema = ir.schema.add_columns(Tuple.Create(typeof(double), "rsi"));
+            MutableBlueRow nr = new MutableBlueRow(schema, ir);
+            nr.setValue("rsi", rsi);
+            return nr;
+         });
+         mySetRealSource(rs);
+      }
+   }
+}

src/DataStreamSuite/SharedStreams/Business/MarketData/Charting/RenkoStream.cs

@@ -1,87 +0,0 @@
-
-using Google.Protobuf.WellKnownTypes;
-
-using System;
-using BCG.Utilities.General;
-using System.Data;
-using bts.api.domain;
-using System.Reactive.Concurrency;
-using mbts.spark;
-
-namespace BCG {
-    // Stream computes the EMA of a column in any stream. It takes in the base stream, the column name and a smoothing factor
-    // as args, and create one row with an ema column, which is the running ema of the column provided 
-    [DataStreamUserRespresentation("renko")]
-    public class RenkoStream : ViewStream {
-        private ViewStream _stream;
-        private readonly string _col_name;
-        private readonly double _brick_sz;
-        private RenkoCalculator _calc;
-
-        private BlueTable _table = new BlueTable(BlueDataUtility.form_schema(
-            "Renko", Tuple.Create("id", TypeCode.Int32),
-            Tuple.Create("entry_price", TypeCode.Double),
-            Tuple.Create("current_price", TypeCode.Double),
-            Tuple.Create("brick_low", TypeCode.Double),
-            Tuple.Create("brick_high", TypeCode.Double),
-            Tuple.Create("brick_transition", TypeCode.String),
-            Tuple.Create("n_updates_within", TypeCode.Int32)
-            ));
-
-        public RenkoStream(ViewStream stream, string col_name, double brick_sz) {
-            _col_name = col_name;
-            _stream = stream;
-            _brick_sz = brick_sz;
-            _calc = new RenkoCalculator(brick_sz);
-        }
-
-        protected override void myOnFirstSubscriber() {
-            _stream.onChange += _onstream_change;
-        }
-        protected override void myOnLastSubscriber() {
-            _stream.onChange -= new ChangeDelegate(_onstream_change);
-        }
-
-
-        private IBlueRow _remove_row(RenkoBrick brick) {
-            var row = _table[brick.id];
-            _table.deleteRow(row);
-            return row;
-        }
-
-        private IBlueRow _create_new_row(RenkoBrick brick) {
-            return new BlueRow(_table.schema, brick.id,
-            brick.entry_price, brick.current_price, brick.low, brick.high,
-            brick.entry.ToString(), brick.n_updates);
-        }
-
-        private void _onstream_change(object o) {
-            // what feeds in is a time series
-            BlueTable deleted_rows = new BlueTable(_table.schema);
-            BlueTable added_rows = new BlueTable(_table.schema);
-            DataPacket dp = o as DataPacket;
-            foreach (IBlueRow r in dp.addedRows) {
-                double val = r.get<double>(_col_name);
-                var new_bricks = _calc.update(val);
-                if (new_bricks.Count() > 0) {
-                    foreach (var nb in new_bricks) {
-                        var new_row = _create_new_row(nb);
-                        added_rows.addRow(new_row);
-                        _table.addRow(new_row);
-                    }
-                } else {
-                    var existing_brick = _calc.current_brick;
-                    var del_row = _remove_row(existing_brick);
-                    deleted_rows.addRow(del_row);
-                    var new_row = _create_new_row(existing_brick);
-                    added_rows.addRow(new_row);
-                    _table.addRow(new_row);
-                }
-            }
-            myFireEvent(new DataPacket(_table, added_rows, deleted_rows));
-        }
-    }
-}
-
-
-

src/DataStreamSuite/SharedStreams/Business/MarketData/Charting/SMAStream.cs

@@ -1,54 +1,24 @@
+namespace bts.SilverPine {
+   using bts;
+   using BCG;
 
-using Google.Protobuf.WellKnownTypes;
-
-using System;
-using BCG.Utilities.General;
-using System.Data;
-using bts.api.domain;
-
-namespace BCG {
-    // Stream computes the SMA of a column in any stream. It takes in the base stream, the column namer and a smoothing factor
-    // as args, and create one row with an sma column, which is the running ema of the column provided 
-    [DataStreamUserRespresentation("sma")]
-    public class SMAStream : ViewStream {
-        private ViewStream _stream;
-        private readonly string _col_name;
-        private Chartutils.SMACalculator _calc;
-        private readonly IBlueTableSchema _schema = BlueDataUtility.form_schema(
-            "SMA", Tuple.Create("id", TypeCode.Int32),
-            Tuple.Create("sma", TypeCode.Double));
-
-        public SMAStream(ViewStream stream, string col_name, int window_size) {
-            _col_name = col_name;
-            _stream = stream;
-            _calc = new Chartutils.SMACalculator(window_size);
-        }
-
-        protected override void myOnFirstSubscriber() {
-            _stream.onChange += new ChangeDelegate(_onstream_change);
-        }
-        protected override void myOnLastSubscriber() {
-            _stream.onChange -= new ChangeDelegate(_onstream_change);
-        }
-
-
-        private void _onstream_change(object o) {
-
-            DataPacket dp = o as DataPacket;
-            foreach (IBlueRow r in dp.addedRows) {
-                double price = r.get<double>(_col_name);
-                if (!Double.IsNaN(price))
-                    _calc.add(r.get<double>(_col_name));
-            }
-            IBlueRow nr = new BlueRow(_schema, 0, _calc.get_SMA());
-            BlueTable state = new BlueTable(_schema);
-            state.addRow(nr);
-            DataPacket dp_out = BlueDataUtility.build_dp_inc(state, myLastValue as DataPacket);
-            myFireEvent(dp_out);
-        }
-    }
-
-}
+   [DataStreamUserRespresentation("SMA")]
+   public class SMAStream : ViewStream {
+      public SMAStream(ViewStream ohlc_str, int period) {
+         GlobalMapStream rs = new GlobalMapStream(ohlc_str, "ts_event", "contract",
+         (rows, idx) => {
+            IBlueRow ir = rows.ElementAt((int)idx);
 
+            IEnumerable<double> closes = BlueDataUtility.get_colvalues<double>(rows, "close", (int)idx + 1, period + 2);
+            double rsi = IndicatorCalc.calculate_RSI(closes.ToList(), period);
+            IBlueTableSchema schema = ir.schema.add_columns(Tuple.Create(typeof(double), "rsi"));
+            MutableBlueRow nr = new MutableBlueRow(schema, ir);
+            nr.setValue("rsi", rsi);
+            return nr;
+         });
+         mySetRealSource(rs);
+      }
 
 
+   }
+}

src/DataStreamSuite/SharedStreams/Business/MarketData/Charting/SecurityRenkoStream.cs

@@ -1,49 +0,0 @@
-using System;
-
-namespace BCG {
-    /*
-       ----------------------------------------------------------------------
-                                   *Expression*
-       ----------------------------------------------------------------------
-           r=filter(bbo(),"sec='BTC-PERPETUAL'")
-           r=math(r,"wt_mid_price = (bid_price*bid_sz + ask_price*ask_sz)/(bid_sz+ask_sz)")
-           r=timeSeries(r)
-           r=math(r,"sz = (bid_sz + ask_sz) * 1.0")
-           r=filter(r,"sz > 0")
-           r=renko(r,"wt_mid_price", 2.0)
-
-       ----------------------------------------------------------------------
-                                     *Schema*
-       ----------------------------------------------------------------------
-           id                              System.Int32
-           entry_price                     System.Double
-           current_price                   System.Double
-           brick_low                       System.Double
-           brick_high                      System.Double
-           brick_transition                System.String
-           n_updates_within                System.Int32
-    */
-
-
-    public class SecurityRenkoStream : ViewStream {
-        public SecurityRenkoStream(int sec_key, double brick_sz) {
-            ViewStream r;
-
-            r = new FilterStream(new BBOStream(),
-                                  $"sec_key={sec_key}");
-            r = new MathStream(r,
-                                "wt_mid_price = (bid_price*bid_sz + ask_price*ask_sz)/(bid_sz+ask_sz)");
-            r = new TimeSeriesStream(r);
-            r = new MathStream(r,
-                                "sz = (bid_sz + ask_sz) * 1.0");
-            r = new FilterStream(r,
-                                  "sz > 0");
-            r = new RenkoStream(r,
-                                 "wt_mid_price",
-                                 brick_sz);
-
-
-            mySetRealSource(r);
-        }
-    }
-}

src/DataStreamSuite/SharedStreams/Business/MarketData/Charting/StandardDeviationStream.cs

@@ -0,0 +1,40 @@
+namespace bts.SilverPine {
+   using BCG;
+
+   using MathNet.Numerics.Statistics;
+
+   // calculates the reazlied standard deviation of a series of prices
+   public class StandardDeviationStream : ViewStream {
+      public StandardDeviationStream(string symbol, string std_dev_period)
+      :
+      this(symbol, TimeBasedFilter.parse_duration(std_dev_period)) { }
+
+      public StandardDeviationStream(string symbol, TimeSpan std_dev_period) {
+         // since this is a 1_day bar str, 90 rows => 90 days, so no need to filter
+         int n_days = std_dev_period.Days;
+         ViewStream ohlcv_str = new FuturesActiveContractStream(symbol);
+         IBlueTableSchema out_schema = null;
+         int id = 0;
+         Func<List<IBlueRow>, uint, IBlueRow> find_std_dev = (rows, idx) => {
+            int iidx = Convert.ToInt32(idx);
+            if (rows == null || rows.Count == 0)
+               return null;
+            IBlueRow ir = rows.ElementAt(iidx);
+            out_schema = out_schema ?? ir.schema.add_column(TypeCode.Double, "std_dev");
+            MutableBlueRow rv = new MutableBlueRow(out_schema, ir);
+            rv.setValue("std_dev", Double.NaN);
+            IEnumerable<double> last_n_close_prices = rows.Skip(iidx).Take(n_days).Select(r => r.get<double>("close"));
+            if (last_n_close_prices.Count() >= n_days) {
+               double std_dev = MathNet.Numerics.Statistics.Statistics.StandardDeviation(last_n_close_prices);
+               rv.setValue("std_dev", std_dev);
+            }
+            //rv.setValue("id", id++);
+            return rv;
+         };
+         ViewStream rs = new GlobalMapStream(ohlcv_str, "ts_event", find_std_dev);
+         rs = new NullFilterStream(rs, "std_dev", true);
+         mySetRealSource(rs);
+      }
+
+   }
+}

src/DataStreamSuite/SharedStreams/Business/MarketData/Charting/TradeBarsStream.cs

@@ -1,161 +0,0 @@
-using BCG;
-using BCG.Utilities.General;
-
-using System;
-
-namespace bts {
-    [DataStreamUserRespresentation("tradebars")]
-    public class TradeBarsStream : ViewStream, IWorker {
-
-        public class TradeBar {
-            static double[] calculate_log_returns(double[] prices) {
-                // Calculate logarithmic returns
-                double[] returns = new double[prices.Length - 1];
-                for (int i = 1; i < prices.Length; i++) {
-                    returns[i - 1] = Math.Log(prices[i] / prices[i - 1]);
-                }
-                return returns;
-            }
-
-            static double calculate_volatility(double[] returns) {
-                // Calculate standard deviation of logarithmic returns
-                double mean = returns.Average();
-                double sum_of_squares = returns.Sum(x => Math.Pow(x - mean, 2));
-                double variance = sum_of_squares / (returns.Length - 1);
-                double volatility = Math.Sqrt(variance);
-
-                return volatility;
-            }
-            public void add_trade(IBlueRow r) {
-                double price = r.get<double>("price");
-                //prices.Add(price);
-                double qty = r.get<double>(col_name: "qty");
-                if (n_entries == 0) {
-                    open = price;
-                    low = price < low ? price : low;
-                    high = price > high ? price : high;
-                }
-                n_entries++;
-                volume += qty;
-                vwap_sum += price * qty;
-                close = price;
-                if ("Buy".Equals(r.get<string>("aggressor"))) {
-                    n_buys_aggs++;
-                } else {
-                    n_sell_aggs++;
-                }
-            }
-            private List<double> prices = new List<double>();
-            public readonly string security;
-            public double low = Double.MaxValue, high, open, close, median, std_dev, vwap_sum, volume;
-            public uint n_buys_aggs, n_sell_aggs, n_entries;
-            public DateTime bar_start = DateTime.Now;
-            public TradeBar(string security_) {
-                security = security_;
-            }
-            public double volatility {
-                get {
-                    //if (prices.Count == 1)
-                    return 0.0;
-                    var log_returns = calculate_log_returns(prices.ToArray());
-                    return calculate_volatility(log_returns);
-                }
-            }
-        }
-
-        private readonly int _bar_period_seconds;
-        private readonly ViewStreamDatabase _vsd;
-        private ViewStream _timesales_stream;
-        private ViewStream _bbo_stream;
-        private IDictionary<uint, TradeBar> _sec_bar_map = new Dictionary<uint, TradeBar>();
-        private bool _dirty = false;
-        private TimeSpan _fire_period;
-        private DateTime _last_fire_time = DateTime.Now;
-        private BlueTable _table = new BlueTable(BlueDataUtility.form_schema("TradeBars",
-        Tuple.Create("id", TypeCode.Int32),
-        Tuple.Create("bar_start", TypeCode.DateTime),
-        Tuple.Create("bar_end", TypeCode.DateTime),
-        Tuple.Create("day_of_week", TypeCode.String),
-        Tuple.Create("n_trades", TypeCode.UInt32),
-        Tuple.Create("security", TypeCode.String),
-        Tuple.Create("sec_key", TypeCode.UInt32),
-        Tuple.Create("low", TypeCode.Double),
-        Tuple.Create("high", TypeCode.Double),
-        Tuple.Create("open", TypeCode.Double),
-        Tuple.Create("close", TypeCode.Double),
-        Tuple.Create("vwap_sum", TypeCode.Double),
-        Tuple.Create("volume", TypeCode.Double),
-        Tuple.Create("n_buy_aggs", TypeCode.UInt32),
-        Tuple.Create("n_sell_aggs", TypeCode.UInt32),
-        Tuple.Create("volatility", TypeCode.Double)));
-
-
-        public TradeBarsStream(int bar_period_seconds)
-        : this(bar_period_seconds, ViewStreamDatabase.instance) { }
-        public TradeBarsStream(int bar_period_seconds, ViewStreamDatabase vsd) {
-            _vsd = vsd;
-            _bar_period_seconds = bar_period_seconds;
-            _timesales_stream = new TimeSalesStream(10000, vsd);
-            _fire_period = TimeSpan.FromSeconds(bar_period_seconds);
-            _bbo_stream = new BBOStream(_vsd);
-        }
-
-        protected override void myOnFirstSubscriber() {
-            _timesales_stream.onChange += _on_timesales;
-            _bbo_stream.onChange += _on_bbo;
-            _vsd.addWorker(this);
-        }
-
-        protected override void myOnLastSubscriber() {
-            _vsd.removeWorker(this);
-            _timesales_stream.onChange -= _on_timesales;
-            _bbo_stream.onChange -= _on_bbo;
-        }
-
-        public void doWork() {
-            if (DateTime.Now - _last_fire_time > _fire_period) {
-                if (_dirty) {
-                    _dirty = false;
-                    _last_fire_time = DateTime.Now;
-                    BlueTable added_rows = new BlueTable(_table.schema);
-                    foreach (var kv in _sec_bar_map) {
-                        var sec_key = kv.Key;
-                        var tbar = kv.Value;
-                        IBlueRow new_row = new BlueRow(_table.schema, _table.Count, tbar.bar_start, DateTime.Now,
-                        tbar.bar_start.DayOfWeek.ToString(),
-                                                        tbar.n_entries, tbar.security, sec_key, tbar.low, tbar.high, tbar.open,
-                                                        tbar.close, tbar.vwap_sum, tbar.volume, tbar.n_sell_aggs, tbar.n_sell_aggs,
-                                                        tbar.volatility);
-                        added_rows.addRow(new_row);
-                        _table.addRow(new_row);
-                    }
-                    _sec_bar_map.Clear();
-                    DataPacket dp_out = new DataPacket(_table, added_rows, new BlueTable(_table.schema));
-                    myFireEvent(dp_out);
-                }
-            }
-        }
-
-        private void _on_timesales(object obj) {
-            DataPacket dp = obj as DataPacket;
-            foreach (IBlueRow r in dp.addedRows) {
-                uint sec_key = r.get<UInt32>("sec_key");
-                TradeBar tbar = null;
-                if (!_sec_bar_map.TryGetValue(sec_key, out tbar)) {
-                    tbar = new TradeBar(r.get<string>("sec"));
-                    _sec_bar_map.Add(sec_key, tbar);
-                }
-                _dirty = true;
-                tbar.add_trade(r);
-            }
-
-
-        }
-
-        private void _on_bbo(object obj) {
-
-        }
-    }
-
-
-}

src/DataStreamSuite/SharedStreams/Business/MarketData/Charting/VWAPStream.cs

@@ -0,0 +1,117 @@
+// namespace bts.SilverPine {
+//    using bts;
+//    using BCG;
+//    using MongoDB.Driver.Linq;
+//    using System.ComponentModel.DataAnnotations;
+//    using ZstdSharp.Unsafe;
+
+//    [DataStreamUserRespresentation("VWAP")]
+//    public class VWAPStream : ViewStream {
+//       // returns the VWAP stream for each contract 
+//       private static ViewStream build_vwap_stream(string symbol, DateTime begin_dt, DateTime end_dt) {
+//          var ohlcv_params = OHLCVStream.get_params(begin_dt, end_dt);
+//          ViewStream ohlc_str = new OHLCVStream(symbol, ohlcv_params);
+//          // stream is grouped per contract and then vwap is calculated per contract
+//          GlobalMapStream rs = new GlobalMapStream(ohlc_str, "ts_event", "contract",
+//          (rows, idx) => {
+//             Log.Gen.info($"VWAPStream.GlobalMapStream delegate n_rows: {rows.Count} idx: {idx}");
+//             IBlueRow ir = rows.ElementAt((int)idx);
+//             var sel_rows = rows.Skip((int)idx).Where(r => r.get_datetime("ts_event") >= end_dt && r.get_datetime("ts_event") <= begin_dt);
+//             var volumeprices = sel_rows.Select(r => Tuple.Create(r.get<int>("volume"), r.get<double>("close"))).ToList();
+//             double vwap = IndicatorCalc.calculate_VWAP(volumeprices);
+//             IBlueTableSchema schema = ir.schema.add_columns(Tuple.Create(typeof(double), "vwap"));
+//             MutableBlueRow nr = new MutableBlueRow(schema, ir);
+//             //vwap = -313.987;
+//             nr.setValue("vwap", vwap);
+//             return nr;
+//          });
+//          return rs;
+//       }
+//       // the args want dt1 > dt2, if its nto we, swap them
+//       static void swap_if_necessary(ref DateTime dt1, ref DateTime dt2) {
+//          if (dt1 < dt2) {
+//             DateTime aux = dt1;
+//             dt1 = dt2;
+//             dt2 = aux;
+//          }
+//       }
+
+//       // this assumes the ohlc_str has diff contracts
+//       public VWAPStream(string symbol_or_contract, DateTime begin_dt, DateTime end_dt) {
+//          swap_if_necessary(ref begin_dt, ref end_dt);
+//          Tuple<ContractUtils.ContractCategory, string> ccat_sym_contr = ContractUtils.get_symbol_category(symbol_or_contract);
+//          string symbol = String.Empty;
+//          symbol = ccat_sym_contr.Item2;
+//          var ohlcv_params = OHLCVStream.get_params(begin_dt, end_dt);
+//          ViewStream vwap_str = build_vwap_stream(symbol, begin_dt, end_dt);
+//          // vwap_str now has vwap for all contracts. if the arg was a symbol, we need to filter it for the active contract,
+//          // whihc we do with FutureActivesContractStream. If the arg was a contract we'll simply filter it
+//          ViewStream rs = vwap_str;
+//          switch (ccat_sym_contr.Item1) {
+//             case ContractUtils.ContractCategory.FutureString:
+//             case ContractUtils.ContractCategory.SpreadString:
+//                rs = new FilterStream(vwap_str, r => r.get<string>("contract").CompareTo(ccat_sym_contr.Item2) == 0);
+//                break;
+//             case ContractUtils.ContractCategory.Symbol:
+//                rs = new FuturesActiveContractStream(vwap_str);
+//                break;
+//          }
+//          rs = new FilterStream(rs, r => r.get_datetime("ts_event").CompareTo(begin_dt) == 0);
+//          rs = new NewProjectStream(rs, "! vwap");
+//          mySetRealSource(rs);
+//       }
+
+//       public VWAPStream(string symbol_or_contract, string begin_dt, string end_dt)
+//       :
+//       this(symbol_or_contract, DateTime.Parse(begin_dt), DateTime.Parse(end_dt)) { }
+//    }
+// }
+
+using System;
+
+using BCG;
+
+using bts.api;
+
+namespace bts.SilverPine {
+   /*
+      ----------------------------------------------------------------------
+                                  *Expression*
+      ----------------------------------------------------------------------
+          //nl("Find Settlement price of the SP500 future in the first week of 2023.")
+
+          r=FutureHistoricalBars("ES", "One_Day", "1/1/2024", "2/1/2024")
+          r=math(r, "vprice = volume * close")
+          r=aggregate(r, "vprice:Sum, volume:Sum")
+          r=math(r, "vwap = vprice_sum / volume_sum")
+          r=newProject(r, "vwap !")
+
+      ----------------------------------------------------------------------
+                                    *Schema*
+      ----------------------------------------------------------------------
+          id                              System.Int32
+          vwap                            System.Double
+   */
+
+   [DataStreamUserRespresentation("VWAP")]
+   public class VWAPStream : ViewStream {
+      public VWAPStream(string symbol_or_contract, DateTime beg_dt, DateTime end_dt)
+      :
+      this(symbol_or_contract, beg_dt, end_dt, ViewStreamDatabase.instance) { }
+
+      public VWAPStream(string symbol_or_contract, DateTime beg_dt, DateTime end_dt, ViewStreamDatabase db) {
+         var fc = new ContractUtils.FutureContract(symbol_or_contract);
+         ViewStream r = new FutureHistoricalBarsStream(fc, BarSize.ONE_DAY, new DateTimeRange(beg_dt, end_dt));
+
+         r = new MathStream(r,
+                 "vprice = volume * close");
+         r = new NonIncGroupByStream(r,
+                                      "vprice:Sum, volume:Sum");
+         r = new MathStream(r,
+                             "vwap = vprice_sum / volume_sum");
+         r = new NewProjectStream(r,
+                                   "vwap !");
+         mySetRealSource(r);
+      }
+   }
+}

src/DataStreamSuite/SharedStreams/Business/MarketData/EventsStream.cs

@@ -0,0 +1,18 @@
+namespace bts.SilverPine {
+   using BCG;
+
+   [DataStreamUserRespresentation("Events")]
+   public class EventsStream : ViewStream {
+      public EventsStream(string event_type) : this(ViewStreamDatabase.instance, event_type) { }
+
+      public EventsStream(ViewStreamDatabase vsd, string event_type) {
+         ViewStream rs =
+            new SQLiteStream("Data Source=C:\\Users\\m_tah\\development\\data\\stream_data.db;Version=3;",
+                              "select * from events");
+         if (!String.IsNullOrEmpty(event_type)) {
+            rs = new FilterStream(rs, r => String.Compare(event_type, r.get<string>("event_type"), true) == 0);
+         }
+         mySetRealSource(rs);
+      }
+   }
+}