Fix: Taifex Time Conversion

FIX: Taifex Crash Fixed, Added Diag Logs
ADD: TBBO & Symbol wise Naming in Recorder
2025-03-26 18:49:17 +00:00 · 2025-03-26 18:48:27 +00:00 · 2025-03-26 18:25:28 +00:00 · 2025-03-26 18:20:39 +00:00 · 2025-03-26 17:59:59 +00:00 · 2025-03-06 05:56:01 +00:00
388 changed files with 19417 additions and 6460 deletions
@@ -87,6 +87,9 @@ RUN dpkg -i /emumba/emumba-deribit-connector-3.3.1-Linux.LICENSE_DISABLED.deb \
 # Pcap
 RUN apt-get update && apt-get install -y libpcap0.8-dev

+# Databento (deps)
+RUN apt-get update && apt-get -y install libzstd-dev
+
 # Set up default executables (update-alternatives)
 RUN \
   update-alternatives --install /usr/bin/clang clang /usr/bin/clang-$clang_major 100; \
@@ -27,7 +27,7 @@
    "-p",
    "8003:8003",
    "-p",
-    "12345:12345",
+    "12345:12345"
  ],
  "mounts": [
    "source=${localEnv:HOME}/.bts/devcontainer-mounts/${devcontainerId},target=/home/bts,type=bind"
@@ -1,3 +1,3 @@
-*.cmd eol=crlf
-*.cs eol=crlf
-*.sh eol=lf
+*.cmd text=auto eol=crlf
+*.cs text=auto eol=crlf
+*.sh text=auto eol=lf
@@ -26,11 +26,23 @@
      "stopAtEntry": false,
      "console": "integratedTerminal"
    },
+    {
+      "name": "(gdb) Launch",
+      "type": "cppdbg",
+      "request": "launch",
+      "program": "/home/bts/spark/build/sparkd-debug-clang/server/sparkd",
+      "args": ["run"],
+      "stopAtEntry": false,
+      "cwd": "/home/bts/spark/build/sparkd-debug-clang/server",
+      "environment": [],
+      "externalConsole": false,
+      "MIMode": "gdb",
+    },
    {
      "name": "DataStreamQueryAnalyzer",
      "type": "coreclr",
      "request": "launch",
-      "preLaunchTask": "Build DataStream (Debug)",
+      //"preLaunchTask": "Build DataStreamQueryAnalyzer (Debug)",
      "program": "",
      "windows": {
        "program": "DataStreamQueryAnalyzer.exe",
@@ -143,4 +155,4 @@
      "default": "http://localhost:8000/config",
    },
  ]
-}
+}
@@ -526,6 +526,28 @@
        "color": "terminal.ansiGreen",
      },
    },
+    {
+      "label": "Run PubSubServer (Debug | Skip Build)",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "PubSubServer",
+      },
+      "windows": {
+        "command": "PubSubServer.exe",
+      },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
+      ],
+      "options": {
+        "cwd": "${workspaceFolder}/build/Debug/bin/net7.0"
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      },
+    },
    {
      "label": "Run PubSubServer (Release)",
      "dependsOn": "Build DataStream (Release)",
@@ -549,6 +571,28 @@
        "color": "terminal.ansiGreen",
      },
    },
+    {
+      "label": "Run PubSubServer (Release | Skip Build)",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "PubSubServer",
+      },
+      "windows": {
+        "command": "PubSubServer.exe",
+      },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
+      ],
+      "options": {
+        "cwd": "${workspaceFolder}/build/Release/bin/net7.0"
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      },
+    },
    {
      "label": "Run Spark",
      "dependsOn": "Build Spark",
@@ -684,6 +728,36 @@
        "id": "wrench"
      },
    },
+    {
+      "label": "View PubSubServer Logs (Last Run)",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "tools/logging/show_last_run.sh",
+        "args": [
+          "${env:HOME}/.local/share/DataStream/data/logging.PubSubServer",
+          "Starting PubSubServer"
+        ]
+      },
+      "windows": {
+        "command": "tools/logging/show_last_run.cmd",
+        "args": [
+          "DataStream",
+          "PubSubServer.exe",
+          "Starting PubSubServer"
+        ]
+      },
+      "presentation": {
+        "reveal": "always",
+        "focus": true,
+        "panel": "shared",
+        "showReuseMessage": false,
+        "close": true,
+      },
+      "icon": {
+        "id": "wrench"
+      },
+    },
  ],
  "inputs": [
    {
@@ -1,5 +1,6 @@
 cmake_minimum_required(VERSION 3.12)
 project(spark)
+include(GNUInstallDirs)
 set(CMAKE_CXX_STANDARD 20)

 if(WIN32)
@@ -49,8 +50,6 @@ else()
  # make sparkd look for libspark.so in its directory
  set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE)
  set(CMAKE_INSTALL_RPATH ".")
-  set(CMAKE_INSTALL_PREFIX "${CMAKE_SOURCE_DIR}/install")
-  set(SPARK_INSTALL_PREFIX "${CMAKE_INSTALL_PREFIX}/$<CONFIG>")

  if(BUILD_SPARKD)
    # Note: All macros that begin w/ 'CCAPI' are ccapi's macros (defined and/or
@@ -132,6 +131,8 @@ else()
      COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/src/server/sparkd/sparkd ${CMAKE_BINARY_DIR}/server
      COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/data_files ${CMAKE_BINARY_DIR}/server
      COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/src/lib/libspark.so ${CMAKE_BINARY_DIR}/server
+      COMMAND ln ARGS -sf ${EXTERNALS_ROOT}/lib/${CMAKE_BUILD_TYPE}/libonixs-fix-engine.so.4 ${CMAKE_BINARY_DIR}/server
+      
      COMMAND
        ln ARGS -sf
        ${EXTERNALS_ROOT}/lib/Release/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.16.0.0
@@ -145,11 +146,14 @@ else()

    install(
      DIRECTORY "${CMAKE_BINARY_DIR}/server/plugins"
-      DESTINATION "${SPARK_INSTALL_PREFIX}/bin")
+      DESTINATION "${CMAKE_INSTALL_BINDIR}")
    install(
      FILES
      "${SPARK_LIB}/libACE-5.7.5.so"
      "${SPARK_LIB}/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.16.0.0"
-      DESTINATION "${SPARK_INSTALL_PREFIX}/bin")
+      "${SPARK_LIB}/libonixs-fix-engine.so.4.10.1"
+      "${SPARK_LIB}/libonixs-fix-engine.so.4"
+      "${SPARK_LIB}/libonixs-fix-engine.so"
+      DESTINATION "${CMAKE_INSTALL_BINDIR}")
  endif()
 endif()
@@ -61,7 +61,7 @@
        },
        "CMAKE_CXX_STANDARD": "20",
        "CMAKE_INSTALL_RPATH": ".",
-        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}/sparkd",
+        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
        "CMAKE_EXE_LINKER_FLAGS": "-Wno-undef",
        "EXTERNALS_ROOT": "${sourceDir}/externals/linux",
        "BUILD_SDK": {
@@ -14,6 +14,34 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "spark_test", "src\client\sp
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Spark.Console", "src\client\Spark.Console\Spark.Console.csproj", "{89D666E6-B500-498F-B026-4D167F68ADFC}"
 EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{267F81EF-A8AB-49FD-9F56-D7EC3C1F7C7C}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "DataStreamSuite", "DataStreamSuite", "{C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamCore", "src\DataStreamSuite\DataStreamCore\DataStreamCore.csproj", "{EF26A475-6101-4C9C-A717-D60BB92A2302}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SharedStreams", "src\DataStreamSuite\SharedStreams\SharedStreams.csproj", "{911B08A3-515A-441D-84C8-8A43740B5FC1}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PubSub", "src\DataStreamSuite\PubSub\PubSub.csproj", "{14140310-2233-430A-821B-F3058A25234B}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BaseUtilities", "src\DataStreamSuite\BaseUtilities\BaseUtilities.csproj", "{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ExpressionCore", "src\DataStreamSuite\ExpressionCore\ExpressionCore.csproj", "{6F22DB96-6EAB-4034-99D9-287021C36F01}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ArithmeticExpressions", "src\DataStreamSuite\ArithmeticExpressions\ArithmeticExpressions.csproj", "{A5C33956-A41F-49B3-A41D-1082CBA59D02}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamExpressions", "src\DataStreamSuite\DataStreamExpressions\DataStreamExpressions.csproj", "{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnSortExpressions", "src\DataStreamSuite\Bts.ColumnSortExpressions\Bts.ColumnSortExpressions.csproj", "{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ColumnMapExpressions", "src\DataStreamSuite\ColumnMapExpressions\ColumnMapExpressions.csproj", "{B945A183-2174-4ACB-B289-8A5C546656FE}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BooleanExpressions", "src\DataStreamSuite\BooleanExpressions\BooleanExpressions.csproj", "{9F6D7DE8-185B-4C1D-B809-3314D092DF28}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnExpressions", "src\DataStreamSuite\Bts.ColumnExpressions\Bts.ColumnExpressions.csproj", "{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStream", "src\DataStreamSuite\DataStream\DataStream.csproj", "{39AEA075-6457-46C0-8821-1158BE6EFA57}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|x64 = Debug|x64
@@ -70,6 +98,102 @@ Global
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.Release|x64.Build.0 = Release|x64
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|x64.ActiveCfg = Debug|x64
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|x64.Build.0 = Debug|x64
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Debug|x64.Build.0 = Debug|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Release|x64.ActiveCfg = Release|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Release|x64.Build.0 = Release|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Debug|x64.Build.0 = Debug|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Release|x64.ActiveCfg = Release|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Release|x64.Build.0 = Release|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.Debug|x64.Build.0 = Debug|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.Release|x64.ActiveCfg = Release|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.Release|x64.Build.0 = Release|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Debug|x64.Build.0 = Debug|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Release|x64.ActiveCfg = Release|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Release|x64.Build.0 = Release|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Debug|x64.Build.0 = Debug|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Release|x64.ActiveCfg = Release|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Release|x64.Build.0 = Release|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Debug|x64.Build.0 = Debug|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Release|x64.ActiveCfg = Release|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Release|x64.Build.0 = Release|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Debug|x64.Build.0 = Debug|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Release|x64.ActiveCfg = Release|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Release|x64.Build.0 = Release|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Debug|x64.Build.0 = Debug|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Release|x64.ActiveCfg = Release|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Release|x64.Build.0 = Release|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.Debug|x64.Build.0 = Debug|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.Release|x64.ActiveCfg = Release|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.Release|x64.Build.0 = Release|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Debug|x64.Build.0 = Debug|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Release|x64.ActiveCfg = Release|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Release|x64.Build.0 = Release|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Debug|x64.Build.0 = Debug|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Release|x64.ActiveCfg = Release|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Release|x64.Build.0 = Release|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Debug|x64.Build.0 = Debug|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Release|x64.ActiveCfg = Release|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Release|x64.Build.0 = Release|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -80,4 +204,19 @@ Global
 	GlobalSection(Performance) = preSolution
 		HasPerformanceSessions = true
 	EndGlobalSection
+	GlobalSection(NestedProjects) = preSolution
+		{C09ED521-D2DF-40BC-A76F-EB3CB870B2F6} = {267F81EF-A8AB-49FD-9F56-D7EC3C1F7C7C}
+		{EF26A475-6101-4C9C-A717-D60BB92A2302} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{911B08A3-515A-441D-84C8-8A43740B5FC1} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{14140310-2233-430A-821B-F3058A25234B} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{6F22DB96-6EAB-4034-99D9-287021C36F01} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{B945A183-2174-4ACB-B289-8A5C546656FE} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{39AEA075-6457-46C0-8821-1158BE6EFA57} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+	EndGlobalSection
 EndGlobal
@@ -18,8 +18,12 @@

  <ItemGroup>
    <PackageReference Include="HtmlAgilityPack" Version="1.11.57" />
+    <PackageReference Include="Microsoft.Data.Analysis" Version="0.21.1" />
    <PackageReference Include="MongoDB.Bson" Version="2.23.1" />
    <PackageReference Include="MongoDB.Driver" Version="2.23.1" />
+    <PackageReference Include="Parquet.Net" Version="5.0.1" />
+    <PackageReference Include="System.Data.SQLite" Version="1.0.118" />
+    <PackageReference Include="MathNet.Numerics" Version="5.0.0" />
  </ItemGroup>

 </Project>
@@ -0,0 +1,32 @@
+namespace bts.SilverPine {
+   using System;
+   using BCG;
+   using System.Collections.Generic;
+   using Bts.DataStreamExpressions;
+
+   // All this does is take in a dsl expression, parse it and fire the data that come out of running it.
+   // So for example, if an expresison is passed 
+   [DataStreamUserRespresentation("dsl")]
+   public class DSLStream : ViewStream {
+      private readonly string _dsl;
+      public DSLStream(string dsl) {
+         Log.Gen.info($"DSLStream.cons dsl: {dsl}");
+         _dsl = dsl;
+         ViewStream rs = build_stream(dsl);
+         rs = rs ?? new LiteralStream("error", $"msg:invalid dsl \"{dsl}\"");
+         mySetRealSource(rs);
+      }
+
+      static ViewStream? build_stream(string stream_expression) {
+         try {
+            var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression);
+            var rv = exp.getView(new Dictionary<string, StatementExpression>(0));
+            return rv;
+
+         } catch (Exception ex) {
+            Log.Gen.error($"Could not build stream: {stream_expression} ex: {ex}");
+            return null;
+         }
+      }
+   }
+}
@@ -1,23 +1,24 @@
+using bts;
+
 using System;
 using System.Collections;

-namespace BCG
-{
+namespace BCG {
 	/// <summary>
 	/// Provides a stream that with distinct rows with respect to the columns
 	/// the user specifies in the argument
 	/// </summary>
-	[DataStreamUserRespresentation ("distinct")]
-	public class DistinctRowStream : ViewStream
-	{
-		public DistinctRowStream (ViewStream stream, params string[] cols)
-		{
-         ArrayList groupByCols = new ArrayList ();
-         foreach (string col in cols)
-            foreach (string c in col.Split (' '))
-               groupByCols.Add (c);
-         mySetRealSource (new GroupByStream (stream, 
-            new DistinctRowEvaluator (groupByCols.ToArray (typeof(string)) as string[]), false));
+	[DataStreamUserRespresentation("distinct")]
+	public class DistinctRowStream : ViewStream {
+		public DistinctRowStream(ViewStream stream, params string[] cols) {
+			var g_by_eval = new DistinctRowEvaluator(cols);
+			ViewStream r = new GroupByStream(stream, g_by_eval);
+			var proj_exp = String.Empty;
+			foreach (var c in cols)
+				proj_exp += $"{c} ";
+			proj_exp = proj_exp.Trim();
+			r = new NewProjectStream(r, proj_exp);
+			mySetRealSource(r);
 		}
 	}
 }
@@ -1,49 +0,0 @@
-// using System;
-// using System.Collections.Generic;
-// using System.Text;
-// using System.IO;
-
-// using System.Runtime.Serialization.Formatters.Binary;
-
-// namespace BCG
-// {
-//    public class DumpLoaderStream : ViewStream, ISideEffectStream
-//    {
-//       //wtf
-//       //public class Exception : System.Exception
-//       //{public Exception(string msg) : base(msg) { }}
-
-//       public DumpLoaderStream(string fileName, int secondsBetweenPolls)
-//       {
-//          mySetRealSource(
-//             new PollStream(() => new LoaderStream(fileName).rows, secondsBetweenPolls, 
-//                ViewStreamDatabase.instance));
-//       }
-
-//       public DumpLoaderStream(string fileName)
-//       {
-//         if (fileName == null)
-//            throw new LoaderStream.Exception("File name cannot be null");
-
-//         for (int i = 0; i < 5; i++)
-//            try
-//            {
-//               using (FileStream fs = new FileStream(fileName, FileMode.Open, FileAccess.Read))
-//               {
-//                  IClientDeserializer deser = new ClientUniversalDeserializer();
-//                  byte[] the_ops;
-//                  Array the_rows;
-//                  deser.deserialize(new Stream[] { fs }, out the_ops, out the_rows);
-
-//                  mySetRealSource(new BlueRowCollectionStaticStream((IBlueRowCollection)the_rows.GetValue(0)));
-//               }
-//               break;
-//            }
-//            catch (IOException)
-//            {
-//               System.Threading.Thread.Sleep(50);
-//            }
-
-//       }
-//    }
-// }
@@ -1,6 +1,7 @@
 using BCG;

 namespace bts {
+   // This is the other side of the SaverStream, its "loads" the stream and fires once
   [DataStreamUserRespresentation("loader")]
   public class LoaderStream : ViewStream {

@@ -5,10 +5,11 @@ using System.Text;
 using System.IO;
 using Google.Protobuf;
 using System.Globalization;
-using bts;
+using BCG;

-namespace BCG {
-   // Writes all the udpates to a stream in a file.
+namespace bts {
+   // Writes all the udpates to a stream in a file or multiples files dependong on the rollover size (as opposed to Saver
+   // which just writes one update, as a create)
   // File naming:
   // token_epoch_yyyyMMdd_HHmmss
   // File location: directory(passed)\token\
@@ -94,10 +95,10 @@ namespace BCG {
      void _on_change(object new_value) {
         DataPacket dp = new_value as DataPacket;
         byte[] serialized_update = null;
-         bool _write_create = _serializer == null || _n_updates_in_current_file == 0;
+         bool first_write = _serializer == null || _n_updates_in_current_file == 0;
         _serializer = _serializer ?? new UniversalStreamIOSerializer();
-         if (_write_create) {
-            // new files always starts with create
+         if (first_write) {
+            // first file always starts with create
            serialized_update = _serializer.serialize(new bts.StreamIOMessage(new bts.StreamIOMessage.CreateMessage(dp.newRows)));
         } else
            serialized_update = _serializer.serialize(new bts.StreamIOMessage(new bts.StreamIOMessage.MultipleRemoveAddMessage(dp.deletedRows, dp.addedRows)));
@@ -105,8 +106,8 @@ namespace BCG {
         long utc_epoch = epoch_now();

         //time, length of data, data
-         bw.Write(utc_epoch);
-         bw.Write(serialized_update.Length);
+         //bw.Write(utc_epoch);
+         //bw.Write(serialized_update.Length);
         bw.Write(serialized_update);
         _out_fs.Flush();
         myFireEvent(dp);
@@ -1,190 +0,0 @@
-// using System;
-// using System.Collections.Generic;
-// using System.Linq;
-// using System.Text;
-// using System.IO;
-// using System.Runtime.Serialization.Formatters.Binary;
-// using BCG.Utilities.General;
-// using bts.collections;
-// using System.Reflection.Metadata;
-
-
-// namespace BCG {
-
-//    public class PlaybackStream : ViewStream, IWorker {
-//       private readonly PlaybackType _type;
-//       private BlueTable _current_table;
-//       private IClientDeserializer _deserilializer = new ClientUniversalDeserializer();
-//       // these are the packets already downloaded from the current file
-//       private List<PlaybackPacket> _packets = new List<PlaybackPacket>();
-//       //private IEnumerable<FileStream> _file_streams = new List<FileStream>();
-//       private IEnumerable<string> _file_names;
-//       private Tuple<int, FileStream> _current_fs = new Tuple<int, FileStream>(-1, null);
-//       private DateTime _playback_t0, _playback_t1;
-//       private bool _all_files_read = false;
-//       SnapshotData _snapshot_data = null; // only initialized when in snapshot mode
-
-//       public PlaybackStream(string datetime_start, string datetime_end, string directory_name, string token) {
-//          _type = PlaybackType.Streaming;
-//          _playback_t0 = PersistorStream.parse_datetime(datetime_start);
-//          _playback_t1 = PersistorStream.parse_datetime(datetime_end);
-//          string token_directory_name = $"{directory_name}\\{token}";
-//          bts.Log.Gen.info($"PlaybackStream t0: {_playback_t0} t1: {_playback_t1} dir: {token_directory_name}");
-//          List<string> files_in_range = new List<string>();
-//          if (Directory.Exists(token_directory_name)) {
-//             _file_names =
-//          Directory.GetFiles(token_directory_name).Where(f => {
-//             DateTime file_time = PersistorStream.parse_datetime_from_filename(f);
-//             return file_time > _playback_t0 && file_time < _playback_t1;
-//          });
-//          }
-//       }
-
-//       protected override void myOnFirstSubscriber() {
-//          FileStream fs = new FileStream(_snapshot_data.snapshot_file, FileMode.Open, FileAccess.Read);
-//          _snapshot_data.packets = StreamPersistorUtils.load_packets_from_file(fs);
-//          PlaybackPacket snapshot_packet = StreamPersistorUtils.find_packet_at_time(_snapshot_data.packets, _snapshot_data.snapshot_time);
-//          if (snapshot_packet != null) {
-//             foreach (PlaybackPacket p in _snapshot_data.packets) {
-//                byte[] ops;
-//                Array row_array;
-//                _deserilializer.deserialize(new Stream[] { new MemoryStream(p.data) }, out ops, out row_array);
-//                for (int i = 0; i < ops.Length; i++) {
-//                   IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
-//                   byte op = ops[i];
-//                   if (op == StreamOpCode.create) {
-//                      IBlueRowCollection full_table = rows_args[0];
-//                      _current_table = new BlueTable(full_table);
-//                   } else if (op == StreamOpCode.multipleRemoveAdd) {
-//                      //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
-//                      IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
-//                      IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
-//                      BlueDataUtility.absorbChanges(_current_table, dels, adds);
-//                   } else {
-//                      bts.Log.assert(false, $"Oprator: {op} is not expected");
-//                   }
-//                   if (p.idx == snapshot_packet.idx) {
-//                      myFireEvent(new DataPacket(_current_table, _current_table, new BlueTable(_current_table.schema)));
-//                      break;
-//                   }
-//                }
-//             }
-//          }
-//       }
-
-
-//       private void _load_packets_from_current_file() {
-//          int curr_idx = _current_fs.Item1;
-//          FileStream curr_fs = _current_fs.Item2;
-//          _packets = new List<PlaybackPacket>();
-//          _packets.AddRange(StreamPersistorUtils.load_packets_from_file(curr_fs));
-//       }
-
-//       private bool current_file_all_replayed {
-//          get {
-//             bool rv = true;
-//             _packets.ForEach(p => rv = rv && p.replayed);
-//             return rv;
-//          }
-//       }
-
-//       protected override void myOnLastSubscriber() {
-//          var fs = _current_fs.Item2;
-//          if (fs != null) {
-//             fs.Close();
-//          }
-//          ViewStreamDatabase.instance.removeWorker(this);
-//       }
-
-//       // public event PlaybackStreamProgress onProgress;
-//       // public event Action onFileExhausted;
-//       private PlaybackPacket _locate_packet_to_replay() {
-//          foreach (var p in _packets) {
-//             // returns the first non replayed packet
-//             if (!p.replayed)
-//                return p;
-//          }
-//          //else
-//          {
-//             double secondsElapsed = new BinaryReader(myFileReader).ReadDouble();
-//             myNextPacketTime = myPlaybackStart + new TimeSpan(0, 0, 0, 0, (int)(1000 * secondsElapsed / myPlaybackSpeed));
-//             if (onProgress != null)
-//                onProgress(myFileReader.Position, myFileReader.Length);
-//          }
-//       }
-
-//       #region IWorker Members
-
-//       public void doWork() {
-//          if (_all_files_read)
-//             return;
-//          if (current_file_all_replayed) {
-//             _load_next_file();
-//          }
-//          PlaybackPacket curr_packet = _locate_packet_to_replay();
-//          if (curr_packet.idx == 0) { // first packet in the file, fire it rightaway 
-//             byte[] ops;
-//             Array row_array;
-//             // what a stupid interface!
-//             _deserilializer.deserialize(new Stream[] { new MemoryStream(curr_packet.data) }, out ops, out row_array);
-//             _fire_packet(ops, row_array);
-//             curr_packet.replay_timestamp = PersistorStream.epoch_now();
-//          } else {
-//             PlaybackPacket prev_packet = _get_previous_packet(curr_packet);
-//             // this is the time we need to fire (replay) this packet
-//             long time_to_fire = prev_packet.replay_timestamp + curr_packet.offset_from_previous;
-//             long now_epoch = PersistorStream.epoch_now();
-//             // fire when we are past the 
-//             if (now_epoch >= time_to_fire) {
-//                byte[] ops;
-//                Array row_array;
-//                _deserilializer.deserialize(new Stream[] { new MemoryStream(curr_packet.data) }, out ops, out row_array);
-//                _fire_packet(ops, row_array);
-//                curr_packet.replay_timestamp = PersistorStream.epoch_now();
-//             }
-//          }
-//       }
-
-//       private void _fire_packet(byte[] ops, Array row_array) {
-//          for (int i = 0; i < ops.Length; i++) {
-//             IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
-//             byte op = ops[i];
-//             if (op == StreamOpCode.create) {
-//                IBlueRowCollection full_table = rows_args[0];
-//                _current_table = new BlueTable(full_table);
-//                //_current_table.addRows(full_table);
-//                myFireEvent(new DataPacket(full_table, full_table, myLastValue == null ? new BlueTable(full_table.schema) :
-//                                           (myLastValue as DataPacket).newRows));
-//             } else if (op == StreamOpCode.multipleRemoveAdd) {
-//                //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
-//                IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
-//                IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
-//                BlueDataUtility.absorbChanges(_current_table, dels, adds);
-//                myFireEvent(new DataPacket(_current_table, adds, dels));
-//             } else {
-//                bts.Log.assert(false, $"Oprator: {op} is not expected");
-//             }
-
-//          }
-//       }
-
-//       private void _load_next_file() {
-//          var new_idx = _current_fs.Item1 + 1;
-//          if (new_idx >= _file_names.Count()) {
-//             _all_files_read = true;
-//             bts.Log.Gen.info("All files read");
-//             return;
-//          }
-//          var prev_fs = _current_fs.Item2;
-//          if (prev_fs != null)
-//             prev_fs.Close();
-//          var new_filename = _file_names.ElementAt(new_idx);
-//          bts.Log.Gen.info("PlaybackStream_load_next_file new_filename=: {new_filename}");
-//          var new_fs = new FileStream(new_filename, FileMode.Open, FileAccess.Read);
-//          _current_fs = new Tuple<int, FileStream>(new_idx, new_fs);
-//          _load_packets_from_current_file();
-//       }
-
-//       #endregion
-//    }
-// }
@@ -1,68 +0,0 @@
-// using System;
-// using System.Collections.Generic;
-// using System.Linq;
-// using System.Text;
-// using System.IO;
-
-// namespace BCG
-// {
-//    public class RecorderStream : ViewStream
-//    {
-//       private ViewStream mySource;
-//       private DateTime myStartTime;
-//       private FileStream myOutFile;
-//       private string myFileName;
-//       private IBlueDataSerializer mySerializer;
-
-//       public RecorderStream(ViewStream src, string recordFile)
-//       {
-//          mySource = src;
-//          myFileName = recordFile;
-
-//          myFileName = myFileName.Replace("%DATE%", DateTime.Now.ToString("yyyyMMdd_HHmmss")); 
-//       }
-
-//       protected override void myOnFirstSubscriber()
-//       {
-//          myStartTime = DateTime.Now;         
-//          mySource.onChange += new ChangeDelegate(mySource_onChange);
-//          myOutFile = new FileStream(myFileName, FileMode.OpenOrCreate);
-//       }
-
-//       protected override void myOnLastSubscriber()
-//       {
-//          mySource.onChange -= mySource_onChange;
-//          myOutFile.Close();
-//       }
-
-//       void mySource_onChange(object newValue)
-//       {
-//          DataPacket dp = (DataPacket)newValue;         
-
-//          BlueTable toWrite = new BlueTable(dp.schema);
-
-//          if (mySerializer == null)
-//          {
-//             mySerializer = new UniversalSerializer(dp.schema);
-//          }
-//          foreach (IBlueRow r in dp.newRows)
-//             toWrite.addRow(r);
-
-//          byte[] result;
-
-//          if (myLastValue == null)
-//             mySerializer.serialize(StreamOpCode.create, new IBlueRowCollection[] { dp.newRows }, out result);
-//          else
-//             mySerializer.serialize(StreamOpCode.multipleRemoveAdd, new IBlueRowCollection[] { dp.deletedRows, dp.addedRows }, out result);
-
-//          BinaryWriter bw = new BinaryWriter(myOutFile);
-//          //write down what time this happened
-//          bw.Write((DateTime.Now - myStartTime).TotalSeconds);
-//          bw.Write(result.Length);
-//          bw.Write(result);         
-//          myOutFile.Flush();
-
-//          myFireEvent(dp);
-//       }
-//    }
-// }
@@ -5,7 +5,8 @@ using System.IO;
 using BCG;

 namespace bts {
-   // this saves the total state in the filr provided in the cons
+   // this saves the updates as a Create on the same file, each update calls -save, which
+   // reqrites the whole data to the packet. The other side of this stream is the Loader
   [DataStreamUserRespresentation("saver")]
   public class SaverStream : ViewStream {
      private ViewStream _stream;
@@ -31,7 +32,6 @@ namespace bts {
         StreamIOMessage.CreateMessage cr_msg = new StreamIOMessage.CreateMessage(dp.newRows);
         byte[] str_bytes = _serializer.serialize(new StreamIOMessage(cr_msg));
         StreamSerializationUtils.write_to_file(str_bytes, _filename);
-
      }
   }
 }
@@ -3,6 +3,7 @@ using System.Collections.Generic;
 using System.Linq;
 using System.Globalization;
 using System.IO;
+using bts;


 namespace BCG {
@@ -0,0 +1,24 @@
+namespace bts.SilverPine {
+   using BCG;
+
+   using System;
+
+   public class NullFilterStream : ViewStream {
+      public NullFilterStream(ViewStream parent_str, string col_name)
+      : this(parent_str, col_name, true) { }
+      public NullFilterStream(ViewStream parent_str, string col_name, bool filter_out_doublenans = false) {
+         ViewStream rs = new FilterStream(parent_str,
+         r => {
+            bool is_dbnull = r.is_null(col_name);
+            if (is_dbnull) {
+               return false;
+            }
+            if (r.schema[col_name].type_code() == TypeCode.Double) {
+               return !Double.IsNaN(r.get<double>(col_name));
+            }
+            return true;
+         });
+         mySetRealSource(rs);
+      }
+   }
+}
@@ -1,102 +1,102 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
+// using System;
+// using System.Collections.Generic;
+// using System.Linq;
+// using System.Text;

-namespace BCG
-{
+// namespace BCG
+// {
   
-    public class AverageGroupsStream : ViewStream
-    {
-        public AverageGroupsStream(ViewStream input, string colToAverage, string keyColumn)
-        {
+//     public class AverageGroupsStream : ViewStream
+//     {
+//         public AverageGroupsStream(ViewStream input, string colToAverage, string keyColumn)
+//         {
            
-            myColToAverage = colToAverage;
-            myInput = input;
-            myKeyColumn = keyColumn;
-        }
+//             myColToAverage = colToAverage;
+//             myInput = input;
+//             myKeyColumn = keyColumn;
+//         }

-        private ViewStream myInput;
-        private string myColToAverage;
-        private BlueTableSchema mySchema;
-        private string myKeyColumn;
+//         private ViewStream myInput;
+//         private string myColToAverage;
+//         private BlueTableSchema mySchema;
+//         private string myKeyColumn;

-        protected override void myOnFirstSubscriber()
-        {
-            myInput.onChange += new ChangeDelegate(myInput_onChange);
-        }
-        protected override void myOnLastSubscriber()
-        {
-            myInput.onChange -= myInput_onChange;
-        }
-        private void myInput_onChange(object newValue)
-        {
-            if (mySchema == null)
-                myBuildSchema(((DataPacket)newValue).schema);
+//         protected override void myOnFirstSubscriber()
+//         {
+//             myInput.onChange += new ChangeDelegate(myInput_onChange);
+//         }
+//         protected override void myOnLastSubscriber()
+//         {
+//             myInput.onChange -= myInput_onChange;
+//         }
+//         private void myInput_onChange(object newValue)
+//         {
+//             if (mySchema == null)
+//                 myBuildSchema(((DataPacket)newValue).schema);

-            IBlueRowCollection newRows = myEvaluate((DataPacket)newValue);
+//             IBlueRowCollection newRows = myEvaluate((DataPacket)newValue);

-            if (myLastValue == null)
-                myFireEvent(new DataPacket(newRows, newRows, new BlueTable(mySchema)));
-            else
-            {
-                IBlueRowCollection added, deleted;
-                BlueDataUtility.computeDifferences(newRows, ((DataPacket) myLastValue).newRows, out deleted, out added);
-                myFireEvent(new DataPacket(newRows, added, deleted));
-            }
-        }
+//             if (myLastValue == null)
+//                 myFireEvent(new DataPacket(newRows, newRows, new BlueTable(mySchema)));
+//             else
+//             {
+//                 IBlueRowCollection added, deleted;
+//                 BlueDataUtility.computeDifferences(newRows, ((DataPacket) myLastValue).newRows, out deleted, out added);
+//                 myFireEvent(new DataPacket(newRows, added, deleted));
+//             }
+//         }

-        private void myBuildSchema(IBlueTableSchema schema)
-        {
-            List<ColumnInfo> columns = new List<ColumnInfo>();
+//         private void myBuildSchema(IBlueTableSchema schema)
+//         {
+//             List<ColumnInfo> columns = new List<ColumnInfo>();

-            foreach (ColumnInfo ci in schema.columnInfo)
-                columns.Add(ci);
+//             foreach (ColumnInfo ci in schema.columnInfo)
+//                 columns.Add(ci);

-            columns.Add(new ColumnInfo("average_"+myColToAverage, columns.Count, typeof(double)));
-            mySchema = new BlueTableSchema("average_of_" + schema.tableName, columns.ToArray());
-        }
+//             columns.Add(new ColumnInfo("average_"+myColToAverage, columns.Count, typeof(double)));
+//             mySchema = new BlueTableSchema("average_of_" + schema.tableName, columns.ToArray());
+//         }



-        private IBlueRowCollection myEvaluate(DataPacket packet)
-        {
+//         private IBlueRowCollection myEvaluate(DataPacket packet)
+//         {
         
-            IDictionary<string, double> colToRunningTotal = new Dictionary<string, double>();
-            IDictionary<string, int> colToCount = new Dictionary<string, int>();
-            IDictionary<string, IBlueRow> colToLastRow = new Dictionary<string, IBlueRow>();
+//             IDictionary<string, double> colToRunningTotal = new Dictionary<string, double>();
+//             IDictionary<string, int> colToCount = new Dictionary<string, int>();
+//             IDictionary<string, IBlueRow> colToLastRow = new Dictionary<string, IBlueRow>();

-            foreach (IBlueRow r in packet.newRows)
-            {
-                double thisVal = Convert.ToDouble(r[myColToAverage]);
+//             foreach (IBlueRow r in packet.newRows)
+//             {
+//                 double thisVal = Convert.ToDouble(r[myColToAverage]);

-                string keyVal = r[myKeyColumn].ToString();
-                if (!colToCount.ContainsKey(keyVal))
-                {
-                    colToRunningTotal[keyVal] = thisVal;
-                    colToCount[keyVal] = 1;
-                }
-                else
-                {
-                    colToRunningTotal[keyVal] += thisVal;
-                    colToCount[keyVal] += 1;
-                }
-                colToLastRow[keyVal] = r;
-            }
+//                 string keyVal = r[myKeyColumn].ToString();
+//                 if (!colToCount.ContainsKey(keyVal))
+//                 {
+//                     colToRunningTotal[keyVal] = thisVal;
+//                     colToCount[keyVal] = 1;
+//                 }
+//                 else
+//                 {
+//                     colToRunningTotal[keyVal] += thisVal;
+//                     colToCount[keyVal] += 1;
+//                 }
+//                 colToLastRow[keyVal] = r;
+//             }

-            BlueTable result = new BlueTable(mySchema);
-            foreach (string key in colToCount.Keys)
-            {
-                object[] values = new object[mySchema.columnInfo.Length];
+//             BlueTable result = new BlueTable(mySchema);
+//             foreach (string key in colToCount.Keys)
+//             {
+//                 object[] values = new object[mySchema.columnInfo.Length];

-                colToLastRow[key].CopyTo(values, 0);
-                values[values.Length - 1] = packet.newRows.Count > 0 ? colToRunningTotal[key] / packet.newRows.Count : 0.0;
-                result.addRow(new BlueRow(mySchema, values));
-            }
+//                 colToLastRow[key].CopyTo(values, 0);
+//                 values[values.Length - 1] = packet.newRows.Count > 0 ? colToRunningTotal[key] / packet.newRows.Count : 0.0;
+//                 result.addRow(new BlueRow(mySchema, values));
+//             }

-            return result;
-        }
+//             return result;
+//         }

      
-    }
-}
+//     }
+// }
@@ -0,0 +1,31 @@
+namespace bts.SilverPine {
+   using BCG;
+
+   using System;
+   using System.Collections.Generic;
+   using System.Linq;
+
+   [DataStreamUserRespresentation("Average")]
+   public class AverageNonIncGroupsStream : ViewStream {
+      public AverageNonIncGroupsStream(ViewStream parent_stream, string inp_col_name, params string[] group_by_cols) {
+         NonIncGroupByStream rs = new NonIncGroupByStream(parent_stream, () => group_by_cols,
+                                                         (g_rows) => _find_mean(inp_col_name, g_rows));
+         mySetRealSource(rs);
+      }
+
+      private List<IBlueRow> _find_mean(string inp_col_name, List<IBlueRow> g_rows) {
+         IBlueRow inp_row = g_rows.ElementAt(0);
+         IEnumerable<double> inp_vals = BlueDataUtility.get_colvalues<double>(g_rows, inp_col_name);
+         double sum = 0.0;
+         inp_vals.ToList().ForEach(d => sum += d);
+         string out_col_name = $"average_{inp_col_name}";
+         double average = sum / inp_vals.Count();
+         IBlueTableSchema schema_out = inp_row.schema.add_column(TypeCode.Double, out_col_name);
+         MutableBlueRow out_row = new MutableBlueRow(schema_out, inp_row);
+         out_row.setValue(out_col_name, average);
+         List<IBlueRow> rv = new List<IBlueRow>();
+         rv.Add(out_row);
+         return rv;
+      }
+   }
+}
@@ -0,0 +1,126 @@
+namespace bts.SilverPine {
+   using BCG;
+
+   using System;
+   using System.Collections.Generic;
+   using System.Linq;
+   //using MathNet.Numerics.Statistics;
+
+
+   public enum MultiRowOp {
+      Mean,
+      Median,
+      Mode,
+      Range,
+      StdDev,
+      Max,
+      Min,
+      First,
+      Last,
+      Sum,
+      EdgeDiff,
+      EdgeDiffPercentage,
+      //IsEdgeDiff,
+   }
+   public class MultiRowOpHelper {
+      public readonly List<Tuple<string, MultiRowOp>> operations;
+
+      public static MultiRowOpHelper parse(string text) {
+         // text is inp_col: op, inp_col: op, inp_col:op
+         List<Tuple<string, MultiRowOp>> rv = new List<Tuple<string, MultiRowOp>>();
+         string[] calcs = text.Split(',');
+         foreach (string c in calcs) {
+            string[] toks = c.Split(':');
+            string inp_col = toks[0].Trim();
+            string op_txt = toks[1].Trim();
+            MultiRowOp op = Enum.Parse<MultiRowOp>(op_txt, true);
+            rv.Add(Tuple.Create(inp_col, op));
+         }
+         return new MultiRowOpHelper(rv);
+      }
+
+      public MultiRowOpHelper(List<Tuple<string, MultiRowOp>> operations_) {
+         operations = operations_;
+      }
+
+      public static IBlueRow group_by(List<IBlueRow> rows, MultiRowOpHelper parameters) {
+         if (rows.Count() == 0) {
+            return null;
+         }
+
+         IBlueRow ir = rows.ElementAt(0);
+         IBlueTableSchema in_schema = ir.schema;
+         IBlueTableSchema out_schema = in_schema;
+         IDictionary<string, Tuple<string, MultiRowOp>> out_col_map =
+            new Dictionary<string, Tuple<string, MultiRowOp>>();
+         foreach (var t in parameters.operations) {
+            string out_col_name = $"{t.Item1}_{t.Item2}".ToLower();
+            out_schema = out_schema.add_column(TypeCode.Double, out_col_name);
+            out_col_map[out_col_name] = t;
+         }
+         MutableBlueRow or = new MutableBlueRow(out_schema, ir);
+         foreach (var kv in out_col_map) {
+            string out_col_name = kv.Key;
+            string in_col_name = kv.Value.Item1;
+            MultiRowOp op = kv.Value.Item2;
+            IEnumerable<double> col_vals = BlueDataUtility.get_colvalues<double>(rows, in_col_name);
+            double comp_res = compute_op(col_vals, op);
+            or.setValue(out_col_name, comp_res);
+         }
+         return or;
+      }
+
+      // rolling_rows are the rolling window of rows. the 0th row is the one thats gettin
+      // calculated. rows 1 - n-1 are its window rows. the schema is based on the operations.
+
+      public IBlueRow rolling_calc(List<IBlueRow> rolling_rows) {
+         if (rolling_rows.Count() == 0) {
+            return null;
+         }
+         IBlueRow ir = rolling_rows.ElementAt(0);
+         IBlueTableSchema in_schema = ir.schema;
+         IBlueTableSchema out_schema = in_schema;
+         IDictionary<string, Tuple<string, MultiRowOp>> out_col_map =
+            new Dictionary<string, Tuple<string, MultiRowOp>>();
+         foreach (var t in operations) {
+            string out_col_name = $"{t.Item1}_{t.Item2}".ToLower();
+            out_schema = out_schema.add_column(TypeCode.Double, out_col_name);
+            out_col_map[out_col_name] = t;
+         }
+         MutableBlueRow or = new MutableBlueRow(out_schema, ir);
+         foreach (var kv in out_col_map) {
+            string out_col_name = kv.Key;
+            string in_col_name = kv.Value.Item1;
+            MultiRowOp op = kv.Value.Item2;
+            IEnumerable<double> col_vals = BlueDataUtility.get_colvalues<double>(rolling_rows, in_col_name);
+            double comp_res = compute_op(col_vals, op);
+            or.setValue(out_col_name, comp_res);
+         }
+         return or;
+      }
+
+      public static double compute_op(IEnumerable<double> col_vals, MultiRowOp op) {
+         switch (op) {
+            case MultiRowOp.Max:
+               return col_vals.Max<double>();
+            case MultiRowOp.Min:
+               return col_vals.Min<double>();
+            case MultiRowOp.Mean:
+               return col_vals.Sum() / col_vals.Count();
+            case MultiRowOp.Sum:
+               return col_vals.Sum();
+            case MultiRowOp.First:
+               return col_vals.First();
+            case MultiRowOp.Last:
+               return col_vals.Last();
+            case MultiRowOp.StdDev:
+               return MathNet.Numerics.Statistics.Statistics.StandardDeviation(col_vals);
+            default:
+               // TODO
+               Log.assert(false);
+               return Double.NaN;
+         }
+      }
+
+   }
+}
@@ -10,6 +10,6 @@ namespace BCG {
        }

        public MaxStream(ViewStream stream, string outputCol)
-           : this(new AddColumnsStream(stream, "fakeCol=0"), outputCol, new string[] { "fakeCol" }) { }
+           : this(new AddColumnsStream(stream, "fake_col: 0"), outputCol, new string[] { "fake_col" }) { }
    }
 }
@@ -0,0 +1,20 @@
+namespace bts.SilverPine {
+   using BCG;
+
+   using System;
+
+   [DataStreamUserRespresentation("AggregationStream")]
+   public class AgregationStream : ViewStream {
+
+      public AgregationStream(ViewStream parent_str, string operation_txt) {
+         ViewStream rs = new NonIncGroupByStream(parent_str, String.Empty, operation_txt);
+         mySetRealSource(rs);
+      }
+
+      public AgregationStream(ViewStream parent_str, string groupby_col, string operation_txt) {
+         ViewStream rs = new NonIncGroupByStream(parent_str, groupby_col, operation_txt);
+         mySetRealSource(rs);
+      }
+   }
+
+}
@@ -1,13 +1,40 @@
 using System;
 using System.Collections.Generic;
 using System.Diagnostics;
+using System.Linq;
+using System.Text.Json;

 using bts;
+using bts.SilverPine;

-namespace BCG
-{
-   public class NonIncGroupByStream : ViewStream
-   {
+namespace BCG {
+   [DataStreamUserRespresentation("AggregateStream")]
+   public class NonIncGroupByStream : ViewStream {
+      private class DelNonIncGroupByEvaluator : INonIncGroupByEvaluator {
+         Func<string[]> _groupby_cols_func;
+         Func<List<IBlueRow>, List<IBlueRow>> _evaluate_func;
+         public DelNonIncGroupByEvaluator(Func<string[]> groupby_cols_func,
+                                          Func<List<IBlueRow>, List<IBlueRow>> evaluate_func) {
+            _groupby_cols_func = groupby_cols_func;
+            _evaluate_func = evaluate_func;
+         }
+         public IBlueRowCollection evaluate(IDictionary<int, List<IBlueRow>> grouped_rows) {
+            BlueTable rv = null;
+            foreach (var kv in grouped_rows) {
+               List<IBlueRow> g_rows = kv.Value;
+               List<IBlueRow> out_rows = _evaluate_func.Invoke(g_rows);
+               if (out_rows != null && out_rows.Count > 0) {
+                  rv = rv ?? new BlueTable(out_rows.ElementAt(0).schema);
+               }
+               rv.addRows(out_rows);
+            }
+            return rv;
+         }
+         public string[] get_group_by_cols() {
+            return _groupby_cols_func();
+         }
+
+      }
      private ViewStream _main_stream;
      private INonIncGroupByEvaluator _groupby_func;
      private BlueRowHasher _row_hasher;
@@ -15,24 +42,50 @@ namespace BCG
      private Stopwatch _timer = new();
      private int _n_total_fires = 0, _n_expensive_fires = 0;

-      public NonIncGroupByStream(ViewStream main_stream, INonIncGroupByEvaluator groupby_func)
-      {
+      public NonIncGroupByStream(ViewStream main_stream, INonIncGroupByEvaluator groupby_func) {
         _main_stream = main_stream;
         _groupby_func = groupby_func;
         _groupby_cols = _groupby_func.get_group_by_cols();
-         _row_hasher = new BlueRowHasher(_groupby_cols.Length);
+         _row_hasher = new BlueRowHasher(_groupby_cols == null ? 0 : _groupby_cols.Length);
      }

+      public NonIncGroupByStream(ViewStream main_stream, Func<string[]> groupby_cols_func,
+                                 Func<List<IBlueRow>, List<IBlueRow>> evaluate_func)
+                                 : this(main_stream, new DelNonIncGroupByEvaluator(groupby_cols_func, evaluate_func)) { }
+
+      public NonIncGroupByStream(ViewStream parent_str, string groupby_col, string standard_group_by_text)
+      : this(parent_str, groupby_col, standard_group_by_text, String.Empty) { }
+
+      public NonIncGroupByStream(ViewStream parent_str, string groupby_col, string standard_group_by_text, string optional_sort_col) {
+         MultiRowOpHelper std_group_by = MultiRowOpHelper.parse(standard_group_by_text);
+         Func<string[]> groupby_cols_func = () => {
+            return String.IsNullOrEmpty(groupby_col) ? null : new string[] { groupby_col };
+         };
+         Func<List<IBlueRow>, List<IBlueRow>> evaluate_func = (group_rows) => {
+            if (!String.IsNullOrEmpty(optional_sort_col))
+               group_rows.Sort((r1, r2) => BlueDataUtility.compare_rows(r2, r1, optional_sort_col));
+            IBlueRow rv = MultiRowOpHelper.group_by(group_rows, std_group_by);
+            return new IBlueRow[] { rv }.ToList();
+         };
+         ViewStream rs = new NonIncGroupByStream(parent_str, groupby_cols_func, evaluate_func);
+         mySetRealSource(rs);
+      }
+      public NonIncGroupByStream(ViewStream parent_str, string standard_group_by_text) :
+      this(parent_str, String.Empty, standard_group_by_text) {
+      }
+
+
      protected override void myOnFirstSubscriber() { _main_stream.onChange += _main_stream_onchange; }

      protected override void myOnLastSubscriber() { _main_stream.onChange -= _main_stream_onchange; }

-      private void _main_stream_onchange(object nv)
-      {
+      private void _main_stream_onchange(object nv) {
         DataPacket dp = nv as DataPacket;
         ///grouping together
         _timer.Restart();
-         IDictionary<int, List<IBlueRow>> grouped_rows = BlueDataUtility.group_together (dp.newRows, _groupby_cols, ref _row_hasher);
+         IDictionary<int, List<IBlueRow>> grouped_rows = (_groupby_cols == null || _groupby_cols.Length == 0) ?
+         new Dictionary<int, List<IBlueRow>>() { { 0, dp.newRows.to_list() } } :
+         BlueDataUtility.group_together(dp.newRows, _groupby_cols, ref _row_hasher);
         _timer.Stop();
         var groupElapsedMs = _timer.ElapsedMilliseconds;
         if (groupElapsedMs >= 50) {
@@ -44,14 +97,15 @@ namespace BCG
         IBlueRowCollection total_state = _groupby_func.evaluate(grouped_rows);
         _timer.Stop();
         var applyElapsedMs = _timer.ElapsedMilliseconds;
-         _n_total_fires++;         
-         if (applyElapsedMs >= 50)
-         {
+         _n_total_fires++;
+         if (applyElapsedMs >= 50) {
            _n_expensive_fires++;
            Log.DataFlow.info(
               $"Time to evaluate {grouped_rows.Count} groups {_timer.ElapsedMilliseconds}ms, ratio {_n_expensive_fires}/{_n_total_fires}");
-         }         
-         myFireEvent(new DataPacket(total_state, total_state, myLastValue == null ? new BlueTable(total_state.schema) : (myLastValue as DataPacket).newRows));             
+         }
+         // TODO,
+         if (total_state != null)
+            myFireEvent(new DataPacket(total_state, total_state, myLastValue == null ? new BlueTable(total_state.schema) : (myLastValue as DataPacket).newRows));
      }
   }
 }
@@ -1,15 +1,16 @@
 using bts;
+using bts.collections;

 using mbts;

 using System;
 using System.Collections.Generic;
 using System.Diagnostics;
+using System.Linq;
+using System.Security.Cryptography;

-namespace BCG
-{
-   public class NonIncJoinBookkeeper : IDisposable
-   {
+namespace BCG {
+   public class NonIncJoinBookkeeper : IDisposable {
      public Action on_new_data;
      private readonly NonIncJoinSpec _join_spec;
      private readonly int _join_position;
@@ -26,41 +27,71 @@ namespace BCG

      private int[] _join_col_indices = null,
                     _data_col_indices = null;
-      
-      public NonIncJoinBookkeeper(NonIncJoinSpec the_join_spec, int join_position, int start_index_in_output, BlueRowHasher row_hasher)
-      {
+
+      public NonIncJoinBookkeeper(NonIncJoinSpec the_join_spec, int join_position, int start_index_in_output, BlueRowHasher row_hasher) {
         _join_spec = the_join_spec;
         _join_position = join_position;
         _start_index_in_output = start_index_in_output;
         _row_hasher = row_hasher;
         _is_left_most = _join_position == 0;
         _is_tag_along = _join_spec.join_col_names.Length == 0;
+      }
+
+      private bool _filled_missing_data = false;
+      private void _fill_missing_data(DataPacket dp) {
+         if (_filled_missing_data)
+            return;
+         Func<string[], bool> null_or_empty = (str_arr) => {
+            if (str_arr == null || str_arr.Length == 0)
+               return true;
+            foreach (string s in str_arr) {
+               if (!String.IsNullOrEmpty(s)) {
+                  return false;
+               }
+            }
+            return true;
+         };
+         Func<string[], string[]> remove_bad_cols = (col_names) => {
+            HashSet<string> schema_col_map = new HashSet<string>(dp.schema.columnNames);
+            List<string> cols_list = new List<string>(col_names);
+            cols_list.RemoveWhere(c => !schema_col_map.Contains(c));
+            return cols_list.ToArray();
+         };
+         if (null_or_empty(_join_spec.data_col_names)) {
+            _join_spec.data_col_names = dp.schema.columnNames;
+         }
+         _join_spec.data_col_names = remove_bad_cols(_join_spec.data_col_names);
+         if (null_or_empty(_join_spec.data_col_names)) {
+            _join_spec.data_col_names = dp.schema.columnNames;
+         }
+         if (null_or_empty(_join_spec.projected_col_names)) {
+            _join_spec.projected_col_names = _join_spec.data_col_names;
+         }
+      }
+
+      public void init() {
         _join_spec.stream.onChange += _on_stream_cb;
      }

-      public void _on_stream_cb(object dp_obj)
-      {
+      public void _on_stream_cb(object dp_obj) {
+         Log.Gen.info($"NonIncJoinBookkeeper._on_stream_cb dp_obj: {dp_obj}");
         var timer = Stopwatch.StartNew();
         DataPacket dp = (DataPacket)dp_obj;
-         if (_schema == null)
-         {
+         _fill_missing_data(dp);
+         if (_schema == null) {
            _schema = dp.schema;
            _join_col_indices = BlueDataUtility.get_column_indices(_schema, _join_spec.join_col_names);
            _data_col_indices = BlueDataUtility.get_column_indices(_schema, _join_spec.data_col_names);
         }

-         if (_is_left_most)
-         {
+         if (_is_left_most) {
            _row_list.Clear();
            foreach (IBlueRow r in dp.newRows)
-               _row_list.Add(new Pair<int, IBlueRow>(_row_hasher.get_hash(r, _join_col_indices), r));            
-         }
-         else
-         {
+               _row_list.Add(new Pair<int, IBlueRow>(_row_hasher.get_hash(r, _join_col_indices), r));
+         } else {
            if (_is_tag_along)
               _solitary_row = BlueDataUtility.extractRow(dp.newRows, 0);
-            else
-            {
+            else {
               _row_map.Clear();
               foreach (IBlueRow r in dp.newRows)
                  _row_map[_row_hasher.get_hash(r, _join_col_indices)] = r;
@@ -79,31 +110,25 @@ namespace BCG

      public bool is_ready { get { return _schema != null; } }

-      public void constribute_cols_to_schema(List<ColumnInfo> cis)
-      {
+      public void constribute_cols_to_schema(List<ColumnInfo> cis) {
         for (int i = 0; i < _join_spec.data_col_names.Length; i++)
            cis.Add(new ColumnInfo(_join_spec.projected_col_names[i], cis.Count, _schema[_join_spec.data_col_names[i]].colType));
      }

      public List<Pair<int, IBlueRow>> get_list() { return _row_list; }

-      public void copy_master_row_to_output_row(int row_index, object[] output_row)
-      {
+      public void copy_master_row_to_output_row(int row_index, object[] output_row) {
         IBlueRow this_row = _row_list[row_index].second;
         BlueDataUtility.copy_to_row(this_row, _data_col_indices, output_row, _start_index_in_output);
      }

-      public void copy_matching_row_to_output_row(int row_hash, object[] output_row)
-      {
-         if (_is_tag_along)
-         {
+      public void copy_matching_row_to_output_row(int row_hash, object[] output_row) {
+         if (_is_tag_along) {
            if (_solitary_row != null)
               BlueDataUtility.copy_to_row(_solitary_row, _data_col_indices, output_row, _start_index_in_output);
            else
               copy_dbnull_to_output_row(output_row);
-         }
-         else
-         {
+         } else {
            IBlueRow this_row = null;
            if (_row_map.TryGetValue(row_hash, out this_row))
               BlueDataUtility.copy_to_row(this_row, _data_col_indices, output_row, _start_index_in_output);
@@ -112,8 +137,7 @@ namespace BCG
         }
      }

-      private void copy_dbnull_to_output_row(object[] output_row)
-      {
+      private void copy_dbnull_to_output_row(object[] output_row) {
         for (int i = 0; i < _data_col_indices.Length; i++)
            output_row[i + _start_index_in_output] = DBNull.Value;
      }
@@ -3,63 +3,63 @@ using System.Collections.Generic;
 using System.Linq;
 using System.Text;
 using System.Threading.Tasks;
+
 using mbts;

-namespace BCG
-{
-   public class NonIncJoinSpec
-   {
+namespace BCG {
+   // this a collection of data, theres no joining logic in here
+   public class NonIncJoinSpec {
      public ViewStream stream;
+      // join col names are the names on which the streams are joined (for equality),
+      // data col names are the names in the original stream
+      // projected col names are their new names, ie if its x:l_x, then x is in the 
+      // data_col_names and l_x in projected cols
      public string[] join_col_names, data_col_names, projected_col_names;

      public NonIncJoinSpec() { }

      public NonIncJoinSpec(ViewStream the_stream, string[] the_join_col_names, string[] the_data_col_names)
-         : this (the_stream, the_join_col_names, the_data_col_names, the_data_col_names)
-      { }
+         : this(the_stream, the_join_col_names, the_data_col_names, the_data_col_names) { }

-      public NonIncJoinSpec(ViewStream the_stream, string[] the_join_col_names, string[] the_data_col_names, string[] the_projected_col_names)
-      {
+      public NonIncJoinSpec(ViewStream the_stream, string[] the_join_col_names, string[] the_data_col_names, string[] the_projected_col_names) {
         stream = the_stream;
         join_col_names = the_join_col_names;
         data_col_names = the_data_col_names;
         projected_col_names = the_projected_col_names;
      }

-      public static NonIncJoinSpec[] from_triplet(params Triplet<ViewStream, string, string>[] join_specs)
-      {
+      public static NonIncJoinSpec[] from_triplet(params Triplet<ViewStream, string, string>[] join_specs) {
         List<NonIncJoinSpec> rv = new List<NonIncJoinSpec>();
-         foreach (Triplet<ViewStream, string, string> r in join_specs)
-         {
-            string[] keys = String.IsNullOrEmpty (r.second) ? new string[] {} : r.second.Split(' ');
+         foreach (Triplet<ViewStream, string, string> r in join_specs) {
+            string[] keys = String.IsNullOrEmpty(r.second) ? new string[] { } : r.second.Split(' ');
            string[] visible_cols = r.third.Split(' ');
            List<string> data_col_names = new List<string>(), mapped_col_names = new List<string>();
-            foreach (string s in visible_cols)
-            {
+            foreach (string s in visible_cols) {
               string[] toks = s.Split(':');
-               if (toks.Length == 1)
-               {
+               if (toks.Length == 1) {
                  data_col_names.Add(toks[0]);
                  mapped_col_names.Add(toks[0]);
-               }
-               else if (toks.Length == 2)
-               {
+               } else if (toks.Length == 2) {
                  data_col_names.Add(toks[0]);
                  mapped_col_names.Add(toks[1]);
               }
            }
-            rv.Add (new NonIncJoinSpec (r.first, keys, data_col_names.ToArray (), mapped_col_names.ToArray()));
+            rv.Add(new NonIncJoinSpec(r.first, keys, data_col_names.ToArray(), mapped_col_names.ToArray()));
         }
-         return rv.ToArray ();
+         return rv.ToArray();
+      }
+
+      public override string ToString() {
+         Func<string[], string> arr_txt = (a) => {
+            string rv = "";
+            foreach (string s in a) {
+               rv += "{s} ";
+            }
+            rv = rv.Trim();
+            return rv;
+         };
+         string rv = $"stream: {stream} join_col_names: {arr_txt(join_col_names)} data_col_names: {arr_txt(data_col_names)} projected_col_names: {arr_txt(projected_col_names)}";
+         return rv;
      }
   }
-
-   //[DataStreamUserRespresentation("ninctest")]
-   //public class NonIncJoinTestStream : ViewStream
-   //{
-   //   public NonIncJoinTestStream (string key_cols, string data_cols)
-   //   {
-   //      NonIncJoinSpec.from_triplet(new Triplet<ViewStream, string, string>(null, key_cols, data_cols));
-   //   }
-   //}
 }
@@ -6,11 +6,9 @@ using System;
 using System.Collections.Generic;
 using System.Diagnostics;

-namespace BCG
-{
+namespace BCG {
   [DataStreamUserRespresentation("fastJoin")]
-   public class NonIncJoinStream : ViewStream, BCG.Utilities.General.IWorker
-   {
+   public class NonIncJoinStream : ViewStream, BCG.Utilities.General.IWorker {
      private readonly NonIncJoinSpec[] _join_specs;
      private readonly INonIncJoinInlineMapper _output_mapper;
      private readonly BlueRowHasher _row_hasher;
@@ -28,24 +26,19 @@ namespace BCG
               {
                  new Triplet<ViewStream, string, string> (left, left_join_cols, left_proj),
                  new Triplet<ViewStream, string, string> (right, right_join_cols, right_proj)
-               })
-      { }
+               }) { }

      public NonIncJoinStream(params Triplet<ViewStream, string, string>[] join_specs)
-         : this(NonIncJoinSpec.from_triplet(join_specs), null, 0)
-      { }
+         : this(NonIncJoinSpec.from_triplet(join_specs), null, 0) { }

      public NonIncJoinStream(ViewStreamDatabase vsd, INonIncJoinInlineMapper output_mapper, int throttles_ms,
                              params Triplet<ViewStream, string, string>[] join_specs)
-         : this(vsd, NonIncJoinSpec.from_triplet(join_specs), output_mapper, throttles_ms)
-      { }
+         : this(vsd, NonIncJoinSpec.from_triplet(join_specs), output_mapper, throttles_ms) { }

      public NonIncJoinStream(NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper output_mapper, int throttles_ms = 0)
-         : this(ViewStreamDatabase.instance, join_specs, output_mapper, throttles_ms)
-      { }
+         : this(ViewStreamDatabase.instance, join_specs, output_mapper, throttles_ms) { }

-      public NonIncJoinStream(ViewStreamDatabase vsd, NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper output_mapper, int throttles_ms = 0)
-      {
+      public NonIncJoinStream(ViewStreamDatabase vsd, NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper output_mapper, int throttles_ms = 0) {
         _join_specs = join_specs;
         _output_mapper = output_mapper;
         _row_hasher = new BlueRowHasher(_join_specs[0].join_col_names.Length);
@@ -53,12 +46,11 @@ namespace BCG
         _vsd = vsd;
      }

-      protected override void myOnFirstSubscriber()
-      {
+      protected override void myOnFirstSubscriber() {
         int data_col_no = 1;
-         for (int jp = 0; jp < _join_specs.Length; jp++)
-         {
+         for (int jp = 0; jp < _join_specs.Length; jp++) {
            NonIncJoinBookkeeper s = new NonIncJoinBookkeeper(_join_specs[jp], jp, data_col_no, _row_hasher);
+            Log.Gen.info($"NonIncJoinStream.myOnFirstSubscriber jp: {jp} spec: {_join_specs[jp]}");
            data_col_no += _join_specs[jp].data_col_names.Length;
            s.on_new_data += _stream_callback;
            _book_keeping_data.Add(s);
@@ -66,10 +58,11 @@ namespace BCG
         if (_output_mapper != null)
            _output_mapper.onMapperChange += _on_mapper_change;
         _vsd.addWorker(this);
+         foreach (var bkd in _book_keeping_data)
+            bkd.init();
      }

-      protected override void myOnLastSubscriber()
-      {
+      protected override void myOnLastSubscriber() {
         foreach (NonIncJoinBookkeeper d in _book_keeping_data)
            d.Dispose();
         if (_output_mapper != null)
@@ -77,30 +70,26 @@ namespace BCG
         _vsd.removeWorker(this);
      }

-      private void _stream_callback()
-      {
+      private void _stream_callback() {
+         Log.Gen.info("NonIncJoinStream._stream_callback received callback");
         _is_dirty = true;
         doWork();
      }

-      private void _on_mapper_change()
-      {
+      private void _on_mapper_change() {
         _is_dirty = true;
         doWork();
      }

-      private bool _is_ready()
-      {
+      private bool _is_ready() {
         bool rv = true;
         foreach (NonIncJoinBookkeeper d in _book_keeping_data)
            rv &= d.is_ready;
         return rv;
      }

-      private void _construct_schema()
-      {
-         if (_output_schema == null)
-         {
+      private void _construct_schema() {
+         if (_output_schema == null) {
            List<ColumnInfo> cis = new List<ColumnInfo>(new ColumnInfo[] { new ColumnInfo("id", 0, typeof(int)) });
            foreach (NonIncJoinBookkeeper d in _book_keeping_data)
               d.constribute_cols_to_schema(cis);
@@ -109,27 +98,23 @@ namespace BCG
      }

      private int _n_total_fires = 0, _n_expensive_fires = 0;
-      private void _form_and_fire()
-      {
-         if (_is_ready())
-         {
+      private void _form_and_fire() {
+         Log.Gen.info($"NonIncJoinStream._form_and_fire _is_ready(): {_is_ready()}");
+         if (_is_ready()) {
            _construct_schema();
            var timer = Stopwatch.StartNew();
            IList<Pair<int, IBlueRow>> left_row_list = _book_keeping_data[0].get_list();
            List<object[]> output_rows = new List<object[]>();

-            for (int i = 0; i < left_row_list.Count; i++)
-            {
+            for (int i = 0; i < left_row_list.Count; i++) {
               object[] or = new object[_output_schema.columnInfo.Length];
               or[0] = i;
               output_rows.Add(or);
            }

-            for (int ji = 0; ji < _book_keeping_data.Count; ji++)
-            {
+            for (int ji = 0; ji < _book_keeping_data.Count; ji++) {
               NonIncJoinBookkeeper bk_data = _book_keeping_data[ji];
-               for (int i = 0; i < left_row_list.Count; i++)
-               {
+               for (int i = 0; i < left_row_list.Count; i++) {
                  if (ji == 0)
                     bk_data.copy_master_row_to_output_row(i, output_rows[i]);
                  else
@@ -139,19 +124,20 @@ namespace BCG
            IBlueRowCollection dp_output_rows = _apply_map(new SequentialIDBlueTable(_output_schema, output_rows));
            timer.Stop();
            _n_total_fires++;
-            if (timer.ElapsedMilliseconds >= 250) {
-               _n_expensive_fires++;
-               string log_msg = $"Join and map time for {_book_keeping_data.Count} "
-                  + $"tables {timer.Elapsed} ratio {_n_expensive_fires}/{_n_total_fires}";
-            }
-            myFireEvent(new DataPacket(dp_output_rows, dp_output_rows, myLastValue == null ? new BlueTable(_output_schema) : (myLastValue as DataPacket).newRows));
+            //if (timer.ElapsedMilliseconds >= 250) {
+            _n_expensive_fires++;
+            string log_msg = $"Join and map time for {_book_keeping_data.Count} "
+               + $"tables {timer.Elapsed.TotalMilliseconds} msecs, ratio {_n_expensive_fires}/{_n_total_fires}";
+            Log.Gen.info(log_msg);
+            //}
+            var dp_out = BlueDataUtility.build_dp_inc(dp_output_rows, myLastValue as DataPacket);
+            myFireEvent(dp_out);
+            //myFireEvent(new DataPacket(dp_output_rows, dp_output_rows, myLastValue == null ? new BlueTable(_output_schema) : (myLastValue as DataPacket).newRows));
         }
      }

-      private IBlueRowCollection _apply_map(IBlueRowCollection unmapped_rows)
-      {
-         if (_output_mapper != null)
-         {
+      private IBlueRowCollection _apply_map(IBlueRowCollection unmapped_rows) {
+         if (_output_mapper != null) {
            IBlueRowCollection output_rows;
            var timer = Stopwatch.StartNew();
            output_rows = _output_mapper.map_rows(unmapped_rows);
@@ -160,17 +146,14 @@ namespace BCG
               Log.Gen.info($"Time to map {unmapped_rows.Count} rows into {output_rows.Count} rows {timer.Elapsed}");
            }
            return output_rows;
-         }
-         else
-         {
+         } else {
            return unmapped_rows;
         }
      }

-      public void doWork()
-      {
-         if (DateTime.Now - _last_fire_time >= _throttle_period && _is_dirty)
-         {
+      public void doWork() {
+         Log.Gen.info($"NonIncJoinStream.doWork _last_fire_time: {_last_fire_time} _is_dirty: {_is_dirty}");
+         if (DateTime.Now - _last_fire_time >= _throttle_period && _is_dirty) {
            _is_dirty = false;
            _last_fire_time = DateTime.Now;
            _form_and_fire();
@@ -0,0 +1,68 @@
+namespace bts {
+   using BCG;
+   using BCG.Utilities.General;
+
+   using System;
+   using System.Diagnostics;
+
+   using ZstdSharp.Unsafe;
+
+   // functon that joins the 2 rows and returns the outputput row. The function
+   // can return a null which means the 2 rows dont need to be joined
+   using JoinFunc = System.Func<BCG.IBlueRow, BCG.IBlueRow, BCG.IBlueRow>;
+   public class ProximityJoinStream : ViewStream, IWorker {
+      private ViewStream _lvs, _rvs;
+      private JoinFunc _join_func;
+      private bool _dirty = false;
+      private Stopwatch _stopwatch = new Stopwatch();
+      private ViewStreamDatabase _vsd = ViewStreamDatabase.instance;
+      public ProximityJoinStream(ViewStream lvs, ViewStream rvs, JoinFunc join_func) {
+         _lvs = lvs;
+         _rvs = rvs;
+         _join_func = join_func;
+      }
+
+      protected override void myOnFirstSubscriber() {
+         _lvs.onChange += _handle_data;
+         _rvs.onChange += _handle_data;
+         _vsd.addWorker(this);
+      }
+
+      protected override void myOnLastSubscriber() {
+         _lvs.onChange -= _handle_data;
+         _rvs.onChange -= _handle_data;
+         _vsd.removeWorker(this);
+      }
+
+      private void _handle_data(object dp_obj) {
+         _dirty = true;
+         doWork();
+      }
+
+      public void doWork() {
+         Func<bool> all_data_revcd = () => {
+            return _lvs.rows != null && _rvs.rows != null;
+         };
+         _stopwatch.Start();
+         if (_dirty && all_data_revcd()) {
+            _dirty = false;
+            BlueTable rv = null;
+            foreach (IBlueRow lr in _lvs.rows) {
+               foreach (IBlueRow rr in _rvs.rows) {
+                  IBlueRow resr = _join_func(lr, rr);
+                  if (resr == null)
+                     continue;
+                  rv = rv ?? new BlueTable(resr.schema);
+                  rv.addRow(resr);
+               }
+            }
+            _stopwatch.Stop();
+            Log.Gen.info($"ProximityJoinStream left_count: {_lvs.rows.Count} right_count: {_rvs.rows.Count} result_count: {rv.Count} time: {_stopwatch.ElapsedMilliseconds} msecs");
+            var dp = BlueDataUtility.build_dp_inc(rv, myLastValue as DataPacket);
+            myFireEvent(dp);
+         }
+
+      }
+
+   }
+}
@@ -0,0 +1,47 @@
+namespace bts.SilverPine {
+   using BCG;
+
+   using System;
+   using System.CodeDom;
+   using System.Configuration;
+   using System.Reactive.Disposables;
+
+   public class TimeProximityJoinStream : ViewStream {
+      private int _id = 0;
+
+      public TimeProximityJoinStream(ViewStream ls, string l_time_col, ViewStream rs, string r_time_col,
+                                       string pre_join_period, string post_join_period) :
+            this(ls, l_time_col, rs, r_time_col, TimeBasedFilter.parse_duration(pre_join_period),
+            TimeBasedFilter.parse_duration(post_join_period)) { }
+
+
+      public TimeProximityJoinStream(ViewStream ls, string l_time_col, ViewStream rs, string r_time_col,
+                                       TimeSpan pre_join_period, TimeSpan post_join_period) {
+         Func<IBlueRow, IBlueRow, IBlueRow> create_row = (lr, rr) => {
+            var id = 0;
+            lock (this) {
+               id = _id++;
+            }
+            IBlueRow rv = BlueDataUtility.create_composite_row(lr, rr, id);
+            return rv;
+         };
+         Func<IBlueRow, IBlueRow, IBlueRow> join_func = (lr, rr) => {
+            DateTime ldt = lr.get_datetime(l_time_col);
+            DateTime rdt = rr.get_datetime(r_time_col);
+            TimeSpan diff = ldt - rdt;
+            if (ldt < rdt) {
+               if (diff.Duration() > pre_join_period) {
+                  return null;
+               }
+            } else {
+               if (diff > post_join_period)
+                  return null;
+            }
+            return create_row(lr, rr);
+         };
+
+         ViewStream rstr = new ProximityJoinStream(ls, rs, join_func);
+         mySetRealSource(rstr);
+      }
+   }
+}
@@ -2,9 +2,13 @@ using System;
 using System.Collections;
 using System.Collections.Generic;

+using Amazon.Runtime.Internal.Util;
+
 using BCG;
 using BCG.Utilities;

+using bts;
+
 namespace BCG {
   public class LiteralStream : ViewStream {

@@ -22,6 +26,34 @@ namespace BCG {
         return rv;
      }

+      //literal("table1", "x:123, y:1.001, z:abc;x:123, y:1.001, z:abc")
+      public LiteralStream(string table_name, string rows) {
+         Func<string, int, IBlueRow> get_row = (json_row, id) => {
+            Dictionary<string, object> map = new Dictionary<string, object>();
+            string[] toks = json_row.Split(',', StringSplitOptions.TrimEntries);
+            //Logger.assert(col_vals.Length == 2);
+            foreach (string t in toks) {
+               var col_vals = t.Split(':', StringSplitOptions.TrimEntries);
+               string col = col_vals[0];
+               string val = col_vals[1];
+               map.Add(col, val);
+            }
+            map["id"] = id;
+            return new BlueRow(map);
+         };
+         string[] row_arr = rows.Split(';', StringSplitOptions.TrimEntries);
+         BlueTable table = null;
+         foreach (string json_string in row_arr) {
+            IBlueRow r = //BlueDataUtility.convert_json_to_row(json_string, table == null ? 0 : table.Count);
+            get_row(json_string, table == null ? 0 : table.Count);
+            table = table ?? new BlueTable(r.schema);
+            table.addRow(r);
+         }
+         ViewStream str = new BlueTableStream(table);
+         str = new SchemaGuesserStream(str);
+         mySetRealSource(str);
+      }
+
      public LiteralStream(string tableName, string schema, string value) {
         string[] columns = schema.Split(' ');

@@ -55,7 +87,7 @@ namespace BCG {
         mySetRealSource(new BlueRowCollectionStaticStream(t));
      }

-      public LiteralStream(string schema, string value) : this("literal", schema, value) { }
+      // public LiteralStream(string schema, string value) : this("literal", schema, value) { }

      // public LiteralStream(string value) {

@@ -0,0 +1,74 @@
+using BCG;
+
+using MongoDB.Driver.Core.WireProtocol.Messages;
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+
+// the first arg is all the rows, the second arg is the index of the row that needs mapping, and 
+// the func returns the mapped row (ie mapped version of the index row)
+using GlobalMap = System.Func<System.Collections.Generic.List<BCG.IBlueRow>, uint, BCG.IBlueRow>;
+
+namespace bts {
+   // In GlobalMap, each row is mapped with the full view of all rows, ie the map functions for a row
+   // can use the state fo all rows.
+   public class GlobalMapStream : ViewStream {
+      private ViewStream _base_stream;
+      private readonly string _sort_col_name, _group_by_col;
+      private GlobalMap _map_func;
+      public GlobalMapStream(ViewStream base_stream, string sort_col_name, string group_by_col, GlobalMap map_func) {
+         _base_stream = base_stream;
+         _sort_col_name = sort_col_name;
+         _group_by_col = group_by_col;
+         _map_func = map_func;
+      }
+
+      public GlobalMapStream(ViewStream base_stream, string sort_col_name, GlobalMap map_func) :
+      this(base_stream, sort_col_name, null, map_func) {
+      }
+
+      protected override void myOnFirstSubscriber() {
+         _base_stream.onChange += _handle_stream_update;
+      }
+
+      protected override void myOnLastSubscriber() {
+         _base_stream.onChange -= _handle_stream_update;
+      }
+
+      void _handle_stream_update(object dp_obj) {
+         DataPacket dp = dp_obj as DataPacket;
+         IDictionary<int, List<IBlueRow>> groups = !String.IsNullOrEmpty(_group_by_col) ?
+         BlueDataUtility.group_together(dp.newRows, new string[] { _group_by_col })
+         :
+         new Dictionary<int, List<IBlueRow>>() { { 0, dp.newRows.to_list() } };
+         BlueTable out_table = null;
+         foreach (var kv in groups) {
+            List<IBlueRow> group_rows = kv.Value;
+            group_rows.Sort((r1, r2) => BlueDataUtility.compare_rows(r2, r1, _sort_col_name));
+            uint i = 0;
+            foreach (IBlueRow r in group_rows) {
+               IBlueRow nr = _map_func(group_rows, i++);
+               out_table = out_table ?? new BlueTable(nr.schema);
+               out_table.addRow(nr);
+            }
+         }
+
+
+         // List<IBlueRow> rows = dp.newRows.to_ienum().ToList();
+         // rows.Sort((r1, r2) => BlueDataUtility.compare_rows(r1, r2, _sort_col_name));
+         // uint i = 0;
+
+         // BlueTable out_table = null;
+         // foreach (IBlueRow r in rows) {
+         //    IBlueRow nr = _map_func(rows, i++);
+         //    out_table = out_table ?? new BlueTable(nr.schema);
+         //    out_table.addRow(nr);
+         // }
+         DataPacket dp_out = BlueDataUtility.build_dp_inc(out_table, myLastValue as DataPacket);
+         myFireEvent(dp_out);
+      }
+   }
+
+
+}
@@ -13,9 +13,8 @@ namespace BCG {
   /// desired row ie newRow = f(row), where f is the mapping fucntion. Note that the id 
   /// is preserved across the rows, ie row["id"] = f(row)["id"]
   /// </summary>
-   [DataStreamUserRespresentation (false)]
-   public class MapStream : MethodDrivenStream
-   {
+   [DataStreamUserRespresentation(false)]
+   public class MapStream : MethodDrivenStream {
      protected IStreamMapper myMapper;
      protected IChangeableStreamMapper myChangeableMapper = null;
      private bool myUseNewMap = Configuration.get<DataStreamConfig>().UseNewMap;
@@ -0,0 +1,14 @@
+namespace bts.SilverPine
+{
+   using BCG;
+
+   [DataStreamUserRespresentation("RollingComputationStream")]
+   public class RollingComputationStream : ViewStream
+   {
+      private ViewStream _parent_str;
+      public RollingComputationStream(ViewStream parent_str, string operation_txt, string roll_column, uint window_sz)
+      {
+
+      }
+   }
+}
@@ -0,0 +1,10 @@
+namespace bts.SilverPine {
+   using BCG;
+
+
+   [DataStreamUserRespresentation("RollingComputationStream")]
+   public class RollingComputationStream : ViewStream {
+      public RollingComputationStream(ViewStream parent_str, string operation_txt, string roll_column, uint window_sz) {
+      }
+   }
+}
@@ -8,26 +8,33 @@ using System;
 using System.IO;
 using System.Linq;

-namespace bts {
-    public class PubSubTestingStream : ViewStream, IWorker {
-        public class PayloadMessage {
+namespace bts
+{
+    public class PubSubTestingStream : ViewStream, IWorker
+    {
+        public class PayloadMessage
+        {
            public int msg_id = 0;
            public string guid;
            public Int64 travel_epoch_ms = 0;
            public byte[] payload = null;
-            public PayloadMessage(int msg_id_args, Int64 send_epoch_args, int payload_size, byte payload_byte) {
+            public PayloadMessage(int msg_id_args, Int64 send_epoch_args, int payload_size, byte payload_byte)
+            {
                msg_id = msg_id_args;
                travel_epoch_ms = send_epoch_args;
                payload = new byte[payload_size];
-                for (int i = 0; i < payload_size; i++) {
+                for (int i = 0; i < payload_size; i++)
+                {
                    payload[i] = payload_byte;
                }
            }
            private PayloadMessage() { }
            public PayloadMessage(int msg_id, int payload_size, byte payload_byte) :
-            this(msg_id, PubSubTestingStream.epoch_now(), payload_size, payload_byte) { }
+            this(msg_id, PubSubTestingStream.epoch_now(), payload_size, payload_byte)
+            { }

-            public byte[] serialize() {
+            public byte[] serialize()
+            {
                var str = new MemoryStream();
                BinaryWriter bw = new BinaryWriter(str);
                bw.Write(msg_id);
@@ -40,7 +47,8 @@ namespace bts {
                return bytes;
            }

-            public static PayloadMessage deserialize(byte[] ser_bytes) {
+            public static PayloadMessage deserialize(byte[] ser_bytes)
+            {
                var str = new MemoryStream(ser_bytes);
                BinaryReader br = new BinaryReader(str);
                int msg_id = br.ReadInt32();
@@ -48,7 +56,8 @@ namespace bts {
                Int64 epoch = br.ReadInt64();
                int len = br.ReadInt32();
                var payload = br.ReadBytes(len);
-                return new PayloadMessage() {
+                return new PayloadMessage()
+                {
                    msg_id = msg_id,
                    guid = guid,
                    travel_epoch_ms = epoch,
@@ -73,7 +82,8 @@ namespace bts {
        private readonly int _payload_sz;
        private const byte _payload_byte = (byte)'A';
        private readonly string _tag_name;
-        public PubSubTestingStream(int publish_period_msecs, int max_pubs, int payload_size, string tag_name, ViewStreamDatabase vsd) {
+        public PubSubTestingStream(int publish_period_msecs, int max_pubs, int payload_size, string tag_name, ViewStreamDatabase vsd)
+        {
            _vsd = vsd;
            _payload_sz = payload_size;
            var schema = BlueDataUtility.form_schema("PubSubTest",
@@ -96,43 +106,53 @@ namespace bts {
        public PubSubTestingStream(int publish_period_seconds, int max_pubs, int msg_size, string tag_name)
        : this(publish_period_seconds, max_pubs, msg_size, tag_name, ViewStreamDatabase.instance) { }

-        protected override void myOnFirstSubscriber() {
+        protected override void myOnFirstSubscriber()
+        {
            _vsd.addWorker(this);
            System.Threading.Thread th = new System.Threading.Thread(_subscribe_thread_func);
            th.IsBackground = true;
            th.Start();
        }

-        protected override void myOnLastSubscriber() {
+        protected override void myOnLastSubscriber()
+        {
            _vsd.removeWorker(this);
            _continue_sub_thread = false;
        }

-        void _log_updates(IEnumerable<PubSubServerMessage> updates) {
+        void _log_updates(IEnumerable<PubSubServerMessage> updates)
+        {
            var idx = 0;
-            foreach (var msg in updates) {
+            foreach (var msg in updates)
+            {
                bts.Log.Gen.info($"PubSubTesStream._subscribe_thread_func idx: {idx} msg: {msg}");
                idx++;
            }
        }

-        private void _subscribe_thread_func() {
+        private void _subscribe_thread_func()
+        {
            // subscription is done in another thread
            SubscriberCore sub_core = new SubscriberCore();
            sub_core.subscribe(_tag_name);
-            while (_continue_sub_thread) {
+            while (_continue_sub_thread)
+            {
                // TODO recv_id
                var updates = sub_core.get_updates(TimeSpan.FromMilliseconds(250));
                if (updates == null || updates.Count() == 0)
                    continue;
                _log_updates(updates);
                _n_updates += updates.Count();
-                if (updates.Count() > 0) {
+                if (updates.Count() > 0)
+                {
                    var server_msg = updates.Last();
-                    if (server_msg.data != null && server_msg.data.Length > 0) {
+                    if (server_msg.data != null && server_msg.data.Length > 0)
+                    {
                        _last_recvd_msg = PayloadMessage.deserialize(server_msg.data);
                        _last_recvd_msg.travel_epoch_ms = epoch_now();
-                    } else {
+                    }
+                    else
+                    {
                        // why are we getting empty or null data ? (server is sending data regardless?)
                    }
                    _fire_dirty = true;
@@ -140,29 +160,36 @@ namespace bts {
            }
        }

-        private void _publish_next_packet() {
-            if (_last_sent_msg == null) {
-                _last_sent_msg = new PayloadMessage(0, _payload_sz, _payload_byte) {
+        private void _publish_next_packet()
+        {
+            if (_last_sent_msg == null)
+            {
+                _last_sent_msg = new PayloadMessage(0, _payload_sz, _payload_byte)
+                {
                    guid = Guid.NewGuid().ToString()
                };
-            } else {
+            }
+            else
+            {
                _last_sent_msg.msg_id++;
                _last_sent_msg.guid = Guid.NewGuid().ToString();
                _last_sent_msg.travel_epoch_ms = epoch_now();
            }
            var ser_bytes = _last_sent_msg.serialize();
            _n_pubs++;
-            _pub_core.publish(String.Empty, _tag_name, ser_bytes);
+            _pub_core.publish("Blob", _tag_name, ser_bytes);
            _fire_dirty = true;
        }

-        private void _fire_dp() {
+        private void _fire_dp()
+        {
            object last_sent_id = _last_sent_msg == null ? (object)DBNull.Value : (object)_last_sent_msg.msg_id;
            object last_recv_id = _last_recvd_msg == null ? (object)DBNull.Value : (object)_last_recvd_msg.msg_id;
            object echo_time = DBNull.Value;
            object last_sent_guid = _last_sent_msg == null ? (object)DBNull.Value : (object)_last_sent_msg.guid;
            object last_recv_guid = _last_recvd_msg == null ? (object)DBNull.Value : (object)_last_recvd_msg.guid;
-            if (_last_sent_msg != null && _last_recvd_msg != null) {
+            if (_last_sent_msg != null && _last_recvd_msg != null)
+            {
                echo_time = _last_recvd_msg.travel_epoch_ms - _last_sent_msg.travel_epoch_ms;
            }
            var match = last_sent_id.Equals(last_recv_id) && last_sent_guid.Equals(last_recv_guid);
@@ -174,21 +201,25 @@ namespace bts {
            myFireEvent(dp);
        }

-        public void doWork() {
+        public void doWork()
+        {
            if (DateTime.Now - _last_publish_time > _publish_period)
-                if (_n_pubs <= _max_pubs) {
+                if (_n_pubs <= _max_pubs)
+                {
                    _publish_next_packet();
                    _last_publish_time = DateTime.Now;
                }
            // publish 
            // TODO: send id
-            if (_fire_dirty) {
+            if (_fire_dirty)
+            {
                _fire_dirty = false;
                _fire_dp();
            }
        }

-        public static long epoch_now() {
+        public static long epoch_now()
+        {
            return DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
        }

@@ -9,17 +9,22 @@ using System.Linq;
 using System.Runtime.CompilerServices;
 using bts.api;

-namespace bts {
+namespace bts
+{
   [DataStreamUserRespresentation("pss_echo")]
-   public class PubsubEchoTestStream : ViewStream, IWorker {
-      public class PayloadMessage {
+   public class PubsubEchoTestStream : ViewStream, IWorker
+   {
+      public class PayloadMessage
+      {
         public readonly string msg_id;
         public readonly Int64 timestamp = 0;
         public readonly byte[] payload = null;

-         public static byte[] create_data(int payload_size, byte payload_byte) {
+         public static byte[] create_data(int payload_size, byte payload_byte)
+         {
            var rv = new byte[payload_size];
-            for (int i = 0; i < payload_size; i++) {
+            for (int i = 0; i < payload_size; i++)
+            {
               rv[i] = payload_byte;
            }
            return rv;
@@ -35,14 +40,16 @@ namespace bts {
         //       payload[i] = payload_byte;
         //    }
         // }
-         public PayloadMessage(string guid_, Int64 timestamp_, byte[] payload_) {
+         public PayloadMessage(string guid_, Int64 timestamp_, byte[] payload_)
+         {
            msg_id = guid_;
            timestamp = timestamp_;
            payload = payload_;
         }

         private PayloadMessage() { }
-         public byte[] serialize() {
+         public byte[] serialize()
+         {
            var str = new MemoryStream();
            BinaryWriter bw = new BinaryWriter(str);
            bw.Write(msg_id);
@@ -54,7 +61,8 @@ namespace bts {
            return bytes;
         }

-         public static PayloadMessage deserialize(byte[] ser_bytes) {
+         public static PayloadMessage deserialize(byte[] ser_bytes)
+         {
            var str = new MemoryStream(ser_bytes);
            BinaryReader br = new BinaryReader(str);
            string guid = br.ReadString();
@@ -80,13 +88,15 @@ namespace bts {
      private readonly string _tag_name;
      private readonly int _payload_sz;
      private readonly IPubSubRouter _router;
-      private enum IOState {
+      private enum IOState
+      {
         SEND = 0,
         RECV = 1
      };
      IOState _state = IOState.SEND;

-      public PubsubEchoTestStream(string server_url, TimeSpan publish_period, int payload_size, string tag_name, ViewStreamDatabase vsd) {
+      public PubsubEchoTestStream(string server_url, TimeSpan publish_period, int payload_size, string tag_name, ViewStreamDatabase vsd)
+      {
         var route = new PubSubRoute(server_url, ".*");
         _router = new PubSubRouter(route);
         _vsd = vsd;
@@ -107,7 +117,8 @@ namespace bts {
      public PubsubEchoTestStream(string server_url, int publish_period_msecs, int payload_size, string tag_name)
      : this(server_url, TimeSpan.FromMilliseconds(publish_period_msecs), payload_size, tag_name, ViewStreamDatabase.instance) { }

-      protected override void myOnFirstSubscriber() {
+      protected override void myOnFirstSubscriber()
+      {
         _pub_core = new PublisherCore(_router);
         _sub_core = new SubscriberCore(_router);
         _sub_core.subscribe(_tag_name);
@@ -115,12 +126,14 @@ namespace bts {

      }

-      protected override void myOnLastSubscriber() {
+      protected override void myOnLastSubscriber()
+      {
         _vsd.removeWorker(this);
         _sub_core.unsubscribe(_tag_name);
      }

-      private void _fire_dp_after_send() {
+      private void _fire_dp_after_send()
+      {
         // as soon as the data is published, its reflected in the stream state
         Log.assert(_last_sent_msg != null);
         BlueRow r = new BlueRow(_table.schema, _table.Count, _last_sent_msg.msg_id, _last_sent_msg.timestamp, DBNull.Value, DBNull.Value,
@@ -130,7 +143,8 @@ namespace bts {
         myFireEvent(dp_out);
      }

-      private void _fire_dp_after_recv() {
+      private void _fire_dp_after_recv()
+      {
         Log.assert(_last_recvd_msg != null);
         // update the last publish row
         int id = _table.Count - 1;
@@ -144,11 +158,13 @@ namespace bts {
         myFireEvent(dp_out);
      }

-      private string _get_next_id() {
+      private string _get_next_id()
+      {
         return _table.Count.ToString();
      }

-      private void _publish_next_packet() {
+      private void _publish_next_packet()
+      {
         var bytes = PayloadMessage.create_data(_payload_sz, _payload_byte);
         _last_sent_msg = new PayloadMessage(_get_next_id(), epoch_now(), bytes);
         var ser_bytes = _last_sent_msg.serialize();
@@ -157,19 +173,25 @@ namespace bts {
         _last_publish_time = DateTime.Now;
      }

-      public void doWork() {
-         Func<IEnumerable<PubSubServerMessage>, PayloadMessage> find_packet = (updates) => {
-            foreach (var u in updates) {
+      public void doWork()
+      {
+         Func<IEnumerable<PubSubServerMessage>, PayloadMessage> find_packet = (updates) =>
+         {
+            foreach (var u in updates)
+            {
               var recv_msg = PayloadMessage.deserialize(u.data);
-               if (recv_msg.msg_id == _last_sent_msg.msg_id) {
+               if (recv_msg.msg_id == _last_sent_msg.msg_id)
+               {
                  return recv_msg;
               }
            }
            return null;
         };
-         switch (_state) {
+         switch (_state)
+         {
            case IOState.SEND:
-               if (DateTime.Now - _last_publish_time >= _publish_period) {
+               if (DateTime.Now - _last_publish_time >= _publish_period)
+               {
                  _publish_next_packet();
                  _last_publish_time = DateTime.Now;
                  _fire_dp_after_send();
@@ -181,7 +203,8 @@ namespace bts {
               var updates = _sub_core.get_updates(TimeSpan.Zero);
               updates = updates.Where(u => u.tag == _tag_name);
               var recv_msg = find_packet(updates);
-               if (recv_msg != null) {
+               if (recv_msg != null)
+               {
                  _last_recvd_msg = recv_msg;
                  _fire_dp_after_recv();
                  _state = IOState.SEND;
@@ -191,7 +214,8 @@ namespace bts {
         }
      }

-      public static long epoch_now() {
+      public static long epoch_now()
+      {
         return DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
      }

@@ -1,9 +1,8 @@
-using Google.Protobuf.WellKnownTypes;
-
 using System;
-using BCG.Utilities.General;

-namespace BCG {
+using BCG;
+
+namespace bts.SilverPine {
   // Keeps adding changing rows to the stream, hence preserving the history
   // and creating a time series, for example this can be used to make bbo updates
   // into a time series of prices/qty 
@@ -41,7 +40,6 @@ namespace BCG {
         }
         _table.addRows(added_rows);
         myFireEvent(new DataPacket(_table, added_rows, new BlueTable(_table.schema)));
-
      }
   }

@@ -7,122 +7,127 @@ using MongoDB.Driver;
 using System;
 using System.Collections.Generic;
 using System.Diagnostics;
+using System.Threading.Tasks;
+using System.Threading;

 namespace bts {
-    [DataStreamUserRespresentation("mongodb_sql")]
-    public class MongoDBSQLStream : ViewStream, IWorker {
-        private ViewStreamDatabase _vsd;
-        private readonly string _connection_string, _db_name, _collection_name;
-        private MongoClient _client;
-        // make this a sorted table
-        private BlueTable _table;
+    // [DataStreamUserRespresentation("mongodb_sql")]
+    // public class MongoDBSQLStream : ViewStream, IWorker {
+    //     private ViewStreamDatabase _vsd;
+    //     private readonly string _connection_string, _db_name, _collection_name;
+    //     private MongoClient _client;
+    //     // make this a sorted table
+    //     private BlueTable _table;


-        // Access the database
-        private IMongoDatabase _database;
-        private IDictionary<string, int> _id_map = new Dictionary<string, int>();
-        // mongodb://192.168.1.80:27017
+    //     // Access the database
+    //     private IMongoDatabase _database;
+    //     private IDictionary<string, int> _id_map = new Dictionary<string, int>();
+    //     // mongodb://192.168.1.80:27017

-        public MongoDBSQLStream(string connection_string, string db_name, string collection_name)
-        : this(connection_string, db_name, collection_name, ViewStreamDatabase.instance) {
+    //     public MongoDBSQLStream(string connection_string, string db_name, string collection_name)
+    //     : this(connection_string, db_name, collection_name, ViewStreamDatabase.instance) {

-        }
-        public MongoDBSQLStream(string connection_string, string db_name, string collection_name, ViewStreamDatabase vsd) {
-            bts.Log.Gen.info($"MongoDBStream constructed connection_string: {connection_string} " +
-                                    $"db_name: {db_name}");
-            _connection_string = connection_string;
-            _db_name = db_name;
-            _collection_name = collection_name;
-            _vsd = vsd;
-            _client = new MongoClient(_connection_string);
-            _database = _client.GetDatabase(_db_name);
-        }
+    //     }
+    //     public MongoDBSQLStream(string connection_string, string db_name, string collection_name, ViewStreamDatabase vsd) {
+    //         bts.Log.Gen.info($"MongoDBStream constructed connection_string: {connection_string} " +
+    //                                 $"db_name: {db_name}");
+    //         _connection_string = connection_string;
+    //         _db_name = db_name;
+    //         _collection_name = collection_name;
+    //         _vsd = vsd;
+    //         _client = new MongoClient(_connection_string);
+    //         _database = _client.GetDatabase(_db_name);
+    //     }

-        protected override void myOnFirstSubscriber() {
-            MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(10));
-        }
+    //     protected override void myOnFirstSubscriber() {
+    //         MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(10));
+    //     }

-        protected override void myOnLastSubscriber() {
-            MasterWorker.getInstance(false).removeWorker(this);
-        }
+    //     protected override void myOnLastSubscriber() {
+    //         MasterWorker.getInstance(false).removeWorker(this);
+    //     }

-        public void doWork() {
-            _read_mogodb();
-        }
+    //     public void doWork() {
+    //         _read_mogodb();
+    //     }

-        private void _read_mogodb() {
-            var timer = Stopwatch.StartNew();
-            // Access the collection
-            IMongoCollection<BsonDocument> collection = _database.GetCollection<BsonDocument>(name: _collection_name);
+    //     private void _read_mogodb() {
+    //         var timer = Stopwatch.StartNew();
+    //         // Access the collection
+    //         IMongoCollection<BsonDocument> collection = _database.GetCollection<BsonDocument>(name: _collection_name);

-            // Fetch all documents from the collection
-            var documents = collection.Find(Builders<BsonDocument>.Filter.Empty).ToCursor();
-            _table = _table == null ? null : new BlueTable(_table.schema);
-            // Iterate through the documents and print values
-            foreach (var document in documents.ToEnumerable()) {
-                bts.Log.Gen.info($"document: {document} json: {document.ToJson()}");
-                if (_table == null) {
-                    _init_table(document);
-                }
-                _add_row(document);
-            }
-            timer.Stop();
-            Log.Gen.info($"MongoDBStream time to read {_collection_name} {timer.ElapsedMilliseconds}ms");
-            timer.Restart();
-            var dp = BlueDataUtility.build_dp_inc(_table, myLastValue == null ? null : myLastValue as DataPacket, true);
-            timer.Stop();
-            Log.Gen.info($"MongoDBStream time to compare tables :{timer.ElapsedMilliseconds}ms");
-            if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
-                return;
-            myFireEvent(dp);
-        }
+    //         // Fetch all documents from the collection
+    //         var documents = collection.Find(Builders<BsonDocument>.Filter.Empty).ToCursor();
+    //         _table = _table == null ? null : new BlueTable(_table.schema);
+    //         // Iterate through the documents and print values
+    //         foreach (var document in documents.ToEnumerable()) {
+    //             bts.Log.Gen.info($"document: {document} json: {document.ToJson()}");
+    //             if (_table == null) {
+    //                 _init_table(document);
+    //             }
+    //             _add_row(document);
+    //         }
+    //         timer.Stop();
+    //         Log.Gen.info($"MongoDBStream time to read {_collection_name} {timer.ElapsedMilliseconds}ms");
+    //         timer.Restart();
+    //         var dp = BlueDataUtility.build_dp_inc(_table, myLastValue == null ? null : myLastValue as DataPacket, true);
+    //         timer.Stop();
+    //         Log.Gen.info($"MongoDBStream time to compare tables :{timer.ElapsedMilliseconds}ms");
+    //         if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
+    //             return;
+    //         myFireEvent(dp);
+    //     }

-        private void _init_table(BsonDocument doc_row) {
-            bts.Log.assert(_table == null);
-            List<ColumnInfo> cis = new List<ColumnInfo>();
-            cis.Add((new ColumnInfo("id", 0, typeof(Int32))));
-            foreach (var n in doc_row.Names) {
-                cis.Add(new ColumnInfo(n, cis.Count, typeof(string)));
-            }
-            IBlueTableSchema schema = new BlueTableSchema(cis.ToArray(), _collection_name);
-            _table = new BlueTable(schema);
-        }
+    //     private void _init_table(BsonDocument doc_row) {
+    //         bts.Log.assert(_table == null);
+    //         List<ColumnInfo> cis = new List<ColumnInfo>();
+    //         cis.Add((new ColumnInfo("id", 0, typeof(Int32))));
+    //         foreach (var n in doc_row.Names) {
+    //             cis.Add(new ColumnInfo(n, cis.Count, typeof(string)));
+    //         }
+    //         IBlueTableSchema schema = new BlueTableSchema(cis.ToArray(), _collection_name);
+    //         _table = new BlueTable(schema);
+    //     }

-        private void _add_row(BsonDocument doc) {
-            var key_val = doc["_id"];
-            var key = key_val.ToString();
-            int id;
-            if (!_id_map.TryGetValue(key, out id)) {
-                id = _id_map.Count;
-                _id_map.Add(key, id);
-            }
-            MutableBlueRow nr = new MutableBlueRow(_table.schema);
-            nr.setValue("id", id);
-            foreach (var n in doc.Names) {
-                var val = doc[n].ToString();
-                nr.setValue(n, val);
-            }
-            _table.addRow(nr);
-        }
-    }
+    //     private void _add_row(BsonDocument doc) {
+    //         var key_val = doc["_id"];
+    //         var key = key_val.ToString();
+    //         int id;
+    //         if (!_id_map.TryGetValue(key, out id)) {
+    //             id = _id_map.Count;
+    //             _id_map.Add(key, id);
+    //         }
+    //         MutableBlueRow nr = new MutableBlueRow(_table.schema);
+    //         nr.setValue("id", id);
+    //         foreach (var n in doc.Names) {
+    //             var val = doc[n].ToString();
+    //             nr.setValue(n, val);
+    //         }
+    //         _table.addRow(nr);
+    //     }
+    // }

    [DataStreamUserRespresentation("mongodb")]
    public class MongoDBStream : ViewStream, IWorker {
        private ViewStreamDatabase _vsd;
        private readonly string _connection_string, _db_name, _collection_name;
+        private readonly TimeSpan _poll_period;
        private MongoClient _client;
        private BlueTable _table;
+        private SyncQueue<DataPacket> _table_queue = new SyncQueue<DataPacket>();
+        private BlockingQueue _work_queue = new BlockingQueue();

        // Access the database
        private IMongoDatabase _database;
        private IDictionary<string, int> _id_map = new Dictionary<string, int>();
        // mongodb://192.168.1.80:27017

-        public MongoDBStream(string connection_string, string db_name, string collection_name)
-        : this(connection_string, db_name, collection_name, ViewStreamDatabase.instance) {
+        public MongoDBStream(string connection_string, string db_name, string collection_name, string poll_period)
+        : this(connection_string, db_name, collection_name, TimeBasedFilter.parse_duration(poll_period), ViewStreamDatabase.instance) {

        }
-        public MongoDBStream(string connection_string, string db_name, string collection_name, ViewStreamDatabase vsd) {
+        public MongoDBStream(string connection_string, string db_name, string collection_name, TimeSpan poll_period, ViewStreamDatabase vsd) {
            bts.Log.Gen.info($"MongoDBStream constructed connection_string: {connection_string} " +
                                    $"db_name: {db_name}");
            _connection_string = connection_string;
@@ -130,19 +135,30 @@ namespace bts {
            _collection_name = collection_name;
            _vsd = vsd;
            _client = new MongoClient(_connection_string);
+            _poll_period = poll_period;
            _database = _client.GetDatabase(_db_name);
        }

        protected override void myOnFirstSubscriber() {
-            MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(10));
+            _vsd.addWorker(this);
+            //MasterWorker.getInstance(false).addWorker(this, _poll_period);
        }

        protected override void myOnLastSubscriber() {
-            MasterWorker.getInstance(false).removeWorker(this);
+            _vsd.removeWorker(this);
+            //MasterWorker.getInstance(false).removeWorker(this);
        }

+        private DateTime _last_poll_time = DateTime.MinValue;
        public void doWork() {
-            _read_mogodb();
+            var now = DateTime.Now;
+            if (now - _last_poll_time > _poll_period) {
+                ThreadPool.QueueUserWorkItem(_ => _read_mogodb());
+                _last_poll_time = DateTime.Now;
+            }
+            var dp = _table_queue.dequeue(0);
+            if (dp != null)
+                myFireEvent(dp);
        }

        private void _read_mogodb() {
@@ -166,10 +182,11 @@ namespace bts {
            timer.Restart();
            var dp = BlueDataUtility.build_dp_inc(_table, myLastValue == null ? null : myLastValue as DataPacket, true);
            timer.Stop();
-            Log.Gen.info($"MongoDBStream time to compare tables: {timer.ElapsedMilliseconds}ms");;
+            Log.Gen.info($"MongoDBStream time to compare tables: {timer.ElapsedMilliseconds}ms"); ;
            if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
                return;
-            myFireEvent(dp);
+            _table_queue.enqueue(dp);
+            //myFireEvent(dp);
        }

        private void _init_table(BsonDocument doc_row) {
@@ -208,46 +225,46 @@ namespace bts {
        }
    }

-    [DataStreamUserRespresentation("mongodb_flat")]
-    public class MongoDBFlatStream : ViewStream {
-        public MongoDBFlatStream(string connection_string, string db_name, string collection_name, string timestamp_col, string ttl_duration)
-        :
-        this(ViewStreamDatabase.instance, connection_string, db_name, collection_name, timestamp_col, TimeBasedFilter.parse_duration(ttl_duration)) { }
+    // [DataStreamUserRespresentation("mongodb_flat")]
+    // public class MongoDBFlatStream : ViewStream {
+    //     public MongoDBFlatStream(string connection_string, string db_name, string collection_name, string timestamp_col, string ttl_duration)
+    //     :
+    //     this(ViewStreamDatabase.instance, connection_string, db_name, collection_name, timestamp_col, TimeBasedFilter.parse_duration(ttl_duration)) { }

-        public MongoDBFlatStream(ViewStreamDatabase vsd, string connection_string, string db_name, string collection_name, string timestamp_col, TimeSpan ttl_duration) {
-            ViewStream rs = new MongoDBStream(connection_string, db_name, collection_name, vsd);
-            rs = new KeyValueToRelationalStream(rs, "doc_id", "key", "val");
-            rs = new DateStandardizationStream(rs, timestamp_col, DateConversion.ISO8601);
-            //rs = new TimeBasedFilterStream(vsd, rs, timestamp_col, ttl_duration);
-            mySetRealSource(rs);
-        }
-    }
+    //     public MongoDBFlatStream(ViewStreamDatabase vsd, string connection_string, string db_name, string collection_name, string timestamp_col, TimeSpan ttl_duration) {
+    //         ViewStream rs = new MongoDBStream(connection_string, db_name, collection_name, vsd);
+    //         rs = new KeyValueToRelationalStream(rs, "doc_id", "key", "val");
+    //         rs = new DateStandardizationStream(rs, timestamp_col, DateConversion.ISO8601);
+    //         //rs = new TimeBasedFilterStream(vsd, rs, timestamp_col, ttl_duration);
+    //         mySetRealSource(rs);
+    //     }
+    // }



-    [DataStreamUserRespresentation("newsreplay")]
-    public class NewsReplayStream : ViewStream {
-        public NewsReplayStream(string connection_string, string db_name, string collection_name,
-                                int buffer_sz, string fire_period) :
-        this(ViewStreamDatabase.instance, connection_string, db_name, collection_name, buffer_sz, TimeBasedFilter.parse_duration(fire_period)) { }
+    // [DataStreamUserRespresentation("newsreplay")]
+    // public class NewsReplayStream : ViewStream {
+    //     public NewsReplayStream(string connection_string, string db_name, string collection_name,
+    //                             int buffer_sz, string fire_period) :
+    //     this(ViewStreamDatabase.instance, connection_string, db_name, collection_name, buffer_sz, TimeBasedFilter.parse_duration(fire_period)) { }

-        public NewsReplayStream(ViewStreamDatabase vsd, string connection_string, string db_name, string collection_name,
-                                int buffer_sz, TimeSpan fire_period) {
-            ViewStream r;
+    //     public NewsReplayStream(ViewStreamDatabase vsd, string connection_string, string db_name, string collection_name,
+    //                             int buffer_sz, TimeSpan fire_period) {
+    //         ViewStream r;

-            r = new MongoDBStream(connection_string, db_name, collection_name, vsd);
-            r = new KeyValueToRelationalStream(r,
-                                                "doc_id",
-                                                "key",
-                                                "value");
-            r = new DateStandardizationStream(r,
-                                               "created_at",
-                                               "ISO8601");
-            r = new SnapshotReplayStream(vsd, r, "created_at", buffer_sz, fire_period);
+    //         r = new MongoDBStream(connection_string, db_name, collection_name, vsd);
+    //         r = new KeyValueToRelationalStream(r,
+    //                                             "doc_id",
+    //                                             "key",
+    //                                             "value");
+    //         r = new DateStandardizationStream(r,
+    //                                            "created_at",
+    //                                            "ISO8601");
+    //         r = new SnapshotReplayStream(vsd, r, "created_at", buffer_sz, fire_period);

-            mySetRealSource(r);
-        }
-    }
+    //         mySetRealSource(r);
+    //     }
+    // }


 }
@@ -6,6 +6,7 @@ namespace bts {
   using System;
   using System.Collections;
   using System.Collections.Generic;
+   using System.Linq;

   public class ProjectData {
      // schema from the base stream
@@ -107,6 +108,7 @@ namespace bts {

   }

+   [DataStreamUserRespresentation("Project")]
   public class NewProjectStream : ViewStream, IStreamMapper {

      private ViewStream _stream;
@@ -136,10 +138,24 @@ namespace bts {
         HashSet<string> removed_cols = new HashSet<string>();
         Dictionary<string, string> mapped_to_orig_map = new Dictionary<string, string>();
         Dictionary<string, string> orig_to_mapped = new Dictionary<string, string>();
+         // by default every col will show up, so if we have the projection as (a:a1)
+         // it will convert a to a1, 
+         // examples: if a schema has cols a b c
+         // project(parent_str, "a:aa ") -> aa b c
+         // project(parent_str, "a:aa !c") -> aa b
+         // project(parent_str, "a:aa !") -> aa
+         // project(parent_str, "a:aa b = (aa*aa)!") -> aa b
+         bool hide_all = false;
         foreach (string p_tok in project_toks) {
            if (p_tok.StartsWith("!")) {
-               string removed_col = p_tok.Substring(1);
-               removed_cols.Add(removed_col);
+               if (p_tok.Length == 1) {
+                  // if it is a "blanket "!"=> hide all cols except the one that
+                  // are stated explicitly
+                  hide_all = true;
+               } else {
+                  string removed_col = p_tok.Substring(1);
+                  removed_cols.Add(removed_col);
+               }
            } else if (p_tok.Contains(":")) {
               string[] map_toks = p_tok.Split(":");
               string orig_col = map_toks[0];
@@ -147,12 +163,22 @@ namespace bts {
               mapped_to_orig_map.Add(mapped_col, orig_col);
               orig_to_mapped.Add(orig_col, mapped_col);
            } else {
-               Log.Gen.error($"NewProjectStream unexpecetd token: {p_tok}");
+               if (input_schema.columnNames.Contains(p_tok)) {
+                  string orig_col = p_tok;
+                  string mapped_col = orig_col;
+                  orig_to_mapped.Add(orig_col, mapped_col);
+               }
            }
         }
+
         // make the new schema
         List<ColumnInfo> new_cols = new List<ColumnInfo>();
         foreach (ColumnInfo ci in input_schema.columnInfo) {
+            if (ci.colName == "id") {
+               ColumnInfo nc = new ColumnInfo(ci.colName, new_cols.Count, ci.colType);
+               new_cols.Add(nc);
+               continue;
+            }
            if (removed_cols.Contains(ci.colName)) {
               continue;
            } else if (orig_to_mapped.ContainsKey(ci.colName)) {
@@ -160,8 +186,13 @@ namespace bts {
               ColumnInfo nc = new ColumnInfo(mapped_name, new_cols.Count, ci.colType);
               new_cols.Add(nc);
            } else {
+               if (hide_all)
+                  continue;
+               //if (ci.colName == "id" || removed_cols.Count > 0) {
+               // unless columns are  explicitely hidden, all columns show up
               ColumnInfo nc = new ColumnInfo(ci.colName, new_cols.Count, ci.colType);
               new_cols.Add(nc);
+               //}
            }
         }
         IBlueTableSchema new_schema = new BlueTableSchema(input_schema.tableName, new_cols.ToArray());
@@ -0,0 +1,122 @@
+using System;
+using System.Data;
+using System.Data.SQLite;
+using System.Diagnostics;
+using System.Collections;
+
+
+using BCG;
+using System.Linq;
+using System.Data.Entity;
+using System.Collections.Generic;
+using System.Data.SQLite;
+
+
+namespace bts.SilverPine {
+
+   public static class SQLiteHelper {
+      public static string location = "C:\\Users\\m_tah\\development\\data\\";
+      public static string db_name = "stream_data.db";
+
+      public static DataTable execute_select_query(string connection_string, string query) {
+         // Create a new DataTable to store the results
+         DataTable dataTable = new DataTable();
+
+         // Create and open a SQLite connection using the provided connection string
+         using (SQLiteConnection connection = new SQLiteConnection(connection_string)) {
+            try {
+               connection.Open();
+
+               // Create a SQLite command with the provided query
+               using (SQLiteCommand command = new SQLiteCommand(query, connection)) {
+                  // Use SQLiteDataAdapter to fill the DataTable
+                  using (SQLiteDataAdapter adapter = new SQLiteDataAdapter(command)) {
+                     adapter.Fill(dataTable);
+                  }
+               }
+            } catch (Exception ex) {
+               Console.WriteLine("An error occurred: " + ex.Message);
+            }
+         }
+         // Return the populated DataTable
+         return dataTable;
+      }
+
+      public static List<Dictionary<string, object>> ExecuteBulkRead(string connectionString, string query) {
+         var result = new List<Dictionary<string, object>>();
+
+         using (var connection = new SQLiteConnection(connectionString)) {
+            connection.Open();
+
+            // Begin a transaction to optimize read speed
+            using (var transaction = connection.BeginTransaction())
+            using (var command = new SQLiteCommand(query, connection, transaction))
+            using (var reader = command.ExecuteReader()) {
+               while (reader.Read()) {
+                  var row = new Dictionary<string, object>();
+
+                  for (int i = 0; i < reader.FieldCount; i++) {
+                     string columnName = reader.GetName(i);
+                     object columnValue = reader.IsDBNull(i) ? null : reader.GetValue(i);
+                     row[columnName] = columnValue;
+                  }
+
+                  result.Add(row);
+               }
+
+               // Commit the transaction
+               transaction.Commit();
+            }
+         }
+
+         return result;
+      }
+
+      public static List<Dictionary<string, object>> get_table(string connectionString, string query) {
+         var result = new List<Dictionary<string, object>>();
+
+         using (var connection = new SQLiteConnection(connectionString)) {
+            try {
+               connection.Open();
+
+               using (var command = new SQLiteCommand(query, connection))
+               using (var reader = command.ExecuteReader()) {
+                  while (reader.Read()) {
+                     var row = new Dictionary<string, object>();
+
+                     for (int i = 0; i < reader.FieldCount; i++) {
+                        string columnName = reader.GetName(i);
+                        object columnValue = reader.IsDBNull(i) ? null : reader.GetValue(i);
+                        row[columnName] = columnValue;
+                     }
+
+                     result.Add(row);
+                  }
+               }
+            } catch (Exception ex) {
+               Console.WriteLine("An error occurred: " + ex.Message);
+            }
+         }
+         return result;
+      }
+
+      public static IEnumerable<Dictionary<string, object>> get_table_2(string connectionString, string query) {
+         using (var connection = new SQLiteConnection(connectionString)) {
+            connection.Open();
+            using (var command = new SQLiteCommand(query, connection))
+            using (var reader = command.ExecuteReader()) {
+               while (reader.Read()) {
+                  var row = new Dictionary<string, object>();
+                  for (int i = 0; i < reader.FieldCount; i++) {
+                     string columnName = reader.GetName(i);
+                     object columnValue = reader.IsDBNull(i) ? null : reader.GetValue(i);
+                     row[columnName] = columnValue;
+                  }
+                  yield return row;
+               }
+            }
+         }
+      }
+   }
+}
+
@@ -0,0 +1,73 @@
+namespace bts.SilverPine {
+   using System;
+   using System.Data;
+   using System.Data.SQLite;
+   using System.Diagnostics;
+   using System.Collections;
+
+
+   using BCG;
+
+   [DataStreamUserRespresentation("SQLite")]
+   public class SQLiteStream : ViewStream {
+      private readonly string _connection_string, _query;
+      private Stopwatch _stopwatch = new Stopwatch();
+
+      private SQLiteStream(string connection_string, string query, bool noop) {
+         _connection_string = connection_string;
+         _query = query;
+      }
+
+      public SQLiteStream(string db_name, string query)
+      :
+      this($"Data Source={SQLiteHelper.location}{db_name};Version=3", query, true) { }
+
+      public SQLiteStream(string query)
+      :
+      this(SQLiteHelper.db_name, query) {
+      }
+
+
+      private void _read_from_sqlite_and_fire() {
+         _stopwatch.Start();
+         DataTable dt = SQLiteHelper.execute_select_query(_connection_string, _query);
+         _stopwatch.Stop();
+         Log.Gen.info($"SQLiteStream.ExecuteSelectQuery time to read db: {_stopwatch.ElapsedMilliseconds} msecs");
+         _stopwatch.Restart();
+         #region perf measures
+         // SQLiteHelper.get_table(_connection_string, _query);
+         // _stopwatch.Stop();
+         // Log.Gen.info($"SQLiteStream.get_table time to read db: {_stopwatch.ElapsedMilliseconds} msecs");
+         // _stopwatch.Restart();
+         // SQLiteHelper.ExecuteBulkRead(_connection_string, _query);
+         // _stopwatch.Stop();
+         // Log.Gen.info($"SQLiteStream.ExecuteBulkRead time to read db: {_stopwatch.ElapsedMilliseconds} msecs");
+         // _stopwatch.Restart();
+         // SQLiteHelper.get_table_2(_connection_string, _query);
+         // _stopwatch.Stop();
+         // Log.Gen.info($"SQLiteStream.get_table_2 time to read db: {_stopwatch.ElapsedMilliseconds} msecs");
+         // _stopwatch.Restart();
+         #endregion
+         IBlueRowCollection rows = BCG.BlueDataUtility.toIBlueRowCollection(dt);
+         //IBlueRowCollection rows = new DataTableWrapperTable(dt);
+         _stopwatch.Stop();
+         Log.Gen.info($"SQLiteStream._read_from_sqlite_and_fire time to load into IBlueRowCollection: {_stopwatch.ElapsedMilliseconds} msecs");
+         DataPacket dp = BlueDataUtility.build_dp_inc(rows, myLastValue as DataPacket);
+         myFireEvent(dp);
+      }
+
+      protected override void myOnFirstSubscriber() {
+         _read_from_sqlite_and_fire();
+      }
+
+      protected override void myOnLastSubscriber() {
+         // this value will be cleared by the base class
+         // but in this particular stream this behaviour
+         // is not usefull since any new subscribers 
+         // would force the query to be recreated
+         //myLastDP = myLastValue;
+      }
+
+   }
+
+}
@@ -3,12 +3,15 @@ using System.Collections.Generic;
 using System.Text;
 using System.IO;
 using BCG.Utilities;
+using System.Diagnostics;
+using Microsoft.Data.Analysis;
+using Parquet;
+using Parquet.Data;

-namespace BCG
-{
+
+namespace BCG {
   [DataStreamUserRespresentation("CSVSaver")]
-   public class CSVSaverStream : ViewStream, ISideEffectStream
-   {
+   public class CSVSaverStream : ViewStream, ISideEffectStream {
      private IBlueTableSchema myResultSchema = new BlueTableSchema("CSVResult", new ColumnInfo("id", 0, typeof(int)),
         new ColumnInfo("fileName", 1, typeof(string)),
         new ColumnInfo("nWrites", 2, typeof(int)),
@@ -20,11 +23,9 @@ namespace BCG
      private int myNWrites = 0, myNMaxWrites = Int32.MaxValue;

      public CSVSaverStream(ViewStream vs, string fullyQualifiedFileName)
-         : this(vs, fullyQualifiedFileName, true, Int32.MaxValue, false)
-      { }
+         : this(vs, fullyQualifiedFileName, true, Int32.MaxValue, false) { }

-      public CSVSaverStream(ViewStream vs, string fullyQualifiedFileName, bool overwrite, int nMaxWrites, bool fireResultSnapshot)
-      {
+      public CSVSaverStream(ViewStream vs, string fullyQualifiedFileName, bool overwrite, int nMaxWrites, bool fireResultSnapshot) {
         myStream = vs;
         myFullyQualifiedFileName = fullyQualifiedFileName;
         myOverwrite = overwrite;
@@ -36,18 +37,14 @@ namespace BCG

      protected override void myOnLastSubscriber() { myStream.onChange -= new ChangeDelegate(myHandleChanges); }

-      private void myHandleChanges(object newValue)
-      {
+      private void myHandleChanges(object newValue) {
         IBlueRowCollection totalState = (newValue as DataPacket).newRows;
-         if (totalState.Count > 0 && myNWrites < myNMaxWrites)
-         {
+         if (totalState.Count > 0 && myNWrites < myNMaxWrites) {
            myNWrites++;
            int i = 0;
            StreamWriter writer = new StreamWriter(new FileStream(myGetNewFileName(), FileMode.Create, FileAccess.Write));
-            foreach (IBlueRow r in totalState)
-            {
-               if (i++ == 0)
-               {
+            foreach (IBlueRow r in totalState) {
+               if (i++ == 0) {
                  string header = String.Empty;
                  foreach (string col in r.schema.columnNames)
                     if (col != "id")
@@ -59,20 +56,17 @@ namespace BCG
               writer.Flush();
            }
            writer.Close();
-            if (myFireResultSnapshot)
-            {
+            if (myFireResultSnapshot) {
               IBlueRow nr = new BlueRow(myResultSchema, 0, myFullyQualifiedFileName, myNWrites, totalState.Count);
               BlueTable nRows = new BlueTable(myResultSchema);
               nRows.addRow(nr);
               myFireEvent(new DataPacket(nRows, nRows, new BlueTable(myLastValue == null ? new BlueTable(myResultSchema) : (myLastValue as DataPacket).newRows)));
-            }
-            else
+            } else
               myFireEvent(newValue);
         }
      }

-      private string myGetNewFileName()
-      {
+      private string myGetNewFileName() {
         string[] tags = myFullyQualifiedFileName.Split('.');
         if (myOverwrite)
            return tags[0] + ".csv";
@@ -80,14 +74,11 @@ namespace BCG
            return tags[0] + "_" + myFormateDateTime(DateTime.Now) + ".csv";
      }

-      private static string myGetCSVRow(IBlueRow r)
-      {
+      private static string myGetCSVRow(IBlueRow r) {
         string rv = String.Empty;
         int i = 0;
-         foreach (string col in r.schema.columnNames)
-         {
-            if (col != "id")
-            {
+         foreach (string col in r.schema.columnNames) {
+            if (col != "id") {
               string colText = String.Empty;
               if (r[col] != null && r[col] != DBNull.Value)
                  colText = r[col].ToString();
@@ -99,8 +90,7 @@ namespace BCG
         return rv;
      }

-      private string myFormateDateTime(DateTime dt)
-      {
+      private string myFormateDateTime(DateTime dt) {
         return
         string.Format("{0}-{1}-{2}-{3}-{4}-{5}-{6}",
                        dt.Year.ToString("0000"),
@@ -114,52 +104,49 @@ namespace BCG
   }

   [DataStreamUserRespresentation("CSVReader")]
-   public class CSVReaderStream : ViewStream
-   {
-      private IBlueRowCollection myRows;
-
-      public CSVReaderStream(string fileAddress)
-      {
-         myRows = BlueDataUtility.createRowsFromCSVFileWithoutTypes(fileAddress);
+   public class CSVReaderStream : ViewStream {
+      private readonly string _file_address;
+      Stopwatch _stopwatch = new Stopwatch();
+      public CSVReaderStream(string fileAddress) {
+         _file_address = fileAddress;
      }

-      protected override void myOnFirstSubscriber()
-      {
-         myFireEvent(new DataPacket(myRows, myRows, new BlueTable(myRows.schema)));
+      protected override void myOnFirstSubscriber() {
+         _stopwatch.Start();
+         DataFrame df = DataFrame.LoadCsv(_file_address);
+         _stopwatch.Stop();
+         bts.Log.Gen.info($"CSVReaderStream.myOnFirstSubscriber stream csv load col_count: {df.Columns.Count} rows, time: {_stopwatch.ElapsedMilliseconds} msecs");
+         _stopwatch.Start();
+         IBlueRowCollection rows = BlueDataUtility.createRowsFromCSVFileWithoutTypes(_file_address);
+         _stopwatch.Stop();
+         bts.Log.Gen.info($"CSVReaderStream.myOnFirstSubscriber stream csv load count: {rows.Count} rows, time: {_stopwatch.ElapsedMilliseconds} msecs");
+         DataPacket dp_out = BlueDataUtility.build_dp_inc(rows, myLastValue as DataPacket);
+         myFireEvent(dp_out);
      }
   }

-   [DataStreamUserRespresentation ("CSVReaderStreamMapper")]
-   public class CSVReaderStreamMapperStream : ViewStream
-   {
-      public CSVReaderStreamMapperStream(ViewStream stream, string output_schema)
-      {
+   [DataStreamUserRespresentation("CSVReaderStreamMapper")]
+   public class CSVReaderStreamMapperStream : ViewStream {
+      public CSVReaderStreamMapperStream(ViewStream stream, string output_schema) {
         mySetRealSource(new MapStream(stream, new CSVReaderStreamMapper(output_schema)));
      }
   }

-   public class CSVReaderStreamMapper : IStreamMapper
-   {
+   public class CSVReaderStreamMapper : IStreamMapper {
      private IBlueTableSchema mySchema;
      private readonly string[] myOutputSchemaStringArray;

-      public CSVReaderStreamMapper(string outputSchema)
-      {
+      public CSVReaderStreamMapper(string outputSchema) {
         myOutputSchemaStringArray = outputSchema.Split(' ');
      }

-      public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema)
-      {
-         if (mySchema == null)
-         {
+      public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema) {
+         if (mySchema == null) {
            ColumnInfo[] ci = inputSchema.columnInfo;
-            foreach (string col in myOutputSchemaStringArray)
-            {
+            foreach (string col in myOutputSchemaStringArray) {
               string colName = col.Substring(0, col.IndexOf('\\')).ToLower();
-               for (int i = 0; i < ci.Length; i++)
-               {
-                  if (ci[i].colName.ToLower().Equals(colName))
-                  {
+               for (int i = 0; i < ci.Length; i++) {
+                  if (ci[i].colName.ToLower().Equals(colName)) {
                     Type t = myGetType(col.Substring(col.IndexOf('\\') + 1));
                     ci[i] = new ColumnInfo(ci[i].colName, i, t);
                     break;
@@ -171,74 +158,52 @@ namespace BCG
         return mySchema;
      }

-      public IBlueRow map(IBlueRow row)
-      {
+      public IBlueRow map(IBlueRow row) {
         object[] vals = new object[row.schema.columnInfo.Length];
         row.Values.CopyTo(vals, 0);
-         for (int i = 0; i < mySchema.columnInfo.Length; i++)
-         {
+         for (int i = 0; i < mySchema.columnInfo.Length; i++) {
            Type t = mySchema.columnInfo[i].colType;
-            try
-            {
+            try {
               vals[i] = System.Convert.ChangeType(vals[i], mySchema.columnInfo[i].colType, System.Globalization.CultureInfo.InvariantCulture);
-            }
-            catch
-            {
-               if (t.Equals(typeof(Int16)))
-               {
+            } catch {
+               if (t.Equals(typeof(Int16))) {
                  Int16 result;
-                  if (Int16.TryParse((string)vals[i], out result))
-                  {
+                  if (Int16.TryParse((string)vals[i], out result)) {
                     vals[i] = result;
                     continue;
                  }
                  double r2;
-                  if (Double.TryParse((string)vals[i], out r2))
-                  {
-                     try
-                     {
+                  if (Double.TryParse((string)vals[i], out r2)) {
+                     try {
                        vals[i] = (Int16)r2;
                        continue;
-                     }
-                     catch { }
+                     } catch { }
                  }
-               }
-               else if (t.Equals(typeof(Int32)))
-               {
+               } else if (t.Equals(typeof(Int32))) {
                  Int32 result;
-                  if (Int32.TryParse((string)vals[i], out result))
-                  {
+                  if (Int32.TryParse((string)vals[i], out result)) {
                     vals[i] = result;
                     continue;
                  }
                  double r2;
-                  if (Double.TryParse((string)vals[i], out r2))
-                  {
-                     try
-                     {
+                  if (Double.TryParse((string)vals[i], out r2)) {
+                     try {
                        vals[i] = (Int32)r2;
                        continue;
-                     }
-                     catch { }
+                     } catch { }
                  }
-               }
-               else if (t.Equals(typeof(Int64)))
-               {
+               } else if (t.Equals(typeof(Int64))) {
                  Int64 result;
-                  if (Int64.TryParse((string)vals[i], out result))
-                  {
+                  if (Int64.TryParse((string)vals[i], out result)) {
                     vals[i] = result;
                     continue;
                  }
                  double r2;
-                  if (Double.TryParse((string)vals[i], out r2))
-                  {
-                     try
-                     {
+                  if (Double.TryParse((string)vals[i], out r2)) {
+                     try {
                        vals[i] = (Int64)r2;
                        continue;
-                     }
-                     catch { }
+                     } catch { }
                  }
               }
               vals[i] = DBNull.Value;
@@ -247,8 +212,7 @@ namespace BCG
         return new BlueRow(mySchema, vals);
      }

-      private Type myGetType(string type)
-      {
+      private Type myGetType(string type) {
         type = type.ToLower();
         if (type.Contains("int"))
            return typeof(int);
@@ -1,31 +1,41 @@
 using System;
 using System.Collections;
 using System.Reflection;
+
 using BCG.Utilities;

-namespace BCG
-{
-   [DataStreamUserRespresentation (false)]
-   public class BlueRowCollectionStaticStream : ViewStream
-   {
-      public BlueRowCollectionStaticStream () {}
-
-      public BlueRowCollectionStaticStream (IBlueRowCollection rows) {loadCollection (rows);}
-
-      public void loadCollection (IBlueRowCollection rows)
-      {
-         myFireEvent (new DataPacket (rows, rows, myLastValue != null ?
-                                                   new BlueTable((myLastValue as DataPacket).newRows) : new BlueTable (rows.schema)));
+namespace BCG {
+   // if a table needs to be fired, it can be wrapped around this
+   public class BlueTableStream : ViewStream {
+      private IBlueRowCollection _rows;
+      public BlueTableStream(IBlueRowCollection rows) {
+         _rows = rows;
      }

-      public void crush_with_single_row(IBlueRow r)
-      {
+      protected override void myOnFirstSubscriber() {
+         var dp_out = BlueDataUtility.build_dp_inc(_rows, myLastValue as DataPacket);
+         myFireEvent(dp_out);
+      }
+   }
+
+   [DataStreamUserRespresentation(false)]
+   public class BlueRowCollectionStaticStream : ViewStream {
+      public BlueRowCollectionStaticStream() { }
+
+      public BlueRowCollectionStaticStream(IBlueRowCollection rows) { loadCollection(rows); }
+
+      public void loadCollection(IBlueRowCollection rows) {
+         myFireEvent(new DataPacket(rows, rows, myLastValue != null ?
+                                                   new BlueTable((myLastValue as DataPacket).newRows) : new BlueTable(rows.schema)));
+      }
+
+      public void crush_with_single_row(IBlueRow r) {
         BlueTable t = new BlueTable(r.schema);
         t.addRow(r);
         loadCollection(t);
      }

-      public void loadPacket (DataPacket dp) {myFireEvent (dp);}
+      public void loadPacket(DataPacket dp) { myFireEvent(dp); }

      /// <summary>
      /// Will work with streams that take ViewStream as first argument ex project, filter, distinct etc
@@ -35,17 +45,16 @@ namespace BCG
      /// <param name="streamType"></param>
      /// <param name="args"></param>
      /// <returns></returns>
-      public static IBlueRowCollection statelessOp (IBlueRowCollection rows, Type streamType, params object[] args)
-      {
-         ArrayList totArgs = new ArrayList (args);
-         BlueRowCollectionStaticStream v1 = new BlueRowCollectionStaticStream ();
-         totArgs.Insert (0, v1);
-         ViewStream retVal = Activator.CreateInstance (streamType, totArgs.ToArray()) as ViewStream;
-         v1.loadCollection (rows);
+      public static IBlueRowCollection statelessOp(IBlueRowCollection rows, Type streamType, params object[] args) {
+         ArrayList totArgs = new ArrayList(args);
+         BlueRowCollectionStaticStream v1 = new BlueRowCollectionStaticStream();
+         totArgs.Insert(0, v1);
+         ViewStream retVal = Activator.CreateInstance(streamType, totArgs.ToArray()) as ViewStream;
+         v1.loadCollection(rows);
         return retVal.rows;
      }

   }

-   
+
 }
@@ -80,7 +80,7 @@ namespace BCG {

      public IBlueRow map(IBlueRow row) {
         MutableBlueRow rv = new MutableBlueRow(getOutputSchema(row.schema), row);
-         rv.setValue(myConversionColName, mapDateTime(row[myConversionColName], myConversion));
+         rv.setValue(myConversionColName, mapDateTime(row.get_datetime(myConversionColName), myConversion));
         return rv;
         //return BlueDataUtility.getNewReplacedRow(row, mapDateTime(row[myConversionColName], myConversion), myConversionColName);
      }
@@ -0,0 +1,38 @@
+namespace bts.SilverPine {
+   using bts;
+
+   using BCG;
+
+   using System;
+
+   public class DateTimeExpanstionStream : ViewStream {
+      public DateTimeExpanstionStream(ViewStream parent_stream, string inp_col) {
+         Func<IBlueTableSchema, IBlueTableSchema> get_schema = (s) => {
+            IBlueTableSchema rv = s.add_column(System.TypeCode.DateTime, $"{inp_col}_date");
+            rv = rv.add_column(System.TypeCode.String, $"{inp_col}_day");
+            rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_day_of_month");
+            rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_month");
+            rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_hour");
+            rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_min");
+            rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_seconds_of_day");
+            return rv;
+         };
+         ViewStream rs = new MapStream(parent_stream,
+         s => get_schema(s),
+         r => {
+            DateTime dt = r.get_datetime(inp_col);
+            IBlueTableSchema s = get_schema(r.schema);
+            MutableBlueRow rv = new MutableBlueRow(s, r);
+            rv.setValue($"{inp_col}_date", dt.Date);
+            rv.setValue($"{inp_col}_day", dt.DayOfWeek.ToString());
+            rv.setValue($"{inp_col}_day_of_month", dt.Day);
+            rv.setValue($"{inp_col}_month", dt.Date.Month);
+            rv.setValue($"{inp_col}_hour", dt.Hour);
+            rv.setValue($"{inp_col}_min", dt.Minute);
+            rv.setValue($"{inp_col}_seconds_of_day", (dt - dt.Date).TotalSeconds);
+            return rv;
+         });
+         mySetRealSource(rs);
+      }
+   }
+}
@@ -0,0 +1,124 @@
+namespace bts.SilverPine {
+   using BCG;
+
+   using bts;
+   using bts.collections;
+
+   using System;
+   using System.CodeDom;
+   using System.Collections.Generic;
+   using System.Linq;
+
+   // Given 2 filters, and 2 columns, finds the different in the value of the 2 columns from the 2 rows
+   // so:
+   // f1(base_stream.rows)[col1] - f1(base_stream.rows)[col2]
+   [DataStreamUserRespresentation("ColDiff")]
+   public class ColumnDifferenceStream : ViewStream {
+      private readonly ViewStream _base_stream;
+      private readonly FilterStream.OnFilter _f1, _f2;
+      private readonly string _c1, _c2;
+
+      public ColumnDifferenceStream(ViewStream base_stream, FilterStream.OnFilter f1, FilterStream.OnFilter f2, string col1, string col2) {
+         _base_stream = base_stream;
+         _f1 = f1;
+         _f2 = f2;
+         _c1 = col1;
+         _c2 = col2;
+      }
+
+      public ColumnDifferenceStream(ViewStream base_stream, string f1, string f2, string col1, string col2) {
+         FilterStream.OnFilter f1_del = r => {
+            BooleanExpressionFilter rv = new BooleanExpressionFilter(f1);
+            return rv.filter(r);
+         };
+         FilterStream.OnFilter f2_del = r => {
+            BooleanExpressionFilter rv = new BooleanExpressionFilter(f2);
+            return rv.filter(r);
+         };
+         ViewStream rs = new ColumnDifferenceStream(base_stream, f1_del, f2_del, col1, col2);
+         mySetRealSource(rs);
+      }
+
+      // for each row, does a row[new_col_name] = row[col1] - rows[row_ffset][col2]
+      // we should generlize this for any math operation
+      public ColumnDifferenceStream(ViewStream parent_stream, string sort_col_name, int row_offset, string col1, string col2, string diff_col_name, double optional_perc_diff_filter = 0.0) {
+         // use GlobalMapStream
+         string diff_col_perc_name = $"{diff_col_name}_percentage";
+         System.Func<System.Collections.Generic.List<BCG.IBlueRow>, uint, BCG.IBlueRow> g_map_func = (rows, idx) => {
+            int minuend_idx = Convert.ToInt32(idx);
+            var inp_schema = rows[0].schema;
+            var out_schema = inp_schema.add_column(TypeCode.Double, diff_col_name);
+            out_schema = out_schema.add_column(TypeCode.Double, diff_col_perc_name);
+            // minuend - subtrahend
+            IBlueRow minuend_row = rows.ElementAt(minuend_idx);
+            MutableBlueRow rv = new MutableBlueRow(out_schema, minuend_row);
+            int subtrahend_index = minuend_idx + row_offset;
+            double diff_val = Double.NaN, diff_val_perc = Double.NaN;
+            if (subtrahend_index >= 0) {
+               try {
+                  IBlueRow subtrahend_row = rows.ElementAt(subtrahend_index);
+                  diff_val = minuend_row.get<double>(col1) - subtrahend_row.get<double>(col2);
+                  diff_val_perc = 100.0 * diff_val / subtrahend_row.get<double>(col1);
+               } catch (Exception ex) {
+                  Log.assert(false, ex.Message);
+               }
+            }
+            rv.setValue(diff_col_name, diff_val);
+            rv.setValue(diff_col_perc_name, diff_val_perc);
+            return rv;
+         };
+         ViewStream rs = new GlobalMapStream(parent_stream, sort_col_name, g_map_func);
+         if (optional_perc_diff_filter != 0) {
+            rs = new FilterStream(rs, r => {
+               double diff_val_perc = r.get<double>(diff_col_perc_name);
+               double abs_diff_val_perc = Math.Abs(diff_val_perc);
+               return abs_diff_val_perc >= optional_perc_diff_filter;
+            });
+         }
+         mySetRealSource(rs);
+      }
+
+      protected override void myOnFirstSubscriber() {
+         _base_stream.onChange += _handle_base_stream_update;
+      }
+
+      protected override void myOnLastSubscriber() {
+         _base_stream.onChange -= _handle_base_stream_update;
+      }
+
+      private void _handle_base_stream_update(object dp_obj) {
+         // could have been done with filter and cartesian prod, and then math
+         DataPacket dp = dp_obj as DataPacket;
+         List<IBlueRow> rows = dp.newRows.to_list();
+         var f1_rows = rows.Where(r => _f1(r));
+         //Log.assert(f1_rows.Count() == 1);
+         var f2_rows = rows.Where(r => _f2(r));
+         //Log.assert(f2_rows.Count() == 1);
+         if (f1_rows != null && f1_rows.Count() > 0 && f2_rows != null && f2_rows.Count() > 0) {
+            IBlueRow f1_r = f1_rows.ElementAt(0);
+            IBlueRow f2_r = f2_rows.ElementAt(0);
+            double diff = f1_r.get<double>(_c1) - f2_r.get<double>(_c2);
+            IBlueTableSchema out_schema = f1_r.schema.add_columns(Tuple.Create(typeof(double), "diff"));
+            BlueTable out_table = new BlueTable(out_schema);
+            MutableBlueRow out_row = new MutableBlueRow(out_schema, f1_r);
+            out_row.setValue("diff", diff);
+            out_table.addRow(out_row);
+            DataPacket dp_out = BlueDataUtility.build_dp(out_table, myLastValue as DataPacket);
+            myFireEvent(dp_out);
+         } else {
+            IBlueRow f1_r = rows.ElementAt(0);
+            IBlueRow f2_r = rows.ElementAt(0);
+            double diff = Double.NaN;
+            IBlueTableSchema out_schema = f1_r.schema.add_columns(Tuple.Create(typeof(double), "diff"));
+            BlueTable out_table = new BlueTable(out_schema);
+            MutableBlueRow out_row = new MutableBlueRow(out_schema, f1_r);
+            out_row.setValue("diff", diff);
+            out_table.addRow(out_row);
+            DataPacket dp_out = BlueDataUtility.build_dp(out_table, myLastValue as DataPacket);
+            myFireEvent(dp_out);
+         }
+
+      }
+   }
+
+}
@@ -0,0 +1,51 @@
+namespace bts.SilverPine {
+   using BCG;
+
+   using bts;
+
+   using System.Collections.Generic;
+   using System.Linq;
+
+   // Given a sort order, an optional group by col, it observes the values of a column 
+   // within that group (or the whole table, if grouping not specified) and returns all rows
+   // where the value changes, ie row[current][col] != row[current-1][col]
+   // TODO: a more general way to express a filter where row[current][col] != row[previous][col],
+   // or even expand it to row[current][col_a] - row[previous][col_b] < 0.5
+   [DataStreamUserRespresentation("ColEvol")]
+   public class ColumnEvolutionStream : ViewStream {
+      private readonly ViewStream _base_stream;
+      private readonly string _sort_col;
+      private readonly string _evol_col;
+      public ColumnEvolutionStream(ViewStream base_stream, string sort_col, string evolution_col) {
+         _base_stream = base_stream;
+         _sort_col = sort_col;
+         _evol_col = evolution_col;
+      }
+
+      protected override void myOnFirstSubscriber() {
+         _base_stream.onChange += _handle_base_stream_update;
+      }
+
+      protected override void myOnLastSubscriber() {
+         base.myOnLastSubscriber();
+      }
+
+      private void _handle_base_stream_update(object dp_obj) {
+         DataPacket dp = dp_obj as DataPacket;
+         List<IBlueRow> rows = dp.newRows.to_list();
+         rows.Sort((r1, r2) => BlueDataUtility.compare_rows(r2, r1, _sort_col));
+         BlueTable rv = new BlueTable(dp.schema);
+         IBlueRow prev_row = rows.ElementAt(0);
+         foreach (IBlueRow r in rows) {
+            var cr = BlueDataUtility.compare_rows(r, prev_row, _evol_col);
+            if (cr != 0) {
+               rv.addRow(r);
+            }
+            prev_row = r;
+         }
+         DataPacket dp_out = BlueDataUtility.build_dp(rv, myLastValue as DataPacket);
+         myFireEvent(dp_out);
+      }
+   }
+
+}
@@ -1,18 +1,15 @@
 using System;
 using System.Collections.Generic;
+
 using mbts;

-namespace BCG
-{
-   internal class MutableAppendTable : IBlueRowCollection
-   {
-      private class MyRow : IBlueRow
-      {
+namespace BCG {
+   internal class MutableAppendTable : IBlueRowCollection {
+      private class MyRow : IBlueRow {
         private IBlueRow myRow;
         private int myID;

-         public MyRow(IBlueRow r, int id)
-         {
+         public MyRow(IBlueRow r, int id) {
            myRow = r;
            myID = id;
         }
@@ -21,10 +18,8 @@ namespace BCG

         public IBlueTableSchema schema { get { return myRow.schema; } }

-         public object this[int index]
-         {
-            get
-            {
+         public object this[int index] {
+            get {
               if (myRow.schema[index].colName == "id")
                  return myID;
               else
@@ -52,20 +47,17 @@ namespace BCG

         public System.Collections.ICollection Keys { get { return myRow.Keys; } }

-         public void Remove(object key) {throw new Exception("The method or operation is not implemented.");}
+         public void Remove(object key) { throw new Exception("The method or operation is not implemented."); }

-         public System.Collections.ICollection Values
-         {
-            get
-            {
+         public System.Collections.ICollection Values {
+            get {
               object[] vals = new object[myRow.Count];
               CopyTo(vals, 0);
               return vals;
            }
         }

-         public object this[object key]
-         {
+         public object this[object key] {
            get { return key is string ? this[key as string] : this[(int)key]; }
            set { throw new Exception("The method or operation is not implemented."); }
         }
@@ -74,8 +66,7 @@ namespace BCG

         #region ICollection Members

-         public void CopyTo(Array array, int index)
-         {
+         public void CopyTo(Array array, int index) {
            myRow.CopyTo(array, index);
            array.SetValue(myID, index + myRow.schema["id"].colNo);
         }
@@ -90,8 +81,7 @@ namespace BCG

         #region IEnumerable Members

-         System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator()
-         { return new BlueRowEnumerator(this); }
+         System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() { return new BlueRowEnumerator(this); }

         #endregion
      }
@@ -106,8 +96,7 @@ namespace BCG

      public MutableAppendTable(IBlueTableSchema schema) { myRows = new BlueTable(schema); }

-      public IBlueRow addRow(IBlueRow r, int tableNo)
-      {
+      public IBlueRow addRow(IBlueRow r, int tableNo) {
         if (!myIDMap.ContainsKey(tableNo))
            myIDMap.Add(tableNo, new Dictionary<int, IBlueRow>());
         IBlueRow nr = new MyRow(r, myID++); // BlueDataUtility.getNewReplacedRow(r, myID++, "id");
@@ -116,20 +105,17 @@ namespace BCG
         return nr;
      }

-      public IBlueRow removeRow(IBlueRow r, int tableNo)
-      {
+      public IBlueRow removeRow(IBlueRow r, int tableNo) {
         IBlueRow dr = myIDMap[tableNo][(int)r["id"]];
         myRows.deleteRow(dr);
         myIDMap[tableNo].Remove((int)r["id"]);
         return dr;
      }

-      public IBlueRowCollection removeTable(int tableNo)
-      {
+      public IBlueRowCollection removeTable(int tableNo) {
         if (!myIDMap.ContainsKey(tableNo))
            return null;
-         else
-         {
+         else {
            BlueTable rv = new BlueTable(myRows.schema);
            foreach (IBlueRow r in myIDMap[tableNo].Values)
               rv.addRow(r);
@@ -139,8 +125,7 @@ namespace BCG
         }
      }

-      public void clear()
-      {
+      public void clear() {
         myRows.clear();
         myIDMap.Clear();
      }
@@ -151,7 +136,7 @@ namespace BCG

      public IBlueTableSchema schema { get { return myRows.schema; } }

-      public bool containsID(int id) {return myRows.containsID(id);}
+      public bool containsID(int id) { return myRows.containsID(id); }

      #endregion

@@ -169,7 +154,7 @@ namespace BCG

      #region IEnumerable Members

-      public System.Collections.IEnumerator GetEnumerator() {return myRows.GetEnumerator();}
+      public System.Collections.IEnumerator GetEnumerator() { return myRows.GetEnumerator(); }

      #endregion
   }
@@ -179,8 +164,8 @@ namespace BCG
   /// to append (or remove) more streams at
   /// runtime
   /// </summary>
-   public class MutableAppendStream : ViewStream
-   {
+   [DataStreamUserRespresentation("UnionStream")]
+   public class MutableAppendStream : ViewStream {
      private bool mySubscriptionActive = false;
      private MutableAppendTable myTable;
      private int myMovingIndex;
@@ -188,22 +173,17 @@ namespace BCG
      private bool _allign_col_orders = true;

      public MutableAppendStream(params ViewStream[] streams)
-         : this(new List<ViewStream>(streams))
-      { }
+         : this(new List<ViewStream>(streams)) { }

      public MutableAppendStream(bool allign_col_order, params ViewStream[] streams)
-         : this(allign_col_order, new List<ViewStream>(streams))
-      { }
+         : this(allign_col_order, new List<ViewStream>(streams)) { }

      public MutableAppendStream(ICollection<ViewStream> streams)
-         : this(true, streams)
-      { }
+         : this(true, streams) { }

-      public MutableAppendStream(bool allign_col_orders, ICollection<ViewStream> streams)
-      {
+      public MutableAppendStream(bool allign_col_orders, ICollection<ViewStream> streams) {
         _allign_col_orders = allign_col_orders;
-         foreach (ViewStream vs in streams)
-         {
+         foreach (ViewStream vs in streams) {
            myMovingIndex++;
            ViewStream pj = new ChangeTableNameStream(vs, myMovingIndex.ToString());
            if (_allign_col_orders)
@@ -212,8 +192,7 @@ namespace BCG
         }
      }

-      public void addStream(ViewStream vs)
-      {
+      public void addStream(ViewStream vs) {
         myMovingIndex++;
         ///the change name is done to identify the callbacks, and the project to
         ///make all the schemas col orders the same
@@ -225,19 +204,14 @@ namespace BCG
            pj.onChange += new ChangeDelegate(myHandleStreamChanges);
      }

-      public void removeStream(ViewStream vs)
-      {
+      public void removeStream(ViewStream vs) {
         int index = -1;
-         foreach (KeyValuePair<int, Pair<ViewStream, ViewStream>> p in myStreamIndex)
-         {
-            if (p.Value.first == vs)
-            {
+         foreach (KeyValuePair<int, Pair<ViewStream, ViewStream>> p in myStreamIndex) {
+            if (p.Value.first == vs) {
               index = p.Key;
-               if (mySubscriptionActive)
-               {
+               if (mySubscriptionActive) {
                  p.Value.second.onChange -= new ChangeDelegate(myHandleStreamChanges);
-                  if (myTable != null)
-                  {
+                  if (myTable != null) {
                     IBlueRowCollection delRows = myTable.removeTable(p.Key);
                     if (delRows != null)
                        myFireEvent(new DataPacket(myTable, new BlueTable(myTable.schema), delRows));
@@ -250,15 +224,13 @@ namespace BCG
         }
      }

-      protected override void myOnFirstSubscriber()
-      {
+      protected override void myOnFirstSubscriber() {
         mySubscriptionActive = true;
         foreach (KeyValuePair<int, Pair<ViewStream, ViewStream>> p in myStreamIndex)
            p.Value.second.onChange += new ChangeDelegate(myHandleStreamChanges);
      }

-      protected override void myOnLastSubscriber()
-      {
+      protected override void myOnLastSubscriber() {
         mySubscriptionActive = false;
         foreach (KeyValuePair<int, Pair<ViewStream, ViewStream>> p in myStreamIndex)
            p.Value.second.onChange -= new ChangeDelegate(myHandleStreamChanges);
@@ -266,20 +238,17 @@ namespace BCG
            myTable.clear();
      }

-      private void myHandleStreamChanges(object newValue)
-      {         
+      private void myHandleStreamChanges(object newValue) {
         DataPacket dp = newValue as DataPacket;
         myTable = myTable ?? new MutableAppendTable(new BlueTableSchema("MutableAppend", dp.schema.columnInfo));
         BlueTable adds = new BlueTable(myTable.schema),
                     dels = new BlueTable(myTable.schema);
         int index = Int32.Parse(dp.schema.tableName);
-         foreach (IBlueRow r in dp.deletedRows)
-         {
+         foreach (IBlueRow r in dp.deletedRows) {
            IBlueRow dr = myTable.removeRow(r, index);
            dels.addRow(dr);
         }
-         foreach (IBlueRow r in dp.addedRows)
-         {
+         foreach (IBlueRow r in dp.addedRows) {
            IBlueRow nr = myTable.addRow(r, index);
            adds.addRow(nr);
         }
@@ -15,10 +15,75 @@ using mbts;
 using BCG.Extensions;
 using System.Text.Json;
 using System.Text.Json.Nodes;
+using Newtonsoft.Json;
+using BCG;
+using System.Runtime.CompilerServices;
+using bts;
+
+namespace bts {
+
+   // public class CompositeRow : IBlueRow {
+   //    private IBlueRow _lrow, _rrow;
+   //    private BlueRow _out_row;
+   //    public CompositeRow(IBlueRow lrow, IBlueRow rrow) {
+   //       _lrow = lrow;
+   //       _rrow = rrow;
+   //       IBlueTableSchema schema = _lrow.schema;
+   //       List<object> vals = new List<object>();
+   //       foreach (var o in _lrow) {
+   //          vals.Add(o);
+   //       }
+   //       foreach (ColumnInfo ci in _rrow.schema.columnInfo) {
+   //          if (ci.colName == "id") {
+   //             continue;
+   //          }
+   //          vals.Add(_rrow[ci.colNo]);
+   //          schema = schema.add_column(ci.type_code(), ci.colName);
+   //       }
+   //       _out_row = new BlueRow(schema, vals.ToArray());
+   //    }
+   //    public IBlueTableSchema schema {
+   //       get { return _out_row.schema; }
+   //    }
+   //    public object? this[int index] {
+   //       get { return _out_row[index]; }
+   //    }
+   //    public object? this[string colName] {
+   //       get {
+   //          return _out_row[colName];
+   //       }
+   //    }
+
+   // }
+}

 namespace BCG {
   public static class BlueDataUtility {
+      public static IBlueRow create_composite_row(IBlueRow lr, IBlueRow rr, int id) {
+         IBlueTableSchema schema = lr.schema;
+         List<object> vals = new List<object>();
+         foreach (ColumnInfo ci in lr.schema.columnInfo) {
+            if (ci.colName == "id") {
+               vals.Add(id);
+            }
+            vals.Add(lr[ci.colName]);
+         }
+         foreach (ColumnInfo ci in rr.schema.columnInfo) {
+            if (ci.colName == "id") {
+               continue;
+            }
+            schema = schema.add_column(ci.type_code(), ci.colName);
+            vals.Add(rr[ci.colName]);
+         }
+         IBlueRow r = new BlueRow(schema, vals.ToArray());
+         return r;
+      }
      #region Extensions
+      public static bool is_null(this IBlueRow r, string col_name) {
+         object col_val = r[col_name];
+         return col_val == null || DBNull.Value.Equals(col_val);
+      }
+
      public static DateTime get_datetime(this IBlueRow r, string col_name, DateTime err_value = default(DateTime)) {
         return BlueDataUtility.get_row_datetime_value(r, col_name, err_value);
      }
@@ -49,6 +114,17 @@ namespace BCG {
            rv.Add(r.get<T>(col));
         return rv;
      }
+      public static IEnumerable<T> get_colvalues<T>(IEnumerable<IBlueRow> rows, string col, int start_index, int count) {
+         List<T> rv = new List<T>();
+         int i = 0;
+         int n_copied = 0;
+         foreach (IBlueRow r in rows)
+            if (i++ >= start_index && n_copied < count) {
+               rv.Add(r.get<T>(col));
+               n_copied++;
+            }
+         return rv;
+      }

      public static IDictionary<T, IEnumerable<IBlueRow>> group_by<T>(IBlueRowCollection rows, string gp_col) {
         Dictionary<T, List<IBlueRow>> rv = new Dictionary<T, List<IBlueRow>>();
@@ -112,13 +188,17 @@ namespace BCG {
         }
      }

-      public static IEnumerable<IBlueRow> to_ienum(this IBlueRowCollection rows) {
+      public static List<IBlueRow> to_list(this IBlueRowCollection rows) {
         List<IBlueRow> rv = new List<IBlueRow>();
         foreach (IBlueRow r in rows)
            rv.Add(r);
         return rv;
      }

+      public static IEnumerable<IBlueRow> to_ienum(this IBlueRowCollection rows) {
+         return rows.to_list();
+      }
+
      public static IDictionary<int, IBlueRow> to_map(this IBlueRowCollection rows) {
         Dictionary<int, IBlueRow> rv = new Dictionary<int, IBlueRow>();
         foreach (IBlueRow r in rows) {
@@ -128,6 +208,10 @@ namespace BCG {
         return rv;
      }

+      public static bool contains_column(this IBlueTableSchema input_schema, string col) {
+         return input_schema.columnNames.Contains(col);
+      }
+
      public static IBlueTableSchema add_columns(this IBlueTableSchema input_schema, params Tuple<Type, string>[] cols) {
         List<ColumnInfo> cis = new List<ColumnInfo>(input_schema.columnInfo);
         foreach (Tuple<Type, string> c in cols)
@@ -135,6 +219,11 @@ namespace BCG {
         return new BlueTableSchema(input_schema.tableName, cis.ToArray());
      }

+      public static IBlueTableSchema add_column(this IBlueTableSchema input_schema, TypeCode type_arg, string col_name) {
+         Tuple<Type, string> tup = new Tuple<Type, string>(_typecode_to_type[type_arg], col_name);
+         return add_columns(input_schema, tup);
+      }
+
      public static IBlueRowCollection to_collection(this IBlueRow r) {
         if (r == null)
            return null;
@@ -191,6 +280,18 @@ namespace BCG {

      #region json

+      public static Dictionary<string, string> convert_json_to_dictionary(string json_string) {
+         Dictionary<string, string> dictionary = JsonConvert.DeserializeObject<Dictionary<string, string>>(json_string);
+         return dictionary;
+      }
+      public static IBlueRow convert_json_to_row(string json_string, int id = 0) {
+         Dictionary<string, object> dictionary = JsonConvert.DeserializeObject<Dictionary<string, object>>(json_string);
+         if (!dictionary.ContainsKey("id")) {
+            dictionary.Add("id", id);
+         }
+         return new BlueRow(dictionary);
+      }
+
      public static List<JsonNode> json_to_json_node_list(string jsonString) {
         var jsonNodeList = new List<JsonNode>();
         bool noe = String.IsNullOrEmpty(jsonString);
@@ -209,7 +310,7 @@ namespace BCG {
            foreach (JsonNode jsonNode in jsonArray) {
               jsonNodeList.Add(jsonNode);
            }
-         } catch (JsonException ex) {
+         } catch (System.Text.Json.JsonException ex) {
            Console.WriteLine($"Invalid JSON: {ex.Message}");
         }

@@ -486,7 +587,16 @@ namespace BCG {
         ColumnInfo[] cis = new ColumnInfo[table.Columns.Count];
         foreach (DataColumn col in table.Columns)
            cis[col.Ordinal] = new ColumnInfo(col.ColumnName, col.Ordinal, col.DataType);
-         return new BlueTableSchema(cis, table.TableName);
+         IBlueTableSchema rv = new BlueTableSchema(cis, table.TableName);
+         rv = rv.add_id_col_if_missing();
+         return rv;
+      }
+      public static IBlueTableSchema add_id_col_if_missing(this IBlueTableSchema schema) {
+         IBlueTableSchema rv = schema;
+         if (!schema.columnNames.Contains("id")) {
+            rv = schema.add_columns(Tuple.Create(typeof(int), "id"));
+         }
+         return rv;
      }

      /// <summary>
@@ -744,10 +854,16 @@ namespace BCG {
      }

      public static IBlueRowCollection toIBlueRowCollection(DataTable dt) {
-         BlueTable retVal = new BlueTable(getSchema(dt));
-         foreach (DataRow dr in dt.Rows)
-            retVal.addRow(dr);
-         return retVal;
+         IBlueTableSchema schema = getSchema(dt);
+         BlueTable rv = new BlueTable(schema);
+         foreach (DataRow dr in dt.Rows) {
+            MutableBlueRow nr = new MutableBlueRow(schema);
+            foreach (DataColumn col in dr.Table.Columns)
+               nr.setValue(col.ColumnName, dr[col.ColumnName]);
+            nr.setValue("id", rv.Count);
+            rv.addRow(nr);
+         }
+         return rv;
      }

      public static IBlueRowCollection toIBlueRowCollection(DataRow[] rows) {
@@ -819,20 +935,20 @@ namespace BCG {
      /// </summary>
      /// <param name="r1"></param>
      /// <param name="r2"></param>
-      /// <param name="colNames"></param>
+      /// <param name="col_names"></param>
      /// <returns></returns>
      public static int compare_rows(IBlueRow r1,
         IBlueRow r2,
-         params string[] colNames) {
-         int retVal = 0;
-         foreach (string colName in colNames) {
-            object val1 = r1[colName];
-            object val2 = r2[colName];
-            retVal = _compare_row_cols(val1, val2);
-            if (retVal != 0)
+         params string[] col_names) {
+         int comp_res = 0;
+         foreach (string cn in col_names) {
+            object val1 = r1[cn];
+            object val2 = r2[cn];
+            comp_res = _compare_row_cols(val1, val2);
+            if (comp_res != 0)
               break;
         }
-         return retVal;
+         return comp_res;
      }

      private static int _compare_row_cols(object v1, object v2) {
@@ -1608,8 +1724,10 @@ namespace BCG {

      public static int[] get_column_indices(IBlueTableSchema schema, string[] col_names) {
         List<int> rv = new List<int>();
-         foreach (string c in col_names)
+         foreach (string c in col_names) {
+            Log.assert(schema.contains_column(c), $"Could not find col: {c} in table: {schema.tableName}");
            rv.Add(schema[c].colNo);
+         }
         return rv.ToArray();
      }

@@ -1783,6 +1901,18 @@ namespace BCG {
         #endregion
      }

+      public static IBlueRow find_max(IEnumerable<IBlueRow> rows, params string[] cols_to_compare) {
+         if (rows == null || rows.Count() == 0)
+            return null;
+         IBlueRow max_row = rows.ElementAt(0);
+         foreach (IBlueRow r in rows) {
+            int c = compare_rows(r, max_row, cols_to_compare);
+            if (c > 0)
+               max_row = r;
+         }
+         return max_row;
+      }
+
      public static int compare_rows(IBlueRow r1, IBlueRow r2, Dictionary<string, bool> sort_map) {
         foreach (KeyValuePair<string, bool> p in sort_map) {
            int comp = compare_send_dontcare_to_last(r1, p.Key, r2, p.Key, p.Value);
@@ -2084,23 +2214,53 @@ namespace BCG {
         try {
            if (v == null || v == DBNull.Value)
               return error_val;
-            else {
-               if (v is DateTime)
-                  return (DateTime)v;
-               else if (v is string) {
-                  DateTime rv_dt = error_val;
-                  DateTime.TryParse((string)v, out rv_dt);
+
+            if (v is DateTime) {
+               return (DateTime)v;
+            } else if (v is string) {
+               DateTime rv_dt;
+
+               // Try parsing the string as a DateTime
+               if (DateTime.TryParse((string)v,
+                                     System.Globalization.CultureInfo.InvariantCulture,
+                                     System.Globalization.DateTimeStyles.RoundtripKind,
+                                     out rv_dt)) {
                  return rv_dt;
               } else {
-                  double oa_date = (double)Convert.ChangeType(v, typeof(Double));
-                  return DateTime.FromOADate(oa_date);
+                  return error_val; // Return the error value if parsing fails
               }
+            } else {
+               // Assume the input can be converted to a double and use FromOADate
+               double oa_date = (double)Convert.ChangeType(v, typeof(Double));
+               return DateTime.FromOADate(oa_date);
            }
         } catch {
            return error_val;
         }
      }

+
+      // private static DateTime _get_datetime_value(object v, DateTime error_val) {
+      //    try {
+      //       if (v == null || v == DBNull.Value)
+      //          return error_val;
+      //       else {
+      //          if (v is DateTime)
+      //             return (DateTime)v;
+      //          else if (v is string) {
+      //             DateTime rv_dt = error_val;
+      //             DateTime.TryParse((string)v, out rv_dt);
+      //             return rv_dt;
+      //          } else {
+      //             double oa_date = (double)Convert.ChangeType(v, typeof(Double));
+      //             return DateTime.FromOADate(oa_date);
+      //          }
+      //       }
+      //    } catch {
+      //       return error_val;
+      //    }
+      // }
+
      #endregion

      #region CSV methods
@@ -8,6 +8,9 @@

  <ItemGroup>
    <PackageReference Include="Google.Protobuf" Version="3.24.3" />
+    <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
+    <PackageReference Include="OpenAI" Version="2.0.0" />
+    <PackageReference Include="SimMetrics.Net" Version="1.0.5" />
  </ItemGroup>

  <ItemGroup>
@@ -47,6 +47,11 @@ namespace BCG {
      /// </summary>
      /// <param name="assemblyName"></param>
      public void loadAssembly(string assemblyName) {
+         Func<Type, bool> is_silverpine_only = (t) => {
+            if (!assemblyName.Contains("SharedStreams.dll"))
+               return true;
+            return t.Namespace == "bts.SilverPine";
+         };
         bts.Log.Gen.info($"PrimitiveManager.loadAssembly assemblyName: {assemblyName}");
         if (myAssemblyNames.Contains(assemblyName))
            return;
@@ -56,7 +61,7 @@ namespace BCG {
            Assembly.LoadFrom(assemblyName);

         foreach (Type type in loadedAssembly.GetTypes())
-            if (!type.IsInterface
+            if (!type.IsInterface && is_silverpine_only(type)
               && !type.IsAbstract
               && type.IsSubclassOf(typeof(ViewStream))
               && type.IsPublic) {
@@ -9,7 +9,15 @@
 Letter                   [a-zA-Z]
 BooleanConstant          'true'|'false'
 CSStringConstant         \"([^\"\\]*|\\.)*\"
-DateTimeConstant         ([1-9]|'10'|'11'|'12')\/([1-9]|([1-2][0-9])|'30'|'31')\/[0-9][0-9][0-9]*
+YearInt                  [0-9]{4}
+MonthInt                 [0-9]{1,2}
+DayInt                   [0-9]{1,2}
+/**
+ * DateTimeConstant supports American (mm/dd/yyyy) and European (yyyy/mm/dd)
+ * formats. Month and day can be 1 or 2 digits. It does not validate that the
+ * numbers represent an actual date.
+ */
+DateTimeConstant         ({MonthInt}\/{DayInt}\/{YearInt}|{YearInt}\/{MonthInt}\/{DayInt})
 IntegerConstant          -?[0-9]+
 NullConstant             'null'
 RealConstant             -?[0-9]+\.[0-9]+
@@ -1,6 +1,6 @@
 {
  "user": "bts",
-  "sparkd": "http://127.0.0.1:8000/config",
+  "sparkd": "http://192.168.1.80:8000/config",
  "pubsub": {
    "milliseconds_between_sub_info_updates": 1000,
    "milliseconds_between_time_slice_updates": 5000,
@@ -105,5 +105,25 @@
        }
      ]
    }
+    // {
+    //   "name": "news_data",
+    //   "streams": [
+    //     {
+    //       "name": "benzinga_news",
+    //       "expression": "benzingaNews(\"mongodb://localhost:27017\",\"local\",\"news_headlines\", \"30s\", \"5d\")",
+    //       "is_on_websocket": true
+    //     },
+    //     {
+    //       "name": "benzinga_news_symbols",
+    //       "expression": "benzingaNewsSymbols(benzinga_news)",
+    //       "is_on_websocket": true
+    //     },
+    //     {
+    //       "name": "benzinga_news_symbol_state",
+    //       "expression": "benzingaNewsSymbolState(benzinga_news)",
+    //       "is_on_websocket": true
+    //     }
+    //   ]
+    // }
  ]
 }
@@ -9,6 +9,21 @@
          "expression": "milliseconds(1000,10)",
          "is_on_websocket": true
        },
+        {
+          "name": "benzinga_news",
+          "expression": "benzingaNews(\"mongodb://localhost:27017\",\"local\",\"news_headlines\", \"30s\", \"5d\")",
+          "is_on_websocket": true
+        },
+        {
+          "name": "benzinga_news_symbols",
+          "expression": "benzingaNewsSymbols(benzinga_news)",
+          "is_on_websocket": true
+        },
+        {
+          "name": "benzinga_news_symbol_state",
+          "expression": "benzingaNewsSymbolState(benzinga_news)",
+          "is_on_websocket": true
+        },
        {
          "name": "test_literal",
          "expression": "literal(\"x\", \"420.0\")",
@@ -10,7 +10,8 @@ using System.IO;
 using System.Text.Json;
 using System.Text.Json.Serialization;

-public class DataStreamPublisherConfig {
+public class DataStreamPublisherConfig
+{
   /// <summary>
   /// The login user for sparkd
   /// </summary>
@@ -38,6 +39,18 @@ public class DataStreamPublisherConfig {
   [JsonPropertyName("publishers")]
   public List<PublisherNode>? Publishers { get; set; }

+   [JsonPropertyName("mongodb_url")]
+   public string MONGO_DB_URL { get; set; }
+
+   [JsonPropertyName("mongodb_db_name")]
+   public string MONGO_DB_NAME { get; set; }
+
+   [JsonPropertyName("mongodb_db_collection")]
+   public string MONGO_DB_COLLECTION { get; set; }
+
+   [JsonPropertyName("benzinga_news_ttl")]
+   public string BENZINGA_NEWS_TTL { get; set; }
+
   [JsonPropertyName("pubsub")]
   public PubSubConfig? PubSub { get; set; }

@@ -49,11 +62,13 @@ public class DataStreamPublisherConfig {
   public string? ConfigFile { get; set; }

   public static DataStreamPublisherConfig Default() =>
-      new() {
+      new()
+      {
         PubSub = PubSubConfig.Default(),
      };

-   public void MergeWith(DataStreamPublisherConfig other) {
+   public void MergeWith(DataStreamPublisherConfig other)
+   {
      User ??= other.User;
      Password ??= other.Password;
      Sparkd ??= other.Sparkd;
@@ -69,8 +84,10 @@ public class DataStreamPublisherConfig {
   /// <exception cref="InvalidOperationException">
   /// JsonSerializer.Deserialize returns null
   /// </exception>
-   public void MergeWithConfigFile() {
-      if (ConfigFile is null) {
+   public void MergeWithConfigFile()
+   {
+      if (ConfigFile is null)
+      {
         return;
      }
      var cfgString = File.ReadAllText(ConfigFile);
@@ -23,8 +23,10 @@ using System.Threading.Tasks;
 await Parser.Default.ParseArguments<DataStreamPublisherConfig>(args)
   .MapResult(RunAsync, errs => Task.FromResult(1));

-async Task<int> RunAsync(DataStreamPublisherConfig cfg) {
-   try {
+async Task<int> RunAsync(DataStreamPublisherConfig cfg)
+{
+   try
+   {
      cfg.MergeWithConfigFile();
      cfg.MergeWith(DataStreamPublisherConfig.Default());
      PubSubConfig.SetGlobalConfig(Log.notnull(cfg.PubSub));
@@ -33,7 +35,9 @@ async Task<int> RunAsync(DataStreamPublisherConfig cfg) {
      var service_ctx = ServiceContext.create(cfg.Sparkd);
      await service_ctx.login(cfg.User ?? "bts", cfg.Password ?? string.Empty);
      _publish_streams(cfg.Publishers ?? new(0));
-   } catch (Exception ex) {
+   }
+   catch (Exception ex)
+   {
      // Write the error to console first, in case the logging system is compromised.
      var error_msg = $"DataStreamPublisher threw exception:\n{ex}";
      Console.Error.WriteLine(error_msg);
@@ -44,29 +48,39 @@ async Task<int> RunAsync(DataStreamPublisherConfig cfg) {
   return 0;
 }

-static ViewStream? build_stream(string? stream_expression) {
-   if (stream_expression is null) {
+static ViewStream? build_stream(string? stream_expression)
+{
+   // Console.WriteLine("Hit key to continue");
+   // Console.ReadLine();
+   if (stream_expression is null)
+   {
      return null;
   }
-   try {
+   try
+   {
      bts.Log.Gen.info($"build_stream stream_expression: {stream_expression}");
      var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression);
      var rv = exp.getView(new Dictionary<string, StatementExpression>(0));
      return rv;

-   } catch (Exception ex) {
+   }
+   catch (Exception ex)
+   {
      bts.Log.assert(false, $"Could not build stream: {stream_expression} ex: {ex}");
      return null;
   }
 }

-void _publish_streams(IEnumerable<PublisherNode> publishers) {
+void _publish_streams(IEnumerable<PublisherNode> publishers)
+{
   PubTableBroker pb_broker = new PubTableBroker();
   RemoteQueryManager csm = new RemoteQueryManager();
   var csvPublishers = new List<CSVPublisher>();

-   foreach (var publisher in publishers) {
-      if (Directory.Exists(publisher.CsvDir)) {
+   foreach (var publisher in publishers)
+   {
+      if (Directory.Exists(publisher.CsvDir))
+      {
         csvPublishers.Add(new(publisher.CsvDir, pb_broker));
      }
   }
@@ -75,7 +89,8 @@ void _publish_streams(IEnumerable<PublisherNode> publishers) {
      .SelectMany(p => p.Streams.empty_if_null().Select(createViewPublisher))
      .ToList();

-   while (true) {
+   while (true)
+   {
      System.Threading.Thread.Sleep(TimeSpan.FromSeconds(1));
      ViewStreamDatabase.instance.doWork();
      csm.doWork();
@@ -83,16 +98,19 @@ void _publish_streams(IEnumerable<PublisherNode> publishers) {
      ManagedSubscriber.instance.doWork();
   }

-   ViewPublisher? createViewPublisher(PublishedStream ps) {
+   ViewPublisher? createViewPublisher(PublishedStream ps)
+   {
      Log.Gen.info(
         $"Instantiating ViewPublisher for: {JsonSerializer.Serialize(ps)}");
-      if (string.IsNullOrEmpty(ps.Name)) {
+      if (string.IsNullOrEmpty(ps.Name))
+      {
         Log.Gen.warn(
            $"Failed to instantiate ViewPublisher. Null/empty 'name': {JsonSerializer.Serialize(ps)}");
         return null;
      }
      var vs = build_stream(ps.Expression);
-      if (vs is null) {
+      if (vs is null)
+      {
         Log.Gen.warn(
            $"Failed to instantiate ViewPublisher. Invalid 'expression': {JsonSerializer.Serialize(ps)}");
         return null;
@@ -103,31 +121,39 @@ void _publish_streams(IEnumerable<PublisherNode> publishers) {
   }
 }

-void on_change(object obj) {
+void on_change(object obj)
+{
   bts.Log.Gen.info($"DSPublisher on_change dp: {obj}");
 }

-public class CSVPublisher : IWorker {
+public class CSVPublisher : IWorker
+{
   private IDictionary<string, ViewPublisher> _pubs = new Dictionary<string, ViewPublisher>();
   private readonly string _csv_directory;
   private PubTableBroker _pub_table_broker;

-   public CSVPublisher(string csv_directory, PubTableBroker pub_table_broker) {
+   public CSVPublisher(string csv_directory, PubTableBroker pub_table_broker)
+   {
      _csv_directory = csv_directory;
      _pub_table_broker = pub_table_broker;
      MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(1));
   }
-   public void doWork() {
+   public void doWork()
+   {
      var file_names = Directory.GetFiles(_csv_directory);
-      foreach (var fn in file_names) {
-         if (!_pubs.ContainsKey(fn)) {
+      foreach (var fn in file_names)
+      {
+         if (!_pubs.ContainsKey(fn))
+         {
            _handle_new_file(fn);
         }
      }
   }

-   private void _handle_new_file(string file_name) {
-      if (file_name.EndsWith(".csv")) {
+   private void _handle_new_file(string file_name)
+   {
+      if (file_name.EndsWith(".csv"))
+      {
         ViewStream vs = new CSVReaderStream(file_name);
         vs = new SchemaGuesserStream(vs);
         var toks = file_name.Split('\\');
@@ -1,5 +1,6 @@
 {
  "user": "bts",
+  "sparkd": "http://192.168.1.80:8000/config",
  "pubsub": {
    "milliseconds_between_sub_info_updates": 1000,
    "milliseconds_between_time_slice_updates": 5000,
@@ -17,14 +18,14 @@
    "publisher_routes": [
      {
        "name": "default",
-        "server_address": "bluto://192.168.1.80:12345",
+        "server_address": "bts-cloud://192.168.1.80:12345",
        "tag": ".*"
      }
    ],
    "subscriber_routes": [
      {
        "name": "default",
-        "server_address": "bluto://192.168.1.80:12345",
+        "server_address": "bts-cloud://192.168.1.80:12345",
        "tag": ".*"
      }
    ],
@@ -169,8 +169,10 @@ namespace BCG
      }
      #endregion

+      private Stopwatch _stopwatch = new Stopwatch();
      private void myTryExpression(string expStr)
      {
+         _stopwatch.Restart();
         myFreezeView(false); // force unfreeze the view
         DataStreamExpression oldExp = myExp;
         try
@@ -344,6 +346,7 @@ namespace BCG
      // this is the stream callback
      private void myCallback(object o)
      {
+         _stopwatch.Stop();
         var timer = Stopwatch.StartNew();
         DataPacket dp = o as DataPacket;
         myCrono.Stop();
@@ -359,7 +362,8 @@ namespace BCG
         myToolStripStatusLabelRows.BackColor = Color.Black;
         myToolStripStatusLabelTime.ForeColor = Color.White;
         myToolStripStatusLabelRows.ForeColor = Color.White;
-         myToolStripStatusLabelTime.Text = "Last update: " + DateTime.Now.ToString("hh:mm:ss:fff tt");
+         string time_text = "Last update: " + DateTime.Now.ToString("hh:mm:ss:fff tt") + $" time: {_stopwatch.ElapsedMilliseconds} msecs\t\t";
+         myToolStripStatusLabelTime.Text = time_text;
         myToolStripStatusLabelRows.Text = "New = " + dp.newRows.Count + "  Added = " + dp.addedRows.Count + "  Deleted = " + dp.deletedRows.Count;
         Log.Ui.diag(Text);
         int hscrollval = myChangeStyle ? 0 : myDataGrid.HorizScrollBarValue;
@@ -117,10 +117,10 @@ public class PubSubConfig {
         WaitPeriodAfterGc = 1000,
         SimpleEchoServer = false,
         PublisherRoutes = new() {
-            new("bts-cloud://127.0.0.1:12345", ".*")
+            new("bluto://127.0.0.1:12345", ".*")
         },
         SubscriberRoutes = new() {
-            new("bts-cloud://127.0.0.1:12345", ".*")
+            new("bluto://127.0.0.1:12345", ".*")
         }
      };
      return rv;
@@ -36,14 +36,17 @@ using System.Linq;
 using Bts.PubSub;
 using Spark.Core;

-namespace bts {
-   public class AsyncTCPConnection_APM : ITCPConnection, IWorker, IDisposable {
+namespace bts
+{
+   public class AsyncTCPConnection_APM : ITCPConnection, IWorker, IDisposable
+   {

      // this class is used as the argument of the completion function for BeginSend/Receive   
      // serves as the base class for both Send and ReeceiveMessage classes.
      // since both TCP sends and receives happen in multiple transfers, this
      // provides a wrapper around copying the partial transfers into the complete buffer
-      public class AsyncMessageBase {
+      public class AsyncMessageBase
+      {
         readonly string msg_id = Guid.NewGuid().ToString();
         // every BeginSend or BeginReceive has a unique key associated with it
         public int key;
@@ -55,37 +58,45 @@ namespace bts {
         public int transfer_ptr;
         protected int _n_transfers = 0;
         public bool completed_sync = false;
-         public AsyncMessageBase(byte[] the_buffer, int the_offset) {
+         public AsyncMessageBase(byte[] the_buffer, int the_offset)
+         {
            buffer = the_buffer;
            transfer_ptr = the_offset;
         }
-         public int remaining_size {
-            get {
+         public int remaining_size
+         {
+            get
+            {
               return buffer.Length - transfer_ptr;
            }
         }

         public int original_size => buffer.Length;

-         public void advance_transfer_pointer(int advance_by) {
+         public void advance_transfer_pointer(int advance_by)
+         {
            transfer_ptr += advance_by;
            _n_transfers++;
-            if (transfer_ptr > original_size) {
+            if (transfer_ptr > original_size)
+            {
               bts.Log.Gen.error($"transfer went beyond initial size {ToString()}");
            }
            // TODO: log if it goes past the buffer
         }

-         public bool fully_transfered() {
+         public bool fully_transfered()
+         {
            return transfer_ptr >= original_size;
         }

-         public override string ToString() {
+         public override string ToString()
+         {
            return $"{GetType().ToString()} msg_id: {msg_id} key: {key} original_length: {original_size} transfer_ptr: {transfer_ptr} n_transfers: {_n_transfers} " +
                   $"fully_transfered: {fully_transfered()} completed_sync: {completed_sync}";
         }
      }
-      public class AsyncSendMessage : AsyncMessageBase {
+      public class AsyncSendMessage : AsyncMessageBase
+      {
         public string message_id;
         public SendConfirmationDelegate on_send_del;
         //private int _n_transfers = 0;
@@ -94,35 +105,43 @@ namespace bts {
            : this(the_buffer, the_offset, String.Empty, null) { }

         public AsyncSendMessage(byte[] the_buffer, int the_transfer_ptr, string the_message_id, SendConfirmationDelegate the_on_send_del)
-         : base(the_buffer, the_transfer_ptr) {
+         : base(the_buffer, the_transfer_ptr)
+         {
            message_id = the_message_id;
            on_send_del = the_on_send_del;
         }
-         public override string ToString() {
+         public override string ToString()
+         {
            return base.ToString();
         }

      }

-      public class AsyncReceiveMessage : AsyncMessageBase {
-         public enum ReadState {
+      public class AsyncReceiveMessage : AsyncMessageBase
+      {
+         public enum ReadState
+         {
            reading_size,
            reading_data
         };
         readonly ReadState read_state;

         public AsyncReceiveMessage(int size, ReadState the_read_state) :
-         this(new byte[size], 0, the_read_state) { }
+         this(new byte[size], 0, the_read_state)
+         { }
         public AsyncReceiveMessage(byte[] the_buffer, int the_transfer_ptr, ReadState the_read_state)
-         : base(the_buffer, the_transfer_ptr) {
+         : base(the_buffer, the_transfer_ptr)
+         {
            read_state = the_read_state;
         }

-         public int read_int32(int start_idx = 0) {
+         public int read_int32(int start_idx = 0)
+         {
            return BitConverter.ToInt32(buffer, start_idx);
         }

-         public byte[] read_bytes(int start_idx) {
+         public byte[] read_bytes(int start_idx)
+         {
            byte[] rv = new byte[buffer.Length - start_idx];
            Array.Copy(buffer, start_idx, rv, 0, rv.Length);
            return rv;
@@ -131,16 +150,19 @@ namespace bts {
         public bool is_reading_size() => read_state == ReadState.reading_size;
      }

-      public class ErrorMessage {
+      public class ErrorMessage
+      {
         public readonly SocketError? socket_error = null;
         public readonly Exception exception = null;
         public readonly bool is_exception = false;
         public readonly string context = String.Empty;
-         public ErrorMessage(SocketError socket_error_, string context_) {
+         public ErrorMessage(SocketError socket_error_, string context_)
+         {
            socket_error = socket_error_;
            context = context_;
         }
-         public ErrorMessage(Exception exception_, string context_) {
+         public ErrorMessage(Exception exception_, string context_)
+         {
            is_exception = true;
            exception = exception_;
            context = context_;
@@ -148,7 +170,8 @@ namespace bts {

         public bool oom => exception != null && exception is OutOfMemoryException;

-         public override string ToString() {
+         public override string ToString()
+         {
            if (is_exception)
               return $"ErrorMessage exception: {exception} context: {context}";
            return $"socket_error: {socket_error} context: {context}";
@@ -174,13 +197,15 @@ namespace bts {
      private bool _hb_received_yet = false, _data_received_yet = false, _hb_sent_once = false;
      private DateTime _last_hb_received_time = new DateTime(1900, 1, 1);

-      public AsyncTCPConnection_APM(Socket sock, TimeSpan hb_period, ITCPConnection.ConnectionSide connection_side) {
+      public AsyncTCPConnection_APM(Socket sock, TimeSpan hb_period, ITCPConnection.ConnectionSide connection_side)
+      {
         _sock = sock;
         _hb_period = hb_period;
         _connection_side = connection_side;
         _sock.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.DontRoute, true);
         #region testing options
-         switch (connection_side) {
+         switch (connection_side)
+         {
            case ITCPConnection.ConnectionSide.Server:
               TCPSocketUtils.set_server_socket_options(ref _sock);
               break;
@@ -195,10 +220,13 @@ namespace bts {
         _detect_connection_status_change();
      }

-      private void _detect_connection_status_change() {
-         Action testing_func = () => {
+      private void _detect_connection_status_change()
+      {
+         Action testing_func = () =>
+         {
            bool connected = _sock.Connected;
-            while (true) {
+            while (true)
+            {
               if (connected != _sock.Connected)
                  Log.Gen.error($"Connection status changed this: {ToString()} {connected} -> {_sock.Connected}");
               connected = _sock.Connected;
@@ -208,26 +236,30 @@ namespace bts {
         ThreadPool.QueueUserWorkItem(_ => testing_func());
      }

-      public override string ToString() {
+      public override string ToString()
+      {
         return $"AsyncTCPConnection_APM side: {_connection_side} peer: {_peer_conn_info}, self: {_self_conn_info}";
      }

      #region ITCPConnection Members

-      public void sendWithConfirmation(byte[] data, ref string conf_id, SendConfirmationDelegate del) {
+      public void sendWithConfirmation(byte[] data, ref string conf_id, SendConfirmationDelegate del)
+      {
         conf_id = Guid.NewGuid().ToString();
         byte[] msg = _build_data_message(data);
         _begin_or_resume_sending(new AsyncSendMessage(msg, 0, conf_id, del));
      }

-      public void send(byte[] data) {
-         Log.Gen.info($"AsyncTCPConnection_APM.send data: {AsyncTCPConnection_AsyncWrapper.to_hex(data, 0, data.Length)}");
+      public void send(byte[] data)
+      {
+         //Log.Gen.info($"AsyncTCPConnection_APM.send data: {AsyncTCPConnection_AsyncWrapper.to_hex(data, 0, data.Length)}");
         byte[] msg = _build_data_message(data);
-         Log.Gen.info($"AsyncTCPConnection_APM.send msg: {AsyncTCPConnection_AsyncWrapper.to_hex(msg, 0, msg.Length)}");
+         //Log.Gen.info($"AsyncTCPConnection_APM.send msg: {AsyncTCPConnection_AsyncWrapper.to_hex(msg, 0, msg.Length)}");
         _begin_or_resume_sending(new AsyncSendMessage(msg, 0));
      }

-      public void register(ITCPMessageHandler messageHandler) {
+      public void register(ITCPMessageHandler messageHandler)
+      {
         bts.Log.assert_d(_message_handler == null, "Message Handler is already assigned");
         //bts.Log.Gen.info($"AsyncTCPConnection_APM for peer = {_peer_conn_info} being registered");
         _message_handler = messageHandler;
@@ -236,15 +268,18 @@ namespace bts {
         _read_next_msg();
      }

-      public RemoteConnectionInfo get_peer_address() {
+      public RemoteConnectionInfo get_peer_address()
+      {
         return TCPSocketUtils.get_peer_address(_sock);
      }

-      public RemoteConnectionInfo get_self_address() {
+      public RemoteConnectionInfo get_self_address()
+      {
         return TCPSocketUtils.get_peer_address(_sock);
      }

-      public bool isDisposed {
+      public bool isDisposed
+      {
         get { return !_is_alive; }
      }

@@ -253,12 +288,14 @@ namespace bts {
      #region IWorker Members

      bool _last_connected = false;
-      public void doWork() {
+      public void doWork()
+      {
         if (!_is_alive)
            return;
         return;
         // disable hbs for now
-         if (!_hb_sent_once) {
+         if (!_hb_sent_once)
+         {
            bts.Log.Gen.info("Sending heartbeat to peer {_peer_conn_info} for the first time");
            _hb_sent_once = true;
         }
@@ -266,12 +303,14 @@ namespace bts {
         _begin_or_resume_sending(new AsyncSendMessage(msg, 0));
      }

-      private static int _tid() {
+      private static int _tid()
+      {
         var tid = System.Threading.Thread.CurrentThread.ManagedThreadId;
         return tid;
      }

-      private static string _cb_txt(IAsyncResult res) {
+      private static string _cb_txt(IAsyncResult res)
+      {
         return $"completed?: {res.IsCompleted} sync?: {res.CompletedSynchronously}";
      }

@@ -279,10 +318,13 @@ namespace bts {

      #region sending data

-      private void _begin_or_resume_sending(AsyncSendMessage message) {
+      private void _begin_or_resume_sending(AsyncSendMessage message)
+      {
         //bts.Log.Gen.info($"AsyncTCPConnection_APM._begin_or_resume_sending this: {ToString()} tid: {_tid()} message: {message} _is_alive: {_is_alive}");
-         if (_is_alive) {
-            try {
+         if (_is_alive)
+         {
+            try
+            {
               SocketError begin_send_err;
               // since we get message on on_send we dont need to keep the state around
               message.key = _send_key++;
@@ -290,35 +332,45 @@ namespace bts {
                                 out begin_send_err, new AsyncCallback(BeginSend_callback_function), message);
               if (begin_send_err != SocketError.Success)
                  _handle_error(new ErrorMessage(begin_send_err, "_begin_or_resume_sending"));
-            } catch (Exception ex) {
+            }
+            catch (Exception ex)
+            {
               _handle_error(new ErrorMessage(ex, "_begin_or_resume_sending"));
            }
         }
      }

-      private void BeginSend_callback_function(IAsyncResult res) {
-         if (res.CompletedSynchronously) {
+      private void BeginSend_callback_function(IAsyncResult res)
+      {
+         if (res.CompletedSynchronously)
+         {
            ThreadPool.QueueUserWorkItem(_ => _on_send_async(res));
-         } else
+         }
+         else
            _on_send_async(res);
      }

-      private void _on_send_async(IAsyncResult res) {
+      private void _on_send_async(IAsyncResult res)
+      {
         AsyncSendMessage message = res.AsyncState as AsyncSendMessage;
-         if (_send_key_map.Contains(message.key)) {
+         if (_send_key_map.Contains(message.key))
+         {
            bts.Log.Gen.warn($"AsyncTCPConnection_APM._on_send key: {message.key} already handled");
            return;
         }
         _send_key_map.Add(message.key);
         // bts.Log.Gen.info($"AsyncTCPConnection_APM._on_send: pre endSend this: {ToString()} tid: {_tid()} " +
         //                         $"res: {_cb_txt(res)} message: {res.AsyncState} _is_alive: {_is_alive}");
-         if (_is_alive) {
-            try {
+         if (_is_alive)
+         {
+            try
+            {
               SocketError send_err;
               int bytes_sent = _sock.EndSend(res, out send_err);
               //message.offset += bytesSent;
               message.advance_transfer_pointer(bytes_sent);
-               if (bytes_sent == 0) {
+               if (bytes_sent == 0)
+               {
                  bts.Log.Gen.error($"AsyncTCPConnection_APM._on_send this: {this} sent 0 bytes, throwing exception");
                  _handle_error(new ErrorMessage(new ApplicationException("Sent 0 bytes, peer socket is likely shutdown"),
                        "_on_send"));
@@ -326,22 +378,31 @@ namespace bts {
               }
               bts.Log.Gen.info($"AsyncTCPConnection_APM._on_send: post endSend this: {ToString()} tid: {_tid()} res: {_cb_txt(res)} " +
               $"message: {res.AsyncState} bytes_sent: {bytes_sent} _is_alive: {_is_alive} send_err: {send_err}");
-               if (send_err == SocketError.Success) {
+               if (send_err == SocketError.Success)
+               {
                  // successful send of some bytes
                  //if (message.offset == message.buffer.Length) {
-                  if (message.fully_transfered()) {
+                  if (message.fully_transfered())
+                  {
                     // fully sent, if confirmation needs to be sent out                     
-                     if (!String.IsNullOrEmpty(message.message_id)) {
+                     if (!String.IsNullOrEmpty(message.message_id))
+                     {
                        message.on_send_del(message.message_id, true);
                     }
-                  } else {
+                  }
+                  else
+                  {
                     // there is more to send
                     _begin_or_resume_sending(message);
                  }
-               } else {
+               }
+               else
+               {
                  _handle_error(new ErrorMessage(send_err, "_on_send"));
               }
-            } catch (Exception ex) {
+            }
+            catch (Exception ex)
+            {
               _handle_error(new ErrorMessage(ex, "_on_send"));
            }
         }
@@ -352,8 +413,10 @@ namespace bts {
      /// future references
      /// </summary>
      /// <returns></returns>
-      private byte[] _build_hb_message() {
-         if (_cached_hb_message == null) {
+      private byte[] _build_hb_message()
+      {
+         if (_cached_hb_message == null)
+         {
            BinaryWriter bw = new BinaryWriter(new MemoryStream());
            // the total msg =[msg_sz, msg_type] 
            int msg_sz = 4;
@@ -365,7 +428,8 @@ namespace bts {
      }

      // writes size prefixed msg
-      private byte[] _build_data_message(byte[] data) {
+      private byte[] _build_data_message(byte[] data)
+      {
         // TODO, avoid unecessary memory creation (write directly into byte[] using BitConverter)
         MemoryStream str = new MemoryStream();
         BinaryWriter bw = new BinaryWriter(str);
@@ -384,28 +448,34 @@ namespace bts {

      #region receiving data

-      private void _read_next_msg() {
+      private void _read_next_msg()
+      {
         _read_size();
      }

-      private void _read_size() {
+      private void _read_size()
+      {
         _begin_or_resume_reading(new AsyncReceiveMessage(4, AsyncReceiveMessage.ReadState.reading_size));
      }

-      private void _read_data(int data_size) {
+      private void _read_data(int data_size)
+      {
         // TODO, this could be a problem is the size is too big - maybe read it in chunks (later)
         _begin_or_resume_reading(new AsyncReceiveMessage(data_size, AsyncReceiveMessage.ReadState.reading_data));
      }

-      private static string IAsyncResult_txt(IAsyncResult res) {
+      private static string IAsyncResult_txt(IAsyncResult res)
+      {
         return $"IsCompleted: {res.IsCompleted} CompletedSynchronously: {res.CompletedSynchronously}";
      }

-      private void _begin_or_resume_reading(AsyncReceiveMessage rcv_msg) {
+      private void _begin_or_resume_reading(AsyncReceiveMessage rcv_msg)
+      {
         //bts.Log.Gen.info($"AsyncTCPConnection_APM._begin_or_resume_reading: this: {ToString()} tid: {_tid()} message: {rcv_msg} _is_alive: {_is_alive}");
         if (!_is_alive)
            return;
-         try {
+         try
+         {
            SocketError read_sock_err;
            rcv_msg.key = _receive_key++;
            var res = _sock.BeginReceive(rcv_msg.buffer, rcv_msg.transfer_ptr,
@@ -413,50 +483,61 @@ namespace bts {
                                 SocketFlags.None, out read_sock_err,
                                 new AsyncCallback(BeginReceive_callback_function),
                                 rcv_msg);
-            if (read_sock_err != SocketError.Success) {
+            if (read_sock_err != SocketError.Success)
+            {
               bts.Log.Gen.error($"_begin_or_resume_reading this: {this} error: {read_sock_err} rcv_msg: {rcv_msg}");
               _handle_error(new ErrorMessage(read_sock_err, "_beging_or_resume_reading"));
            }
-         } catch (Exception ex) {
+         }
+         catch (Exception ex)
+         {
            _handle_error(new ErrorMessage(ex, "_beging_or_resume_reading"));
         }
      }

-      private void _log_incoming_msg(IAsyncResult res) {
-         AsyncReceiveMessage rcv_message = res.AsyncState as AsyncReceiveMessage;
-         var buff_txt = AsyncTCPConnection_AsyncWrapper.to_hex(rcv_message.buffer, 0, rcv_message.buffer.Length);
-         Log.Gen.info($"BeginReceive_callback_function _log_incoming_msg buff: {buff_txt}");
+      private void _log_incoming_msg(IAsyncResult res)
+      {
+         // AsyncReceiveMessage rcv_message = res.AsyncState as AsyncReceiveMessage;
+         // var buff_txt = AsyncTCPConnection_AsyncWrapper.to_hex(rcv_message.buffer, 0, rcv_message.buffer.Length);
+         // Log.Gen.info($"BeginReceive_callback_function _log_incoming_msg buff: {buff_txt}");

      }

      // this is the function passed to BeginReceive, we 
-      private void BeginReceive_callback_function(IAsyncResult res) {
+      private void BeginReceive_callback_function(IAsyncResult res)
+      {
         _log_incoming_msg(res);
-         if (res.CompletedSynchronously) {
+         if (res.CompletedSynchronously)
+         {
            ThreadPool.QueueUserWorkItem(_ => _on_read_async(res));
-         } else
+         }
+         else
            _on_read_async(res);
      }

      // this will always be called on the non main-thread. the reason this is on its own thread is that this function ends up
      // calling _begin_or_resume_reading, which in tuns calls _on_read_async. So to not cause a stack overflow, this is broken
      // always called on a different thread than _begin_or_resume_reading
-      private void _on_read_async(IAsyncResult res) {
+      private void _on_read_async(IAsyncResult res)
+      {
         AsyncReceiveMessage rcv_message = res.AsyncState as AsyncReceiveMessage;
-         if (_read_key_map.Contains(rcv_message.key)) {
+         if (_read_key_map.Contains(rcv_message.key))
+         {
            bts.Log.Gen.warn($"AsyncTCPConnection_APM._on_read key: {rcv_message.key} already handled");
            return;
         }
         _read_key_map.Add(rcv_message.key);
         if (!_is_alive)
            return;
-         try {
+         try
+         {
            SocketError read_err;
            Log.Gen.info($"AsyncTCPConnection_APM._on_read pre-EndReceive rcv_message: {rcv_message} res: {IAsyncResult_txt(res)}");
            int bytes_read = _sock.EndReceive(res, out read_err);
            rcv_message.advance_transfer_pointer(bytes_read);
            Log.Gen.info($"AsyncTCPConnection_APM._on_read post-EndReceive rcv_message: {rcv_message} res: {IAsyncResult_txt(res)} bytes_read: {bytes_read}");
-            if (bytes_read == 0) {
+            if (bytes_read == 0)
+            {
               bts.Log.Gen.error($"AsyncTCPConnection_APM._on_read this: {this} read 0 bytes, throwing exception");
               //bts.Log.Gen.error($"AsyncTCPConnection_APM._on_read this: {this} read 0 bytes, sleeping and ignoring");
               //System.Threading.Thread.Sleep(50);
@@ -464,23 +545,32 @@ namespace bts {
               return;
            }
            //return;
-            if (read_err == SocketError.Success) {
-               if (rcv_message.fully_transfered()) {
-                  if (rcv_message.is_reading_size()) {
+            if (read_err == SocketError.Success)
+            {
+               if (rcv_message.fully_transfered())
+               {
+                  if (rcv_message.is_reading_size())
+                  {
                     var msg_size = _on_read_size_completely(rcv_message);
-                     if (msg_size != null && msg_size.HasValue) {
+                     if (msg_size != null && msg_size.HasValue)
+                     {
                        _read_data(msg_size.Value);
                     }
                     return;
-                  } else {
+                  }
+                  else
+                  {
                     // data is read
                     int msg_type = rcv_message.read_int32();
                     bts.Log.assert(msg_type == HB_MSG_SIGNAL || msg_type == DATA_MSG_SIGNAL);
-                     if (msg_type == DATA_MSG_SIGNAL) {
+                     if (msg_type == DATA_MSG_SIGNAL)
+                     {
                        _on_read_data_completely(rcv_message);
                        _read_next_msg();
                        return;
-                     } else if (msg_type == HB_MSG_SIGNAL) {
+                     }
+                     else if (msg_type == HB_MSG_SIGNAL)
+                     {
                        _on_read_hb_completely(rcv_message);
                        _read_next_msg();
                        return;
@@ -488,45 +578,57 @@ namespace bts {
                     // go back to reading size for a new message
                     //_read_size();
                  }
-               } else {
+               }
+               else
+               {
                  // msg has not been fully rd, we have already advanced the transfer ptr
                  _begin_or_resume_reading(rcv_msg: rcv_message);
               }
-            } else {
+            }
+            else
+            {
               _handle_error(new ErrorMessage(read_err, "_on_read_async"));
            }
-         } catch (Exception ex) {
+         }
+         catch (Exception ex)
+         {
            _handle_error(new ErrorMessage(ex, "_on_read_async"));
         }
      }

      #endregion

-      private int? _on_read_size_completely(AsyncReceiveMessage msg) {
+      private int? _on_read_size_completely(AsyncReceiveMessage msg)
+      {
         bts.Log.assert(msg.is_reading_size());
         bts.Log.assert(msg.fully_transfered());
         bts.Log.assert(msg.original_size == 4);
         var msg_sz = msg.read_int32();
-         var buff_txt = AsyncTCPConnection_AsyncWrapper.to_hex(msg.buffer, 0, msg.buffer.Length);
-         bts.Log.Gen.info($"AsyncTCPConnection_APM._on_read_size_completely buff: {buff_txt}");
+         //var buff_txt = AsyncTCPConnection_AsyncWrapper.to_hex(msg.buffer, 0, msg.buffer.Length);
+         //bts.Log.Gen.info($"AsyncTCPConnection_APM._on_read_size_completely buff: {buff_txt}");
         bts.Log.Gen.info($"AsyncTCPConnection_APM._on_read_size_completely msg: {msg} msg_sz: {msg_sz}");
-         if (msg_sz < 0 || msg_sz > _max_msg_size) {
+         if (msg_sz < 0 || msg_sz > _max_msg_size)
+         {
            string err_msg = $"Encountered malformed message, prefixed_size: {msg_sz}";
            _handle_error(new ErrorMessage(new ApplicationException(err_msg), "_on_read_size_completely"));
            return null;
-         } else {
+         }
+         else
+         {
            // sz is determined, now we read that many bytes
            //_read_data(msg_sz);
            return msg_sz;
         }
      }

-      private void _on_read_hb_completely(AsyncReceiveMessage msg) {
+      private void _on_read_hb_completely(AsyncReceiveMessage msg)
+      {
         bts.Log.assert(!msg.is_reading_size());
         bts.Log.assert(msg.fully_transfered());
         bts.Log.assert(msg.original_size == 4);
         bts.Log.assert(msg.read_int32(0) == HB_MSG_SIGNAL);
-         if (!_hb_received_yet) {
+         if (!_hb_received_yet)
+         {
            _hb_received_yet = true;
            //bts.Log.Gen.info($"AsyncTCPConnection_APM HB received for the first time from peer {_peer_conn_info}");
         }
@@ -534,7 +636,8 @@ namespace bts {
         _message_handler.onData(new RemoteMessage(null, RemoteMessageType.Heartbeat, _peer_conn_info));
      }

-      private void _on_read_data_completely(AsyncReceiveMessage msg) {
+      private void _on_read_data_completely(AsyncReceiveMessage msg)
+      {
         bts.Log.assert(!msg.is_reading_size());
         bts.Log.assert(msg.fully_transfered());
         var type = msg.read_int32(0);
@@ -543,7 +646,8 @@ namespace bts {
         byte[] data = msg.read_bytes(4);
         ///callbacks can be made through thread pool to avoid this
         ///thread from being blocked, by this callback
-         if (!_data_received_yet) {
+         if (!_data_received_yet)
+         {
            _data_received_yet = true;
            //bts.Log.Gen.info($"AsyncTCPConnection_APM data received for the first time from peer {_peer_conn_info}");
         }
@@ -553,18 +657,24 @@ namespace bts {

      #region IDisposable Members

-      public void Dispose() {
+      public void Dispose()
+      {
         // socket is closed and disposed here
-         lock (this) {
+         lock (this)
+         {
            bts.Log.Gen.info($"AsyncTCPConnection_APM.Dispose this: {this} Dispose called _dipose_impetus: {_dipose_impetus}");
-            if (_is_alive) {
+            if (_is_alive)
+            {
               _master_worker.removeWorker(this);
               _is_alive = false;
               bts.Log.Gen.info($"AsyncTCPConnection_APM.Dispose this: {this} being disposed, last HB was received at {_last_hb_received_time}");
-               try {
+               try
+               {
                  bts.Log.Gen.info($"AsyncTCPConnection_APM.Dispose this: {this} shutting down socket now");
                  _sock.Shutdown(SocketShutdown.Both);
-               } catch (Exception ex) {
+               }
+               catch (Exception ex)
+               {
                  String.Format("AsyncTCPConnection_APM.Dispose exception: {ex} thrown when closing socket");
                  if (ex is OutOfMemoryException)
                     throw ex;
@@ -577,8 +687,10 @@ namespace bts {
      #endregion

      private string _dipose_impetus = String.Empty;
-      private void _handle_error(ErrorMessage err_msg) {
-         lock (this) {
+      private void _handle_error(ErrorMessage err_msg)
+      {
+         lock (this)
+         {
            Log.Gen.error($"AsyncTCPConnection_APM._handle_error this: {this} error_msg: {err_msg}");
            if (err_msg.oom)
               throw err_msg.exception;
@@ -1,4 +1,5 @@
-namespace bts {
+namespace bts
+{
   using BCG;
   using BCG.Utilities.General;
   using System;
@@ -11,15 +12,19 @@ namespace bts {

   // this ITCPConnection uses the TCPClient class instead of a raw socket (in the hope, that it would configure the socket correctly)

-   public class AsyncTCPConnection_AsyncWrapper : ITCPConnection, IWorker, IDisposable {
-      public class ConnectionID {
+   public class AsyncTCPConnection_AsyncWrapper : ITCPConnection, IWorker, IDisposable
+   {
+      public class ConnectionID
+      {
         public readonly RemoteConnectionInfo self;
         public readonly RemoteConnectionInfo peer;
         public readonly Type conn_type;
-         public override string ToString() {
+         public override string ToString()
+         {
            return $"{conn_type} self: {self} peer: {peer}";
         }
-         public ConnectionID(RemoteConnectionInfo self_, RemoteConnectionInfo peer_, Type conn_type_) {
+         public ConnectionID(RemoteConnectionInfo self_, RemoteConnectionInfo peer_, Type conn_type_)
+         {
            self = self_;
            peer = peer_;
            conn_type = conn_type_;
@@ -33,30 +38,34 @@ namespace bts {
      private bool _is_alive = true;
      ConnectionID _connection_id;

-      public static string to_hex(byte[] byte_arr, int start_pos, int end_pos) {
-         var len = end_pos - start_pos;
-         StringBuilder hex_builder = new StringBuilder(len * 2);
-         int i = 0;
+      // public static string to_hex(byte[] byte_arr, int start_pos, int end_pos) {
+      //    var len = end_pos - start_pos;
+      //    StringBuilder hex_builder = new StringBuilder(len * 2);
+      //    int i = 0;

-         foreach (byte b in byte_arr) {
-            if (i >= start_pos && i < end_pos)
-               hex_builder.Append($"{b:X2} ");
-            i++;
-         }
-         return hex_builder.ToString().Trim();
-      }
+      //    foreach (byte b in byte_arr) {
+      //       if (i >= start_pos && i < end_pos)
+      //          hex_builder.Append($"{b:X2} ");
+      //       i++;
+      //    }
+      //    return hex_builder.ToString().Trim();
+      // }

-      public AsyncTCPConnection_AsyncWrapper(TcpClient tcp_client) {
+      public AsyncTCPConnection_AsyncWrapper(TcpClient tcp_client)
+      {
         _tcp_client = tcp_client;
         _nw_stream = _tcp_client.GetStream();
         _connection_id = new ConnectionID(TCPSocketUtils.get_self_address(_tcp_client.Client), TCPSocketUtils.get_peer_address(_tcp_client.Client), GetType());
         Log.Gen.info("AsyncTCPConnection_AsyncWrapper.cons connection: {_connection_id}");
      }

-      private void _detect_connection_status_change() {
-         Action testing_func = () => {
+      private void _detect_connection_status_change()
+      {
+         Action testing_func = () =>
+         {
            bool connected = _tcp_client.Connected;
-            while (true) {
+            while (true)
+            {
               if (connected != _tcp_client.Connected)
                  Log.Gen.error($"Connection status changed this: {ToString()} {connected} -> {_tcp_client.Connected}");
               connected = _tcp_client.Connected;
@@ -69,14 +78,17 @@ namespace bts {
      }

      #region Sending 
-      public void _send_fixed_size_msg(byte[] buffer) {
-         //Log.Gen.info($"AsyncTCPConnection_AsyncWrapper._send_fixed_size_msg buffer: {to_hex(buffer, 0, buffer.Length)}");
+      public void _send_fixed_size_msg(byte[] buffer)
+      {
+         // string buff_txt = to_hex(buffer, 0, buffer.Length);
+         // Log.Gen.info($"AsyncTCPConnection_AsyncWrapper._send_fixed_size_msg buffer: {buff_txt}");

         int total_bytes_sent = 0;
         int n_bytes = buffer.Length;

         int n_transfers = 0;
-         while (_is_alive && total_bytes_sent < n_bytes) {
+         while (_is_alive && total_bytes_sent < n_bytes)
+         {
            int remaining_bytes = n_bytes - total_bytes_sent;
            _nw_stream.Write(buffer, total_bytes_sent, remaining_bytes);
            total_bytes_sent += remaining_bytes;
@@ -85,8 +97,10 @@ namespace bts {
         Log.Gen.info($"AsyncTCPConnection_AsyncWrapper._send_fixed_size_msg sent {buffer.Length} bytes in {n_transfers} transfers");
      }

-      private void _send_one_msg(byte[] msg) {
-         try {
+      private void _send_one_msg(byte[] msg)
+      {
+         try
+         {
            // the total msg = [msg_sz, msg_type, data] 
            int sz = msg.Length + 4;
            MemoryStream str = new MemoryStream();
@@ -97,15 +111,20 @@ namespace bts {
            str.Position = 0;
            byte[] buff = str.ToArray();
            _send_fixed_size_msg(buff);
-         } catch (Exception ex) {
+         }
+         catch (Exception ex)
+         {
            _handle_error(new ErrorMessage(ex, $"_send_one_msg call_stacK: {ex.StackTrace}"));
         }
      }

-      private void _send_forever() {
-         while (_is_alive) {
+      private void _send_forever()
+      {
+         while (_is_alive)
+         {
            byte[] data = _send_queue.dequeue();
-            if (data != null) {
+            if (data != null)
+            {
               _send_one_msg(data);
            }
         }
@@ -118,39 +137,47 @@ namespace bts {
      }
      #endregion

-      public void register(ITCPMessageHandler message_handler) {
+      public void register(ITCPMessageHandler message_handler)
+      {
         _msg_handler = message_handler;
         // starting thread pool threads to keep reading and sending
         ThreadPool.QueueUserWorkItem(_ => _read_msgs_forever());
         ThreadPool.QueueUserWorkItem(_ => _send_forever());
      }

-      public RemoteConnectionInfo get_peer_address() {
+      public RemoteConnectionInfo get_peer_address()
+      {
         return _connection_id.peer;
      }

-      public RemoteConnectionInfo get_self_address() {
+      public RemoteConnectionInfo get_self_address()
+      {
         return _connection_id.self;
      }

      public bool isDisposed { get { return !_is_alive; } }

-      public void doWork() {
+      public void doWork()
+      {
         // send hb
      }

      #region Reading
      // This makes the blocking IO call, the callstack is _read_msgs_forever -> _read_one_message -> _read_fixed_size_msg
-      private byte[] _read_fixed_size_msg(int n_bytes) {
-         if (n_bytes <= 0) {
+      private byte[] _read_fixed_size_msg(int n_bytes)
+      {
+         if (n_bytes <= 0)
+         {
            throw new ArgumentException("Number of bytes to read must be greater than zero.", nameof(n_bytes));
         }
         byte[] buffer = new byte[n_bytes];
         int total_bytes_read = 0;
         int n_transfers = 0;
-         while (_is_alive && total_bytes_read < n_bytes) {
+         while (_is_alive && total_bytes_read < n_bytes)
+         {
            int bytes_read = _nw_stream.Read(buffer, total_bytes_read, n_bytes - total_bytes_read);
-            if (bytes_read <= 0) {
+            if (bytes_read <= 0)
+            {
               throw new IOException("Read 0 bytes, the connection was closed before all data could be read.");
            }
            n_transfers++;
@@ -161,37 +188,44 @@ namespace bts {
         return buffer;
      }

-      private class OneRecvMessage {
+      private class OneRecvMessage
+      {
         public int msg_type;
         public byte[] data;
-         public OneRecvMessage(int msg_type_, byte[] data_) {
+         public OneRecvMessage(int msg_type_, byte[] data_)
+         {
            msg_type = msg_type_;
            data = data_;
         }
-         public OneRecvMessage(byte[] msg) {
+         public OneRecvMessage(byte[] msg)
+         {
            MemoryStream str = new MemoryStream(msg);
            BinaryReader br = new BinaryReader(str);
            int msg_type = br.ReadInt32();
-            if (msg_type == DATA_MSG_SIGNAL) {
+            if (msg_type == DATA_MSG_SIGNAL)
+            {
               // TODO, polish it, too much memory bneing allocated
               data = br.ReadBytes(msg.Length - 4);
            }
         }

-         public override string ToString() {
+         public override string ToString()
+         {
            string rv = $"OneMessage msg_type: {msg_type} data_len: {data.Length}";
            return rv;
         }
      }

-      private OneRecvMessage _read_one_message() {
+      private OneRecvMessage _read_one_message()
+      {
         // read size first
         byte[] sz_arr = _read_fixed_size_msg(4);
         BinaryReader br = new BinaryReader(new MemoryStream(sz_arr));
         br.BaseStream.Position = 0;
         int msg_sz = br.ReadInt32();
         Log.Gen.info($"AsyncTCPConnection_AsyncWrapper._read_one_message read msg_sz: {msg_sz}");
-         if (msg_sz < 0) {
+         if (msg_sz < 0)
+         {
            throw new IOException("Read 0 bytes, the connection was closed before all data could be read.");
         }
         byte[] msg = _read_fixed_size_msg(msg_sz);
@@ -201,19 +235,29 @@ namespace bts {
         return rv;
      }

-      private void _read_msgs_forever() {
-         while (_is_alive) {
-            try {
+      private void _read_msgs_forever()
+      {
+         while (_is_alive)
+         {
+            try
+            {
               OneRecvMessage one_msg = _read_one_message();
-               if (one_msg.msg_type == DATA_MSG_SIGNAL) {
+               if (one_msg.msg_type == DATA_MSG_SIGNAL)
+               {
                  _msg_handler.onData(new RemoteMessage(one_msg.data, RemoteMessageType.Data, _connection_id.peer));
-               } else if (one_msg.msg_type == HB_MSG_SIGNAL) {
+               }
+               else if (one_msg.msg_type == HB_MSG_SIGNAL)
+               {
                  // keep a last hb time an dudpate it here atomically
                  _msg_handler.onData(new RemoteMessage(null, RemoteMessageType.Heartbeat, _connection_id.peer));
-               } else {
+               }
+               else
+               {
                  throw new ApplicationException($"Unknown message type: {one_msg.msg_type}");
               }
-            } catch (Exception ex) {
+            }
+            catch (Exception ex)
+            {
               _handle_error(new ErrorMessage(ex, $"_read_msgs_forever, call_stack: {ex.StackTrace}"));
            }
         }
@@ -223,16 +267,19 @@ namespace bts {

      public bool is_disposed => !_is_alive;

-      public class ErrorMessage {
+      public class ErrorMessage
+      {
         public readonly SocketError? socket_error = null;
         public readonly Exception exception = null;
         public readonly bool is_exception = false;
         public readonly string context = String.Empty;
-         public ErrorMessage(SocketError socket_error_, string context_) {
+         public ErrorMessage(SocketError socket_error_, string context_)
+         {
            socket_error = socket_error_;
            context = context_;
         }
-         public ErrorMessage(Exception exception_, string context_) {
+         public ErrorMessage(Exception exception_, string context_)
+         {
            is_exception = true;
            exception = exception_;
            context = context_;
@@ -240,23 +287,30 @@ namespace bts {

         public bool oom => exception != null && exception is OutOfMemoryException;

-         public override string ToString() {
+         public override string ToString()
+         {
            if (is_exception)
               return $"ErrorMessage exception: {exception} context: {context}";
            return $"socket_error: {socket_error} context: {context}";
         }
      }

-      public void Dispose() {
+      public void Dispose()
+      {
         // socket is closed and disposed here
-         lock (this) {
-            if (_is_alive) {
+         lock (this)
+         {
+            if (_is_alive)
+            {
               //_master_worker.removeWorker(this);
               _is_alive = false;
-               try {
+               try
+               {
                  bts.Log.Gen.info($"AsyncTCPConnection_APM.Dispose this: {this} closing tcp_client nin an orderly fashion now");
                  _tcp_client.Close();
-               } catch (Exception ex) {
+               }
+               catch (Exception ex)
+               {
                  String.Format("AsyncTCPConnection_APM.Dispose exception: {ex} thrown when closing socket");
                  if (ex is OutOfMemoryException)
                     throw ex;
@@ -266,8 +320,10 @@ namespace bts {
         }
      }

-      private void _handle_error(ErrorMessage err_msg) {
-         lock (this) {
+      private void _handle_error(ErrorMessage err_msg)
+      {
+         lock (this)
+         {
            Log.Gen.error($"AsyncTCPConnection_APM._handle_error this: {this} error_msg: {err_msg}");
            if (err_msg.oom)
               throw err_msg.exception;
@@ -275,7 +331,8 @@ namespace bts {
         }
      }

-      public override string ToString() {
+      public override string ToString()
+      {
         return _connection_id.ToString();
      }
   }
@@ -285,4 +285,4 @@ namespace BCG {



-}
+}
@@ -1,126 +1,126 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
+// using System;
+// using System.Collections.Generic;
+// using System.Linq;
+// using System.Text;

-/*
- *  This class contains the recent channges of state of a single pubsub tag, with respect to who has subscribed, who has published, etc. 
- *  It has a list of new subscription events. 
- * 
- */
-namespace BCG {
-   public class PubSubSingleTagState {
-      private string myTag;
-      public string tag {
-         get { return myTag; }
-         set { myTag = value; }
+// /*
+//  *  This class contains the recent channges of state of a single pubsub tag, with respect to who has subscribed, who has published, etc. 
+//  *  It has a list of new subscription events. 
+//  * 
+//  */
+// namespace BCG {
+//    public class PubSubSingleTagState {
+//       private string myTag;
+//       public string tag {
+//          get { return myTag; }
+//          set { myTag = value; }

-      }
-      //we have this here as SortedList so that we can compare for equality
-      private SortedList<string, string> myPublishers;
-      private SortedList<string, string> mySubscribers;
-      private Dictionary<string, PubSubSingleTagStateEvent> myNewSubscribers;
-      private Dictionary<string, PubSubSingleTagStateEvent> myRemovedSubscribers;
-      private Dictionary<string, PubSubSingleTagStateEvent> myNewPublishers;
-      private Dictionary<string, PubSubSingleTagStateEvent> myRemovedPublishers;
+//       }
+//       //we have this here as SortedList so that we can compare for equality
+//       private SortedList<string, string> myPublishers;
+//       private SortedList<string, string> mySubscribers;
+//       private Dictionary<string, PubSubSingleTagStateEvent> myNewSubscribers;
+//       private Dictionary<string, PubSubSingleTagStateEvent> myRemovedSubscribers;
+//       private Dictionary<string, PubSubSingleTagStateEvent> myNewPublishers;
+//       private Dictionary<string, PubSubSingleTagStateEvent> myRemovedPublishers;



-      public PubSubSingleTagState() {
-         //the total state of the tag
-         myPublishers = new SortedList<string, string>();
-         mySubscribers = new SortedList<string, string>();
+//       public PubSubSingleTagState() {
+//          //the total state of the tag
+//          myPublishers = new SortedList<string, string>();
+//          mySubscribers = new SortedList<string, string>();

-         //how has the state changed recentlty
-         myNewSubscribers = new Dictionary<string, PubSubSingleTagStateEvent>();
-         myNewPublishers = new Dictionary<string, PubSubSingleTagStateEvent>();
-         myRemovedPublishers = new Dictionary<string, PubSubSingleTagStateEvent>();
-         myRemovedSubscribers = new Dictionary<string, PubSubSingleTagStateEvent>();
-      }
+//          //how has the state changed recentlty
+//          myNewSubscribers = new Dictionary<string, PubSubSingleTagStateEvent>();
+//          myNewPublishers = new Dictionary<string, PubSubSingleTagStateEvent>();
+//          myRemovedPublishers = new Dictionary<string, PubSubSingleTagStateEvent>();
+//          myRemovedSubscribers = new Dictionary<string, PubSubSingleTagStateEvent>();
+//       }

-      //returns a list of all the events that need to be published.
-      public PubSubSingleTagStateEvent[] getNewEventsForPublishing() {
-         PubSubSingleTagStateEvent[] newEvents = new PubSubSingleTagStateEvent[myNewSubscribers.Count + myNewPublishers.Count + myRemovedPublishers.Count + myRemovedSubscribers.Count];
+//       //returns a list of all the events that need to be published.
+//       public PubSubSingleTagStateEvent[] getNewEventsForPublishing() {
+//          PubSubSingleTagStateEvent[] newEvents = new PubSubSingleTagStateEvent[myNewSubscribers.Count + myNewPublishers.Count + myRemovedPublishers.Count + myRemovedSubscribers.Count];

-         myNewSubscribers.Values.CopyTo(newEvents, 0);
-         int index = myNewSubscribers.Count;
-         myNewPublishers.Values.CopyTo(newEvents, index);
-         index += myNewPublishers.Count;
-         myRemovedSubscribers.Values.CopyTo(newEvents, index);
-         index += myRemovedSubscribers.Count;
-         myRemovedPublishers.Values.CopyTo(newEvents, index);
+//          myNewSubscribers.Values.CopyTo(newEvents, 0);
+//          int index = myNewSubscribers.Count;
+//          myNewPublishers.Values.CopyTo(newEvents, index);
+//          index += myNewPublishers.Count;
+//          myRemovedSubscribers.Values.CopyTo(newEvents, index);
+//          index += myRemovedSubscribers.Count;
+//          myRemovedPublishers.Values.CopyTo(newEvents, index);

-         return newEvents;
+//          return newEvents;

-      }
+//       }

-      //call this after you have published the new events
-      //this is made public for code readability reasons 
-      //i.e. so that someone reading the SubInfoStateWatcher can understand what's going on more easily
-      public void resetNewEvents() {
-         myNewSubscribers = new Dictionary<string, PubSubSingleTagStateEvent>();
-         myNewPublishers = new Dictionary<string, PubSubSingleTagStateEvent>();
-         myRemovedSubscribers = new Dictionary<string, PubSubSingleTagStateEvent>();
-         myRemovedPublishers = new Dictionary<string, PubSubSingleTagStateEvent>();
-      }
+//       //call this after you have published the new events
+//       //this is made public for code readability reasons 
+//       //i.e. so that someone reading the SubInfoStateWatcher can understand what's going on more easily
+//       public void resetNewEvents() {
+//          myNewSubscribers = new Dictionary<string, PubSubSingleTagStateEvent>();
+//          myNewPublishers = new Dictionary<string, PubSubSingleTagStateEvent>();
+//          myRemovedSubscribers = new Dictionary<string, PubSubSingleTagStateEvent>();
+//          myRemovedPublishers = new Dictionary<string, PubSubSingleTagStateEvent>();
+//       }

-      public void newPublish(PubSubSingleTagStateEvent newPublisher) {
-         //don't do anything here if we already knew about the publishers because we were already dealing with this publishers
-         if (!myPublishers.ContainsKey(newPublisher.app)) {
-            myPublishers.Add(newPublisher.app, newPublisher.app);
+//       public void newPublish(PubSubSingleTagStateEvent newPublisher) {
+//          //don't do anything here if we already knew about the publishers because we were already dealing with this publishers
+//          if (!myPublishers.ContainsKey(newPublisher.app)) {
+//             myPublishers.Add(newPublisher.app, newPublisher.app);

-            //if we were going to delete it, that means it was already being published by this app
-            //that means we should no logner delete it, but we don't have to announce that it was published again
-            if (myRemovedPublishers.ContainsKey(newPublisher.app))
-               myRemovedPublishers.Remove(newPublisher.app);
-            else
-               myNewPublishers[newPublisher.app] = newPublisher;
+//             //if we were going to delete it, that means it was already being published by this app
+//             //that means we should no logner delete it, but we don't have to announce that it was published again
+//             if (myRemovedPublishers.ContainsKey(newPublisher.app))
+//                myRemovedPublishers.Remove(newPublisher.app);
+//             else
+//                myNewPublishers[newPublisher.app] = newPublisher;


-         }
-      }
+//          }
+//       }

-      public void removePublisher(PubSubSingleTagStateEvent removedPublisher) {
-         if (myPublishers.ContainsKey(removedPublisher.app)) {
-            myPublishers.Remove(removedPublisher.app);
+//       public void removePublisher(PubSubSingleTagStateEvent removedPublisher) {
+//          if (myPublishers.ContainsKey(removedPublisher.app)) {
+//             myPublishers.Remove(removedPublisher.app);

-            //same kind of logic here. If we were about to publish this new publisher, we no longer need to announce it
-            if (myNewPublishers.ContainsKey(removedPublisher.app))
-               myNewPublishers.Remove(removedPublisher.app);
-            else
-               myRemovedPublishers[removedPublisher.app] = removedPublisher;
-         }
+//             //same kind of logic here. If we were about to publish this new publisher, we no longer need to announce it
+//             if (myNewPublishers.ContainsKey(removedPublisher.app))
+//                myNewPublishers.Remove(removedPublisher.app);
+//             else
+//                myRemovedPublishers[removedPublisher.app] = removedPublisher;
+//          }

-      }
+//       }

-      public void newSubscription(PubSubSingleTagStateEvent newSubscriber) {
-         if (!mySubscribers.ContainsKey(newSubscriber.app)) {
-            mySubscribers.Add(newSubscriber.app, newSubscriber.app);
+//       public void newSubscription(PubSubSingleTagStateEvent newSubscriber) {
+//          if (!mySubscribers.ContainsKey(newSubscriber.app)) {
+//             mySubscribers.Add(newSubscriber.app, newSubscriber.app);

-            if (myRemovedSubscribers.ContainsKey(newSubscriber.app))
-               myRemovedSubscribers.Remove(newSubscriber.app);
-            else
-               myNewSubscribers[newSubscriber.app] = newSubscriber;
+//             if (myRemovedSubscribers.ContainsKey(newSubscriber.app))
+//                myRemovedSubscribers.Remove(newSubscriber.app);
+//             else
+//                myNewSubscribers[newSubscriber.app] = newSubscriber;


-         }
-      }
+//          }
+//       }

-      public void removeSubscriber(PubSubSingleTagStateEvent removedSubscriber) {
-         if (mySubscribers.ContainsKey(removedSubscriber.app)) {
-            mySubscribers.Remove(removedSubscriber.app);
-            //if we were going to add this subscriber, we no logner want to
-            //this means no event needs to be published
-            if (myNewSubscribers.ContainsKey(removedSubscriber.app))
-               myNewSubscribers.Remove(removedSubscriber.app);
-            else
-               //we have to remove this guy in this updated
-               myRemovedSubscribers[removedSubscriber.app] = removedSubscriber;
-         }
+//       public void removeSubscriber(PubSubSingleTagStateEvent removedSubscriber) {
+//          if (mySubscribers.ContainsKey(removedSubscriber.app)) {
+//             mySubscribers.Remove(removedSubscriber.app);
+//             //if we were going to add this subscriber, we no logner want to
+//             //this means no event needs to be published
+//             if (myNewSubscribers.ContainsKey(removedSubscriber.app))
+//                myNewSubscribers.Remove(removedSubscriber.app);
+//             else
+//                //we have to remove this guy in this updated
+//                myRemovedSubscribers[removedSubscriber.app] = removedSubscriber;
+//          }


-      }
+//       }


-   }
-}
+//    }
+// }
@@ -0,0 +1,61 @@
+{
+   "user": "bts",
+   "pubsub": {
+      "milliseconds_between_sub_info_updates": 1000,
+      "milliseconds_between_time_slice_updates": 5000,
+      "milliseconds_between_no_buffer_space_retries": 100,
+      "retries_on_no_buffer_space": 1,
+      "do_it_all_seconds_between_logs": 0,
+      "heartbeat_interval": 15000,
+      "missing_heartbeats_before_gc": 50,
+      "use_managed_sockets": true,
+      "state_update_period": 5000,
+      "statistics_update_period": 5000,
+      "disposed_connection_check_period": 1000,
+      "message_processing_warning_threshold": 1000,
+      "server_port": 12345,
+      "publisher_routes": [
+         {
+            "name": "default",
+            "server_address": "bts-cloud://192.168.1.80:12345",
+            "tag": ".*"
+         }
+      ],
+      "subscriber_routes": [
+         {
+            "name": "default",
+            "server_address": "bts-cloud://192.168.1.80:12345",
+            "tag": ".*"
+         }
+      ],
+      "max_packet_size": 500000000,
+      "blue_comm_must_die": true,
+      "crushable_tags": [],
+      "tag_type_to_assembly_map": [
+         {
+            "handler_class": "BCG.PubSubServerTagHandler",
+            "assembly": "DataStreamCore.dll"
+         }
+      ],
+      "server_queue": "PubSubQueue",
+      "port_range": {
+         "first": 30400,
+         "last": 31700
+      },
+      "bind_to_random_port": false,
+      "debug_tags": "^NOTAGS$",
+      "latency_report_threshold_ms": 1500,
+      "use_substations": false,
+      "use_dynamic_clients": false,
+      "is_substation": false,
+      "internally_publish_sub_info": true,
+      "use_managed_clients_in_native_code": false,
+      "compressed_tags": "^NOTAGS$",
+      "tags_to_report_compression_ratio": "^NOTAGS$",
+      "disable_msmq": false,
+      "use_prepublished_server_state": false,
+      "ip_translation": null,
+      "gc_activation_threshold": 0,
+      "wait_period_after_gc": 1000
+   }
+}
@@ -7,26 +7,45 @@ namespace bts {
   using System.Text.Json;
   using System.Text.Json.Nodes;
   [DataStreamUserRespresentation("benzingaNews")]
-   public class BenzingaNewStream : ViewStream {
+   public class BenzingaNewsStream : ViewStream {
      private IBlueTableSchema _schema = null;
      private int _id = 0;
-      // constructor takes in the raw mongo db stream, and the time to live (ttl), "7d" for example
-      public BenzingaNewStream(ViewStream mongodb_raw_stream, string ttl) {
+
+
+      private static ViewStream _pre_process_raw_kvstream(ViewStream mongo_db_str) {
+         ViewStream r = new KeyValueToRelationalStream(mongo_db_str, "doc_id", "key", "value");
+         r = new DateStandardizationStream(r, "created_at", "ISO8601");
+         return r;
+      }
+
+      public BenzingaNewsStream(string connection_string, string db_name, string collection_name, string poll_period, string ttl)
+      : this(new MongoDBStream(connection_string, db_name, collection_name, poll_period), ttl) {
+      }
+
+      // public BenzingaNewsStream()
+      // : this(new MongoDBStream(DataStreamPub connection_string, db_name, collection_name, poll_period), ttl) {
+      // }
+
+      // assumes it gets a raw mongodb_raw_kv_stream with key value pairs
+      public BenzingaNewsStream(ViewStream mongodb_raw_kv_stream, string ttl) {
+         ViewStream output_stream = _pre_process_raw_kvstream(mongodb_raw_kv_stream);
         // Hiding body (with project, columns can be hidden, or renamed)
-         ViewStream output_stream = new ProjectStream(mongodb_raw_stream, "!body");
+         output_stream = new ProjectStream(output_stream, "!body");
         // add column news_source = 'Benzinga', so we can append other news sources
         output_stream = new AddColumnsStream(output_stream, @"news_source : Benzinga");
         // now SuperMapStream is used to map one row into multiple, one per symbol,
         // (SuperMapStream takes 2 lambdas, one to get the enw schema and another to 
         // return an array of rows, per one row)
-         output_stream = new SuperMapStream(mongodb_raw_stream,
+         output_stream = new SuperMapStream(output_stream,

        (IBlueTableSchema s) => {
           // this lambda returns the new schema (ie with extra cols symbol, exchange, is_primary_exchange)
           if (_schema == null) {
              var out_schema = s;
              out_schema = BlueDataUtility.remove_columns(out_schema, "securities");
-              out_schema = BlueDataUtility.add_column(s, "doc_id_internal", typeof(string));
+              out_schema = BlueDataUtility.remove_columns(out_schema, "body");
+              out_schema = BlueDataUtility.remove_columns(out_schema, "tags");
+              out_schema = BlueDataUtility.add_column(out_schema, "doc_id_internal", typeof(string));
              out_schema = BlueDataUtility.add_column(out_schema, "symbol", typeof(string));
              out_schema = BlueDataUtility.add_column(out_schema, "exchange", typeof(string));
              out_schema = BlueDataUtility.add_column(out_schema, "is_primary_exchange", typeof(bool));
@@ -44,11 +63,13 @@ namespace bts {
           }
           foreach (JsonNode jnode in json_objs) {
              if (jnode is JsonObject json_object) {
+                 string symbol = jnode["symbol"] == null ? String.Empty : jnode["symbol"].ToString();
+                 string exchange = jnode["exchange"] == null ? String.Empty : jnode["exchange"].ToString();
                 MutableBlueRow nr = new MutableBlueRow(_schema, r);
                 nr.setValue("id", _id++);
                 nr.setValue("doc_id_internal", $"{r["doc_id"]}_{jnode["symbol"]}");
-                 nr.setValue("symbol", jnode["symbol"]);
-                 nr.setValue("exchange", jnode["exchange"]);
+                 nr.setValue("symbol", symbol);
+                 nr.setValue("exchange", exchange);
                 nr.setValue("is_primary_exchange", Boolean.Parse(jnode["primary"].ToString()));
                 expanded_rows.Add(nr);
              }
@@ -0,0 +1,33 @@
+namespace bts {
+   using BCG;
+
+   using System.Collections.Generic;
+   using System.Linq;
+   using System.Security.Cryptography;
+
+   //[DataStreamUserRespresentation("benzingaNewsSymbols")]
+   public class BenzingaNewsSymbolStateStream : ViewStream, INonIncGroupByEvaluator {
+      public BenzingaNewsSymbolStateStream(ViewStream news_str) {
+         ViewStream r = new NonIncGroupByStream(news_str, this);
+         mySetRealSource(r);
+      }
+      public BenzingaNewsSymbolStateStream(string connection_string, string db_name, string collection_name, string poll_period, string ttl) :
+      this(new BenzingaNewsStream(connection_string, db_name, collection_name, poll_period, ttl)) {
+      }
+
+      public IBlueRowCollection evaluate(IDictionary<int, List<IBlueRow>> grouped_rows) {
+         BlueTable rv = null;
+         foreach (var kv in grouped_rows) {
+            var symbol_rows = kv.Value;
+            IBlueRow state_row = BlueDataUtility.find_max(symbol_rows, "created_at");
+            rv = rv == null ? new BlueTable(state_row.schema) : rv;
+            rv.addRow(state_row);
+         }
+         return rv;
+      }
+
+      public string[] get_group_by_cols() {
+         return new string[] { "symbol" };
+      }
+   }
+}
@@ -0,0 +1,14 @@
+namespace bts {
+   using BCG;
+   [DataStreamUserRespresentation("benzingaNewsSymbols")]
+
+   public class BenzingaNewsSymbolsStream : ViewStream {
+      public BenzingaNewsSymbolsStream(ViewStream news_str) {
+         ViewStream rs = new DistinctRowStream(news_str, "symbol");
+         mySetRealSource(rs);
+      }
+      public BenzingaNewsSymbolsStream(string connection_string, string db_name, string collection_name, string poll_period, string ttl) :
+      this(new BenzingaNewsStream(connection_string, db_name, collection_name, poll_period, ttl)) {
+      }
+   }
+}
@@ -0,0 +1,252 @@
+using bts.SilverPine;
+
+namespace bts.SilverPine {
+   using bts.api;
+   using bts.spark;
+
+   using System;
+
+   using ZstdSharp.Unsafe;
+
+   public static class ContractUtils {
+      public class FutureTradableContract {
+         public string symbol;
+         public DateTime expiration;
+         public Exchange exchange;
+         public string contract_text;
+      }
+      public class FutureSpreadContract {
+         public List<Tuple<double, bts.SilverPine.ContractUtils.FutureTradableContract>> legs;
+         public string symbol {
+            get {
+               return legs == null || legs.Count == 0 ? String.Empty : legs.ElementAt(0).Item2.symbol;
+            }
+         }
+         public string contract_text;
+      }
+      public class FutureContinousContract {
+         public string symbol;
+         public Exchange exchange;
+      }
+      // this is for all future conytracts belonging to a symbol
+      public class FutureWildcardContract {
+         public string symbol;
+         public Exchange exchange;
+      }
+      public class FutureContract {
+         public FutureWildcardContract wildcard_contract = null;
+         public FutureTradableContract future_tradable_contract = null;
+         public FutureSpreadContract spread_tradable_contract = null;
+         public FutureContinousContract continous_constract = null;
+
+         public FutureContract(string txt) {
+            wildcard_contract = parse_wildcard_contract(txt);
+            if (wildcard_contract != null)
+               return;
+            FutureTradableContract fc = parse_future_contract(txt);
+            if (fc != null) {
+               future_tradable_contract = fc;
+               return;
+            }
+            FutureSpreadContract sc = parse_future_spread_contract(txt);
+            if (sc != null) {
+               spread_tradable_contract = sc;
+               return;
+            }
+            // this is not totally true, since ParseFutureSpreadContract does not
+            // parse ICS spreads
+            continous_constract = new FutureContinousContract() { symbol = txt };
+         }
+         public bool is_continous_contract => continous_constract != null;
+         public string symbol() {
+            if (wildcard_contract != null)
+               return wildcard_contract.symbol;
+            if (continous_constract != null) {
+               return continous_constract.symbol;
+            } else if (future_tradable_contract != null) {
+               return future_tradable_contract.symbol;
+            } else if (spread_tradable_contract != null) {
+               return spread_tradable_contract.symbol;
+            }
+            Log.assert(false);
+            return String.Empty;
+         }
+         public string contract_text {
+            get {
+               if (future_tradable_contract != null)
+                  return future_tradable_contract.contract_text;
+               if (spread_tradable_contract != null)
+                  return spread_tradable_contract.contract_text;
+               return null;
+            }
+         }
+      }
+      #region Parsing
+      public static FutureWildcardContract parse_wildcard_contract(string contract_txt, Exchange exchange = Exchange.CME) {
+         if (contract_txt.EndsWith("*")) {
+            contract_txt = contract_txt.Replace("*", String.Empty);
+            return new FutureWildcardContract() {
+               symbol = contract_txt,
+               exchange = exchange
+            };
+         }
+         return null;
+      }
+      public static FutureTradableContract parse_future_contract(string contract_txt, Exchange exchange = Exchange.CME) {
+         // Validate the future string length
+         if (!contract_txt.All(c => Char.IsLetterOrDigit(c)))
+            return null;
+         if (contract_txt.Length < 4) {
+            return null;
+         }
+
+         // Extract the symbol and the expiration month/year codes
+         string symbol = contract_txt.Substring(0, contract_txt.Length - 2);
+         char monthCode = contract_txt[contract_txt.Length - 2];
+         char yearCode = contract_txt[contract_txt.Length - 1];
+
+         // Map month code to month, return null if the month code is invalid
+         int month = monthCode switch {
+            'F' => 1,  // January
+            'G' => 2,  // February
+            'H' => 3,  // March
+            'J' => 4,  // April
+            'K' => 5,  // May
+            'M' => 6,  // June
+            'N' => 7,  // July
+            'Q' => 8,  // August
+            'U' => 9,  // September
+            'V' => 10, // October
+            'X' => 11, // November
+            'Z' => 12, // December
+            _ => 0 // Return 0 for invalid month code
+         };
+
+         if (month == 0) {
+            return null; // Invalid month code
+         }
+
+         // Calculate the year based on the current decade
+         int currentYear = DateTime.Now.Year;
+         int decadeStart = currentYear / 10 * 10;
+         int year = decadeStart + (yearCode - '0');
+
+         // If the year is in the past, assume the next decade
+         if (year < currentYear) {
+            year += 10;
+         }
+
+         // Create the FutureContract instance and set its properties
+         DateTime expiration = new DateTime(year, month, 1);
+         var futureContract = new FutureTradableContract {
+            symbol = symbol,
+            expiration = expiration,
+            exchange = exchange,
+            contract_text = contract_txt
+         };
+
+         return futureContract;
+      }
+      // Method to parse a futures spread, e.g. "CLX4-CLZ4"
+      public static FutureSpreadContract parse_future_spread_contract(string contract_txt, Exchange exchange = Exchange.CME) {
+         // Split the string by '-' to handle each leg
+         var spreadParts = contract_txt.Split('-');
+         if (spreadParts.Length < 2) {
+            return null; // Not a valid spread format
+         }
+
+         var spreadContract = new FutureSpreadContract { legs = new List<Tuple<double, FutureTradableContract>>() };
+
+         foreach (var part in spreadParts) {
+            // Split each leg by '*', assuming the format is weight*future or just future
+            string[] legParts = part.Split('*');
+            double weight = 1.0;
+            string futurePart;
+
+            if (legParts.Length == 2) {
+               if (!double.TryParse(legParts[0], out weight)) {
+                  return null; // Invalid weight, parsing failed
+               }
+               futurePart = legParts[1];
+            } else if (legParts.Length == 1) {
+               futurePart = legParts[0];
+            } else {
+               return null; // Invalid format, too many '*' parts
+            }
+
+            // Parse the future contract
+            var futureContract = parse_future_contract(futurePart, exchange);
+            if (futureContract == null) {
+               return null; // Failed to parse the future part
+            }
+
+            // Add the leg to the spread contract
+            spreadContract.legs.Add(new Tuple<double, FutureTradableContract>(weight, futureContract));
+         }
+         return spreadContract;
+      }
+
+      // public static ContractType DetermineFutureOrSpread(string contract) {
+      //    if (contract.Contains("-")) {
+      //       return ContractType.FutureSpread;
+      //    } else {
+      //       return ContractType.Future;
+      //    }
+      // }
+      #endregion
+
+      public static ContractType get_type(string contract_txt) {
+         FutureTradableContract fc = parse_future_contract(contract_txt);
+         if (fc != null) {
+            return ContractType.Future;
+         }
+         FutureSpreadContract sc = parse_future_spread_contract(contract_txt);
+         if (sc != null) {
+            return ContractType.FutureSpread;
+         }
+         // tehre are some non calendar spreads we cant parse
+         //Log.assert(false);
+         return bts.SilverPine.ContractUtils.ContractType.FutureSpread;
+      }
+
+      public static ContractCategory get_category(string symbol) {
+         var tp = get_symbol_category(symbol);
+         return tp.Item1;
+      }
+
+      public static Tuple<ContractCategory, string> get_symbol_category(string symbol) {
+         ContractUtils.FutureTradableContract fc = parse_future_contract(symbol);
+         if (fc != null) {
+            return Tuple.Create(ContractCategory.FutureString, fc.symbol);
+         }
+         var sc = parse_future_spread_contract(symbol);
+         if (sc != null) {
+            return Tuple.Create(ContractCategory.FutureString, sc.symbol);
+         }
+         return Tuple.Create(ContractCategory.Symbol, symbol);
+      }
+
+
+
+      public enum ContractType {
+         Spot,
+         Option,
+         Future,
+         FutureSpread,
+         OptionSpread
+      }
+      public enum ContractCategory {
+         Symbol = 0,
+         FutureString,
+         SpreadString
+      }
+      public enum CallPut {
+         Call,
+         Put
+      }
+      public enum Exchange {
+         CME
+      }
+
+   }
+}
@@ -0,0 +1,23 @@
+namespace bts.SilverPine {
+   using BCG;
+
+   // Finds the regions around specified events 
+   [DataStreamUserRespresentation("EventsPriceAction")]
+   public class EventsPriceActionStream : ViewStream {
+      public EventsPriceActionStream(string symbol, string event_type, string pre_event_period,
+            string post_event_period, int n_events) :
+         this(symbol, event_type, TimeBasedFilter.parse_duration(pre_event_period), TimeBasedFilter.parse_duration(post_event_period),
+         n_events, ViewStreamDatabase.instance) { }
+
+      public EventsPriceActionStream(string symbol, string event_type, TimeSpan pre_event_period,
+      TimeSpan post_event_period, int n_events, ViewStreamDatabase vsd) {
+         // TODO: figure out which bars to use, 1 min or 1 day or live, based on pre_event_period / post_event_period
+         FuturesActiveContractStream olhcv_str = new FuturesActiveContractStream(symbol);
+         ViewStream events_str = new EventsStream(vsd, event_type);
+         // todo
+         //events_str = new TopStream(events_str, n_events, "-ts");
+         ViewStream rs = new TimeProximityJoinStream(olhcv_str, "ts_event", events_str, "ts", pre_event_period, post_event_period);
+         mySetRealSource(rs);
+      }
+   }
+}
@@ -0,0 +1,20 @@
+namespace bts.SilverPine {
+   using BCG;
+
+   public class IndicatorStream : ViewStream {
+      enum Indicator {
+         StandardDeviation = 0
+      }
+      public class IndicatorInfo {
+         Indicator indicator;
+         IDictionary<string, object> parameters;
+      }
+      public IndicatorStream(string symbol_name, string indicator_name, string indicator_params_json) {
+
+      }
+      public IndicatorStream(string symbol_name, IndicatorInfo info) {
+
+      }
+
+   }
+}
@@ -0,0 +1,57 @@
+namespace bts.SilverPine {
+   using BCG;
+
+   using ZstdSharp.Unsafe;
+
+   [DataStreamUserRespresentation("AbsVolume")]
+   // Calculates the volume within a time window (this does not do rates, like highest daily volume etc,
+   // but with some thought can perhaps be extended)
+   public class AbsoluteVolumeStream : ViewStream {
+      // TODO, abstract this out
+      // adds contract_type to OHLCV stream
+      private static ViewStream _get_ohlcv_steam(string symbol, DateTime d1, DateTime d2) {
+         var ohlcv_params = OHLCVStream.get_params(d1, d2);
+         ViewStream rs = new OHLCVStream(symbol, ohlcv_params);
+         Func<IBlueTableSchema, IBlueTableSchema> create_schema =
+         s => BlueDataUtility.add_columns(s, Tuple.Create(typeof(string), "contract_type"));
+         rs = new SuperMapStream(rs,
+         s => create_schema(s),
+         r => {
+            IBlueTableSchema out_schema = create_schema(r.schema);
+            string contract_txt = r.get<string>("contract");
+            var contr_type = ContractUtils.get_type(contract_txt);
+            MutableBlueRow out_row = new MutableBlueRow(out_schema, r);
+            out_row.setValue("contract_type", contr_type.ToString());
+            return new IBlueRow[] { out_row };
+         });
+         return rs;
+      }
+
+      public AbsoluteVolumeStream(string symbol, string ctype, DateTime d1)
+      : this(symbol, ctype, d1, d1) { }
+
+      public AbsoluteVolumeStream(string symbol, string ctype, DateTime d1, DateTime d2)
+      : this(symbol, Enum.Parse<ContractUtils.ContractType>(ctype), d1, d2) {
+      }
+
+      static void swap_if_necessary(ref DateTime dt1, ref DateTime dt2) {
+         if (dt1 < dt2) {
+            DateTime aux = dt1;
+            dt1 = dt2;
+            dt2 = aux;
+         }
+      }
+
+      private AbsoluteVolumeStream(string symbol, ContractUtils.ContractType ctype, DateTime d1, DateTime d2) {
+         swap_if_necessary(ref d1, ref d2);
+         ViewStream ohlcv_str = _get_ohlcv_steam(symbol, d1, d2);
+         ohlcv_str = new FilterStream(ohlcv_str,
+            r => r.get_datetime("ts_event") <= d1 && r.get_datetime("ts_event") >= d2);
+         ohlcv_str = new FilterStream(ohlcv_str, r => String.Compare(ctype.ToString(), r.get<string>("contract_type"), true) == 0);
+         ViewStream rs = new MaxStream(ohlcv_str, "volume");
+         rs = new NewProjectStream(rs, "volume:total_volume contract !");
+         mySetRealSource(rs);
+      }
+
+   }
+}
@@ -1,35 +1,105 @@
-namespace BCG {
+namespace bts.SilverPine {
   using BCG.Utilities.General;

-   using bts.api;
-   using bts.api.domain;
-   using bts.api.synthetic;
   using bts.md;
-   using bts.spark;
-   using bts.utils;
-
-   using mbts;
-   using mbts.co2;
-   using mbts.config;
-   using mbts.spark;
-   using mbts.ui;
-
-   using Spark.Core.Data;
-   using Spark.Core.Disposables;
+   using BCG;
   using Spark.Core.Reactive;

   using System;
-   using System.Collections.Concurrent;
   using System.Collections.Generic;
-   using System.Collections.ObjectModel;
   using System.Linq;
   using System.Reactive.Linq;
   using bts.api.core;
   using Spark.Api.MarketData;

   using SecurityKey = System.UInt32;
-   using System.Data;
-   using System.Reflection.Metadata;
+   using System.Data.Entity.Infrastructure;
+
+   public class FakeBBOStream : ViewStream, IWorker {
+      private string _symbol;
+      public BlueTable _table = new BlueTable(BBOStream._bbo_schema);
+      private DateTime _last_fired_time = DateTime.MinValue;
+      private TimeSpan _period = TimeSpan.FromMilliseconds(333);
+      private double _start_price;
+      private double _inc;
+      private int _total_count = 5;
+      private Random _rnd = new Random(DateTime.Now.Millisecond);
+      private string[] _months = new string[]{
+         "F","G", "H", "J","K", "M", "N","Q","U","V","X","Z"
+      };
+      private ViewStreamDatabase _vsd;
+      public FakeBBOStream(string contract_type, string exchange, string symbol, double start_price, double inc)
+      :
+      this(contract_type, exchange, symbol, start_price, inc, ViewStreamDatabase.instance) { }
+
+      public FakeBBOStream(string contract_type, string exchange, string symbol, double start_price, double inc, ViewStreamDatabase vsd) {
+         _symbol = symbol;
+         _start_price = start_price;
+         _inc = inc;
+         _vsd = vsd;
+         Log.assert(contract_type.ToLower() == "future");
+      }
+
+      protected override void myOnFirstSubscriber() {
+         _vsd.addWorker(this);
+      }
+
+      protected override void myOnLastSubscriber() {
+         _vsd.removeWorker(this);
+      }
+
+      private bool _random_truefase() {
+         int r = _rnd.Next(0, 100);
+         return r % 2 == 0;
+      }
+
+      private IBlueRow _perturb_row(IBlueRow er) {
+         MutableBlueRow rv = new MutableBlueRow(er);
+         double bid_price = rv.get<double>("bid_price");
+         bid_price += _inc;
+         double ask_price = bid_price + _inc;
+         rv.setValue("bid_price", bid_price);
+         rv.setValue("ask_price", ask_price);
+         return rv;
+      }
+
+      public void doWork() {
+         if (DateTime.Now - _last_fired_time < _period)
+            return;
+         _last_fired_time = DateTime.Now;
+         if (_table.Count == 0) {
+            // first fire_symbol
+            string year_code = DateTime.Now.Year.ToString().Last().ToString();
+            for (int i = 0; i < _total_count; i++) {
+               string month_code = _months[i % _months.Length];
+               string sec = $"{_symbol}{month_code}{year_code}";
+               double bid_price = _start_price + _rnd.Next(-10, 10) * _inc;
+               double ask_price = bid_price + _inc;
+               BlueRow nr = new BlueRow(_table.schema, _table.Count, _table.Count, sec,
+                                       bid_price, _rnd.Next(1, 10),
+                                       ask_price, _rnd.Next(1, 10));
+               _table.addRow(nr);
+               var dp = BlueDataUtility.build_dp(new BlueTable(_table), myLastValue as DataPacket);
+               myFireEvent(dp);
+            }
+         } else {
+            BlueTable deleted_rows = new BlueTable(_table.schema);
+            BlueTable added_rows = new BlueTable(_table.schema);
+            // go through all rows, select some at random, add them to deleted, get
+            // the replaced one and add it to added, updates table state and fire
+            for (int i = 0; i < _table.Count; i++) {
+               if (_random_truefase()) {
+                  IBlueRow er = _table[i];
+                  IBlueRow nr = _perturb_row(er);
+                  _table.deleteRow(er);
+                  _table.addRow(nr);
+               }
+            }
+            var dp = BlueDataUtility.build_dp_inc(new BlueTable(_table), myLastValue as DataPacket, true);
+            myFireEvent(dp);
+         }
+      }
+   }

   [DataStreamUserRespresentation("bbo")]
   public class BBOStream : ViewStream, IWorker {
@@ -39,7 +109,7 @@ namespace BCG {
      private int _id = 0;
      IReadOnlyDictionary<SecurityKey, BookTuple> _cached_state = null;
      MarketDataClient _md_client;
-      private readonly IBlueTableSchema _bbo_schema = BlueDataUtility.form_schema("bbo",
+      internal static IBlueTableSchema _bbo_schema = BlueDataUtility.form_schema("bbo",
                                                Tuple.Create("id", TypeCode.Int32),
                                                Tuple.Create("sec_key", TypeCode.UInt32),
                                                Tuple.Create("sec", TypeCode.String),
@@ -54,9 +124,11 @@ namespace BCG {
      }

      public BBOStream(ViewStreamDatabase vsd) {
-         bts.Log.Gen.info("BBOStream.cons");
-         _vsd = vsd;
-         _md_client = new MarketDataClient(new MDSource(TimeSpan.FromSeconds(0.5)));
+         // bts.Log.Gen.info("BBOStream.cons");
+         // _vsd = vsd;
+         // _md_client = new MarketDataClient(new MDSource(TimeSpan.FromSeconds(0.5)));
+         FakeBBOStream rs = new FakeBBOStream("Future", "CME", "ES", 5000, 0.25, vsd);
+         mySetRealSource(rs);
      }

      protected override void myOnFirstSubscriber() {
@@ -0,0 +1,254 @@
+namespace bts.SilverPine {
+   using BCG;
+
+   using System.Runtime.CompilerServices;
+
+   using ZstdSharp.Unsafe;
+
+   public class DateTimeRange {
+      public readonly DateTime range_beg;
+      public readonly DateTime range_end;
+      public DateTimeRange(DateTime range_beg_, DateTime range_end_) {
+         range_beg = range_beg_;
+         range_end = range_end_;
+      }
+      public static DateTimeRange parse(string range_beg_, string range_end_) {
+         DateTime range_beg = String.IsNullOrEmpty(range_beg_) ? DateTime.MinValue : DateTime.Parse(range_beg_);
+         DateTime range_end = String.IsNullOrEmpty(range_end_) ? DateTime.MaxValue : DateTime.Parse(range_end_);
+         return new DateTimeRange(range_beg, range_end);
+      }
+   }
+
+   public class FutureBarsLoader {
+      public static string get_db_table_name(BarSize bar_size, string symbol, bool is_continous) {
+         switch (bar_size) {
+            case BarSize.ONE_DAY: {
+                  if (is_continous) {
+                     return $"bars_future_{symbol}_continous_1day";
+                  } else
+                     return $"bars_future_{symbol}_all_1day";
+               }
+            case BarSize.ONE_MIN: {
+                  if (is_continous) {
+                     return $"bars_future_{symbol}_continous_1min";
+                  } else
+                     return $"bars_future_{symbol}_all_1min";
+               }
+         }
+         Log.assert(false);
+         return null;
+      }
+   }
+   public enum BarSize {
+      ONE_DAY = 0,
+      ONE_MIN
+   }
+
+   [DataStreamUserRespresentation("FuturesHistorical1DayBars")]
+   public class FuturesHistorical1DayBarsStream : ViewStream {
+
+      // this gets the raw bars for a symbol (perhaps make it into a raw bars stream?)
+      public FuturesHistorical1DayBarsStream(string symbol) :
+      this(symbol, DateTime.MinValue, DateTime.MaxValue) { }
+
+      // creates the non raw stream (ie for tradable contract, or continous contract)
+      public FuturesHistorical1DayBarsStream(string future_contract, DateTime range_beg, DateTime range_end)
+     : this(new ContractUtils.FutureContract(future_contract), new DateTimeRange(range_beg, range_end)) { }
+
+      // this creates a tradable contracts bars  streams or continous contracts stream. if get_all_rows is true, 
+      // it gets the raw table with all contracts for that symbol
+      public FuturesHistorical1DayBarsStream(ContractUtils.FutureContract future_contract,
+                                             DateTimeRange dt_range) {
+         DateTime beg = dt_range.range_beg;//.ToUniversalTime(),
+         DateTime end = dt_range.range_end;//.ToUniversalTime();
+         beg = beg.Date;
+         end = end.Date.Add(new TimeSpan(23, 59, 59));
+         // this returns rows from beg and end inclusive
+         Func<string, SQLiteStream> get_sqlite_str = (table_name) => {
+            string range_beg_iso8601 = beg.ToString("yyyy-MM-ddTHH:mm:ss"),
+               range_end_iso8601 = end.ToString("yyyy-MM-ddTHH:mm:ss");
+            string sql_str = $"select * from {table_name} where ts_event >= '{range_beg_iso8601}' AND ts_event <= '{range_end_iso8601}'";
+            return new SQLiteStream(SQLiteHelper.db_name, sql_str);
+         };
+         string symbol = future_contract.symbol();
+         if (future_contract.wildcard_contract != null) {
+            string table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_DAY, future_contract.symbol(), false);
+            ViewStream rs = get_sqlite_str(table_name);
+            //rs = new DateStandardizationStream(rs, "ts_event", DateConversion.ChopTime);
+            mySetRealSource(rs);
+            return;
+         }
+         if (future_contract.wildcard_contract != null) {
+            string table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_DAY, future_contract.symbol(), false);
+            ViewStream rs = get_sqlite_str(table_name);
+            //rs = new DateStandardizationStream(rs, "ts_event", DateConversion.ChopTime);
+            mySetRealSource(rs);
+            return;
+         }
+         if (future_contract.is_continous_contract) {
+            string table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_DAY, symbol, true);
+            ViewStream rs = get_sqlite_str(table_name);
+            //rs = new DateStandardizationStream(rs, "ts_event", DateConversion.ChopTime);
+            mySetRealSource(rs);
+            return;
+         } else {
+            string table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_DAY, future_contract.symbol(), false);
+            ViewStream rs = get_sqlite_str(table_name);
+            //rs = new DateStandardizationStream(rs, "ts_event", DateConversion.ChopTime);
+            //TODO: filter it 
+            rs = new FilterStream(rs, $"symbol = '{future_contract.contract_text}'");
+            mySetRealSource(rs);
+         }
+      }
+   }
+
+   [DataStreamUserRespresentation("FuturesHistorical1MinBars")]
+   public class FuturesHistorical1MinuteBarsStream : ViewStream {
+      // same as the above but for one min
+      // this gets the raw bars for a symbol
+      public FuturesHistorical1MinuteBarsStream(string symbol) :
+      this(symbol, DateTime.MinValue, DateTime.MaxValue) { }
+
+      // creates the non raw stream (ie for tradable contract, or continous contract)
+      public FuturesHistorical1MinuteBarsStream(string future_contract, DateTime range_beg, DateTime range_end)
+     : this(new ContractUtils.FutureContract(future_contract), new DateTimeRange(range_beg, range_end)) { }
+
+      // this creates a tradable contracts bars  streams or continous contracts stream. if get_all_rows is true, 
+      // it gets the raw table with all contracts for that symbol
+      public FuturesHistorical1MinuteBarsStream(ContractUtils.FutureContract future_contract,
+                                             DateTimeRange dt_range) {
+         Func<string, SQLiteStream> get_sqlite_str = (table_name) => {
+            string range_beg_iso8601 = dt_range.range_beg.ToString("yyyy-MM-ddTHH:mm:ss"),
+               range_end_iso8601 = dt_range.range_end.ToString("yyyy-MM-ddTHH:mm:ss");
+            //string sql_str = $"select * from {table_name} where DATE(ts_event) >= '{range_beg_iso8601}' AND DATE(ts_event) <= '{range_end_iso8601}'";
+            string sql_str = $"select * from {table_name} where ts_event >= '{range_beg_iso8601}' AND ts_event <= '{range_end_iso8601}'";
+            return new SQLiteStream(SQLiteHelper.db_name, sql_str);
+         };
+         string symbol = future_contract.symbol();
+         if (future_contract.wildcard_contract != null) {
+            string table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_MIN, future_contract.symbol(), false);
+            ViewStream rs = get_sqlite_str(table_name);
+            mySetRealSource(rs);
+            return;
+         }
+         if (future_contract.is_continous_contract) {
+            string table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_MIN, symbol, true);
+            ViewStream rs = get_sqlite_str(table_name);
+            mySetRealSource(rs);
+            return;
+         } else {
+            string table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_MIN, future_contract.symbol(), false);
+            ViewStream rs = get_sqlite_str(table_name);
+            // TODO: filter
+            rs = new FilterStream(rs, $"symbol = '{future_contract.contract_text}'");
+            mySetRealSource(rs);
+         }
+      }
+   }
+
+   [DataStreamUserRespresentation("FutureHistoricalBars")]
+   public class FutureHistoricalBarsStream : ViewStream {
+      public FutureHistoricalBarsStream(string fut_contract, string bar_size_txt, string range_beg, string range_end) {
+         DateTime dt_range_beg = DateTime.Parse(range_beg);
+         DateTime dt_range_end = DateTime.Parse(range_end);
+         TimeSpan diff = dt_range_end - dt_range_beg;
+         BarSize bar_size = Enum.Parse<BarSize>(bar_size_txt, true);
+         ViewStream rs = new FutureHistoricalBarsStream(fut_contract, bar_size.ToString(), dt_range_beg, dt_range_end);
+         mySetRealSource(rs);
+      }
+
+      public FutureHistoricalBarsStream(string fut_contract, string range_beg, string range_end) {
+         DateTime dt_range_beg = DateTime.Parse(range_beg);
+         DateTime dt_range_end = DateTime.Parse(range_end);
+         TimeSpan diff = dt_range_end - dt_range_beg;
+         BarSize bar_size = diff < TimeSpan.FromDays(1.0) ? BarSize.ONE_DAY : BarSize.ONE_MIN;
+         ViewStream rs = new FutureHistoricalBarsStream(fut_contract, bar_size.ToString(), dt_range_beg, dt_range_end);
+         mySetRealSource(rs);
+      }
+
+      public FutureHistoricalBarsStream(string fut_contract, string bar_size)
+      : this(fut_contract, bar_size, DateTime.MinValue, DateTime.MaxValue) { }
+
+      public FutureHistoricalBarsStream(string fut_contract, string bar_size,
+                                        DateTime dt_range_beg, DateTime dt_range_end)
+                                        :
+      this(new ContractUtils.FutureContract(fut_contract), Enum.Parse<BarSize>(bar_size, true),
+         new DateTimeRange(dt_range_beg, dt_range_end)) { }
+
+      public FutureHistoricalBarsStream(ContractUtils.FutureContract fut_contract, BarSize bar_size,
+                                        DateTimeRange dt_range) {
+         switch (bar_size) {
+            case BarSize.ONE_DAY: {
+                  ViewStream rs = new FuturesHistorical1DayBarsStream(fut_contract, dt_range);
+                  mySetRealSource(rs);
+                  break;
+               }
+            case BarSize.ONE_MIN: {
+                  ViewStream rs = new FuturesHistorical1MinuteBarsStream(fut_contract, dt_range);
+                  mySetRealSource(rs);
+                  break;
+               }
+         }
+      }
+   }
+
+   [DataStreamUserRespresentation("FuturePriceDiff")]
+   public class FutureHistoricalBarsPriceDiffStream : ViewStream {
+
+      public FutureHistoricalBarsPriceDiffStream(string future_contract, string dt1, string dt2)
+      :
+      this(new ContractUtils.FutureContract(future_contract), DateTime.Parse(dt1), DateTime.Parse(dt2), "close", ViewStreamDatabase.instance) {
+      }
+
+
+      public FutureHistoricalBarsPriceDiffStream(string future_contract, DateTime dt1, DateTime dt2)
+      :
+      this(new ContractUtils.FutureContract(future_contract), dt1, dt2, "close", ViewStreamDatabase.instance) {
+      }
+
+      public FutureHistoricalBarsPriceDiffStream(string future_contract, DateTime dt1, DateTime dt2, string col)
+      :
+      this(new ContractUtils.FutureContract(future_contract), dt1, dt2, col, ViewStreamDatabase.instance) {
+      }
+
+
+      public FutureHistoricalBarsPriceDiffStream(ContractUtils.FutureContract fc, DateTime dt1, DateTime dt2, string col, ViewStreamDatabase vsd) {
+         BarSize bsize = dt2 - dt1 >= TimeSpan.FromDays(1.0) ? BarSize.ONE_DAY : BarSize.ONE_MIN;
+         ViewStream rs = new FutureHistoricalBarsStream(fc, BarSize.ONE_DAY, new DateTimeRange(dt1, dt2));
+
+         rs = new ColumnDifferenceStream(rs, r => {
+            DateTime r_dt = r.get_datetime("ts_event");
+            var rv = dt1.Equals(r_dt);
+            return rv;
+         },
+         //r => dt2.Equals(r.get_datetime("ts_event")), col, col);
+         r => {
+            DateTime r_dt = r.get_datetime("ts_event");
+            var rv = dt2.Equals(r_dt);
+            return rv;
+         }, col, col);
+         rs = new NewProjectStream(rs, "diff !");
+         mySetRealSource(rs);
+      }
+   }
+
+   // takes the contract name and expands it into multuiple columns corresponding to the contracts.
+   // the contract columns are a union, of all properties with some of them empty
+   public class ContractExpanderStream : ViewStream {
+      private static IBlueTableSchema map_schema(IBlueTableSchema schema) {
+         return schema.add_column(TypeCode.String, "contract_type");
+      }
+      public ContractExpanderStream(ViewStream parent_str, string contract_col_name) {
+         ViewStream rs = new MapStream(parent_str, (schema) => map_schema(schema),
+         r => {
+            string contract = r.get<string>(contract_col_name);
+            var ct = ContractUtils.get_type(contract);
+            MutableBlueRow rv = new MutableBlueRow(map_schema(r.schema), r);
+            rv.setValue("contract_type", ct.ToString());
+            return rv;
+         });
+         mySetRealSource(rs);
+      }
+   }
+
+}
@@ -0,0 +1,53 @@
+using BCG;
+
+namespace bts.SilverPine {
+   /*
+      ----------------------------------------------------------------------
+                                  *Expression*
+      ----------------------------------------------------------------------
+          //r=FutureHistoricalBars("ES*", "02/01/2024 09:30 AM", "02/01/2024 09:30 PM")
+          r=FutureHistoricalBars("ES*","One_Day", 02/01/2024, 02/01/2025)
+          r=contractExpander(r,"symbol")
+          r=filter(r,"contract_type='FutureSpread'")
+          r=aggregate(r,"symbol","volume:Sum","ts_event")
+          r=aggregate(r,"volume_sum:Max")
+          r=newProject(r,"volume_sum_max:volume symbol !")
+
+      ----------------------------------------------------------------------
+                                    *Schema*
+      ----------------------------------------------------------------------
+          symbol                          System.String
+          id                              System.Int32
+          volume                          System.Double
+   */
+
+   [DataStreamUserRespresentation("VolumeAgg")]
+   public class VolumeAggregationStream : ViewStream {
+      public VolumeAggregationStream(string future_contract, string contract_type, DateTime range_beg, DateTime range_end)
+      : this(future_contract, contract_type, range_beg, range_end, ViewStreamDatabase.instance) { }
+
+      public VolumeAggregationStream(string future_contract, string contract_type, string range_beg, string range_end)
+      :
+      this(future_contract, contract_type, DateTime.Parse(range_beg), DateTime.Parse(range_end), ViewStreamDatabase.instance) { }
+
+      public VolumeAggregationStream(string future_contract, string contract_type, DateTime range_beg, DateTime range_end, ViewStreamDatabase db) {
+         future_contract = future_contract.EndsWith("*") ? future_contract : future_contract + "*";
+         ViewStream r;
+         r = new FutureHistoricalBarsStream(future_contract,
+                                             range_beg.ToString(), range_end.ToString());
+         r = new ContractExpanderStream(r, "symbol");
+         r = new FilterStream(r, $"contract_type='{contract_type}'");
+         r = new NonIncGroupByStream(r, "symbol", "volume:Sum", "ts_event");
+         r = new NonIncGroupByStream(r, "volume_sum:Max");
+         r = new NewProjectStream(r, "volume_sum_max:volume symbol !");
+         mySetRealSource(r);
+      }
+   }
+
+   [DataStreamUserRespresentation("PeriodicVolumeAgg")]
+   public class PeriodicVolumeAggregationStream : ViewStream {
+      public PeriodicVolumeAggregationStream(string future_contract, string contract_type, DateTime range_beg, DateTime range_end,
+                                             TimeSpan period, ViewStreamDatabase db) {
+      }
+   }
+}
@@ -0,0 +1,152 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+
+using bts;
+
+namespace bts.SilverPine {
+   public static class IndicatorCalc {
+      public static double calculate_pearson_correlation(List<double> prices1, List<double> prices2) {
+         if (prices1 == null)
+            throw new ArgumentNullException(nameof(prices1), "First price list cannot be null.");
+         if (prices2 == null)
+            throw new ArgumentNullException(nameof(prices2), "Second price list cannot be null.");
+         if (prices1.Count != prices2.Count)
+            throw new ArgumentException("Price lists must have the same number of elements.");
+         if (prices1.Count < 2)
+            throw new ArgumentException("At least two data points are required to calculate correlation.");
+
+         int n = prices1.Count;
+         double sumX = 0;
+         double sumY = 0;
+         double sumX2 = 0;
+         double sumY2 = 0;
+         double sumXY = 0;
+
+         for (int i = 0; i < n; i++) {
+            double x = prices1[i];
+            double y = prices2[i];
+
+            sumX += x;
+            sumY += y;
+            sumX2 += x * x;
+            sumY2 += y * y;
+            sumXY += x * y;
+         }
+
+         double numerator = (n * sumXY) - (sumX * sumY);
+         double denominatorPart1 = (n * sumX2) - (sumX * sumX);
+         double denominatorPart2 = (n * sumY2) - (sumY * sumY);
+         double denominator = Math.Sqrt(denominatorPart1 * denominatorPart2);
+
+         if (denominator == 0)
+            throw new InvalidOperationException("Denominator is zero, correlation is undefined.");
+
+         return numerator / denominator;
+      }
+      public static double calculate_VWAP(List<Tuple<int, double>> volumePriceData) {
+         Log.Gen.info($"ChartingHelper.calculate_VWAP n_vals: {volumePriceData.Count}");
+         double totalVolume = 0;
+         double totalPriceVolume = 0;
+
+         foreach (var data in volumePriceData) {
+            int volume = data.Item1; // Volume
+            double price = data.Item2; // Closing price
+
+            // Accumulate total price-volume (price * volume) and total volume
+            totalPriceVolume += price * volume;
+            totalVolume += volume;
+         }
+
+         // VWAP is totalPriceVolume divided by totalVolume
+         return totalVolume > 0 ? totalPriceVolume / totalVolume : 0;
+      }
+
+      // Calculate Volatility (Standard Deviation of returns) over a given period
+      public static double calculate_volatility(IList<double> prices, int period = 252) {
+         if (prices == null || prices.Count < period + 1) {
+            throw new ArgumentException("Insufficient data to calculate volatility.");
+         }
+
+         // Calculate daily returns
+         var returns = new List<double>();
+         for (int i = 1; i < prices.Count; i++) {
+            double dailyReturn = (prices[i] - prices[i - 1]) / prices[i - 1];
+            returns.Add(dailyReturn);
+         }
+
+         // Get the returns over the desired period
+         var recentReturns = returns.Skip(returns.Count - period).Take(period).ToList();
+
+         // Calculate the mean of returns
+         double meanReturn = recentReturns.Average();
+
+         // Calculate the variance (sum of squared differences from the mean)
+         double variance = recentReturns.Sum(r => Math.Pow(r - meanReturn, 2)) / period;
+
+         // Volatility is the square root of variance (standard deviation)
+         double volatility = Math.Sqrt(variance);
+
+         // Annualize volatility by multiplying by the square root of 252 (number of trading days in a year)
+         return volatility * Math.Sqrt(252);
+      }
+
+      // Calculate SMA (Simple Moving Average) for a given period (e.g., 50, 100, or 200)
+      public static double calculate_SMA(IList<double> prices, int period = 50) {
+         if (prices == null || prices.Count < period) {
+            throw new ArgumentException("Insufficient data to calculate SMA.");
+         }
+
+         // Sum up the prices over the given period
+         double sum = prices.Skip(prices.Count - period).Take(period).Sum();
+
+         // Return the average
+         return sum / period;
+      }
+
+      public static double calculate_RSI(IList<double> prices, int period = 14) {
+         if (prices == null || prices.Count < period + 1) {
+            return Double.NaN;
+            //throw new ArgumentException("Insufficient data to calculate RSI.");
+         }
+
+         double gainSum = 0;
+         double lossSum = 0;
+
+         // Calculate the initial average gain and loss over the first 'period' days
+         for (int i = 1; i <= period; i++) {
+            double change = prices[i] - prices[i - 1];
+            if (change > 0) {
+               gainSum += change;
+            } else {
+               lossSum -= change; // Subtract since loss is negative
+            }
+         }
+
+         double avgGain = gainSum / period;
+         double avgLoss = lossSum / period;
+
+         // Calculate RSI using smoothed values
+         for (int i = period + 1; i < prices.Count; i++) {
+            double change = prices[i] - prices[i - 1];
+            if (change > 0) {
+               avgGain = (avgGain * (period - 1) + change) / period;
+               avgLoss = (avgLoss * (period - 1)) / period;
+            } else {
+               avgGain = (avgGain * (period - 1)) / period;
+               avgLoss = (avgLoss * (period - 1) - change) / period;
+            }
+         }
+
+         // Prevent division by zero in case of no losses
+         if (avgLoss == 0) {
+            return 100;
+         }
+
+         double rs = avgGain / avgLoss;
+         double rsi = 100 - (100 / (1 + rs));
+
+         return rsi;
+      }
+   }
+}
@@ -0,0 +1,31 @@
+namespace bts.SilverPine {
+   using BCG;
+
+   using bts;
+
+   [DataStreamUserRespresentation("DailyChange")]
+   // takes in the ohlcv stream and computes the closing vs closing difference
+   public class DailyChange : ViewStream {
+      public DailyChange(ViewStream ohlc_str) {
+         GlobalMapStream rs = new GlobalMapStream(ohlc_str, "ts_event", "contract",
+         (rows, idx) => {
+            double daily_change = Double.NaN;
+            double daily_change_perc = Double.NaN;
+            IBlueRow ir = rows.ElementAt((int)idx);
+            if (idx < rows.Count - 1) {
+               IBlueRow pr = rows.ElementAt((int)idx + 1);
+               double close = ir.get<double>("close"), prev_close = pr.get<double>("close");
+               daily_change = close - prev_close;
+               daily_change_perc = 100.0 * daily_change / prev_close;
+            }
+            IBlueTableSchema schema = ir.schema.add_columns(Tuple.Create(typeof(double), "daily_change"));
+            schema = schema.add_columns(Tuple.Create(typeof(double), "daily_change_perc"));
+            MutableBlueRow nr = new MutableBlueRow(schema, ir);
+            nr.setValue("daily_change", daily_change);
+            nr.setValue("daily_change_perc", daily_change_perc);
+            return nr;
+         });
+         mySetRealSource(rs);
+      }
+   }
+}
@@ -1,54 +0,0 @@
-
-using Google.Protobuf.WellKnownTypes;
-
-using System;
-using BCG.Utilities.General;
-using System.Data;
-using bts.api.domain;
-
-namespace BCG {
-    // Stream computes the EMA of a column in any stream. It takes in the base stream, the column name and a smoothing factor
-    // as args, and create one row with an ema column, which is the running ema of the column provided 
-    [DataStreamUserRespresentation("ema")]
-    public class EMAStream : ViewStream {
-        private ViewStream _stream;
-        private readonly string _col_name;
-        private EMACalculator _calc;
-        private readonly IBlueTableSchema _schema = BlueDataUtility.form_schema(
-            "SMA", Tuple.Create("id", TypeCode.Int32),
-            Tuple.Create("ema", TypeCode.Double));
-
-        public EMAStream(ViewStream stream, string col_name, double smoothing_factor) {
-            _col_name = col_name;
-            _stream = stream;
-            _calc = new EMACalculator(smoothing_factor);
-        }
-
-        protected override void myOnFirstSubscriber() {
-            _stream.onChange += new ChangeDelegate(_onstream_change);
-        }
-        protected override void myOnLastSubscriber() {
-            _stream.onChange -= new ChangeDelegate(_onstream_change);
-        }
-
-
-        private void _onstream_change(object o) {
-
-            DataPacket dp = o as DataPacket;
-            foreach (IBlueRow r in dp.addedRows) {
-                double price = r.get<double>(_col_name);
-                if (!Double.IsNaN(price))
-                    _calc.add(price);
-            }
-            IBlueRow nr = new BlueRow(_schema, 0, _calc.get_EMA());
-            BlueTable state = new BlueTable(_schema);
-            state.addRow(nr);
-            DataPacket dp_out = BlueDataUtility.build_dp_inc(state, myLastValue as DataPacket);
-            myFireEvent(dp_out);
-        }
-    }
-
-}
-
-
-
@@ -0,0 +1,26 @@
+namespace bts.SilverPine {
+   using BCG;
+
+   using bts;
+
+   [DataStreamUserRespresentation("RSI")]
+   public class RSIStream : ViewStream {
+      // public RSIStream(ViewStreamDatabase vsd, string symbol, int n_days) {
+
+      // }
+
+      public RSIStream(ViewStream instrument_ohlc_str, int period) {
+         GlobalMapStream rs = new GlobalMapStream(instrument_ohlc_str, "ts_event", "contract",
+         (rows, idx) => {
+            IBlueRow ir = rows.ElementAt((int)idx);
+            IEnumerable<double> closes = BlueDataUtility.get_colvalues<double>(rows, "close", (int)idx + 1, period + 2);
+            double rsi = IndicatorCalc.calculate_RSI(closes.ToList(), period);
+            IBlueTableSchema schema = ir.schema.add_columns(Tuple.Create(typeof(double), "rsi"));
+            MutableBlueRow nr = new MutableBlueRow(schema, ir);
+            nr.setValue("rsi", rsi);
+            return nr;
+         });
+         mySetRealSource(rs);
+      }
+   }
+}
@@ -1,87 +0,0 @@
-
-using Google.Protobuf.WellKnownTypes;
-
-using System;
-using BCG.Utilities.General;
-using System.Data;
-using bts.api.domain;
-using System.Reactive.Concurrency;
-using mbts.spark;
-
-namespace BCG {
-    // Stream computes the EMA of a column in any stream. It takes in the base stream, the column name and a smoothing factor
-    // as args, and create one row with an ema column, which is the running ema of the column provided 
-    [DataStreamUserRespresentation("renko")]
-    public class RenkoStream : ViewStream {
-        private ViewStream _stream;
-        private readonly string _col_name;
-        private readonly double _brick_sz;
-        private RenkoCalculator _calc;
-
-        private BlueTable _table = new BlueTable(BlueDataUtility.form_schema(
-            "Renko", Tuple.Create("id", TypeCode.Int32),
-            Tuple.Create("entry_price", TypeCode.Double),
-            Tuple.Create("current_price", TypeCode.Double),
-            Tuple.Create("brick_low", TypeCode.Double),
-            Tuple.Create("brick_high", TypeCode.Double),
-            Tuple.Create("brick_transition", TypeCode.String),
-            Tuple.Create("n_updates_within", TypeCode.Int32)
-            ));
-
-        public RenkoStream(ViewStream stream, string col_name, double brick_sz) {
-            _col_name = col_name;
-            _stream = stream;
-            _brick_sz = brick_sz;
-            _calc = new RenkoCalculator(brick_sz);
-        }
-
-        protected override void myOnFirstSubscriber() {
-            _stream.onChange += _onstream_change;
-        }
-        protected override void myOnLastSubscriber() {
-            _stream.onChange -= new ChangeDelegate(_onstream_change);
-        }
-
-
-        private IBlueRow _remove_row(RenkoBrick brick) {
-            var row = _table[brick.id];
-            _table.deleteRow(row);
-            return row;
-        }
-
-        private IBlueRow _create_new_row(RenkoBrick brick) {
-            return new BlueRow(_table.schema, brick.id,
-            brick.entry_price, brick.current_price, brick.low, brick.high,
-            brick.entry.ToString(), brick.n_updates);
-        }
-
-        private void _onstream_change(object o) {
-            // what feeds in is a time series
-            BlueTable deleted_rows = new BlueTable(_table.schema);
-            BlueTable added_rows = new BlueTable(_table.schema);
-            DataPacket dp = o as DataPacket;
-            foreach (IBlueRow r in dp.addedRows) {
-                double val = r.get<double>(_col_name);
-                var new_bricks = _calc.update(val);
-                if (new_bricks.Count() > 0) {
-                    foreach (var nb in new_bricks) {
-                        var new_row = _create_new_row(nb);
-                        added_rows.addRow(new_row);
-                        _table.addRow(new_row);
-                    }
-                } else {
-                    var existing_brick = _calc.current_brick;
-                    var del_row = _remove_row(existing_brick);
-                    deleted_rows.addRow(del_row);
-                    var new_row = _create_new_row(existing_brick);
-                    added_rows.addRow(new_row);
-                    _table.addRow(new_row);
-                }
-            }
-            myFireEvent(new DataPacket(_table, added_rows, deleted_rows));
-        }
-    }
-}
-
-
-
@@ -1,54 +1,24 @@
+namespace bts.SilverPine {
+   using bts;
+   using BCG;

-using Google.Protobuf.WellKnownTypes;
-
-using System;
-using BCG.Utilities.General;
-using System.Data;
-using bts.api.domain;
-
-namespace BCG {
-    // Stream computes the SMA of a column in any stream. It takes in the base stream, the column namer and a smoothing factor
-    // as args, and create one row with an sma column, which is the running ema of the column provided 
-    [DataStreamUserRespresentation("sma")]
-    public class SMAStream : ViewStream {
-        private ViewStream _stream;
-        private readonly string _col_name;
-        private Chartutils.SMACalculator _calc;
-        private readonly IBlueTableSchema _schema = BlueDataUtility.form_schema(
-            "SMA", Tuple.Create("id", TypeCode.Int32),
-            Tuple.Create("sma", TypeCode.Double));
-
-        public SMAStream(ViewStream stream, string col_name, int window_size) {
-            _col_name = col_name;
-            _stream = stream;
-            _calc = new Chartutils.SMACalculator(window_size);
-        }
-
-        protected override void myOnFirstSubscriber() {
-            _stream.onChange += new ChangeDelegate(_onstream_change);
-        }
-        protected override void myOnLastSubscriber() {
-            _stream.onChange -= new ChangeDelegate(_onstream_change);
-        }
-
-
-        private void _onstream_change(object o) {
-
-            DataPacket dp = o as DataPacket;
-            foreach (IBlueRow r in dp.addedRows) {
-                double price = r.get<double>(_col_name);
-                if (!Double.IsNaN(price))
-                    _calc.add(r.get<double>(_col_name));
-            }
-            IBlueRow nr = new BlueRow(_schema, 0, _calc.get_SMA());
-            BlueTable state = new BlueTable(_schema);
-            state.addRow(nr);
-            DataPacket dp_out = BlueDataUtility.build_dp_inc(state, myLastValue as DataPacket);
-            myFireEvent(dp_out);
-        }
-    }
-
-}
+   [DataStreamUserRespresentation("SMA")]
+   public class SMAStream : ViewStream {
+      public SMAStream(ViewStream ohlc_str, int period) {
+         GlobalMapStream rs = new GlobalMapStream(ohlc_str, "ts_event", "contract",
+         (rows, idx) => {
+            IBlueRow ir = rows.ElementAt((int)idx);

+            IEnumerable<double> closes = BlueDataUtility.get_colvalues<double>(rows, "close", (int)idx + 1, period + 2);
+            double rsi = IndicatorCalc.calculate_RSI(closes.ToList(), period);
+            IBlueTableSchema schema = ir.schema.add_columns(Tuple.Create(typeof(double), "rsi"));
+            MutableBlueRow nr = new MutableBlueRow(schema, ir);
+            nr.setValue("rsi", rsi);
+            return nr;
+         });
+         mySetRealSource(rs);
+      }


+   }
+}
@@ -1,49 +0,0 @@
-using System;
-
-namespace BCG {
-    /*
-       ----------------------------------------------------------------------
-                                   *Expression*
-       ----------------------------------------------------------------------
-           r=filter(bbo(),"sec='BTC-PERPETUAL'")
-           r=math(r,"wt_mid_price = (bid_price*bid_sz + ask_price*ask_sz)/(bid_sz+ask_sz)")
-           r=timeSeries(r)
-           r=math(r,"sz = (bid_sz + ask_sz) * 1.0")
-           r=filter(r,"sz > 0")
-           r=renko(r,"wt_mid_price", 2.0)
-
-       ----------------------------------------------------------------------
-                                     *Schema*
-       ----------------------------------------------------------------------
-           id                              System.Int32
-           entry_price                     System.Double
-           current_price                   System.Double
-           brick_low                       System.Double
-           brick_high                      System.Double
-           brick_transition                System.String
-           n_updates_within                System.Int32
-    */
-
-
-    public class SecurityRenkoStream : ViewStream {
-        public SecurityRenkoStream(int sec_key, double brick_sz) {
-            ViewStream r;
-
-            r = new FilterStream(new BBOStream(),
-                                  $"sec_key={sec_key}");
-            r = new MathStream(r,
-                                "wt_mid_price = (bid_price*bid_sz + ask_price*ask_sz)/(bid_sz+ask_sz)");
-            r = new TimeSeriesStream(r);
-            r = new MathStream(r,
-                                "sz = (bid_sz + ask_sz) * 1.0");
-            r = new FilterStream(r,
-                                  "sz > 0");
-            r = new RenkoStream(r,
-                                 "wt_mid_price",
-                                 brick_sz);
-
-
-            mySetRealSource(r);
-        }
-    }
-}
@@ -0,0 +1,40 @@
+namespace bts.SilverPine {
+   using BCG;
+
+   using MathNet.Numerics.Statistics;
+
+   // calculates the reazlied standard deviation of a series of prices
+   public class StandardDeviationStream : ViewStream {
+      public StandardDeviationStream(string symbol, string std_dev_period)
+      :
+      this(symbol, TimeBasedFilter.parse_duration(std_dev_period)) { }
+
+      public StandardDeviationStream(string symbol, TimeSpan std_dev_period) {
+         // since this is a 1_day bar str, 90 rows => 90 days, so no need to filter
+         int n_days = std_dev_period.Days;
+         ViewStream ohlcv_str = new FuturesActiveContractStream(symbol);
+         IBlueTableSchema out_schema = null;
+         int id = 0;
+         Func<List<IBlueRow>, uint, IBlueRow> find_std_dev = (rows, idx) => {
+            int iidx = Convert.ToInt32(idx);
+            if (rows == null || rows.Count == 0)
+               return null;
+            IBlueRow ir = rows.ElementAt(iidx);
+            out_schema = out_schema ?? ir.schema.add_column(TypeCode.Double, "std_dev");
+            MutableBlueRow rv = new MutableBlueRow(out_schema, ir);
+            rv.setValue("std_dev", Double.NaN);
+            IEnumerable<double> last_n_close_prices = rows.Skip(iidx).Take(n_days).Select(r => r.get<double>("close"));
+            if (last_n_close_prices.Count() >= n_days) {
+               double std_dev = MathNet.Numerics.Statistics.Statistics.StandardDeviation(last_n_close_prices);
+               rv.setValue("std_dev", std_dev);
+            }
+            //rv.setValue("id", id++);
+            return rv;
+         };
+         ViewStream rs = new GlobalMapStream(ohlcv_str, "ts_event", find_std_dev);
+         rs = new NullFilterStream(rs, "std_dev", true);
+         mySetRealSource(rs);
+      }
+
+   }
+}
@@ -1,161 +0,0 @@
-using BCG;
-using BCG.Utilities.General;
-
-using System;
-
-namespace bts {
-    [DataStreamUserRespresentation("tradebars")]
-    public class TradeBarsStream : ViewStream, IWorker {
-
-        public class TradeBar {
-            static double[] calculate_log_returns(double[] prices) {
-                // Calculate logarithmic returns
-                double[] returns = new double[prices.Length - 1];
-                for (int i = 1; i < prices.Length; i++) {
-                    returns[i - 1] = Math.Log(prices[i] / prices[i - 1]);
-                }
-                return returns;
-            }
-
-            static double calculate_volatility(double[] returns) {
-                // Calculate standard deviation of logarithmic returns
-                double mean = returns.Average();
-                double sum_of_squares = returns.Sum(x => Math.Pow(x - mean, 2));
-                double variance = sum_of_squares / (returns.Length - 1);
-                double volatility = Math.Sqrt(variance);
-
-                return volatility;
-            }
-            public void add_trade(IBlueRow r) {
-                double price = r.get<double>("price");
-                //prices.Add(price);
-                double qty = r.get<double>(col_name: "qty");
-                if (n_entries == 0) {
-                    open = price;
-                    low = price < low ? price : low;
-                    high = price > high ? price : high;
-                }
-                n_entries++;
-                volume += qty;
-                vwap_sum += price * qty;
-                close = price;
-                if ("Buy".Equals(r.get<string>("aggressor"))) {
-                    n_buys_aggs++;
-                } else {
-                    n_sell_aggs++;
-                }
-            }
-            private List<double> prices = new List<double>();
-            public readonly string security;
-            public double low = Double.MaxValue, high, open, close, median, std_dev, vwap_sum, volume;
-            public uint n_buys_aggs, n_sell_aggs, n_entries;
-            public DateTime bar_start = DateTime.Now;
-            public TradeBar(string security_) {
-                security = security_;
-            }
-            public double volatility {
-                get {
-                    //if (prices.Count == 1)
-                    return 0.0;
-                    var log_returns = calculate_log_returns(prices.ToArray());
-                    return calculate_volatility(log_returns);
-                }
-            }
-        }
-
-        private readonly int _bar_period_seconds;
-        private readonly ViewStreamDatabase _vsd;
-        private ViewStream _timesales_stream;
-        private ViewStream _bbo_stream;
-        private IDictionary<uint, TradeBar> _sec_bar_map = new Dictionary<uint, TradeBar>();
-        private bool _dirty = false;
-        private TimeSpan _fire_period;
-        private DateTime _last_fire_time = DateTime.Now;
-        private BlueTable _table = new BlueTable(BlueDataUtility.form_schema("TradeBars",
-        Tuple.Create("id", TypeCode.Int32),
-        Tuple.Create("bar_start", TypeCode.DateTime),
-        Tuple.Create("bar_end", TypeCode.DateTime),
-        Tuple.Create("day_of_week", TypeCode.String),
-        Tuple.Create("n_trades", TypeCode.UInt32),
-        Tuple.Create("security", TypeCode.String),
-        Tuple.Create("sec_key", TypeCode.UInt32),
-        Tuple.Create("low", TypeCode.Double),
-        Tuple.Create("high", TypeCode.Double),
-        Tuple.Create("open", TypeCode.Double),
-        Tuple.Create("close", TypeCode.Double),
-        Tuple.Create("vwap_sum", TypeCode.Double),
-        Tuple.Create("volume", TypeCode.Double),
-        Tuple.Create("n_buy_aggs", TypeCode.UInt32),
-        Tuple.Create("n_sell_aggs", TypeCode.UInt32),
-        Tuple.Create("volatility", TypeCode.Double)));
-
-
-        public TradeBarsStream(int bar_period_seconds)
-        : this(bar_period_seconds, ViewStreamDatabase.instance) { }
-        public TradeBarsStream(int bar_period_seconds, ViewStreamDatabase vsd) {
-            _vsd = vsd;
-            _bar_period_seconds = bar_period_seconds;
-            _timesales_stream = new TimeSalesStream(10000, vsd);
-            _fire_period = TimeSpan.FromSeconds(bar_period_seconds);
-            _bbo_stream = new BBOStream(_vsd);
-        }
-
-        protected override void myOnFirstSubscriber() {
-            _timesales_stream.onChange += _on_timesales;
-            _bbo_stream.onChange += _on_bbo;
-            _vsd.addWorker(this);
-        }
-
-        protected override void myOnLastSubscriber() {
-            _vsd.removeWorker(this);
-            _timesales_stream.onChange -= _on_timesales;
-            _bbo_stream.onChange -= _on_bbo;
-        }
-
-        public void doWork() {
-            if (DateTime.Now - _last_fire_time > _fire_period) {
-                if (_dirty) {
-                    _dirty = false;
-                    _last_fire_time = DateTime.Now;
-                    BlueTable added_rows = new BlueTable(_table.schema);
-                    foreach (var kv in _sec_bar_map) {
-                        var sec_key = kv.Key;
-                        var tbar = kv.Value;
-                        IBlueRow new_row = new BlueRow(_table.schema, _table.Count, tbar.bar_start, DateTime.Now,
-                        tbar.bar_start.DayOfWeek.ToString(),
-                                                        tbar.n_entries, tbar.security, sec_key, tbar.low, tbar.high, tbar.open,
-                                                        tbar.close, tbar.vwap_sum, tbar.volume, tbar.n_sell_aggs, tbar.n_sell_aggs,
-                                                        tbar.volatility);
-                        added_rows.addRow(new_row);
-                        _table.addRow(new_row);
-                    }
-                    _sec_bar_map.Clear();
-                    DataPacket dp_out = new DataPacket(_table, added_rows, new BlueTable(_table.schema));
-                    myFireEvent(dp_out);
-                }
-            }
-        }
-
-        private void _on_timesales(object obj) {
-            DataPacket dp = obj as DataPacket;
-            foreach (IBlueRow r in dp.addedRows) {
-                uint sec_key = r.get<UInt32>("sec_key");
-                TradeBar tbar = null;
-                if (!_sec_bar_map.TryGetValue(sec_key, out tbar)) {
-                    tbar = new TradeBar(r.get<string>("sec"));
-                    _sec_bar_map.Add(sec_key, tbar);
-                }
-                _dirty = true;
-                tbar.add_trade(r);
-            }
-
-
-        }
-
-        private void _on_bbo(object obj) {
-
-        }
-    }
-
-
-}
@@ -0,0 +1,117 @@
+// namespace bts.SilverPine {
+//    using bts;
+//    using BCG;
+//    using MongoDB.Driver.Linq;
+//    using System.ComponentModel.DataAnnotations;
+//    using ZstdSharp.Unsafe;
+
+//    [DataStreamUserRespresentation("VWAP")]
+//    public class VWAPStream : ViewStream {
+//       // returns the VWAP stream for each contract 
+//       private static ViewStream build_vwap_stream(string symbol, DateTime begin_dt, DateTime end_dt) {
+//          var ohlcv_params = OHLCVStream.get_params(begin_dt, end_dt);
+//          ViewStream ohlc_str = new OHLCVStream(symbol, ohlcv_params);
+//          // stream is grouped per contract and then vwap is calculated per contract
+//          GlobalMapStream rs = new GlobalMapStream(ohlc_str, "ts_event", "contract",
+//          (rows, idx) => {
+//             Log.Gen.info($"VWAPStream.GlobalMapStream delegate n_rows: {rows.Count} idx: {idx}");
+//             IBlueRow ir = rows.ElementAt((int)idx);
+//             var sel_rows = rows.Skip((int)idx).Where(r => r.get_datetime("ts_event") >= end_dt && r.get_datetime("ts_event") <= begin_dt);
+//             var volumeprices = sel_rows.Select(r => Tuple.Create(r.get<int>("volume"), r.get<double>("close"))).ToList();
+//             double vwap = IndicatorCalc.calculate_VWAP(volumeprices);
+//             IBlueTableSchema schema = ir.schema.add_columns(Tuple.Create(typeof(double), "vwap"));
+//             MutableBlueRow nr = new MutableBlueRow(schema, ir);
+//             //vwap = -313.987;
+//             nr.setValue("vwap", vwap);
+//             return nr;
+//          });
+//          return rs;
+//       }
+//       // the args want dt1 > dt2, if its nto we, swap them
+//       static void swap_if_necessary(ref DateTime dt1, ref DateTime dt2) {
+//          if (dt1 < dt2) {
+//             DateTime aux = dt1;
+//             dt1 = dt2;
+//             dt2 = aux;
+//          }
+//       }
+
+//       // this assumes the ohlc_str has diff contracts
+//       public VWAPStream(string symbol_or_contract, DateTime begin_dt, DateTime end_dt) {
+//          swap_if_necessary(ref begin_dt, ref end_dt);
+//          Tuple<ContractUtils.ContractCategory, string> ccat_sym_contr = ContractUtils.get_symbol_category(symbol_or_contract);
+//          string symbol = String.Empty;
+//          symbol = ccat_sym_contr.Item2;
+//          var ohlcv_params = OHLCVStream.get_params(begin_dt, end_dt);
+//          ViewStream vwap_str = build_vwap_stream(symbol, begin_dt, end_dt);
+//          // vwap_str now has vwap for all contracts. if the arg was a symbol, we need to filter it for the active contract,
+//          // whihc we do with FutureActivesContractStream. If the arg was a contract we'll simply filter it
+//          ViewStream rs = vwap_str;
+//          switch (ccat_sym_contr.Item1) {
+//             case ContractUtils.ContractCategory.FutureString:
+//             case ContractUtils.ContractCategory.SpreadString:
+//                rs = new FilterStream(vwap_str, r => r.get<string>("contract").CompareTo(ccat_sym_contr.Item2) == 0);
+//                break;
+//             case ContractUtils.ContractCategory.Symbol:
+//                rs = new FuturesActiveContractStream(vwap_str);
+//                break;
+//          }
+//          rs = new FilterStream(rs, r => r.get_datetime("ts_event").CompareTo(begin_dt) == 0);
+//          rs = new NewProjectStream(rs, "! vwap");
+//          mySetRealSource(rs);
+//       }
+
+//       public VWAPStream(string symbol_or_contract, string begin_dt, string end_dt)
+//       :
+//       this(symbol_or_contract, DateTime.Parse(begin_dt), DateTime.Parse(end_dt)) { }
+//    }
+// }
+
+using System;
+
+using BCG;
+
+using bts.api;
+
+namespace bts.SilverPine {
+   /*
+      ----------------------------------------------------------------------
+                                  *Expression*
+      ----------------------------------------------------------------------
+          //nl("Find Settlement price of the SP500 future in the first week of 2023.")
+
+          r=FutureHistoricalBars("ES", "One_Day", "1/1/2024", "2/1/2024")
+          r=math(r, "vprice = volume * close")
+          r=aggregate(r, "vprice:Sum, volume:Sum")
+          r=math(r, "vwap = vprice_sum / volume_sum")
+          r=newProject(r, "vwap !")
+
+      ----------------------------------------------------------------------
+                                    *Schema*
+      ----------------------------------------------------------------------
+          id                              System.Int32
+          vwap                            System.Double
+   */
+
+   [DataStreamUserRespresentation("VWAP")]
+   public class VWAPStream : ViewStream {
+      public VWAPStream(string symbol_or_contract, DateTime beg_dt, DateTime end_dt)
+      :
+      this(symbol_or_contract, beg_dt, end_dt, ViewStreamDatabase.instance) { }
+
+      public VWAPStream(string symbol_or_contract, DateTime beg_dt, DateTime end_dt, ViewStreamDatabase db) {
+         //var fc = new ContractUtils.FutureContract(symbol_or_contract);
+         ViewStream r = new FutureHistoricalBarsStream(symbol_or_contract, beg_dt.ToString(), end_dt.ToString());
+         // since this is already filtered, we can groupBy iit without a groupBy column
+         r = new MathStream(r,
+                 "vprice = volume * close");
+         r = new NonIncGroupByStream(r,
+                                      "vprice:Sum, volume:Sum");
+         r = new MathStream(r,
+                             "vwap = vprice_sum / volume_sum");
+         r = new NewProjectStream(r,
+                                   "vwap !");
+         mySetRealSource(r);
+      }
+   }
+}
@@ -0,0 +1,18 @@
+namespace bts.SilverPine {
+   using BCG;
+
+   [DataStreamUserRespresentation("Events")]
+   public class EventsStream : ViewStream {
+      public EventsStream(string event_type) : this(ViewStreamDatabase.instance, event_type) { }
+
+      public EventsStream(ViewStreamDatabase vsd, string event_type) {
+         ViewStream rs =
+            new SQLiteStream("Data Source=C:\\Users\\m_tah\\development\\data\\stream_data.db;Version=3;",
+                              "select * from events");
+         if (!String.IsNullOrEmpty(event_type)) {
+            rs = new FilterStream(rs, r => String.Compare(event_type, r.get<string>("event_type"), true) == 0);
+         }
+         mySetRealSource(rs);
+      }
+   }
+}
@@ -0,0 +1,105 @@
+namespace bts.SilverPine {
+   using BCG;
+
+   using bts;
+   // Given volume data per date for all futures (all exps + spreads) it selects the
+   // the future with the highest volume  
+   [DataStreamUserRespresentation("FuturesActiveContract")]
+   public class FuturesActiveContractStream : ViewStream, INonIncGroupByEvaluator {
+
+      public FuturesActiveContractStream(string future_name)
+      // the active contract's resoulution is a day, so we used 
+      : this(new OHLCVStream(future_name,
+            // uses Historic Daily bars to find the acive contract
+            new OHLCVStream.OHLCVParams(OHLCVStream.Source.SQL, OHLCVStream.OHLCVType.HISTORIC, OHLCVStream.OHLCVBarSize.ONE_DAY,
+                                       OHLCVStream.get_default_historic_range()))) { }
+
+      // simply finds the most active (highest volume) contract per day
+      public FuturesActiveContractStream(ViewStream ohlcv_stream) {
+         var rs = new FilterStream(ohlcv_stream, r => {
+            ContractUtils.ContractType t = ContractUtils.get_type(r.get<string>("contract"));
+            return t == ContractUtils.ContractType.Future;
+         });
+         mySetRealSource(new NonIncGroupByStream(rs, this));
+      }
+
+      public IBlueRowCollection evaluate(IDictionary<int, List<IBlueRow>> grouped_rows) {
+         Func<string, bool> is_future = (c) => ContractUtils.get_category(c) == ContractUtils.ContractCategory.FutureString;
+         BlueTable rv = null;
+         Func<List<IBlueRow>, IBlueRow> reduce = (rows) => {
+            var future_rows = rows.Where(r => is_future(r.get<string>("contract")));
+            IBlueRow rv = future_rows.ElementAt(0);
+            foreach (IBlueRow r in future_rows) {
+               if (r.get<int>("volume") > rv.get<int>("volume"))
+                  rv = r;
+            }
+            return rv;
+         };
+         foreach (var kv in grouped_rows) {
+            var rows = kv.Value;
+            if (rows == null || rows.Count == 0)
+               continue;
+            IBlueRow rr = reduce(rows);
+            rv = rv ?? new BlueTable(rr.schema);
+            rv.addRow(rr);
+         }
+         return rv;
+      }
+      public string[] get_group_by_cols() {
+         return new string[] { "ts_event" };
+      }
+   }
+
+   // Reduces the OHLCV stream by date and by either contract (if given explicty) or active contract
+   // is the first arg is a symbol and not a contract
+   public class FilteredOHLCVStream : ViewStream {
+      public FilteredOHLCVStream(string symbol_or_contract)
+      :
+      this(symbol_or_contract, DateTime.Today.Date.Subtract(TimeSpan.FromDays(1)), DateTime.MinValue) { }
+
+      public FilteredOHLCVStream(string symbol_or_contract, DateTime dt1, DateTime dt2) {
+         Tuple<ContractUtils.ContractCategory, string> tup = ContractUtils.get_symbol_category(symbol_or_contract);
+         var ohlcv_params = OHLCVStream.get_params(dt1, dt2);
+         ViewStream rs = new OHLCVStream(tup.Item2, ohlcv_params);
+         switch (tup.Item1) {
+            case ContractUtils.ContractCategory.FutureString:
+            case ContractUtils.ContractCategory.SpreadString:
+               rs = new FilterStream(rs, $"contract = '{symbol_or_contract}'");
+               break;
+            case ContractUtils.ContractCategory.Symbol:
+               rs = new FuturesActiveContractStream(rs);
+               break;
+         }
+         rs = new FilterStream(rs, r => r.get_datetime("ts_event") <= dt1 &&
+                                          r.get_datetime("ts_event") >= dt2);
+         mySetRealSource(rs);
+      }
+   }
+}
+
+
+namespace bts.SilverPine {
+   using BCG;
+
+   using bts.api;
+
+   using Newtonsoft.Json.Serialization;
+
+   [DataStreamUserRespresentation("FuturesContinousContract")]
+   public class FuturesContinousContractStream : ViewStream {
+      // the argument is symbol, like "ES" etc and not the contract itself
+      public FuturesContinousContractStream(string symbol) : this(symbol, ViewStreamDatabase.instance) { }
+      public FuturesContinousContractStream(string symbol, ViewStreamDatabase db) {
+         symbol = symbol.EndsWith("*") ? symbol : symbol + "*";
+         ViewStream r = new FuturesHistorical1DayBarsStream(symbol);
+         r = new ContractExpanderStream(r, "symbol");
+         r = new FilterStream(r, "contract_type='Future'");
+         // finding the max volume across symbols, if we need to do this for open_interest, then join here 
+         // with the open interest stream on symbol/contract
+         r = new NonIncGroupByStream(r, "ts_event", "volume:Max");
+         r = new NewProjectStream(r, "ts_event symbol !");
+         r = new SortStream(r, "-ts_event");
+         mySetRealSource(r);
+      }
+   }
+}
@@ -0,0 +1,16 @@
+namespace bts.SilverPine {
+   using bts;
+   using BCG;
+
+   // it uses the volume (better to use open interest perhap) to detect the rollover, ie when a new 
+   // future exp becomes the highest volume one
+   [DataStreamUserRespresentation("FuturesRoll")]
+   public class FuturesRollStream : ViewStream {
+      public FuturesRollStream(string future_symbol) {
+         ViewStream rs = new FuturesContinousContractStream(future_symbol);
+         rs = new ColumnEvolutionStream(rs, "ts_event", "symbol");
+         rs = new NewProjectStream(rs, "symbol ts_event:roll_date !");
+         mySetRealSource(rs);
+      }
+   }
+}
@@ -0,0 +1,37 @@
+namespace bts.SilverPine {
+   using BCG;
+
+   [DataStreamUserRespresentation("OHLCVPriceDiff")]
+   public class OHLCVPriceDiffStream : ViewStream {
+      // TODO, abstract this out
+      private static ViewStream _get_ohlcv_steam(string symbol_or_contract, DateTime d1, DateTime d2) {
+         var tup = ContractUtils.get_symbol_category(symbol_or_contract);
+         var ohlcv_params = OHLCVStream.get_params(d1, d2);
+         var cont_cat = tup.Item1;
+         string symbol = tup.Item2;
+         ViewStream rs = new OHLCVStream(symbol, ohlcv_params);
+         switch (cont_cat) {
+            case ContractUtils.ContractCategory.FutureString:
+            case ContractUtils.ContractCategory.SpreadString:
+               rs = new FilterStream(rs, $"contract = '{symbol_or_contract}'");
+               break;
+            case ContractUtils.ContractCategory.Symbol:
+               rs = new FuturesActiveContractStream(rs);
+               break;
+         }
+         return rs;
+      }
+
+      public OHLCVPriceDiffStream(string symbol_or_contract, DateTime d1, DateTime d2)
+      : this(symbol_or_contract, d1, d2, "close") { }
+
+      public OHLCVPriceDiffStream(string symbol_or_contract, DateTime d1, DateTime d2, string col) {
+         ViewStream rs = _get_ohlcv_steam(symbol_or_contract, d1, d2);
+         rs = new ColumnDifferenceStream(rs, r => d1.Equals(r.get_datetime("ts_event")),
+                                              r => d2.Equals(r.get_datetime("ts_event")), col, col);
+         rs = new NewProjectStream(rs, "diff !");
+         mySetRealSource(rs);
+      }
+
+   }
+}
@@ -0,0 +1,210 @@
+namespace bts.SilverPine {
+   using bts;
+   using BCG;
+   using MongoDB.Driver;
+
+   // OHLCV request breakdown
+   // symbol_or_contract (if symbol, get continous, otherwise get raw and filter it)
+   // bar_size (one min, one day)
+   // live/historic
+   // time range
+
+   // internal code oranization only
+   public class OHLCV_1min_stream : ViewStream {
+
+      public OHLCV_1min_stream(string symbol_or_contract, string range_beg, string range_end) :
+      this(symbol_or_contract, DateTime.Parse(range_beg), DateTime.Parse(range_end)) { }
+
+      public OHLCV_1min_stream(string symbol_or_contract, DateTime range_beg, DateTime range_end) {
+         Func<string, bool, SQLiteStream> get_sqlite_str = (symbol, is_continous_contracts) => {
+            string table_name = is_continous_contracts ? $"ohlcv_1min_{symbol}_continous" : $"{symbol}_ohlcv_1min";
+            string db_str = $"Data Source={SQLiteHelper.location}{SQLiteHelper.db_name};Version=3";
+            string range_beg_iso8601 = range_beg.ToString("yyyy-MM-ddTHH:mm:ss"),
+               range_end_iso8601 = range_end.ToString("yyyy-MM-ddTHH:mm:ss");
+            string sql_str = $"select * from {table_name} where DATE(ts_event) >= '{range_beg_iso8601}' AND DATE(ts_event) <= '{range_end_iso8601}'";
+            return new SQLiteStream(db_str, sql_str);
+         };
+
+         Tuple<ContractUtils.ContractCategory, string> tup = ContractUtils.get_symbol_category(symbol_or_contract);
+         var contract_cat = tup.Item1;
+         switch (contract_cat) {
+            case ContractUtils.ContractCategory.Symbol: {
+                  ViewStream res = get_sqlite_str(tup.Item2, true);
+                  mySetRealSource(res);
+                  // this creates the continous stream for 1 min bars, but we store it in the DB, so we just read it from it
+                  // ViewStream ohlcv_1min = get_sqlite_str(tup.Item2, false);
+                  // // the following is whats written to the DB in the _contionous table
+                  // ohlcv_1min = new DateTimeExpanstionStream(ohlcv_1min, "ts_event");
+                  // ViewStream active_contr_str = new FuturesActiveContractStream(tup.Item2);
+                  // ViewStream res = new NonIncJoinStream(ohlcv_1min, "ts_event_date symbol", "ts_event ts_event_date symbol open high low close volume",
+                  //                                     active_contr_str, "ts_event contract", "ts_event:date contract:continous_contract");
+                  // res = new NullFilterStream(res, "continous_contract", true);
+                  // res = new NewProjectStream(res, "ts_event date continous_contract open high low close volume !");
+                  // mySetRealSource(res);
+                  break;
+               }
+            default: {
+                  ViewStream sql_str = get_sqlite_str(tup.Item2, false);
+                  ViewStream rs = new FilterStream(sql_str, $"symbol='{symbol_or_contract}'");
+                  // filter here
+                  mySetRealSource(rs);
+                  break;
+               }
+         }
+      }
+   }
+
+   // this is to make bigger bars form smaller bars. Right now the assumption it only works
+   // with symbols vs contracts, and the 1min <= aggregation_period < 1day
+   [DataStreamUserRespresentation("OHLCVAggregated")]
+   public class OHLCVAggregatedStream : ViewStream {
+      public OHLCVAggregatedStream(string symbol, string agregation_period, string range_beg, string range_end) :
+      this(symbol, TimeBasedFilter.parse_duration(agregation_period), DateTime.Parse(range_beg), DateTime.Parse(range_end)) {
+      }
+
+      public OHLCVAggregatedStream(string symbol, TimeSpan aggregation_period, DateTime range_beg, DateTime range_end) {
+         Tuple<ContractUtils.ContractCategory, string> tup = ContractUtils.get_symbol_category(symbol);
+         Log.assert(tup.Item1 == ContractUtils.ContractCategory.Symbol);
+         if (aggregation_period < TimeSpan.FromDays(1.0) &&
+            aggregation_period > TimeSpan.FromMinutes(1.0)) {
+            ViewStream r = new OHLCV_1min_stream(symbol, range_beg, range_end);
+            r = new DateTimeExpanstionStream(r,
+                                              "ts_event");
+            double tot_secs = aggregation_period.TotalSeconds;
+            r = new MathStream(r,
+                                $"bucket = Floor(ts_event_seconds_of_day / {tot_secs})");
+            r = new NonIncGroupByStream(r,
+                                         "bucket",
+                                         // column:operation
+                                         "open:first, high:max, low:min, close:last, volume:sum",
+                                         "ts_event");
+            r = new SortStream(r,
+                                "-ts_event");
+            mySetRealSource(r);
+         } else {
+            // fix it later
+            Log.assert(false);
+         }
+      }
+   }
+
+
+   [DataStreamUserRespresentation("OHLCV")]
+   // This stream retreives the OHLCV table as is, and fires it as-is. 
+   // The only logic here is as follows:
+   // 1. deciding bw sql table or pubsub
+   // 2. deciding bw live, historic or append(live, historic)
+   // 3. deciding size of bar
+   // (the logic for2 and 3 is in get_params)
+   public class OHLCVStream : ViewStream {
+      public enum Source {
+         SQL,
+         PubSub
+      }
+      public enum OHLCVType {
+         LIVE,
+         HISTORIC,
+         LIVE_AND_HISTORIC
+
+      };
+      public enum OHLCVBarSize {
+         ONE_DAY = 0,
+         ONE_HOUR,
+         ONE_MIN,
+         ONE_SEC
+      };
+      public class OHLCVParams {
+         public readonly Source source = Source.SQL;
+         public readonly OHLCVType type = OHLCVType.HISTORIC;
+         public readonly OHLCVBarSize bar_size;
+         public readonly Tuple<DateTime, DateTime> optional_dt_range;
+         public OHLCVParams(Source source_, OHLCVType type_, OHLCVBarSize bar_size_,
+                            Tuple<DateTime, DateTime> optional_dt_range_) {
+            source = source_;
+            type = type_;
+            bar_size = bar_size_;
+            optional_dt_range = optional_dt_range_;
+         }
+      }
+
+      public static OHLCVParams get_params(DateTime dt1, DateTime dt2) {
+         Func<DateTime, bool> today = (dt) => dt.Date >= DateTime.Now.Date;
+         TimeSpan period = dt1 > dt2 ? dt1 - dt2 : dt2 - dt1;
+         bool has_today = today(dt1) || today(dt2);
+         bool has_history = !today(dt1) && !today(dt2);
+         Source source = Source.SQL;
+         OHLCVType type = (has_today && has_history) ? OHLCVType.LIVE_AND_HISTORIC :
+         ((has_today && !has_history) ? OHLCVType.LIVE : OHLCVType.HISTORIC);
+         // TODO, for less than a day, its defaulted to 1 min
+         OHLCVBarSize bar_size = period >= TimeSpan.FromDays(1.0) ? OHLCVBarSize.ONE_DAY : OHLCVBarSize.ONE_MIN;
+         Tuple<DateTime, DateTime> optional_dt_range = new Tuple<DateTime, DateTime>(dt1, dt2);
+         return new OHLCVParams(source, type, bar_size, optional_dt_range);
+      }
+
+      public static Tuple<DateTime, DateTime> get_default_historic_range() {
+         return Tuple.Create(DateTime.Now.Date.Subtract(TimeSpan.FromDays(1)), DateTime.MinValue);
+      }
+
+      private static string get_table_from_barsize(OHLCVBarSize bar_size, string symbol) {
+         switch (bar_size) {
+            case OHLCVBarSize.ONE_DAY:
+               return $"{symbol}_OHLCV_1DAY";
+            case OHLCVBarSize.ONE_SEC:
+            case OHLCVBarSize.ONE_MIN:
+            case OHLCVBarSize.ONE_HOUR:
+               return $"{symbol}_OHLCV_1MIN";
+         }
+         Log.assert(false);
+         return String.Empty;
+      }
+
+      private Source _sourse = Source.SQL; // get from config
+      private readonly string _location = "C:\\Users\\m_tah\\development\\data\\";
+      private readonly string _db_name = "stream_data.db";
+      // the symbol is used ot retreive the correct table, the data flows as-is from the table
+      // with some name / type changes
+      public OHLCVStream(string symbol, OHLCVParams ohlcv_params) {
+         switch (ohlcv_params.type) {
+            case OHLCVType.LIVE:
+            case OHLCVType.LIVE_AND_HISTORIC:
+               // TODO
+               Log.assert(false);
+               break;
+            case OHLCVType.HISTORIC:
+               // TODO: assuming 1 day only for historic               
+               string table_name = get_table_from_barsize(ohlcv_params.bar_size, symbol);// $"{symbol}_OHLCV_1DAY";
+                                                                                         //string table_name = $"{symbol}_{ohlcv_params.bar_size}";
+               string db_str = $"Data Source={_location}{_db_name};Version=3";
+               string sql_str = $"select * from {table_name}";
+               // "Data Source=C:\\Users\\m_tah\\development\\data\\stream_data.db;Version=3;"
+               ViewStream rs = new SQLiteStream(db_str, sql_str);
+               rs = new SchemaGuesserStream(rs);
+               if (ohlcv_params.bar_size == OHLCVBarSize.ONE_DAY)
+                  rs = new DateStandardizationStream(rs, "ts_event", DateConversion.ChopTime);
+               rs = new NewProjectStream(rs, "symbol:contract");
+               mySetRealSource(rs);
+               break;
+         }
+      }
+      public OHLCVStream(string symbol)
+      :
+      this(symbol, new OHLCVParams(Source.SQL, OHLCVType.HISTORIC, OHLCVBarSize.ONE_DAY, get_default_historic_range())) { }
+
+      public OHLCVStream(string symbol, OHLCVBarSize bar_size)
+      :
+      this(symbol, new OHLCVParams(Source.SQL, OHLCVType.HISTORIC, bar_size, get_default_historic_range())) {
+      }
+
+      public OHLCVStream(string symbol, string bar_size)
+      :
+      this(symbol, Enum.Parse<OHLCVBarSize>(bar_size)) { }
+
+      // aggregates the bars
+      public OHLCVStream(string symbol, TimeSpan agregation_period) {
+
+      }
+
+
+
+   }
+}
@@ -0,0 +1,26 @@
+namespace bts.SilverPine {
+   using BCG;
+
+   using bts;
+   // used directly in blueorb
+   [DataStreamUserRespresentation("Settlement")]
+   public class SettlementPriceStream : ViewStream {
+
+      // public SettlementPriceStream(string future_name, string date_txt)
+      // : this(future_name, date_txt, date_txt) { }
+
+      // public SettlementPriceStream(string future_name, string begin_dt, string end_dt)
+      // : this(future_name, DateTime.Parse(begin_dt).Date, DateTime.Parse(end_dt).Date) { }
+
+      public SettlementPriceStream(string future_contract, DateTime begin_dt)
+      : this(future_contract, begin_dt, begin_dt) { }
+
+      public SettlementPriceStream(string future_contract, DateTime begin_dt, DateTime end_dt) {
+         ContractUtils.FutureContract fc = new ContractUtils.FutureContract(future_contract);
+         ViewStream rs = new FutureHistoricalBarsStream(fc, BarSize.ONE_DAY, new DateTimeRange(begin_dt, end_dt));
+         rs = new NewProjectStream(rs, "symbol ts_event:date close:settlement !");
+         mySetRealSource(rs);
+      }
+
+   }
+}
@@ -0,0 +1,32 @@
+namespace bts.SilverPine {
+   using BCG;
+   // Takes multiple bars and collapses them into one, which is aggregated over a bigger time period.
+   // All bars belonging to a common group_by_col, are aggregated
+   public class BarsAggregatorStream : ViewStream {
+      public BarsAggregatorStream(ViewStream ohlcv_stream, params string[] group_by_cols) {
+         ViewStream rs = new NonIncGroupByStream(ohlcv_stream,
+            () => { return group_by_cols; },
+            (group_rows) => {
+               var sorted_rows = group_rows.OrderBy(r => r.get_datetime("ts_event"));
+               IBlueRow first_row = sorted_rows.First(),
+                        last_row = sorted_rows.Last();
+               double open = first_row.get<double>("open");
+               double close = last_row.get<double>("close");
+               double low = sorted_rows.Min(r => r.get<double>("low"));
+               double high = sorted_rows.Max(r => r.get<double>("high"));
+               double volume = 0.0;
+               foreach (IBlueRow r in sorted_rows)
+                  volume += r.get<double>("volume");
+               MutableBlueRow rv = new MutableBlueRow(first_row);
+               rv.setValue("open", open);
+               rv.setValue("close", close);
+               rv.setValue("low", low);
+               rv.setValue("high", high);
+               rv.setValue("volume", volume);
+               return new IBlueRow[] { rv }.ToList();
+            }
+         );
+         mySetRealSource(rs);
+      }
+   }
+}
@@ -0,0 +1,32 @@
+namespace bts.SilverPine {
+   using BCG;
+
+   // this is an internal stream. its job is to take an OHLCV stream
+   // and filter it for 
+   // 1. symbol (table_name and active contract) and 
+   // 2. time  (can be a day of the week, month, and a time of the day period)
+   public class BarsSearcherStream : ViewStream {
+      public BarsSearcherStream(string symbol, OHLCVStream.OHLCVBarSize bar_size,
+                                 string day_of_week, TimeSpan tod_begin, TimeSpan tod_end) {
+         ViewStream rs = new OHLCVStream(symbol, bar_size);
+         rs = new DateTimeExpanstionStream(rs, "ts_event");
+         rs = new FilterStream(rs, r => {
+            DateTime ts = r.get_datetime("ts_event");
+            return r.get<string>("ts_event_day") == day_of_week.ToString() &&
+                  ts.TimeOfDay >= tod_begin && ts.TimeOfDay <= tod_end;
+         });
+         // look at symbol and either filter ir ot do the active contracts
+         // assuming its symbol
+         rs = new NonIncJoinStream(rs, "ts_event_date", "open high low close volume contract ts_event",
+                                 new FuturesActiveContractStream(symbol), "ts_event", "ts_event:date contract:active_contract");
+         // filter it for the active contract only
+         rs = new FilterStream(rs, r => String.Compare(r.get<string>("contract"), r.get<string>("active_contract")) == 0);
+         mySetRealSource(rs);
+      }
+      public BarsSearcherStream(string symbol, string bar_size,
+                                 string day_of_week, string tod_begin, string tod_end)
+                                 :
+                                 this(symbol, Enum.Parse<OHLCVStream.OHLCVBarSize>(bar_size), day_of_week,
+                                       TimeSpan.Parse(tod_begin), TimeSpan.Parse(tod_end)) { }
+   }
+}
@@ -1,130 +0,0 @@
-namespace BCG {
-   using bts.api.orderman;
-
-   enum TSGroupByOperation {
-      OHLC = 0,
-      Statistics
-   }
-
-   [DataStreamUserRespresentation("timeseriesanalysis")]
-   public class TimeSeriesGroupByStream : ViewStream {
-      public TimeSeriesGroupByStream(ViewStream md_str, string timebucket_col, string output_col, string operation) {
-         ViewStream rs = new NonIncGroupByStream(md_str, new OHLCNonIncGroupByEvaluator(timebucket_col, output_col));
-         mySetRealSource(rs);
-      }
-   }
-   public class OHLC {
-      public OHLC(double open_args, double high_args, double low_args, double close_args) {
-         open = open_args;
-         high = high_args;
-         low = low_args;
-         close = close_args;
-      }
-      public readonly double open, high, low, close;
-   }
-   public class OHLCNonIncGroupByEvaluator : INonIncGroupByEvaluator {
-      private string _timestamp_col;
-      private string _output_col;
-      private IBlueTableSchema _out_schema;
-      public OHLCNonIncGroupByEvaluator(string timestamp_col, string output_col) {
-         _timestamp_col = timestamp_col;
-         _output_col = output_col;
-      }
-
-      string open_col {
-         get {
-            return $"{_output_col}_open"; ;
-         }
-      }
-      string high_col {
-         get {
-            return $"{_output_col}_high"; ;
-         }
-      }
-      string low_col {
-         get {
-            return $"{_output_col}_low"; ;
-         }
-      }
-      string close_col {
-         get {
-            return $"{_output_col}_close"; ;
-         }
-      }
-      private void _build_schema(IBlueTableSchema input_schema) {
-         List<Tuple<Type, string>> new_cols = new List<Tuple<Type, string>>();
-         ColumnInfo ci = input_schema[_output_col];
-         new_cols.Add(Tuple.Create(ci.colType, open_col));
-         new_cols.Add(Tuple.Create(ci.colType, high_col));
-         new_cols.Add(Tuple.Create(ci.colType, low_col));
-         new_cols.Add(Tuple.Create(ci.colType, close_col));
-         // should hide the output col
-         _out_schema = BlueDataUtility.add_columns(input_schema, new_cols.ToArray());
-      }
-
-      public IBlueRowCollection evaluate(IDictionary<int, List<IBlueRow>> grouped_rows) {
-         if (_out_schema == null) {
-            if (grouped_rows.Count > 0) {
-               foreach (var kv in grouped_rows) {
-                  var rows = kv.Value;
-                  if (rows != null && rows.Count > 0) {
-                     _build_schema(rows.ElementAt(0).schema);
-                  }
-               }
-            }
-         }
-         BlueTable rv = new BlueTable(_out_schema);
-         foreach (var kv in grouped_rows) {
-            int id = kv.Key;
-            IEnumerable<IBlueRow> rows = kv.Value;
-            // sort rows on time to compute o,c correctly??
-            if (rows != null && rows.Count() > 0) {
-               IBlueRow reprow = rows.ElementAt(0);
-               IEnumerable<double> vals = BlueDataUtility.get_colvalues<double>(rows, _output_col);
-               OHLC ohlc = compute_ohlc(vals);
-               MutableBlueRow nr = new MutableBlueRow(_out_schema, reprow);
-               nr.setValue("id", kv.Key);
-               nr.setValue(open_col, ohlc.open);
-               nr.setValue(high_col, ohlc.high);
-               nr.setValue(low_col, ohlc.low);
-               nr.setValue(close_col, ohlc.close);
-               rv.addRow(nr);
-            }
-         }
-         return rv;
-      }
-
-      public string[] get_group_by_cols() {
-         return new string[] { _timestamp_col };
-      }
-
-      static OHLC compute_ohlc(IEnumerable<double> prices) {
-         double o = prices.ElementAt(0);
-         double c = prices.ElementAt(prices.Count() - 1);
-         double l = prices.Min();
-         double h = prices.Max();
-         return new(o, h, l, c);
-      }
-   }
-   [DataStreamUserRespresentation("RSI")]
-   public class RSIStream : ViewStream {
-      // for ms_str expects a timeseries with a timestamp col and an output col (price col) to calculate the ohlc for
-      public RSIStream(ViewStream md_str, string timebucket_column, string output_col) {
-
-      }
-   }
-   [DataStreamUserRespresentation("Bollinger")]
-   public class BollingerBandStream : ViewStream {
-      // for ms_str expects a timeseries with a timestamp col and an output col (price col) to calculate the ohlc for
-      public BollingerBandStream(ViewStream md_str, string timebucket_column, string output_col) {
-      }
-   }
-   [DataStreamUserRespresentation("OHLC")]
-   public class OHLCStream : ViewStream {
-      // for ms_str expects a timeseries with a timestamp col and an output col (price col) to calculate the ohlc for
-      public OHLCStream(ViewStream md_str, string timebucket_column, string output_col) {
-         ViewStream rs = new TimeSeriesGroupByStream(md_str, timebucket_column, output_col, TSGroupByOperation.OHLC.ToString());
-         mySetRealSource(rs);
-      }
-   }
-}
@@ -0,0 +1,48 @@
+namespace Bts.DataStreamPublisher;
+
+using Bts.PubSub;
+
+//using CommandLine;
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+public class SharedStreamConfig {
+
+   [JsonPropertyName("mongodb_url")]
+   public string MONGO_DB_URL { get; set; }
+
+   [JsonPropertyName("mongodb_db_name")]
+   public string MONGO_DB_NAME { get; set; }
+
+   [JsonPropertyName("mongodb_db_collection")]
+   public string MONGO_DB_COLLECTION { get; set; }
+
+   [JsonPropertyName("benzinga_news_ttl")]
+   public string BENZINGA_NEWS_TTL { get; set; }
+
+   [JsonPropertyName("pubsub")]
+   public PubSubConfig? PubSub { get; set; }
+
+
+   /// <summary>
+   /// Merges options with those defined in `ConfigFile` (does not overwrite
+   /// non-null existing values).
+   /// </summary>
+   /// <exception cref="InvalidOperationException">
+   /// JsonSerializer.Deserialize returns null
+   /// </exception>
+   public void MergeWithConfigFile() {
+      // if (ConfigFile is null) {
+      //    return;
+      // }
+      // var cfgString = File.ReadAllText(ConfigFile);
+      // var parsedCfg = JsonSerializer.Deserialize<DataStreamPublisherConfig>(cfgString) ??
+      //    throw new InvalidOperationException($"Failed to parse json from {ConfigFile}");
+      // MergeWith(parsedCfg);
+      // ConfigFile = null;
+   }
+}
@@ -0,0 +1,276 @@
+# **DSL Master Learning Prompt**
+
+This prompt is designed to teach the foundational concepts of the DSL for financial data streaming and transformation. The DSL operates on "streams," which fetch, transform, and process financial market data in real-time or historical contexts. Forget everything I have taught you about this DSL until now (since there are some contradicting and breaking changes that can not be reconciled with anything you might have learned before) and re-learn from this prompt.
+
+# **Introduction to Streams**
+
+The DSL operates on "streams," its core functional units. Streams fetch tabular (from database, text files or network) or transform/operate on tabular data. They either act as **Datasources** (e.g., FutureHistoricalBars), that bootstrap the data or **Operators** (e.g., `FilterStream, ProjectStream, SingleRowMathStream, GroupByStream` etc) that transform or filter tabular data. Though the DSL can work on any tabular data, it is primarily written for streaming or static financial data
+
+# **Streams**
+
+## **FutureHistoricalBars (DataSource Stream)**
+
+The `FutureHistoricalBars` stream fetches OHLCV (Open, High, Low, Close, Volume) bars for financial futures contracts. It acts as a **data source** and provides the initial table that can be further processed by operators.
+
+#### **Stream arguments (parameters):**
+
+1. **`Future_contract` (string):** Specifies the contract to fetch data for. This can be one of the following 3 types  
+1. **Continuous contracts:** Use the base symbol (e.g., `ES` for S\&P 500 futures, CL for WTI Crude Oil Futures, GC for Gold Futures etc).  
+2. **Specific contracts:** Use a detailed symbol (e.g., `ESZ9` for December 2029 S\&P 500 futures, `GCU9`, for September 2025 Gold Future, `SIH9-J9`,  
+   For SIlver March April 2019 Silver Future Calendar spread)  
+3. **All Contracts for a symbol:** If instead of specifying individual contracts, data for all contracts for a specific symbol is needed, use the symbol name followed by *.* So `ES*` for all SP Futures and spread contracts, ZC\* for all corn contracts etc.  
+2. **`Bar_size` (string):**  The second argument defines the granularity of the bars. Accepted values:  
+   * `ONE_DAY` – Daily bars.  
+     * `ONE_MIN` – Minute bars.  
+3. **Optional:**  
+   * **`range_begin` (datetime):** Start of the desired time range.  
+   * **`range_end` (datetime):** End of the desired time range.
+
+#### **Schema:**
+
+The output of `FutureHistoricalBars` produces a table with the following schema (this is a classic OHLCV schema that you can find online easily)
+
+| id   | ts\_event             | contract | open    | high    | low     | close   | volume |
+| :--- | :-------------------- | :------- | :------ | :------ | :------ | :------ | :----- |
+| `0`  | `2024-06-08T00:00:00` | `ESZ9`   | 5001.50 | 5052.25 | 4990.75 | 5025.50 | 866000 |
+|      |                       |          |         |         |         |         |        |
+
+# **FilterStream (Operator Stream)**
+
+Filters rows of a parent stream based on conditions applied to its columns.
+
+#### **Stream arguments (parameters):**
+
+1. `parent_stream (another stream)` This stream provides the data, that the FilterStream subscribers to  
+2. `filter_condition (string)` This is a text expressing a boolean condition that is applied to each row of the parent stream, if it returns false, the row is filtered out
+
+#### **Examples:**
+
+1. **NL prompt:** Show me all the days when the open price for SP 500 Futures was greater than or equal to 5000:  
+   **DSL:** `r = FilterStream(FutureHistoricalBars("ES", "ONE_DAY"), "open >= 5000")`  
+2. **NL prompt:** Show me all the days when the close price for SP 500 Futures was less than 3000 and volume was greater than half a million   
+   **DSL:** `r = FilterStream(FutureHistoricalBars("ES", "ONE_MIN"), "close < 3000 and volume > 500000)`  
+3. **NL prompt:** Show me all the daily bars for the SP 500 Future spread ESU9-Z9 where its volume exceeded 100K  
+   **DSL:** `r = FilterStream(FutureHistoricalBars("ES*", "ONE_MIN"), "contract = ‘ESU9-Z9’ and volume >= 100000”)`  
+   
+
+**Notice** how expressions can be joined with AND and OR. These expressions can be nested with parenthesis.Also string needs to be in single quotes.
+
+# **ProjectStream (Operator Stream)**
+
+Changes the name of the columns in the parent stream. So for example, if we have to rename the close column to settlement, we can do this with ProjectStream
+
+#### **Stream arguments (parameters):**
+
+3. `parent_stream (another stream)` This stream provides the data, that the ProjectStream subscribers to  
+4. `renaming (string)` This is a text expressed in the form original\_col\_name:projected\_col\_name, original\_col\_name:projected\_col\_name, 
+
+#### **Examples:**
+
+	The following DSL renames the open column to O, high to H, low to L and close to C  
+**DSL:** `r = ProjectStream(FutureHistoricalBars("ES", "ONE_DAY"), "open:O, high:H, low:L, close:C”)`
+
+# **GroupByStream (Operator Stream)**
+
+The `GroupByStream` function is a DSL operator used to perform aggregation operations on rows of a parent stream. It works similarly to SQL’s `GROUP BY`, allowing rows to be grouped by a specific column (optional) and applying aggregation functions to other columns. 
+
+---
+
+#### **Stream arguments (parameters):**
+
+1. **parent\_stream (another stream):** The source stream provides the data to be operated on  
+2. **Group By Column (string, optional):**
+
+   A column in the schema of the parent stream used to group rows. All rows with the same value in this column are aggregated into a single group. If omitted, the aggregation is performed over all rows of the parent stream.
+
+3. **Operation Text (string):** A string defining the column(s) to be aggregated and the operation(s) to apply, in the format:  
+   `<column_name: operation>, <column_name: operation>, …` Each `<column_name: operation>` pair produces a new column in the output stream. The new column name is formatted as `<column_name>_<operation>` in lowercase.
+
+---
+
+#### **Valid Aggregation Operations:**
+
+* `Mean`: Arithmetic mean of the values.  
+* `Median`: Median of the values.  
+* `Mode`: Most frequently occurring value.  
+* `Range`: Difference between the max and min.  
+* `StdDev`: Standard deviation of the values.  
+* `Max`: Maximum value.  
+* `Min`: Minimum value.  
+* `First`: First value in the group.  
+* `Last`: Last value in the group.  
+* `Sum`: Sum of the values.  
+* `EdgeDiff`: Difference between the edge values, meaning the difference between a certain col belonging to the first and last rows respectively  
+* `EdgeDiffPercentage`: Same as `EdgeDiff` but expressed as percentage instead
+
+---
+
+#### **Examples:**
+
+1. **NL:**   
+   Find the total SP 500 volume executed in the first week of 2021\.  
+   **DSL:**   
+   `bars = FutureHistoricalBars("ES", “ONE_DAY”, “2021-01-03T00:00:00”,“2021-01-11T00:00:00”)`   
+   `result = GroupByStream(bars,”volume_Sum”)`  
+2. **NL:**   
+   **Find the highest price and lowest price of the  WTI Crude Oil futures over the first half of 2024**  
+   **DSL:**   
+   `bars = FutureHistoricalBars("CL", “ONE_DAY”, “2021-01-03T00:00:00”,“2021-01-06T00:00:00”)`   
+   `result = GroupByStream(bars,”high:Max low:Min”)`  
+   `result = ProjectStream(result,”high_max:highest_price min_low: lowest_price”)`  
+3. **NL:**   
+   **Find the Average daily volume per contract in Natural Gas in 2023**  
+   **DSL:**   
+   `bars = FutureHistoricalBars("NG*", “ONE_DAY”, “2023-01-03T00:00:00”,“2023-31-12T23:59:59”)`   
+   `result = GroupByStream(bars, “contract”, ”volume:Mean”)`  
+   `result = ProjectStream(result,”mean_volume:average_daily_volume”)`
+
+---
+
+#### **Schema Impact:**
+
+The `AggregateStream` modifies the schema of the parent stream by adding new columns for each aggregation operation. Existing columns are unaffected.
+
+# **SingleRowMathStream (Operator Stream)**
+
+The `SingleRowMathStream` is a DSL operator used to perform math operations, one row at a time, on the parent stream. 
+
+---
+
+#### **Stream arguments (parameters):**
+
+1. **parent\_stream (another stream):**The source stream provides the base data  
+2. **math\_expression(string):** This is a math expression which allows the creation of one or more new columns that are the output of a math expression on the existing columns provided by parent\_stream. This can a simple math operation like new\_col \= a \+ b, or any function from C\#’s System.Math namespace, like new\_column \= Math.Log(Math.Pow(volume, 1.1))
+
+#### **Examples:**
+
+4. **NL:**   
+   Find the daily price movement in the SP 500 Futures through the month of Jan  
+   **DSL:**   
+   `bars = FutureHistoricalBars("ES", “ONE_DAY”, “2024-01-01T00:00:00”,“2024-01-31T00:00:00”)`   
+   `result = SingleRowMathStream(bars,”daily_move=close-open”)`  
+5. **NL:**   
+   Find the daily typical price of the  WTI Crude Oil futures over the first half of 2024  
+   **DSL:**   
+   `bars = FutureHistoricalBars("CL", “ONE_DAY”, “2021-01-03T00:00:00”,“2021-01-06T00:00:00”)`   
+   `result = SingleRowMathStream(bars, “typical_price = (high+low+close)/3.0”)`
+
+#### **Input Space (Scope):** The scope of this operation is a **single row.** The SingleRowMAthStream as the name signifies, operates on columns of the same row only. **No multi row operations are allowed.** For multi row operations using what follows.
+
+**Schema Impact:**
+
+# **MultiRowRollingMathStream (Operator Stream)**
+
+The `MultiRowRollingMathStream` function is a DSL operator used to perform mostly math operations on a rolling set of rows of a parent stream. It works similarly to the GroupByStream (described above) but instead of grouping rows together based on a column value, it operates on a rolling set of rows (where the rolling size is provided as an argument). 
+
+---
+
+#### **Stream arguments (parameters):**
+
+1. **Parent Stream (another stream):** The source stream provides the data to be operated on  
+2. **Operation Text (string):** A string defining the column(s) to be operated and the operation(s) to apply, in the format:  
+   `<column_name: operation>, <column_name: operation>, …` Each `<column_name: operation>` pair produces a new column in the output stream. The new column name is formatted as `<column_name>_<operation>` in lowercase. This is like the GroupByStream’s operations string.   
+3. **Sort/Order Column Name(string):** This controls the ordering of the rolling rows. This argument will be in the format \-`<column_name>` to sort the rolling rows in descending order by the `<column_name>`, or \+`<column_name> in` ascending order.  
+4. **Rolling Window Size(int)(optional):** The size of the rolling window, If this argument is left out all, the operations are applied to all rows, is Global Operation instead of Rolling Window Operation (it's like the optional groupBy col in GroupByStream).
+
+---
+
+#### **Valid Aggregation Operations:**
+
+* `Mean`: Arithmetic mean of the values.  
+* `Median`: Median of the values.  
+* `Mode`: Most frequently occurring value.  
+* `Range`: Difference between the max and min.  
+* `StdDev`: Standard deviation of the values.  
+* `Max`: Maximum value.  
+* `Min`: Minimum value.  
+* `First`: First value in the group.  
+* `Last`: Last value in the group.  
+* `Sum`: Sum of the values.  
+* `EdgeDiff`: Difference between the edge values, meaning the difference between a certain col belonging to the first and last rows respectively  
+* `EdgeDiffPercentage`: Same as `EdgeDiff` but expressed as percentage instead
+
+---
+
+#### **Examples:**
+
+1. **NL:**   
+   **Find the 20 d MA of SP 500 Futures close prices**  
+   `bars = FutureHistoricalBars("ES", “ONE_DAY”, “2021-01-03T00:00:00”,“2021-01-11T00:00:00”)`   
+   `result = MultirowRollingMathStream(bars, “close:Mean”, “-ts_event”, 20)`  
+2. **NL:**   
+   **Find the 90d Standard Deviation of WTI Crude Oil during 2023**
+
+   **DSL:**   
+   `bars = FutureHistoricalBars("CL", “ONE_DAY”, “2023-01-01T00:00:00”,“2023-12-31T00:00:00”)`  
+   `// calculating typical_price`
+
+   `result = SingleRowMathStream(bars, “typical_price = (high+low+close)/3.0”)`   
+   `result = MultirowRollingMathStream(bars,“typical_price:StdDev”, “-ts_event”, 90)`
+
+   `result=ProjectStream(result, “stddev_typical_price:90dStdDev”)`
+
+3. **NL:**   
+   **Find the Average daily volume of Natural Gas in 2023**  
+   **DSL:**   
+   `bars = FutureHistoricalBars("NG", “ONE_DAY”, “2023-01-03T00:00:00”,“2023-31-12T23:59:59”)`  
+   `result = MultirowRollingMathStream(bars, ”volume:Mean”, “-ts_event”)`  
+   `result = ProjectStream(result,”mean_volume:average_daily_volume”)`
+
+---
+
+#### **Schema Impact:**
+
+The `MultirowRollingMathStream` modifies the schema of the parent stream by adding new columns for each operation. Existing columns are unaffected.
+
+# **UnionStream (Operator Stream)**
+
+Like the SQL Union appends all the rows from 2 streams together.
+
+#### **Stream arguments (parameters):**
+
+5. `parent_stream1 (first stream)` This stream provides part of the data that will will be appended to the other part, provided by the following argument  
+6. `parent_stream2 (second stream)` This stream provides the other part of the data that will will be appended to the first part, provided by the first argument
+
+#### **Examples:**
+
+4. **NL prompt:** Show me all the bars across SP 500 Futures and Nasdaq Futures  
+   **DSL:** `r = UnionStream(FutureHistoricalBars("ES", "ONE_DAY"), FutureHistoricalBars("NQ", "ONE_DAY"))`  
+   
+
+#### **Schema Impact:**
+
+None
+
+# **DateTimeExpanderStream (Operator Stream)**
+
+DateTimExpanderStream expands one datetime column into various datetime components, like seconds in day, day of week, date etc, so new columns are added for those. The existing columns remain as is.
+
+#### **Stream arguments (parameters):**
+
+1. `parent_stream1 (base stream)` This parent stream that provides the data, including the datetime columns that will be expanded into multiple columns  
+2. `Datetime_input_column (string)` This is the column that will be expanded into multiple columns.
+
+**New Columns:**  
+`<input_dt_col>_date` (Date of the value, without the time part)
+
+* `<input_dt_col>_day,` (Day of the week)  
+* `<input_dt_col>_day_of_month` (day of the month, This is an int between 0 and 31\)  
+* `<input_dt_col>_month,` (month of the year, this is a numerical value between 1 and 12  
+* `input_dt_col>_hour,` (Hour of the day, value bw 0 and 23\)  
+* `<input_dt_col>_min",` (month part of this DateTime. Value is integer between 1 and 12).  
+* `<input_dt_col>_seconds_of_day",` (Total seconds of the day) 
+
+#### **Examples:**
+
+**DSL:** `r = DateTimeExpanderStream(FutureHistoricalBars("ES", "ONE_DAY"), “ts_event”)`   
+**Input Schema:**
+
+| id   | ts\_event | contract | open | high | low  | close | volume |
+| :--- | :-------- | :------- | :--- | :--- | :--- | :---- | :----- |
+
+**Output Schema:**
+
+| id   | ts\_event | contract | open | high | low  | close | volume |      |      |      |      |      |      |      |
+| :--- | :-------- | :------- | :--- | :--- | :--- | :---- | :----- | :--- | :--- | :--- | :--- | :--- | :--- | :--- |
+
@@ -0,0 +1,250 @@
+{
+   "prompts": [
+      {
+         "category": "Settlement",
+         "description": "This is a few shot learning example for an NL to DSL. The DSL function in question is Settlement (symbol_or_contract, datetime1). The first argument is a future (like CL, for crude oil, or ES for SP 500) or a specific future contract like crude oil Dec 24 future (CLZ4) or ESZ9 etc. The second argument is the date at which the settlement is to be fetched. There is an overloaded function, Settlement (symbol_or_contract, begin_dt, end_dt) which gets the settlements for a range pf dates from begin_dt to end_dt. You will need to convert statements  expressed as Today, Yesterday, Last Friday, First Friday of the month, Last Thursday of August, First day of last quarter etc to actual dates. The function Settlement's second and third arguments are dates, so never pass a string like FirstDayOfLastQUarter. You will need to know what Today is, since there will be references to last week, yesterday etc.",
+         "prompt": "What was the settlement for SP 500 Future on June 8th?",
+         "examples": [
+            {
+               "natural_language": "What was the settlement for SP 500 Future on June 8th?",
+               "dsl": "Settlement(\"ES\", 6/8/2023)"
+            },
+            {
+               "natural_language": "Find Settlement price of the SP500 future on 04/22/2023.",
+               "dsl": "Settlement(\"ES\", 4/22/2023)"
+            },
+            {
+               "natural_language": "Find Settlement price of the Silver Dec 24 Future on 04/22/2023.",
+               "dsl": "Settlement(\"SIZ4\", 4/22/2023)"
+            },
+            {
+               "natural_language": "Find Yesterday's Settlement of the Crude Oil Future.",
+               "dsl": "Settlement(\"CL\", 10/10/2024)"
+            },
+            {
+               "natural_language": "Find today's settlement of the 10 year treasury future.",
+               "dsl": "Settlement(\"ZN\", 10/11/2024)"
+            },
+            {
+               "natural_language": "Find the Settlement of Natural Gas Dec 24 future on the first Friday of Sept.",
+               "dsl": "Settlement(\"NGZ4\", 9/6/2024)"
+            },
+            {
+               "natural_language": "Find the Settlement of WTI Jan 25 future in the first Week of Sept.",
+               "dsl": "Settlement(\"CLF5\", 9/2/2024, 9/6/2024)"
+            },
+            {
+               "natural_language": "Find the Settlement price for the Sp 500 Dec 24 Mar 25  spread for the first two weeks of September.",
+               "dsl": "Settlement(\"ESZ4-ESH5\", 9/14/2024, 9/1/2024)"
+            },
+            {
+               "natural_language": "Find the Settlement of Natural Gas future on the first Friday of Sept.",
+               "dsl": "Settlement(\"NG\", 9/6/2024)"
+            }
+         ]
+      },
+      {
+         "category": "VWAP",
+         "description": "This is a few shot learning example for an NL to DSL. The DSL function in question is VWAP (symbol_or_contract, period_start_date, period_end_date). The first argument is a future (like CL, for crude oil, or ES for SP 500) or a specific future contract like crude oil Dec 24 future (CLZ4) or SP 500 Dec 2024 (ESZ4). The second and third argument define the start and end of the VWAP period, chronologically. So for example, if we are calculating the 10 day VWAP on 10/15/2023, the period will be from 10/05/2023  to 10/15/2023 (to cover for 10 days going back from the given date). so the second argument will 10/05/2023 and the third argument will be 10/15/2023. The NL prompt  will also have datetimes expressed as Today, Yesterday, Last Friday, Last Sunday, First day of quarter, Today etc. You will need to translate these as dates that need to be passed to the VWAP DSL function. ",
+         "prompt": "What was the VWAP for SP 500 Future over the past 30 days?",
+         "examples": [
+            {
+               "natural_language": "Find the 20 day VWAP of the SP future on 04/22/2023.",
+               "dsl": "VWAP(\"ES\", 4/02/2023, 4/22/2023)"
+            },
+            {
+               "natural_language": "Find the 10 day VWAP of the Crude Oil Future on 10/10/2023.",
+               "dsl": "VWAP(\"CL\", 10/01/2023, 10/10/2023)"
+            },
+            {
+               "natural_language": "Find today's 30-day VWAP of the 10 year treasury future on 9/11/2023.",
+               "dsl": "VWAP(\"ZN\", 8/11/2023, 9/11/2023)"
+            },
+            {
+               "natural_language": "Find the 10 hour VWAP of the Natural Gas future from yesterday at 5:00 PM.",
+               "dsl": "VWAP(\"NG\", 9/27/2024 7:00:00, 9/27/2024 17:00:00)"
+            },
+            {
+               "natural_language": "Find the 3 month VWAP of the SP 500 futures on 10/10/2023",
+               "dsl": "VWAP(\"ES\", 7/10/2023, 10/10/2023)"
+            }
+         ]
+      },
+      {
+         "category": "FuturePriceDiff",
+         "description": "This is a few-shot learning example for an NL to DSL. The DSL function in question is FuturePriceDiff(symbol_or_contract, datetime1, datetime2). The first argument is a future (like CL, for crude oil, or ES for SP 500 and so on and so forth) or a specific future contract like crude oil Dec 24 future (CLZ4). The second and third arguments are datetimes (usually dates) that the function finds the difference of prices at. If the dates are given like \"first day of quarter\", \"first week of August\" I expect you to convert them into exact dates since the function only takes specific dates. The second argument should be the smaller datetime and the 3rd argument should be the end of the range, ie the higher one. Any prompt asking for price difference or price move should call FuturePriceDiff. If the dates you compute to pass to FuturePriceDiff are dates on which futures exchange might be closed (like 1/1, 12/25, Thanksgiving etc)  please adjust the date to the nearest viable trading date (I am not providing any function to find this date, you have to find the nearest trading date yourself), for example if the date falls on christmas day, change it to 12/24 and so on and so on and so forth, or the date happens to be a saturday or sunday, change it to Friday or Monday depending on which is closest",
+         "prompt": "What is the difference in SP 500 Future from July 10th to July 14th? (or how much did the SP 500 Future price move from July 10th to July 14th?)",
+         "examples": [
+            {
+               "natural_language": "What is the difference in ZN price from July 10th to July 14th?",
+               "dsl": "FuturePriceDiff(\"ZN\", 7/10/2024, 7/14/2024)"
+            },
+            {
+               "natural_language": "What is the difference in SP 500 Future price from the first day of the first quarter of 2020 to the last day of the first quarter of 2020?",
+               "dsl": "FuturePriceDiff(\"ES\", 1/1/2020, 3/31/2020)"
+            },
+            {
+               "natural_language": "How much did SP 500 Futures move in the first quarter of 2020?",
+               "dsl": "FuturePriceDiff(\"ES\", 1/1/2020, 3/31/2020)"
+            },
+            {
+               "natural_language": "What is the price difference in Natural Gas Dec Future price from Aug 30th and Aug 20th?",
+               "dsl": "FuturePriceDiff(\"NGZ4\", 8/20/2024, 8/30/2024)"
+            },
+            {
+               "natural_language": "What is the price move in  SP 500 Future from Aug 30th 2023 to October 10th 2023",
+               "dsl": "FuturePriceDiff(\"ES\", 8/30/2023, 10/10/2023)"
+            },
+            {
+               "natural_language": "How much did  SP 500 Future move during the first quarter of last year",
+               "dsl": "FuturePriceDiff(\"ES\", 1/1/2023, 3/31/2023)"
+            }
+         ]
+      },
+      {
+         "category": "Contract Aggregate Volume",
+         "description": "this is a few shot learning example for an NL to DSL. The DSL function in question is VolumeAgg(symbol, contract_type, datetime1, datetime2), which computes the highest volume in the date range, specified by datetime1, and datetime2. The first argument is a future symbol (like CL for crude oil, ES for SP 500 etc).  The second argument is the ContractType, which is \"Future\" for futures and \"FutureSpread\" for future spreads. the 3rd argument is the begin date and the 4th argument is the end date, between which the max volume is computed. If no future is specified use \"*\" from the first arugment.",
+         "prompt": "Which  SP 500 Future traded the most between 7/10/2024 and 7/14/2024",
+         "examples": [
+            {
+               "natural_language": "Which SP Future traded the most between 7/10/2024 and 7/14/2024",
+               "dsl": "VolumeAgg(\"ES\", \"Future\", 7/10/2024, 7/14/2024)"
+            },
+            {
+               "natural_language": "Which Gold Spread traded the most in the first month of this year",
+               "dsl": "VolumeAgg(\"GC\", \"FutureSpread\", 1/1/2024, 1/31/2024)"
+            },
+            {
+               "natural_language": "Which 10 year Spread traded the highest volume last week to now?(in this example(only) we are assuming today is 11/26/2024)",
+               "dsl": "VolumeAgg(\"ZN\", \"FutureSpread\", 11/19/2024, 11/26/2024)"
+            },
+            {
+               "natural_language": "Which Corn Spread traded the most today (in this example(only) we are assuming today is 11/26/2024)?",
+               "dsl": "VolumeAgg(\"ZC\", \"FutureSpread\", 11/26/2024, 11/26/2024)"
+            }
+         ]
+      },
+      {
+         "category": "Roll",
+         "description": "This is a few shot learning example for an NL to DSL. The DSL function in question is FuturesRoll(symbol) which computes the rollover date for a future, ie when the highest volume swtches from one expiration to the next one. ",
+         "prompt": "When does the roll typically occur In  SP 500?",
+         "examples": [
+            {
+               "natural_language": "When does the roll typically occur In SP 500?",
+               "dsl": "FuturesRoll(\"ES\")"
+            },
+            {
+               "natural_language": "When does the roll typically occur for 10 year treasury?",
+               "dsl": "FuturesRoll(\"ZN\")"
+            },
+            {
+               "natural_language": "When does the roll usually occur for Gold Future",
+               "dsl": "FuturesRoll(\"GC\")"
+            },
+            {
+               "natural_language": "At what date does the future rollover happens for SP500?",
+               "dsl": "FuturesRoll(\"ES\")"
+            },
+            {
+               "natural_language": "When does the future rollover happens for the Corn Future?",
+               "dsl": "FuturesRoll(\"ZC\")"
+            }
+         ]
+      },
+      {
+         "category": "Daily Move",
+         "description": "This is a few shot learning example for an NL to DSL. ",
+         "prompt": "How often did Sp 500 Future settle + or - 10% over the previous day?",
+         "examples": [
+            {
+               "natural_language": "List all the days when the SP 500 closed + or -5% from the previous day's close",
+               "dsl": "ColDiff(filteredOHLCV(\"ES\"),\"ts_event\", -1, \"close\", \"close\", \"close_diff\", 5)"
+            },
+            {
+               "natural_language": "List all the days when the Crude Oil closed + or -2.5% from the previous day's close",
+               "dsl": "ColDiff(filteredOHLCV(\"CL\"),\"ts_event\", -1, \"close\", \"close\", \"close_diff\", 2.5)"
+            },
+            {
+               "natural_language": "Show me all the days when the Ten year Treaury Future opened + or -1% from the previous day's open",
+               "dsl": "ColDiff(filteredOHLCV(\"ZN\"),\"ts_event\", -1, \"open\", \"open\", \"open_diff\", 1.0)"
+            },
+            {
+               "natural_language": "List all the trading days when the Silver Future opened + or -1% from the previous day's close",
+               "dsl": "ColDiff(filteredOHLCV(\"SI\"),\"ts_event\", -1, \"open\", \"open\", \"open_diff\", 1.0)"
+            }
+         ]
+      },
+      {
+         "category": "Open Interest",
+         "description": "This is a few shot learning example for an NL to DSL.",
+         "prompt": "When does the most open interest typically shift from one contract to another in SP 500 Future?",
+         "examples": [
+            {
+               "natural_language": "When does the most open interest typically shift from one contract to another in e-minis? ",
+               "dsl": "FuturesRoll(\"ES\")"
+            },
+            {
+               "natural_language": "Usually when does the open interest move from one contract to another in WTI?",
+               "dsl": "FuturesRoll(\"CL\")"
+            }
+         ]
+      },
+      {
+         "category": "Events",
+         "description": "This is a few shot learning example for an NL to DSL. In the NL prompt the user is asking to show the price of a future (treasury future, or SP 500 future etc) around an event type. The event type could be \"FOMC Announcement\" or \"Storage numbers\" etc. The DSL is as follows: EventsPriceAction(future_name, event_type, pre_event_window, post_event_window, number_of_events) first arg is the name of the future, (ex ES or ZN etc), the second argument is the event type, for example, \"FOMC announcement\", the 3rd argument is a time window for how much pre-event data to include in the end result, the 4th argument is how much post-event data to include in the end result. Both are strings of the form \"5d\" for 5 days, \"5h\" for 5 hours and \"120m\" for 120 minutes and so on. Usually, only one of the event periods is provided. Set the other to be half of what the user provided. The last argument is the number of events to consider. So if it’s 7, that means the most recent 7 events. If the user does not specify that, pass a large number.",
+         "prompt": "What did the volume and price of SP500 Future do following the release of the last 10 FOMC rates?",
+         "examples": [
+            {
+               "natural_language": "How did SP 500 Future do in the 5 days following the FOMC announcements?",
+               "dsl": "EventsPriceAction(\"ES\", \"FOMC Announcement\", \"2d\", \"5d\", 10)"
+            },
+            {
+               "natural_language": "How did the price of Treasury Future evolve in the 2 days following the Job Reports?",
+               "dsl": "EventsPriceAction(\"ZN\", \"Job Reports\", \"1d\", \"2d\", 10)"
+            },
+            {
+               "natural_language": "Show me the price movement and volume of Crude Oil futures during the 5-minute window following the last 5 OPEC meetings.",
+               "dsl": "EventsPriceAction(\"CL\", \"OPEC Meeting\", \"2m\", \"5m\", 5)"
+            },
+            {
+               "natural_language": "Show me the price movement and volume of SP 500 futures preceding the 10-minute window of the last 5 OPEC meetings.",
+               "dsl": "EventsPriceAction(\"CL\", \"OPEC Meeting\", \"10m\", \"5m\", 5)"
+            }
+         ]
+      },
+      {
+         "category": "Indicator, Standard Deviation",
+         "description": "This is a few shot learning example for an NL to DSL. In the NL prompt the user is asking to show all the days when the price of a future (treasury future, or SP 500 future etc) moved more than m times the N days standard deviation, where both m and N are provided. For example, the DSL for \"Using a 90d standard deviation, give me the dates and amounts of the last 4+ standard deviation moves in WTI futures\" would be: r=standardDeviation(\"ES\", \"90d\")\nr=math(r, \"abs_daily_change = Abs(close - open) comp = std_dev * 4\")\nr=filter(r,\"abs_daily_change >= comp\"). The first line, creates a stream for StandardDeviation, based on the number of days, for example 90d standard deviation, and assigned it to r, the second line takes r from the first line and passes it to a math stream that creates 2 new columns, abs_daily_change and comp. comp is the factor (4, from 4+ in the prompt) times standard deviation. The third line filters (like a sql where) of rows where abs_daily_change >= comp",
+         "prompt": "Using a 90d standard deviation, give me the dates and amounts of the last 1+ standard deviation moves in SP  futures ?",
+         "examples": [
+            {
+               "natural_language": "Using a 90d standard deviation, give me the dates and amounts of the last 1+ standard deviation moves in WTI futures ?",
+               "dsl": "r=standardDeviation(\"CL\", \"90d\")\nr=math(r, \"abs_daily_change = Abs(close - open) comp = std_dev * 1.0\")\nr=filter(r,\"abs_daily_change >= comp\")"
+            },
+            {
+               "natural_language": "Show me the times the SP Futures moved twice the 60 day standard deviation",
+               "dsl": "r=standardDeviation(\"ES\", \"60d\")\nr=math(r, \"abs_daily_change = Abs(close - open) comp = std_dev * 2.0\")\nr=filter(r,\"abs_daily_change >= comp\")"
+            },
+            {
+               "natural_language": "List all the days the Gold Futures moved 1.5 times the 45 day standard deviation",
+               "dsl": "r=standardDeviation(\"GC\", \"45d\")\nr=math(r, \"abs_daily_change = Abs(close - open) comp = std_dev * 1.5\")\nr=filter(r,\"abs_daily_change >= comp\")"
+            },
+            {
+               "natural_language": "List all the days the 10 year treasury future moved more than twice the 1 month standard deviation",
+               "dsl": "r=standardDeviation(\"ZN\", \"30d\")\nr=math(r, \"abs_daily_change = Abs(close - open) comp = std_dev * 2.0\")\nr=filter(r,\"abs_daily_change >= comp\")"
+            }
+         ]
+      },
+      {
+         "category": "Price Movement",
+         "description": "This is a few shot learning example for an NL to DSL. In the NL prompt the user is asking for the average price movement for e-minis (aka ES aka SP Futures aka SP 500 Futures)) on Wednesday between 9:30 am and 9:40 AM?  the multiline DSL is as follows (along with the explaination in comments before each line, dont include the comments in the asnwer):  //following line gets the 1 min ohlcv bars of ES \n r=OHLCV_1min_stream(\"ES\")\n //this passes it to the filter stream, which filters the rows for Wednesday, and time bw 9:30 and 9:40.\nr=filter(r,\"ts_event_day = 'Wednesday' and ts_event_hour = 9 and ts_event_min >= 30 and ts_event_min < 40\")\n// this aggregates it using the date as the agregation key, it find the max of the column called high, and the low of the column called min, to find the max and the min prices in that time window\nr=aggregate(r, \"ts_event_date\", \"high: max, low: min\")\n/// this then passes the previous result to the math stream operator which calculates the move, which is the difference between high_max and low_min (columns that were added from the previous aggregate)\nr=math(r,\"move = high_max - low_min\")\n// At the end we compute the mean of the move column, notice this is the total mean, so we are not providing an aggregation col to agregate\nr=aggregate(r, \"move:mean\")\n// we project out every col by the ! operator and call rename move_mean as average_vol\nr=newProject(r,\"move_mean:average_move !\")",
+         "prompt": "What is the average price movement for SP Futures (ES) on Wednesday between 9:30 am and 9:40 AM ?",
+         "examples": [
+            {
+               "natural_language": "What is the average price movement for SP Futures (ES) on Wednesday between 9:30 am and 9:40 AM?",
+               "dsl": "r=OHLCV_1min_stream(\"ES\")\nr=filter(r,\"ts_event_day = 'Wednesday' and ts_event_hour = 9 and ts_event_min >= 30 and ts_event_min < 40\")\nr=aggregate(r, \"ts_event_date\", \"high: max, low: min\")\nr=math(r,\"move = high_max - low_min\")\nr=aggregate(r, \"move:mean\")\nr=newProject(r,\"move_mean:average_move !\")"
+            }
+         ]
+      }
+   ]
+}