CLEANUP: BYBIT Symbols.conf.

Originally, this spark.conf was hardcoded into compose.yaml, and copied
into the sparkd container on each deploy. It hasn't been used since
config profiles were introduced.

.env

@@ -34,7 +34,7 @@
 #   src/server/sparkd/Dockerfile
 SPARKD__IMAGE=sparkd
 # The config dir for this service to use
-SPARKD__CONFIG_DIR=src/app_sparkd/configurations/dev
+SPARKD__CONFIG_DIR=src/server/sparkd/configurations/dev
 # Mount a specific volume or dir to "~/sparkdata/data" inside the docker.
 # Defaults to a volume named "sparkd_<config_port>" (e.g., sparkd_8000).
 # SPARKD__DATA_DIR=/home/bts/sparkdata/data
@@ -44,6 +44,5 @@ SPARKD__CONFIG_PORT=8000
 SPARKD__QUICKBUS_PORT=8001
 SPARKD__SIM_FIX_PORT=8002
 SPARKD__SHM_SIZE=4gb
-SPARKD__PYTHON_REQUIREMENTS=requirements.txt
 
 # TODO: add config for other services (pubsub, md-publisher, webapp, api-server, etc.)

.gitignore

@@ -11,7 +11,6 @@ cmake-build-debug/
 **.vscode/*
 !.vscode/launch.json
 !.vscode/tasks.json
-!.vscode/*.env
 .vs
 obj/
 *.csproj.user

.vscode/python.env

@@ -1 +0,0 @@
-PYTHONPATH=/workspaces/spark/build/sparkd-debug-clang/src/lib_python_interop;/workspaces/spark/src/lib_python_interop/python_interop_plugins

.vscode/tasks.json

@@ -363,7 +363,7 @@
           "${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
         ],
         "options": {
-          "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0-windows"
+          "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0-windows"
         },
       },
       "icon": {
@@ -383,7 +383,7 @@
           "${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
         ],
         "options": {
-          "cwd": "${workspaceFolder}/build/Debug/bin/net8.0-windows"
+          "cwd": "${workspaceFolder}/build/Debug/bin/net7.0-windows"
         },
       },
       "icon": {
@@ -403,7 +403,7 @@
           "${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
         ],
         "options": {
-          "cwd": "${workspaceFolder}/build/Release/bin/net8.0-windows"
+          "cwd": "${workspaceFolder}/build/Release/bin/net7.0-windows"
         },
       },
       "icon": {
@@ -427,7 +427,7 @@
         "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
       ],
       "options": {
-        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0"
+        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0"
       },
       "icon": {
         "id": "run",
@@ -450,7 +450,7 @@
         "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
       ],
       "options": {
-        "cwd": "${workspaceFolder}/build/Debug/bin/net8.0"
+        "cwd": "${workspaceFolder}/build/Debug/bin/net7.0"
       },
       "icon": {
         "id": "run",
@@ -473,7 +473,7 @@
         "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
       ],
       "options": {
-        "cwd": "${workspaceFolder}/build/Release/bin/net8.0"
+        "cwd": "${workspaceFolder}/build/Release/bin/net7.0"
       },
       "icon": {
         "id": "run",
@@ -496,7 +496,7 @@
         "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
       ],
       "options": {
-        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0"
+        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0"
       },
       "icon": {
         "id": "run",
@@ -519,7 +519,7 @@
         "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
       ],
       "options": {
-        "cwd": "${workspaceFolder}/build/Debug/bin/net8.0"
+        "cwd": "${workspaceFolder}/build/Debug/bin/net7.0"
       },
       "icon": {
         "id": "run",
@@ -541,7 +541,7 @@
         "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
       ],
       "options": {
-        "cwd": "${workspaceFolder}/build/Debug/bin/net8.0"
+        "cwd": "${workspaceFolder}/build/Debug/bin/net7.0"
       },
       "icon": {
         "id": "run",
@@ -564,7 +564,7 @@
         "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
       ],
       "options": {
-        "cwd": "${workspaceFolder}/build/Release/bin/net8.0"
+        "cwd": "${workspaceFolder}/build/Release/bin/net7.0"
       },
       "icon": {
         "id": "run",
@@ -586,7 +586,7 @@
         "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
       ],
       "options": {
-        "cwd": "${workspaceFolder}/build/Release/bin/net8.0"
+        "cwd": "${workspaceFolder}/build/Release/bin/net7.0"
       },
       "icon": {
         "id": "run",
@@ -601,7 +601,7 @@
       "windows": {
         "command": "Spark.exe",
         "options": {
-          "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0-windows"
+          "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0-windows"
         },
       },
       "icon": {
@@ -617,7 +617,7 @@
       "windows": {
         "command": "Spark.exe",
         "options": {
-          "cwd": "${workspaceFolder}/build/Debug/bin/net8.0-windows"
+          "cwd": "${workspaceFolder}/build/Debug/bin/net7.0-windows"
         },
       },
       "icon": {
@@ -633,7 +633,7 @@
       "windows": {
         "command": "Spark.exe",
         "options": {
-          "cwd": "${workspaceFolder}/build/Release/bin/net8.0-windows"
+          "cwd": "${workspaceFolder}/build/Release/bin/net7.0-windows"
         },
       },
       "icon": {
@@ -648,7 +648,7 @@
       "windows": {
         "command": "Spark.exe",
         "options": {
-          "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0-windows"
+          "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0-windows"
         },
       },
       "icon": {
@@ -673,7 +673,7 @@
         "${input:sparkConfigQuery}",
       ],
       "options": {
-        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0"
+        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0"
       },
       "icon": {
         "id": "run",
@@ -754,7 +754,7 @@
           "${input:sparkTest}"
         ],
         "options": {
-          "cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
+          "cwd": "${workspaceFolder}/build/Debug/bin/net7.0",
         }
       },
     },

CMakeLists.txt

@@ -113,15 +113,15 @@ else()
   endif()
 
   # subdirs
-  add_subdirectory(src/lib_spark)
   if (BUILD_SPARKD)
-    add_subdirectory(src/lib_python_interop)
     add_subdirectory(src/lib_onixs_fix)
-    add_subdirectory(src/lib_binance)
+    add_subdirectory(src/lib_binance_execution)
     add_subdirectory(src/lib_okx_execution)
+    add_subdirectory(src/lib_python_md)
     add_subdirectory(src/app_sparkd)
     add_subdirectory(src/data_files)
   endif()
+  add_subdirectory(src/lib_spark)
   if (BUILD_ADMIN_WEB_APP)
     add_subdirectory(src/Spark.Admin.WebApp)
   endif()

cmake/InitPythonModule.cmake

@@ -1,71 +0,0 @@
-include_guard(DIRECTORY)
-include(GNUInstallDirs)
-include(${CMAKE_SOURCE_DIR}/cmake/ExportToParent.cmake)
-include(${CMAKE_SOURCE_DIR}/cmake/SetFirstDefined.cmake)
-
-function(init_python_module)
-  set(_options)
-  set(_one_value_args
-    NAME
-  )
-  set(_multi_value_args
-    SOURCES
-  )
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-  set_first_defined(_name _ARGS_NAME PROJECT_NAME)
-  set_first_defined(_sources _ARGS_SOURCES ${_name}_SOURCES)
-
-  set(module_proj ${_name}_module)
-  set(${module_proj}_SOURCES ${_sources})
-  pybind11_add_module(${module_proj} SHARED ${_sources})
-  target_link_libraries(
-    ${module_proj}
-    PUBLIC
-      pybind11::embed
-      spark
-      ${_name}
-  )
-  target_include_directories(
-    ${module_proj}
-    PRIVATE
-      $<BUILD_INTERFACE:${pybind11_INCLUDE_DIRS}>
-  )
-  target_include_directories(
-    ${module_proj}
-    PUBLIC
-      ${Python3_INCLUDE_DIRS}
-      $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
-      $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>
-  )
-  add_custom_command(
-    OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/stubs/${_name}.pyi
-    DEPENDS ${_name} ${module_proj}
-    COMMAND PYTHONPATH=${CMAKE_CURRENT_BINARY_DIR} pybind11-stubgen --numpy-array-use-type-var --ignore-invalid-expressions '.*' ${_name}_module
-    COMMENT "Generating py stubs"
-  )
-  add_custom_target(
-    "${_name}_py_stubs"
-    ALL
-    DEPENDS
-      ${CMAKE_CURRENT_BINARY_DIR}/stubs/${_name}.pyi
-  )
-  install(
-    TARGETS ${module_proj}
-    EXPORT ${module_proj}Targets
-    RUNTIME
-      DESTINATION ${CMAKE_INSTALL_BINDIR}
-    LIBRARY
-      DESTINATION ${CMAKE_INSTALL_LIBDIR}
-    ARCHIVE
-      DESTINATION ${CMAKE_INSTALL_LIBDIR}
-  )
-  foreach(dir ${pybind11_INCLUDE_DIRS})
-    install(DIRECTORY ${dir} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
-  endforeach()
-  install(
-    EXPORT ${module_proj}Targets
-    FILE ${module_proj}Targets.cmake
-    NAMESPACE ${_name}::
-    DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name}
-  )
-endfunction()

cmake/InitPythonPlugin.cmake

@@ -1,65 +0,0 @@
-include_guard(DIRECTORY)
-include(GNUInstallDirs)
-
-function(init_python_plugin)
-  set(_options)
-  set(_one_value_args NAME PLUGIN_NAME)
-  set(_multi_value_args SOURCES)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-  set_first_defined(_name _ARGS_NAME PROJECT_NAME)
-  set(_plugin_name ${_ARGS_PLUGIN_NAME})
-  if (NOT DEFINED _ARGS_PLUGIN_NAME)
-    set(_plugin_name ${_name}_plugins)
-  endif()
-  set(${_plugin_name}_SOURCES ${_ARGS_SOURCES})
-  if (NOT DEFINED _ARGS_SOURCES)
-    file(
-      GLOB_RECURSE ${_plugin_name}_SOURCES
-      RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
-      ${_plugin_name}/*.py)
-  endif()
-  set(src_outs)
-  foreach(_src ${${_plugin_name}_SOURCES})
-    set(_src_in ${CMAKE_CURRENT_SOURCE_DIR}/${_src})
-    set(_src_out ${CMAKE_CURRENT_BINARY_DIR}/${_src})
-    add_custom_command(
-      OUTPUT ${_src_out}
-      COMMAND ${CMAKE_COMMAND} -E copy ${_src_in} ${_src_out}
-      DEPENDS ${_src_in}
-      COMMENT "Copying ${_plugin_name} plugin ${_src} to ${_src_out}"
-    )
-    list(APPEND src_outs ${_src_out})
-  endforeach()
-  
-  add_custom_target(copy_${_plugin_name} ALL DEPENDS ${src_outs})
-  
-  add_library(${_plugin_name} INTERFACE)
-  add_library(${_name}::${_plugin_name} ALIAS ${_plugin_name})
-  
-  set_target_properties(
-    ${_plugin_name}
-    PROPERTIES
-      INTERFACE_PYTHON_SCRIPTS_DIR
-        $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/${_plugin_name}>$<INSTALL_INTERFACE:${CMAKE_INSTALL_BINDIR}/${_plugin_name}>
-  )
-  
-  set_target_properties(
-    ${_plugin_name}
-    PROPERTIES
-    INTERFACE_PYTHON_SCRIPTS $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/${${_plugin_name}_SOURCES}>$<INSTALL_INTERFACE:${CMAKE_INSTALL_BINDIR}/${${_plugin_name}_SOURCES}>
-  )
-  
-  install(
-    TARGETS ${_plugin_name}
-    EXPORT ${_plugin_name}Targets
-    DESTINATION ${CMAKE_INSTALL_BINDIR}
-  )
-  
-  install(
-    EXPORT ${_plugin_name}Targets
-    FILE ${_plugin_name}Targets.cmake
-    NAMESPACE ${_name}::
-    DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name}
-  )
-  install(DIRECTORY ${_plugin_name} DESTINATION ${CMAKE_INSTALL_BINDIR})
-endfunction()

cmake/SymLinkPythonModule.cmake

@@ -1,39 +0,0 @@
-include_guard(DIRECTORY)
-include(${CMAKE_SOURCE_DIR}/cmake/SymLink.cmake)
-
-function(sym_link_python_module)
-  set(_options)
-  set(_one_value_args LIB_NAME MODULE_NAME PLUGIN_NAME)
-  set(_multi_value_args)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}"
-                        "${_multi_value_args}" ${ARGN})
-  if(NOT DEFINED _ARGS_LIB_NAME)
-    message(
-      FATAL_ERROR "LIB_NAME must be specified for sym_link_python_module()")
-  endif()
-  set(_lib_name ${_ARGS_LIB_NAME})
-  set(_module_name ${_ARGS_MODULE_NAME})
-  if(NOT DEFINED _ARGS_MODULE_NAME)
-    set(_module_name ${_lib_name}_module)
-  endif()
-  set(_plugin_name ${_ARGS_PLUGIN_NAME})
-  set(_deps ${_lib_name} ${_module_name})
-  if(DEFINED _ARGS_PLUGIN_NAME)
-    list(APPEND _deps ${_plugin_name} copy_${_plugin_name})
-  endif()
-  add_dependencies(${PROJECT_NAME} ${_deps})
-  sym_link_local_lib(${_lib_name})
-  set(_module_suffix cpython-310-x86_64-linux-gnu.so)
-  sym_link_file(
-    SOURCE
-    ${CMAKE_BINARY_DIR}/src/lib_${_lib_name}/${_module_name}.${_module_suffix}
-    DEST
-    ${CMAKE_CURRENT_BINARY_DIR}/${_module_name}.${_module_suffix}
-    INSTALL_DEST
-    ${CMAKE_INSTALL_BINDIR}/)
-  if(DEFINED _ARGS_PLUGIN_NAME)
-    get_target_property(plugins ${_plugin_name} INTERFACE_PYTHON_SCRIPTS_DIR)
-    sym_link_dir(TARGET_NAME ${PROJECT_NAME}_link_${_plugin_name} SOURCE
-                 ${plugins} DEST ${CMAKE_CURRENT_BINARY_DIR}/${_plugin_name})
-  endif()
-endfunction()

compose.yaml

@@ -8,7 +8,6 @@ services:
         - IMAGE=${SPARKD__IMAGE}
         - SOURCE_DIR=${SPARKD__SOURCE_DIR}
         - CONFIG_DIR=${SPARKD__CONFIG_DIR}
-        - PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
     shm_size: "${SPARKD__SHM_SIZE:-512mb}"
     command: >
       sh -c "
@@ -22,10 +21,6 @@ services:
           . /home/bts/sparkdata/conf/.env.local
           set +a
         fi
-        if [ -d /home/bts/sparkdata/conf/plugins ]; then
-          echo 'Copying /home/bts/sparkdata/conf/plugins to runtime dir (/home/bts/bin)'
-          cp -r /home/bts/sparkdata/conf/plugins /home/bts/bin/
-        fi
         ./sparkd run"
     env_file:
       - .env
@@ -44,12 +39,11 @@ services:
       - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/sparkdata/data
       - ${SPARKD__RUNTIME_CONF_DIR:-sparkd_conf}:/home/bts/sparkdata/conf
     build:
-      dockerfile: src/app_sparkd/Dockerfile
+      dockerfile: src/server/sparkd/Dockerfile
       args:
         - IMAGE=${SPARKD__IMAGE}
         - SOURCE_DIR=${SPARKD__SOURCE_DIR}
         - CONFIG_DIR=${SPARKD__CONFIG_DIR}
-        - PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
     shm_size: "${SPARKD__SHM_SIZE:-512mb}"
     command: >
       sh -c "
@@ -65,10 +59,6 @@ services:
           . /home/bts/sparkdata/conf/.env.local
           set +a
         fi
-        if [ -d /home/bts/sparkdata/conf/plugins ]; then
-          echo 'Copying /home/bts/sparkdata/conf/plugins to runtime dir (/home/bts/bin)'
-          cp -r /home/bts/sparkdata/conf/plugins /home/bts/bin/
-        fi
         ./sparkd run"
     env_file:
       - .env

requirements.txt

@@ -16,16 +16,11 @@ jmespath==1.0.1
 lazy-object-proxy==1.6.0
 MarkupSafe==3.0.2
 mccabe==0.6.1
-numpy==2.2.6
 platformdirs==2.3.0
 protobuf==4.21.12
 pybars3==0.9.7
-pybind11==2.13.6
-pybind11-stubgen==2.5.3
-pybit==5.11.0
 pycodestyle==2.7.0
 pycparser==2.20
-pycryptodome==3.23.0
 pylint==2.10.2
 PyMeta3==0.5.1
 python-dateutil==2.9.0.post0
@@ -36,7 +31,6 @@ six==1.17.0
 termcolor==1.1.0
 toml==0.10.2
 urllib3==2.0.3
-websocket-client==1.8.0
 Werkzeug==3.1.3
 wrapt==1.12.1
 yapf==0.31.0

spark_client.sln

@@ -18,14 +18,6 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_websocket", "src\clie
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_api_server", "src\client\spark_api_server\spark_api_server.csproj", "{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamCore", "src\DataStreamSuite\DataStreamCore\DataStreamCore.csproj", "{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStream", "src\DataStreamSuite\DataStream\DataStream.csproj", "{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SharedStreams", "src\DataStreamSuite\SharedStreams\SharedStreams.csproj", "{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PubSub", "src\DataStreamSuite\PubSub\PubSub.csproj", "{51C40BA9-F226-4A35-939D-1ED6F5FF7299}"
-EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|x64 = Debug|x64
@@ -98,38 +90,6 @@ Global
 		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x64.Build.0 = Release|x64
 		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x64.ActiveCfg = Release|x64
 		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Debug|x64.Build.0 = Debug|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Release|x64.ActiveCfg = Release|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Release|x64.Build.0 = Release|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Debug|x64.Build.0 = Debug|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Release|x64.ActiveCfg = Release|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Release|x64.Build.0 = Release|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Debug|x64.Build.0 = Debug|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Release|x64.ActiveCfg = Release|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Release|x64.Build.0 = Release|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Debug|x64.Build.0 = Debug|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Release|x64.ActiveCfg = Release|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Release|x64.Build.0 = Release|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE

src/DataStreamSuite/DataStream/BlueOrb/AddColumnsStream.cs

@@ -1,11 +0,0 @@
-namespace bts.BlueOrb {
-   using BCG;
-
-   [DataStreamUserRespresentation("AddColumnsStream")]
-   public class AddColumnsStream: ViewStream {
-      public AddColumnsStream(ViewStream parent_str, string add_col_expr) {
-         ViewStream rs = new BCG.AddColumnsStream(parent_str, add_col_expr);
-         mySetRealSource(rs);
-      }
-   }
-}

src/DataStreamSuite/DataStream/BlueOrb/CountStream.cs

@@ -1,49 +0,0 @@
-using System;
-
-namespace BCG {
-   [DataStreamUserRespresentation("CountStream")]
-   public class CountStream : ViewStream {
-      private DataPacket myLastFiredDataPacket;
-      private IBlueTableSchema mySchema = new BlueTableSchema("count",
-         new ColumnInfo("id", 0, typeof(int)),
-         new ColumnInfo("count", 1, typeof(int)));
-      private int myId = 0;
-      private ViewStream myVS;
-
-      public CountStream(ViewStream vs) {
-         myVS = vs;
-      }
-
-      public CountStream(string tableName)
-         : this(new TableStream(tableName)) { }
-
-      protected override void myOnFirstSubscriber() {
-         myVS.onChange += new ChangeDelegate(myHandleChanges);
-      }
-      protected override void myOnLastSubscriber() {
-         myVS.onChange -= new ChangeDelegate(myHandleChanges);
-      }
-
-
-      private void myHandleChanges(object newValue) {
-         myFireEvent(
-            myLastFiredDataPacket =
-               myLastFiredDataPacket == null ?
-                  new DataPacket(myGetRows(newValue, ++myId), myGetRows(newValue, myId),
-                  new BlueTable(mySchema))
-                  :
-                  new DataPacket(myGetRows(newValue, ++myId), myGetRows(newValue, myId),
-                  myLastFiredDataPacket.newRows)
-            );
-      }
-
-      private IBlueRowCollection myGetRows(object dp, int id) {
-         BlueTable retVal = new BlueTable(mySchema);
-         retVal.addRow
-            (new BlueRow(mySchema, id, (dp as DataPacket).newRows.Count));
-         return retVal;
-      }
-
-
-   }
-}

src/DataStreamSuite/DataStream/BlueOrb/DateTimeExpanderStream.cs

@@ -1,38 +0,0 @@
-namespace bts.BlueOrb {
-   using bts;
-
-   using BCG;
-
-   using System;
-   [DataStreamUserRespresentation("DateTimeExpanderStream")]
-   public class DateTimeExpanstionStream : ViewStream {
-      public DateTimeExpanstionStream(ViewStream parent_stream, string inp_col) {
-         Func<IBlueTableSchema, IBlueTableSchema> get_schema = (s) => {
-            IBlueTableSchema rv = s.add_column(System.TypeCode.DateTime, $"{inp_col}_date");
-            rv = rv.add_column(System.TypeCode.String, $"{inp_col}_day_of_week");
-            rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_day_of_month");
-            rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_month");
-            rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_hour");
-            rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_minute");
-            rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_seconds_of_day");
-            return rv;
-         };
-         ViewStream rs = new MapStream(parent_stream,
-         s => get_schema(s),
-         r => {
-            DateTime dt = r.get_datetime(inp_col);
-            IBlueTableSchema s = get_schema(r.schema);
-            MutableBlueRow rv = new MutableBlueRow(s, r);
-            rv.setValue($"{inp_col}_date", dt.Date);
-            rv.setValue($"{inp_col}_day_of_week", dt.DayOfWeek.ToString());
-            rv.setValue($"{inp_col}_day_of_month", dt.Day);
-            rv.setValue($"{inp_col}_month", dt.Date.Month);
-            rv.setValue($"{inp_col}_hour", dt.Hour);
-            rv.setValue($"{inp_col}_minute", dt.Minute);
-            rv.setValue($"{inp_col}_seconds_of_day", (dt - dt.Date).TotalSeconds);
-            return rv;
-         });
-         mySetRealSource(rs);
-      }
-   }
-}

src/DataStreamSuite/DataStream/BlueOrb/FilterStream.cs

@@ -1,156 +0,0 @@
-namespace bts.BlueOrb {
-   using BCG;
-   using System;
-   using System.Text.RegularExpressions;
-
-   [DataStreamUserRespresentation("FilterStream")]
-   public class FilterStream : ViewStream {
-      public class RegExpFilter : IStreamFilter {
-         private Regex[] myRegExps;
-         private string[] myCols;
-
-         public RegExpFilter(string col, string regExp)
-            : this(new string[] { col }, new string[] { regExp }) { }
-
-         public RegExpFilter(string[] cols, string[] regExps) {
-            myCols = cols;
-            myRegExps = new Regex[regExps.Length];
-            for (int i = 0; i < regExps.Length; i++)
-               myRegExps[i] = new Regex(regExps[i]);
-         }
-
-         #region IStreamFilter Members
-         public bool filter(IBlueRow dr) {
-            bool rv = true;
-            for (int i = 0; i < myCols.Length; i++) {
-               rv &= myRegExps[i].IsMatch(dr[myCols[i]] is DBNull ? "" : dr[myCols[i]].ToString());
-               if (!rv)
-                  return rv;
-            }
-            return rv;
-         }
-         #endregion
-      }
-      private BlueTable myLastFiredRows;
-      private IBlueRowCollection myLastReceivedRows;
-      private IStreamFilter myFilter;
-      private ViewStream myParentStream;
-      private string myFilterString;
-      private bool myUseNewFilter = false;// Configuration.get<DataStreamConfig>().UseNewFilter;
-
-      #region constructors
-      public delegate bool OnFilter(IBlueRow r);
-      private class OnFilterWrapper : IStreamFilter {
-         public OnFilterWrapper(OnFilter f) { this.f = f; }
-
-         public bool filter(IBlueRow dr) { return f(dr); }
-
-         private OnFilter f;
-      }
-      public FilterStream(ViewStream vs, OnFilter f)
-         : this(vs, new OnFilterWrapper(f)) { }
-
-
-      public FilterStream(ViewStream vs, IStreamFilter filter) {
-         {
-            myParentStream = vs;
-            myFilter = filter;
-            myFilterString = filter.ToString();
-            if (filter is IDynamicStreamFilter)
-               (filter as IDynamicStreamFilter).onFilterChange += new FilterChangeDelegate(myHandleFilterChange);
-         }
-      }
-
-      public FilterStream(ViewStream vs, string filterExpression)
-         : this(vs, new BooleanExpressionFilter(filterExpression)) {
-         myFilterString = filterExpression;
-      }
-
-      public FilterStream(ViewStream vs, string col, string regExp)
-         : this(vs, new RegExpFilter(col, regExp)) {
-         myFilterString = col + " , " + regExp;
-      }
-
-      public FilterStream(ViewStream vs)
-         : this(vs, new BinaryFilter(true)) {
-         myFilterString = true.ToString();
-      }
-      #endregion
-
-      #region overridden methods
-      protected override void myOnFirstSubscriber() { myParentStream.onChange += new ChangeDelegate(myHandleCallbacks); }
-
-      protected override void myOnLastSubscriber() {
-         myLastFiredRows = null;
-         myParentStream.onChange -= new ChangeDelegate(myHandleCallbacks);
-      }
-
-      protected override void myFireEvent(object o) {
-         DataPacket dp = o as DataPacket;
-         if (myLastValue == null || dp.addedRows.Count + dp.deletedRows.Count > 0)
-            base.myFireEvent(o);
-      }
-      #endregion
-
-      private void myHandleCallbacks(object newValue) {
-         DataPacket retVal;
-         DataPacket arrivedPacket = newValue as DataPacket;
-         myLastReceivedRows = arrivedPacket.newRows;
-         if (myLastFiredRows != null) {
-            retVal = new DataPacket(myLastFiredRows, new BlueTable(myLastFiredRows.schema),
-               new BlueTable(myLastFiredRows.schema));
-            foreach (IBlueRow r in arrivedPacket.deletedRows)
-               if (myLastFiredRows.containsID((int)r["id"])) {
-                  myLastFiredRows.deleteRow(r);
-                  (retVal.deletedRows as BlueTable).addRow(r);
-               }
-            foreach (IBlueRow r in arrivedPacket.addedRows)
-               if (myFilter.filter(r)) {
-                  myLastFiredRows.addRow(r);
-                  (retVal.addedRows as BlueTable).addRow(r);
-               }
-         } else {
-            myLastFiredRows = new BlueTable(arrivedPacket.schema);
-            retVal = new DataPacket(myLastFiredRows, myLastFiredRows, new BlueTable(myLastFiredRows.schema));
-            foreach (IBlueRow r in arrivedPacket.newRows)
-               if (myFilter.filter(r))
-                  myLastFiredRows.addRow(r);
-         }
-         myFireEvent(retVal);
-      }
-
-      public IStreamFilter filter {
-         set {
-            {
-               myFilter = value;
-               if (numberOfSubscribers > 0) {
-                  if (myLastReceivedRows != null) {
-                     BlueTable nRows = new BlueTable(myLastReceivedRows.schema);
-                     foreach (IBlueRow r in myLastReceivedRows)
-                        if (myFilter.filter(r))
-                           nRows.addRow(r);
-                     IBlueRowCollection dRows = new BlueTable(nRows.schema),
-                        aRows = new BlueTable(nRows.schema);
-                     BlueDataUtility.computeDifferences(nRows, myLastFiredRows, out dRows, out aRows);
-                     myLastFiredRows = nRows;
-                     myFireEvent(new DataPacket(nRows, aRows, dRows));
-                  }
-               }
-            }
-         }
-         get {            
-               return myFilter;
-         }
-      }
-
-      //public string filterString { set { filter = new BooleanExpressionFilter(value); } }
-
-      public override string ToString() {
-         return GetType().ToString() + " " + myFilterString;
-      }
-
-      private void myHandleFilterChange() {
-         filter = myFilter;
-      }
-   }
-}

src/DataStreamSuite/DataStream/BlueOrb/GroupByStream.cs

@@ -1,121 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Diagnostics;
-using System.Linq;
-using System.Text.Json;
-
-using bts;
-//using bts.SilverPine;
-using BCG;
-
-namespace bts.BlueOrb {
-   [DataStreamUserRespresentation("GroupByStream")]
-   public class GroupByStream : ViewStream {
-
-      public interface INonIncGroupByEvaluator {
-         // grouped_rows
-         IBlueRowCollection evaluate(IDictionary<int, List<IBlueRow>> grouped_rows);
-         string[] get_group_by_cols();
-      }
-
-      private class DelNonIncGroupByEvaluator : INonIncGroupByEvaluator {
-         Func<string[]> _groupby_cols_func;
-         Func<List<IBlueRow>, List<IBlueRow>> _evaluate_func;
-         public DelNonIncGroupByEvaluator(Func<string[]> groupby_cols_func,
-                                          Func<List<IBlueRow>, List<IBlueRow>> evaluate_func) {
-            _groupby_cols_func = groupby_cols_func;
-            _evaluate_func = evaluate_func;
-         }
-         public IBlueRowCollection evaluate(IDictionary<int, List<IBlueRow>> grouped_rows) {
-            BlueTable rv = null;
-            foreach (var kv in grouped_rows) {
-               List<IBlueRow> g_rows = kv.Value;
-               List<IBlueRow> out_rows = _evaluate_func.Invoke(g_rows);
-               if (out_rows != null && out_rows.Count > 0) {
-                  rv = rv ?? new BlueTable(out_rows.ElementAt(0).schema);
-               }
-               rv.addRows(out_rows);
-            }
-            return rv;
-         }
-         public string[] get_group_by_cols() {
-            return _groupby_cols_func();
-         }
-
-      }
-      private ViewStream _main_stream;
-      private INonIncGroupByEvaluator _groupby_func;
-      private BlueRowHasher _row_hasher;
-      private string[] _groupby_cols;
-      private Stopwatch _timer = new();
-      private int _n_total_fires = 0, _n_expensive_fires = 0;
-
-      public GroupByStream(ViewStream main_stream, INonIncGroupByEvaluator groupby_func) {
-         _main_stream = main_stream;
-         _groupby_func = groupby_func;
-         _groupby_cols = _groupby_func.get_group_by_cols();
-         _row_hasher = new BlueRowHasher(_groupby_cols == null ? 0 : _groupby_cols.Length);
-      }
-
-      public GroupByStream(ViewStream main_stream, Func<string[]> groupby_cols_func,
-                                 Func<List<IBlueRow>, List<IBlueRow>> evaluate_func)
-                                 : this(main_stream, new DelNonIncGroupByEvaluator(groupby_cols_func, evaluate_func)) { }
-
-      public GroupByStream(ViewStream parent_str, string groupby_col, string standard_group_by_text)
-      : this(parent_str, groupby_col, standard_group_by_text, String.Empty) { }
-
-      // this is the main constructor that does everything
-      public GroupByStream(ViewStream parent_str, string groupby_col, string standard_group_by_ops, string optional_sort_col) {
-         MultiRowOpHelper std_group_by = MultiRowOpHelper.parse_ops(standard_group_by_ops);
-         Func<string[]> groupby_cols_func = () => {
-            return String.IsNullOrEmpty(groupby_col) ? null : new string[] { groupby_col };
-         };
-         Func<List<IBlueRow>, List<IBlueRow>> evaluate_func = (group_rows) => {
-            if (!String.IsNullOrEmpty(optional_sort_col))
-               group_rows.Sort((r1, r2) => BlueDataUtility.compare_rows(r2, r1, optional_sort_col));
-            // this si where the group by aggregation happens
-            IBlueRow rv = MultiRowOpHelper.group_by_compute(group_rows, std_group_by);
-            return new IBlueRow[] { rv }.ToList();
-         };
-         ViewStream rs = new GroupByStream(parent_str, groupby_cols_func, evaluate_func);
-         mySetRealSource(rs);
-      }
-      public GroupByStream(ViewStream parent_str, string standard_group_by_text) :
-      this(parent_str, String.Empty, standard_group_by_text) {
-      }
-
-
-      protected override void myOnFirstSubscriber() { _main_stream.onChange += _main_stream_onchange; }
-
-      protected override void myOnLastSubscriber() { _main_stream.onChange -= _main_stream_onchange; }
-
-      private void _main_stream_onchange(object nv) {
-         DataPacket dp = nv as DataPacket;
-         ///grouping together
-         _timer.Restart();
-         IDictionary<int, List<IBlueRow>> grouped_rows = (_groupby_cols == null || _groupby_cols.Length == 0) ?
-         new Dictionary<int, List<IBlueRow>>() { { 0, dp.newRows.to_list() } } :
-         BlueDataUtility.group_together(dp.newRows, _groupby_cols, ref _row_hasher);
-         _timer.Stop();
-         var groupElapsedMs = _timer.ElapsedMilliseconds;
-         if (groupElapsedMs >= 50) {
-            Log.DataFlow.info($"Time to group {dp.newRows.Count} rows into {grouped_rows.Count} groups: {_timer.ElapsedMilliseconds}ms");
-         }
-         // Log.log(bts.Log.Level.Info, bts.log.Subsystem.gen, String.Format("Time to group {0} rows into {1} groups = {2}", dp.newRows.Count, grouped_rows.Count, _timer.lastIntervalTime));
-         /// reducing
-         _timer.Restart();
-         IBlueRowCollection total_state = _groupby_func.evaluate(grouped_rows);
-         _timer.Stop();
-         var applyElapsedMs = _timer.ElapsedMilliseconds;
-         _n_total_fires++;
-         if (applyElapsedMs >= 50) {
-            _n_expensive_fires++;
-            Log.DataFlow.info(
-               $"Time to evaluate {grouped_rows.Count} groups {_timer.ElapsedMilliseconds}ms, ratio {_n_expensive_fires}/{_n_total_fires}");
-         }
-         // TODO,
-         if (total_state != null)
-            myFireEvent(new DataPacket(total_state, total_state, myLastValue == null ? new BlueTable(total_state.schema) : (myLastValue as DataPacket).newRows));
-      }
-   }
-}

src/DataStreamSuite/DataStream/BlueOrb/Join/SimpleJoinStream.cs

@@ -1,187 +0,0 @@
-namespace bts.BlueOrb {
-   using BCG;
-   using RowHash = System.Int32;
-   using System;
-   using System.Collections.Generic;
-   using System.Linq;
-   using System.Text;
-   using System.Threading.Tasks;
-   using System.Collections;
-   using System.Data.Entity.Core.Mapping;
-
-   [DataStreamUserRespresentation("JoinStream")]
-   public class SimpleJoinStream : ViewStream {
-      public class JoinBookkeepingData {
-         // initialized in cons
-         private readonly List<string> _join_col_names;
-         private readonly bool _is_left;
-         private readonly BlueRowHasher _hasher;
-         // initialized on first udpate         
-         private IBlueTableSchema _schema;
-         private int[] _join_cols;
-         private IDictionary<RowHash, List<IBlueRow>> _hash_row_map = new Dictionary<RowHash, List<IBlueRow>>();
-         
-         public JoinBookkeepingData(ViewStream vs, List<string> join_cols, bool is_left, BlueRowHasher row_hasher) {
-            _join_col_names = join_cols.Where (c => !String.IsNullOrEmpty(c)).ToList ();
-            _is_left = is_left;
-            _hasher = row_hasher;
-         }
-
-         public IBlueTableSchema schema => _schema;
-
-         public IDictionary<RowHash, List<IBlueRow>> rows => _hash_row_map;
-
-         public List<IBlueRow> get_matching_rows (int hash) {
-            List<IBlueRow> rv = null;
-            _hash_row_map.TryGetValue(hash, out rv);
-            return rv;
-         }
-
-         private RowHash _get_hash(IBlueRow r) {
-            if (_join_cols == null || _join_cols.Length == 0) { 
-               return 0;
-            }
-            return _hasher.get_hash(r, _join_cols);
-         }
-
-         public void update_data(DataPacket dp) {
-            if (!ready) {               
-               _schema = dp.newRows.schema;
-               _join_cols = _join_col_names.Count == 0 ? new int[] { }: _join_col_names.Select(n => dp.schema[n].colNo).ToArray();
-            }
-            _hash_row_map.Clear();
-            foreach (IBlueRow r in dp.newRows) {
-               int hash = _get_hash(r);
-               List<IBlueRow> hash_rows = null;
-               if (!_hash_row_map.TryGetValue(hash, out hash_rows)) {
-                  hash_rows = new List<IBlueRow>();
-                  _hash_row_map.Add(hash, hash_rows);
-               }
-               hash_rows.Add(r);
-            }
-         }
-
-         public bool ready {
-            get {
-               return _schema != null;
-            }
-         }
-
-      }
-
-      private ViewStream _ls, _rs;
-      private JoinBookkeepingData _ldata, _rdata;
-      private ViewStreamDatabase _vsd;
-      private IBlueTableSchema _schema = null;
-
-      public SimpleJoinStream(ViewStream ls, string ljc, ViewStream rs, string rjc) :
-         this(ViewStreamDatabase.instance, ls, ljc, rs, rjc) { }
-
-      public SimpleJoinStream(ViewStreamDatabase vsd, ViewStream ls, string ljc, ViewStream rs, string rjc) {
-         _ls = ls;
-         _rs = rs;
-         List<string> lj_col_list = ljc.Split(',').Where(x => !String.IsNullOrEmpty(x)).ToList();
-         List<string> rj_col_list = rjc.Split(',').Where(x => !String.IsNullOrEmpty(x)).ToList();
-         Log.assert(lj_col_list.Count ==  rj_col_list.Count);
-         int n_cols = lj_col_list.Count;
-         BlueRowHasher hasher = new BlueRowHasher(n_cols);
-         _ldata = new JoinBookkeepingData(ls, lj_col_list, true, hasher);
-         _rdata = new JoinBookkeepingData(rs, rj_col_list, false, hasher);
-      }
-
-      protected override void myOnFirstSubscriber() {
-         _ls.onChange += _ls_onchange;
-         _rs.onChange += _rs_onchange;
-      }
-
-      private void _ls_onchange(object dp_obj) {
-         DataPacket dp = dp_obj as DataPacket;
-         _ldata.update_data(dp);
-         _form_and_fire();
-      }
-
-      private void _rs_onchange(object dp_obj) {
-         DataPacket dp = dp_obj as DataPacket ;
-         _rdata.update_data(dp);
-         _form_and_fire();
-      }
-
-      protected override void myOnLastSubscriber() {
-         _ls.onChange -= _ls_onchange;
-         _rs.onChange -= _rs_onchange;
-      }
-
-      private void _build_schema() {
-         if (_schema != null)
-            return;
-         List<ColumnInfo> cis = new List<ColumnInfo>();
-         cis.Add(new ColumnInfo("id", 0, TypeCode.Int32));
-         foreach (ColumnInfo ci in _ldata.schema.columnInfo) {
-            string col_name = _rdata.schema.contains_column(ci.colName) ? $"left_{ci.colName}" : ci.colName;
-            cis.Add(new ColumnInfo(col_name, cis.Count, ci.colType));
-         }
-         foreach (ColumnInfo ci in _rdata.schema.columnInfo) {
-            string col_name = _ldata.schema.contains_column(ci.colName) ? $"right_{ci.colName}" : ci.colName;
-            cis.Add(new ColumnInfo(col_name, cis.Count, ci.colType));
-         }
-         _schema = new BlueTableSchema($"{_ldata.schema.tableName}_join_{_rdata.schema.tableName}", cis.ToArray());
-      }
-
-      private void _form_and_fire() {
-         if (!_ldata.ready||!_rdata.ready)
-            return;
-         _build_schema();
-         List<object[]> res_rows = new List<object[]>();
-         
-         Action<object[], IBlueRow, bool> copy_row = (rval, r, is_left) => {
-            int offset = 1;
-            JoinBookkeepingData sdata = _ldata;
-            if (is_left) {
-               offset = 1;
-               sdata = _ldata;
-            } else {
-               sdata = _rdata;
-               offset = 1 + _ldata.schema.columnInfo.Length;
-            }
-            for (int i = 0; i < sdata.schema.columnInfo.Length; i++) {
-               rval[offset + i] = r[i];
-            }
-         };
-         Func<int, IBlueRow, object[]> build_row_array = (id, l_row) => {
-            object[] rv = new object[_schema.columnNames.Length];            
-            for (int i = 0; i < _schema.columnNames.Length; i++) {
-               rv[i] = DBNull.Value;
-            }
-            rv[0] = id;
-            copy_row(rv, l_row, true);
-            return rv;
-         };
-         // this is where the join happens
-         foreach (var kv in _ldata.rows) {
-            RowHash hash = kv.Key;
-            List<IBlueRow> l_rows = kv.Value;
-            List<IBlueRow> r_rows = _rdata.get_matching_rows(hash);
-
-            foreach (IBlueRow lr in l_rows) {                              
-               if (r_rows != null) {
-                  foreach (IBlueRow r in r_rows) {
-                     object[] rowvals = build_row_array(res_rows.Count, lr);
-                     copy_row(rowvals, r, false);
-                     res_rows.Add(rowvals);
-                  }
-               } else {
-                  object[] rowvals = build_row_array(res_rows.Count, lr);
-                  res_rows.Add(rowvals);
-               }
-            }
-         }
-         BlueTable rv = new BlueTable(_schema);
-         foreach (object[] rval in res_rows) {
-            rv.addRow(new BlueRow(_schema, rval));
-         }
-         var dp = BlueDataUtility.build_dp(rv, myLastValue as DataPacket);
-         myFireEvent(dp);
-      }
-
-   }
-}

src/DataStreamSuite/DataStream/BlueOrb/JsonSaverStream.cs

@@ -1,28 +0,0 @@
-//namespace bts.Blueorb {
-//   using BCG;
-
-//   public class JsonSaverStream : ViewStream {
-//      private ViewStream _stream;
-//      private string _file_name;
-//      public JsonSaverStream(ViewStream vs, string file_name) {
-//         _stream = vs;
-//         _file_name = file_name;
-//      }
-
-//      protected override void myOnFirstSubscriber() {
-//         _stream.onChange += _on_data;
-//      }
-
-//      private void _on_data(object dp_obj) {
-//         if (dp_obj == null) {
-//            return;
-//         }
-//         DataPacket dp = dp_obj as DataPacket;
-//         dp.newRows.write_to_json_file(dp.newRows, _file_name);
-//      }
-
-//      protected override void myOnLastSubscriber() {
-//         _stream.onChange -= _on_data;
-//      }
-//   }
-//}

src/DataStreamSuite/DataStream/BlueOrb/LiteralStream.cs

@@ -1,91 +0,0 @@
-using System;
-using System.Collections;
-using System.Collections.Generic;
-
-using Amazon.Runtime.Internal.Util;
-
-using BCG;
-using BCG.Utilities;
-
-using bts;
-
-namespace bts.BlueOrb{
-   public class LiteralStream : ViewStream {
-
-      public static List<Tuple<ColumnInfo, object>> parse_columns(string key_value_pairs) {
-         List<Tuple<ColumnInfo, object>> rv = new List<Tuple<ColumnInfo, object>>();
-         string[] pairs = key_value_pairs.Split(' ');
-         foreach (string p in pairs) {
-            string[] kvp = p.Split(':');
-            string col_name = kvp[0];
-            string val = kvp[1];
-            var tup = BlueDataUtility.guess_type(val);
-            var ci = new ColumnInfo(col_name, rv.Count, tup.Item1);
-            rv.Add(Tuple.Create(ci, tup.Item2));
-         }
-         return rv;
-      }
-
-      //literal("table1", "x:123, y:1.001, z:abc;x:123, y:1.001, z:abc")
-      // col_name: col_value ... ; col_name: col_value
-      public LiteralStream(string table_name, string rows) {
-         Func<string, int, IBlueRow> get_row = (json_row, id) => {
-            Dictionary<string, object> map = new Dictionary<string, object>();
-            string[] toks = json_row.Split(',', StringSplitOptions.TrimEntries);
-            //Logger.assert(col_vals.Length == 2);
-            foreach (string t in toks) {
-               var col_vals = t.Split(':', StringSplitOptions.TrimEntries);
-               string col = col_vals[0];
-               string val = col_vals[1];
-               map.Add(col, val);
-            }
-            map["id"] = id;
-            return new BlueRow(map);
-         };
-         string[] row_arr = rows.Split(';', StringSplitOptions.TrimEntries);
-         BlueTable table = null;
-         foreach (string json_string in row_arr) {
-            IBlueRow r = //BlueDataUtility.convert_json_to_row(json_string, table == null ? 0 : table.Count);
-            get_row(json_string, table == null ? 0 : table.Count);
-            table = table ?? new BlueTable(r.schema);
-            table.addRow(r);
-         }
-         ViewStream str = new BlueTableStream(table);
-         str = new SchemaGuesserStream(str);
-         mySetRealSource(str);
-      }
-
-      public LiteralStream(string tableName, string schema, string value) {
-         string[] columns = schema.Split(' ');
-
-
-         ColumnInfo[] ci = new ColumnInfo[columns.Length + 1];
-         ci[0] = new ColumnInfo("id", 0, typeof(int));
-         int n = 1;
-         foreach (string c in columns) {
-            string[] columnAndType = c.Split('/');
-            if (columnAndType.Length == 1)
-               ci[n] = new ColumnInfo(c, n, typeof(double));
-            else
-               ci[n] = new ColumnInfo(columnAndType[0], n, Type.GetType("System." + columnAndType[1], true));
-            n++;
-         }
-
-
-         BlueTable t = new BlueTable(new BlueTableSchema(tableName, ci));
-         string[] rows = value.Split(';');
-         n = 0;
-         foreach (string r in rows)
-            try {
-               if (r == string.Empty)
-                  continue;
-               t.addRow(
-                  BlueDataUtility.createRowFromCSVFileRow(r, t.schema, ++n));
-            } catch (Exception e) {
-               throw (new Exception("Error occured at line: " + n.ToString() + " " + e.Message));
-            }
-
-         mySetRealSource(new BlueRowCollectionStaticStream(t));
-      }
-   }
-}

src/DataStreamSuite/DataStream/BlueOrb/MillisecondStream.cs

@@ -1,89 +0,0 @@
-using System;
-
-using BCG.Utilities.General;
-using BCG;
-using bts;
-
-namespace bts.BlueOrb {
-   [DataStreamUserRespresentation("MillisecondStream")]
-   public class MillisecondStream : ViewStream, IWorker {
-      private DateTime _last_fire_time = DateTime.MinValue;
-      private readonly TimeSpan _period;
-      private readonly int _size;
-      private readonly ViewStreamDatabase _vsd;
-      private IBlueTableSchema _schema;
-      private bts.CircularBufferTable _table;
-
-      public MillisecondStream(int interval_msecs, int size)
-      : this(ViewStreamDatabase.instance, interval_msecs, size) {
-      }
-
-      public MillisecondStream(ViewStreamDatabase vsd, int interval_msecs, int size) {
-         _vsd = vsd;
-         _period = TimeSpan.FromMilliseconds(interval_msecs);
-         _size = size;
-         _schema = BlueDataUtility.form_schema("Milliseconds",
-               Tuple.Create("id", TypeCode.Int32),
-               Tuple.Create("time", TypeCode.DateTime),
-               Tuple.Create("display_time", TypeCode.String));
-         _table = new CircularBufferTable(_schema, _size);
-      }
-
-      protected override void myOnFirstSubscriber() {
-         _vsd.addWorker(this);
-      }
-
-      protected override void myOnLastSubscriber() {
-         _vsd.removeWorker(this);
-      }
-
-      private int _id = 0;
-      public void doWork() {
-         if (DateTime.Now - _last_fire_time >= _period) {
-            _last_fire_time = DateTime.Now;
-            IBlueRow nr = new BlueRow(_schema, _id++, _last_fire_time,
-            _last_fire_time.ToString("yyyy-MM-dd HH:mm:ss.fff"));
-            _table.add_row(nr);
-            var dp = BlueDataUtility.build_dp_inc(_table, myLastValue as DataPacket);
-            myFireEvent(dp);
-         }
-      }
-   }
-   internal class MSecondStream : TimeUnitStream {
-      public MSecondStream(int intervalInms, int size)
-         : base(DateTime.Now, new TimeSpan(0, 0, 0, 0, intervalInms), size, ViewStreamDatabase.instance) { }
-
-      protected override DateTime myRoundTime(DateTime dt) { return dt; }
-   }
-
-   internal class DisplayableMillisecondStream : ViewStream, IStreamMapper {
-      private IBlueTableSchema mySchema = new BlueTableSchema("ms",
-                                                               new ColumnInfo("id", 0, typeof(int)),
-                                                               new ColumnInfo("time", 1, typeof(string)));
-      public DisplayableMillisecondStream(int intervalInMs, int sz) {
-         mySetRealSource(new MapStream(new MSecondStream(intervalInMs, sz), this));
-      }
-      #region IStreamMapper Members
-
-      public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema) {
-         return mySchema;
-      }
-
-      #endregion
-
-      #region IDynamicStreamMapper Members
-
-      public IBlueRow map(IBlueRow row) {
-         DateTime time = row.get_datetime("time");
-         var time_txt = time.ToString("yyyy-MM-dd HH:mm:ss.fff");
-         return new BlueRow(mySchema, row["id"], time_txt);
-      }
-
-      #endregion
-
-      private string myToString(DateTime dt) {
-         string sep = ":";
-         return dt.Hour + sep + dt.Minute + sep + dt.Second + sep + dt.Millisecond;
-      }
-   }
-}

src/DataStreamSuite/DataStream/BlueOrb/MultiRowOpHelper.cs

@@ -1,161 +0,0 @@
-namespace bts.BlueOrb {
-   using BCG;
-
-   using System;
-   using System.Collections.Generic;
-   using System.Linq;
-   //using MathNet.Numerics.Statistics;
-
-
-   public enum MultiRowOp {
-      Mean,
-      Median,
-      Mode,
-      Range,
-      StdDev,
-      Max,
-      Min,
-      First,
-      Last,
-      Sum,
-      EdgeDiff,
-      EdgeDiffPercentage,
-   }
-   public class MultiRowOpHelper {
-      // input + op
-      public readonly List<Tuple<string, MultiRowOp>> operations;
-
-      public static MultiRowOpHelper parse_ops(string text) {
-         // text is inp_col: op, inp_col: op, inp_col:op
-         // ex: "open:First, close:Last, high:Max, low:Min, volume:Sum")
-         List<Tuple<string, MultiRowOp>> rv = new List<Tuple<string, MultiRowOp>>();
-         string[] calcs = text.Split(',');
-         foreach (string c in calcs) {
-            string[] toks = c.Split(':');
-            string inp_col = toks[0].Trim();
-            string op_txt = toks[1].Trim();
-            MultiRowOp op = Enum.Parse<MultiRowOp>(op_txt, true);
-            rv.Add(Tuple.Create(inp_col, op));
-         }
-         return new MultiRowOpHelper(rv);
-      }
-
-      public MultiRowOpHelper(List<Tuple<string, MultiRowOp>> operations_) {
-         operations = operations_;
-      }
-
-      // aggregates multiple rows into one - ie does group_by compute
-      public static IBlueRow group_by_compute(List<IBlueRow> rows, MultiRowOpHelper parameters) {
-         if (rows.Count() == 0) {
-            return null;
-         }
-
-         IBlueRow ir = rows.ElementAt(0);
-         IBlueTableSchema in_schema = ir.schema;
-         IBlueTableSchema out_schema = in_schema;
-         IDictionary<string, Tuple<string, MultiRowOp>> out_col_map =
-            new Dictionary<string, Tuple<string, MultiRowOp>>();
-         foreach (var t in parameters.operations) {
-            string out_col_name = $"{t.Item1}_{t.Item2}".ToLower();
-            out_schema = out_schema.add_column(TypeCode.Double, out_col_name);
-            out_col_map[out_col_name] = t;
-         }
-         MutableBlueRow or = new MutableBlueRow(out_schema, ir);
-         foreach (var kv in out_col_map) {
-            string out_col_name = kv.Key;
-            string in_col_name = kv.Value.Item1;
-            MultiRowOp op = kv.Value.Item2;
-            IEnumerable<double> col_vals = BlueDataUtility.get_colvalues<double>(rows, in_col_name);
-            double comp_res = compute_op(col_vals, op);
-            or.setValue(out_col_name, comp_res);
-         }
-         return or;
-      }
-
-      // aggregates multiple rows into one, used by MultiRowRollingMathStream
-      public static IBlueRow group_by_compute(List<IBlueRow> rows, int start_idx, int n_rows, MultiRowOpHelper parameters) {
-         if (rows.Count() == 0) {
-            return null;
-         }
-         IBlueRow ir = rows.ElementAt(start_idx);
-         IBlueTableSchema in_schema = ir.schema;
-         IBlueTableSchema out_schema = in_schema;
-         // out_col vs inp_cal + type
-         IDictionary<string, Tuple<string, MultiRowOp>> out_col_map =
-            new Dictionary<string, Tuple<string, MultiRowOp>>();
-         foreach (var t in parameters.operations) {
-            string out_col_name = $"{t.Item1}_{t.Item2}".ToLower();
-            out_schema = out_schema.add_column(TypeCode.Double, out_col_name);
-            out_col_map[out_col_name] = t;
-         }
-         MutableBlueRow or = new MutableBlueRow(out_schema, ir);
-         foreach (var kv in out_col_map) {
-            string out_col_name = kv.Key;
-            string in_col_name = kv.Value.Item1;
-            MultiRowOp op = kv.Value.Item2;
-            IEnumerable<double> col_vals = BlueDataUtility.get_colvalues<double>(rows, start_idx, n_rows, in_col_name);
-            double comp_res = compute_op(col_vals, op);
-            or.setValue(out_col_name, comp_res);
-            or.setValue("id", ir["id"]);
-         }
-         return or;
-      }
-
-      // rolling_rows are the rolling window of rows. the 0th row is the one thats gettin
-      // calculated. rows 1 - n-1 are its window rows. the schema is based on the operations.
-
-      public IBlueRow rolling_calc(List<IBlueRow> rolling_rows) {
-         if (rolling_rows.Count() == 0) {
-            return null;
-         }
-         IBlueRow ir = rolling_rows.ElementAt(0);
-         IBlueTableSchema in_schema = ir.schema;
-         IBlueTableSchema out_schema = in_schema;
-         IDictionary<string, Tuple<string, MultiRowOp>> out_col_map =
-            new Dictionary<string, Tuple<string, MultiRowOp>>();
-         foreach (var t in operations) {
-            string out_col_name = $"{t.Item1}_{t.Item2}".ToLower();
-            out_schema = out_schema.add_column(TypeCode.Double, out_col_name);
-            out_col_map[out_col_name] = t;
-         }
-         MutableBlueRow or = new MutableBlueRow(out_schema, ir);
-         foreach (var kv in out_col_map) {
-            string out_col_name = kv.Key;
-            string in_col_name = kv.Value.Item1;
-            MultiRowOp op = kv.Value.Item2;
-            IEnumerable<double> col_vals = BlueDataUtility.get_colvalues<double>(rolling_rows, in_col_name);
-            double comp_res = compute_op(col_vals, op);
-            or.setValue(out_col_name, comp_res);
-         }
-         return or;
-      }
-
-      public static double compute_op(IEnumerable<double> col_vals, MultiRowOp op) {
-         switch (op) {
-            case MultiRowOp.Max:
-               return col_vals.Max<double>();
-            case MultiRowOp.Min:
-               return col_vals.Min<double>();
-            case MultiRowOp.Mean:
-               return MathNet.Numerics.Statistics.Statistics.Mean(col_vals);
-            case MultiRowOp.Sum:
-               return col_vals.Sum();
-            case MultiRowOp.Median:
-               return MathNet.Numerics.Statistics.Statistics.Median(col_vals);
-            //case MultiRowOp.Mode:
-            //   return MathNet.Numerics.Statistics.Statistics.(col_vals);
-            case MultiRowOp.First:
-               return col_vals.First();
-            case MultiRowOp.Last:
-               return col_vals.Last();
-            case MultiRowOp.StdDev:
-               return MathNet.Numerics.Statistics.Statistics.StandardDeviation(col_vals);
-            default:
-               // TODO
-               Log.assert(false);
-               return Double.NaN;
-         }
-      }
-
-   }
-}

src/DataStreamSuite/DataStream/BlueOrb/MultiRowRollingMathStream.cs

@@ -1,40 +0,0 @@
-namespace bts.BlueOrb {
-   using BCG;
-
-   using System.Collections.Generic;
-   using System.Net;
-
-   [DataStreamUserRespresentation("MultiRowRollingMathStream")]
-   public class MultiRowRollingMathStream : ViewStream {
-      private ViewStream _sorted_parent_str;
-      private MultiRowOpHelper _op;
-      private int _rolling_window_sz;
-      public MultiRowRollingMathStream(ViewStream parent_str, string op_txt, string sort_exp, int rolling_window_sz) {
-         _sorted_parent_str = new SortStream(parent_str, sort_exp);
-         _op = MultiRowOpHelper.parse_ops(op_txt);
-         _rolling_window_sz = rolling_window_sz;
-      }
-
-      protected override void myOnFirstSubscriber() {
-         _sorted_parent_str.onChange += _parent_str_onchange;
-      }      
-
-      protected override void myOnLastSubscriber() {
-         _sorted_parent_str.onChange -= _parent_str_onchange;
-      }
-
-      private void _parent_str_onchange(object dp_obj) {
-         DataPacket dp = dp_obj as DataPacket;
-         var rows = dp.newRows.to_list();
-         BlueTable out_table = null;
-         for (int i = 0; i < rows.Count; i++) {
-            var out_r = MultiRowOpHelper.group_by_compute(rows, i, _rolling_window_sz, _op);
-            if (out_table == null)
-               out_table = new BlueTable(out_r.schema);
-            out_table.addRow(out_r);
-         }            
-         var dp_out = BlueDataUtility.build_dp_inc(out_table, myLastValue as DataPacket, true);
-         myFireEvent(dp_out);
-      }
-   }
-}

src/DataStreamSuite/DataStream/BlueOrb/ProjectStream.cs

@@ -1,188 +0,0 @@
-namespace bts.BlueOrb {
-   using BCG;
-   using System;
-   using System.Collections;
-   using System.Collections.Generic;
-   using System.Linq;
-
-   #region mappings
-   internal class ProjectData {
-      // schema from the base stream
-      public readonly IBlueTableSchema in_schema;
-      // this is the projected schema
-      public readonly IBlueTableSchema out_schema;
-      // projected cols to the input cols, for cols who's names are changed
-      public readonly IDictionary<string, string> mapped_to_orig_col_map = new Dictionary<string, string>();
-      public ProjectData(IBlueTableSchema in_schema_, IBlueTableSchema out_schema_,
-                  IDictionary<string, string> mapped_to_orig_col_map_) {
-         in_schema = in_schema_;
-         out_schema = out_schema_;
-         mapped_to_orig_col_map = mapped_to_orig_col_map_;
-      }
-   }
-   #endregion
-
-
-#region Row with mappings build in
-   internal class NewProjectRow : IBlueRow {
-      #region IBlueRow members
-      private ProjectData _project_data;
-      private IBlueRow _row;
-      public NewProjectRow(ProjectData project_data, IBlueRow row) {
-         _project_data = project_data;
-         _row = row;
-      }
-      public IBlueTableSchema schema { get { return _project_data.out_schema; } }
-      #endregion
-
-      #region ICollection members
-      public int Count { get { return schema.columnNames.Length; } }
-
-      public bool IsSynchronized { get { return false; } }
-
-      public object SyncRoot { get { return this; } }
-
-      public void CopyTo(Array array, int index) {
-         int i = index;
-         foreach (object o in this as IEnumerable)
-            array.SetValue(o, i++);
-      }
-      #endregion
-
-      #region IDictionary members
-      public bool IsFixedSize { get { return true; } }
-
-      public bool IsReadOnly { get { return true; } }
-
-      public object this[object key] {
-         get {
-            if (key is string)
-               return this[(string)key];
-            return this[(int)key];
-         }
-         set { throw new NotSupportedException("The IBlueRow is readonly"); }
-      }
-
-      public object this[int index] {
-         get {
-            return this[schema.columnInfo[index].colName];
-         }
-      }
-
-      public object this[string col_name] {
-         get {
-            string mapped_to_col = null;
-            if (_project_data.mapped_to_orig_col_map.TryGetValue(col_name, out mapped_to_col)) {
-               return _row[mapped_to_col];
-            }
-            return _row[col_name];
-         }
-      }
-
-      public ICollection Keys { get { return schema.columnNames; } }
-
-      public ICollection Values {
-         get {
-            object[] retVal = new object[schema.columnNames.Length];
-            CopyTo(retVal, 0);
-            return retVal;
-         }
-      }
-
-      public void Add(object key, object value) { throw new NotSupportedException("The IBlueRow is readonly"); }
-
-      public void Clear() { throw new NotSupportedException("The IBlueRow is readonly"); }
-
-      public bool Contains(object key) { return schema[(string)key] != null; }
-
-      /// <summary>
-      /// Note, the default enumerator is the IDictionary's enumerator, so if user needs to
-      /// iterate through the values, typecast this into an IEnumerable
-      /// </summary>
-      /// <returns></returns>
-      public IDictionaryEnumerator GetEnumerator() { return new BlueRowDictionaryEnumerator(this); }
-
-      IEnumerator IEnumerable.GetEnumerator() { return new BlueRowEnumerator(this); }
-
-      public void Remove(object key) { throw new NotSupportedException("The IBlueRow is readonly"); }
-      #endregion
-   }
-   #endregion
-
-   [DataStreamUserRespresentation("ProjectStream")]
-   public class NewProjectStream : ViewStream, IStreamMapper {
-
-      private ViewStream _stream;
-      private string _project_expression;
-      private ProjectData _project_data = null;
-      //public ProjectData 
-      public NewProjectStream(ViewStream vs, string project_expression) {
-         _stream = vs;
-         _project_expression = project_expression;
-         _stream = new MapStream(vs, this);
-         mySetRealSource(_stream);
-      }
-
-      public IBlueTableSchema getOutputSchema(IBlueTableSchema input_schema) {
-         _project_data = _create_project_data(input_schema, _project_expression);
-         return _project_data.out_schema;
-      }
-
-      public IBlueRow map(IBlueRow row) {
-         return new NewProjectRow(_project_data, row);
-      }
-
-      // this is where most of the logic is, in terms of translating cols
-      private static ProjectData _create_project_data(IBlueTableSchema input_schema,
-                                                      string project_expression) {
-         project_expression = project_expression.Trim();
-         string[] project_toks = project_expression.Split(' ', ',');
-         HashSet<string> removed_cols = new HashSet<string>();
-         Dictionary<string, string> orig_to_mapped = new Dictionary<string, string>();      
-         // by default every col will show up, so if we have the projection as (a:a1)
-         // it will convert a to a1, 
-         // modifications (6/30/2025):
-         // only the columns represented here, either mapped
-         // or not, will show
-         // examples: if a schema has cols a b c
-         // project(parent_str, "a:aa b") -> aa b
-         foreach (string o_p_tok in project_toks) {
-            string p_tok = o_p_tok.Trim();
-            if (p_tok.Contains(":")) {
-               // explicit mapping
-               string[] map_toks = p_tok.Split(":");
-               string orig_col = map_toks[0];
-               string mapped_col = map_toks[1];
-               orig_to_mapped.Add(orig_col, mapped_col);  
-            } else {
-               // this is a col_name explicitely stated
-               if (input_schema.columnNames.Contains(p_tok)) {
-                  string orig_col = p_tok;
-                  string mapped_col = orig_col;
-                  orig_to_mapped.Add(orig_col, mapped_col);
-               }
-            }
-         }
-
-         // make the new schema
-         List<ColumnInfo> new_cols = new List<ColumnInfo>();
-         foreach (ColumnInfo ci in input_schema.columnInfo) {
-            if (ci.colName == "id") {
-               ColumnInfo nc = new ColumnInfo(ci.colName, new_cols.Count, ci.colType);
-               new_cols.Add(nc);
-               continue;           
-            } else if (orig_to_mapped.ContainsKey(ci.colName)) {
-               string mapped_name = orig_to_mapped[ci.colName];
-               ColumnInfo nc = new ColumnInfo(mapped_name, new_cols.Count, ci.colType);
-               new_cols.Add(nc);
-            }
-         }
-         IBlueTableSchema new_schema = new BlueTableSchema(input_schema.tableName, new_cols.ToArray());
-         var mapped_to_orig_map = orig_to_mapped.ToDictionary(kv => kv.Value, kv => kv.Key);
-         return new ProjectData(input_schema, new_schema, mapped_to_orig_map);
-      }
-
-   
-
-   }
-}

src/DataStreamSuite/DataStream/BlueOrb/PythonDataframeStream.cs

@@ -1,127 +0,0 @@
-namespace bts.BlueOrb {
-   using BCG;
-   using System.IO;
-   using System.Diagnostics;
-   using System;
-
-   [DataStreamUserRespresentation("PythonDataframeStream")]
-   public class PythonDataframeStream : ViewStream {
-      private ViewStream _parent_str;
-      private readonly string _in_csv_file_name;
-      private readonly string _out_csv_filename;
-      private readonly string _transform_function;
-      public PythonDataframeStream(ViewStream parent_str, string transform_function) {
-         var guid = System.Guid.NewGuid();
-         _in_csv_file_name = $"pdfs_in_{guid}.csv";
-         _out_csv_filename = $"pdfs_out_{guid}.csv";
-         _parent_str = new CSVSaverStream(parent_str, _in_csv_file_name);
-         _transform_function = transform_function;
-      }
-
-      protected override void myOnFirstSubscriber() {
-         _parent_str.onChange += _parent_str_onchange;
-      }
-
-      protected override void myOnLastSubscriber() {
-         _parent_str.onChange -= _parent_str_onchange;
-      }
-
-      private void _parent_str_onchange(object dp_obj) {
-         Log.assert(dp_obj != null);
-         // the file _in_csv_file_name is now populated.
-         // call py_func_wrapper_main and pass this file name as well
-         // as the output file name
-         // py_func_wrapper_main reads _in_csv_file_name into the df and calls 
-         // transform function, which returns the tranformed df. py_func_wrapper_main then
-         // writes it into the _out_csv_filename file.
-         // now this function should read _out_csv_filename into a dp
-         // and fire it
-         _compute_and_write();
-         ViewStream rs = new SchemaGuesserStream(new CSVReaderStream(_out_csv_filename));
-         IBlueRowCollection nrows = null;
-         rs.onChange += (dp_obj) => {
-            nrows = (dp_obj as DataPacket).newRows;
-         };
-         var dp = BlueDataUtility.build_dp(nrows, myLastValue as DataPacket);
-         myFireEvent(dp);
-      }
-
-      private void _compute_and_write() {
-         File.WriteAllText("user_code.py", _transform_function);
-         // wrapper function
-         string py_func_wrapper_main = @"
-import sys
-from user_code import transform
-import pandas as pd
-
-if __name__ == '__main__':
-   input_file_name = sys.argv[1]
-   output_file_name = sys.argv[2]
-# 1. load input_file_name into df
-# 2. do some test calc
-# 3. write the output of 2 into output_file_name
-   df = pd.read_csv(input_file_name)   
-   df_out = transform(df)
-   df_out.to_csv(output_file_name, index=False)";
-
-
-         File.WriteAllText("runner.py", py_func_wrapper_main);
-
-         ProcessStartInfo psi = new ProcessStartInfo {
-            FileName = "python",
-            Arguments = $"runner.py {_in_csv_file_name} {_out_csv_filename}",
-            RedirectStandardOutput = false,
-            UseShellExecute = false
-         };
-         var proc = Process.Start(psi);
-         proc.WaitForExit();
-
-      }
-
-      #region Unused test code
-      private void _simple_test() {
-
-         // actual function that needs to be called ////////////////////////
-         string py_real_function = @"
-def test(x, out_path):
-   rv = x * 10
-   with open(out_path, 'w') as f:
-      f.write(str(rv))
-   return rv";
-         // the function is in a string object.
-         File.WriteAllText("user_code.py", py_real_function);
-
-
-         // runner app (top level main function) //////////////////////////////
-         string py_func_wrapper_main = @"
-import sys
-from user_code import test
-
-if __name__ == '__main__':
-   arg = int(sys.argv[1])
-   out_path = sys.argv[2]
-   rv = test(arg, out_path)
-   rv = rv + 12
-   with open(out_path, 'w') as f:to \
-      f.write(str(rv))";
-   
-         File.WriteAllText("runner.py", py_func_wrapper_main);
-
-         string outputPath = "py_out.txt";
-         ProcessStartInfo psi = new ProcessStartInfo {
-            FileName = "python",
-            Arguments = $"runner.py 4 {outputPath}",
-            RedirectStandardOutput = false,
-            UseShellExecute = false
-         };
-         var proc = Process.Start(psi);
-         proc.WaitForExit();
-
-         // Now read the output written by the function
-         string result = File.ReadAllText(outputPath);
-         Console.WriteLine(result); // Should print "6"
-
-      }
-      #endregion
-   }
-}

src/DataStreamSuite/DataStream/BlueOrb/SchemaGuesserStream.cs

@@ -1,30 +0,0 @@
-using BCG;
-
-namespace bts.BlueOrb {
-   public class SchemaGuesserStream : ViewStream {
-
-      private IBlueRowCollection _rows = null;
-      private ViewStream _vs;
-      public SchemaGuesserStream(ViewStream vs) {
-         _vs = vs;
-      }
-
-      protected override void myOnFirstSubscriber() {
-         _vs.onChange += _on_data;
-      }
-      protected override void myOnLastSubscriber() {
-         _vs.onChange -= _on_data;
-      }
-
-      private void _on_data(object obj) {
-         DataPacket dp = obj as DataPacket;
-         if (_rows == null) {
-            _rows = BlueDataUtility.get_typed_table(dp.newRows);
-         }
-         var dp_out = BlueDataUtility.build_dp_inc(_rows, myLastValue as DataPacket);
-         myFireEvent(dp_out);
-
-      }
-
-   }
-}

src/DataStreamSuite/DataStream/BlueOrb/SingleRowMathStream.cs

@@ -1,67 +0,0 @@
-using Bts.ArithmeticExpressions;
-
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using BCG;
-
-namespace bts.BlueOrb {
-
-   [DataStreamUserRespresentation("SingleRowMathStream")]
-   public class MathStream : ViewStream, IStreamMapper {
-      private IArithmeticProgram myExp;
-      private BooleanExpressionFilter _bool;
-      List<string> myExtraColumns = new List<string>();
-      BlueTableSchema mySchema;
-
-
-      public MathStream(ViewStream vs, string mathExpression)
-         : this(vs, mathExpression, string.Empty) {
-      }
-
-      public MathStream(ViewStream vs, string mathExpression, string boolean_expression) {
-         myExp = ArithmeticProgramParser.Parse(mathExpression) ??
-            throw new Exception($"Failed to parse ArithmeticExpression '{mathExpression}'");
-         myExtraColumns = myExp.OutputIdentifiers.ToList();
-         if (!string.IsNullOrEmpty(boolean_expression))
-            _bool = new BooleanExpressionFilter(boolean_expression);
-         mySetRealSource(new MapStream(vs, this));
-      }
-
-      #region IStreamMapper Members
-
-      public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema) {
-         if (mySchema == null) {
-            Dictionary<string, ColumnInfo> colIdx = new Dictionary<string, ColumnInfo>();
-            foreach (ColumnInfo ci in inputSchema.columnInfo)
-               colIdx[ci.colName] = ci;
-
-            foreach (string ncn in myExtraColumns) {
-               ColumnInfo newC;
-               if (colIdx.ContainsKey(ncn))
-                  newC = new ColumnInfo(colIdx[ncn].colName, colIdx[ncn].colNo, typeof(double));
-               else
-                  newC = new ColumnInfo(ncn, colIdx.Count, typeof(double));
-               colIdx[ncn] = newC;
-            }
-            mySchema = new BlueTableSchema(colIdx.Values.ToArray(), "math(" + inputSchema.tableName + ")");
-         }
-         return mySchema;
-      }
-
-      #endregion
-
-      #region IDynamicStreamMapper Members
-
-      public IBlueRow map(IBlueRow row) {
-         MutableBlueRow result = new MutableBlueRow(mySchema, row);
-         if (_bool == null || _bool.filter(row)) {
-            myExp.Evaluate(result.AsDictionary());
-         }
-         return result;
-      }
-
-      #endregion
-   }
-}

src/DataStreamSuite/DataStream/BlueOrb/SortStream.cs

@@ -1,17 +0,0 @@
-namespace bts.BlueOrb {
-   using BCG;
-
-   using System;
-   using System.Collections.Generic;
-   using System.Linq;
-   using System.Text;
-   using System.Threading.Tasks;
-
-   [DataStreamUserRespresentation("SortStream")]
-   public class SortStream: ViewStream {
-      public SortStream (ViewStream parent_stream, string sort_expr) {
-         ViewStream rs = new BCG.SortStream(parent_stream, sort_expr);
-         mySetRealSource(rs);
-      }
-   }
-}

src/DataStreamSuite/DataStream/BlueOrb/TimeBasedRollingFilterStream.cs

@@ -1,16 +0,0 @@
-namespace bts.BlueOrb {
-   using BCG;
-
-   using System;
-   using System.Collections.Generic;
-   using System.Linq;
-   using System.Text;
-   using System.Threading.Tasks;
-   [DataStreamUserRespresentation("TimeBasedRollingFilterStream")]
-   public class TimeBasedRollingFilterStream : ViewStream {
-      public TimeBasedRollingFilterStream(ViewStream parent_str, string time_col, double n_seconds) {
-         ViewStream rs = new BCG.TimeBasedFilterStream(ViewStreamDatabase.instance, parent_str, time_col, TimeSpan.FromSeconds(n_seconds));
-         mySetRealSource(rs);
-      }
-   }
-}

src/DataStreamSuite/DataStream/BlueOrb/TopStream.cs

@@ -1,11 +0,0 @@
-namespace bts.BlueOrb {
-   using BCG;
-   [DataStreamUserRespresentation("TopStream")]
-   public class TopStream : ViewStream { 
-      public TopStream(ViewStream vs, int n_rows, string sort_col) {
-         ViewStream rs = new BCG.TopStream (vs, n_rows, sort_col);
-         mySetRealSource(rs);
-      }
-   }
-
-}

src/DataStreamSuite/DataStream/BlueOrb/UnionStream.cs

@@ -1,16 +0,0 @@
-namespace bts.BlueOrb {
-   using System;
-   using System.Collections.Generic;
-
-   using mbts;
-   using BCG;
-
-   [DataStreamUserRespresentation("UnionStream")]
-   public class UnionStream : ViewStream {
-      public UnionStream(ViewStream left_stream, ViewStream right_stream) {
-         ViewStream rs = new BCG.MutableAppendStream(left_stream, right_stream);
-         mySetRealSource(rs);
-      }
-   }
-
-}

src/DataStreamSuite/DataStream/DataStream.csproj

@@ -18,7 +18,6 @@
 
   <ItemGroup>
     <PackageReference Include="HtmlAgilityPack" Version="1.11.57" />
-    <PackageReference Include="IronPython" Version="3.4.2" />
     <PackageReference Include="Microsoft.Data.Analysis" Version="0.21.1" />
     <PackageReference Include="MongoDB.Bson" Version="2.23.1" />
     <PackageReference Include="MongoDB.Driver" Version="2.23.1" />

src/DataStreamSuite/DataStream/DataStreamExtensions.cs

@@ -2,27 +2,15 @@ namespace Bts.DataStream;
 
 using BCG;
 
-using bts.api.core;
-
-using Bts.DataStreamCore;
-
-using Microsoft.Extensions.Logging;
-
 using Spark.Core.Disposables;
 
 using System;
-using System.Collections;
-using System.Collections.Generic;
-using System.Linq;
 using System.Reactive.Linq;
 
 public static class DataStreamExtensions {
    public static ViewStream AsViewStream(this IObservable<DataPacket> source) =>
       source as ViewStream ?? new ObservableViewStream(source);
 
-   public static ViewStream AsViewStream<T>(this IObservable<IDataPacket<T>> source, string? name = null, ILogger? logger = null) where T : IBlueRow =>
-      source as ViewStream ?? new ObservableViewStream(source.Select(dp => new DataPacket(dp.newRows, dp.addedRows, dp.deletedRows)), name, logger);
-
    public static IObservable<DataPacket> AsObservable(this ViewStream source) {
       return source as IObservable<DataPacket> ?? Observable.Create<DataPacket>(Subscribe);
 
@@ -35,100 +23,4 @@ public static class DataStreamExtensions {
                $"Expected DataPacket, got {newValue?.GetType()}", nameof(newValue)));
       }
    }
-
-   public static IObservable<IDataPacket<T>> SelectDataPacket<T>(this IObservable<IDataPacket> source, Func<IBlueRow, T> selector) where T : IBlueRow =>
-      source.Select(dp => {
-         dp.get_new_added_deleted(out var newRows, out var addedRows, out var deletedRows);
-         return new DataPacket<T>(CastRows(newRows), CastRows(addedRows), CastRows(deletedRows));
-
-         IBlueRowCollection<T> CastRows(ICollection rows) =>
-            new BlueTableSelectorWrapper<T>(
-               rows as IBlueRowCollection ?? new BlueRowCollection(rows, dp.Schema), selector);
-      });
-
-   public static IObservable<TableUpdate<IBlueRow, int>> AsTables(this IObservable<IDataPacket> source) =>
-      source.AsTables<IBlueRow, int>(r => r, r => r.Id());
-
-   public static IObservable<TableUpdate<T, int>> AsTables<T>(this IObservable<IDataPacket> source, Func<IBlueRow, T> valueSelector) where T : IBlueRow =>
-      source.AsTables<T, int>(valueSelector, r => r.Id());
-
-   public static IObservable<TableUpdate<T, int>> AsTables<T>(
-      this IObservable<IDataPacket> source, Func<IBlueRow, T> valueSelector, Func<T, int> keySelector) =>
-      source.AsTables<T, int>(valueSelector, keySelector);
-
-   public static IObservable<TableUpdate<T, TKey>> AsTables<T, TKey>(
-      this IObservable<IDataPacket> source, Func<IBlueRow, T> valueSelector, Func<T, TKey> keySelector, Func<IBlueRow, TKey>? rowKeySelector = null) where TKey : IEquatable<TKey> {
-      rowKeySelector ??= r => keySelector(valueSelector(r));
-      return source.Scan(TableUpdate<T, TKey>.mk_empty(keySelector), (res, dp) => {
-         var bd = res.to_builder();
-         dp.get_new_added_deleted(out var newRows, out var addedRows, out var deletedRows);
-         foreach (IBlueRow r in addedRows) {
-            bd.add_or_update(valueSelector(r));
-         }
-         foreach (IBlueRow r in deletedRows) {
-            bd.remove(rowKeySelector(r));
-         }
-         return bd.to_table();
-      });
-   }
-
-   public static IObservable<IDataPacket<T>> AsDataPackets<T>(this IObservable<TableUpdate<T, int>> source, IBlueTableSchema schema) where T : IBlueRow =>
-      source.Scan(
-         (Table: new BlueTableImmutable<T>(schema), Dp: (IDataPacket<T>)DataPacket<T>.Empty(schema)),
-         (res, table) => {
-            var bd = res.Table.ToDataPacketBuilder();
-            foreach (var r in table.removed) {
-               bd.Remove(r.Id());
-            }
-            foreach (var r in table.added) {
-               bd.AddOrUpdate(r);
-            }
-            var resDp = bd.ToDataPacketT(out var newRows, out var addedRows, out var deletedRows);
-            return (newRows, resDp);
-         })
-         .Where(t => t.Dp.addedRows.Any() || t.Dp.deletedRows.Any())
-         .Select(t => t.Dp);
-
-   public static ViewStream AsViewStream<T>(this IObservable<TableUpdate<T, int>> source, IBlueTableSchema schema) where T : IBlueRow =>
-      source.AsDataPackets(schema).AsViewStream();
-
-   public static int ResolveIndices(this IBlueTableSchema schema, string name) {
-      var indices = ResolveIndices(schema, new[] { name });
-      return indices[0];
-   }
-
-   public static (int, int) ResolveIndices(this IBlueTableSchema schema, string name1, string name2) {
-      var indices = ResolveIndices(schema, new[] { name1, name2 });
-      return (indices[0], indices[1]);
-   }
-
-   public static (int, int, int) ResolveIndices(this IBlueTableSchema schema, string name1, string name2, string name3) {
-      var indices = ResolveIndices(schema, new[] { name1, name2, name3 });
-      return (indices[0], indices[1], indices[2]);
-   }
-
-   public static (int, int, int, int) ResolveIndices(this IBlueTableSchema schema, string name1, string name2, string name3, string name4) {
-      var indices = ResolveIndices(schema, new[] { name1, name2, name3, name4 });
-      return (indices[0], indices[1], indices[2], indices[3]);
-   }
-
-   public static int[] ResolveIndices(this IBlueTableSchema schema, string[] columnNames) {
-      var columns = schema.columnInfo;
-      var indices = new int[columnNames.Length];
-      int nResolved = 0;
-      for (int i = 0; i < columnNames.Length; i++) {
-         int origNResolved = nResolved;
-         for (int j = 0; j < columns.Length; j++) {
-            if (columns[j].colName == columnNames[i]) {
-               indices[i] = j;
-               nResolved++;
-               break;
-            }
-         }
-         if (nResolved == origNResolved) {
-            throw new ArgumentException($"No column named '{columnNames[i]}'", nameof(schema));
-         }
-      }
-      return indices;
-   }
 }

src/DataStreamSuite/DataStream/ObservableViewStream.cs

@@ -4,21 +4,14 @@ using BCG;
 
 using bts;
 
-using Microsoft.Extensions.Logging;
-
 using System;
 
 public class ObservableViewStream : ViewStream, IObservable<DataPacket>, IDisposable {
    readonly IObservable<DataPacket> source;
-   readonly string? name;
-   readonly ILogger? logger;
    IDisposable? subscription;
 
-   public ObservableViewStream(IObservable<DataPacket> source, string? name = null, ILogger? logger = null) {
+   public ObservableViewStream(IObservable<DataPacket> source) =>
       this.source = source;
-      this.name = name;
-      this.logger = logger;
-   }
 
    public void Dispose() {
       GC.SuppressFinalize(this);
@@ -30,13 +23,7 @@ public class ObservableViewStream : ViewStream, IObservable<DataPacket>, IDispos
 
    protected override void myOnFirstSubscriber() {
       Log.isnull(subscription);
-      subscription = source.Subscribe(myFireEvent, OnError, OnCompleted);
-
-      void OnError(Exception exception) =>
-         logger?.LogError("Error in ObservableViewStream. name: {Name}, error: {Exception}", name, exception);
-
-      void OnCompleted() =>
-         logger?.LogInformation("ObservableViewStream completed. name: {Name}", name);
+      subscription = source.Subscribe(myFireEvent);
    }
 
    protected override void myOnLastSubscriber() {

src/DataStreamSuite/DataStream/Primitives/AddColumns/AddColumnsStream.cs

@@ -1,51 +1,33 @@
 using System;
-using System.Collections.Generic;
 
-namespace BCG {   
+namespace BCG {
    public class AddColumnsStream : ViewStream {
-      public static Tuple<string, Type, object> parse_single_tok(string exp) {
+      public static Tuple<string, Type, object> parse_exp(string exp) {
          string[] toks = exp.Split(':');
-         string col_name = toks[0].Trim();
-         string col_value = toks[1].Trim();
+         string col_name = toks[0];
+         string col_value = toks[1];
          Tuple<Type, object> tup = BlueDataUtility.guess_type(col_value);
          return Tuple.Create(col_name, tup.Item1, tup.Item2);
       }
 
-      public static List<Tuple<string, Type, object>> parse_exp(string exp) {
-         string[] toks = exp.Split(",");
-         List<Tuple<string, Type, object>> rv = new List<Tuple<string, Type, object>>();
-         foreach (string s in toks) {
-            var tup = parse_single_tok(s.Trim());
-            rv.Add(tup);
-         }
-         return rv;
-      }
-
       private Tuple<string, Type, object> _new_col;
-      private List<Tuple<string, Type, object>> _new_col_map;
       private IBlueTableSchema _out_schema;
 
       public AddColumnsStream(ViewStream vs, string col_def) {
-         _new_col_map = parse_exp(col_def);
+         _new_col = parse_exp(col_def);
          ViewStream rs = new MapStream(vs, (IBlueTableSchema s) => _get_schema(s), (IBlueRow r) => _map(r));
          mySetRealSource(rs);
       }
 
       private IBlueTableSchema _get_schema(IBlueTableSchema input_schema) {
-         if (_out_schema == null) {
-            _out_schema = input_schema;
-            foreach (var m in _new_col_map) {
-               _out_schema = BlueDataUtility.add_column(_out_schema, m.Item1, m.Item2);
-            }
-         }
+         if (_out_schema == null)
+            _out_schema = BlueDataUtility.add_column(input_schema, _new_col.Item1, _new_col.Item2);
          return _out_schema;
       }
 
       private IBlueRow _map(IBlueRow r) {
          MutableBlueRow rv = new MutableBlueRow(_get_schema(r.schema), r);
-         foreach (var m in _new_col_map) {
-            rv.setValue(m.Item1, m.Item3);
-         }
+         rv.setValue(_new_col.Item1, _new_col.Item3);
          return rv;
       }
    }

src/DataStreamSuite/DataStream/Primitives/Count/CountStream.cs

@@ -1,47 +1,52 @@
 using System;
 
-namespace bts.BlueOrb {
-   using BCG;
-   [DataStreamUserRespresentation("CountStream")]
-   public class CountStream : ViewStream {
+namespace BCG
+{
+   [DataStreamUserRespresentation ("count")]
+	public class CountStream : ViewStream
+	{
       private DataPacket myLastFiredDataPacket;
-      private IBlueTableSchema mySchema = new BlueTableSchema("count",
-         new ColumnInfo("id", 0, typeof(int)),
-         new ColumnInfo("count", 1, typeof(int)));
+      private IBlueTableSchema mySchema = new BlueTableSchema ("count", 
+         new ColumnInfo ("id", 0, typeof(int)),
+         new ColumnInfo ("count", 1, typeof(int)));
       private int myId = 0;
       private ViewStream myVS;
 
-      public CountStream(ViewStream vs) {
+      public CountStream (ViewStream vs)
+      {
          myVS = vs;
       }
 
-      public CountStream(string tableName)
-         : this(new TableStream(tableName)) { }
+      public CountStream (string tableName)
+         : this (new TableStream (tableName))
+      {}
 
-      protected override void myOnFirstSubscriber() {
+      protected override void myOnFirstSubscriber () {
          myVS.onChange += new ChangeDelegate(myHandleChanges);
       }
-      protected override void myOnLastSubscriber() {
+      protected override void myOnLastSubscriber () {
          myVS.onChange -= new ChangeDelegate(myHandleChanges);
       }
 
 
-      private void myHandleChanges(object newValue) {
-         myFireEvent(
+      private void myHandleChanges(object newValue)
+      {
+         myFireEvent (
             myLastFiredDataPacket =
                myLastFiredDataPacket == null ?
-                  new DataPacket(myGetRows(newValue, ++myId), myGetRows(newValue, myId),
-                  new BlueTable(mySchema))
+                  new DataPacket (myGetRows (newValue, ++myId), myGetRows (newValue, myId), 
+                  new BlueTable (mySchema))
                   :
-                  new DataPacket(myGetRows(newValue, ++myId), myGetRows(newValue, myId),
+                  new DataPacket (myGetRows (newValue, ++myId), myGetRows (newValue, myId), 
                   myLastFiredDataPacket.newRows)
             );
       }
 
-      private IBlueRowCollection myGetRows(object dp, int id) {
-         BlueTable retVal = new BlueTable(mySchema);
+      private IBlueRowCollection myGetRows (object dp, int id)
+      {
+         BlueTable retVal = new BlueTable (mySchema);
          retVal.addRow
-            (new BlueRow(mySchema, id, (dp as DataPacket).newRows.Count));
+            (new BlueRow (mySchema, id, (dp as DataPacket).newRows.Count));
          return retVal;
       }
 

src/DataStreamSuite/DataStream/Primitives/Filter/FilterStream.cs

@@ -5,20 +5,25 @@ using mbts.config;
 using System;
 using System.Text.RegularExpressions;
 
-namespace BCG.StreamWorld.Addition {
-   [DataStreamUserRespresentation(false)]
-   public class FilterStream : ViewStream {
+namespace BCG.StreamWorld.Addition
+{
+   [DataStreamUserRespresentation (false)]
+   public class FilterStream : ViewStream
+   {
       private ViewStream myBaseStream;
       private BlueTable myBaseTable, myStateTable;
       private IStreamFilter myFilter;
 
       public FilterStream(ViewStream vs)
-         : this(vs, "1=1") { }
+         : this(vs, "1=1")
+      { }
 
       public FilterStream(ViewStream vs, string filterExp)
-         : this(vs, new BooleanExpressionFilter(filterExp)) { }
+         : this(vs, new BooleanExpressionFilter(filterExp))
+      { }
 
-      public FilterStream(ViewStream vs, IStreamFilter filter) {
+      public FilterStream(ViewStream vs, IStreamFilter filter)
+      {
          myBaseStream = vs;
          myFilter = filter;
          if (filter is IDynamicStreamFilter)
@@ -26,35 +31,43 @@ namespace BCG.StreamWorld.Addition {
       }
 
 
-      protected override void myOnFirstSubscriber() {
+      protected override void myOnFirstSubscriber()
+      {
          myBaseStream.onChange += new ChangeDelegate(myHandleChanges);
       }
 
-      protected override void myOnLastSubscriber() {
+      protected override void myOnLastSubscriber()
+      {
          myBaseStream.onChange -= new ChangeDelegate(myHandleChanges);
          myBaseTable.clear();
          myStateTable.clear();
       }
 
-      private void myHandleChanges(object newValue) {
+      private void myHandleChanges(object newValue)
+      {
          DataPacket dp = newValue as DataPacket;
-         if (myStateTable == null) {
+         if (myStateTable == null)
+         {
             bts.Log.assert(myBaseTable == null);
             myStateTable = new BlueTable(dp.schema);
             myBaseTable = new BlueTable(dp.schema);
          }
          BlueTable dels = new BlueTable(dp.schema),
                      adds = new BlueTable(dp.schema);
-         foreach (IBlueRow r in dp.deletedRows) {
+         foreach (IBlueRow r in dp.deletedRows)
+         {
             myBaseTable.deleteRow(r);
-            if (myStateTable.containsID((int)r["id"])) {
+            if (myStateTable.containsID((int)r["id"]))
+            {
                dels.addRow(r);
                myStateTable.deleteRow(r);
             }
          }
-         foreach (IBlueRow r in dp.addedRows) {
+         foreach (IBlueRow r in dp.addedRows)
+         {
             myBaseTable.addRow(r);
-            if (myFilter.filter(r)) {
+            if (myFilter.filter(r))
+            {
                adds.addRow(r);
                myStateTable.addRow(r);
             }
@@ -63,10 +76,13 @@ namespace BCG.StreamWorld.Addition {
             myFireEvent(new DataPacket(myStateTable, adds, dels));
       }
 
-      public IStreamFilter filter {
-         set {
+      public IStreamFilter filter
+      {
+         set
+         {
             myFilter = value;
-            if (myBaseTable != null) {
+            if (myBaseTable != null)
+            {
                IBlueRowCollection oldState = new BlueTable(myStateTable);
                myStateTable.clear();
                foreach (IBlueRow r in myBaseTable)

src/DataStreamSuite/DataStream/Primitives/Filter/TimeBasedFilter.cs

@@ -52,14 +52,14 @@ namespace BCG {
       }
 
       public TimeBasedFilter(ViewStreamDatabase vsd, string rowTimeColumnName, TimeSpan rowTimeToLive)
-      : this(vsd, rowTimeColumnName, rowTimeToLive, rowTimeToLive) {
+      : this(vsd, rowTimeColumnName, rowTimeToLive, new TimeSpan(0, 1, 0) /* 1 minute */) {
       }
 
-      public TimeBasedFilter(ViewStreamDatabase vsd, string rowTimeColumnName, TimeSpan rowTimeToLive, TimeSpan filter_period) {
+      public TimeBasedFilter(ViewStreamDatabase vsd, string rowTimeColumnName, TimeSpan rowTimeToLive, TimeSpan filterFrequency) {
          myVsd = vsd;
          _time_col = rowTimeColumnName;
          _ttl = rowTimeToLive;
-         _filter_period = filter_period;
+         myFilterFrequency = filterFrequency;
          myNextFilter = DateTime.MinValue;
 
          // add worker
@@ -92,7 +92,7 @@ namespace BCG {
             // fire event
             onFilterChange();
 
-            myNextFilter = now.Add(_filter_period);
+            myNextFilter = now.Add(myFilterFrequency);
          }
       }
       #endregion
@@ -108,7 +108,7 @@ namespace BCG {
       private ViewStreamDatabase myVsd;
       private string _time_col;
       private TimeSpan _ttl;
-      private TimeSpan _filter_period;
+      private TimeSpan myFilterFrequency;
       private DateTime myNextFilter;
       #endregion
    }

src/DataStreamSuite/DataStream/Primitives/GroupBy/GroupByStream.cs

@@ -5,7 +5,7 @@ using System.Collections.Generic;
 using System.Linq;
 
 namespace BCG {
-   internal class GroupByStream : MethodDrivenStream {
+   public class GroupByStream : MethodDrivenStream {
       private class MyFuncGroupByEvaluator : IGroupByEvaluator {
          private string[] _groupby_cols;
          private Func<DataPacket, IBlueRow> _groupby_func;

src/DataStreamSuite/DataStream/Primitives/GroupBy/NonIncremental/NonIncGroupByStream.cs

@@ -8,7 +8,7 @@ using bts;
 using bts.SilverPine;
 
 namespace BCG {
-   //[DataStreamUserRespresentation("AggregateStream")]
+   [DataStreamUserRespresentation("AggregateStream")]
    public class NonIncGroupByStream : ViewStream {
       private class DelNonIncGroupByEvaluator : INonIncGroupByEvaluator {
          Func<string[]> _groupby_cols_func;

src/DataStreamSuite/DataStream/Primitives/Join/FullJoinStream.cs

@@ -1,31 +1,31 @@
-//using System;
-//using System.Collections.Generic;
-//using System.Linq;
-//using System.Text;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
 
-//namespace BCG
-//{
-//   public class FullJoinStream : ViewStream
-//   {
-//      private ViewStream myMainStream;
-//      public FullJoinStream(ViewStream v1, string joinCol1, string cmExp1,
-//         ViewStream v2, string joinCol2, string cmExp2, string empty_col)
-//      {
-//         myMainStream = new LeftJoinStream(v1, joinCol1, cmExp1, v2, joinCol2, cmExp2);
-//         ViewStream filtered = new LeftJoinStream(v2, joinCol2, cmExp2, v1, joinCol1, cmExp1);
-//         filtered = new FilterStream(filtered, String.Format("{0}=null", empty_col));
-//         myMainStream = new AppendStream(myMainStream, filtered);
-//      }
+namespace BCG
+{
+   public class FullJoinStream : ViewStream
+   {
+      private ViewStream myMainStream;
+      public FullJoinStream(ViewStream v1, string joinCol1, string cmExp1,
+         ViewStream v2, string joinCol2, string cmExp2, string empty_col)
+      {
+         myMainStream = new LeftJoinStream(v1, joinCol1, cmExp1, v2, joinCol2, cmExp2);
+         ViewStream filtered = new LeftJoinStream(v2, joinCol2, cmExp2, v1, joinCol1, cmExp1);
+         filtered = new FilterStream(filtered, String.Format("{0}=null", empty_col));
+         myMainStream = new AppendStream(myMainStream, filtered);
+      }
 
-//      protected override void myOnFirstSubscriber()
-//      {
-//         myMainStream.onChange += myHandleMainStreamChange;
-//      }
+      protected override void myOnFirstSubscriber()
+      {
+         myMainStream.onChange += myHandleMainStreamChange;
+      }
 
-//      private void myHandleMainStreamChange(object newData)
-//      {
-//         IBlueRowCollection result = (newData as DataPacket).newRows;
-//         myFireEvent(new DataPacket(result, result, myLastValue == null ? new BlueTable(result.schema) : (myLastValue as DataPacket).newRows));
-//      }
-//   }
-//}
+      private void myHandleMainStreamChange(object newData)
+      {
+         IBlueRowCollection result = (newData as DataPacket).newRows;
+         myFireEvent(new DataPacket(result, result, myLastValue == null ? new BlueTable(result.schema) : (myLastValue as DataPacket).newRows));
+      }
+   }
+}

src/DataStreamSuite/DataStream/Primitives/Misc/TimeSeriesStream.cs

@@ -2,7 +2,7 @@ using System;
 
 using BCG;
 
-namespace bts.BlueOrb {
+namespace bts.SilverPine {
    // Keeps adding changing rows to the stream, hence preserving the history
    // and creating a time series, for example this can be used to make bbo updates
    // into a time series of prices/qty 

src/DataStreamSuite/DataStream/Primitives/SQL/SQLite/SQLiteHelper.cs

@@ -12,7 +12,7 @@ using System.Collections.Generic;
 using System.Data.SQLite;
 
 
-namespace bts.BlueOrb{
+namespace bts.SilverPine {
 
    public static class SQLiteHelper {
       public static string location = "C:\\Users\\m_tah\\development\\data\\";

src/DataStreamSuite/DataStream/Primitives/SQL/SQLite/SqliteStream.cs

@@ -1,12 +1,14 @@
-namespace bts.BlueOrb {
+namespace bts.SilverPine {
    using System;
    using System.Data;
    using System.Data.SQLite;
    using System.Diagnostics;
    using System.Collections;
+
+
    using BCG;
 
-   [DataStreamUserRespresentation("SQLiteStream")]
+   [DataStreamUserRespresentation("SQLite")]
    public class SQLiteStream : ViewStream {
       private readonly string _connection_string, _query;
       private Stopwatch _stopwatch = new Stopwatch();
@@ -68,5 +70,4 @@ namespace bts.BlueOrb {
 
    }
 
-}
-
+}

src/DataStreamSuite/DataStream/Primitives/Time/MillisecondStream.cs

@@ -1,6 +1,5 @@
 using System;
 
-using BCG;
 using BCG.Utilities.General;
 
 using bts;
@@ -88,17 +87,3 @@ namespace BCG {
       }
    }
 }
-
-//namespace bts.BlueOrb {
-//   [DataStreamUserRespresentation("MillisecondStream")]
-//   public class MillisecondStream : ViewStream {
-//      public MillisecondStream(ViewStreamDatabase vsd, int interval_msecs, int size) {
-//         ViewStream rs = new BCG.MillisecondStream(vsd, interval_msecs, size);
-//         mySetRealSource(rs);
-//      }
-
-//      public MillisecondStream(int interval_msecs, int size)
-//         : this(interval_msecs, size) {
-//      }
-//   }
-//}

src/DataStreamSuite/DataStream/Primitives/Timeseries/ColumnDifferenceStream.cs

@@ -1,124 +1,124 @@
-//namespace bts.SilverPine {
-//   using BCG;
+namespace bts.SilverPine {
+   using BCG;
 
-//   using bts;
-//   //using bts.collections;
+   using bts;
+   using bts.collections;
 
-//   using System;
-//   using System.CodeDom;
-//   using System.Collections.Generic;
-//   using System.Linq;
+   using System;
+   using System.CodeDom;
+   using System.Collections.Generic;
+   using System.Linq;
 
-//   // Given 2 filters, and 2 columns, finds the different in the value of the 2 columns from the 2 rows
-//   // so:
-//   // f1(base_stream.rows)[col1] - f1(base_stream.rows)[col2]
-//   [DataStreamUserRespresentation("ColDiff")]
-//   public class ColumnDifferenceStream : ViewStream {
-//      private readonly ViewStream _base_stream;
-//      private readonly FilterStream.OnFilter _f1, _f2;
-//      private readonly string _c1, _c2;
+   // Given 2 filters, and 2 columns, finds the different in the value of the 2 columns from the 2 rows
+   // so:
+   // f1(base_stream.rows)[col1] - f1(base_stream.rows)[col2]
+   [DataStreamUserRespresentation("ColDiff")]
+   public class ColumnDifferenceStream : ViewStream {
+      private readonly ViewStream _base_stream;
+      private readonly FilterStream.OnFilter _f1, _f2;
+      private readonly string _c1, _c2;
 
-//      public ColumnDifferenceStream(ViewStream base_stream, FilterStream.OnFilter f1, FilterStream.OnFilter f2, string col1, string col2) {
-//         _base_stream = base_stream;
-//         _f1 = f1;
-//         _f2 = f2;
-//         _c1 = col1;
-//         _c2 = col2;
-//      }
+      public ColumnDifferenceStream(ViewStream base_stream, FilterStream.OnFilter f1, FilterStream.OnFilter f2, string col1, string col2) {
+         _base_stream = base_stream;
+         _f1 = f1;
+         _f2 = f2;
+         _c1 = col1;
+         _c2 = col2;
+      }
 
-//      public ColumnDifferenceStream(ViewStream base_stream, string f1, string f2, string col1, string col2) {
-//         FilterStream.OnFilter f1_del = r => {
-//            BooleanExpressionFilter rv = new BooleanExpressionFilter(f1);
-//            return rv.filter(r);
-//         };
-//         FilterStream.OnFilter f2_del = r => {
-//            BooleanExpressionFilter rv = new BooleanExpressionFilter(f2);
-//            return rv.filter(r);
-//         };
-//         ViewStream rs = new ColumnDifferenceStream(base_stream, f1_del, f2_del, col1, col2);
-//         mySetRealSource(rs);
-//      }
+      public ColumnDifferenceStream(ViewStream base_stream, string f1, string f2, string col1, string col2) {
+         FilterStream.OnFilter f1_del = r => {
+            BooleanExpressionFilter rv = new BooleanExpressionFilter(f1);
+            return rv.filter(r);
+         };
+         FilterStream.OnFilter f2_del = r => {
+            BooleanExpressionFilter rv = new BooleanExpressionFilter(f2);
+            return rv.filter(r);
+         };
+         ViewStream rs = new ColumnDifferenceStream(base_stream, f1_del, f2_del, col1, col2);
+         mySetRealSource(rs);
+      }
 
-//      // for each row, does a row[new_col_name] = row[col1] - rows[row_ffset][col2]
-//      // we should generlize this for any math operation
-//      public ColumnDifferenceStream(ViewStream parent_stream, string sort_col_name, int row_offset, string col1, string col2, string diff_col_name, double optional_perc_diff_filter = 0.0) {
-//         // use GlobalMapStream
-//         string diff_col_perc_name = $"{diff_col_name}_percentage";
-//         System.Func<System.Collections.Generic.List<BCG.IBlueRow>, uint, BCG.IBlueRow> g_map_func = (rows, idx) => {
-//            int minuend_idx = Convert.ToInt32(idx);
-//            var inp_schema = rows[0].schema;
-//            var out_schema = inp_schema.add_column(TypeCode.Double, diff_col_name);
-//            out_schema = out_schema.add_column(TypeCode.Double, diff_col_perc_name);
-//            // minuend - subtrahend
-//            IBlueRow minuend_row = rows.ElementAt(minuend_idx);
-//            MutableBlueRow rv = new MutableBlueRow(out_schema, minuend_row);
-//            int subtrahend_index = minuend_idx + row_offset;
-//            double diff_val = Double.NaN, diff_val_perc = Double.NaN;
-//            if (subtrahend_index >= 0) {
-//               try {
-//                  IBlueRow subtrahend_row = rows.ElementAt(subtrahend_index);
-//                  diff_val = minuend_row.get<double>(col1) - subtrahend_row.get<double>(col2);
-//                  diff_val_perc = 100.0 * diff_val / subtrahend_row.get<double>(col1);
-//               } catch (Exception ex) {
-//                  Log.assert(false, ex.Message);
-//               }
-//            }
-//            rv.setValue(diff_col_name, diff_val);
-//            rv.setValue(diff_col_perc_name, diff_val_perc);
-//            return rv;
-//         };
-//         ViewStream rs = new GlobalMapStream(parent_stream, sort_col_name, g_map_func);
-//         if (optional_perc_diff_filter != 0) {
-//            rs = new FilterStream(rs, r => {
-//               double diff_val_perc = r.get<double>(diff_col_perc_name);
-//               double abs_diff_val_perc = Math.Abs(diff_val_perc);
-//               return abs_diff_val_perc >= optional_perc_diff_filter;
-//            });
-//         }
-//         mySetRealSource(rs);
-//      }
+      // for each row, does a row[new_col_name] = row[col1] - rows[row_ffset][col2]
+      // we should generlize this for any math operation
+      public ColumnDifferenceStream(ViewStream parent_stream, string sort_col_name, int row_offset, string col1, string col2, string diff_col_name, double optional_perc_diff_filter = 0.0) {
+         // use GlobalMapStream
+         string diff_col_perc_name = $"{diff_col_name}_percentage";
+         System.Func<System.Collections.Generic.List<BCG.IBlueRow>, uint, BCG.IBlueRow> g_map_func = (rows, idx) => {
+            int minuend_idx = Convert.ToInt32(idx);
+            var inp_schema = rows[0].schema;
+            var out_schema = inp_schema.add_column(TypeCode.Double, diff_col_name);
+            out_schema = out_schema.add_column(TypeCode.Double, diff_col_perc_name);
+            // minuend - subtrahend
+            IBlueRow minuend_row = rows.ElementAt(minuend_idx);
+            MutableBlueRow rv = new MutableBlueRow(out_schema, minuend_row);
+            int subtrahend_index = minuend_idx + row_offset;
+            double diff_val = Double.NaN, diff_val_perc = Double.NaN;
+            if (subtrahend_index >= 0) {
+               try {
+                  IBlueRow subtrahend_row = rows.ElementAt(subtrahend_index);
+                  diff_val = minuend_row.get<double>(col1) - subtrahend_row.get<double>(col2);
+                  diff_val_perc = 100.0 * diff_val / subtrahend_row.get<double>(col1);
+               } catch (Exception ex) {
+                  Log.assert(false, ex.Message);
+               }
+            }
+            rv.setValue(diff_col_name, diff_val);
+            rv.setValue(diff_col_perc_name, diff_val_perc);
+            return rv;
+         };
+         ViewStream rs = new GlobalMapStream(parent_stream, sort_col_name, g_map_func);
+         if (optional_perc_diff_filter != 0) {
+            rs = new FilterStream(rs, r => {
+               double diff_val_perc = r.get<double>(diff_col_perc_name);
+               double abs_diff_val_perc = Math.Abs(diff_val_perc);
+               return abs_diff_val_perc >= optional_perc_diff_filter;
+            });
+         }
+         mySetRealSource(rs);
+      }
 
-//      protected override void myOnFirstSubscriber() {
-//         _base_stream.onChange += _handle_base_stream_update;
-//      }
+      protected override void myOnFirstSubscriber() {
+         _base_stream.onChange += _handle_base_stream_update;
+      }
 
-//      protected override void myOnLastSubscriber() {
-//         _base_stream.onChange -= _handle_base_stream_update;
-//      }
+      protected override void myOnLastSubscriber() {
+         _base_stream.onChange -= _handle_base_stream_update;
+      }
 
-//      private void _handle_base_stream_update(object dp_obj) {
-//         // could have been done with filter and cartesian prod, and then math
-//         DataPacket dp = dp_obj as DataPacket;
-//         List<IBlueRow> rows = dp.newRows.to_list();
-//         var f1_rows = rows.Where(r => _f1(r));
-//         //Log.assert(f1_rows.Count() == 1);
-//         var f2_rows = rows.Where(r => _f2(r));
-//         //Log.assert(f2_rows.Count() == 1);
-//         if (f1_rows != null && f1_rows.Count() > 0 && f2_rows != null && f2_rows.Count() > 0) {
-//            IBlueRow f1_r = f1_rows.ElementAt(0);
-//            IBlueRow f2_r = f2_rows.ElementAt(0);
-//            double diff = f1_r.get<double>(_c1) - f2_r.get<double>(_c2);
-//            IBlueTableSchema out_schema = f1_r.schema.add_columns(Tuple.Create(typeof(double), "diff"));
-//            BlueTable out_table = new BlueTable(out_schema);
-//            MutableBlueRow out_row = new MutableBlueRow(out_schema, f1_r);
-//            out_row.setValue("diff", diff);
-//            out_table.addRow(out_row);
-//            DataPacket dp_out = BlueDataUtility.build_dp(out_table, myLastValue as DataPacket);
-//            myFireEvent(dp_out);
-//         } else {
-//            IBlueRow f1_r = rows.ElementAt(0);
-//            IBlueRow f2_r = rows.ElementAt(0);
-//            double diff = Double.NaN;
-//            IBlueTableSchema out_schema = f1_r.schema.add_columns(Tuple.Create(typeof(double), "diff"));
-//            BlueTable out_table = new BlueTable(out_schema);
-//            MutableBlueRow out_row = new MutableBlueRow(out_schema, f1_r);
-//            out_row.setValue("diff", diff);
-//            out_table.addRow(out_row);
-//            DataPacket dp_out = BlueDataUtility.build_dp(out_table, myLastValue as DataPacket);
-//            myFireEvent(dp_out);
-//         }
+      private void _handle_base_stream_update(object dp_obj) {
+         // could have been done with filter and cartesian prod, and then math
+         DataPacket dp = dp_obj as DataPacket;
+         List<IBlueRow> rows = dp.newRows.to_list();
+         var f1_rows = rows.Where(r => _f1(r));
+         //Log.assert(f1_rows.Count() == 1);
+         var f2_rows = rows.Where(r => _f2(r));
+         //Log.assert(f2_rows.Count() == 1);
+         if (f1_rows != null && f1_rows.Count() > 0 && f2_rows != null && f2_rows.Count() > 0) {
+            IBlueRow f1_r = f1_rows.ElementAt(0);
+            IBlueRow f2_r = f2_rows.ElementAt(0);
+            double diff = f1_r.get<double>(_c1) - f2_r.get<double>(_c2);
+            IBlueTableSchema out_schema = f1_r.schema.add_columns(Tuple.Create(typeof(double), "diff"));
+            BlueTable out_table = new BlueTable(out_schema);
+            MutableBlueRow out_row = new MutableBlueRow(out_schema, f1_r);
+            out_row.setValue("diff", diff);
+            out_table.addRow(out_row);
+            DataPacket dp_out = BlueDataUtility.build_dp(out_table, myLastValue as DataPacket);
+            myFireEvent(dp_out);
+         } else {
+            IBlueRow f1_r = rows.ElementAt(0);
+            IBlueRow f2_r = rows.ElementAt(0);
+            double diff = Double.NaN;
+            IBlueTableSchema out_schema = f1_r.schema.add_columns(Tuple.Create(typeof(double), "diff"));
+            BlueTable out_table = new BlueTable(out_schema);
+            MutableBlueRow out_row = new MutableBlueRow(out_schema, f1_r);
+            out_row.setValue("diff", diff);
+            out_table.addRow(out_row);
+            DataPacket dp_out = BlueDataUtility.build_dp(out_table, myLastValue as DataPacket);
+            myFireEvent(dp_out);
+         }
 
-//      }
-//   }
+      }
+   }
 
-//}
+}

src/DataStreamSuite/DataStream/Primitives/Top/TopStream.cs

@@ -40,4 +40,4 @@ namespace BCG {
             myFireSwitch = myLastFiredPacket.addedRows.Count + myLastFiredPacket.deletedRows.Count == 0 ? false : true;
         }
     }
-}
+}

src/DataStreamSuite/DataStream/Primitives/Union/AppendStream.cs

@@ -1,118 +1,118 @@
-//using System;
-//using System.Data;
-//using System.Collections;
-//using BCG.Utilities;
-//using BCG.Utilities.General;
-//namespace BCG
-//{
-//   /// <summary>
-//   /// AppendStream appends 2 tables with each other, with a generated id having no bearing to the original id.
-//   /// The column ordering the 2 tables need not be the same
-//   /// </summary>
-//   public class AppendStream : UnionStream
-//   {
-//      private bool myUseNewAppend = DataStreamLoadFileConfig.instance.useNewAppend;
-//      public AppendStream(ViewStream v1, ViewStream v2)
-//      {
-//         if (myUseNewAppend)
-//            mySetRealSource(new MutableAppendStream(v1, v2));
-//         else
-//         {
-//            myLeftVs = v1;
-//            myRightVs = v2;
-//         }
-//      }
+using System;
+using System.Data;
+using System.Collections;
+using BCG.Utilities;
+using BCG.Utilities.General;
+namespace BCG
+{
+   /// <summary>
+   /// AppendStream appends 2 tables with each other, with a generated id having no bearing to the original id.
+   /// The column ordering the 2 tables need not be the same
+   /// </summary>
+   public class AppendStream : UnionStream
+   {
+      private bool myUseNewAppend = DataStreamLoadFileConfig.instance.useNewAppend;
+      public AppendStream(ViewStream v1, ViewStream v2)
+      {
+         if (myUseNewAppend)
+            mySetRealSource(new MutableAppendStream(v1, v2));
+         else
+         {
+            myLeftVs = v1;
+            myRightVs = v2;
+         }
+      }
 
-//      public AppendStream (params ViewStream[] vss)
-//      {
+      public AppendStream (params ViewStream[] vss)
+      {
 
-//         if (myUseNewAppend)
-//            mySetRealSource(new MutableAppendStream(vss));
-//         else
-//         {
-//            if (vss.Length == 1)
-//               mySetRealSource(vss[0]);
-//            else if (vss.Length > 1) 
-//            {
-//               AppendStream rv = null;
-//               foreach (ViewStream s in vss)
-//                  rv = rv == null ? new AppendStream(new ViewStream[] { s }) : new AppendStream(rv, s);
-//               mySetRealSource(rv);
-//            }
-//            else
-//               //don't do anything if they didnt' supply any viewstreams
-//               mySetRealSource(null);
-//         }
+         if (myUseNewAppend)
+            mySetRealSource(new MutableAppendStream(vss));
+         else
+         {
+            if (vss.Length == 1)
+               mySetRealSource(vss[0]);
+            else if (vss.Length > 1) 
+            {
+               AppendStream rv = null;
+               foreach (ViewStream s in vss)
+                  rv = rv == null ? new AppendStream(new ViewStream[] { s }) : new AppendStream(rv, s);
+               mySetRealSource(rv);
+            }
+            else
+               //don't do anything if they didnt' supply any viewstreams
+               mySetRealSource(null);
+         }
           
-//      }
+      }
 
-//      public override void add (IBlueRowCollection rows, bool isLeft) 
-//      {
-//         IDictionary index = isLeft ? myLeftIndex : myRightIndex;
-//         int[] indexToIndexMap = isLeft ? myLeftIndexToIndexMap : myRightIndexToIndexMap;
-//         foreach (IBlueRow row in rows)
-//         {
-//            UnionRow newRow = new UnionRow (myLastFiredPacket.newRows.schema, row, 
-//               myIdCounter ++, myIdIndex, indexToIndexMap);
-//            ((BlueTable)myLastFiredPacket.newRows).addRow (newRow);
-//            index[row["id"]] =  newRow;
-//            ((BlueTable)(myLastFiredPacket.addedRows)).addRow (newRow);
-//         }
-//      }
+      public override void add (IBlueRowCollection rows, bool isLeft) 
+      {
+         IDictionary index = isLeft ? myLeftIndex : myRightIndex;
+         int[] indexToIndexMap = isLeft ? myLeftIndexToIndexMap : myRightIndexToIndexMap;
+         foreach (IBlueRow row in rows)
+         {
+            UnionRow newRow = new UnionRow (myLastFiredPacket.newRows.schema, row, 
+               myIdCounter ++, myIdIndex, indexToIndexMap);
+            ((BlueTable)myLastFiredPacket.newRows).addRow (newRow);
+            index[row["id"]] =  newRow;
+            ((BlueTable)(myLastFiredPacket.addedRows)).addRow (newRow);
+         }
+      }
 
-//      public override void delete (IBlueRowCollection rows, bool isLeft) 
-//      {
-//         IDictionary index = isLeft ? myLeftIndex : myRightIndex;
-//         foreach (IBlueRow axedRow in rows)
-//         {
-//            int rowId = (int)axedRow["id"];
-//            IBlueRow row = (IBlueRow)index[rowId];
-//            index.Remove (rowId);
-//            ((BlueTable)myLastFiredPacket.newRows).deleteRow (row);
-//            ((BlueTable)(myLastFiredPacket.deletedRows)).addRow (row);
-//         }
-//      }
+      public override void delete (IBlueRowCollection rows, bool isLeft) 
+      {
+         IDictionary index = isLeft ? myLeftIndex : myRightIndex;
+         foreach (IBlueRow axedRow in rows)
+         {
+            int rowId = (int)axedRow["id"];
+            IBlueRow row = (IBlueRow)index[rowId];
+            index.Remove (rowId);
+            ((BlueTable)myLastFiredPacket.newRows).deleteRow (row);
+            ((BlueTable)(myLastFiredPacket.deletedRows)).addRow (row);
+         }
+      }
 
-//      public override void mergeChanges (IBlueRowCollection dels, IBlueRowCollection adds, bool isLeft)
-//      {
-//         IDictionary temp = new Hashtable ();
-//         IDictionary index = isLeft ? myLeftIndex : myRightIndex;
-//         int[] indexToIndexMap = isLeft ? myLeftIndexToIndexMap : myRightIndexToIndexMap;
-//         IBlueRow corrRow;
-//         foreach (IBlueRow dRow in dels)
-//         {
-//            corrRow = index[dRow["id"]] as IBlueRow;
-//            (myLastFiredPacket.newRows as BlueTable).deleteRow (corrRow);
-//            (myLastFiredPacket.deletedRows as BlueTable).addRow (corrRow);
-//            index.Remove (dRow["id"]);
-//            if (adds.containsID ((int)dRow["id"]))
-//            {
-//               IBlueRow r = adds[(int)dRow["id"]];
-//               corrRow = new UnionRow (myLastFiredPacket.schema, r, (int)corrRow["id"], myIdIndex, indexToIndexMap);
-//               (myLastFiredPacket.newRows as BlueTable).addRow (corrRow);
-//               (myLastFiredPacket.addedRows as BlueTable).addRow (corrRow);
-//               index.Add (r["id"], corrRow);
-//               temp.Add (r["id"], true);
-//            }
-//         }
-//         foreach (IBlueRow aRow in adds)
-//         {
-//            if (!temp.Contains (aRow["id"]))
-//            {
-//               corrRow = new UnionRow (myLastFiredPacket.schema, aRow, myIdCounter++, myIdIndex, indexToIndexMap);
-//               (myLastFiredPacket.newRows as BlueTable).addRow (corrRow);
-//               (myLastFiredPacket.addedRows as BlueTable).addRow (corrRow);
-//               index.Add (aRow["id"], corrRow);
-//            }
-//         }
-//      }
-//   }
+      public override void mergeChanges (IBlueRowCollection dels, IBlueRowCollection adds, bool isLeft)
+      {
+         IDictionary temp = new Hashtable ();
+         IDictionary index = isLeft ? myLeftIndex : myRightIndex;
+         int[] indexToIndexMap = isLeft ? myLeftIndexToIndexMap : myRightIndexToIndexMap;
+         IBlueRow corrRow;
+         foreach (IBlueRow dRow in dels)
+         {
+            corrRow = index[dRow["id"]] as IBlueRow;
+            (myLastFiredPacket.newRows as BlueTable).deleteRow (corrRow);
+            (myLastFiredPacket.deletedRows as BlueTable).addRow (corrRow);
+            index.Remove (dRow["id"]);
+            if (adds.containsID ((int)dRow["id"]))
+            {
+               IBlueRow r = adds[(int)dRow["id"]];
+               corrRow = new UnionRow (myLastFiredPacket.schema, r, (int)corrRow["id"], myIdIndex, indexToIndexMap);
+               (myLastFiredPacket.newRows as BlueTable).addRow (corrRow);
+               (myLastFiredPacket.addedRows as BlueTable).addRow (corrRow);
+               index.Add (r["id"], corrRow);
+               temp.Add (r["id"], true);
+            }
+         }
+         foreach (IBlueRow aRow in adds)
+         {
+            if (!temp.Contains (aRow["id"]))
+            {
+               corrRow = new UnionRow (myLastFiredPacket.schema, aRow, myIdCounter++, myIdIndex, indexToIndexMap);
+               (myLastFiredPacket.newRows as BlueTable).addRow (corrRow);
+               (myLastFiredPacket.addedRows as BlueTable).addRow (corrRow);
+               index.Add (aRow["id"], corrRow);
+            }
+         }
+      }
+   }
 
-//   public class MergeStream : ViewStream
-//   {
-//      public MergeStream (ViewStream v1, ViewStream v2, string[] mergeCols)
-//      {
-//         mySetRealSource (new DistinctRowStream (new AppendStream (v1, v2), mergeCols));
-//      }
-//   }
-//}
+   public class MergeStream : ViewStream
+   {
+      public MergeStream (ViewStream v1, ViewStream v2, string[] mergeCols)
+      {
+         mySetRealSource (new DistinctRowStream (new AppendStream (v1, v2), mergeCols));
+      }
+   }
+}

src/DataStreamSuite/DataStream/Primitives/Union/MutableAppendStream.cs

@@ -2,171 +2,170 @@ using System;
 using System.Collections.Generic;
 
 using mbts;
-//using BCG;
 
 namespace BCG {
-    /// <summary>
-   /// MutableStream can append multiple streams
-   /// as well as give the user the flexibility
-   /// to append (or remove) more streams at
-   /// runtime
-   /// </summary>
-   //[DataStreamUserRespresentation("UnionStream")]
-   public class MutableAppendStream : ViewStream {
-      internal class MutableAppendTable : IBlueRowCollection {
-         private class MyRow : IBlueRow {
-            private IBlueRow myRow;
-            private int myID;
+   internal class MutableAppendTable : IBlueRowCollection {
+      private class MyRow : IBlueRow {
+         private IBlueRow myRow;
+         private int myID;
 
-            public MyRow(IBlueRow r, int id) {
-               myRow = r;
-               myID = id;
-            }
-
-            #region IBlueRow Members
-
-            public IBlueTableSchema schema { get { return myRow.schema; } }
-
-            public object this[int index] {
-               get {
-                  if (myRow.schema[index].colName == "id")
-                     return myID;
-                  else
-                     return myRow[index];
-               }
-            }
-
-            public object this[string colName] { get { return colName == "id" ? (object)myID : myRow[colName]; } }
-
-            #endregion
-
-            #region IDictionary Members
-
-            public void Add(object key, object value) { throw new Exception("Row is read only."); }
-
-            public void Clear() { throw new Exception("Row is read only."); }
-
-            public bool Contains(object key) { return myRow.Contains(key); }
-
-            public System.Collections.IDictionaryEnumerator GetEnumerator() { return new BlueRowDictionaryEnumerator(this); }
-
-            public bool IsFixedSize { get { return true; } }
-
-            public bool IsReadOnly { get { return true; } }
-
-            public System.Collections.ICollection Keys { get { return myRow.Keys; } }
-
-            public void Remove(object key) { throw new Exception("The method or operation is not implemented."); }
-
-            public System.Collections.ICollection Values {
-               get {
-                  object[] vals = new object[myRow.Count];
-                  CopyTo(vals, 0);
-                  return vals;
-               }
-            }
-
-            public object this[object key] {
-               get { return key is string ? this[key as string] : this[(int)key]; }
-               set { throw new Exception("The method or operation is not implemented."); }
-            }
-
-            #endregion
-
-            #region ICollection Members
-
-            public void CopyTo(Array array, int index) {
-               myRow.CopyTo(array, index);
-               array.SetValue(myID, index + myRow.schema["id"].colNo);
-            }
-
-            public int Count { get { return myRow.Count; } }
-
-            public bool IsSynchronized { get { return myRow.IsSynchronized; } }
-
-            public object SyncRoot { get { return myRow.SyncRoot; } }
-
-            #endregion
-
-            #region IEnumerable Members
-
-            System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() { return new BlueRowEnumerator(this); }
-
-            #endregion
+         public MyRow(IBlueRow r, int id) {
+            myRow = r;
+            myID = id;
          }
 
-         private BlueTable myRows;
-         private int myID = 0;
-         /// <summary>
-         /// maps the table no to a dictionary containing the
-         /// local IDs to a row in the final big table
-         /// </summary>
-         private Dictionary<int, Dictionary<int, IBlueRow>> myIDMap = new Dictionary<int, Dictionary<int, IBlueRow>>();
+         #region IBlueRow Members
 
-         public MutableAppendTable(IBlueTableSchema schema) { myRows = new BlueTable(schema); }
+         public IBlueTableSchema schema { get { return myRow.schema; } }
 
-         public IBlueRow addRow(IBlueRow r, int tableNo) {
-            if (!myIDMap.ContainsKey(tableNo))
-               myIDMap.Add(tableNo, new Dictionary<int, IBlueRow>());
-            IBlueRow nr = new MyRow(r, myID++); // BlueDataUtility.getNewReplacedRow(r, myID++, "id");
-            myRows.addRow(nr);
-            myIDMap[tableNo].Add((int)r["id"], nr);
-            return nr;
-         }
-
-         public IBlueRow removeRow(IBlueRow r, int tableNo) {
-            IBlueRow dr = myIDMap[tableNo][(int)r["id"]];
-            myRows.deleteRow(dr);
-            myIDMap[tableNo].Remove((int)r["id"]);
-            return dr;
-         }
-
-         public IBlueRowCollection removeTable(int tableNo) {
-            if (!myIDMap.ContainsKey(tableNo))
-               return null;
-            else {
-               BlueTable rv = new BlueTable(myRows.schema);
-               foreach (IBlueRow r in myIDMap[tableNo].Values)
-                  rv.addRow(r);
-               myRows.deleteRows(rv);
-               myIDMap.Remove(tableNo);
-               return rv;
+         public object this[int index] {
+            get {
+               if (myRow.schema[index].colName == "id")
+                  return myID;
+               else
+                  return myRow[index];
             }
          }
 
-         public void clear() {
-            myRows.clear();
-            myIDMap.Clear();
+         public object this[string colName] { get { return colName == "id" ? (object)myID : myRow[colName]; } }
+
+         #endregion
+
+         #region IDictionary Members
+
+         public void Add(object key, object value) { throw new Exception("Row is read only."); }
+
+         public void Clear() { throw new Exception("Row is read only."); }
+
+         public bool Contains(object key) { return myRow.Contains(key); }
+
+         public System.Collections.IDictionaryEnumerator GetEnumerator() { return new BlueRowDictionaryEnumerator(this); }
+
+         public bool IsFixedSize { get { return true; } }
+
+         public bool IsReadOnly { get { return true; } }
+
+         public System.Collections.ICollection Keys { get { return myRow.Keys; } }
+
+         public void Remove(object key) { throw new Exception("The method or operation is not implemented."); }
+
+         public System.Collections.ICollection Values {
+            get {
+               object[] vals = new object[myRow.Count];
+               CopyTo(vals, 0);
+               return vals;
+            }
          }
 
-         #region IBlueRowCollection Members
-
-         public IBlueRow this[int id] { get { return myRows[id]; } }
-
-         public IBlueTableSchema schema { get { return myRows.schema; } }
-
-         public bool containsID(int id) { return myRows.containsID(id); }
+         public object this[object key] {
+            get { return key is string ? this[key as string] : this[(int)key]; }
+            set { throw new Exception("The method or operation is not implemented."); }
+         }
 
          #endregion
 
          #region ICollection Members
 
-         public void CopyTo(Array array, int index) { myRows.CopyTo(array, index); }
+         public void CopyTo(Array array, int index) {
+            myRow.CopyTo(array, index);
+            array.SetValue(myID, index + myRow.schema["id"].colNo);
+         }
 
-         public int Count { get { return myRows.Count; } }
+         public int Count { get { return myRow.Count; } }
 
-         public bool IsSynchronized { get { return myRows.IsSynchronized; } }
+         public bool IsSynchronized { get { return myRow.IsSynchronized; } }
 
-         public object SyncRoot { get { return myRows.SyncRoot; } }
+         public object SyncRoot { get { return myRow.SyncRoot; } }
 
          #endregion
 
          #region IEnumerable Members
 
-         public System.Collections.IEnumerator GetEnumerator() { return myRows.GetEnumerator(); }
+         System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() { return new BlueRowEnumerator(this); }
 
          #endregion
       }
+
+      private BlueTable myRows;
+      private int myID = 0;
+      /// <summary>
+      /// maps the table no to a dictionary containing the
+      /// local IDs to a row in the final big table
+      /// </summary>
+      private Dictionary<int, Dictionary<int, IBlueRow>> myIDMap = new Dictionary<int, Dictionary<int, IBlueRow>>();
+
+      public MutableAppendTable(IBlueTableSchema schema) { myRows = new BlueTable(schema); }
+
+      public IBlueRow addRow(IBlueRow r, int tableNo) {
+         if (!myIDMap.ContainsKey(tableNo))
+            myIDMap.Add(tableNo, new Dictionary<int, IBlueRow>());
+         IBlueRow nr = new MyRow(r, myID++); // BlueDataUtility.getNewReplacedRow(r, myID++, "id");
+         myRows.addRow(nr);
+         myIDMap[tableNo].Add((int)r["id"], nr);
+         return nr;
+      }
+
+      public IBlueRow removeRow(IBlueRow r, int tableNo) {
+         IBlueRow dr = myIDMap[tableNo][(int)r["id"]];
+         myRows.deleteRow(dr);
+         myIDMap[tableNo].Remove((int)r["id"]);
+         return dr;
+      }
+
+      public IBlueRowCollection removeTable(int tableNo) {
+         if (!myIDMap.ContainsKey(tableNo))
+            return null;
+         else {
+            BlueTable rv = new BlueTable(myRows.schema);
+            foreach (IBlueRow r in myIDMap[tableNo].Values)
+               rv.addRow(r);
+            myRows.deleteRows(rv);
+            myIDMap.Remove(tableNo);
+            return rv;
+         }
+      }
+
+      public void clear() {
+         myRows.clear();
+         myIDMap.Clear();
+      }
+
+      #region IBlueRowCollection Members
+
+      public IBlueRow this[int id] { get { return myRows[id]; } }
+
+      public IBlueTableSchema schema { get { return myRows.schema; } }
+
+      public bool containsID(int id) { return myRows.containsID(id); }
+
+      #endregion
+
+      #region ICollection Members
+
+      public void CopyTo(Array array, int index) { myRows.CopyTo(array, index); }
+
+      public int Count { get { return myRows.Count; } }
+
+      public bool IsSynchronized { get { return myRows.IsSynchronized; } }
+
+      public object SyncRoot { get { return myRows.SyncRoot; } }
+
+      #endregion
+
+      #region IEnumerable Members
+
+      public System.Collections.IEnumerator GetEnumerator() { return myRows.GetEnumerator(); }
+
+      #endregion
+   }
+   /// <summary>
+   /// MutableStream can append multiple streams
+   /// as well as give the user the flexibility
+   /// to append (or remove) more streams at
+   /// runtime
+   /// </summary>
+   [DataStreamUserRespresentation("UnionStream")]
+   public class MutableAppendStream : ViewStream {
       private bool mySubscriptionActive = false;
       private MutableAppendTable myTable;
       private int myMovingIndex;
@@ -174,8 +173,7 @@ namespace BCG {
       private bool _allign_col_orders = true;
 
       public MutableAppendStream(params ViewStream[] streams)
-         : this(new List<ViewStream>(streams)) {
-      }
+         : this(new List<ViewStream>(streams)) { }
 
       public MutableAppendStream(bool allign_col_order, params ViewStream[] streams)
          : this(allign_col_order, new List<ViewStream>(streams)) { }
@@ -258,13 +256,3 @@ namespace BCG {
       }
    }
 }
-
-
-//namespace bts.BlueOrb {
-//   public class UnionStream : ViewStream {
-//      public UnionStream(ViewStream left_stream, ViewStream right_stream) {
-//         ViewStream rs = new BCG.MutableAppendStream(left_stream, right_stream);
-//         mySetRealSource(rs);
-//      }
-//   }
-//}

src/DataStreamSuite/DataStream/Primitives/Union/UnionOperand.cs

@@ -1,63 +1,63 @@
-//using System;
+using System;
 
-//namespace BCG
-//{
-//   /// <summary>
-//   /// Respresents a Union Operand. Hooks up to the Stream delegate, and alerts the UnionStream
-//   /// on receiving updates
-//   /// </summary>
-//   public class UnionOperand 
-//   {
-//      private bool myIsLeft;
-//      private DataPacket myLastArrivedPacket;
-//      private UnionStream myMaster;
+namespace BCG
+{
+   /// <summary>
+   /// Respresents a Union Operand. Hooks up to the Stream delegate, and alerts the UnionStream
+   /// on receiving updates
+   /// </summary>
+   public class UnionOperand 
+   {
+      private bool myIsLeft;
+      private DataPacket myLastArrivedPacket;
+      private UnionStream myMaster;
        
-//      public UnionOperand (bool isLeft, UnionStream master)
-//      {
-//         myIsLeft = isLeft;
-//         myMaster = master;
-//      }
+      public UnionOperand (bool isLeft, UnionStream master)
+      {
+         myIsLeft = isLeft;
+         myMaster = master;
+      }
 
-//      internal void myChangeHandler (object o)
-//      {
-//         if (myLastArrivedPacket != null)
-//            mySubsequentChangeHandler ((DataPacket)o);
-//         else
-//         {
-//            myLastArrivedPacket = (DataPacket)o;
-//            myFirstChangeHandler ((DataPacket)o);
-//         }
-//      }
+      internal void myChangeHandler (object o)
+      {
+         if (myLastArrivedPacket != null)
+            mySubsequentChangeHandler ((DataPacket)o);
+         else
+         {
+            myLastArrivedPacket = (DataPacket)o;
+            myFirstChangeHandler ((DataPacket)o);
+         }
+      }
 
-//      private void myFirstChangeHandler(DataPacket dp)
-//      {
-//         myMaster.startSession ();
-//         myMaster.add (dp.newRows, myIsLeft);
-//         myMaster.endSession ();
-//      }
+      private void myFirstChangeHandler(DataPacket dp)
+      {
+         myMaster.startSession ();
+         myMaster.add (dp.newRows, myIsLeft);
+         myMaster.endSession ();
+      }
 
-//      private void mySubsequentChangeHandler (DataPacket dp)
-//      {
-//         myMaster.startSession ();  
-//         //myMaster.delete (dp.deletedRows, myIsLeft);
-//         //myMaster.add (dp.addedRows, myIsLeft);
-//         myMaster.mergeChanges (dp.deletedRows, dp.addedRows, myIsLeft);
-//         myMaster.endSession ();
-//      }
+      private void mySubsequentChangeHandler (DataPacket dp)
+      {
+         myMaster.startSession ();  
+         //myMaster.delete (dp.deletedRows, myIsLeft);
+         //myMaster.add (dp.addedRows, myIsLeft);
+         myMaster.mergeChanges (dp.deletedRows, dp.addedRows, myIsLeft);
+         myMaster.endSession ();
+      }
 
-//      public IBlueTableSchema opSchema
-//      {
-//         get
-//         {
-//            if (myLastArrivedPacket == null  || myLastArrivedPacket.newRows == null)
-//               return null;
-//            return myLastArrivedPacket.newRows.schema;
-//         }
-//      }
+      public IBlueTableSchema opSchema
+      {
+         get
+         {
+            if (myLastArrivedPacket == null  || myLastArrivedPacket.newRows == null)
+               return null;
+            return myLastArrivedPacket.newRows.schema;
+         }
+      }
 
-//      public void init ()
-//      {
-//         myLastArrivedPacket = null;
-//      }
-//   }
-//}
+      public void init ()
+      {
+         myLastArrivedPacket = null;
+      }
+   }
+}

src/DataStreamSuite/DataStream/Primitives/Union/UnionStream.cs

@@ -1,119 +1,119 @@
-//using System;
-//using System.Data;
-//using System.Collections;
-//using BCG.Utilities;
-//using BCG.Utilities.General;
-//namespace BCG
-//{
-//   /// <summary>
-//   /// Provides a live stream of Union of 2 streams
-//   /// </summary>
-//   public abstract class UnionStream : ViewStream 
-//   {
-//      protected UnionOperand myLeftOp, myRightOp;
-//      protected ViewStream myLeftVs, myRightVs;
-//      protected DataPacket myLastFiredPacket;
-//      protected IDictionary myLeftIndex = new Hashtable (), myRightIndex = new Hashtable ();
-//      protected int[] myLeftIndexToIndexMap, myRightIndexToIndexMap;
-//      protected int myIdCounter, myIdIndex;
+using System;
+using System.Data;
+using System.Collections;
+using BCG.Utilities;
+using BCG.Utilities.General;
+namespace BCG
+{
+   /// <summary>
+   /// Provides a live stream of Union of 2 streams
+   /// </summary>
+   public abstract class UnionStream : ViewStream 
+   {
+      protected UnionOperand myLeftOp, myRightOp;
+      protected ViewStream myLeftVs, myRightVs;
+      protected DataPacket myLastFiredPacket;
+      protected IDictionary myLeftIndex = new Hashtable (), myRightIndex = new Hashtable ();
+      protected int[] myLeftIndexToIndexMap, myRightIndexToIndexMap;
+      protected int myIdCounter, myIdIndex;
 
-//      protected override void myOnFirstSubscriber() 
-//      {
-//         myLeftOp = new UnionOperand (true, this);
-//         myRightOp = new UnionOperand (false, this);
-//         myLeftVs.onChange += new DataStream.ChangeDelegate (myLeftOp.myChangeHandler);
-//         myRightVs.onChange += new DataStream.ChangeDelegate (myRightOp.myChangeHandler);
-//      }
+      protected override void myOnFirstSubscriber() 
+      {
+         myLeftOp = new UnionOperand (true, this);
+         myRightOp = new UnionOperand (false, this);
+         myLeftVs.onChange += new DataStream.ChangeDelegate (myLeftOp.myChangeHandler);
+         myRightVs.onChange += new DataStream.ChangeDelegate (myRightOp.myChangeHandler);
+      }
 
-//      protected override void myOnLastSubscriber () 
-//      {
-//         myInit ();
-//         myLeftVs.onChange -= new DataStream.ChangeDelegate (myLeftOp.myChangeHandler);
-//         myRightVs.onChange -= new DataStream.ChangeDelegate (myRightOp.myChangeHandler);
-//      }
+      protected override void myOnLastSubscriber () 
+      {
+         myInit ();
+         myLeftVs.onChange -= new DataStream.ChangeDelegate (myLeftOp.myChangeHandler);
+         myRightVs.onChange -= new DataStream.ChangeDelegate (myRightOp.myChangeHandler);
+      }
 
-//      public void startSession ()
-//      {
-//         if (myLastFiredPacket == null)
-//         {
-//            myLastFiredPacket = new DataPacket ();
-//            if (myLeftOp.opSchema != null)//ie call back initiated by left operand
-//            {
-//               myLeftIndexToIndexMap = myInitIndexToIndex (myLeftOp.opSchema, myLeftOp.opSchema);
-//               myLastFiredPacket.newRows = new BlueTable (myLeftOp.opSchema);
-//               myIdIndex = myLeftOp.opSchema["id"].colNo;
-//            }
-//            else
-//            {
-//               myRightIndexToIndexMap = myInitIndexToIndex (myRightOp.opSchema, myRightOp.opSchema);
-//               myLastFiredPacket.newRows = new BlueTable (myRightOp.opSchema);
-//               myIdIndex = myRightOp.opSchema["id"].colNo;
-//            }
-//         }
-//         else if (myRightIndexToIndexMap == null && myRightOp.opSchema != null)
-//            myRightIndexToIndexMap = myInitIndexToIndex (myLeftOp.opSchema, myRightOp.opSchema);
-//         else if (myLeftIndexToIndexMap == null && myLeftOp.opSchema != null)
-//            myLeftIndexToIndexMap = myInitIndexToIndex (myRightOp.opSchema, myLeftOp.opSchema);
-//         myLastFiredPacket.addedRows = new BlueTable (myLastFiredPacket.newRows.schema);
-//         myLastFiredPacket.deletedRows = new BlueTable (myLastFiredPacket.newRows.schema);
-//      }
+      public void startSession ()
+      {
+         if (myLastFiredPacket == null)
+         {
+            myLastFiredPacket = new DataPacket ();
+            if (myLeftOp.opSchema != null)//ie call back initiated by left operand
+            {
+               myLeftIndexToIndexMap = myInitIndexToIndex (myLeftOp.opSchema, myLeftOp.opSchema);
+               myLastFiredPacket.newRows = new BlueTable (myLeftOp.opSchema);
+               myIdIndex = myLeftOp.opSchema["id"].colNo;
+            }
+            else
+            {
+               myRightIndexToIndexMap = myInitIndexToIndex (myRightOp.opSchema, myRightOp.opSchema);
+               myLastFiredPacket.newRows = new BlueTable (myRightOp.opSchema);
+               myIdIndex = myRightOp.opSchema["id"].colNo;
+            }
+         }
+         else if (myRightIndexToIndexMap == null && myRightOp.opSchema != null)
+            myRightIndexToIndexMap = myInitIndexToIndex (myLeftOp.opSchema, myRightOp.opSchema);
+         else if (myLeftIndexToIndexMap == null && myLeftOp.opSchema != null)
+            myLeftIndexToIndexMap = myInitIndexToIndex (myRightOp.opSchema, myLeftOp.opSchema);
+         myLastFiredPacket.addedRows = new BlueTable (myLastFiredPacket.newRows.schema);
+         myLastFiredPacket.deletedRows = new BlueTable (myLastFiredPacket.newRows.schema);
+      }
 
-//      private int[] myInitIndexToIndex (IBlueTableSchema sch1, IBlueTableSchema sch2)
-//      {
-//         //BlueEventLog.logAssert (sch1.columnNames.Length.Equals (sch2.columnNames.Length));
-//         //foreach (string colName in sch1.columnNames)
-//         //   BlueEventLog.logAssert (sch2[colName] != null);
-//         if(!sch1.columnNames.Length.Equals(sch2.columnNames.Length))
-//            throw new BlueDataException("Union stream: different schemas");
-//         foreach(string colName in sch1.columnNames)
-//         {
-//            if(sch2[colName] == null)
-//               throw new BlueDataException("Union stream: can't find " + colName);
-//         }
-//         int[] retVal = new int[sch1.columnNames.Length];
-//         for (int i = 0; i < retVal.Length; i++)
-//            retVal[i] = sch2[sch1[i].colName].colNo;
-//         return retVal;
-//      }
+      private int[] myInitIndexToIndex (IBlueTableSchema sch1, IBlueTableSchema sch2)
+      {
+         //BlueEventLog.logAssert (sch1.columnNames.Length.Equals (sch2.columnNames.Length));
+         //foreach (string colName in sch1.columnNames)
+         //   BlueEventLog.logAssert (sch2[colName] != null);
+         if(!sch1.columnNames.Length.Equals(sch2.columnNames.Length))
+            throw new BlueDataException("Union stream: different schemas");
+         foreach(string colName in sch1.columnNames)
+         {
+            if(sch2[colName] == null)
+               throw new BlueDataException("Union stream: can't find " + colName);
+         }
+         int[] retVal = new int[sch1.columnNames.Length];
+         for (int i = 0; i < retVal.Length; i++)
+            retVal[i] = sch2[sch1[i].colName].colNo;
+         return retVal;
+      }
 
-//      public virtual void endSession ()
-//      {
-//         myFireEvent (myLastFiredPacket.Clone ());
-//      }
+      public virtual void endSession ()
+      {
+         myFireEvent (myLastFiredPacket.Clone ());
+      }
 
-//      public abstract void add (IBlueRowCollection rows, bool isLeft);
+      public abstract void add (IBlueRowCollection rows, bool isLeft);
 
-//      public abstract void delete (IBlueRowCollection rows, bool isLeft);
+      public abstract void delete (IBlueRowCollection rows, bool isLeft);
 
-//      public abstract void mergeChanges (IBlueRowCollection dels, IBlueRowCollection adds, bool isLeft);
+      public abstract void mergeChanges (IBlueRowCollection dels, IBlueRowCollection adds, bool isLeft);
       
-//      private void myInit ()
-//      {
-//         myLastFiredPacket = null;
-//         myLeftOp.init ();
-//         myRightOp.init ();
-//         myLeftIndex.Clear ();
-//         myRightIndex.Clear ();
-//         myIdCounter = myIdIndex = 0;
-//         myLeftIndexToIndexMap = myRightIndexToIndexMap = null;
-//      }
+      private void myInit ()
+      {
+         myLastFiredPacket = null;
+         myLeftOp.init ();
+         myRightOp.init ();
+         myLeftIndex.Clear ();
+         myRightIndex.Clear ();
+         myIdCounter = myIdIndex = 0;
+         myLeftIndexToIndexMap = myRightIndexToIndexMap = null;
+      }
 
-//      /// <summary>
-//      /// Used for test purposes only. Users of this class specifically, should ignore it 
-//      /// (too bad, the NUnit cant see the private methods)
-//      /// </summary>
-//      /// <param name="dp"></param>
-//      /// <param name="isLeft"></param>
-//      internal void receiveStream (DataPacket dp, bool isLeft)
-//      {
-//         if (isLeft)
-//            myLeftOp.myChangeHandler (dp);
-//         else
-//            myRightOp.myChangeHandler (dp);
-//      }
-//      public void receiveStreamTest(DataPacket dp, bool isLeft)
-//      {
-//          receiveStream(dp, isLeft);
-//      }
-//   }
-//}
+      /// <summary>
+      /// Used for test purposes only. Users of this class specifically, should ignore it 
+      /// (too bad, the NUnit cant see the private methods)
+      /// </summary>
+      /// <param name="dp"></param>
+      /// <param name="isLeft"></param>
+      internal void receiveStream (DataPacket dp, bool isLeft)
+      {
+         if (isLeft)
+            myLeftOp.myChangeHandler (dp);
+         else
+            myRightOp.myChangeHandler (dp);
+      }
+      public void receiveStreamTest(DataPacket dp, bool isLeft)
+      {
+          receiveStream(dp, isLeft);
+      }
+   }
+}

src/DataStreamSuite/DataStream/TableUpdateDataPacket.cs

@@ -1,73 +0,0 @@
-namespace Bts.DataStream;
-
-using BCG;
-
-using bts.api.core;
-
-using Bts.DataStreamCore;
-
-using System;
-using System.Collections;
-using System.Collections.Generic;
-
-
-public class TableUpdateDataPacket<T> : IDataPacket<T> where T : IBlueRow {
-   readonly TableUpdate<T, int> table;
-   readonly IBlueTableSchema schema;
-
-   public TableUpdateDataPacket(TableUpdate<T, int> table, IBlueTableSchema schema) {
-      this.table = table;
-      this.schema = schema;
-   }
-
-   public IBlueRowCollection<T> newRows {
-      get {
-         var rv = new BlueTableImmutable<T>(schema);
-         return rv;
-      }
-   }
-
-   public IBlueRowCollection<T> addedRows => throw new NotImplementedException();
-
-   public IBlueRowCollection<T> deletedRows => throw new NotImplementedException();
-
-   class MyRowCollection : IBlueRowCollection<T> {
-      readonly TableUpdate<T, int> table;
-      readonly IEnumerable<T> items;
-      Lazy<SortedList<int, T>> itemList;
-
-      public MyRowCollection(IEnumerable<T> items, IBlueTableSchema schema) {
-         this.items = items;
-         this.schema = schema;
-         itemList = new(() => {
-            var rv = new SortedList<int, T>();
-            foreach (var item in this.items) {
-               rv.Add(item.Id(), item);
-            }
-            return rv;
-         });
-      }
-
-      public int Count => itemList.Value.Count;
-      public bool IsSynchronized => false;
-      public object SyncRoot => this;
-
-      public T this[int id] => itemList.Value[id];
-
-      public IBlueTableSchema schema { get; }
-
-      public bool containsID(int id) =>
-         itemList.Value.ContainsKey(id);
-
-      public void CopyTo(Array array, int index) {
-      }
-
-      public IEnumerator<T> GetEnumerator() {
-         throw new NotImplementedException();
-      }
-
-      IEnumerator IEnumerable.GetEnumerator() {
-         throw new NotImplementedException();
-      }
-   }
-}

src/DataStreamSuite/DataStreamCore/BlueData/BlueDataUtility.cs

@@ -114,15 +114,6 @@ namespace BCG {
             rv.Add(r.get<T>(col));
          return rv;
       }
-      public static IEnumerable<T> get_colvalues<T>(IEnumerable<IBlueRow> rows, int start_idx, int n_items,string col) {
-         List<T> rv = new List<T>();
-         int end_idx = Math.Min(start_idx + n_items, rows.Count());
-         for (int i = start_idx; i < end_idx; i++) {
-            var r = rows.ElementAt(i);
-            rv.Add(r.get<T>(col));
-         }
-         return rv;
-      }
       public static IEnumerable<T> get_colvalues<T>(IEnumerable<IBlueRow> rows, string col, int start_index, int count) {
          List<T> rv = new List<T>();
          int i = 0;
@@ -289,28 +280,6 @@ namespace BCG {
 
       #region json
 
-      public static void write_to_json_file(this IBlueRowCollection table, string file_path) { 
-         write_dict_to_json_file(file_path, table.to_namevalue_pairs());
-      }
-
-      public static void write_dict_to_json_file(string file_path, List<Dictionary<string, string>> data) {
-         var options = new JsonSerializerOptions { WriteIndented = true };
-         string json = System.Text.Json.JsonSerializer.Serialize(data, options);
-         File.WriteAllText(file_path, json);
-      }
-
-      public static List<Dictionary<string, string>> ReadFromFile(string filePath) {
-         string json = File.ReadAllText(filePath);
-         return System.Text.Json.JsonSerializer.Deserialize<List<Dictionary<string, string>>>(json);
-      }
-
-      public static void write_table_to_json(IBlueRowCollection table) {
-         List<Dictionary<string, string>> rows = new List<Dictionary<string, string>>();
-         foreach (IBlueRow row in table) {
-            row.to_json();
-         }
-      }
-
       public static Dictionary<string, string> convert_json_to_dictionary(string json_string) {
          Dictionary<string, string> dictionary = JsonConvert.DeserializeObject<Dictionary<string, string>>(json_string);
          return dictionary;
@@ -379,14 +348,6 @@ namespace BCG {
          }
          return rv;
       }
-
-      public static List<Dictionary<string,string>> to_namevalue_pairs(this IBlueRowCollection table) {
-         List<Dictionary<string, string>> rv = new List<Dictionary<string, string>>();
-         foreach (IBlueRow r in table) {
-            rv.Add(r.to_dict());
-         }
-         return rv;
-      }
       #region Schema types
       // these are the only types allowed in schemas
       private static IReadOnlyDictionary<TypeCode, Type> _typecode_to_type = new Dictionary<TypeCode, Type>
@@ -1264,20 +1225,6 @@ namespace BCG {
          return rv;
       }
 
-      public static Dictionary<string, string> to_dict(this IBlueRow r) {
-         Dictionary<string, string> map = new Dictionary<string, string>();
-         foreach (ColumnInfo c in r.schema.columnInfo) {
-            map.Add(c.colName, r[c.colName].ToString());
-         }
-         return map;
-      }
-
-      public static string to_json(this IBlueRow r) {
-         Dictionary<string, string> map = r.to_dict();
-         string json = System.Text.Json.JsonSerializer.Serialize(map);
-         return json;
-      }
-
       public static void compute_differences_fast(IBlueRowCollection new_rows, IBlueRowCollection old_rows,
                                                   out IBlueRowCollection out_d_rows, out IBlueRowCollection out_a_rows) {
          if (old_rows == null || old_rows.Count == 0) {

src/DataStreamSuite/DataStreamCore/BlueData/BlueRowCollection.cs

@@ -1,56 +0,0 @@
-#nullable enable
-namespace Bts.DataStreamCore;
-
-using BCG;
-
-using System;
-using System.Collections;
-using System.Collections.Generic;
-using System.Linq;
-
-public class BlueRowCollection : IBlueRowCollection {
-   readonly ICollection source;
-   readonly Func<int, IBlueRow> findRow;
-   readonly Func<int, bool> containsKey;
-
-   public BlueRowCollection(ICollection source, IBlueTableSchema schema, Func<int, IBlueRow>? findRow = null, Func<int, bool>? containsKey = null) {
-      this.source = source;
-      this.schema = schema;
-      if (findRow is null || containsKey is null) {
-         if (source is IBlueRowCollection brc) {
-            findRow ??= id => brc[id];
-            containsKey ??= brc.containsID;
-         } else if (source is IDictionary dict) {
-            findRow ??= id => dict[id] as IBlueRow ?? throw new KeyNotFoundException();
-            containsKey ??= id => dict.Contains(id);
-         } else if (source is IDictionary<int, IBlueRow> genDict) {
-            findRow ??= id => genDict[id];
-            containsKey ??= genDict.ContainsKey;
-         } else if (source is IReadOnlyDictionary<int, IBlueRow> roGenDict) {
-            findRow ??= id => roGenDict[id];
-            containsKey = roGenDict.ContainsKey;
-         } else {
-            var cachedRows = new Lazy<Dictionary<int, IBlueRow>>(() =>
-               source.Cast<IBlueRow>().ToDictionary(r => r.Id()));
-            findRow ??= id => cachedRows.Value[id];
-            // Don't remove this lambda expression. Intellisense may suggest it, but that would
-            // instantiate 'cachedRows' immediately and defeat the purpose of using Lazy<T>.
-            containsKey ??= id => cachedRows.Value.ContainsKey(id);
-         }
-      }
-      this.findRow = findRow;
-      this.containsKey = containsKey;
-   }
-
-   public IBlueRow this[int id] => findRow(id);
-
-   public IBlueTableSchema schema { get; }
-
-   public int Count => source.Count;
-
-   public bool containsID(int id) =>
-      containsKey(id);
-
-   public IEnumerator GetEnumerator() =>
-      source.GetEnumerator();
-}

src/DataStreamSuite/DataStreamCore/BlueData/BlueTableImmutable.cs

@@ -96,125 +96,3 @@ public class BlueTableImmutable : IBlueRowCollection, IImmutableRowCollection {
          new(schema, table.ToImmutable());
    }
 }
-
-public class BlueTableImmutable<T> : IBlueRowCollection<T> where T : IBlueRow {
-   readonly ImmutableDictionary<int, T> rows;
-
-   public BlueTableImmutable(IBlueTableSchema schema)
-      : this(schema, ImmutableDictionary<int, T>.Empty) { }
-
-   public BlueTableImmutable(IBlueTableSchema schema, ImmutableDictionary<int, T> rows) {
-      this.schema = schema;
-      this.rows = rows;
-   }
-
-   public int Count => rows.Count;
-   public bool IsSynchronized => false;
-   public IBlueTableSchema schema { get; }
-   public object SyncRoot => this;
-
-   public T this[int id] => rows[id];
-
-   public bool containsID(int id) =>
-      rows.ContainsKey(id);
-
-   public void CopyTo(Array array, int index) {
-      foreach (var r in rows.Values) {
-         array.SetValue(r, index++);
-      }
-   }
-
-   public IEnumerator<T> GetEnumerator() =>
-      rows.Values.GetEnumerator();
-
-   IEnumerator IEnumerable.GetEnumerator() =>
-      GetEnumerator();
-
-   public DataPacketBuilder ToDataPacketBuilder() =>
-      new(this);
-
-   public class DataPacketBuilder {
-      readonly IBlueTableSchema schema;
-      readonly ImmutableDictionary<int, T>.Builder table;
-      readonly ImmutableDictionary<int, T>.Builder added =
-         ImmutableDictionary<int, T>.Empty.ToBuilder();
-      readonly ImmutableDictionary<int, T>.Builder removed =
-         ImmutableDictionary<int, T>.Empty.ToBuilder();
-
-      public DataPacketBuilder(BlueTableImmutable<T> table) {
-         schema = table.schema;
-         this.table = table.rows.ToBuilder();
-      }
-
-      public void AddOrUpdate(T row, IEqualityComparer<IBlueRow>? comparer = null) {
-         var id = row.Id();
-         if (table.TryGetValue(id, out var orig)) {
-            if (comparer?.Equals(row, orig) ?? false) {
-               // No op: This row was already in the table (according to 'comparer').
-               // Remove it from 'added' and 'removed'.
-               added.Remove(id);
-               removed.Remove(id);
-               return;
-            }
-            // Change op: A row with this id was already in the table,
-            // but its values have changed (according to 'comparer').
-            removed[id] = orig;
-         }
-         added[id] = row;
-      }
-
-      public void Remove(int id) {
-         added.Remove(id);
-         if (table.TryGetValue(id, out var orig)) {
-            removed[id] = orig;
-         }
-      }
-
-      public DataPacket ToDataPacket(bool reset = true) {
-         table.RemoveRange(removed.Keys);
-         table.AddRange(added);
-         var newRows = new BlueTableImmutable<T>(schema, table.ToImmutable());
-         var addedRows = new BlueTableImmutable<T>(schema, added.ToImmutable());
-         var removedRows = new BlueTableImmutable<T>(schema, removed.ToImmutable());
-         if (reset) {
-            added.Clear();
-            removed.Clear();
-         }
-         return new(newRows, addedRows, removedRows);
-      }
-
-      public IDataPacket<T> ToDataPacketT(bool reset = true) {
-         table.RemoveRange(removed.Keys);
-         table.AddRange(added);
-         var newRows = new BlueTableImmutable<T>(schema, table.ToImmutable());
-         var addedRows = new BlueTableImmutable<T>(schema, added.ToImmutable());
-         var removedRows = new BlueTableImmutable<T>(schema, removed.ToImmutable());
-         if (reset) {
-            added.Clear();
-            removed.Clear();
-         }
-         return new DataPacket<T>(newRows, addedRows, removedRows);
-      }
-
-      public IDataPacket<T> ToDataPacketT(
-         out BlueTableImmutable<T> newRows,
-         out BlueTableImmutable<T> addedRows,
-         out BlueTableImmutable<T> removedRows,
-         bool reset = true
-      ) {
-         table.RemoveRange(removed.Keys);
-         table.AddRange(added);
-         newRows = new BlueTableImmutable<T>(schema, table.ToImmutable());
-         addedRows = new BlueTableImmutable<T>(schema, added.ToImmutable());
-         removedRows = new BlueTableImmutable<T>(schema, removed.ToImmutable());
-         if (reset) {
-            added.Clear();
-            removed.Clear();
-         }
-         return new DataPacket<T>(newRows, addedRows, removedRows);
-      }
-
-      public BlueTableImmutable<T> ToImmutable() =>
-         new(schema, table.ToImmutable());
-   }
-}

src/DataStreamSuite/DataStreamCore/BlueData/BlueTableSchemaBuilder.cs

@@ -1,29 +0,0 @@
-#nullable enable
-namespace Bts.DataStreamCore;
-
-using BCG;
-
-using System;
-using System.Collections.Generic;
-
-public class BlueTableSchemaBuilder {
-   readonly string? tableName;
-   readonly List<(string Name, Type Type)> colDefs = new();
-
-   public BlueTableSchemaBuilder(string? tableName = null) =>
-      this.tableName = tableName;
-
-   public void Add<T>(string Name) =>
-      colDefs.Add((Name, typeof(T)));
-
-   public void Add(string Name, Type Type) =>
-      colDefs.Add((Name, Type));
-
-   public IBlueTableSchema ToSchema(string? tableName = null) {
-      var colInfos = new ColumnInfo[colDefs.Count];
-      for (int i = 0; i < colInfos.Length; i++) {
-         colInfos[i] = new(colDefs[i].Name, i, colDefs[i].Type);
-      }
-      return new BlueTableSchema(colInfos, tableName ?? this.tableName);
-   }
-}

src/DataStreamSuite/DataStreamCore/BlueData/BlueTableSchemaFactory.cs

@@ -1,16 +0,0 @@
-#nullable enable
-namespace Bts.DataStreamCore;
-
-using BCG;
-
-using System;
-
-public class BlueTableSchemaFactory {
-   public static IBlueTableSchema Create(string tableName, params (string Name, Type Type)[] columns) {
-      var colInfos = new ColumnInfo[columns.Length];
-      for (int i = 0; i < columns.Length; i++) {
-         colInfos[i] = new(columns[i].Name, i, columns[i].Type);
-      }
-      return new BlueTableSchema(colInfos, tableName);
-   }
-}

src/DataStreamSuite/DataStreamCore/BlueData/BlueTableSelectorWrapper.cs

@@ -1,39 +0,0 @@
-#nullable enable
-namespace Bts.DataStreamCore;
-
-using BCG;
-
-using System;
-using System.Collections.Generic;
-
-public static class BlueTableSelectorWrapper {
-   public static BlueTableSelectorWrapper<T> Create<T>(
-      IBlueRowCollection source, Func<IBlueRow, T> selector, IBlueTableSchema? schema = null) where T : IBlueRow =>
-      new(source, selector, schema);
-}
-
-public class BlueTableSelectorWrapper<T> : IBlueRowCollection<T> where T : IBlueRow {
-   readonly IBlueRowCollection source;
-   readonly Func<IBlueRow, T> selector;
-
-   public BlueTableSelectorWrapper(IBlueRowCollection source, Func<IBlueRow, T> selector, IBlueTableSchema? schema = null) {
-      this.source = source;
-      this.selector = selector;
-      this.schema = schema ?? source.schema;
-   }
-
-   public T this[int id] => selector(source[id]);
-
-   public IBlueTableSchema schema { get; }
-
-   public int Count => source.Count;
-
-   public bool containsID(int id) =>
-      source.containsID(id);
-
-   public IEnumerator<T> GetEnumerator() {
-      foreach (IBlueRow r in source) {
-         yield return selector(r);
-      }
-   }
-}

src/DataStreamSuite/DataStreamCore/BlueData/ColumnInfo.cs

@@ -1,5 +1,4 @@
 using System;
-using System.Text.Json.Serialization;
 
 namespace BCG {
    /// <summary>
@@ -57,16 +56,9 @@ namespace BCG {
       private Type myColType;
       private object myColValue;
 
-      [JsonPropertyName("colNo")]
       public int colNo { get { return myColNo; } }
-
-      [JsonPropertyName("colName")]
       public string colName { get { return myColName; } }
-
-      [JsonPropertyName("colType")]
       public Type colType { get { return myColType; } }
-
-      [JsonIgnore]
       public object colValue {get{return myColValue;}}
    }
 }

src/DataStreamSuite/DataStreamCore/BlueData/IBlueRowCollection.cs

@@ -1,6 +1,5 @@
 using System;
 using System.Collections;
-using System.Collections.Generic;
 
 namespace BCG
 {
@@ -16,23 +15,5 @@ namespace BCG
       IBlueRow this[int id]{get;}
       IBlueTableSchema schema {get;}
       bool containsID (int id);
-
-      bool ICollection.IsSynchronized => false;
-      object ICollection.SyncRoot => this;
-
-      void ICollection.CopyTo(Array array, int index) {
-         foreach (IBlueRow r in this) {
-            array.SetValue(r, index++);
-         }
-      }
-   }
-
-   public interface IBlueRowCollection<T> : IBlueRowCollection, IReadOnlyCollection<T> where T : IBlueRow {
-      new T this[int id] { get; }
-
-      IBlueRow IBlueRowCollection.this[int id] => this[id];
-
-      IEnumerator IEnumerable.GetEnumerator() =>
-         GetEnumerator();
    }
 }

src/DataStreamSuite/DataStreamCore/DataPacket.cs

@@ -3,8 +3,6 @@ using System.Collections;
 using System.Data;
 using BCG.Utilities.General;
 
-using Bts.DataStreamCore;
-
 namespace BCG
 {
 	[Serializable]
@@ -185,7 +183,6 @@ namespace BCG
             return retVal;
          }
       }
-      IBlueTableSchema IDataPacket.Schema => schema;
 
       /*          
        status  D total  A total  D this   A This   D' total A' Total 
@@ -317,24 +314,6 @@ namespace BCG
       }
    }
 
-   public class DataPacket<T> : IDataPacket<T> where T : IBlueRow {
-      public DataPacket(IBlueRowCollection<T> newRows, IBlueRowCollection<T> addedRows, IBlueRowCollection<T> deletedRows) {
-         this.newRows = newRows;
-         this.addedRows = addedRows;
-         this.deletedRows = deletedRows;
-      }
-
-      public IBlueRowCollection<T> newRows { get; }
-      public IBlueRowCollection<T> addedRows { get; }
-      public IBlueRowCollection<T> deletedRows { get; }
-
-      public static DataPacket<T> Empty(IBlueTableSchema schema) =>
-         new(
-            new BlueTableImmutable<T>(schema),
-            new BlueTableImmutable<T>(schema),
-            new BlueTableImmutable<T>(schema));
-   }
-
    public static class DataPacketExtensions {
       public static bool HasChanges(this DataPacket dataPacket) =>
          dataPacket.addedRows.Count > 0 || dataPacket.deletedRows.Count > 0;

src/DataStreamSuite/DataStreamCore/DynamicLoading/PrimitiveManager.cs

@@ -47,10 +47,10 @@ namespace BCG {
       /// </summary>
       /// <param name="assemblyName"></param>
       public void loadAssembly(string assemblyName) {
-         Func<Type, bool> is_blue_orb = (t) => {
-            //if (!assemblyName.Contains("SharedStreams.dll"))
-            //   return true;
-            return t.Namespace == "bts.BlueOrb";// || t.Namespace == "BCG";
+         Func<Type, bool> is_silverpine_only = (t) => {
+            if (!assemblyName.Contains("SharedStreams.dll"))
+               return true;
+            return t.Namespace == "bts.SilverPine";
          };
          bts.Log.Gen.info($"PrimitiveManager.loadAssembly assemblyName: {assemblyName}");
          if (myAssemblyNames.Contains(assemblyName))
@@ -61,7 +61,7 @@ namespace BCG {
             Assembly.LoadFrom(assemblyName);
 
          foreach (Type type in loadedAssembly.GetTypes())
-            if (!type.IsInterface && is_blue_orb(type)
+            if (!type.IsInterface && is_silverpine_only(type)
                && !type.IsAbstract
                && type.IsSubclassOf(typeof(ViewStream))
                && type.IsPublic) {
@@ -77,7 +77,7 @@ namespace BCG {
                      break;
                   }
                if (isVisible) {
-                 // Log.assert_d(myTypeLookupTable[userRep] == null, type.FullName);
+                  Log.assert_d(myTypeLookupTable[userRep] == null, type.FullName);
                   myTypeLookupTable[userRep] = type;
                }
             }

src/DataStreamSuite/DataStreamCore/IDataPacket.cs

@@ -9,22 +9,7 @@ namespace BCG
 {
    public interface IDataPacket
    {
-      IBlueTableSchema Schema { get; }
       //this is an ICollection of IDictionary
       void get_new_added_deleted(out ICollection new_rows, out ICollection added_rows, out ICollection deleted_rows);
    }
-
-   public interface IDataPacket<T> : IDataPacket where T : IBlueRow {
-      IBlueRowCollection<T> newRows { get; }
-      IBlueRowCollection<T> addedRows { get; }
-      IBlueRowCollection<T> deletedRows { get; }
-
-      IBlueTableSchema IDataPacket.Schema => newRows.schema;
-
-      void IDataPacket.get_new_added_deleted(out ICollection new_rows, out ICollection added_rows, out ICollection deleted_rows) {
-         new_rows = newRows;
-         added_rows = addedRows;
-         deleted_rows = deletedRows;
-      }
-   }
 }

src/DataStreamSuite/SharedStreams/Business/ContractUtils.cs

@@ -1,8 +1,8 @@
 using bts.SilverPine;
 
-namespace bts.BlueOrb {
-   //using bts.api;
-   //using bts.spark;
+namespace bts.SilverPine {
+   using bts.api;
+   using bts.spark;
 
    using System;
 
@@ -16,7 +16,7 @@ namespace bts.BlueOrb {
          public string contract_text;
       }
       public class FutureSpreadContract {
-         public List<Tuple<double, ContractUtils.FutureTradableContract>> legs;
+         public List<Tuple<double, bts.SilverPine.ContractUtils.FutureTradableContract>> legs;
          public string symbol {
             get {
                return legs == null || legs.Count == 0 ? String.Empty : legs.ElementAt(0).Item2.symbol;
@@ -206,7 +206,7 @@ namespace bts.BlueOrb {
          }
          // tehre are some non calendar spreads we cant parse
          //Log.assert(false);
-         return ContractUtils.ContractType.FutureSpread;
+         return bts.SilverPine.ContractUtils.ContractType.FutureSpread;
       }
 
       public static ContractCategory get_category(string symbol) {

src/DataStreamSuite/SharedStreams/Business/EventsPriceActionStream.cs

@@ -1,23 +1,23 @@
-//namespace bts.BlueOrb {
-//   using BCG;
+namespace bts.SilverPine {
+   using BCG;
 
-//   // Finds the regions around specified events 
-//   [DataStreamUserRespresentation("EventsPriceAction")]
-//   public class EventsPriceActionStream : ViewStream {
-//      public EventsPriceActionStream(string symbol, string event_type, string pre_event_period,
-//            string post_event_period, int n_events) :
-//         this(symbol, event_type, TimeBasedFilter.parse_duration(pre_event_period), TimeBasedFilter.parse_duration(post_event_period),
-//         n_events, ViewStreamDatabase.instance) { }
+   // Finds the regions around specified events 
+   [DataStreamUserRespresentation("EventsPriceAction")]
+   public class EventsPriceActionStream : ViewStream {
+      public EventsPriceActionStream(string symbol, string event_type, string pre_event_period,
+            string post_event_period, int n_events) :
+         this(symbol, event_type, TimeBasedFilter.parse_duration(pre_event_period), TimeBasedFilter.parse_duration(post_event_period),
+         n_events, ViewStreamDatabase.instance) { }
 
-//      public EventsPriceActionStream(string symbol, string event_type, TimeSpan pre_event_period,
-//      TimeSpan post_event_period, int n_events, ViewStreamDatabase vsd) {
-//         // TODO: figure out which bars to use, 1 min or 1 day or live, based on pre_event_period / post_event_period
-//         FuturesActiveContractStream olhcv_str = new FuturesActiveContractStream(symbol);
-//         ViewStream events_str = new EventsStream(vsd, event_type);
-//         // todo
-//         //events_str = new TopStream(events_str, n_events, "-ts");
-//         ViewStream rs = new TimeProximityJoinStream(olhcv_str, "ts_event", events_str, "ts", pre_event_period, post_event_period);
-//         mySetRealSource(rs);
-//      }
-//   }
-//}
+      public EventsPriceActionStream(string symbol, string event_type, TimeSpan pre_event_period,
+      TimeSpan post_event_period, int n_events, ViewStreamDatabase vsd) {
+         // TODO: figure out which bars to use, 1 min or 1 day or live, based on pre_event_period / post_event_period
+         FuturesActiveContractStream olhcv_str = new FuturesActiveContractStream(symbol);
+         ViewStream events_str = new EventsStream(vsd, event_type);
+         // todo
+         //events_str = new TopStream(events_str, n_events, "-ts");
+         ViewStream rs = new TimeProximityJoinStream(olhcv_str, "ts_event", events_str, "ts", pre_event_period, post_event_period);
+         mySetRealSource(rs);
+      }
+   }
+}

src/DataStreamSuite/SharedStreams/Business/Indicators/IndicatorStream.cs

@@ -1,20 +1,20 @@
-//namespace bts.SilverPine {
-//   using BCG;
+namespace bts.SilverPine {
+   using BCG;
 
-//   public class IndicatorStream : ViewStream {
-//      enum Indicator {
-//         StandardDeviation = 0
-//      }
-//      public class IndicatorInfo {
-//         Indicator indicator;
-//         IDictionary<string, object> parameters;
-//      }
-//      public IndicatorStream(string symbol_name, string indicator_name, string indicator_params_json) {
+   public class IndicatorStream : ViewStream {
+      enum Indicator {
+         StandardDeviation = 0
+      }
+      public class IndicatorInfo {
+         Indicator indicator;
+         IDictionary<string, object> parameters;
+      }
+      public IndicatorStream(string symbol_name, string indicator_name, string indicator_params_json) {
 
-//      }
-//      public IndicatorStream(string symbol_name, IndicatorInfo info) {
+      }
+      public IndicatorStream(string symbol_name, IndicatorInfo info) {
 
-//      }
+      }
 
-//   }
-//}
+   }
+}

src/DataStreamSuite/SharedStreams/Business/MarketData/AbsoluteVolumeStream.cs

@@ -1,57 +1,57 @@
-//namespace bts.SilverPine {
-//   using BCG;
+namespace bts.SilverPine {
+   using BCG;
 
-//   using ZstdSharp.Unsafe;
+   using ZstdSharp.Unsafe;
 
-//   [DataStreamUserRespresentation("AbsVolume")]
-//   // Calculates the volume within a time window (this does not do rates, like highest daily volume etc,
-//   // but with some thought can perhaps be extended)
-//   public class AbsoluteVolumeStream : ViewStream {
-//      // TODO, abstract this out
-//      // adds contract_type to OHLCV stream
-//      private static ViewStream _get_ohlcv_steam(string symbol, DateTime d1, DateTime d2) {
-//         var ohlcv_params = OHLCVStream.get_params(d1, d2);
-//         ViewStream rs = new OHLCVStream(symbol, ohlcv_params);
-//         Func<IBlueTableSchema, IBlueTableSchema> create_schema =
-//         s => BlueDataUtility.add_columns(s, Tuple.Create(typeof(string), "contract_type"));
-//         rs = new SuperMapStream(rs,
-//         s => create_schema(s),
-//         r => {
-//            IBlueTableSchema out_schema = create_schema(r.schema);
-//            string contract_txt = r.get<string>("contract");
-//            var contr_type = ContractUtils.get_type(contract_txt);
-//            MutableBlueRow out_row = new MutableBlueRow(out_schema, r);
-//            out_row.setValue("contract_type", contr_type.ToString());
-//            return new IBlueRow[] { out_row };
-//         });
-//         return rs;
-//      }
+   [DataStreamUserRespresentation("AbsVolume")]
+   // Calculates the volume within a time window (this does not do rates, like highest daily volume etc,
+   // but with some thought can perhaps be extended)
+   public class AbsoluteVolumeStream : ViewStream {
+      // TODO, abstract this out
+      // adds contract_type to OHLCV stream
+      private static ViewStream _get_ohlcv_steam(string symbol, DateTime d1, DateTime d2) {
+         var ohlcv_params = OHLCVStream.get_params(d1, d2);
+         ViewStream rs = new OHLCVStream(symbol, ohlcv_params);
+         Func<IBlueTableSchema, IBlueTableSchema> create_schema =
+         s => BlueDataUtility.add_columns(s, Tuple.Create(typeof(string), "contract_type"));
+         rs = new SuperMapStream(rs,
+         s => create_schema(s),
+         r => {
+            IBlueTableSchema out_schema = create_schema(r.schema);
+            string contract_txt = r.get<string>("contract");
+            var contr_type = ContractUtils.get_type(contract_txt);
+            MutableBlueRow out_row = new MutableBlueRow(out_schema, r);
+            out_row.setValue("contract_type", contr_type.ToString());
+            return new IBlueRow[] { out_row };
+         });
+         return rs;
+      }
 
-//      public AbsoluteVolumeStream(string symbol, string ctype, DateTime d1)
-//      : this(symbol, ctype, d1, d1) { }
+      public AbsoluteVolumeStream(string symbol, string ctype, DateTime d1)
+      : this(symbol, ctype, d1, d1) { }
 
-//      public AbsoluteVolumeStream(string symbol, string ctype, DateTime d1, DateTime d2)
-//      : this(symbol, Enum.Parse<ContractUtils.ContractType>(ctype), d1, d2) {
-//      }
+      public AbsoluteVolumeStream(string symbol, string ctype, DateTime d1, DateTime d2)
+      : this(symbol, Enum.Parse<ContractUtils.ContractType>(ctype), d1, d2) {
+      }
 
-//      static void swap_if_necessary(ref DateTime dt1, ref DateTime dt2) {
-//         if (dt1 < dt2) {
-//            DateTime aux = dt1;
-//            dt1 = dt2;
-//            dt2 = aux;
-//         }
-//      }
+      static void swap_if_necessary(ref DateTime dt1, ref DateTime dt2) {
+         if (dt1 < dt2) {
+            DateTime aux = dt1;
+            dt1 = dt2;
+            dt2 = aux;
+         }
+      }
 
-//      private AbsoluteVolumeStream(string symbol, ContractUtils.ContractType ctype, DateTime d1, DateTime d2) {
-//         swap_if_necessary(ref d1, ref d2);
-//         ViewStream ohlcv_str = _get_ohlcv_steam(symbol, d1, d2);
-//         ohlcv_str = new FilterStream(ohlcv_str,
-//            r => r.get_datetime("ts_event") <= d1 && r.get_datetime("ts_event") >= d2);
-//         ohlcv_str = new FilterStream(ohlcv_str, r => String.Compare(ctype.ToString(), r.get<string>("contract_type"), true) == 0);
-//         ViewStream rs = new MaxStream(ohlcv_str, "volume");
-//         rs = new NewProjectStream(rs, "volume:total_volume contract !");
-//         mySetRealSource(rs);
-//      }
+      private AbsoluteVolumeStream(string symbol, ContractUtils.ContractType ctype, DateTime d1, DateTime d2) {
+         swap_if_necessary(ref d1, ref d2);
+         ViewStream ohlcv_str = _get_ohlcv_steam(symbol, d1, d2);
+         ohlcv_str = new FilterStream(ohlcv_str,
+            r => r.get_datetime("ts_event") <= d1 && r.get_datetime("ts_event") >= d2);
+         ohlcv_str = new FilterStream(ohlcv_str, r => String.Compare(ctype.ToString(), r.get<string>("contract_type"), true) == 0);
+         ViewStream rs = new MaxStream(ohlcv_str, "volume");
+         rs = new NewProjectStream(rs, "volume:total_volume contract !");
+         mySetRealSource(rs);
+      }
 
-//   }
-//}
+   }
+}

src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpatialArbScoreStream.cs

@@ -1,5 +1,4 @@
-namespace bts.BlueOrb {
-   using bts.SilverPine;
+namespace bts.SilverPine {
    using BCG;
    using System;
 

src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpatialArbStatsStream.cs

@@ -1,4 +1,4 @@
-namespace bts.BlueOrb {
+namespace bts.SilverPine {
    using BCG;
 
    using SharpCompress.Common.Rar;

src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpatialFuturePairsStream.cs

@@ -1,4 +1,4 @@
-namespace bts.BlueOrb {
+namespace bts.SilverPine {
    using BCG;
 
    using bts.api;

src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpreadPerformanceStream.cs

@@ -1,4 +1,4 @@
-namespace bts.BlueOrb {
+namespace bts.SilverPine {
    using BCG;
 
    using System.Diagnostics;

src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpreadStatArbsWeightLinearRegressionStream.cs

@@ -1,4 +1,4 @@
-namespace bts.BlueOrb{
+namespace bts.SilverPine{
    using BCG;
    public class SpreadStatArbsWeightLinearRegressionStream : ViewStream {
       private IBlueTableSchema _schema;

src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpreadStatsStream.cs

@@ -1,4 +1,4 @@
-namespace bts.BlueOrb {
+namespace bts.SilverPine {
    using BCG;
 
    using MathNet.Numerics.LinearAlgebra.Factorization;

src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpreadWeightCalculator.cs

@@ -2,7 +2,7 @@
 using System.Collections.Generic;
 using MathNet.Numerics.LinearRegression;
 
-namespace bts.BlueOrb {
+namespace bts.SilverPine {
 
    public static class SpreadWeightCalculator {
       /// <summary>

src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/StandardDeviationStream.cs

@@ -1,40 +1,40 @@
-//namespace bts.SilverPine {
-//   using BCG;
+namespace bts.SilverPine {
+   using BCG;
 
-//   using MathNet.Numerics.Statistics;
+   using MathNet.Numerics.Statistics;
 
-//   // calculates the reazlied standard deviation of a series of prices
-//   public class StandardDeviationStream : ViewStream {
-//      public StandardDeviationStream(string symbol, string std_dev_period)
-//      :
-//      this(symbol, TimeBasedFilter.parse_duration(std_dev_period)) { }
+   // calculates the reazlied standard deviation of a series of prices
+   public class StandardDeviationStream : ViewStream {
+      public StandardDeviationStream(string symbol, string std_dev_period)
+      :
+      this(symbol, TimeBasedFilter.parse_duration(std_dev_period)) { }
 
-//      public StandardDeviationStream(string symbol, TimeSpan std_dev_period) {
-//         // since this is a 1_day bar str, 90 rows => 90 days, so no need to filter
-//         int n_days = std_dev_period.Days;
-//         ViewStream ohlcv_str = new FuturesActiveContractStream(symbol);
-//         IBlueTableSchema out_schema = null;
-//         int id = 0;
-//         Func<List<IBlueRow>, uint, IBlueRow> find_std_dev = (rows, idx) => {
-//            int iidx = Convert.ToInt32(idx);
-//            if (rows == null || rows.Count == 0)
-//               return null;
-//            IBlueRow ir = rows.ElementAt(iidx);
-//            out_schema = out_schema ?? ir.schema.add_column(TypeCode.Double, "std_dev");
-//            MutableBlueRow rv = new MutableBlueRow(out_schema, ir);
-//            rv.setValue("std_dev", Double.NaN);
-//            IEnumerable<double> last_n_close_prices = rows.Skip(iidx).Take(n_days).Select(r => r.get<double>("close"));
-//            if (last_n_close_prices.Count() >= n_days) {
-//               double std_dev = MathNet.Numerics.Statistics.Statistics.StandardDeviation(last_n_close_prices);
-//               rv.setValue("std_dev", std_dev);
-//            }
-//            //rv.setValue("id", id++);
-//            return rv;
-//         };
-//         ViewStream rs = new GlobalMapStream(ohlcv_str, "ts_event", find_std_dev);
-//         rs = new NullFilterStream(rs, "std_dev", true);
-//         mySetRealSource(rs);
-//      }
+      public StandardDeviationStream(string symbol, TimeSpan std_dev_period) {
+         // since this is a 1_day bar str, 90 rows => 90 days, so no need to filter
+         int n_days = std_dev_period.Days;
+         ViewStream ohlcv_str = new FuturesActiveContractStream(symbol);
+         IBlueTableSchema out_schema = null;
+         int id = 0;
+         Func<List<IBlueRow>, uint, IBlueRow> find_std_dev = (rows, idx) => {
+            int iidx = Convert.ToInt32(idx);
+            if (rows == null || rows.Count == 0)
+               return null;
+            IBlueRow ir = rows.ElementAt(iidx);
+            out_schema = out_schema ?? ir.schema.add_column(TypeCode.Double, "std_dev");
+            MutableBlueRow rv = new MutableBlueRow(out_schema, ir);
+            rv.setValue("std_dev", Double.NaN);
+            IEnumerable<double> last_n_close_prices = rows.Skip(iidx).Take(n_days).Select(r => r.get<double>("close"));
+            if (last_n_close_prices.Count() >= n_days) {
+               double std_dev = MathNet.Numerics.Statistics.Statistics.StandardDeviation(last_n_close_prices);
+               rv.setValue("std_dev", std_dev);
+            }
+            //rv.setValue("id", id++);
+            return rv;
+         };
+         ViewStream rs = new GlobalMapStream(ohlcv_str, "ts_event", find_std_dev);
+         rs = new NullFilterStream(rs, "std_dev", true);
+         mySetRealSource(rs);
+      }
 
-//   }
-//}
+   }
+}

src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/VWAPStream.cs

@@ -1,118 +1,117 @@
-// TODO: Remove
-//// namespace bts.SilverPine 
-////    using bts;
-////    using BCG;
-////    using MongoDB.Driver.Linq;
-////    using System.ComponentModel.DataAnnotations;
-////    using ZstdSharp.Unsafe;
+// namespace bts.SilverPine {
+//    using bts;
+//    using BCG;
+//    using MongoDB.Driver.Linq;
+//    using System.ComponentModel.DataAnnotations;
+//    using ZstdSharp.Unsafe;
 
-////    [DataStreamUserRespresentation("VWAP")]
-////    public class VWAPStream : ViewStream {
-////       // returns the VWAP stream for each contract 
-////       private static ViewStream build_vwap_stream(string symbol, DateTime begin_dt, DateTime end_dt) {
-////          var ohlcv_params = OHLCVStream.get_params(begin_dt, end_dt);
-////          ViewStream ohlc_str = new OHLCVStream(symbol, ohlcv_params);
-////          // stream is grouped per contract and then vwap is calculated per contract
-////          GlobalMapStream rs = new GlobalMapStream(ohlc_str, "ts_event", "contract",
-////          (rows, idx) => {
-////             Log.Gen.info($"VWAPStream.GlobalMapStream delegate n_rows: {rows.Count} idx: {idx}");
-////             IBlueRow ir = rows.ElementAt((int)idx);
-////             var sel_rows = rows.Skip((int)idx).Where(r => r.get_datetime("ts_event") >= end_dt && r.get_datetime("ts_event") <= begin_dt);
-////             var volumeprices = sel_rows.Select(r => Tuple.Create(r.get<int>("volume"), r.get<double>("close"))).ToList();
-////             double vwap = IndicatorCalc.calculate_VWAP(volumeprices);
-////             IBlueTableSchema schema = ir.schema.add_columns(Tuple.Create(typeof(double), "vwap"));
-////             MutableBlueRow nr = new MutableBlueRow(schema, ir);
-////             //vwap = -313.987;
-////             nr.setValue("vwap", vwap);
-////             return nr;
-////          });
-////          return rs;
-////       }
-////       // the args want dt1 > dt2, if its nto we, swap them
-////       static void swap_if_necessary(ref DateTime dt1, ref DateTime dt2) {
-////          if (dt1 < dt2) {
-////             DateTime aux = dt1;
-////             dt1 = dt2;
-////             dt2 = aux;
-////          }
-////       }
+//    [DataStreamUserRespresentation("VWAP")]
+//    public class VWAPStream : ViewStream {
+//       // returns the VWAP stream for each contract 
+//       private static ViewStream build_vwap_stream(string symbol, DateTime begin_dt, DateTime end_dt) {
+//          var ohlcv_params = OHLCVStream.get_params(begin_dt, end_dt);
+//          ViewStream ohlc_str = new OHLCVStream(symbol, ohlcv_params);
+//          // stream is grouped per contract and then vwap is calculated per contract
+//          GlobalMapStream rs = new GlobalMapStream(ohlc_str, "ts_event", "contract",
+//          (rows, idx) => {
+//             Log.Gen.info($"VWAPStream.GlobalMapStream delegate n_rows: {rows.Count} idx: {idx}");
+//             IBlueRow ir = rows.ElementAt((int)idx);
+//             var sel_rows = rows.Skip((int)idx).Where(r => r.get_datetime("ts_event") >= end_dt && r.get_datetime("ts_event") <= begin_dt);
+//             var volumeprices = sel_rows.Select(r => Tuple.Create(r.get<int>("volume"), r.get<double>("close"))).ToList();
+//             double vwap = IndicatorCalc.calculate_VWAP(volumeprices);
+//             IBlueTableSchema schema = ir.schema.add_columns(Tuple.Create(typeof(double), "vwap"));
+//             MutableBlueRow nr = new MutableBlueRow(schema, ir);
+//             //vwap = -313.987;
+//             nr.setValue("vwap", vwap);
+//             return nr;
+//          });
+//          return rs;
+//       }
+//       // the args want dt1 > dt2, if its nto we, swap them
+//       static void swap_if_necessary(ref DateTime dt1, ref DateTime dt2) {
+//          if (dt1 < dt2) {
+//             DateTime aux = dt1;
+//             dt1 = dt2;
+//             dt2 = aux;
+//          }
+//       }
 
-////       // this assumes the ohlc_str has diff contracts
-////       public VWAPStream(string symbol_or_contract, DateTime begin_dt, DateTime end_dt) {
-////          swap_if_necessary(ref begin_dt, ref end_dt);
-////          Tuple<ContractUtils.ContractCategory, string> ccat_sym_contr = ContractUtils.get_symbol_category(symbol_or_contract);
-////          string symbol = String.Empty;
-////          symbol = ccat_sym_contr.Item2;
-////          var ohlcv_params = OHLCVStream.get_params(begin_dt, end_dt);
-////          ViewStream vwap_str = build_vwap_stream(symbol, begin_dt, end_dt);
-////          // vwap_str now has vwap for all contracts. if the arg was a symbol, we need to filter it for the active contract,
-////          // whihc we do with FutureActivesContractStream. If the arg was a contract we'll simply filter it
-////          ViewStream rs = vwap_str;
-////          switch (ccat_sym_contr.Item1) {
-////             case ContractUtils.ContractCategory.FutureString:
-////             case ContractUtils.ContractCategory.SpreadString:
-////                rs = new FilterStream(vwap_str, r => r.get<string>("contract").CompareTo(ccat_sym_contr.Item2) == 0);
-////                break;
-////             case ContractUtils.ContractCategory.Symbol:
-////                rs = new FuturesActiveContractStream(vwap_str);
-////                break;
-////          }
-////          rs = new FilterStream(rs, r => r.get_datetime("ts_event").CompareTo(begin_dt) == 0);
-////          rs = new NewProjectStream(rs, "! vwap");
-////          mySetRealSource(rs);
-////       }
+//       // this assumes the ohlc_str has diff contracts
+//       public VWAPStream(string symbol_or_contract, DateTime begin_dt, DateTime end_dt) {
+//          swap_if_necessary(ref begin_dt, ref end_dt);
+//          Tuple<ContractUtils.ContractCategory, string> ccat_sym_contr = ContractUtils.get_symbol_category(symbol_or_contract);
+//          string symbol = String.Empty;
+//          symbol = ccat_sym_contr.Item2;
+//          var ohlcv_params = OHLCVStream.get_params(begin_dt, end_dt);
+//          ViewStream vwap_str = build_vwap_stream(symbol, begin_dt, end_dt);
+//          // vwap_str now has vwap for all contracts. if the arg was a symbol, we need to filter it for the active contract,
+//          // whihc we do with FutureActivesContractStream. If the arg was a contract we'll simply filter it
+//          ViewStream rs = vwap_str;
+//          switch (ccat_sym_contr.Item1) {
+//             case ContractUtils.ContractCategory.FutureString:
+//             case ContractUtils.ContractCategory.SpreadString:
+//                rs = new FilterStream(vwap_str, r => r.get<string>("contract").CompareTo(ccat_sym_contr.Item2) == 0);
+//                break;
+//             case ContractUtils.ContractCategory.Symbol:
+//                rs = new FuturesActiveContractStream(vwap_str);
+//                break;
+//          }
+//          rs = new FilterStream(rs, r => r.get_datetime("ts_event").CompareTo(begin_dt) == 0);
+//          rs = new NewProjectStream(rs, "! vwap");
+//          mySetRealSource(rs);
+//       }
 
-////       public VWAPStream(string symbol_or_contract, string begin_dt, string end_dt)
-////       :
-////       this(symbol_or_contract, DateTime.Parse(begin_dt), DateTime.Parse(end_dt)) { }
-////    }
-//// }
+//       public VWAPStream(string symbol_or_contract, string begin_dt, string end_dt)
+//       :
+//       this(symbol_or_contract, DateTime.Parse(begin_dt), DateTime.Parse(end_dt)) { }
+//    }
+// }
 
-//using System;
+using System;
 
-//using BCG;
+using BCG;
 
-//using bts.api;
+using bts.api;
 
-//namespace bts.SilverPine {
-//   /*
-//      ----------------------------------------------------------------------
-//                                  *Expression*
-//      ----------------------------------------------------------------------
-//          //nl("Find Settlement price of the SP500 future in the first week of 2023.")
+namespace bts.SilverPine {
+   /*
+      ----------------------------------------------------------------------
+                                  *Expression*
+      ----------------------------------------------------------------------
+          //nl("Find Settlement price of the SP500 future in the first week of 2023.")
 
-//          r=FutureHistoricalBars("ES", "One_Day", "1/1/2024", "2/1/2024")
-//          r=math(r, "vprice = volume * close")
-//          r=aggregate(r, "vprice:Sum, volume:Sum")
-//          r=math(r, "vwap = vprice_sum / volume_sum")
-//          r=newProject(r, "vwap !")
+          r=FutureHistoricalBars("ES", "One_Day", "1/1/2024", "2/1/2024")
+          r=math(r, "vprice = volume * close")
+          r=aggregate(r, "vprice:Sum, volume:Sum")
+          r=math(r, "vwap = vprice_sum / volume_sum")
+          r=newProject(r, "vwap !")
 
-//      ----------------------------------------------------------------------
-//                                    *Schema*
-//      ----------------------------------------------------------------------
-//          id                              System.Int32
-//          vwap                            System.Double
-//   */
+      ----------------------------------------------------------------------
+                                    *Schema*
+      ----------------------------------------------------------------------
+          id                              System.Int32
+          vwap                            System.Double
+   */
 
-//   [DataStreamUserRespresentation("VWAP")]
-//   public class VWAPStream : ViewStream {
-//      public VWAPStream(string symbol_or_contract, DateTime beg_dt, DateTime end_dt)
-//      :
-//      this(symbol_or_contract, beg_dt, end_dt, ViewStreamDatabase.instance) { }
+   [DataStreamUserRespresentation("VWAP")]
+   public class VWAPStream : ViewStream {
+      public VWAPStream(string symbol_or_contract, DateTime beg_dt, DateTime end_dt)
+      :
+      this(symbol_or_contract, beg_dt, end_dt, ViewStreamDatabase.instance) { }
 
-//      public VWAPStream(string symbol_or_contract, DateTime beg_dt, DateTime end_dt, ViewStreamDatabase db) {
-//         //var fc = new ContractUtils.FutureContract(symbol_or_contract);
-//         ViewStream r = new FutureHistoricalBarsStream(symbol_or_contract, beg_dt.ToString(), end_dt.ToString());
-//         // since this is already filtered, we can groupBy iit without a groupBy column
-//         r = new MathStream(r,
-//                 "vprice = volume * close");
-//         r = new NonIncGroupByStream(r,
-//                                      "vprice:Sum, volume:Sum");
-//         r = new MathStream(r,
-//                             "vwap = vprice_sum / volume_sum");
-//         r = new NewProjectStream(r,
-//                                   "vwap !");
-//         mySetRealSource(r);
-//      }
-//   }
-//}
+      public VWAPStream(string symbol_or_contract, DateTime beg_dt, DateTime end_dt, ViewStreamDatabase db) {
+         //var fc = new ContractUtils.FutureContract(symbol_or_contract);
+         ViewStream r = new FutureHistoricalBarsStream(symbol_or_contract, beg_dt.ToString(), end_dt.ToString());
+         // since this is already filtered, we can groupBy iit without a groupBy column
+         r = new MathStream(r,
+                 "vprice = volume * close");
+         r = new NonIncGroupByStream(r,
+                                      "vprice:Sum, volume:Sum");
+         r = new MathStream(r,
+                             "vwap = vprice_sum / volume_sum");
+         r = new NewProjectStream(r,
+                                   "vwap !");
+         mySetRealSource(r);
+      }
+   }
+}

src/DataStreamSuite/SharedStreams/Business/MarketData/BBOStream.cs

@@ -1,4 +1,4 @@
-namespace bts.BlueOrb {
+namespace bts.SilverPine {
    using BCG.Utilities.General;
 
    using bts.md;

src/DataStreamSuite/SharedStreams/Business/MarketData/Bars/HistoricalBarsStream.cs

@@ -1,5 +1,4 @@
-namespace bts.BlueOrb {
-   using bts.SilverPine;
+namespace bts.SilverPine {
    using BCG;
 
    using System.Runtime.CompilerServices;
@@ -46,7 +45,7 @@ namespace bts.BlueOrb {
    }
 
    [DataStreamUserRespresentation("FuturesHistorical1DayBars")]
-   internal class FuturesHistorical1DayBarsStream : ViewStream {
+   public class FuturesHistorical1DayBarsStream : ViewStream {
 
       // this gets the raw bars for a symbol (perhaps make it into a raw bars stream?)
       public FuturesHistorical1DayBarsStream(string symbol) :
@@ -104,7 +103,7 @@ namespace bts.BlueOrb {
    }
 
    [DataStreamUserRespresentation("FuturesHistorical1MinBars")]
-   internal class FuturesHistorical1MinuteBarsStream : ViewStream {
+   public class FuturesHistorical1MinuteBarsStream : ViewStream {
       // same as the above but for one min
       // this gets the raw bars for a symbol
       public FuturesHistorical1MinuteBarsStream(string symbol) :
@@ -147,7 +146,7 @@ namespace bts.BlueOrb {
       }
    }
 
-   [DataStreamUserRespresentation("FutureHistoricalBarsStream")]
+   [DataStreamUserRespresentation("FutureHistoricalBars")]
    public class FutureHistoricalBarsStream : ViewStream {
       public FutureHistoricalBarsStream(string fut_contract, string range_beg, string range_end) {
          DateTime dt_range_beg = DateTime.Parse(range_beg);
@@ -188,6 +187,46 @@ namespace bts.BlueOrb {
       }
    }
 
+   [DataStreamUserRespresentation("FuturePriceDiff")]
+   public class FutureHistoricalBarsPriceDiffStream : ViewStream {
+
+      public FutureHistoricalBarsPriceDiffStream(string future_contract, string dt1, string dt2)
+      :
+      this(new ContractUtils.FutureContract(future_contract), DateTime.Parse(dt1), DateTime.Parse(dt2), "close", ViewStreamDatabase.instance) {
+      }
+
+
+      public FutureHistoricalBarsPriceDiffStream(string future_contract, DateTime dt1, DateTime dt2)
+      :
+      this(new ContractUtils.FutureContract(future_contract), dt1, dt2, "close", ViewStreamDatabase.instance) {
+      }
+
+      public FutureHistoricalBarsPriceDiffStream(string future_contract, DateTime dt1, DateTime dt2, string col)
+      :
+      this(new ContractUtils.FutureContract(future_contract), dt1, dt2, col, ViewStreamDatabase.instance) {
+      }
+
+
+      public FutureHistoricalBarsPriceDiffStream(ContractUtils.FutureContract fc, DateTime dt1, DateTime dt2, string col, ViewStreamDatabase vsd) {
+         BarSize bsize = dt2 - dt1 >= TimeSpan.FromDays(1.0) ? BarSize.ONE_DAY : BarSize.ONE_MIN;
+         ViewStream rs = new FutureHistoricalBarsStream(fc, BarSize.ONE_DAY, new DateTimeRange(dt1, dt2));
+
+         rs = new ColumnDifferenceStream(rs, r => {
+            DateTime r_dt = r.get_datetime("ts_event");
+            var rv = dt1.Equals(r_dt);
+            return rv;
+         },
+         //r => dt2.Equals(r.get_datetime("ts_event")), col, col);
+         r => {
+            DateTime r_dt = r.get_datetime("ts_event");
+            var rv = dt2.Equals(r_dt);
+            return rv;
+         }, col, col);
+         rs = new NewProjectStream(rs, "diff !");
+         mySetRealSource(rs);
+      }
+   }
+
    // takes the contract name and expands it into multuiple columns corresponding to the contracts.
    // the contract columns are a union, of all properties with some of them empty
    public class ContractExpanderStream : ViewStream {
@@ -208,29 +247,3 @@ namespace bts.BlueOrb {
    }
 
 }
-
-namespace bts.BlueOrb {
-   using BCG;
-
-   using bts.api;
-
-   using Newtonsoft.Json.Serialization;
-
-   [DataStreamUserRespresentation("FuturesContinousContract")]
-   public class FuturesContinousContractStream : ViewStream {
-      // the argument is symbol, like "ES" etc and not the contract itself
-      public FuturesContinousContractStream(string symbol) : this(symbol, ViewStreamDatabase.instance) { }
-      public FuturesContinousContractStream(string symbol, ViewStreamDatabase db) {
-         symbol = symbol.EndsWith("*") ? symbol : symbol + "*";
-         ViewStream r = new FuturesHistorical1DayBarsStream(symbol);
-         r = new ContractExpanderStream(r, "symbol");
-         r = new FilterStream(r, "contract_type='Future'");
-         // finding the max volume across symbols, if we need to do this for open_interest, then join here 
-         // with the open interest stream on symbol/contract
-         r = new NonIncGroupByStream(r, "ts_event", "volume:Max");
-         r = new NewProjectStream(r, "ts_event symbol !");
-         r = new SortStream(r, "-ts_event");
-         mySetRealSource(r);
-      }
-   }
-}

src/DataStreamSuite/SharedStreams/Business/MarketData/Bars/VolumeAggregationStream.cs

@@ -1,53 +1,53 @@
-//using BCG;
+using BCG;
 
-//namespace bts.SilverPine {
-//   /*
-//      ----------------------------------------------------------------------
-//                                  *Expression*
-//      ----------------------------------------------------------------------
-//          //r=FutureHistoricalBars("ES*", "02/01/2024 09:30 AM", "02/01/2024 09:30 PM")
-//          r=FutureHistoricalBars("ES*","One_Day", 02/01/2024, 02/01/2025)
-//          r=contractExpander(r,"symbol")
-//          r=filter(r,"contract_type='FutureSpread'")
-//          r=aggregate(r,"symbol","volume:Sum","ts_event")
-//          r=aggregate(r,"volume_sum:Max")
-//          r=newProject(r,"volume_sum_max:volume symbol !")
+namespace bts.SilverPine {
+   /*
+      ----------------------------------------------------------------------
+                                  *Expression*
+      ----------------------------------------------------------------------
+          //r=FutureHistoricalBars("ES*", "02/01/2024 09:30 AM", "02/01/2024 09:30 PM")
+          r=FutureHistoricalBars("ES*","One_Day", 02/01/2024, 02/01/2025)
+          r=contractExpander(r,"symbol")
+          r=filter(r,"contract_type='FutureSpread'")
+          r=aggregate(r,"symbol","volume:Sum","ts_event")
+          r=aggregate(r,"volume_sum:Max")
+          r=newProject(r,"volume_sum_max:volume symbol !")
 
-//      ----------------------------------------------------------------------
-//                                    *Schema*
-//      ----------------------------------------------------------------------
-//          symbol                          System.String
-//          id                              System.Int32
-//          volume                          System.Double
-//   */
+      ----------------------------------------------------------------------
+                                    *Schema*
+      ----------------------------------------------------------------------
+          symbol                          System.String
+          id                              System.Int32
+          volume                          System.Double
+   */
 
-//   [DataStreamUserRespresentation("VolumeAgg")]
-//   public class VolumeAggregationStream : ViewStream {
-//      public VolumeAggregationStream(string future_contract, string contract_type, DateTime range_beg, DateTime range_end)
-//      : this(future_contract, contract_type, range_beg, range_end, ViewStreamDatabase.instance) { }
+   [DataStreamUserRespresentation("VolumeAgg")]
+   public class VolumeAggregationStream : ViewStream {
+      public VolumeAggregationStream(string future_contract, string contract_type, DateTime range_beg, DateTime range_end)
+      : this(future_contract, contract_type, range_beg, range_end, ViewStreamDatabase.instance) { }
 
-//      public VolumeAggregationStream(string future_contract, string contract_type, string range_beg, string range_end)
-//      :
-//      this(future_contract, contract_type, DateTime.Parse(range_beg), DateTime.Parse(range_end), ViewStreamDatabase.instance) { }
+      public VolumeAggregationStream(string future_contract, string contract_type, string range_beg, string range_end)
+      :
+      this(future_contract, contract_type, DateTime.Parse(range_beg), DateTime.Parse(range_end), ViewStreamDatabase.instance) { }
 
-//      public VolumeAggregationStream(string future_contract, string contract_type, DateTime range_beg, DateTime range_end, ViewStreamDatabase db) {
-//         future_contract = future_contract.EndsWith("*") ? future_contract : future_contract + "*";
-//         ViewStream r;
-//         r = new FutureHistoricalBarsStream(future_contract,
-//                                             range_beg.ToString(), range_end.ToString());
-//         r = new ContractExpanderStream(r, "symbol");
-//         r = new FilterStream(r, $"contract_type='{contract_type}'");
-//         r = new NonIncGroupByStream(r, "symbol", "volume:Sum", "ts_event");
-//         r = new NonIncGroupByStream(r, "volume_sum:Max");
-//         r = new NewProjectStream(r, "volume_sum_max:volume symbol !");
-//         mySetRealSource(r);
-//      }
-//   }
+      public VolumeAggregationStream(string future_contract, string contract_type, DateTime range_beg, DateTime range_end, ViewStreamDatabase db) {
+         future_contract = future_contract.EndsWith("*") ? future_contract : future_contract + "*";
+         ViewStream r;
+         r = new FutureHistoricalBarsStream(future_contract,
+                                             range_beg.ToString(), range_end.ToString());
+         r = new ContractExpanderStream(r, "symbol");
+         r = new FilterStream(r, $"contract_type='{contract_type}'");
+         r = new NonIncGroupByStream(r, "symbol", "volume:Sum", "ts_event");
+         r = new NonIncGroupByStream(r, "volume_sum:Max");
+         r = new NewProjectStream(r, "volume_sum_max:volume symbol !");
+         mySetRealSource(r);
+      }
+   }
 
-//   [DataStreamUserRespresentation("PeriodicVolumeAgg")]
-//   public class PeriodicVolumeAggregationStream : ViewStream {
-//      public PeriodicVolumeAggregationStream(string future_contract, string contract_type, DateTime range_beg, DateTime range_end,
-//                                             TimeSpan period, ViewStreamDatabase db) {
-//      }
-//   }
-//}
+   [DataStreamUserRespresentation("PeriodicVolumeAgg")]
+   public class PeriodicVolumeAggregationStream : ViewStream {
+      public PeriodicVolumeAggregationStream(string future_contract, string contract_type, DateTime range_beg, DateTime range_end,
+                                             TimeSpan period, ViewStreamDatabase db) {
+      }
+   }
+}

src/DataStreamSuite/SharedStreams/Business/MarketData/BookStream.cs

@@ -1,6 +1,5 @@
-namespace bts.BlueOrb{
+namespace BCG {
    using BCG.Utilities.General;
-   using BCG;
 
    using bts.md;
 

src/DataStreamSuite/SharedStreams/Business/MarketData/EventsStream.cs

@@ -1,4 +1,4 @@
-namespace bts.BlueOrb {
+namespace bts.SilverPine {
    using BCG;
 
    [DataStreamUserRespresentation("Events")]
@@ -15,4 +15,4 @@ namespace bts.BlueOrb {
          mySetRealSource(rs);
       }
    }
-}
+}

src/DataStreamSuite/SharedStreams/Business/MarketData/FuturesActiveContractStream.cs

@@ -1,4 +1,4 @@
-namespace bts.BlueOrb{
+namespace bts.SilverPine {
    using BCG;
 
    using bts;
@@ -76,3 +76,30 @@ namespace bts.BlueOrb{
       }
    }
 }
+
+
+namespace bts.SilverPine {
+   using BCG;
+
+   using bts.api;
+
+   using Newtonsoft.Json.Serialization;
+
+   [DataStreamUserRespresentation("FuturesContinousContract")]
+   public class FuturesContinousContractStream : ViewStream {
+      // the argument is symbol, like "ES" etc and not the contract itself
+      public FuturesContinousContractStream(string symbol) : this(symbol, ViewStreamDatabase.instance) { }
+      public FuturesContinousContractStream(string symbol, ViewStreamDatabase db) {
+         symbol = symbol.EndsWith("*") ? symbol : symbol + "*";
+         ViewStream r = new FuturesHistorical1DayBarsStream(symbol);
+         r = new ContractExpanderStream(r, "symbol");
+         r = new FilterStream(r, "contract_type='Future'");
+         // finding the max volume across symbols, if we need to do this for open_interest, then join here 
+         // with the open interest stream on symbol/contract
+         r = new NonIncGroupByStream(r, "ts_event", "volume:Max");
+         r = new NewProjectStream(r, "ts_event symbol !");
+         r = new SortStream(r, "-ts_event");
+         mySetRealSource(r);
+      }
+   }
+}

src/DataStreamSuite/SharedStreams/Business/MarketData/FuturesRollStream.cs

@@ -1,18 +1,16 @@
+namespace bts.SilverPine {
+   using bts;
+   using BCG;
 
-//TODO: Remove
-//namespace bts.SilverPine {
-//   using bts;
-//   using BCG;
-
-//   // it uses the volume (better to use open interest perhap) to detect the rollover, ie when a new 
-//   // future exp becomes the highest volume one
-//   [DataStreamUserRespresentation("FuturesRoll")]
-//   public class FuturesRollStream : ViewStream {
-//      public FuturesRollStream(string future_symbol) {
-//         ViewStream rs = new FuturesContinousContractStream(future_symbol);
-//         rs = new ColumnEvolutionStream(rs, "ts_event", "symbol");
-//         rs = new NewProjectStream(rs, "symbol ts_event:roll_date !");
-//         mySetRealSource(rs);
-//      }
-//   }
-//}
+   // it uses the volume (better to use open interest perhap) to detect the rollover, ie when a new 
+   // future exp becomes the highest volume one
+   [DataStreamUserRespresentation("FuturesRoll")]
+   public class FuturesRollStream : ViewStream {
+      public FuturesRollStream(string future_symbol) {
+         ViewStream rs = new FuturesContinousContractStream(future_symbol);
+         rs = new ColumnEvolutionStream(rs, "ts_event", "symbol");
+         rs = new NewProjectStream(rs, "symbol ts_event:roll_date !");
+         mySetRealSource(rs);
+      }
+   }
+}

src/DataStreamSuite/SharedStreams/Business/MarketData/InstrumentsStream.cs

@@ -1,4 +1,4 @@
-namespace bts.BlueOrb {
+namespace bts.SilverPine {
    using BCG.Utilities.General;
 
    using bts.api;

src/DataStreamSuite/SharedStreams/Business/MarketData/OHLCVPriceDiffStream.cs

@@ -1,37 +1,37 @@
-//namespace bts.SilverPine {
-//   using BCG;
+namespace bts.SilverPine {
+   using BCG;
 
-//   [DataStreamUserRespresentation("OHLCVPriceDiff")]
-//   public class OHLCVPriceDiffStream : ViewStream {
-//      // TODO, abstract this out
-//      private static ViewStream _get_ohlcv_steam(string symbol_or_contract, DateTime d1, DateTime d2) {
-//         var tup = ContractUtils.get_symbol_category(symbol_or_contract);
-//         var ohlcv_params = OHLCVStream.get_params(d1, d2);
-//         var cont_cat = tup.Item1;
-//         string symbol = tup.Item2;
-//         ViewStream rs = new OHLCVStream(symbol, ohlcv_params);
-//         switch (cont_cat) {
-//            case ContractUtils.ContractCategory.FutureString:
-//            case ContractUtils.ContractCategory.SpreadString:
-//               rs = new FilterStream(rs, $"contract = '{symbol_or_contract}'");
-//               break;
-//            case ContractUtils.ContractCategory.Symbol:
-//               rs = new FuturesActiveContractStream(rs);
-//               break;
-//         }
-//         return rs;
-//      }
+   [DataStreamUserRespresentation("OHLCVPriceDiff")]
+   public class OHLCVPriceDiffStream : ViewStream {
+      // TODO, abstract this out
+      private static ViewStream _get_ohlcv_steam(string symbol_or_contract, DateTime d1, DateTime d2) {
+         var tup = ContractUtils.get_symbol_category(symbol_or_contract);
+         var ohlcv_params = OHLCVStream.get_params(d1, d2);
+         var cont_cat = tup.Item1;
+         string symbol = tup.Item2;
+         ViewStream rs = new OHLCVStream(symbol, ohlcv_params);
+         switch (cont_cat) {
+            case ContractUtils.ContractCategory.FutureString:
+            case ContractUtils.ContractCategory.SpreadString:
+               rs = new FilterStream(rs, $"contract = '{symbol_or_contract}'");
+               break;
+            case ContractUtils.ContractCategory.Symbol:
+               rs = new FuturesActiveContractStream(rs);
+               break;
+         }
+         return rs;
+      }
 
-//      public OHLCVPriceDiffStream(string symbol_or_contract, DateTime d1, DateTime d2)
-//      : this(symbol_or_contract, d1, d2, "close") { }
+      public OHLCVPriceDiffStream(string symbol_or_contract, DateTime d1, DateTime d2)
+      : this(symbol_or_contract, d1, d2, "close") { }
 
-//      public OHLCVPriceDiffStream(string symbol_or_contract, DateTime d1, DateTime d2, string col) {
-//         ViewStream rs = _get_ohlcv_steam(symbol_or_contract, d1, d2);
-//         rs = new ColumnDifferenceStream(rs, r => d1.Equals(r.get_datetime("ts_event")),
-//                                              r => d2.Equals(r.get_datetime("ts_event")), col, col);
-//         rs = new NewProjectStream(rs, "diff !");
-//         mySetRealSource(rs);
-//      }
+      public OHLCVPriceDiffStream(string symbol_or_contract, DateTime d1, DateTime d2, string col) {
+         ViewStream rs = _get_ohlcv_steam(symbol_or_contract, d1, d2);
+         rs = new ColumnDifferenceStream(rs, r => d1.Equals(r.get_datetime("ts_event")),
+                                              r => d2.Equals(r.get_datetime("ts_event")), col, col);
+         rs = new NewProjectStream(rs, "diff !");
+         mySetRealSource(rs);
+      }
 
-//   }
-//}
+   }
+}

src/DataStreamSuite/SharedStreams/Business/MarketData/OHLCVStream.cs

@@ -1,4 +1,4 @@
-namespace bts.BlueOrb {
+namespace bts.SilverPine {
    using bts;
    using BCG;
    using MongoDB.Driver;

src/DataStreamSuite/SharedStreams/Business/MarketData/SettlementPriceStream.cs

@@ -1,27 +1,26 @@
-// TODO: Remove
-//namespace bts.SilverPine {
-//   using BCG;
+namespace bts.SilverPine {
+   using BCG;
 
-//   using bts;
-//   // used directly in blueorb
-//   [DataStreamUserRespresentation("Settlement")]
-//   public class SettlementPriceStream : ViewStream {
+   using bts;
+   // used directly in blueorb
+   [DataStreamUserRespresentation("Settlement")]
+   public class SettlementPriceStream : ViewStream {
 
-//      // public SettlementPriceStream(string future_name, string date_txt)
-//      // : this(future_name, date_txt, date_txt) { }
+      // public SettlementPriceStream(string future_name, string date_txt)
+      // : this(future_name, date_txt, date_txt) { }
 
-//      // public SettlementPriceStream(string future_name, string begin_dt, string end_dt)
-//      // : this(future_name, DateTime.Parse(begin_dt).Date, DateTime.Parse(end_dt).Date) { }
+      // public SettlementPriceStream(string future_name, string begin_dt, string end_dt)
+      // : this(future_name, DateTime.Parse(begin_dt).Date, DateTime.Parse(end_dt).Date) { }
 
-//      public SettlementPriceStream(string future_contract, DateTime begin_dt)
-//      : this(future_contract, begin_dt, begin_dt) { }
+      public SettlementPriceStream(string future_contract, DateTime begin_dt)
+      : this(future_contract, begin_dt, begin_dt) { }
 
-//      public SettlementPriceStream(string future_contract, DateTime begin_dt, DateTime end_dt) {
-//         ContractUtils.FutureContract fc = new ContractUtils.FutureContract(future_contract);
-//         ViewStream rs = new FutureHistoricalBarsStream(fc, BarSize.ONE_DAY, new DateTimeRange(begin_dt, end_dt));
-//         rs = new NewProjectStream(rs, "symbol ts_event:date close:settlement !");
-//         mySetRealSource(rs);
-//      }
+      public SettlementPriceStream(string future_contract, DateTime begin_dt, DateTime end_dt) {
+         ContractUtils.FutureContract fc = new ContractUtils.FutureContract(future_contract);
+         ViewStream rs = new FutureHistoricalBarsStream(fc, BarSize.ONE_DAY, new DateTimeRange(begin_dt, end_dt));
+         rs = new NewProjectStream(rs, "symbol ts_event:date close:settlement !");
+         mySetRealSource(rs);
+      }
 
-//   }
-//}
+   }
+}

src/DataStreamSuite/SharedStreams/Business/MarketData/SyntheticSpreadStream.cs

@@ -1,7 +1,8 @@
 using System;
 using BCG;
 
-namespace bts.BlueOrb {
+namespace bts.SilverPine
+{
 /*
    ----------------------------------------------------------------------
                                *Expression*

src/DataStreamSuite/SharedStreams/Business/MarketData/TimSeriesOperations/BarsSearcherStream.cs

@@ -1,32 +1,32 @@
-//namespace bts.SilverPine {
-//   using BCG;
+namespace bts.SilverPine {
+   using BCG;
 
-//   // this is an internal stream. its job is to take an OHLCV stream
-//   // and filter it for 
-//   // 1. symbol (table_name and active contract) and 
-//   // 2. time  (can be a day of the week, month, and a time of the day period)
-//   public class BarsSearcherStream : ViewStream {
-//      public BarsSearcherStream(string symbol, OHLCVStream.OHLCVBarSize bar_size,
-//                                 string day_of_week, TimeSpan tod_begin, TimeSpan tod_end) {
-//         ViewStream rs = new OHLCVStream(symbol, bar_size);
-//         rs = new DateTimeExpanstionStream(rs, "ts_event");
-//         rs = new FilterStream(rs, r => {
-//            DateTime ts = r.get_datetime("ts_event");
-//            return r.get<string>("ts_event_day") == day_of_week.ToString() &&
-//                  ts.TimeOfDay >= tod_begin && ts.TimeOfDay <= tod_end;
-//         });
-//         // look at symbol and either filter ir ot do the active contracts
-//         // assuming its symbol
-//         rs = new NonIncJoinStream(rs, "ts_event_date", "open high low close volume contract ts_event",
-//                                 new FuturesActiveContractStream(symbol), "ts_event", "ts_event:date contract:active_contract");
-//         // filter it for the active contract only
-//         rs = new FilterStream(rs, r => String.Compare(r.get<string>("contract"), r.get<string>("active_contract")) == 0);
-//         mySetRealSource(rs);
-//      }
-//      public BarsSearcherStream(string symbol, string bar_size,
-//                                 string day_of_week, string tod_begin, string tod_end)
-//                                 :
-//                                 this(symbol, Enum.Parse<OHLCVStream.OHLCVBarSize>(bar_size), day_of_week,
-//                                       TimeSpan.Parse(tod_begin), TimeSpan.Parse(tod_end)) { }
-//   }
-//}
+   // this is an internal stream. its job is to take an OHLCV stream
+   // and filter it for 
+   // 1. symbol (table_name and active contract) and 
+   // 2. time  (can be a day of the week, month, and a time of the day period)
+   public class BarsSearcherStream : ViewStream {
+      public BarsSearcherStream(string symbol, OHLCVStream.OHLCVBarSize bar_size,
+                                 string day_of_week, TimeSpan tod_begin, TimeSpan tod_end) {
+         ViewStream rs = new OHLCVStream(symbol, bar_size);
+         rs = new DateTimeExpanstionStream(rs, "ts_event");
+         rs = new FilterStream(rs, r => {
+            DateTime ts = r.get_datetime("ts_event");
+            return r.get<string>("ts_event_day") == day_of_week.ToString() &&
+                  ts.TimeOfDay >= tod_begin && ts.TimeOfDay <= tod_end;
+         });
+         // look at symbol and either filter ir ot do the active contracts
+         // assuming its symbol
+         rs = new NonIncJoinStream(rs, "ts_event_date", "open high low close volume contract ts_event",
+                                 new FuturesActiveContractStream(symbol), "ts_event", "ts_event:date contract:active_contract");
+         // filter it for the active contract only
+         rs = new FilterStream(rs, r => String.Compare(r.get<string>("contract"), r.get<string>("active_contract")) == 0);
+         mySetRealSource(rs);
+      }
+      public BarsSearcherStream(string symbol, string bar_size,
+                                 string day_of_week, string tod_begin, string tod_end)
+                                 :
+                                 this(symbol, Enum.Parse<OHLCVStream.OHLCVBarSize>(bar_size), day_of_week,
+                                       TimeSpan.Parse(tod_begin), TimeSpan.Parse(tod_end)) { }
+   }
+}

src/DataStreamSuite/SharedStreams/NL/DSL_master_learnign_prompt.md

@@ -1,571 +1,276 @@
-# BlueOrbDSL Master Learning Prompt v1.1 (Optimized)
+# **DSL Master Learning Prompt**
 
-**This version is the *final, frozen* canonical Master Learning Prompt for BlueOrbDSL model training and generation.**  
-**No further edits are required. This is the single source of truth for Learning Mode.**  
-**Reviewed and validated as optimal on \[insert date here, e.g. June 9, 2025\] after full Meta Mode review.**
+This prompt is designed to teach the foundational concepts of the DSL for financial data streaming and transformation. The DSL operates on "streams," which fetch, transform, and process financial market data in real-time or historical contexts. Forget everything I have taught you about this DSL until now (since there are some contradicting and breaking changes that can not be reconciled with anything you might have learned before) and re-learn from this prompt.
 
-**Change Log — v1 → v1.1**  
-(June 7, 2025\)
+# **Introduction to Streams**
 
-✅ Applied full Meta Mode review recommendations:
+The DSL operates on "streams," its core functional units. Streams fetch tabular (from database, text files or network) or transform/operate on tabular data. They either act as **Datasources** (e.g., FutureHistoricalBars), that bootstrap the data or **Operators** (e.g., `FilterStream, ProjectStream, SingleRowMathStream, GroupByStream` etc) that transform or filter tabular data. Though the DSL can work on any tabular data, it is primarily written for streaming or static financial data
 
-- Standardized **Arguments** section (bullet lists, first arg \= `parent_stream` for operators)  
-- Added **Adds Columns?** and **Modifies Schema?** rows for *all* streams  
-- Clarified **Special Notes** for all streams  
-- Clearly stated UnionStream schema requirements  
-- Explicitly documented JoinStream as inner join, with left/right prefix behavior  
-- Clarified behavior of **TimeBasedRollingFilterStream**  
-- Changed DateTimeExpanderStream new columns to a bullet list for clarity  
-- Standardized all **Example Usage** to fenced `dsl` code blocks  
-- Shortened **PythonDataframeStream** example; big one should go to appendix if needed  
-- Minor language and consistency cleanups across stream definitions
+# **Streams**
 
-**This version is now ready to serve as the authoritative MLP for BlueOrbDSL training and generation.**  
-When instructed to "Switch to Learning Mode," this will become the model’s single source of truth.
+## **FutureHistoricalBars (DataSource Stream)**
+
+The `FutureHistoricalBars` stream fetches OHLCV (Open, High, Low, Close, Volume) bars for financial futures contracts. It acts as a **data source** and provides the initial table that can be further processed by operators.
+
+#### **Stream arguments (parameters):**
+
+1. **`Future_contract` (string):** Specifies the contract to fetch data for. This can be one of the following 3 types  
+1. **Continuous contracts:** Use the base symbol (e.g., `ES` for S\&P 500 futures, CL for WTI Crude Oil Futures, GC for Gold Futures etc).  
+2. **Specific contracts:** Use a detailed symbol (e.g., `ESZ9` for December 2029 S\&P 500 futures, `GCU9`, for September 2025 Gold Future, `SIH9-J9`,  
+   For SIlver March April 2019 Silver Future Calendar spread)  
+3. **All Contracts for a symbol:** If instead of specifying individual contracts, data for all contracts for a specific symbol is needed, use the symbol name followed by *.* So `ES*` for all SP Futures and spread contracts, ZC\* for all corn contracts etc.  
+2. **`Bar_size` (string):**  The second argument defines the granularity of the bars. Accepted values:  
+   * `ONE_DAY` – Daily bars.  
+     * `ONE_MIN` – Minute bars.  
+3. **Optional:**  
+   * **`range_begin` (datetime):** Start of the desired time range.  
+   * **`range_end` (datetime):** End of the desired time range.
+
+#### **Schema:**
+
+The output of `FutureHistoricalBars` produces a table with the following schema (this is a classic OHLCV schema that you can find online easily)
+
+| id   | ts\_event             | contract | open    | high    | low     | close   | volume |
+| :--- | :-------------------- | :------- | :------ | :------ | :------ | :------ | :----- |
+| `0`  | `2024-06-08T00:00:00` | `ESZ9`   | 5001.50 | 5052.25 | 4990.75 | 5025.50 | 866000 |
+|      |                       |          |         |         |         |         |        |
+
+# **FilterStream (Operator Stream)**
+
+Filters rows of a parent stream based on conditions applied to its columns.
+
+#### **Stream arguments (parameters):**
+
+1. `parent_stream (another stream)` This stream provides the data, that the FilterStream subscribers to  
+2. `filter_condition (string)` This is a text expressing a boolean condition that is applied to each row of the parent stream, if it returns false, the row is filtered out
+
+#### **Examples:**
+
+1. **NL prompt:** Show me all the days when the open price for SP 500 Futures was greater than or equal to 5000:  
+   **DSL:** `r = FilterStream(FutureHistoricalBars("ES", "ONE_DAY"), "open >= 5000")`  
+2. **NL prompt:** Show me all the days when the close price for SP 500 Futures was less than 3000 and volume was greater than half a million   
+   **DSL:** `r = FilterStream(FutureHistoricalBars("ES", "ONE_MIN"), "close < 3000 and volume > 500000)`  
+3. **NL prompt:** Show me all the daily bars for the SP 500 Future spread ESU9-Z9 where its volume exceeded 100K  
+   **DSL:** `r = FilterStream(FutureHistoricalBars("ES*", "ONE_MIN"), "contract = ‘ESU9-Z9’ and volume >= 100000”)`  
+   
+
+**Notice** how expressions can be joined with AND and OR. These expressions can be nested with parenthesis.Also string needs to be in single quotes.
+
+# **ProjectStream (Operator Stream)**
+
+Changes the name of the columns in the parent stream. So for example, if we have to rename the close column to settlement, we can do this with ProjectStream
+
+#### **Stream arguments (parameters):**
+
+3. `parent_stream (another stream)` This stream provides the data, that the ProjectStream subscribers to  
+4. `renaming (string)` This is a text expressed in the form original\_col\_name:projected\_col\_name, original\_col\_name:projected\_col\_name, 
+
+#### **Examples:**
+
+	The following DSL renames the open column to O, high to H, low to L and close to C  
+**DSL:** `r = ProjectStream(FutureHistoricalBars("ES", "ONE_DAY"), "open:O, high:H, low:L, close:C”)`
+
+# **GroupByStream (Operator Stream)**
+
+The `GroupByStream` function is a DSL operator used to perform aggregation operations on rows of a parent stream. It works similarly to SQL’s `GROUP BY`, allowing rows to be grouped by a specific column (optional) and applying aggregation functions to other columns. 
 
 ---
 
-# Table of Contents
+#### **Stream arguments (parameters):**
 
-1. Preamble  
-2. Streams Overview  
-3. Data Sources  
-   3.1 FutureHistoricalBarsStream  
-   3.2 LiveBBOStream  
-   3.3 LiteralStream  
-4. Operators  
-   4.1 FilterStream  
-   4.2 ProjectStream  
-   4.3 GroupByStream  
-   4.4 SingleRowMathStream  
-   4.5 MultiRowRollingMathStream  
-   4.6 UnionStream  
-   4.7 JoinStream  
-   4.8 DateTimeExpanderStream  
-   4.9 ContractExpanderStream  
-   4.10 CountStream  
-   4.11 TimeSeriesStream  
-   4.12 TimeBasedRollingFilterStream  
-   4.13 PythonDataframeStream  
-5. Reserved Words & Enumerations  
-6. Common Pitfalls
+1. **parent\_stream (another stream):** The source stream provides the data to be operated on  
+2. **Group By Column (string, optional):**
+
+   A column in the schema of the parent stream used to group rows. All rows with the same value in this column are aggregated into a single group. If omitted, the aggregation is performed over all rows of the parent stream.
+
+3. **Operation Text (string):** A string defining the column(s) to be aggregated and the operation(s) to apply, in the format:  
+   `<column_name: operation>, <column_name: operation>, …` Each `<column_name: operation>` pair produces a new column in the output stream. The new column name is formatted as `<column_name>_<operation>` in lowercase.
 
 ---
 
-## 1 Preamble
+#### **Valid Aggregation Operations:**
 
-BlueOrbDSL turns natural-language finance questions into **stream pipelines** that operate on tabular market data (historical or real-time).  
-Each *stream* is either a **Data Source** (fetches data) or an **Operator** (transforms data).  
-Streams compose: the output table of one becomes the input of the next.
-
-**Convention:** For all operators, the **first argument** is always the `parent_stream`. Subsequent arguments are constructor parameters.
+* `Mean`: Arithmetic mean of the values.  
+* `Median`: Median of the values.  
+* `Mode`: Most frequently occurring value.  
+* `Range`: Difference between the max and min.  
+* `StdDev`: Standard deviation of the values.  
+* `Max`: Maximum value.  
+* `Min`: Minimum value.  
+* `First`: First value in the group.  
+* `Last`: Last value in the group.  
+* `Sum`: Sum of the values.  
+* `EdgeDiff`: Difference between the edge values, meaning the difference between a certain col belonging to the first and last rows respectively  
+* `EdgeDiffPercentage`: Same as `EdgeDiff` but expressed as percentage instead
 
 ---
 
-## 2 Streams Overview
+#### **Examples:**
 
-All stream definitions follow this table layout:
-
-| Field                | Meaning                                                |
-| :------------------- | :----------------------------------------------------- |
-| **Type**             | DataSource / Operator                                  |
-| **Purpose**          | One-sentence description                               |
-| **Arguments**        | Bullet list; first arg \= parent\_stream (if Operator) |
-| **Output Schema**    | Columns created or removed                             |
-| **Adds Columns?**    | Yes / No                                               |
-| **Modifies Schema?** | Yes / No                                               |
-| **Example Usage**    | Fenced `dsl` block                                     |
-| **Special Notes**    | Constraints / caveats                                  |
+1. **NL:**   
+   Find the total SP 500 volume executed in the first week of 2021\.  
+   **DSL:**   
+   `bars = FutureHistoricalBars("ES", “ONE_DAY”, “2021-01-03T00:00:00”,“2021-01-11T00:00:00”)`   
+   `result = GroupByStream(bars,”volume_Sum”)`  
+2. **NL:**   
+   **Find the highest price and lowest price of the  WTI Crude Oil futures over the first half of 2024**  
+   **DSL:**   
+   `bars = FutureHistoricalBars("CL", “ONE_DAY”, “2021-01-03T00:00:00”,“2021-01-06T00:00:00”)`   
+   `result = GroupByStream(bars,”high:Max low:Min”)`  
+   `result = ProjectStream(result,”high_max:highest_price min_low: lowest_price”)`  
+3. **NL:**   
+   **Find the Average daily volume per contract in Natural Gas in 2023**  
+   **DSL:**   
+   `bars = FutureHistoricalBars("NG*", “ONE_DAY”, “2023-01-03T00:00:00”,“2023-31-12T23:59:59”)`   
+   `result = GroupByStream(bars, “contract”, ”volume:Mean”)`  
+   `result = ProjectStream(result,”mean_volume:average_daily_volume”)`
 
 ---
 
-## 3 Data Sources
+#### **Schema Impact:**
 
-### 3.1 FutureHistoricalBarsStream
+The `AggregateStream` modifies the schema of the parent stream by adding new columns for each aggregation operation. Existing columns are unaffected.
 
-| Field                | Description                                                                                                                                                                                                |
-| :------------------- | :--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| **Type**             | DataSource                                                                                                                                                                                                 |
-| **Purpose**          | Fetch historical OHLCV bars for futures contracts                                                                                                                                                          |
-| **Arguments**        | • `contract` — continuous (`ES`), specific (`ESZ9`), or wildcard (`ES*`) • \``bar_size`\` — ONE\_DAY, ONE\_MIN • `` `range_begin` `` — ISO datetime, optional • `` `range_end` `` — ISO datetime, optional |
-| **Output Schema**    | `id, ts_event, date, contract, open, high, low, close, volume`                                                                                                                                             |
-| **Adds Columns?**    | Yes                                                                                                                                                                                                        |
-| **Modifies Schema?** | No                                                                                                                                                                                                         |
-| **Example Usage**    | `bars = FutureHistoricalBarsStream("ES", "ONE_DAY", "2023-01-01", "2023-12-31")`                                                                                                                           |
-| **Special Notes**    | Whereas the ts\_event column is the timestamp of the column, date is the date of the day.                                                                                                                  |
+# **SingleRowMathStream (Operator Stream)**
 
-### 3.2 LiveBBOStream
-
-| Field                | Description                                                                                                                                                                |
-| :------------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| **Type**             | DataSource                                                                                                                                                                 |
-| **Purpose**          | Stream live best-bid/offer quotes                                                                                                                                          |
-| **Arguments**        | • `contract` — e.g. `"NGU5"` or `"CL*"`                                                                                                                                    |
-| **Output Schema**    | `ts_event, contract, bid_price, bid_size, ask_price, ask_size`                                                                                                             |
-| **Adds Columns?**    | Yes                                                                                                                                                                        |
-| **Modifies Schema?** | No                                                                                                                                                                         |
-| **Example Usage**    | `bbo = LiveBBOStream("NGU5")`                                                                                                                                              |
-| **Special Notes**    | LiveBBOStream produces the latest value of the bid/ask price and size. It does not maintain history. Each new row overwrites the old one. For history use TimeSeriesStream |
-
-### 3.3 LiteralStream
-
-| Field                | Description                                                                                                       |
-| :------------------- | :---------------------------------------------------------------------------------------------------------------- |
-| **Type**             | DataSource                                                                                                        |
-| **Purpose**          | Inject small static tables via JSON records                                                                       |
-| **Arguments**        | • `json_text` — string                                                                                            |
-| **Output Schema**    | Inferred from JSON keys                                                                                           |
-| **Adds Columns?**    | Yes                                                                                                               |
-| **Modifies Schema?** | No                                                                                                                |
-| **Example Usage**    | `meta = LiteralStream('[{"event":"OPEC","date":"2024-06-01"}]')`                                                  |
-| **Special Notes**    | If you download the data from the internet, and want to use it conjunction with the BlueOrbDSL this can be useful |
+The `SingleRowMathStream` is a DSL operator used to perform math operations, one row at a time, on the parent stream. 
 
 ---
 
-## 4 Operators
+#### **Stream arguments (parameters):**
 
-### 4.1 FilterStream
+1. **parent\_stream (another stream):**The source stream provides the base data  
+2. **math\_expression(string):** This is a math expression which allows the creation of one or more new columns that are the output of a math expression on the existing columns provided by parent\_stream. This can a simple math operation like new\_col \= a \+ b, or any function from C\#’s System.Math namespace, like new\_column \= Math.Log(Math.Pow(volume, 1.1))
 
-| Field                | Description                                                                                                                  |
-| :------------------- | :--------------------------------------------------------------------------------------------------------------------------- |
-| **Type**             | Operator                                                                                                                     |
-| **Purpose**          | Keep rows where a boolean condition is true                                                                                  |
-| **Arguments**        | • `parent_stream` •`` `condition` `` — string                                                                                |
-| **Output Schema**    | Same as input                                                                                                                |
-| **Adds Columns?**    | No                                                                                                                           |
-| **Modifies Schema?** | No                                                                                                                           |
-| **Special Notes**    | Condition syntax: use ASCII single quotes (`contract = 'ESZ9'`). Supports `and`, `or`, parentheses, equality and inequality. |
+#### **Examples:**
 
-### 4.2 ProjectStream
+4. **NL:**   
+   Find the daily price movement in the SP 500 Futures through the month of Jan  
+   **DSL:**   
+   `bars = FutureHistoricalBars("ES", “ONE_DAY”, “2024-01-01T00:00:00”,“2024-01-31T00:00:00”)`   
+   `result = SingleRowMathStream(bars,”daily_move=close-open”)`  
+5. **NL:**   
+   Find the daily typical price of the  WTI Crude Oil futures over the first half of 2024  
+   **DSL:**   
+   `bars = FutureHistoricalBars("CL", “ONE_DAY”, “2021-01-03T00:00:00”,“2021-01-06T00:00:00”)`   
+   `result = SingleRowMathStream(bars, “typical_price = (high+low+close)/3.0”)`
 
-| Field                | Description                                                                         |
-| :------------------- | :---------------------------------------------------------------------------------- |
-| **Type**             | Operator                                                                            |
-| **Purpose**          | Rename or drop columns                                                              |
-| **Arguments**        | • `parent_stream` • `` `column_map ``\` — string                                    |
-| **Output Schema**    | Renamed subset                                                                      |
-| **Adds Columns?**    | No                                                                                  |
-| **Modifies Schema?** | Yes                                                                                 |
-| **Example Usage**    | `clean = ProjectStream(bars, "close:settlement, volume")`                           |
-| **Special Notes**    | **Cannot add new columns or constants.** Use LiteralStream \+ JoinStream if needed. |
+#### **Input Space (Scope):** The scope of this operation is a **single row.** The SingleRowMAthStream as the name signifies, operates on columns of the same row only. **No multi row operations are allowed.** For multi row operations using what follows.
+
+**Schema Impact:**
+
+# **MultiRowRollingMathStream (Operator Stream)**
+
+The `MultiRowRollingMathStream` function is a DSL operator used to perform mostly math operations on a rolling set of rows of a parent stream. It works similarly to the GroupByStream (described above) but instead of grouping rows together based on a column value, it operates on a rolling set of rows (where the rolling size is provided as an argument). 
 
 ---
 
-### 4.3 GroupByStream
+#### **Stream arguments (parameters):**
 
-| Field                | Description                                                                                            |
-| :------------------- | :----------------------------------------------------------------------------------------------------- |
-| **Type**             | Operator                                                                                               |
-| **Purpose**          | Aggregates rows by key (or globally)                                                                   |
-| **Arguments**        | • `parent_stream` • `` `group_by` `` — string (blank for global) • `` `aggregation_string` `` — string |
-| **Output Schema**    | Adds `<column>_<aggregation>` columns                                                                  |
-| **Adds Columns?**    | Yes                                                                                                    |
-| **Modifies Schema?** | Yes                                                                                                    |
-| **Example Usage**    | `daily = GroupByStream(bars, "date", "volume:Sum")`                                                    |
-| **Special Notes**    | None                                                                                                   |
-
-### 4.4 SingleRowMathStream
-
-| Field                | Description                                                                  |
-| :------------------- | :--------------------------------------------------------------------------- |
-| **Type**             | Operator                                                                     |
-| **Purpose**          | Add new columns from row-wise math expressions                               |
-| **Arguments**        | • `parent_stream` • `` `math_expression` `` — string                         |
-| **Output Schema**    | Adds new column(s)                                                           |
-| **Adds Columns?**    | Yes                                                                          |
-| **Modifies Schema?** | No                                                                           |
-| **Example Usage**    | `r = SingleRowMathStream(bars, "daily_return = close - open")`               |
-| **Special Notes**    | Operates on one row at a time only. Cannot reference values from other rows. |
-
-### 4.5 MultiRowRollingMathStream
-
-| Field                | Description                                                                                                                                 |
-| :------------------- | :------------------------------------------------------------------------------------------------------------------------------------------ |
-| **Type**             | Operator                                                                                                                                    |
-| **Purpose**          | Aggregates over rolling window of rows                                                                                                      |
-| **Arguments**        | • `parent_stream` • `` `aggregation_string` `` — string • `` `sort_column` `` — string with \+/- prefix • `` `rolling_window_size` `` — int |
-| **Output Schema**    | Adds `<column>_<aggregation>` columns                                                                                                       |
-| **Adds Columns?**    | Yes                                                                                                                                         |
-| **Modifies Schema?** | Yes                                                                                                                                         |
-| **Example Usage**    | `ma20 = MultiRowRollingMathStream(bars, "close:Mean", "-ts_event", 20)`                                                                     |
-| **Special Notes**    | sort\_column should be prefixed with \+ (ascending) or \- (descending). Example: "-ts\_event".                                              |
-
-### 4.6 UnionStream
-
-| Field                | Description                                              |
-| :------------------- | :------------------------------------------------------- |
-| **Type**             | Operator                                                 |
-| **Purpose**          | Append rows from two streams                             |
-| **Arguments**        | • `parent_stream1` • `parent_stream2`                    |
-| **Output Schema**    | Same as inputs                                           |
-| **Adds Columns?**    | No                                                       |
-| **Modifies Schema?** | No                                                       |
-| **Example Usage**    | `unioned = UnionStream(bars1, bars2)`                    |
-| **Special Notes**    | **Schemas must match exactly (column names AND types).** |
-
-### 4.7 JoinStream
-
-| Field                | Description                                                                                                                                                           |
-| :------------------- | :-------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| **Type**             | Operator                                                                                                                                                              |
-| **Purpose**          | Join two streams on equality of selected columns                                                                                                                      |
-| **Arguments**        | • `` `left_parent_stream` ``   • `` `left_join_cols` `` — comma-separated string   • `` `right_parent_stream` ``   • `` `right_join_cols` `` — comma-separated string |
-| **Output Schema**    | Combined columns from both streams                                                                                                                                    |
-| **Adds Columns?**    | Yes                                                                                                                                                                   |
-| **Modifies Schema?** | Yes                                                                                                                                                                   |
-| **Example Usage**    | `joined = JoinStream(a, "ts_event", b, "ts_event")`                                                                                                                   |
-| **Special Notes**    | Always performs an **inner join** on equality. If columns overlap, prefix applied (`left_`, `right_`). If join columns args left empty, does cartesian product        |
-
-### 4.8 DateTimeExpanderStream
-
-| Field                | Description                                                                                                                        |
-| :------------------- | :--------------------------------------------------------------------------------------------------------------------------------- |
-| **Type**             | Operator                                                                                                                           |
-| **Purpose**          | Adds columns with date/time components from a datetime column                                                                      |
-| **Arguments**        | • `parent_stream` • `datetime_column` — string                                                                                     |
-| **Output Schema**    | Adds: • \`\_date\` • \`\_day\_of\_week\` • \`\_day\_of\_month\` • \`\_month\` • \`\_hour\` • \`\_minute\` • \`\_seconds\_of\_day\` |
-| **Adds Columns?**    | Yes                                                                                                                                |
-| **Modifies Schema?** | Yes                                                                                                                                |
-| **Example Usage**    | `expanded = DateTimeExpanderStream(bars, "ts_event")`                                                                              |
-| **Special Notes**    | None                                                                                                                               |
-
-### 4.9 ContractExpanderStream
-
-| Field                | Description                                           |
-| :------------------- | :---------------------------------------------------- |
-| **Type**             | Operator                                              |
-| **Purpose**          | Adds metadata columns based on `contract`             |
-| **Arguments**        | • `parent_stream` • contract\_column — string         |
-| **Output Schema**    | Adds: `• contract_type`                               |
-| **Adds Columns?**    | Yes                                                   |
-| **Modifies Schema?** | Yes                                                   |
-| **Example Usage**    | `expanded = ContractExpanderStream(bars, "contract")` |
-| **Special Notes**    | None                                                  |
-
-### 4.10 CountStream
-
-| Field                | Description                       |
-| :------------------- | :-------------------------------- |
-| **Type**             | Operator                          |
-| **Purpose**          | Count number of rows              |
-| **Arguments**        | • `parent_stream`                 |
-| **Output Schema**    | `id, count`                       |
-| **Adds Columns?**    | Yes                               |
-| **Modifies Schema?** | Yes                               |
-| **Example Usage**    | `total_count = CountStream(bars)` |
-| **Special Notes**    | None                              |
-
-### 4.11 TimeSeriesStream
-
-| Field                | Description                                                          |
-| :------------------- | :------------------------------------------------------------------- |
-| **Type**             | Operator                                                             |
-| **Purpose**          | Append each update from parent stream into time series               |
-| **Arguments**        | • `parent_stream` • \``buffer_size`\` — int                          |
-| **Output Schema**    | Parent schema \+ `timestamp` (if not already present)                |
-| **Adds Columns?**    | Yes                                                                  |
-| **Modifies Schema?** | No                                                                   |
-| **Example Usage**    | `ts_bbo = TimeSeriesStream(bbo, 10000)`                              |
-| **Special Notes**    | Append-only stream, does not deduplicate or collapse repeated values |
-
-### 4.12 TimeBasedRollingFilterStream
-
-| Field                | Description                                                                  |
-| :------------------- | :--------------------------------------------------------------------------- |
-| **Type**             | Operator                                                                     |
-| **Purpose**          | Keep rows newer than rolling window of N seconds                             |
-| **Arguments**        | • `parent_stream` • \``no_of_seconds`\` — int                                |
-| **Output Schema**    | Same as input                                                                |
-| **Adds Columns?**    | No                                                                           |
-| **Modifies Schema?** | No                                                                           |
-| **Example Usage**    | `recent = TimeBasedRollingFilterStream(trades, 60)`                          |
-| **Special Notes**    | Drops rows older than `no_of_seconds` compared to current max timestamp seen |
-
-### 4.13 PythonDataframeStream
-
-| Field                | Description                                                                                                             |
-| :------------------- | :---------------------------------------------------------------------------------------------------------------------- |
-| **Type**             | Operator                                                                                                                |
-| **Purpose**          | Apply arbitrary Python transformation to DataFrame                                                                      |
-| **Arguments**        | • `parent_stream` • `` `python_function` `` — string                                                                    |
-| **Output Schema**    | Defined by function                                                                                                     |
-| **Adds Columns?**    | Yes/No — depends on function                                                                                            |
-| **Modifies Schema?** | Yes/No — depends on function                                                                                            |
-| **Example Usage**    | `r = PythonDataframeStream(bars, "def transform(df): return df[df.volume > 1e6]")`                                      |
-| **Special Notes**    | **Use only as last resort.** Function should be small and well-documented. Avoid large multi-step logic where possible. |
+1. **Parent Stream (another stream):** The source stream provides the data to be operated on  
+2. **Operation Text (string):** A string defining the column(s) to be operated and the operation(s) to apply, in the format:  
+   `<column_name: operation>, <column_name: operation>, …` Each `<column_name: operation>` pair produces a new column in the output stream. The new column name is formatted as `<column_name>_<operation>` in lowercase. This is like the GroupByStream’s operations string.   
+3. **Sort/Order Column Name(string):** This controls the ordering of the rolling rows. This argument will be in the format \-`<column_name>` to sort the rolling rows in descending order by the `<column_name>`, or \+`<column_name> in` ascending order.  
+4. **Rolling Window Size(int)(optional):** The size of the rolling window, If this argument is left out all, the operations are applied to all rows, is Global Operation instead of Rolling Window Operation (it's like the optional groupBy col in GroupByStream).
 
 ---
 
-## 5 Reserved Words & Enumerations
+#### **Valid Aggregation Operations:**
 
-- **Bar\_size:** `ONE_DAY`, `ONE_MIN`  
-- **Aggregation Operations:** `Mean`, `Median`, `Mode`, `Sum`, `Max`, `Min`, `StdDev`, `Range`, `First`, `Last`, `EdgeDiff`, `EdgeDiffPercentage`
+* `Mean`: Arithmetic mean of the values.  
+* `Median`: Median of the values.  
+* `Mode`: Most frequently occurring value.  
+* `Range`: Difference between the max and min.  
+* `StdDev`: Standard deviation of the values.  
+* `Max`: Maximum value.  
+* `Min`: Minimum value.  
+* `First`: First value in the group.  
+* `Last`: Last value in the group.  
+* `Sum`: Sum of the values.  
+* `EdgeDiff`: Difference between the edge values, meaning the difference between a certain col belonging to the first and last rows respectively  
+* `EdgeDiffPercentage`: Same as `EdgeDiff` but expressed as percentage instead
 
 ---
 
-## 6 Common Pitfalls
+#### **Examples:**
 
-| Pitfall                                         | Correct Approach                                     |
-| :---------------------------------------------- | :--------------------------------------------------- |
-| Adding literal columns in ProjectStream         | Use LiteralStream → JoinStream                       |
-| Smart quotes in conditions                      | Use `'...'` ASCII quotes                             |
-| Large functions in PythonDataframeStream        | Keep minimal or move to appendix                     |
-| Ambiguous TimeBasedRollingFilterStream behavior | Now clearly stated: drops rows older than N seconds. |
+1. **NL:**   
+   **Find the 20 d MA of SP 500 Futures close prices**  
+   `bars = FutureHistoricalBars("ES", “ONE_DAY”, “2021-01-03T00:00:00”,“2021-01-11T00:00:00”)`   
+   `result = MultirowRollingMathStream(bars, “close:Mean”, “-ts_event”, 20)`  
+2. **NL:**   
+   **Find the 90d Standard Deviation of WTI Crude Oil during 2023**
 
-## 7 Reinforced Learning DSL Examples
+   **DSL:**   
+   `bars = FutureHistoricalBars("CL", “ONE_DAY”, “2023-01-01T00:00:00”,“2023-12-31T00:00:00”)`  
+   `// calculating typical_price`
 
-This section contains **complete worked examples** of BlueOrbDSL pipelines answering complex natural language queries.
+   `result = SingleRowMathStream(bars, “typical_price = (high+low+close)/3.0”)`   
+   `result = MultirowRollingMathStream(bars,“typical_price:StdDev”, “-ts_event”, 90)`
 
-Each example shows:
+   `result=ProjectStream(result, “stddev_typical_price:90dStdDev”)`
 
-- The **NL Query** (natural language question)  
-- The **DSL Solution** in full  
-- Optional **Notes**
+3. **NL:**   
+   **Find the Average daily volume of Natural Gas in 2023**  
+   **DSL:**   
+   `bars = FutureHistoricalBars("NG", “ONE_DAY”, “2023-01-03T00:00:00”,“2023-31-12T23:59:59”)`  
+   `result = MultirowRollingMathStream(bars, ”volume:Mean”, “-ts_event”)`  
+   `result = ProjectStream(result,”mean_volume:average_daily_volume”)`
 
 ---
 
-### Example 1 — On which dates did SP 500 Futures open 2% higher than the previous day's close?
+#### **Schema Impact:**
 
-#### DSL Solution
+The `MultirowRollingMathStream` modifies the schema of the parent stream by adding new columns for each operation. Existing columns are unaffected.
 
-```
-bars = FutureHistoricalBarsStream("ES", "ONE_DAY", "1900-01-01T00:00:00", "2100-12-31T23:59:59")
-rolling_bars = MultiRowRollingMathStream(bars, "close:First, open:Last", "+ts_event", 2)
-percent_change = SingleRowMathStream(rolling_bars, "percent_change = 100 * (open_last - close_first) / close_first")
-filtered_bars = FilterStream(percent_change, "percent_change >= 2")
-result = ProjectStream(filtered_bars, "ts_event, percent_change")
-```
+# **UnionStream (Operator Stream)**
 
----
+Like the SQL Union appends all the rows from 2 streams together.
 
-### Example 2 — What is the average price movement for e-minis on Wednesday between 9:30 am and 9:40 AM CST?
+#### **Stream arguments (parameters):**
 
-#### DSL Solution
+5. `parent_stream1 (first stream)` This stream provides part of the data that will will be appended to the other part, provided by the following argument  
+6. `parent_stream2 (second stream)` This stream provides the other part of the data that will will be appended to the first part, provided by the first argument
 
-```
-bars = FutureHistoricalBarsStream("ES", "ONE_MIN", "1900-01-01T00:00:00", "2100-12-31T23:59:59")
-expanded_bars = DateTimeExpanderStream(bars, "ts_event")
-filtered_bars = FilterStream(expanded_bars, "ts_event_day_of_week = 'Wednesday' and ts_event_hour = 9 and ts_event_minute >= 30 and ts_event_minute < 40")
-price_movement = SingleRowMathStream(filtered_bars, "movement = close - open")
-average_movement = GroupByStream(price_movement, "", "movement:Mean")
-result = ProjectStream(average_movement, "movement_mean:average_price_movement")
-```
+#### **Examples:**
 
----
+4. **NL prompt:** Show me all the bars across SP 500 Futures and Nasdaq Futures  
+   **DSL:** `r = UnionStream(FutureHistoricalBars("ES", "ONE_DAY"), FutureHistoricalBars("NQ", "ONE_DAY"))`  
+   
 
-### Example 3 — What did the volume and price of ZN in the 5 minutes following the last 10 FOMC rates?
+#### **Schema Impact:**
 
-#### DSL Solution
+None
 
-```
-bars_1 = FutureHistoricalBarsStream("ZN", "ONE_MIN", "2023-11-01T14:00:00", "2023-11-01T14:05:00")
-bars_2 = FutureHistoricalBarsStream("ZN", "ONE_MIN", "2023-09-20T14:00:00", "2023-09-20T14:05:00")
-bars_3 = FutureHistoricalBarsStream("ZN", "ONE_MIN", "2023-07-26T14:00:00", "2023-07-26T14:05:00")
-bars_4 = FutureHistoricalBarsStream("ZN", "ONE_MIN", "2023-06-14T14:00:00", "2023-06-14T14:05:00")
-bars_5 = FutureHistoricalBarsStream("ZN", "ONE_MIN", "2023-05-03T14:00:00", "2023-05-03T14:05:00")
-bars_6 = FutureHistoricalBarsStream("ZN", "ONE_MIN", "2023-03-22T14:00:00", "2023-03-22T14:05:00")
-bars_7 = FutureHistoricalBarsStream("ZN", "ONE_MIN", "2023-02-01T14:00:00", "2023-02-01T14:05:00")
-bars_8 = FutureHistoricalBarsStream("ZN", "ONE_MIN", "2022-12-14T14:00:00", "2022-12-14T14:05:00")
-bars_9 = FutureHistoricalBarsStream("ZN", "ONE_MIN", "2022-11-02T14:00:00", "2022-11-02T14:05:00")
-bars_10 = FutureHistoricalBarsStream("ZN", "ONE_MIN", "2022-09-21T14:00:00", "2022-09-21T14:05:00")
+# **DateTimeExpanderStream (Operator Stream)**
 
-all_bars = UnionStream(bars_1, bars_2)
-all_bars = UnionStream(all_bars, bars_3)
-all_bars = UnionStream(all_bars, bars_4)
-all_bars = UnionStream(all_bars, bars_5)
-all_bars = UnionStream(all_bars, bars_6)
-all_bars = UnionStream(all_bars, bars_7)
-all_bars = UnionStream(all_bars, bars_8)
-all_bars = UnionStream(all_bars, bars_9)
-all_bars = UnionStream(all_bars, bars_10)
+DateTimExpanderStream expands one datetime column into various datetime components, like seconds in day, day of week, date etc, so new columns are added for those. The existing columns remain as is.
 
-grouped_data = GroupByStream(all_bars, "date", "open:First, close:Last, high:Max, low:Min, volume:Sum")
-result = ProjectStream(grouped_data, "ts_event_date, open_first:5_min_open, close_last:5_min_close, high_max:5_min_high, low_min:5_min_low, volume_sum:5_min_volume")
-```
+#### **Stream arguments (parameters):**
 
----
+1. `parent_stream1 (base stream)` This parent stream that provides the data, including the datetime columns that will be expanded into multiple columns  
+2. `Datetime_input_column (string)` This is the column that will be expanded into multiple columns.
 
-### Example 4 — How much volume of RBOB typically trades during the settlement window?
+**New Columns:**  
+`<input_dt_col>_date` (Date of the value, without the time part)
 
-#### DSL Solution
+* `<input_dt_col>_day,` (Day of the week)  
+* `<input_dt_col>_day_of_month` (day of the month, This is an int between 0 and 31\)  
+* `<input_dt_col>_month,` (month of the year, this is a numerical value between 1 and 12  
+* `input_dt_col>_hour,` (Hour of the day, value bw 0 and 23\)  
+* `<input_dt_col>_min",` (month part of this DateTime. Value is integer between 1 and 12).  
+* `<input_dt_col>_seconds_of_day",` (Total seconds of the day) 
 
-```
-rbob_bars = FutureHistoricalBarsStream("RB", "ONE_MIN", "2023-01-01T00:00:00", "2023-12-31T23:59:59")
-expanded_bars = DateTimeExpanderStream(rbob_bars, "ts_event")
-settlement_window = FilterStream(expanded_bars, "(ts_event_hour = 14 and ts_event_minute >= 28) or (ts_event_hour = 14 and ts_event_minute < 30)")
-typical_settlement_volume = GroupByStream(settlement_window, "ts_event_date", "volume:Sum")
-result = ProjectStream(typical_settlement_volume, "ts_event_date, volume_sum:daily_settlement_volume")
-```
+#### **Examples:**
 
----
+**DSL:** `r = DateTimeExpanderStream(FutureHistoricalBars("ES", "ONE_DAY"), “ts_event”)`   
+**Input Schema:**
 
-### Example 5 — How much SP Futures during the settlement window, average per day, during the 2 week windows before future expiration, in 2023?
+| id   | ts\_event | contract | open | high | low  | close | volume |
+| :--- | :-------- | :------- | :--- | :--- | :--- | :---- | :----- |
 
-#### DSL Solution
+**Output Schema:**
 
-```
-sp_bars = FutureHistoricalBarsStream("ES", "ONE_MIN", "2023-01-01T00:00:00", "2023-12-31T23:59:59")
-expanded_bars = DateTimeExpanderStream(sp_bars, "ts_event")
-settlement_window = FilterStream(expanded_bars, "(ts_event_hour = 15 and ts_event_minute >= 58) or (ts_event_hour = 16 and ts_event_minute < 1)")
-filtered_bars = FilterStream(settlement_window, "(ts_event_date >= '2023-03-03' and ts_event_date < '2023-03-17') or (ts_event_date >= '2023-06-02' and ts_event_date < '2023-06-16') or (ts_event_date >= '2023-09-01' and ts_event_date < '2023-09-15') or (ts_event_date >= '2023-12-01' and ts_event_date < '2023-12-15')")
-daily_volume = GroupByStream(filtered_bars, "ts_event_date", "volume:Sum")
-average_daily_volume = GroupByStream(daily_volume, "", "volume_sum:Mean")
-result = ProjectStream(average_daily_volume, "volume_sum_mean:average_daily_volume")
-```
+| id   | ts\_event | contract | open | high | low  | close | volume |      |      |      |      |      |      |      |
+| :--- | :-------- | :------- | :--- | :--- | :--- | :---- | :----- | :--- | :--- | :--- | :--- | :--- | :--- | :--- |
 
----
-
-### Example 6 — Find the difference between the number of days in 2023 that SP 500 went up 2% and when it went down 2%
-
-#### DSL Solution
-
-```
-bars = FutureHistoricalBarsStream("ES", "ONE_DAY", "2023-01-01T00:00:00", "2023-12-31T23:59:59")
-percentage_change = SingleRowMathStream(bars, "percent_change = 100 * (close - open) / open")
-days_up_2_percent = FilterStream(percentage_change, "percent_change >= 2")
-days_up_count = CountStream(days_up_2_percent)
-days_down_2_percent = FilterStream(percentage_change, "percent_change <= -2")
-days_down_count = CountStream(days_down_2_percent)
-joined_counts = JoinStream(days_up_count, "", days_down_count, "")
-difference_result = SingleRowMathStream(joined_counts, "difference = days_up_count - days_down_count")
-result = ProjectStream(difference_result, "difference")
-```
-
----
-
-### Example 7 — Monitor which CL contracts (CL) are trading with bid size \>= 1000, and record the last 1000 events\*
-
-#### DSL Solution
-
-```
-bbo = LiveBBOStream("CL*")
-large_bid = FilterStream(bbo, "bid_size >= 1000")
-large_bid_ts = TimeSeriesStream(large_bid, 1000)
-result = ProjectStream(large_bid_ts, "timestamp, ts_event, contract, bid_size, bid_price, ask_price")
-```
-
----
-
-### Example 8 — Identify FutureSpreads in CrudeOil and monitor their spread width
-
-#### DSL Solution
-
-```
-bbo = LiveBBOStream("CL*")
-expanded_bbo = ContractExpanderStream(bbo, "contract")
-future_spreads = FilterStream(expanded_bbo, "contract_type = 'FutureSpread'")
-spread_stream = SingleRowMathStream(future_spreads, "spread = ask_price - bid_price")
-result = ProjectStream(spread_stream, "ts_event, contract, spread")
-```
-
----
-
-### Example 9 — Track when bid prices in NaturalGas Futures cross above 3.00 and keep only the last 10 seconds of such events
-
-#### DSL Solution
-
-```
-bbo = LiveBBOStream("NG")
-high_bid = FilterStream(bbo, "bid_price > 3.00")
-high_bid_ts = TimeSeriesStream(high_bid, 10000)
-recent_high_bid = TimeBasedRollingFilterStream(high_bid_ts, 10)
-result = ProjectStream(recent_high_bid, "ts_event, contract, bid_price, bid_size, ask_price")
-```
-
----
-
-### Example 11 — Detect sequences in NaturalGas future where bid\_price increased 3 times consecutively within a rolling 30-second window
-
-#### DSL Solution
-
-```
-bbo = LiveBBOStream("NG")
-bbo_ts = TimeSeriesStream(bbo, 10000)
-recent_bbo = TimeBasedRollingFilterStream(bbo, 30)  // 30-second window of updates
-
-result = PythonDataframeStream(recent_bbo, "
-def transform(df):
-    import pandas as pd
-    df = df.sort_values('ts_event')
-    df['bid_increase'] = df['bid_price'].diff() > 0
-    df['consec_increase'] = df['bid_increase'].rolling(window=3).sum()
-    # Find rows where we just reached 3 consecutive increases
-    df['trigger'] = (df['consec_increase'] == 3) & (df['consec_increase'].shift(1) < 3)
-    return df[df['trigger']][['ts_event', 'contract', 'bid_price', 'bid_size']]
-")
-```
-
----
-
-### Example 12 — When we get a 5+ level bid sweep in SP 500, show me how the bid behaves in the 10 minutes after — mean, stddev, min/max bid\_price
-
-#### DSL Solution
-
-```
-bbo = LiveBBOStream("ES")
-bbo_ts = TimeSeriesStream(bbo, 10000)
-recent_bbo = TimeSeriesStream(bbo, 5000)  // keep large enough window for event tracking
-
-result = PythonDataframeStream(recent_bbo, "
-def transform(df):
-    import pandas as pd
-    df = df.sort_values('ts_event')
-    rows = []
-    in_measurement = False
-    measurement_start_time = None
-    measurement_end_time = None
-    bid_prices_in_window = []
-    min_bid_size_threshold = 100  # defines what we consider 'stable large bid'
-    min_levels_stable = 5
-    prior_bid_sizes = []
-
-    for i, row in df.iterrows():
-        bid_price = row['bid_price']
-        bid_size = row['bid_size']
-        ts = pd.to_datetime(row['ts_event'])
-
-        # Maintain rolling window of prior bid_sizes
-        prior_bid_sizes.append(bid_size)
-        if len(prior_bid_sizes) > min_levels_stable:
-            prior_bid_sizes.pop(0)
-
-        if not in_measurement:
-            # Detect sweep trigger
-            if all(x > min_bid_size_threshold for x in prior_bid_sizes[:-1]) and bid_size < min_bid_size_threshold:
-                # Sweep detected → start measurement window
-                measurement_start_time = ts
-                measurement_end_time = measurement_start_time + pd.Timedelta(minutes=10)
-                bid_prices_in_window = []
-                in_measurement = True
-        else:
-            # Inside measurement window
-            if ts <= measurement_end_time:
-                bid_prices_in_window.append(bid_price)
-            else:
-                # Window complete → emit stats
-                if bid_prices_in_window:
-                    mean_bid = pd.Series(bid_prices_in_window).mean()
-                    stddev_bid = pd.Series(bid_prices_in_window).std()
-                    min_bid = pd.Series(bid_prices_in_window).min()
-                    max_bid = pd.Series(bid_prices_in_window).max()
-                    rows.append({
-                        'sweep_time': measurement_start_time,
-                        'measurement_end': measurement_end_time,
-                        'mean_bid_price': mean_bid,
-                        'stddev_bid_price': stddev_bid,
-                        'min_bid_price': min_bid,
-                        'max_bid_price': max_bid
-                    })
-                # Reset state
-                in_measurement = False
-                measurement_start_time = None
-                measurement_end_time = None
-                bid_prices_in_window = []
-                prior_bid_sizes = []
-
-    return pd.DataFrame(rows)
-")
-
-```
-
-*End of Section 7 — Reinforced Learning DSL Examples*  

src/DataStreamSuite/SharedStreams/NL/DSL_meta_prompt,md

@@ -1,27 +0,0 @@
-**Enter Meta Mode**  
- You are not learning the DSL yet.  
- You are reviewing the following **BlueOrbDSL Master Learning Prompt** (MLP) as a *training document for yourself*.
-
-Your goal is to optimize it to make it easier for you — or any future LLM — to learn from it and generate DSL code with high accuracy.
-
-Please analyze the MLP and provide:
-
-1. **Clarity review** — is each stream definition clear, unambiguous, easy to parse?
-
-2. **Consistency review** — do all stream definitions follow the same structure and style?
-
-3. **Terminology check** — are key terms (parent stream, arguments, schema impact, example usage, etc.) used consistently?
-
-4. **Structural feedback** — is the overall document structured in a way that supports learning (headings, bold, tables, code blocks)?
-
-5. **Formatting feedback** — are important warnings / "cannot do X" constraints highlighted clearly?
-
-6. **Missing info** — is anything missing from any stream definition that would help you understand how to use it?
-
-7. **Generation risk spots** — any parts where you foresee yourself making common mistakes due to ambiguity?
-
-**Do not attempt to "learn" this DSL yet.** Stay in Meta Mode and provide feedback as an *editor and optimizer*.
-
-When I later say:  
- → **Switch to Learning Mode — this is now the final BlueOrbDSL MLP** → you will forget your Meta comments and treat the MLP as the single authoritative source for DSL generation.
-

src/DataStreamSuite/SharedStreams/NL/NLStream.cs

@@ -2,38 +2,34 @@ namespace bts.SilverPine {
    using BCG;
 
    using bts;
-   //using bts.SilverPine;
+   using bts.SilverPine;
 
-   namespace bts.BlueOrb {
+   [DataStreamUserRespresentation("nl")]
+   public class NLStream : ViewStream {
+      private readonly string _nl_prompt;
+      private ViewStream _nl_to_dsl_stream, _dsl_stream;
+      public NLStream(string nl_prompt) {
+         _nl_prompt = nl_prompt;
+      }
 
-      [DataStreamUserRespresentation("nl")]
-      public class NLStream : ViewStream {
-         private readonly string _nl_prompt;
-         private ViewStream _nl_to_dsl_stream, _dsl_stream;
-         public NLStream(string nl_prompt) {
-            _nl_prompt = nl_prompt;
-         }
+      protected override void myOnFirstSubscriber() {
+         _nl_to_dsl_stream = new NLToDSLStream(_nl_prompt);
+         _nl_to_dsl_stream.onChange += _handle_dsl_callback;
+      }
 
-         protected override void myOnFirstSubscriber() {
-            _nl_to_dsl_stream = new NLToDSLStream(_nl_prompt);
-            _nl_to_dsl_stream.onChange += _handle_dsl_callback;
-         }
+      private void _handle_dsl_callback(object dp_obj) {
+         DataPacket dp = dp_obj as DataPacket;
+         var r = dp.newRows.to_list().ElementAt(0);
+         string dsl = r.get<string>("response");
+         Log.Gen.info($"NLStream._handle_dsl_callback dsl: {dsl}");
+         _dsl_stream = new DSLStream(dsl);
+         _dsl_stream.onChange += _handle_data_cb;
+      }
 
-         private void _handle_dsl_callback(object dp_obj) {
-            DataPacket dp = dp_obj as DataPacket;
-            var r = dp.newRows.to_list().ElementAt(0);
-            string dsl = r.get<string>("response");
-            Log.Gen.info($"NLStream._handle_dsl_callback dsl: {dsl}");
-            _dsl_stream = new DSLStream(dsl);
-            _dsl_stream.onChange += _handle_data_cb;
-         }
-
-         private void _handle_data_cb(object dp_obj) {
-            var dp = dp_obj as DataPacket;
-            DataPacket dp_out = BlueDataUtility.build_dp_inc(dp.newRows, myLastValue as DataPacket);
-            myFireEvent(dp_out);
-         }
+      private void _handle_data_cb(object dp_obj) {
+         var dp = dp_obj as DataPacket;
+         DataPacket dp_out = BlueDataUtility.build_dp_inc(dp.newRows, myLastValue as DataPacket);
+         myFireEvent(dp_out);
       }
    }
-
-}
+}

src/Spark.Admin.WebApp/src/Accounts/Types.fs

@@ -17,7 +17,7 @@ type Tab =
   | PositionManagementTab
   | MarginTab
 
-let venues = [| Venue.BINANCE; Venue.BinanceUs; Venue.BinanceUsdsFutures; Venue.BYBIT; Venue.CBOE; Venue.COINBASE; Venue.DERIBIT; Venue.GATEIO; Venue.OKX; Venue.GLBX; Venue.ICE; Venue.TAIFEX; Venue.TWSE; Venue.SIM; |]
+let venues = [| Venue.BINANCE; Venue.BinanceUs; Venue.BinanceUsdsFutures; Venue.CBOE; Venue.COINBASE; Venue.DERIBIT; Venue.OKX; Venue.GLBX; Venue.ICE; Venue.TAIFEX; Venue.TWSE; Venue.SIM; |]
 
 let securityTypes = [| SecurityType.SPOT; SecurityType.FUTURE; SecurityType.OPTION; SecurityType.SPREAD; |]
 
@@ -50,11 +50,6 @@ type BinanceUsdsFUTURESExchangeField =
   | Origin of CustomerOrFirm
   | ClearingAccount of string
 
-type BybitExchangeField =
-  | CTI of CTICode
-  | Origin of CustomerOrFirm
-  | ClearingAccount of string
-
 type COINBASEExchangeField =
   | CTI of CTICode
   | Origin of CustomerOrFirm
@@ -65,11 +60,6 @@ type DERIBITExchangeField =
   | Origin of CustomerOrFirm
   | ClearingAccount of string
 
-type GateioExchangeField =
-  | CTI of CTICode
-  | Origin of CustomerOrFirm
-  | ClearingAccount of string
-
 type ICEExchangeField =
   | CTI of CTICode
   | Origin of CustomerOrFirm

src/Spark.Admin.WebApp/src/Accounts/View.fs

@@ -797,8 +797,6 @@ let venueToString (venue:bts.api.Venue) =
     | bts.api.Venue.BINANCE -> "BINANCE"
     | bts.api.Venue.BinanceUs -> "BinanceUs"
     | bts.api.Venue.BinanceUsdsFutures -> "BinanceUsdsFutures"
-    | bts.api.Venue.BYBIT -> "BYBIT"
-    | bts.api.Venue.GATEIO -> "GATEIO"
     | bts.api.Venue.FTX -> "FTX"
     | bts.api.Venue.OKX -> "OKX"
     | bts.api.Venue.TAIFEX -> "TAIFEX"

src/app_sparkd/CMakeLists.txt

@@ -107,25 +107,39 @@ target_link_directories(
 # The dirs are relative to sparkd's path, represented by "$ORIGIN".
 set_target_properties(
   ${PROJECT_NAME} PROPERTIES
-  BUILD_RPATH "$ORIGIN/../lib_spark;$ORIGIN/../lib_python_interop"
+  BUILD_RPATH "$ORIGIN/../lib_spark;$ORIGIN/../lib_python_md"
   INSTALL_RPATH "$ORIGIN/../${CMAKE_INSTALL_LIBDIR}"
 )
 
-include(${CMAKE_SOURCE_DIR}/cmake/SymLink.cmake)
-include(${CMAKE_SOURCE_DIR}/cmake/SymLinkPythonModule.cmake)
-# Py interop
-sym_link_python_module(
-  LIB_NAME python_interop
-  MODULE_NAME python_interop_module
-  PLUGIN_NAME python_interop_plugins
+# Python interop (for python md providers)
+add_dependencies(${PROJECT_NAME}
+  python_md
+  python_md_module
+  python_md_providers
+  copy_python_md_providers
 )
+include(${CMAKE_SOURCE_DIR}/cmake/SymLink.cmake)
+
+sym_link_local_lib(python_md)
+sym_link_file(
+  SOURCE ${CMAKE_BINARY_DIR}/src/lib_python_md/python_md_module.cpython-310-x86_64-linux-gnu.so
+  DEST ${CMAKE_CURRENT_BINARY_DIR}/python_md_module.cpython-310-x86_64-linux-gnu.so
+  INSTALL_DEST ${CMAKE_INSTALL_BINDIR}/
+)
+get_target_property(providers python_md_providers INTERFACE_PYTHON_SCRIPTS_DIR)
+sym_link_dir(
+  TARGET_NAME link_python_md_providers
+  SOURCE ${providers}
+  DEST ${CMAKE_CURRENT_BINARY_DIR}/python_md_providers
+)
+
 target_link_libraries(
   sparkd
   spark
-  binance
+  binance_execution
   okx_execution
-  python_interop
-  python_interop_module
+  python_md
+  python_md_module
   databento::databento
   ${emumba-deribit-connector-library_LIBRARIES}
   tbb

src/app_sparkd/Dockerfile

@@ -1,53 +1,19 @@
-ARG IMAGE="sparkd"
-ARG CONFIG_DIR="src/app_sparkd/configurations/dev"
+FROM bluetradingsystems/linux_dotnet7
+
 ARG SOURCE_DIR="install/sparkd-release-gcc"
-ARG PYTHON_REQUIREMENTS="requirements.txt"
-
-# sparkd_base
-FROM bluetradingsystems/linux_dotnet7 AS sparkd_base
-ARG CONFIG_DIR
-ARG SOURCE_DIR
-ARG PYTHON_REQUIREMENTS
+ARG CONF_FILE="src/app_sparkd/spark.conf"
 
 USER "root"
-RUN apt-get update && apt-get install -y liblttng-ust-dev
-RUN apt-get update && apt-get install -y libpcap0.8-dev
-RUN apt-get update && apt-get install -y python3-dev
-RUN apt-get update && apt-get install -y python3-pip
-RUN python3 -m pip install --upgrade pip
-COPY "${PYTHON_REQUIREMENTS}" "/home/bts/bin/requirements.txt"
-RUN pip3 install -r "/home/bts/bin/requirements.txt"
+RUN apt-get update \
+  && apt-get install -y liblttng-ust-dev
+
+RUN apt-get update \
+  && apt-get install -y libpcap0.8-dev
+
+RUN apt-get update \
+  && apt-get install -y python3.10-dev
 
 USER "bts"
 COPY --chown=bts:bts "${SOURCE_DIR}" "."
-COPY --chown=bts:bts "${CONFIG_DIR}" "/home/bts/sparkdata/conf"
+COPY --chown=bts:bts "${CONF_FILE}" ".local/share/bts/sparkd/conf/spark.conf"
 WORKDIR "/home/bts/bin"
-
-# sparkd
-FROM sparkd_base AS sparkd
-
-# sparkd_md_replay
-FROM sparkd_base AS sparkd_md_replay
-ARG MD_RECORDING_DIR="src/app_sparkd/.md_recording"
-COPY --chown=bts:bts "${MD_RECORDING_DIR}" "/home/bts/sparkdata/data/md_recording"
-
-# sparkd_symbols - like sparkd, but doesn't copy a config dir into the image
-FROM bluetradingsystems/linux_dotnet7 AS sparkd_symbols
-ARG SOURCE_DIR
-ARG PYTHON_REQUIREMENTS="requirements.txt"
-
-USER "root"
-RUN apt-get update && apt-get install -y liblttng-ust-dev
-RUN apt-get update && apt-get install -y libpcap0.8-dev
-RUN apt-get update && apt-get install -y python3-dev
-RUN apt-get update && apt-get install -y python3-pip
-RUN python3 -m pip install --upgrade pip
-COPY "${PYTHON_REQUIREMENTS}" "/home/bts/bin/requirements.txt"
-RUN pip3 install -r "/home/bts/bin/requirements.txt"
-
-USER "bts"
-COPY --chown=bts:bts "${SOURCE_DIR}" "."
-WORKDIR "/home/bts/bin"
-
-# selected image
-FROM ${IMAGE}

src/app_sparkd/configurations/alphaworx_prod/spark.conf

@@ -112,7 +112,7 @@ OkxExeCfg {
     secret_key: "240038D7A28D7F9461247CFCB8E43710"
     api_key: "dc188532-c74c-4d40-affe-3c6bf4d0ca9a"
     passphrase: ":,7i%tA%ivfj%SCbX?Ut"
-    ws_url: "wss://ws.okx.com:8443"
+    ws_url: "wss://wsaws.okx.com:8443"
     rest_url: "https://www.okx.com"
     ws_ping_timeout_seconds: 10 
     ws_ping_interval_seconds: 25

src/app_sparkd/configurations/alphaworx_zen_hk_live-preview/.env

@@ -1,6 +0,0 @@
-SPARKD__CONFIG_DIR=src/app_sparkd/configurations/alphaworx_zen_hk_live-preview
-SPARKD__PYTHON_REQUIREMENTS=src/app_sparkd/configurations/alphaworx_zen_hk_live-preview/requirements.txt
-SPARKD__SHM_SIZE=8gb
-SPARKD__MEMORY=8gb
-SPARKD__IMAGE=sparkd_symbols
-SPARKD__RUNTIME_CONF_DIR=/home/bts/8000/sparkdata/conf

src/app_sparkd/configurations/alphaworx_zen_hk_live-preview/README.txt

@@ -1,6 +0,0 @@
-* Connects to Binance, Okx, and Bybit PRODUCTION sites with small accounts designated for test trades
-* Api keys NOT included
-* Configured for Alphaworx and Bts testing in a production environment
-
-To retrieve api keys, run "python init_env.py". It should dump them to a file
-called  ".env.local".

src/app_sparkd/configurations/alphaworx_zen_hk_live-preview/binance_1.json

@@ -1,9 +0,0 @@
-{
-  "api_key": "${BINANCE_API_KEY}",
-  "secret_key": "${BINANCE_SECRET_KEY}",
-  "account": "PROD",
-  "rest_endpoint": "https://api.binance.com",
-  "ws_endpoint": "wss://ws-api.binance.com:443/ws-api/v3",
-  "ws_stream_endpoint": "wss://stream.binance.com:9443",
-  "use_cross_margin_api": true
-}

src/app_sparkd/configurations/alphaworx_zen_hk_live-preview/binance_usds_futures_1.json

@@ -1,9 +0,0 @@
-{
-  "api_key": "${BINANCE_API_KEY}",
-  "secret_key": "${BINANCE_SECRET_KEY}",
-  "account": "PROD",
-  "rest_endpoint": "https://fapi.binance.com",
-  "ws_endpoint": "wss://ws-fapi.binance.com/ws-fapi/v1",
-  "ws_stream_endpoint": "wss://fstream.binance.com",
-  "use_cross_margin_api": false
-}