cleanup: Remove unnecessary py_module.cpp files

We used to have separate modules for md and execution. I forgot to remove these after unifying them to a single module.
diag: Reduce book log noise in md plugins
2025-06-16 14:31:28 +00:00 · 2025-06-16 14:28:09 +00:00 · 2025-06-16 14:17:45 +00:00 · 2025-06-13 21:26:36 +00:00 · 2025-06-13 20:50:30 +00:00 · 2025-06-13 20:50:30 +00:00
2204 changed files with 23348 additions and 10545 deletions
@@ -34,7 +34,7 @@
 #   src/server/sparkd/Dockerfile
 SPARKD__IMAGE=sparkd
 # The config dir for this service to use
-SPARKD__CONFIG_DIR=src/server/sparkd/configurations/dev
+SPARKD__CONFIG_DIR=src/app_sparkd/configurations/dev
 # Mount a specific volume or dir to "~/sparkdata/data" inside the docker.
 # Defaults to a volume named "sparkd_<config_port>" (e.g., sparkd_8000).
 # SPARKD__DATA_DIR=/home/bts/sparkdata/data
@@ -44,5 +44,6 @@ SPARKD__CONFIG_PORT=8000
 SPARKD__QUICKBUS_PORT=8001
 SPARKD__SIM_FIX_PORT=8002
 SPARKD__SHM_SIZE=4gb
+SPARKD__PYTHON_REQUIREMENTS=requirements.txt

 # TODO: add config for other services (pubsub, md-publisher, webapp, api-server, etc.)
@@ -199,24 +199,27 @@
      "request": "launch",
      "program": "",
      "linux": {
-        "preLaunchTask": "Build Sparkd (Debug)",
-        "program": "${workspaceFolder}/build/sparkd-debug-clang/server/sparkd",
-        "args": [
-          "run"
-        ],
-        "stopAtEntry": false,
-        "cwd": "${workspaceFolder}/build/sparkd-debug-clang/server",
-        "environment": [],
-        "externalConsole": false,
-        "MIMode": "gdb",
-        "setupCommands": [
-          {
-            "description": "Enable pretty-printing for gdb",
-            "text": "-enable-pretty-printing",
-            "ignoreFailures": true
-          }
-        ]
-      }
+        "program": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd/sparkd",
+      },
+      "args": [
+        "run"
+      ],
+      "cwd": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd",
+      "stopAtEntry": false,
+      "externalConsole": false,
+      "MIMode": "gdb",
+      "setupCommands": [
+        {
+          "description": "Enable pretty-printing for gdb",
+          "text": "-enable-pretty-printing",
+          "ignoreFailures": true
+        },
+        {
+          "description": "Set Disassembly Flavor to Intel",
+          "text": "-gdb-set disassembly-flavor intel",
+          "ignoreFailures": true
+        }
+      ]
    },
    {
      "name": "Sparkd (Skip Build)",
@@ -224,74 +227,27 @@
      "request": "launch",
      "program": "",
      "linux": {
-        "program": "${workspaceFolder}/build/sparkd-debug-clang/server/sparkd",
-        "args": [
-          "run"
-        ],
-        "stopAtEntry": false,
-        "cwd": "${workspaceFolder}/build/sparkd-debug-clang/server",
-        "environment": [],
-        "externalConsole": false,
-        "MIMode": "gdb",
-        "setupCommands": [
-          {
-            "description": "Enable pretty-printing for gdb",
-            "text": "-enable-pretty-printing",
-            "ignoreFailures": true
-          }
-        ]
-      }
-    },
-    {
-      "name": "Sparkd Test",
-      "type": "cppdbg",
-      "request": "launch",
-      "program": "",
-      "linux": {
-        "preLaunchTask": "Build Sparkd (Debug)",
-        "program": "${workspaceFolder}/build/sparkd-debug-clang/server/sparkd",
-        "args": [
-          "test",
-          "${input:sparkdTest}",
-        ],
-        "stopAtEntry": false,
-        "cwd": "${workspaceFolder}/build/sparkd-debug-clang/server",
-        "environment": [],
-        "externalConsole": false,
-        "MIMode": "gdb",
-        "setupCommands": [
-          {
-            "description": "Enable pretty-printing for gdb",
-            "text": "-enable-pretty-printing",
-            "ignoreFailures": true
-          }
-        ]
-      }
-    },
-    {
-      "name": "Sparkd Test (Skip Build)",
-      "type": "cppdbg",
-      "request": "launch",
-      "program": "",
-      "linux": {
-        "program": "${workspaceFolder}/build/sparkd-debug-clang/server/sparkd",
-        "args": [
-          "test",
-          "${input:sparkdTest}",
-        ],
-        "stopAtEntry": false,
-        "cwd": "${workspaceFolder}/build/sparkd-debug-clang/server",
-        "environment": [],
-        "externalConsole": false,
-        "MIMode": "gdb",
-        "setupCommands": [
-          {
-            "description": "Enable pretty-printing for gdb",
-            "text": "-enable-pretty-printing",
-            "ignoreFailures": true
-          }
-        ]
-      }
+        "program": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd/sparkd",
+      },
+      "args": [
+        "run"
+      ],
+      "cwd": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd",
+      "stopAtEntry": false,
+      "externalConsole": false,
+      "MIMode": "gdb",
+      "setupCommands": [
+        {
+          "description": "Enable pretty-printing for gdb",
+          "text": "-enable-pretty-printing",
+          "ignoreFailures": true
+        },
+        {
+          "description": "Set Disassembly Flavor to Intel",
+          "text": "-gdb-set disassembly-flavor intel",
+          "ignoreFailures": true
+        }
+      ]
    },
  ],
  "inputs": [
@@ -22,9 +22,9 @@ set(
 # can run simultaneously without fighting over the same shm files (e.g., sparkd,
 # api server, etc.).
 set(SHARED_MEMORY_PREFIX "bts" CACHE STRING "Prefix to use when creating shared memory files")
-set(SPARK_BIN "${EXTERNALS_ROOT}/bin")
-set(SPARK_INCLUDE "${EXTERNALS_ROOT}/include")
-set(SPARK_LIB "${EXTERNALS_ROOT}/lib/$<CONFIG>")
+set(EXTERNALS_BIN "${EXTERNALS_ROOT}/bin")
+set(EXTERNALS_INCLUDE "${EXTERNALS_ROOT}/include")
+set(EXTERNALS_LIB "${EXTERNALS_ROOT}/lib/$<CONFIG>")
 set(BUILD_DOCS OFF CACHE BOOL "Build documentation")
 set(BUILD_SDK OFF CACHE BOOL "Build the sdk")
 set(EXPORT_MD_CONSUMER OFF CACHE BOOL "Export (extern) C functions for MD consumer interop")
@@ -36,8 +36,8 @@ else()
  set(BUILD_ADMIN_WEB_APP ON CACHE BOOL "Build the Admin WebApp")
 endif()

-include_directories(BEFORE SYSTEM ${SPARK_INCLUDE} ${EXTERNALS_ROOT}/include/zlib)
-link_directories(${SPARK_LIB})
+include_directories(BEFORE SYSTEM ${EXTERNALS_INCLUDE} ${EXTERNALS_ROOT}/include/zlib)
+link_directories(${EXTERNALS_LIB})

 add_compile_definitions(SHARED_MEMORY_PREFIX="${SHARED_MEMORY_PREFIX}")
 add_compile_definitions(APP_DATA_DIR="${APP_DATA_DIR}")
@@ -47,12 +47,8 @@ endif()
 add_compile_definitions(EXPORT_MD_CONSUMER="${EXPORT_MD_CONSUMER}")
 if(WIN32)
  # subdirs
-  add_subdirectory(src/lib)
+  add_subdirectory(src/lib_spark)
 else()
-  # make sparkd look for libspark.so in its directory
-  set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE)
-  set(CMAKE_INSTALL_RPATH ".")
-
  if(BUILD_SPARKD)
    # Note: All macros that begin w/ 'CCAPI' are ccapi's macros (defined and/or
    # read by their code). We override some below.
@@ -117,12 +113,15 @@ else()
  endif()

  # subdirs
+  add_subdirectory(src/lib_spark)
  if (BUILD_SPARKD)
+    add_subdirectory(src/lib_python_interop)
+    add_subdirectory(src/lib_onixs_fix)
    add_subdirectory(src/lib_binance_execution)
-    add_subdirectory(src/server/sparkd)
+    add_subdirectory(src/lib_okx_execution)
+    add_subdirectory(src/app_sparkd)
    add_subdirectory(src/data_files)
  endif()
-  add_subdirectory(src/lib)
  if (BUILD_ADMIN_WEB_APP)
    add_subdirectory(src/Spark.Admin.WebApp)
  endif()
@@ -132,37 +131,4 @@ else()
  if(BUILD_SDK)
    add_subdirectory(src/sdk)
  endif()
-
-  if(BUILD_SPARKD)
-    add_custom_command(
-      OUTPUT ${CMAKE_BINARY_DIR}/server
-      COMMAND mkdir -p ${CMAKE_BINARY_DIR}/server
-      COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/src/server/sparkd/sparkd ${CMAKE_BINARY_DIR}/server
-      COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/data_files ${CMAKE_BINARY_DIR}/server
-      COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/src/lib/libspark.so ${CMAKE_BINARY_DIR}/server
-      COMMAND ln ARGS -sf ${EXTERNALS_ROOT}/lib/${CMAKE_BUILD_TYPE}/libonixs-fix-engine.so.4 ${CMAKE_BINARY_DIR}/server
-      
-      COMMAND
-        ln ARGS -sf
-        ${EXTERNALS_ROOT}/lib/Release/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.16.0.0
-        ${CMAKE_BINARY_DIR}/server
-      COMMAND ln ARGS -sf ${EXTERNALS_ROOT}/lib/Release/libACE-5.7.5.so ${CMAKE_BINARY_DIR}/server
-      COMMAND mkdir ARGS -p ${CMAKE_BINARY_DIR}/server/plugins
-      COMMENT "Creating server runtime layout"
-      DEPENDS sparkd data_files_dir)
-
-    add_custom_target(server_layout ALL DEPENDS ${CMAKE_BINARY_DIR}/server)
-
-    install(
-      DIRECTORY "${CMAKE_BINARY_DIR}/server/plugins"
-      DESTINATION "${CMAKE_INSTALL_BINDIR}")
-    install(
-      FILES
-      "${SPARK_LIB}/libACE-5.7.5.so"
-      "${SPARK_LIB}/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.16.0.0"
-      "${SPARK_LIB}/libonixs-fix-engine.so.4.10.1"
-      "${SPARK_LIB}/libonixs-fix-engine.so.4"
-      "${SPARK_LIB}/libonixs-fix-engine.so"
-      DESTINATION "${CMAKE_INSTALL_BINDIR}")
-  endif()
 endif()
@@ -31,9 +31,8 @@
        "CMAKE_SHARED_LINKER_FLAGS_RELEASE": "/OPT:REF /OPT:ICF /LTCG",
        "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
        "EXTERNALS_ROOT": "${sourceDir}/externals/windows",
-        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0",
-        "DOTNET_WINDOWS_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0-windows",
+        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf",
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0;${sourceDir}/build/$<CONFIG>/bin/net8.0-windows",
        "EXPORT_MD_CONSUMER": {
          "type": "BOOL",
          "value": "ON"
@@ -58,11 +57,12 @@
        "APP_DATA_DIR_OVERRIDE": "${dollar}{SPARKD_DATA_DIR}",
        "CMAKE_BUILD_WITH_INSTALL_RPATH": {
          "type": "BOOL",
-          "value": "ON"
+          "value": "OFF"
        },
        "CMAKE_CXX_STANDARD": "20",
        "CMAKE_INSTALL_RPATH": ".",
        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
+        "CMAKE_PREFIX_PATH": "${sourceDir}/cmake/;${sourceDir}/.venv/lib/python3.10/site-packages/pybind11/share/cmake/pybind11",
        "CMAKE_EXE_LINKER_FLAGS": "-Wno-undef",
        "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
        "DATABENTO_USE_EXTERNAL_JSON": {
@@ -93,7 +93,7 @@
          "value": "ON"
        },
        "CCAPI_LOG_LEVEL": "Info",
-        "CONFIG_DEFAULTS_PATH": "config/profiles/md_dev/*.conf;config/profiles/default/*.conf"
+        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/md_dev/*.conf;spark/config/profiles/default/*.conf"
      }
    },

@@ -104,7 +104,7 @@
      "cacheVariables": {
        "CMAKE_BUILD_TYPE": "Release",
        "CCAPI_LOG_LEVEL": "Warn",
-        "CONFIG_DEFAULTS_PATH": "config/profiles/md_prod/*.conf;config/profiles/default/*.conf"
+        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/md_prod/*.conf;spark/config/profiles/default/*.conf"
      }
    },

@@ -186,7 +186,11 @@
        "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
-        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
+        "CMAKE_EXPORT_COMPILE_COMMANDS": {
+          "type": "BOOL",
+          "value": "ON"
+        },
+        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf",
        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0",
        "EXPORT_MD_CONSUMER": {
          "type": "BOOL",
@@ -227,7 +231,7 @@
      "cacheVariables": {
        "APP_DATA_DIR": "${dollar}{APPDATA}/bts/DataStream",
        "APP_DATA_DIR_OVERRIDE": "${dollar}{DATA_STREAM_DATA_DIR}",
-        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf",
+        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf;spark/config/profiles/data_stream/*.conf",
        "SHARED_MEMORY_PREFIX": "data_stream"
      }
    },
@@ -262,8 +266,7 @@
        "CMAKE_CXX_COMPILER": "clang++",
        "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
-        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
-        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf",
+        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf;spark/config/profiles/data_stream/*.conf",
        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
        "EXPORT_MD_CONSUMER": {
          "type": "BOOL",
@@ -0,0 +1,7 @@
+include_guard(GLOBAL)
+
+macro(export_to_parent)
+  foreach(var IN ITEMS ${ARGN})
+    set(${var} ${${var}} PARENT_SCOPE)
+  endforeach()
+endmacro()
@@ -0,0 +1,142 @@
+include_guard(DIRECTORY)
+include(${CMAKE_SOURCE_DIR}/cmake/ExportToParent.cmake)
+include(${CMAKE_SOURCE_DIR}/cmake/InstallBasic.cmake)
+include(${CMAKE_SOURCE_DIR}/cmake/ProtobufGenerateCpp.cmake)
+include(${CMAKE_SOURCE_DIR}/cmake/SetFirstDefined.cmake)
+include(${CMAKE_SOURCE_DIR}/cmake/safe_dep_glob.cmake)
+
+function(init_library)
+  set(_options SHARED STATIC)
+  set(_one_value_args
+    NAME
+    SOURCE_DIR
+    INCLUDE_DIR
+  )
+  set(_multi_value_args)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+  if (_ARGS_SHARED AND _ARGS_STATIC)
+    message(FATAL_ERROR "Invalid args passed to init_library: cannot specify STATIC and SHARED")
+  endif()
+  set_first_defined(_name _ARGS_NAME PROJECT_NAME)
+  set_first_defined(_source_dir _ARGS_SOURCE_DIR _name)
+  set_first_defined(_include_dir _ARGS_INCLUDE_DIR _source_dir)
+
+  # Cpp sources
+  file(
+    GLOB_RECURSE ${_name}_SOURCES
+    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
+    ${_source_dir}/*.cpp
+  )
+  # Note: cpp files inside 'tests' dir(s) are considered tests.
+  # Remove tests from main sources, and add them to test sources
+  foreach(item ${${_name}_SOURCES})
+    if(${item} MATCHES "tests/.*\.cpp$")
+      list(APPEND ${_name}_TEST_SOURCES ${item})
+    endif()
+  endforeach()
+  foreach(item ${${_name}_TEST_SOURCES})
+    list(REMOVE_ITEM ${_name}_SOURCES ${item})
+  endforeach()
+
+  # Proto sources 
+  file(
+    GLOB_RECURSE ${_name}_PROTO_SOURCES
+    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
+    ${_source_dir}/protobufs/*.proto
+  )
+  update_deps_file("${${_name}_PROTO_SOURCES};")
+  protobuf_generate_cpp(
+    CXX_SOURCES ${_name}_CXX_PROTO_SOURCES
+    CXX_HEADERS ${_name}_CXX_PROTO_HEADERS
+    CXX_OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}
+    PROTO_INCLUDE_DIRS
+      ${EXTERNALS_INCLUDE}
+      ${CMAKE_CURRENT_SOURCE_DIR}
+    PROTO_SOURCES
+      ${${_name}_PROTO_SOURCES}
+  )
+
+  export_to_parent(
+    ${_name}_SOURCES
+    ${_name}_TEST_SOURCES
+    ${_name}_PROTO_SOURCES
+    ${_name}_CXX_PROTO_SOURCES
+    ${_name}_CXX_PROTO_HEADERS
+  )
+
+  # Add main lib
+  if (_ARGS_STATIC)
+    add_library(${_name} STATIC ${${_name}_SOURCES} ${${_name}_CXX_PROTO_SOURCES})
+  elseif(_ARGS_SHARED)
+    add_library(${_name} SHARED ${${_name}_SOURCES} ${${_name}_CXX_PROTO_SOURCES})
+  else()
+    add_library(${_name} ${${_name}_SOURCES} ${${_name}_CXX_PROTO_SOURCES})
+  endif()
+  target_include_directories(
+    ${_name}
+    SYSTEM
+    BEFORE
+    PUBLIC
+      $<BUILD_INTERFACE:${EXTERNALS_ROOT}/include>
+      $<BUILD_INTERFACE:${EXTERNALS_ROOT}/include/zlib>
+  )
+  target_include_directories(
+    ${_name}
+    PUBLIC
+      $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
+      $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>
+  )
+  # Include protos (if any)
+  if (${_name}_PROTO_SOURCES)
+    target_include_directories(
+      ${_name}
+      PUBLIC
+        $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
+    )
+  endif()
+  target_link_directories(
+    ${_name}
+    PUBLIC
+      $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
+      $<BUILD_INTERFACE:${EXTERNALS_ROOT}/lib/$<CONFIG>>
+      $<INSTALL_INTERFACE:${CMAKE_INSTALL_LIBDIR}>
+  )
+  # Install main lib
+  foreach(_hdr ${${_name}_CXX_PROTO_HEADERS})
+    file(RELATIVE_PATH _relative_path ${CMAKE_CURRENT_BINARY_DIR} ${_hdr})
+    install(FILES ${_hdr} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/${_relative_path})
+  endforeach()
+  install_basic(
+    NAME ${_name}
+    INCLUDE_DIR ${_include_dir}
+  )
+
+  # Add tests executable (if there are any)
+  if (${_name}_TEST_SOURCES AND NOT WIN32)
+    add_executable(${_name}-tests ${${_name}_TEST_SOURCES})
+
+    get_target_property(_lib_type ${_name} TYPE)
+    if (_lib_type STREQUAL "SHARED_LIBRARY")
+      # Set RPATH so the tests exe knows where to find the shared lib.
+      set_target_properties(
+        ${_name}-tests
+        PROPERTIES
+          BUILD_RPATH "$ORIGIN"
+          INSTALL_RPATH "$ORIGIN/../${CMAKE_INSTALL_LIBDIR}"
+      )
+    endif()
+
+    # Link main lib
+    target_link_directories(
+      ${_name}-tests
+      PRIVATE
+        $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
+        $<INSTALL_INTERFACE:${CMAKE_INSTALL_LIBDIR}>
+    )
+    target_link_libraries(${_name}-tests PRIVATE ${_name})
+
+    # Catch2
+    find_package(Catch2 CONFIG REQUIRED COMPONENTS Catch2WithMain)
+    target_link_libraries(${_name}-tests PRIVATE Catch2::Catch2WithMain)
+  endif()
+endfunction()
@@ -0,0 +1,71 @@
+include_guard(DIRECTORY)
+include(GNUInstallDirs)
+include(${CMAKE_SOURCE_DIR}/cmake/ExportToParent.cmake)
+include(${CMAKE_SOURCE_DIR}/cmake/SetFirstDefined.cmake)
+
+function(init_python_module)
+  set(_options)
+  set(_one_value_args
+    NAME
+  )
+  set(_multi_value_args
+    SOURCES
+  )
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+  set_first_defined(_name _ARGS_NAME PROJECT_NAME)
+  set_first_defined(_sources _ARGS_SOURCES ${_name}_SOURCES)
+
+  set(module_proj ${_name}_module)
+  set(${module_proj}_SOURCES ${_sources})
+  pybind11_add_module(${module_proj} SHARED ${_sources})
+  target_link_libraries(
+    ${module_proj}
+    PUBLIC
+      pybind11::embed
+      spark
+      ${_name}
+  )
+  target_include_directories(
+    ${module_proj}
+    PRIVATE
+      $<BUILD_INTERFACE:${pybind11_INCLUDE_DIRS}>
+  )
+  target_include_directories(
+    ${module_proj}
+    PUBLIC
+      ${Python3_INCLUDE_DIRS}
+      $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
+      $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>
+  )
+  add_custom_command(
+    OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/stubs/${_name}.pyi
+    DEPENDS ${_name} ${module_proj}
+    COMMAND PYTHONPATH=${CMAKE_CURRENT_BINARY_DIR} pybind11-stubgen --numpy-array-use-type-var --ignore-invalid-expressions '.*' ${_name}_module
+    COMMENT "Generating py stubs"
+  )
+  add_custom_target(
+    "${_name}_py_stubs"
+    ALL
+    DEPENDS
+      ${CMAKE_CURRENT_BINARY_DIR}/stubs/${_name}.pyi
+  )
+  install(
+    TARGETS ${module_proj}
+    EXPORT ${module_proj}Targets
+    RUNTIME
+      DESTINATION ${CMAKE_INSTALL_BINDIR}
+    LIBRARY
+      DESTINATION ${CMAKE_INSTALL_LIBDIR}
+    ARCHIVE
+      DESTINATION ${CMAKE_INSTALL_LIBDIR}
+  )
+  foreach(dir ${pybind11_INCLUDE_DIRS})
+    install(DIRECTORY ${dir} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
+  endforeach()
+  install(
+    EXPORT ${module_proj}Targets
+    FILE ${module_proj}Targets.cmake
+    NAMESPACE ${_name}::
+    DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name}
+  )
+endfunction()
@@ -0,0 +1,65 @@
+include_guard(DIRECTORY)
+include(GNUInstallDirs)
+
+function(init_python_plugin)
+  set(_options)
+  set(_one_value_args NAME PLUGIN_NAME)
+  set(_multi_value_args SOURCES)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+  set_first_defined(_name _ARGS_NAME PROJECT_NAME)
+  set(_plugin_name ${_ARGS_PLUGIN_NAME})
+  if (NOT DEFINED _ARGS_PLUGIN_NAME)
+    set(_plugin_name ${_name}_plugins)
+  endif()
+  set(${_plugin_name}_SOURCES ${_ARGS_SOURCES})
+  if (NOT DEFINED _ARGS_SOURCES)
+    file(
+      GLOB_RECURSE ${_plugin_name}_SOURCES
+      RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
+      ${_plugin_name}/*.py)
+  endif()
+  set(src_outs)
+  foreach(_src ${${_plugin_name}_SOURCES})
+    set(_src_in ${CMAKE_CURRENT_SOURCE_DIR}/${_src})
+    set(_src_out ${CMAKE_CURRENT_BINARY_DIR}/${_src})
+    add_custom_command(
+      OUTPUT ${_src_out}
+      COMMAND ${CMAKE_COMMAND} -E copy ${_src_in} ${_src_out}
+      DEPENDS ${_src_in}
+      COMMENT "Copying ${_plugin_name} plugin ${_src} to ${_src_out}"
+    )
+    list(APPEND src_outs ${_src_out})
+  endforeach()
+  
+  add_custom_target(copy_${_plugin_name} ALL DEPENDS ${src_outs})
+  
+  add_library(${_plugin_name} INTERFACE)
+  add_library(${_name}::${_plugin_name} ALIAS ${_plugin_name})
+  
+  set_target_properties(
+    ${_plugin_name}
+    PROPERTIES
+      INTERFACE_PYTHON_SCRIPTS_DIR
+        $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/${_plugin_name}>$<INSTALL_INTERFACE:${CMAKE_INSTALL_BINDIR}/${_plugin_name}>
+  )
+  
+  set_target_properties(
+    ${_plugin_name}
+    PROPERTIES
+    INTERFACE_PYTHON_SCRIPTS $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/${${_plugin_name}_SOURCES}>$<INSTALL_INTERFACE:${CMAKE_INSTALL_BINDIR}/${${_plugin_name}_SOURCES}>
+  )
+  
+  install(
+    TARGETS ${_plugin_name}
+    EXPORT ${_plugin_name}Targets
+    DESTINATION ${CMAKE_INSTALL_BINDIR}
+  )
+  
+  install(
+    EXPORT ${_plugin_name}Targets
+    FILE ${_plugin_name}Targets.cmake
+    NAMESPACE ${_name}::
+    DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name}
+  )
+  install(DIRECTORY ${_plugin_name} DESTINATION ${CMAKE_INSTALL_BINDIR})
+endfunction()
@@ -0,0 +1,39 @@
+include_guard(DIRECTORY)
+include(GNUInstallDirs)
+include(${CMAKE_SOURCE_DIR}/cmake/PackBasic.cmake)
+include(${CMAKE_SOURCE_DIR}/cmake/SetFirstDefined.cmake)
+
+function(install_basic)
+  set(_options)
+  set(_one_value_args NAME INCLUDE_DIR)
+  set(_multi_value_args)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+  set_first_defined(_name _ARGS_NAME PROJECT_NAME)
+  set_first_defined(_include_dir _ARGS_INCLUDE_DIR _name)
+
+  file(
+    GLOB_RECURSE ${_name}_HEADERS
+    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
+    ${_include_dir}/*.h)
+
+  target_sources(${_name}
+    PUBLIC
+      FILE_SET HEADERS FILES ${${_name}_HEADERS})
+
+  install(
+    TARGETS ${_name}
+    EXPORT ${_name}Targets
+    RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
+    LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
+    ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
+    FILE_SET HEADERS DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}
+  )
+
+  install(
+    EXPORT ${_name}Targets
+    FILE ${_name}Targets.cmake
+    NAMESPACE ${_name}::
+    DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name})
+
+  pack_basic()
+endfunction()
@@ -0,0 +1,44 @@
+include_guard(DIRECTORY)
+include(CMakePackageConfigHelpers)
+include(${CMAKE_SOURCE_DIR}/cmake/SetFirstDefined.cmake)
+
+function(pack_basic)
+  set(_options)
+  set(_one_value_args NAME)
+  set(_multi_value_args)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+  set_first_defined(_name _ARGS_NAME PROJECT_NAME)
+
+  find_package(Git QUIET)
+
+  if(GIT_FOUND)
+    execute_process(
+      COMMAND ${GIT_EXECUTABLE} describe --tags --always --dirty
+      WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
+      OUTPUT_VARIABLE PROJECT_VERSION
+      OUTPUT_STRIP_TRAILING_WHITESPACE
+    )
+  endif()
+
+  if(NOT GIT_FOUND OR PROJECT_VERSION STREQUAL "")
+    set(PROJECT_VERSION "0.0.0")
+  endif()
+
+  write_basic_package_version_file(
+    "${CMAKE_CURRENT_BINARY_DIR}/${_name}ConfigVersion.cmake"
+    VERSION ${PROJECT_VERSION}
+    COMPATIBILITY AnyNewerVersion
+  )
+
+  configure_package_config_file(
+    "${CMAKE_CURRENT_SOURCE_DIR}/Config.cmake.in"
+    "${CMAKE_CURRENT_BINARY_DIR}/${_name}Config.cmake"
+    INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name}
+  )
+
+  install(
+    FILES
+      "${CMAKE_CURRENT_BINARY_DIR}/${_name}Config.cmake"
+      "${CMAKE_CURRENT_BINARY_DIR}/${_name}ConfigVersion.cmake"
+    DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name})
+endfunction()
@@ -1,12 +1,23 @@
-set(Protobuf_PROTOC_EXECUTABLE "${SPARK_BIN}/protoc")
-set(PROTOBUF_INCLUDE_DIRS "${SPARK_INCLUDE}")
+set(Protobuf_PROTOC_EXECUTABLE "${EXTERNALS_BIN}/protoc")
+set(PROTOBUF_INCLUDE_DIRS "${EXTERNALS_INCLUDE}")
 set(PROTOBUF_STATIC_LIB "libprotobuf")
+set(_protobuf_src_dir ${CMAKE_SOURCE_DIR}/src/lib_spark)
+set(_protobuf_bin_dir ${CMAKE_BINARY_DIR}/src/lib_spark)

-function(Protobuf_Generate_Cpp SRCS HDRS)
-  set(${SRCS})
-  set(${HDRS})
+function(Protobuf_Generate_Cpp)
+  set(_options)
+  set(_one_value_args CXX_SOURCES CXX_HEADERS CXX_OUT_DIR)
+  set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+  set(${_ARGS_CXX_SOURCES})
+  set(${_ARGS_CXX_HEADERS})

-  foreach(FIL ${ARGN})
+  set(_proto_include_cli_args)
+  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
+    list(APPEND _proto_include_cli_args -I ${_dir})
+  endforeach()
+
+  foreach(FIL ${_ARGS_PROTO_SOURCES})
    get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
    get_filename_component(FIL_WE ${FIL} NAME_WE)
    get_filename_component(FIL_DIR ${FIL} DIRECTORY)
@@ -14,48 +25,56 @@ function(Protobuf_Generate_Cpp SRCS HDRS)
      set(FIL_WE "${FIL_DIR}/${FIL_WE}")
    endif()

-    list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc")
-    list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h")
+    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc")
+    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h")

    add_custom_command(
-      OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc"
-             "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h"
+      OUTPUT "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
+             "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
      COMMAND
-        ${Protobuf_PROTOC_EXECUTABLE} ARGS --cpp_out=${CMAKE_BINARY_DIR}/src -I
-        ${PROTOBUF_INCLUDE_DIRS} -I ${CMAKE_SOURCE_DIR}/src ${ABS_FIL}
+        ${Protobuf_PROTOC_EXECUTABLE} ARGS --cpp_out=${_ARGS_CXX_OUT_DIR} ${_proto_include_cli_args} ${ABS_FIL}
      DEPENDS ${ABS_FIL}
      COMMENT "Running C++ protocol buffer compiler on ${FIL}"
      VERBATIM)
  endforeach()

-  set_source_files_properties(${${SRCS}} ${${HDRS}} PROPERTIES GENERATED TRUE)
+  set_source_files_properties(${${_ARGS_CXX_SOURCES}} ${${_ARGS_CXX_HEADERS}} PROPERTIES GENERATED TRUE)

  # remove some warnings from protobuf generated files
  if(WIN32)
-    set_source_files_properties(${${SRCS}} PROPERTIES COMPILE_FLAGS
+    set_source_files_properties(${${_ARGS_CXX_SOURCES}} PROPERTIES COMPILE_FLAGS
                                                      "/wd4309 /wd4838")
  else()
    set_source_files_properties(
-      ${${SRCS}}
+      ${${_ARGS_CXX_SOURCES}}
      PROPERTIES
        COMPILE_FLAGS
        "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
    )
  endif()

-  set(${SRCS}
-      ${${SRCS}}
+  set(${_ARGS_CXX_SOURCES}
+      ${${_ARGS_CXX_SOURCES}}
      PARENT_SCOPE)
-  set(${HDRS}
-      ${${HDRS}}
+  set(${_ARGS_CXX_HEADERS}
+      ${${_ARGS_CXX_HEADERS}}
      PARENT_SCOPE)
 endfunction()

-function(Service_Compiler_Generate_Cpp SRCS HDRS)
-  set(${SRCS})
-  set(${HDRS})
+function(Service_Compiler_Generate_Cpp)
+  set(_options)
+  set(_one_value_args CXX_SOURCES CXX_HEADERS CXX_OUT_DIR)
+  set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+  set(${_ARGS_CXX_SOURCES})
+  set(${_ARGS_CXX_HEADERS})

-  foreach(FIL ${ARGN})
+  set(_proto_include_cli_args)
+  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
+    list(APPEND _proto_include_cli_args -I ${_dir})
+  endforeach()
+
+  foreach(FIL ${_ARGS_PROTO_SOURCES})
    get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
    get_filename_component(FIL_WE ${FIL} NAME_WE)
    get_filename_component(FIL_DIR ${FIL} DIRECTORY)
@@ -63,58 +82,58 @@ function(Service_Compiler_Generate_Cpp SRCS HDRS)
      set(FIL_WE "${FIL_DIR}/${FIL_WE}")
    endif()

-    list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc")
-    list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h")
+    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc")
+    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h")

-    list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.cc")
-    list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.h")
+    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc")
+    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h")

-    list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.cc")
-    list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.h")
+    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc")
+    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h")

    set(TEMPLATES
-        ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/templates/cpp/client.pb.cc
-        ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/templates/cpp/client.pb.h
-        ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/templates/cpp/server.pb.cc
-        ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/templates/cpp/server.pb.h
+        ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/client.pb.cc
+        ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/client.pb.h
+        ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/server.pb.cc
+        ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/server.pb.h
    )

+    set(_plugin_py ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/plugin.py)
    if(WIN32)
+      set(_plugin_cmd ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/plugin.cmd)
      add_custom_command(
-        OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc"
-               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h"
-               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.cc"
-               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.h"
-               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.cc"
-               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.h"
+        OUTPUT "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h"
        COMMAND SET PYTHONPATH=${SPARK_PYTHON}
        COMMAND ${CMAKE_SOURCE_DIR}/.venv/Scripts/activate.bat
        COMMAND
-          ${Protobuf_PROTOC_EXECUTABLE} ARGS --cpp_out=${CMAKE_BINARY_DIR}/src
-          --plugin=protoc-gen-service=${CMAKE_SOURCE_DIR}/src/lib/api/service_compiler/plugin.cmd
-          --service_out=lang=cpp:${CMAKE_BINARY_DIR}/src -I
-          ${PROTOBUF_INCLUDE_DIRS} -I ${CMAKE_SOURCE_DIR}/src ${ABS_FIL}
-        DEPENDS ${ABS_FIL} ${TEMPLATES}
-                ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/plugin.py
+          ${Protobuf_PROTOC_EXECUTABLE} ARGS --cpp_out=${_ARGS_CXX_OUT_DIR}
+          --plugin=protoc-gen-service=${_plugin_cmd}
+          --service_out=lang=cpp:${_ARGS_CXX_OUT_DIR} -I
+          ${PROTOBUF_INCLUDE_DIRS} -I ${_protobuf_src_dir} ${ABS_FIL}
+        DEPENDS ${ABS_FIL} ${TEMPLATES} ${_plugin_cmd} ${_plugin_py}
        COMMENT "Running C++ service compiler on ${FIL}"
        VERBATIM)
    else()
      add_custom_command(
-        OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc"
-               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h"
-               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.cc"
-               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.h"
-               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.cc"
-               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.h"
+        OUTPUT "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h"
        COMMAND ${CMAKE_SOURCE_DIR}/tools/venv_exec.sh
        ARGS
          ${Protobuf_PROTOC_EXECUTABLE}
-          --cpp_out=${CMAKE_BINARY_DIR}/src
-          --plugin=protoc-gen-service=${CMAKE_SOURCE_DIR}/src/lib/api/service_compiler/plugin.py
-          --service_out=lang=cpp,make_file_name=${ABS_FIL}:${CMAKE_BINARY_DIR}/src
-          -I ${PROTOBUF_INCLUDE_DIRS} -I ${CMAKE_SOURCE_DIR}/src ${ABS_FIL}
-        DEPENDS ${ABS_FIL} ${TEMPLATES}
-                ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/plugin.py
+          --cpp_out=${_ARGS_CXX_OUT_DIR}
+          --plugin=protoc-gen-service=${_plugin_py}
+          --service_out=lang=cpp,make_file_name=${ABS_FIL}:${_ARGS_CXX_OUT_DIR}
+          ${_proto_include_cli_args} ${ABS_FIL}
+        DEPENDS ${ABS_FIL} ${TEMPLATES} ${_plugin_py}
        COMMENT "Running C++ service compiler on ${FIL}"
        VERBATIM)
    endif()
@@ -123,22 +142,22 @@ function(Service_Compiler_Generate_Cpp SRCS HDRS)

  # remove some warnings from protobuf generated files
  if(WIN32)
-    set_source_files_properties(${${SRCS}} PROPERTIES COMPILE_FLAGS
+    set_source_files_properties(${${_ARGS_CXX_SOURCES}} PROPERTIES COMPILE_FLAGS
                                                      "/wd4309 /wd4838")
  else()
    set_source_files_properties(
-      ${${SRCS}}
+      ${${_ARGS_CXX_SOURCES}}
      PROPERTIES
        COMPILE_FLAGS
        "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
    )
  endif()

-  set(${SRCS}
-      ${${SRCS}}
+  set(${_ARGS_CXX_SOURCES}
+      ${${_ARGS_CXX_SOURCES}}
      PARENT_SCOPE)
-  set(${HDRS}
-      ${${HDRS}}
+  set(${_ARGS_CXX_HEADERS}
+      ${${_ARGS_CXX_HEADERS}}
      PARENT_SCOPE)

 endfunction()
@@ -0,0 +1,83 @@
+include_guard(GLOBAL)
+
+function(protobuf_generate_cpp)
+  set(_options)
+  set(_one_value_args
+    CXX_SOURCES
+    CXX_HEADERS
+    CXX_OUT_DIR
+    PROTOC
+  )
+  set(_multi_value_args
+    PROTO_INCLUDE_DIRS
+    PROTO_SOURCES
+  )
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+
+  if (NOT DEFINED _ARGS_PROTOC)
+    if (WIN32)
+      set(_ARGS_PROTOC "${EXTERNALS_BIN}/protoc.exe")
+    else()
+      set(_ARGS_PROTOC "${EXTERNALS_BIN}/protoc")
+    endif()
+  endif()
+  set(${_ARGS_CXX_SOURCES})
+  set(${_ARGS_CXX_HEADERS})
+
+  set(_proto_include_cli_args)
+  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
+    list(APPEND _proto_include_cli_args -I ${_dir})
+  endforeach()
+
+  foreach(_fil ${_ARGS_PROTO_SOURCES})
+    get_filename_component(_abs_fil ${_fil} ABSOLUTE)
+    get_filename_component(_fil_we ${_fil} NAME_WE)
+    get_filename_component(_fil_dir ${_fil} DIRECTORY)
+    if(_fil_dir)
+      set(_fil_we "${_fil_dir}/${_fil_we}")
+    endif()
+    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.cc")
+    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.h")
+    message(STATUS "add_custom_command for ${_fil} -> ${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.h")
+    # TODO: Pass proto sources by relative path (so this fn can be called from
+    # arbitrary subdirs).
+    add_custom_command(
+      OUTPUT
+        "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.cc"
+        "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.h"
+      COMMAND
+        ${_ARGS_PROTOC}
+      ARGS
+        --cpp_out=${_ARGS_CXX_OUT_DIR} ${_proto_include_cli_args} ${_abs_fil}
+      DEPENDS ${_abs_fil}
+      COMMENT "Running C++ protocol buffer compiler on ${_fil}"
+      VERBATIM
+    )
+  endforeach()
+
+  set_source_files_properties(
+    ${${_ARGS_CXX_SOURCES}}
+    ${${_ARGS_CXX_HEADERS}}
+    PROPERTIES
+      GENERATED TRUE
+  )
+
+  # remove some warnings from protobuf generated files
+  if(WIN32)
+    set_source_files_properties(
+      ${${_ARGS_CXX_SOURCES}}
+      PROPERTIES
+        COMPILE_FLAGS "/wd4309 /wd4838"
+    )
+  else()
+    set_source_files_properties(
+      ${${_ARGS_CXX_SOURCES}}
+      PROPERTIES
+        COMPILE_FLAGS "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
+    )
+  endif()
+
+  set(${_ARGS_CXX_SOURCES} ${${_ARGS_CXX_SOURCES}} PARENT_SCOPE)
+  set(${_ARGS_CXX_HEADERS} ${${_ARGS_CXX_HEADERS}} PARENT_SCOPE)
+endfunction()
+
@@ -0,0 +1,61 @@
+include_guard(GLOBAL)
+
+function(protobuf_generate_cpp_config)
+  set(_options)
+  set(_one_value_args
+    PROTO_SOURCES_HEADER
+    PROTO_DEFAULTS_HEADER
+  )
+  set(_multi_value_args
+    PROTO_SOURCES
+    PROTO_DEFAULTS
+  )
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+
+  # Configure the 'config_proto_files' header. It provides an array of strings populated with the
+  # files in PROTO_SOURCES. At runtime, we use this array to generate a special 'config' proto
+  # message with fields for each message defined in said files. A global instance of this 'config'
+  # proto is then constructed, and populated with [compile-time] defaults, and [runtime] overrides
+  # (i.e., in spark.conf). Calls to `bts::config::get<T>()` retrieve the value from the field
+  # associated with type `T` in this global instance.
+  message(STATUS "PROTO_SOURCES: ${_ARGS_PROTO_SOURCES}")
+  message(STATUS "PROTO_SOURCES_HEADER: ${_ARGS_PROTO_SOURCES_HEADER}")
+  message(STATUS "PROTO_DEFAULTS_HEADER: ${_ARGS_PROTO_DEFAULTS_HEADER}")
+  if (_ARGS_PROTO_SOURCES_HEADER)
+    set(PROTO_SOURCES)
+    set(PROTO_SOURCES_CXX_ARRAY_ITEMS "")
+    foreach(_proto_path ${_ARGS_PROTO_SOURCES})
+      list(APPEND PROTO_SOURCES ${_proto_path})
+      string(APPEND PROTO_SOURCES_CXX_ARRAY_ITEMS "\"${_proto_path}\",\n")
+    endforeach()
+    list(LENGTH PROTO_SOURCES PROTO_SOURCES_LENGTH)
+    message(STATUS "CONFIGURING PROTO_SOURCES_HEADER): ${_ARGS_PROTO_SOURCES_HEADER}")
+    configure_file(${CMAKE_SOURCE_DIR}/cmake/config_proto_files.h.in ${_ARGS_PROTO_SOURCES_HEADER})
+  endif()
+
+  # Generate static defaults. Ideally, this would be a single string, but we use an array because
+  # the compiler complains if the string is longer than 65k.
+  set(_cfg_defaults_str "inline const char * config_static_defaults[] = { ")
+
+  foreach(_cfg_file_path ${_ARGS_PROTO_DEFAULTS})
+    file(READ ${_cfg_file_path} _cfg_file_content)
+    string(REPLACE "\"" "\\\"" _cfg_file_content_escaped ${_cfg_file_content})
+    string(REPLACE "\n" ";" _cfg_lines ${_cfg_file_content_escaped})
+
+    foreach(_line ${_cfg_lines})
+      string(APPEND _cfg_defaults_str "\n  \"${_line}\",")
+    endforeach(_line)
+  endforeach(_cfg_file_path)
+
+  string(APPEND _cfg_defaults_str "};")
+  set(_old_cfg_defaults_str "")
+
+  if(EXISTS ${_ARGS_PROTO_DEFAULTS_HEADER})
+    file(READ ${_ARGS_PROTO_DEFAULTS_HEADER} _old_cfg_defaults_str)
+  endif()
+
+  if(NOT _cfg_defaults_str STREQUAL _old_cfg_defaults_str)
+    file(WRITE "${_ARGS_PROTO_DEFAULTS_HEADER}" "${_cfg_defaults_str}")
+  endif()
+endfunction()
+
@@ -10,7 +10,7 @@ function(protobuf_generate_fs)
    set(_ARGS_TARGET_NAME "${PROJECT_NAME}_generate_fs_protos")
  endif()
  if (NOT DEFINED _ARGS_PROTOC)
-    set(_ARGS_PROTOC "${SPARK_BIN}/protoc")
+    set(_ARGS_PROTOC "${EXTERNALS_BIN}/protoc")
  endif()

  set(_proto_include_cli_args)
@@ -27,8 +27,8 @@ function(protobuf_generate_fs)
  endforeach()

  set(_venv_exec ${CMAKE_SOURCE_DIR}/tools/venv_exec.sh)
-  set(_plugin_py ${CMAKE_SOURCE_DIR}/src/lib/api/service_compiler/plugin.py)
-  set(_template_fs ${CMAKE_SOURCE_DIR}/src/lib/api/service_compiler/templates/fs/fable.fs)
+  set(_plugin_py ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/plugin.py)
+  set(_template_fs ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/fs/fable.fs)
  add_custom_command(
    OUTPUT ${_outputs}
    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
@@ -0,0 +1,10 @@
+include_guard(GLOBAL)
+
+macro(set_first_defined RESULT)
+  foreach(_arg ${ARGN})
+    if (DEFINED ${_arg})
+      set(${RESULT} ${${_arg}})
+      break()
+    endif()
+  endforeach()
+endmacro()
@@ -0,0 +1,39 @@
+include_guard(DIRECTORY)
+include(${CMAKE_SOURCE_DIR}/cmake/SymLink.cmake)
+
+function(sym_link_python_module)
+  set(_options)
+  set(_one_value_args LIB_NAME MODULE_NAME PLUGIN_NAME)
+  set(_multi_value_args)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}"
+                        "${_multi_value_args}" ${ARGN})
+  if(NOT DEFINED _ARGS_LIB_NAME)
+    message(
+      FATAL_ERROR "LIB_NAME must be specified for sym_link_python_module()")
+  endif()
+  set(_lib_name ${_ARGS_LIB_NAME})
+  set(_module_name ${_ARGS_MODULE_NAME})
+  if(NOT DEFINED _ARGS_MODULE_NAME)
+    set(_module_name ${_lib_name}_module)
+  endif()
+  set(_plugin_name ${_ARGS_PLUGIN_NAME})
+  set(_deps ${_lib_name} ${_module_name})
+  if(DEFINED _ARGS_PLUGIN_NAME)
+    list(APPEND _deps ${_plugin_name} copy_${_plugin_name})
+  endif()
+  add_dependencies(${PROJECT_NAME} ${_deps})
+  sym_link_local_lib(${_lib_name})
+  set(_module_suffix cpython-310-x86_64-linux-gnu.so)
+  sym_link_file(
+    SOURCE
+    ${CMAKE_BINARY_DIR}/src/lib_${_lib_name}/${_module_name}.${_module_suffix}
+    DEST
+    ${CMAKE_CURRENT_BINARY_DIR}/${_module_name}.${_module_suffix}
+    INSTALL_DEST
+    ${CMAKE_INSTALL_BINDIR}/)
+  if(DEFINED _ARGS_PLUGIN_NAME)
+    get_target_property(plugins ${_plugin_name} INTERFACE_PYTHON_SCRIPTS_DIR)
+    sym_link_dir(TARGET_NAME ${PROJECT_NAME}_link_${_plugin_name} SOURCE
+                 ${plugins} DEST ${CMAKE_CURRENT_BINARY_DIR}/${_plugin_name})
+  endif()
+endfunction()
@@ -0,0 +1,11 @@
+#pragma once
+
+#include <array>
+
+namespace bts::@PROJECT_NAME@::protobufs {
+
+inline std::array<const char *, @PROTO_SOURCES_LENGTH@> config_proto_files = {
+@PROTO_SOURCES_CXX_ARRAY_ITEMS@
+};
+
+} // namespace bts::@PROJECT_NAME@::protobufs
@@ -1,24 +0,0 @@
-cmake_minimum_required(VERSION 3.5)
-
-# Explicitly set policy to convert relative paths to absolute (although this is a default policy,
-# setting it explicitly prevents a compile-warning)
-cmake_policy(SET CMP0076 NEW)
-
-# add protos to the library to facilitate source_group
-target_sources(libspark PUBLIC ${Config_Protos} ${NonConfig_Protos} ${Service_Protos})
-
-function(assign_source_group)
-  foreach(_source IN ITEMS ${ARGN})
-    if(IS_ABSOLUTE "${_source}")
-      file(RELATIVE_PATH _source_rel "${CMAKE_CURRENT_SOURCE_DIR}" "${_source}")
-    else()
-      set(_source_rel "${_source}")
-    endif()
-    get_filename_component(_source_path "${_source_rel}" PATH)
-    string(REPLACE "/" "\\" _source_path_msvc "${_source_path}")
-    source_group("${_source_path_msvc}" FILES "${_source}")
-    set(SOURCE_FILES "${_source}")
-  endforeach()
-endfunction(assign_source_group)
-
-assign_source_group(${SparkLib_Sources} ${Config_Protos} ${NonConfig_Protos} ${Service_Protos})
@@ -3,11 +3,12 @@ services:
    volumes:
      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/sparkdata/data
    build:
-      dockerfile: src/server/sparkd/Dockerfile
+      dockerfile: src/app_sparkd/Dockerfile
      args:
        - IMAGE=${SPARKD__IMAGE}
        - SOURCE_DIR=${SPARKD__SOURCE_DIR}
        - CONFIG_DIR=${SPARKD__CONFIG_DIR}
+        - PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
    shm_size: "${SPARKD__SHM_SIZE:-512mb}"
    command: >
      sh -c "
@@ -39,11 +40,12 @@ services:
      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/sparkdata/data
      - ${SPARKD__RUNTIME_CONF_DIR:-sparkd_conf}:/home/bts/sparkdata/conf
    build:
-      dockerfile: src/server/sparkd/Dockerfile
+      dockerfile: src/app_sparkd/Dockerfile
      args:
        - IMAGE=${SPARKD__IMAGE}
        - SOURCE_DIR=${SPARKD__SOURCE_DIR}
        - CONFIG_DIR=${SPARKD__CONFIG_DIR}
+        - PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
    shm_size: "${SPARKD__SHM_SIZE:-512mb}"
    command: >
      sh -c "
@@ -124,8 +124,8 @@ service OrderService {
 ```
 ### C++ Example
 ```cpp
-#include "lib/api/public/ServiceContext.h"
-#include "lib/api/services/OrderService.client.pb.h"
+#include "spark/api/public/ServiceContext.h"
+#include "spark/api/services/OrderService.client.pb.h"
 using namespace bts::api;
 ServiceContext ctx;
 auto orders = ctx.locate<orderman::OrderService>();
@@ -29,9 +29,9 @@ namespace sample {
 ```

 ```cpp
-#include "lib/api/public/ServiceContext.h"
-#include "spark/lib/domain_model/Security.h"
-#include "lib/api/services/MarketDataService.client.pb.h"
+#include "spark/api/public/ServiceContext.h"
+#include "spark/domain_model/Security.h"
+#include "spark/api/services/MarketDataService.client.pb.h"
 #include <thread>

 using namespace bts::api;
@@ -16,11 +16,16 @@ jmespath==1.0.1
 lazy-object-proxy==1.6.0
 MarkupSafe==3.0.2
 mccabe==0.6.1
+numpy==2.2.6
 platformdirs==2.3.0
 protobuf==4.21.12
 pybars3==0.9.7
+pybind11==2.13.6
+pybind11-stubgen==2.5.3
+pybit==5.11.0
 pycodestyle==2.7.0
 pycparser==2.20
+pycryptodome==3.23.0
 pylint==2.10.2
 PyMeta3==0.5.1
 python-dateutil==2.9.0.post0
@@ -31,6 +36,7 @@ six==1.17.0
 termcolor==1.1.0
 toml==0.10.2
 urllib3==2.0.3
+websocket-client==1.8.0
 Werkzeug==3.1.3
 wrapt==1.12.1
 yapf==0.31.0
@@ -0,0 +1,49 @@
+using System;
+
+namespace BCG {
+   [DataStreamUserRespresentation("count")]
+   public class CountStream : ViewStream {
+      private DataPacket myLastFiredDataPacket;
+      private IBlueTableSchema mySchema = new BlueTableSchema("count",
+         new ColumnInfo("id", 0, typeof(int)),
+         new ColumnInfo("count", 1, typeof(int)));
+      private int myId = 0;
+      private ViewStream myVS;
+
+      public CountStream(ViewStream vs) {
+         myVS = vs;
+      }
+
+      public CountStream(string tableName)
+         : this(new TableStream(tableName)) { }
+
+      protected override void myOnFirstSubscriber() {
+         myVS.onChange += new ChangeDelegate(myHandleChanges);
+      }
+      protected override void myOnLastSubscriber() {
+         myVS.onChange -= new ChangeDelegate(myHandleChanges);
+      }
+
+
+      private void myHandleChanges(object newValue) {
+         myFireEvent(
+            myLastFiredDataPacket =
+               myLastFiredDataPacket == null ?
+                  new DataPacket(myGetRows(newValue, ++myId), myGetRows(newValue, myId),
+                  new BlueTable(mySchema))
+                  :
+                  new DataPacket(myGetRows(newValue, ++myId), myGetRows(newValue, myId),
+                  myLastFiredDataPacket.newRows)
+            );
+      }
+
+      private IBlueRowCollection myGetRows(object dp, int id) {
+         BlueTable retVal = new BlueTable(mySchema);
+         retVal.addRow
+            (new BlueRow(mySchema, id, (dp as DataPacket).newRows.Count));
+         return retVal;
+      }
+
+
+   }
+}
@@ -0,0 +1,38 @@
+namespace bts.BlueOrb {
+   using bts;
+
+   using BCG;
+
+   using System;
+   [DataStreamUserRespresentation("DateTimeExpanderStream")]
+   public class DateTimeExpanstionStream : ViewStream {
+      public DateTimeExpanstionStream(ViewStream parent_stream, string inp_col) {
+         Func<IBlueTableSchema, IBlueTableSchema> get_schema = (s) => {
+            IBlueTableSchema rv = s.add_column(System.TypeCode.DateTime, $"{inp_col}_date");
+            rv = rv.add_column(System.TypeCode.String, $"{inp_col}_day_of_week");
+            rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_day_of_month");
+            rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_month");
+            rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_hour");
+            rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_minute");
+            rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_seconds_of_day");
+            return rv;
+         };
+         ViewStream rs = new MapStream(parent_stream,
+         s => get_schema(s),
+         r => {
+            DateTime dt = r.get_datetime(inp_col);
+            IBlueTableSchema s = get_schema(r.schema);
+            MutableBlueRow rv = new MutableBlueRow(s, r);
+            rv.setValue($"{inp_col}_date", dt.Date);
+            rv.setValue($"{inp_col}_day_of_week", dt.DayOfWeek.ToString());
+            rv.setValue($"{inp_col}_day_of_month", dt.Day);
+            rv.setValue($"{inp_col}_month", dt.Date.Month);
+            rv.setValue($"{inp_col}_hour", dt.Hour);
+            rv.setValue($"{inp_col}_minute", dt.Minute);
+            rv.setValue($"{inp_col}_seconds_of_day", (dt - dt.Date).TotalSeconds);
+            return rv;
+         });
+         mySetRealSource(rs);
+      }
+   }
+}
@@ -0,0 +1,158 @@
+namespace bts.BlueOrb {
+   using BCG;
+   using System;
+   using System.Text.RegularExpressions;
+
+
+   public class RegExpFilter : IStreamFilter {
+      private Regex[] myRegExps;
+      private string[] myCols;
+
+      public RegExpFilter(string col, string regExp)
+         : this(new string[] { col }, new string[] { regExp }) { }
+
+      public RegExpFilter(string[] cols, string[] regExps) {
+         myCols = cols;
+         myRegExps = new Regex[regExps.Length];
+         for (int i = 0; i < regExps.Length; i++)
+            myRegExps[i] = new Regex(regExps[i]);
+      }
+
+      #region IStreamFilter Members
+      public bool filter(IBlueRow dr) {
+         bool rv = true;
+         for (int i = 0; i < myCols.Length; i++) {
+            rv &= myRegExps[i].IsMatch(dr[myCols[i]] is DBNull ? "" : dr[myCols[i]].ToString());
+            if (!rv)
+               return rv;
+         }
+         return rv;
+      }
+      #endregion
+   }
+
+   //[DataStreamUserReprsentation("FilterStream")]
+   public class FilterStream : ViewStream {
+      private BlueTable myLastFiredRows;
+      private IBlueRowCollection myLastReceivedRows;
+      private IStreamFilter myFilter;
+      private ViewStream myParentStream;
+      private string myFilterString;
+      private bool myUseNewFilter = false;// Configuration.get<DataStreamConfig>().UseNewFilter;
+
+      #region constructors
+      public delegate bool OnFilter(IBlueRow r);
+      private class OnFilterWrapper : IStreamFilter {
+         public OnFilterWrapper(OnFilter f) { this.f = f; }
+
+         public bool filter(IBlueRow dr) { return f(dr); }
+
+         private OnFilter f;
+      }
+      public FilterStream(ViewStream vs, OnFilter f)
+         : this(vs, new OnFilterWrapper(f)) { }
+
+
+      public FilterStream(ViewStream vs, IStreamFilter filter) {
+         {
+            myParentStream = vs;
+            myFilter = filter;
+            myFilterString = filter.ToString();
+            if (filter is IDynamicStreamFilter)
+               (filter as IDynamicStreamFilter).onFilterChange += new FilterChangeDelegate(myHandleFilterChange);
+         }
+      }
+
+      public FilterStream(ViewStream vs, string filterExpression)
+         : this(vs, new BooleanExpressionFilter(filterExpression)) {
+         myFilterString = filterExpression;
+      }
+
+      public FilterStream(ViewStream vs, string col, string regExp)
+         : this(vs, new RegExpFilter(col, regExp)) {
+         myFilterString = col + " , " + regExp;
+      }
+
+      public FilterStream(ViewStream vs)
+         : this(vs, new BinaryFilter(true)) {
+         myFilterString = true.ToString();
+      }
+      #endregion
+
+      #region overridden methods
+      protected override void myOnFirstSubscriber() { myParentStream.onChange += new ChangeDelegate(myHandleCallbacks); }
+
+      protected override void myOnLastSubscriber() {
+         myLastFiredRows = null;
+         myParentStream.onChange -= new ChangeDelegate(myHandleCallbacks);
+      }
+
+      protected override void myFireEvent(object o) {
+         DataPacket dp = o as DataPacket;
+         if (myLastValue == null || dp.addedRows.Count + dp.deletedRows.Count > 0)
+            base.myFireEvent(o);
+      }
+      #endregion
+
+      private void myHandleCallbacks(object newValue) {
+         DataPacket retVal;
+         DataPacket arrivedPacket = newValue as DataPacket;
+         myLastReceivedRows = arrivedPacket.newRows;
+         if (myLastFiredRows != null) {
+            retVal = new DataPacket(myLastFiredRows, new BlueTable(myLastFiredRows.schema),
+               new BlueTable(myLastFiredRows.schema));
+            foreach (IBlueRow r in arrivedPacket.deletedRows)
+               if (myLastFiredRows.containsID((int)r["id"])) {
+                  myLastFiredRows.deleteRow(r);
+                  (retVal.deletedRows as BlueTable).addRow(r);
+               }
+            foreach (IBlueRow r in arrivedPacket.addedRows)
+               if (myFilter.filter(r)) {
+                  myLastFiredRows.addRow(r);
+                  (retVal.addedRows as BlueTable).addRow(r);
+               }
+         } else {
+            myLastFiredRows = new BlueTable(arrivedPacket.schema);
+            retVal = new DataPacket(myLastFiredRows, myLastFiredRows, new BlueTable(myLastFiredRows.schema));
+            foreach (IBlueRow r in arrivedPacket.newRows)
+               if (myFilter.filter(r))
+                  myLastFiredRows.addRow(r);
+         }
+         myFireEvent(retVal);
+      }
+
+      public IStreamFilter filter {
+         set {
+            {
+               myFilter = value;
+               if (numberOfSubscribers > 0) {
+                  if (myLastReceivedRows != null) {
+                     BlueTable nRows = new BlueTable(myLastReceivedRows.schema);
+                     foreach (IBlueRow r in myLastReceivedRows)
+                        if (myFilter.filter(r))
+                           nRows.addRow(r);
+                     IBlueRowCollection dRows = new BlueTable(nRows.schema),
+                        aRows = new BlueTable(nRows.schema);
+                     BlueDataUtility.computeDifferences(nRows, myLastFiredRows, out dRows, out aRows);
+                     myLastFiredRows = nRows;
+                     myFireEvent(new DataPacket(nRows, aRows, dRows));
+                  }
+               }
+            }
+         }
+         get {            
+               return myFilter;
+         }
+      }
+
+      //public string filterString { set { filter = new BooleanExpressionFilter(value); } }
+
+      public override string ToString() {
+         return GetType().ToString() + " " + myFilterString;
+      }
+
+      private void myHandleFilterChange() {
+         filter = myFilter;
+      }
+   }
+}
@@ -0,0 +1,121 @@
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using System.Text.Json;
+
+using bts;
+//using bts.SilverPine;
+using BCG;
+
+namespace bts.BlueOrb {
+   [DataStreamUserRespresentation("GroupByStream")]
+   public class GroupByStream : ViewStream {
+
+      public interface INonIncGroupByEvaluator {
+         // grouped_rows
+         IBlueRowCollection evaluate(IDictionary<int, List<IBlueRow>> grouped_rows);
+         string[] get_group_by_cols();
+      }
+
+      private class DelNonIncGroupByEvaluator : INonIncGroupByEvaluator {
+         Func<string[]> _groupby_cols_func;
+         Func<List<IBlueRow>, List<IBlueRow>> _evaluate_func;
+         public DelNonIncGroupByEvaluator(Func<string[]> groupby_cols_func,
+                                          Func<List<IBlueRow>, List<IBlueRow>> evaluate_func) {
+            _groupby_cols_func = groupby_cols_func;
+            _evaluate_func = evaluate_func;
+         }
+         public IBlueRowCollection evaluate(IDictionary<int, List<IBlueRow>> grouped_rows) {
+            BlueTable rv = null;
+            foreach (var kv in grouped_rows) {
+               List<IBlueRow> g_rows = kv.Value;
+               List<IBlueRow> out_rows = _evaluate_func.Invoke(g_rows);
+               if (out_rows != null && out_rows.Count > 0) {
+                  rv = rv ?? new BlueTable(out_rows.ElementAt(0).schema);
+               }
+               rv.addRows(out_rows);
+            }
+            return rv;
+         }
+         public string[] get_group_by_cols() {
+            return _groupby_cols_func();
+         }
+
+      }
+      private ViewStream _main_stream;
+      private INonIncGroupByEvaluator _groupby_func;
+      private BlueRowHasher _row_hasher;
+      private string[] _groupby_cols;
+      private Stopwatch _timer = new();
+      private int _n_total_fires = 0, _n_expensive_fires = 0;
+
+      public GroupByStream(ViewStream main_stream, INonIncGroupByEvaluator groupby_func) {
+         _main_stream = main_stream;
+         _groupby_func = groupby_func;
+         _groupby_cols = _groupby_func.get_group_by_cols();
+         _row_hasher = new BlueRowHasher(_groupby_cols == null ? 0 : _groupby_cols.Length);
+      }
+
+      public GroupByStream(ViewStream main_stream, Func<string[]> groupby_cols_func,
+                                 Func<List<IBlueRow>, List<IBlueRow>> evaluate_func)
+                                 : this(main_stream, new DelNonIncGroupByEvaluator(groupby_cols_func, evaluate_func)) { }
+
+      public GroupByStream(ViewStream parent_str, string groupby_col, string standard_group_by_text)
+      : this(parent_str, groupby_col, standard_group_by_text, String.Empty) { }
+
+      // this is the main constructor that does everything
+      public GroupByStream(ViewStream parent_str, string groupby_col, string standard_group_by_ops, string optional_sort_col) {
+         MultiRowOpHelper std_group_by = MultiRowOpHelper.parse_ops(standard_group_by_ops);
+         Func<string[]> groupby_cols_func = () => {
+            return String.IsNullOrEmpty(groupby_col) ? null : new string[] { groupby_col };
+         };
+         Func<List<IBlueRow>, List<IBlueRow>> evaluate_func = (group_rows) => {
+            if (!String.IsNullOrEmpty(optional_sort_col))
+               group_rows.Sort((r1, r2) => BlueDataUtility.compare_rows(r2, r1, optional_sort_col));
+            // this si where the group by aggregation happens
+            IBlueRow rv = MultiRowOpHelper.group_by_compute(group_rows, std_group_by);
+            return new IBlueRow[] { rv }.ToList();
+         };
+         ViewStream rs = new GroupByStream(parent_str, groupby_cols_func, evaluate_func);
+         mySetRealSource(rs);
+      }
+      public GroupByStream(ViewStream parent_str, string standard_group_by_text) :
+      this(parent_str, String.Empty, standard_group_by_text) {
+      }
+
+
+      protected override void myOnFirstSubscriber() { _main_stream.onChange += _main_stream_onchange; }
+
+      protected override void myOnLastSubscriber() { _main_stream.onChange -= _main_stream_onchange; }
+
+      private void _main_stream_onchange(object nv) {
+         DataPacket dp = nv as DataPacket;
+         ///grouping together
+         _timer.Restart();
+         IDictionary<int, List<IBlueRow>> grouped_rows = (_groupby_cols == null || _groupby_cols.Length == 0) ?
+         new Dictionary<int, List<IBlueRow>>() { { 0, dp.newRows.to_list() } } :
+         BlueDataUtility.group_together(dp.newRows, _groupby_cols, ref _row_hasher);
+         _timer.Stop();
+         var groupElapsedMs = _timer.ElapsedMilliseconds;
+         if (groupElapsedMs >= 50) {
+            Log.DataFlow.info($"Time to group {dp.newRows.Count} rows into {grouped_rows.Count} groups: {_timer.ElapsedMilliseconds}ms");
+         }
+         // Log.log(bts.Log.Level.Info, bts.log.Subsystem.gen, String.Format("Time to group {0} rows into {1} groups = {2}", dp.newRows.Count, grouped_rows.Count, _timer.lastIntervalTime));
+         /// reducing
+         _timer.Restart();
+         IBlueRowCollection total_state = _groupby_func.evaluate(grouped_rows);
+         _timer.Stop();
+         var applyElapsedMs = _timer.ElapsedMilliseconds;
+         _n_total_fires++;
+         if (applyElapsedMs >= 50) {
+            _n_expensive_fires++;
+            Log.DataFlow.info(
+               $"Time to evaluate {grouped_rows.Count} groups {_timer.ElapsedMilliseconds}ms, ratio {_n_expensive_fires}/{_n_total_fires}");
+         }
+         // TODO,
+         if (total_state != null)
+            myFireEvent(new DataPacket(total_state, total_state, myLastValue == null ? new BlueTable(total_state.schema) : (myLastValue as DataPacket).newRows));
+      }
+   }
+}
@@ -0,0 +1,17 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using BCG;
+
+namespace bts.BlueOrb {   
+   public interface INonIncJoinInlineMapper
+   {      
+      IBlueTableSchema get_output_schema(IBlueTableSchema schema);
+
+      IBlueRowCollection map_rows(IBlueRowCollection rows);
+      
+      event MapperChangeDelegate onMapperChange;
+   }
+}
@@ -0,0 +1,152 @@
+using bts;
+using bts.collections;
+
+using mbts;
+
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using System.Security.Cryptography;
+using BCG;
+
+namespace bts.BlueOrb {   
+   internal class NonIncJoinBookkeeper : IDisposable {
+      public Action on_new_data;
+      private readonly NonIncJoinSpec _join_spec;
+      private readonly int _join_position;
+      private readonly int _start_index_in_output;
+      private readonly BlueRowHasher _row_hasher;
+      private readonly bool _is_left_most;
+      private readonly bool _is_tag_along;
+      private int _n_total_fires = 0, _n_expensive_fires = 0;
+
+      private IBlueTableSchema _schema = null;
+      private IDictionary<int, IBlueRow> _row_map = new Dictionary<int, IBlueRow>();
+      private List<Pair<int, IBlueRow>> _row_list = new List<Pair<int, IBlueRow>>();
+      private IBlueRow _solitary_row = null;
+
+      private int[] _join_col_indices = null,
+                     _data_col_indices = null;
+
+      public NonIncJoinBookkeeper(NonIncJoinSpec the_join_spec, int join_position, int start_index_in_output, BlueRowHasher row_hasher) {
+         _join_spec = the_join_spec;
+         _join_position = join_position;
+         _start_index_in_output = start_index_in_output;
+         _row_hasher = row_hasher;
+         _is_left_most = _join_position == 0;
+         _is_tag_along = _join_spec.join_col_names.Length == 0;
+      }
+
+      private bool _filled_missing_data = false;
+      private void _fill_missing_data(DataPacket dp) {
+         if (_filled_missing_data)
+            return;
+         Func<string[], bool> null_or_empty = (str_arr) => {
+            if (str_arr == null || str_arr.Length == 0)
+               return true;
+            foreach (string s in str_arr) {
+               if (!String.IsNullOrEmpty(s)) {
+                  return false;
+               }
+            }
+            return true;
+         };
+         Func<string[], string[]> remove_bad_cols = (col_names) => {
+            HashSet<string> schema_col_map = new HashSet<string>(dp.schema.columnNames);
+            List<string> cols_list = new List<string>(col_names);
+            cols_list.RemoveWhere(c => !schema_col_map.Contains(c));
+            return cols_list.ToArray();
+         };
+         if (null_or_empty(_join_spec.data_col_names)) {
+            _join_spec.data_col_names = dp.schema.columnNames;
+         }
+         _join_spec.data_col_names = remove_bad_cols(_join_spec.data_col_names);
+         if (null_or_empty(_join_spec.data_col_names)) {
+            _join_spec.data_col_names = dp.schema.columnNames;
+         }
+         if (null_or_empty(_join_spec.projected_col_names)) {
+            _join_spec.projected_col_names = _join_spec.data_col_names;
+         }
+      }
+
+      public void init() {
+         _join_spec.stream.onChange += _on_stream_cb;
+      }
+
+      public void _on_stream_cb(object dp_obj) {
+         //Log.Gen.info($"NonIncJoinBookkeeper._on_stream_cb dp_obj: {dp_obj}");
+         var timer = Stopwatch.StartNew();
+         DataPacket dp = (DataPacket)dp_obj;
+         _fill_missing_data(dp);
+         if (_schema == null) {
+            _schema = dp.schema;
+            _join_col_indices = BlueDataUtility.get_column_indices(_schema, _join_spec.join_col_names);
+            _data_col_indices = BlueDataUtility.get_column_indices(_schema, _join_spec.data_col_names);
+         }
+
+         if (_is_left_most) {
+            _row_list.Clear();
+            foreach (IBlueRow r in dp.newRows)
+               _row_list.Add(new Pair<int, IBlueRow>(_row_hasher.get_hash(r, _join_col_indices), r));
+         } else {
+            if (_is_tag_along)
+               _solitary_row = BlueDataUtility.extractRow(dp.newRows, 0);
+            else {
+               _row_map.Clear();
+               foreach (IBlueRow r in dp.newRows)
+                  _row_map[_row_hasher.get_hash(r, _join_col_indices)] = r;
+            }
+         }
+         timer.Stop();
+         _n_total_fires++;
+         if (timer.ElapsedMilliseconds >= 50) {
+            _n_expensive_fires++;
+            Log.Gen.info($"NonIncJoinBookkeepingData._on_stream_cb {dp.newRows.Count} rows {_join_col_indices.Length} "
+               + $"key cols {_join_position} join pos time {timer.Elapsed} ratio {_n_expensive_fires}/{_n_total_fires}");
+         }
+         if (on_new_data != null)
+            on_new_data();
+      }
+
+      public bool is_ready { get { return _schema != null; } }
+
+      public void constribute_cols_to_schema(List<ColumnInfo> cis) {
+         for (int i = 0; i < _join_spec.data_col_names.Length; i++)
+            cis.Add(new ColumnInfo(_join_spec.projected_col_names[i], cis.Count, _schema[_join_spec.data_col_names[i]].colType));
+      }
+
+      public List<Pair<int, IBlueRow>> get_list() { return _row_list; }
+
+      public void copy_master_row_to_output_row(int row_index, object[] output_row) {
+         IBlueRow this_row = _row_list[row_index].second;
+         BlueDataUtility.copy_to_row(this_row, _data_col_indices, output_row, _start_index_in_output);
+      }
+
+      public void copy_matching_row_to_output_row(int row_hash, object[] output_row) {
+         if (_is_tag_along) {
+            if (_solitary_row != null)
+               BlueDataUtility.copy_to_row(_solitary_row, _data_col_indices, output_row, _start_index_in_output);
+            else
+               copy_dbnull_to_output_row(output_row);
+         } else {
+            IBlueRow this_row = null;
+            if (_row_map.TryGetValue(row_hash, out this_row))
+               BlueDataUtility.copy_to_row(this_row, _data_col_indices, output_row, _start_index_in_output);
+            else
+               copy_dbnull_to_output_row(output_row);
+         }
+      }
+
+      private void copy_dbnull_to_output_row(object[] output_row) {
+         for (int i = 0; i < _data_col_indices.Length; i++)
+            output_row[i + _start_index_in_output] = DBNull.Value;
+      }
+
+      #region IDisposable
+
+      public void Dispose() { _join_spec.stream.onChange -= _on_stream_cb; }
+
+      #endregion
+   }
+}
@@ -0,0 +1,67 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+using BCG;
+
+using mbts;
+
+namespace bts.BlueOrb {   
+   // this a collection of spec, theres no joining logic in here
+   public class NonIncJoinSpec {
+      public ViewStream stream;
+      // join col names are the names on which the streams are joined (for equality),
+      // data col names are the names in the original stream
+      // projected col names are their new names, ie if its x:l_x, then x is in the 
+      // data_col_names and l_x in projected cols
+      public string[] join_col_names, data_col_names, projected_col_names;
+
+      public NonIncJoinSpec() { }
+
+      public NonIncJoinSpec(ViewStream the_stream, string[] the_join_col_names, string[] the_data_col_names)
+         : this(the_stream, the_join_col_names, the_data_col_names, the_data_col_names) { }
+
+      public NonIncJoinSpec(ViewStream the_stream, string[] the_join_col_names, string[] the_data_col_names, string[] the_projected_col_names) {
+         stream = the_stream;
+         join_col_names = the_join_col_names;
+         data_col_names = the_data_col_names;
+         projected_col_names = the_projected_col_names;
+      }
+
+      public static NonIncJoinSpec[] from_triplet(params Triplet<ViewStream, string, string>[] join_specs) {
+         List<NonIncJoinSpec> rv = new List<NonIncJoinSpec>();
+         foreach (Triplet<ViewStream, string, string> r in join_specs) {
+            string[] keys = String.IsNullOrEmpty(r.second) ? new string[] { } : r.second.Split(' ');
+            string[] visible_cols = r.third.Split(' ');
+            List<string> data_col_names = new List<string>(), mapped_col_names = new List<string>();
+            foreach (string s in visible_cols) {
+               string[] toks = s.Split(':');
+               if (toks.Length == 1) {
+                  data_col_names.Add(toks[0]);
+                  mapped_col_names.Add(toks[0]);
+               } else if (toks.Length == 2) {
+                  data_col_names.Add(toks[0]);
+                  mapped_col_names.Add(toks[1]);
+               }
+            }
+            rv.Add(new NonIncJoinSpec(r.first, keys, data_col_names.ToArray(), mapped_col_names.ToArray()));
+         }
+         return rv.ToArray();
+      }
+
+      public override string ToString() {
+         Func<string[], string> arr_txt = (a) => {
+            string rv = "";
+            foreach (string s in a) {
+               rv += "{s} ";
+            }
+            rv = rv.Trim();
+            return rv;
+         };
+         string rv = $"stream: {stream} join_col_names: {arr_txt(join_col_names)} data_col_names: {arr_txt(data_col_names)} projected_col_names: {arr_txt(projected_col_names)}";
+         return rv;
+      }
+   }
+}
@@ -0,0 +1,165 @@
+using bts;
+
+using mbts;
+
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using BCG;
+
+namespace bts.BlueOrb {   
+   [DataStreamUserRespresentation("JoinStream")]
+   public class NonIncJoinStream : ViewStream, BCG.Utilities.General.IWorker {
+      private readonly NonIncJoinSpec[] _join_specs;
+      private readonly INonIncJoinInlineMapper _output_mapper;
+      private readonly BlueRowHasher _row_hasher;
+      private readonly TimeSpan _throttle_period;
+      private readonly ViewStreamDatabase _vsd;
+
+      private List<NonIncJoinBookkeeper> _book_keeping_data = new List<NonIncJoinBookkeeper>();
+      private IBlueTableSchema _output_schema = null;
+      private DateTime _last_fire_time = DateTime.MinValue;
+      private bool _is_dirty = false;
+
+      public NonIncJoinStream(ViewStream left, string left_join_cols, string left_proj,
+                              ViewStream right, string right_join_cols, string right_proj)
+         : this(new Triplet<ViewStream, string, string>[]
+               {
+                  new Triplet<ViewStream, string, string> (left, left_join_cols, left_proj),
+                  new Triplet<ViewStream, string, string> (right, right_join_cols, right_proj)
+               }) { }
+
+      public NonIncJoinStream(params Triplet<ViewStream, string, string>[] join_specs)
+         : this(NonIncJoinSpec.from_triplet(join_specs), null, 0) { }
+
+      public NonIncJoinStream(ViewStreamDatabase vsd, INonIncJoinInlineMapper output_mapper, int throttles_ms,
+                              params Triplet<ViewStream, string, string>[] join_specs)
+         : this(vsd, NonIncJoinSpec.from_triplet(join_specs), output_mapper, throttles_ms) { 
+      }
+
+      public NonIncJoinStream(NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper output_mapper, int throttles_ms = 0)
+         : this(ViewStreamDatabase.instance, join_specs, output_mapper, throttles_ms) { }
+
+      public NonIncJoinStream(ViewStreamDatabase vsd, NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper output_mapper, int throttles_ms = 0) {
+         _join_specs = join_specs;
+         _output_mapper = output_mapper;
+         _row_hasher = new BlueRowHasher(_join_specs[0].join_col_names.Length);
+         _throttle_period = throttles_ms == 0 ? TimeSpan.Zero : new TimeSpan(0, 0, 0, 0, throttles_ms);
+         _vsd = vsd;
+      }
+
+      protected override void myOnFirstSubscriber() {
+         int data_col_no = 1;
+         for (int jp = 0; jp < _join_specs.Length; jp++) {
+            NonIncJoinBookkeeper s = new NonIncJoinBookkeeper(_join_specs[jp], jp, data_col_no, _row_hasher);
+            //Log.Gen.info($"NonIncJoinStream.myOnFirstSubscriber jp: {jp} spec: {_join_specs[jp]}");
+            data_col_no += _join_specs[jp].data_col_names.Length;
+            s.on_new_data += _stream_callback;
+            _book_keeping_data.Add(s);
+         }
+         if (_output_mapper != null)
+            _output_mapper.onMapperChange += _on_mapper_change;
+         _vsd.addWorker(this);
+         foreach (var bkd in _book_keeping_data)
+            bkd.init();
+      }
+
+      protected override void myOnLastSubscriber() {
+         foreach (NonIncJoinBookkeeper d in _book_keeping_data)
+            d.Dispose();
+         if (_output_mapper != null)
+            _output_mapper.onMapperChange -= _on_mapper_change;
+         _vsd.removeWorker(this);
+      }
+
+      private void _stream_callback() {
+         //Log.Gen.info("NonIncJoinStream._stream_callback received callback");
+         _is_dirty = true;
+         doWork();
+      }
+
+      private void _on_mapper_change() {
+         _is_dirty = true;
+         doWork();
+      }
+
+      private bool _is_ready() {
+         bool rv = true;
+         foreach (NonIncJoinBookkeeper d in _book_keeping_data)
+            rv &= d.is_ready;
+         return rv;
+      }
+
+      private void _construct_schema() {
+         if (_output_schema == null) {
+            List<ColumnInfo> cis = new List<ColumnInfo>(new ColumnInfo[] { new ColumnInfo("id", 0, typeof(int)) });
+            foreach (NonIncJoinBookkeeper d in _book_keeping_data)
+               d.constribute_cols_to_schema(cis);
+            _output_schema = new BlueTableSchema("NonIncJoinStream", cis.ToArray());
+         }
+      }
+
+      private int _n_total_fires = 0, _n_expensive_fires = 0;
+      private void _form_and_fire() {
+         //Log.Gen.info($"NonIncJoinStream._form_and_fire _is_ready(): {_is_ready()}");
+         if (_is_ready()) {
+            _construct_schema();
+            var timer = Stopwatch.StartNew();
+            IList<Pair<int, IBlueRow>> left_row_list = _book_keeping_data[0].get_list();
+            List<object[]> output_rows = new List<object[]>();
+
+            for (int i = 0; i < left_row_list.Count; i++) {
+               object[] or = new object[_output_schema.columnInfo.Length];
+               or[0] = i;
+               output_rows.Add(or);
+            }
+
+            for (int ji = 0; ji < _book_keeping_data.Count; ji++) {
+               NonIncJoinBookkeeper bk_data = _book_keeping_data[ji];
+               for (int i = 0; i < left_row_list.Count; i++) {
+                  if (ji == 0)
+                     bk_data.copy_master_row_to_output_row(i, output_rows[i]);
+                  else
+                     bk_data.copy_matching_row_to_output_row(left_row_list[i].first, output_rows[i]);
+               }
+            }
+            IBlueRowCollection dp_output_rows = _apply_map(new SequentialIDBlueTable(_output_schema, output_rows));
+            timer.Stop();
+            _n_total_fires++;
+            //if (timer.ElapsedMilliseconds >= 250) {
+            _n_expensive_fires++;
+            //string log_msg = $"Join and map time for {_book_keeping_data.Count} "
+            //   + $"tables {timer.Elapsed.TotalMilliseconds} msecs, ratio {_n_expensive_fires}/{_n_total_fires}";
+            //Log.Gen.info(log_msg);
+            //}
+            var dp_out = BlueDataUtility.build_dp_inc(dp_output_rows, myLastValue as DataPacket, true);
+            myFireEvent(dp_out);
+            //myFireEvent(new DataPacket(dp_output_rows, dp_output_rows, myLastValue == null ? new BlueTable(_output_schema) : (myLastValue as DataPacket).newRows));
+         }
+      }
+
+      private IBlueRowCollection _apply_map(IBlueRowCollection unmapped_rows) {
+         if (_output_mapper != null) {
+            IBlueRowCollection output_rows;
+            var timer = Stopwatch.StartNew();
+            output_rows = _output_mapper.map_rows(unmapped_rows);
+            timer.Stop();
+            if (timer.ElapsedMilliseconds >= 50) {
+               Log.Gen.info($"Time to map {unmapped_rows.Count} rows into {output_rows.Count} rows {timer.Elapsed}");
+            }
+            return output_rows;
+         } else {
+            return unmapped_rows;
+         }
+      }
+
+      public void doWork() {
+         //Log.Gen.info($"NonIncJoinStream.doWork _last_fire_time: {_last_fire_time} _is_dirty: {_is_dirty}");
+         if (DateTime.Now - _last_fire_time >= _throttle_period && _is_dirty) {
+            _is_dirty = false;
+            _last_fire_time = DateTime.Now;
+            _form_and_fire();
+         }
+      }
+   }
+}
@@ -0,0 +1,28 @@
+//namespace bts.Blueorb {
+//   using BCG;
+
+//   public class JsonSaverStream : ViewStream {
+//      private ViewStream _stream;
+//      private string _file_name;
+//      public JsonSaverStream(ViewStream vs, string file_name) {
+//         _stream = vs;
+//         _file_name = file_name;
+//      }
+
+//      protected override void myOnFirstSubscriber() {
+//         _stream.onChange += _on_data;
+//      }
+
+//      private void _on_data(object dp_obj) {
+//         if (dp_obj == null) {
+//            return;
+//         }
+//         DataPacket dp = dp_obj as DataPacket;
+//         dp.newRows.write_to_json_file(dp.newRows, _file_name);
+//      }
+
+//      protected override void myOnLastSubscriber() {
+//         _stream.onChange -= _on_data;
+//      }
+//   }
+//}
@@ -0,0 +1,91 @@
+using System;
+using System.Collections;
+using System.Collections.Generic;
+
+using Amazon.Runtime.Internal.Util;
+
+using BCG;
+using BCG.Utilities;
+
+using bts;
+
+namespace bts.BlueOrb{
+   public class LiteralStream : ViewStream {
+
+      public static List<Tuple<ColumnInfo, object>> parse_columns(string key_value_pairs) {
+         List<Tuple<ColumnInfo, object>> rv = new List<Tuple<ColumnInfo, object>>();
+         string[] pairs = key_value_pairs.Split(' ');
+         foreach (string p in pairs) {
+            string[] kvp = p.Split(':');
+            string col_name = kvp[0];
+            string val = kvp[1];
+            var tup = BlueDataUtility.guess_type(val);
+            var ci = new ColumnInfo(col_name, rv.Count, tup.Item1);
+            rv.Add(Tuple.Create(ci, tup.Item2));
+         }
+         return rv;
+      }
+
+      //literal("table1", "x:123, y:1.001, z:abc;x:123, y:1.001, z:abc")
+      // col_name: col_value ... ; col_name: col_value
+      public LiteralStream(string table_name, string rows) {
+         Func<string, int, IBlueRow> get_row = (json_row, id) => {
+            Dictionary<string, object> map = new Dictionary<string, object>();
+            string[] toks = json_row.Split(',', StringSplitOptions.TrimEntries);
+            //Logger.assert(col_vals.Length == 2);
+            foreach (string t in toks) {
+               var col_vals = t.Split(':', StringSplitOptions.TrimEntries);
+               string col = col_vals[0];
+               string val = col_vals[1];
+               map.Add(col, val);
+            }
+            map["id"] = id;
+            return new BlueRow(map);
+         };
+         string[] row_arr = rows.Split(';', StringSplitOptions.TrimEntries);
+         BlueTable table = null;
+         foreach (string json_string in row_arr) {
+            IBlueRow r = //BlueDataUtility.convert_json_to_row(json_string, table == null ? 0 : table.Count);
+            get_row(json_string, table == null ? 0 : table.Count);
+            table = table ?? new BlueTable(r.schema);
+            table.addRow(r);
+         }
+         ViewStream str = new BlueTableStream(table);
+         str = new SchemaGuesserStream(str);
+         mySetRealSource(str);
+      }
+
+      public LiteralStream(string tableName, string schema, string value) {
+         string[] columns = schema.Split(' ');
+
+
+         ColumnInfo[] ci = new ColumnInfo[columns.Length + 1];
+         ci[0] = new ColumnInfo("id", 0, typeof(int));
+         int n = 1;
+         foreach (string c in columns) {
+            string[] columnAndType = c.Split('/');
+            if (columnAndType.Length == 1)
+               ci[n] = new ColumnInfo(c, n, typeof(double));
+            else
+               ci[n] = new ColumnInfo(columnAndType[0], n, Type.GetType("System." + columnAndType[1], true));
+            n++;
+         }
+
+
+         BlueTable t = new BlueTable(new BlueTableSchema(tableName, ci));
+         string[] rows = value.Split(';');
+         n = 0;
+         foreach (string r in rows)
+            try {
+               if (r == string.Empty)
+                  continue;
+               t.addRow(
+                  BlueDataUtility.createRowFromCSVFileRow(r, t.schema, ++n));
+            } catch (Exception e) {
+               throw (new Exception("Error occured at line: " + n.ToString() + " " + e.Message));
+            }
+
+         mySetRealSource(new BlueRowCollectionStaticStream(t));
+      }
+   }
+}
@@ -0,0 +1,89 @@
+using System;
+
+using BCG.Utilities.General;
+using BCG;
+using bts;
+
+namespace bts.BlueOrb {
+   [DataStreamUserRespresentation("MillisecondStream")]
+   public class MillisecondStream : ViewStream, IWorker {
+      private DateTime _last_fire_time = DateTime.MinValue;
+      private readonly TimeSpan _period;
+      private readonly int _size;
+      private readonly ViewStreamDatabase _vsd;
+      private IBlueTableSchema _schema;
+      private bts.CircularBufferTable _table;
+
+      public MillisecondStream(int interval_msecs, int size)
+      : this(ViewStreamDatabase.instance, interval_msecs, size) {
+      }
+
+      public MillisecondStream(ViewStreamDatabase vsd, int interval_msecs, int size) {
+         _vsd = vsd;
+         _period = TimeSpan.FromMilliseconds(interval_msecs);
+         _size = size;
+         _schema = BlueDataUtility.form_schema("Milliseconds",
+               Tuple.Create("id", TypeCode.Int32),
+               Tuple.Create("time", TypeCode.DateTime),
+               Tuple.Create("display_time", TypeCode.String));
+         _table = new CircularBufferTable(_schema, _size);
+      }
+
+      protected override void myOnFirstSubscriber() {
+         _vsd.addWorker(this);
+      }
+
+      protected override void myOnLastSubscriber() {
+         _vsd.removeWorker(this);
+      }
+
+      private int _id = 0;
+      public void doWork() {
+         if (DateTime.Now - _last_fire_time >= _period) {
+            _last_fire_time = DateTime.Now;
+            IBlueRow nr = new BlueRow(_schema, _id++, _last_fire_time,
+            _last_fire_time.ToString("yyyy-MM-dd HH:mm:ss.fff"));
+            _table.add_row(nr);
+            var dp = BlueDataUtility.build_dp_inc(_table, myLastValue as DataPacket);
+            myFireEvent(dp);
+         }
+      }
+   }
+   public class MSecondStream : TimeUnitStream {
+      public MSecondStream(int intervalInms, int size)
+         : base(DateTime.Now, new TimeSpan(0, 0, 0, 0, intervalInms), size, ViewStreamDatabase.instance) { }
+
+      protected override DateTime myRoundTime(DateTime dt) { return dt; }
+   }
+
+   public class DisplayableMillisecondStream : ViewStream, IStreamMapper {
+      private IBlueTableSchema mySchema = new BlueTableSchema("ms",
+                                                               new ColumnInfo("id", 0, typeof(int)),
+                                                               new ColumnInfo("time", 1, typeof(string)));
+      public DisplayableMillisecondStream(int intervalInMs, int sz) {
+         mySetRealSource(new MapStream(new MSecondStream(intervalInMs, sz), this));
+      }
+      #region IStreamMapper Members
+
+      public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema) {
+         return mySchema;
+      }
+
+      #endregion
+
+      #region IDynamicStreamMapper Members
+
+      public IBlueRow map(IBlueRow row) {
+         DateTime time = row.get_datetime("time");
+         var time_txt = time.ToString("yyyy-MM-dd HH:mm:ss.fff");
+         return new BlueRow(mySchema, row["id"], time_txt);
+      }
+
+      #endregion
+
+      private string myToString(DateTime dt) {
+         string sep = ":";
+         return dt.Hour + sep + dt.Minute + sep + dt.Second + sep + dt.Millisecond;
+      }
+   }
+}
@@ -0,0 +1,127 @@
+namespace bts.BlueOrb {
+   using BCG;
+
+   using System;
+   using System.Collections.Generic;
+   using System.Linq;
+   //using MathNet.Numerics.Statistics;
+
+
+   public enum MultiRowOp {
+      Mean,
+      Median,
+      Mode,
+      Range,
+      StdDev,
+      Max,
+      Min,
+      First,
+      Last,
+      Sum,
+      EdgeDiff,
+      EdgeDiffPercentage,
+      //IsEdgeDiff,
+   }
+   public class MultiRowOpHelper {
+      public readonly List<Tuple<string, MultiRowOp>> operations;
+
+      public static MultiRowOpHelper parse_ops(string text) {
+         // text is inp_col: op, inp_col: op, inp_col:op
+         List<Tuple<string, MultiRowOp>> rv = new List<Tuple<string, MultiRowOp>>();
+         string[] calcs = text.Split(',');
+         foreach (string c in calcs) {
+            string[] toks = c.Split(':');
+            string inp_col = toks[0].Trim();
+            string op_txt = toks[1].Trim();
+            MultiRowOp op = Enum.Parse<MultiRowOp>(op_txt, true);
+            rv.Add(Tuple.Create(inp_col, op));
+         }
+         return new MultiRowOpHelper(rv);
+      }
+
+      public MultiRowOpHelper(List<Tuple<string, MultiRowOp>> operations_) {
+         operations = operations_;
+      }
+
+      // aggregates multiple rows into one - ie does group_by compute
+      public static IBlueRow group_by_compute(List<IBlueRow> rows, MultiRowOpHelper parameters) {
+         if (rows.Count() == 0) {
+            return null;
+         }
+
+         IBlueRow ir = rows.ElementAt(0);
+         IBlueTableSchema in_schema = ir.schema;
+         IBlueTableSchema out_schema = in_schema;
+         IDictionary<string, Tuple<string, MultiRowOp>> out_col_map =
+            new Dictionary<string, Tuple<string, MultiRowOp>>();
+         foreach (var t in parameters.operations) {
+            string out_col_name = $"{t.Item1}_{t.Item2}".ToLower();
+            out_schema = out_schema.add_column(TypeCode.Double, out_col_name);
+            out_col_map[out_col_name] = t;
+         }
+         MutableBlueRow or = new MutableBlueRow(out_schema, ir);
+         foreach (var kv in out_col_map) {
+            string out_col_name = kv.Key;
+            string in_col_name = kv.Value.Item1;
+            MultiRowOp op = kv.Value.Item2;
+            IEnumerable<double> col_vals = BlueDataUtility.get_colvalues<double>(rows, in_col_name);
+            double comp_res = compute_op(col_vals, op);
+            or.setValue(out_col_name, comp_res);
+         }
+         return or;
+      }
+
+      // rolling_rows are the rolling window of rows. the 0th row is the one thats gettin
+      // calculated. rows 1 - n-1 are its window rows. the schema is based on the operations.
+
+      public IBlueRow rolling_calc(List<IBlueRow> rolling_rows) {
+         if (rolling_rows.Count() == 0) {
+            return null;
+         }
+         IBlueRow ir = rolling_rows.ElementAt(0);
+         IBlueTableSchema in_schema = ir.schema;
+         IBlueTableSchema out_schema = in_schema;
+         IDictionary<string, Tuple<string, MultiRowOp>> out_col_map =
+            new Dictionary<string, Tuple<string, MultiRowOp>>();
+         foreach (var t in operations) {
+            string out_col_name = $"{t.Item1}_{t.Item2}".ToLower();
+            out_schema = out_schema.add_column(TypeCode.Double, out_col_name);
+            out_col_map[out_col_name] = t;
+         }
+         MutableBlueRow or = new MutableBlueRow(out_schema, ir);
+         foreach (var kv in out_col_map) {
+            string out_col_name = kv.Key;
+            string in_col_name = kv.Value.Item1;
+            MultiRowOp op = kv.Value.Item2;
+            IEnumerable<double> col_vals = BlueDataUtility.get_colvalues<double>(rolling_rows, in_col_name);
+            double comp_res = compute_op(col_vals, op);
+            or.setValue(out_col_name, comp_res);
+         }
+         return or;
+      }
+
+      public static double compute_op(IEnumerable<double> col_vals, MultiRowOp op) {
+         switch (op) {
+            case MultiRowOp.Max:
+               return col_vals.Max<double>();
+            case MultiRowOp.Min:
+               return col_vals.Min<double>();
+            case MultiRowOp.Mean:
+               return col_vals.Sum() / col_vals.Count();
+            case MultiRowOp.Sum:
+               return col_vals.Sum();
+            case MultiRowOp.First:
+               return col_vals.First();
+            case MultiRowOp.Last:
+               return col_vals.Last();
+            case MultiRowOp.StdDev:
+               return MathNet.Numerics.Statistics.Statistics.StandardDeviation(col_vals);
+            default:
+               // TODO
+               Log.assert(false);
+               return Double.NaN;
+         }
+      }
+
+   }
+}
@@ -0,0 +1,207 @@
+namespace bts.BlueOrb {
+   using BCG;
+
+   using ProtoBuf;
+
+   using System;
+   using System.Collections;
+   using System.Collections.Generic;
+   using System.Linq;
+
+   #region mappings
+   internal class ProjectData {
+      // schema from the base stream
+      public readonly IBlueTableSchema in_schema;
+      // this is the projected schema
+      public readonly IBlueTableSchema out_schema;
+      // projected cols to the input cols, for cols who's names are changed
+      public readonly IDictionary<string, string> mapped_to_orig_col_map = new Dictionary<string, string>();
+      public ProjectData(IBlueTableSchema in_schema_, IBlueTableSchema out_schema_,
+                  IDictionary<string, string> mapped_to_orig_col_map_) {
+         in_schema = in_schema_;
+         out_schema = out_schema_;
+         mapped_to_orig_col_map = mapped_to_orig_col_map_;
+      }
+   }
+   #endregion
+
+
+#region Row with mapipings build in
+   internal class NewProjectRow : IBlueRow {
+      #region IBlueRow members
+      private ProjectData _project_data;
+      private IBlueRow _row;
+      public NewProjectRow(ProjectData project_data, IBlueRow row) {
+         _project_data = project_data;
+         _row = row;
+      }
+      public IBlueTableSchema schema { get { return _project_data.out_schema; } }
+      #endregion
+
+      #region ICollection members
+      public int Count { get { return schema.columnNames.Length; } }
+
+      public bool IsSynchronized { get { return false; } }
+
+      public object SyncRoot { get { return this; } }
+
+      public void CopyTo(Array array, int index) {
+         int i = index;
+         foreach (object o in this as IEnumerable)
+            array.SetValue(o, i++);
+      }
+      #endregion
+
+      #region IDictionary members
+      public bool IsFixedSize { get { return true; } }
+
+      public bool IsReadOnly { get { return true; } }
+
+      public object this[object key] {
+         get {
+            if (key is string)
+               return this[(string)key];
+            return this[(int)key];
+         }
+         set { throw new NotSupportedException("The IBlueRow is readonly"); }
+      }
+
+      public object this[int index] {
+         get {
+            return this[schema.columnInfo[index].colName];
+         }
+      }
+
+      public object this[string col_name] {
+         get {
+            string mapped_to_col = null;
+            if (_project_data.mapped_to_orig_col_map.TryGetValue(col_name, out mapped_to_col)) {
+               return _row[mapped_to_col];
+            }
+            return _row[col_name];
+         }
+      }
+
+      public ICollection Keys { get { return schema.columnNames; } }
+
+      public ICollection Values {
+         get {
+            object[] retVal = new object[schema.columnNames.Length];
+            CopyTo(retVal, 0);
+            return retVal;
+         }
+      }
+
+      public void Add(object key, object value) { throw new NotSupportedException("The IBlueRow is readonly"); }
+
+      public void Clear() { throw new NotSupportedException("The IBlueRow is readonly"); }
+
+      public bool Contains(object key) { return schema[(string)key] != null; }
+
+      /// <summary>
+      /// Note, the default enumerator is the IDictionary's enumerator, so if user needs to
+      /// iterate through the values, typecast this into an IEnumerable
+      /// </summary>
+      /// <returns></returns>
+      public IDictionaryEnumerator GetEnumerator() { return new BlueRowDictionaryEnumerator(this); }
+
+      IEnumerator IEnumerable.GetEnumerator() { return new BlueRowEnumerator(this); }
+
+      public void Remove(object key) { throw new NotSupportedException("The IBlueRow is readonly"); }
+      #endregion
+   }
+   #endregion
+
+   [DataStreamUserRespresentation("ProjectStream")]
+   public class NewProjectStream : ViewStream, IStreamMapper {
+
+      private ViewStream _stream;
+      private string _project_expression;
+      private ProjectData _project_data = null;
+      //public ProjectData 
+      public NewProjectStream(ViewStream vs, string project_expression) {
+         _stream = vs;
+         _project_expression = project_expression;
+         _stream = new MapStream(vs, this);
+         mySetRealSource(_stream);
+      }
+
+      public IBlueTableSchema getOutputSchema(IBlueTableSchema input_schema) {
+         _project_data = _create_project_data(input_schema, _project_expression);
+         return _project_data.out_schema;
+      }
+
+      public IBlueRow map(IBlueRow row) {
+         return new NewProjectRow(_project_data, row);
+      }
+
+      private static ProjectData _create_project_data(IBlueTableSchema input_schema,
+                                                      string project_expression) {
+         project_expression = project_expression.Trim();
+         string[] project_toks = project_expression.Split(' ');
+         HashSet<string> removed_cols = new HashSet<string>();
+         Dictionary<string, string> mapped_to_orig_map = new Dictionary<string, string>();
+         Dictionary<string, string> orig_to_mapped = new Dictionary<string, string>();
+         // by default every col will show up, so if we have the projection as (a:a1)
+         // it will convert a to a1, 
+         // examples: if a schema has cols a b c
+         // project(parent_str, "a:aa ") -> aa b c
+         // project(parent_str, "a:aa !c") -> aa b
+         // project(parent_str, "a:aa !") -> aa
+         // project(parent_str, "a:aa b")) -> aa b
+         bool hide_all = false;
+         foreach (string p_tok in project_toks) {
+            if (p_tok.StartsWith("!")) {
+               if (p_tok.Length == 1) {
+                  // if it is a "blanket "!"=> hide all cols except the one that
+                  // are stated explicitly
+                  hide_all = true;
+               } else {
+                  string removed_col = p_tok.Substring(1);
+                  removed_cols.Add(removed_col);
+               }
+            } else if (p_tok.Contains(":")) {
+               string[] map_toks = p_tok.Split(":");
+               string orig_col = map_toks[0];
+               string mapped_col = map_toks[1];
+               mapped_to_orig_map.Add(mapped_col, orig_col);
+               orig_to_mapped.Add(orig_col, mapped_col);
+            } else {
+               if (input_schema.columnNames.Contains(p_tok)) {
+                  string orig_col = p_tok;
+                  string mapped_col = orig_col;
+                  orig_to_mapped.Add(orig_col, mapped_col);
+               }
+            }
+         }
+
+         // make the new schema
+         List<ColumnInfo> new_cols = new List<ColumnInfo>();
+         foreach (ColumnInfo ci in input_schema.columnInfo) {
+            if (ci.colName == "id") {
+               ColumnInfo nc = new ColumnInfo(ci.colName, new_cols.Count, ci.colType);
+               new_cols.Add(nc);
+               continue;
+            }
+            if (removed_cols.Contains(ci.colName)) {
+               continue;
+            } else if (orig_to_mapped.ContainsKey(ci.colName)) {
+               string mapped_name = orig_to_mapped[ci.colName];
+               ColumnInfo nc = new ColumnInfo(mapped_name, new_cols.Count, ci.colType);
+               new_cols.Add(nc);
+            } else {
+               if (hide_all)
+                  continue;
+               //if (ci.colName == "id" || removed_cols.Count > 0) {
+               // unless columns are  explicitely hidden, all columns show up
+               ColumnInfo nc = new ColumnInfo(ci.colName, new_cols.Count, ci.colType);
+               new_cols.Add(nc);
+               //}
+            }
+         }
+         IBlueTableSchema new_schema = new BlueTableSchema(input_schema.tableName, new_cols.ToArray());
+         return new ProjectData(input_schema, new_schema, mapped_to_orig_map);
+      }
+
+   }
+}
@@ -0,0 +1,30 @@
+using BCG;
+
+namespace bts.BlueOrb {
+   public class SchemaGuesserStream : ViewStream {
+
+      private IBlueRowCollection _rows = null;
+      private ViewStream _vs;
+      public SchemaGuesserStream(ViewStream vs) {
+         _vs = vs;
+      }
+
+      protected override void myOnFirstSubscriber() {
+         _vs.onChange += _on_data;
+      }
+      protected override void myOnLastSubscriber() {
+         _vs.onChange -= _on_data;
+      }
+
+      private void _on_data(object obj) {
+         DataPacket dp = obj as DataPacket;
+         if (_rows == null) {
+            _rows = BlueDataUtility.get_typed_table(dp.newRows);
+         }
+         var dp_out = BlueDataUtility.build_dp_inc(_rows, myLastValue as DataPacket);
+         myFireEvent(dp_out);
+
+      }
+
+   }
+}
@@ -0,0 +1,67 @@
+using Bts.ArithmeticExpressions;
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using BCG;
+
+namespace bts.BlueOrb {
+
+   [DataStreamUserRespresentation("SingleRowMathStream")]
+   public class MathStream : ViewStream, IStreamMapper {
+      private IArithmeticProgram myExp;
+      private BooleanExpressionFilter _bool;
+      List<string> myExtraColumns = new List<string>();
+      BlueTableSchema mySchema;
+
+
+      public MathStream(ViewStream vs, string mathExpression)
+         : this(vs, mathExpression, string.Empty) {
+      }
+
+      public MathStream(ViewStream vs, string mathExpression, string boolean_expression) {
+         myExp = ArithmeticProgramParser.Parse(mathExpression) ??
+            throw new Exception($"Failed to parse ArithmeticExpression '{mathExpression}'");
+         myExtraColumns = myExp.OutputIdentifiers.ToList();
+         if (!string.IsNullOrEmpty(boolean_expression))
+            _bool = new BooleanExpressionFilter(boolean_expression);
+         mySetRealSource(new MapStream(vs, this));
+      }
+
+      #region IStreamMapper Members
+
+      public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema) {
+         if (mySchema == null) {
+            Dictionary<string, ColumnInfo> colIdx = new Dictionary<string, ColumnInfo>();
+            foreach (ColumnInfo ci in inputSchema.columnInfo)
+               colIdx[ci.colName] = ci;
+
+            foreach (string ncn in myExtraColumns) {
+               ColumnInfo newC;
+               if (colIdx.ContainsKey(ncn))
+                  newC = new ColumnInfo(colIdx[ncn].colName, colIdx[ncn].colNo, typeof(double));
+               else
+                  newC = new ColumnInfo(ncn, colIdx.Count, typeof(double));
+               colIdx[ncn] = newC;
+            }
+            mySchema = new BlueTableSchema(colIdx.Values.ToArray(), "math(" + inputSchema.tableName + ")");
+         }
+         return mySchema;
+      }
+
+      #endregion
+
+      #region IDynamicStreamMapper Members
+
+      public IBlueRow map(IBlueRow row) {
+         MutableBlueRow result = new MutableBlueRow(mySchema, row);
+         if (_bool == null || _bool.filter(row)) {
+            myExp.Evaluate(result.AsDictionary());
+         }
+         return result;
+      }
+
+      #endregion
+   }
+}
@@ -0,0 +1,11 @@
+namespace bts.BlueOrb {
+   using BCG;
+   [DataStreamUserRespresentation("TopStream")]
+   public class TopStream : ViewStream { 
+      public TopStream(ViewStream vs, int n_rows, string sort_col) {
+         ViewStream rs = new BCG.TopStream (vs, n_rows, sort_col);
+         mySetRealSource(rs);
+      }
+   }
+
+}
@@ -0,0 +1,261 @@
+using System;
+using System.Collections.Generic;
+
+using mbts;
+using BCG;
+
+namespace bts.BlueOrb {
+   internal class MutableAppendTable : IBlueRowCollection {
+      private class MyRow : IBlueRow {
+         private IBlueRow myRow;
+         private int myID;
+
+         public MyRow(IBlueRow r, int id) {
+            myRow = r;
+            myID = id;
+         }
+
+         #region IBlueRow Members
+
+         public IBlueTableSchema schema { get { return myRow.schema; } }
+
+         public object this[int index] {
+            get {
+               if (myRow.schema[index].colName == "id")
+                  return myID;
+               else
+                  return myRow[index];
+            }
+         }
+
+         public object this[string colName] { get { return colName == "id" ? (object)myID : myRow[colName]; } }
+
+         #endregion
+
+         #region IDictionary Members
+
+         public void Add(object key, object value) { throw new Exception("Row is read only."); }
+
+         public void Clear() { throw new Exception("Row is read only."); }
+
+         public bool Contains(object key) { return myRow.Contains(key); }
+
+         public System.Collections.IDictionaryEnumerator GetEnumerator() { return new BlueRowDictionaryEnumerator(this); }
+
+         public bool IsFixedSize { get { return true; } }
+
+         public bool IsReadOnly { get { return true; } }
+
+         public System.Collections.ICollection Keys { get { return myRow.Keys; } }
+
+         public void Remove(object key) { throw new Exception("The method or operation is not implemented."); }
+
+         public System.Collections.ICollection Values {
+            get {
+               object[] vals = new object[myRow.Count];
+               CopyTo(vals, 0);
+               return vals;
+            }
+         }
+
+         public object this[object key] {
+            get { return key is string ? this[key as string] : this[(int)key]; }
+            set { throw new Exception("The method or operation is not implemented."); }
+         }
+
+         #endregion
+
+         #region ICollection Members
+
+         public void CopyTo(Array array, int index) {
+            myRow.CopyTo(array, index);
+            array.SetValue(myID, index + myRow.schema["id"].colNo);
+         }
+
+         public int Count { get { return myRow.Count; } }
+
+         public bool IsSynchronized { get { return myRow.IsSynchronized; } }
+
+         public object SyncRoot { get { return myRow.SyncRoot; } }
+
+         #endregion
+
+         #region IEnumerable Members
+
+         System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() { return new BlueRowEnumerator(this); }
+
+         #endregion
+      }
+
+      private BlueTable myRows;
+      private int myID = 0;
+      /// <summary>
+      /// maps the table no to a dictionary containing the
+      /// local IDs to a row in the final big table
+      /// </summary>
+      private Dictionary<int, Dictionary<int, IBlueRow>> myIDMap = new Dictionary<int, Dictionary<int, IBlueRow>>();
+
+      public MutableAppendTable(IBlueTableSchema schema) { myRows = new BlueTable(schema); }
+
+      public IBlueRow addRow(IBlueRow r, int tableNo) {
+         if (!myIDMap.ContainsKey(tableNo))
+            myIDMap.Add(tableNo, new Dictionary<int, IBlueRow>());
+         IBlueRow nr = new MyRow(r, myID++); // BlueDataUtility.getNewReplacedRow(r, myID++, "id");
+         myRows.addRow(nr);
+         myIDMap[tableNo].Add((int)r["id"], nr);
+         return nr;
+      }
+
+      public IBlueRow removeRow(IBlueRow r, int tableNo) {
+         IBlueRow dr = myIDMap[tableNo][(int)r["id"]];
+         myRows.deleteRow(dr);
+         myIDMap[tableNo].Remove((int)r["id"]);
+         return dr;
+      }
+
+      public IBlueRowCollection removeTable(int tableNo) {
+         if (!myIDMap.ContainsKey(tableNo))
+            return null;
+         else {
+            BlueTable rv = new BlueTable(myRows.schema);
+            foreach (IBlueRow r in myIDMap[tableNo].Values)
+               rv.addRow(r);
+            myRows.deleteRows(rv);
+            myIDMap.Remove(tableNo);
+            return rv;
+         }
+      }
+
+      public void clear() {
+         myRows.clear();
+         myIDMap.Clear();
+      }
+
+      #region IBlueRowCollection Members
+
+      public IBlueRow this[int id] { get { return myRows[id]; } }
+
+      public IBlueTableSchema schema { get { return myRows.schema; } }
+
+      public bool containsID(int id) { return myRows.containsID(id); }
+
+      #endregion
+
+      #region ICollection Members
+
+      public void CopyTo(Array array, int index) { myRows.CopyTo(array, index); }
+
+      public int Count { get { return myRows.Count; } }
+
+      public bool IsSynchronized { get { return myRows.IsSynchronized; } }
+
+      public object SyncRoot { get { return myRows.SyncRoot; } }
+
+      #endregion
+
+      #region IEnumerable Members
+
+      public System.Collections.IEnumerator GetEnumerator() { return myRows.GetEnumerator(); }
+
+      #endregion
+   }
+   /// <summary>
+   /// MutableStream can append multiple streams
+   /// as well as give the user the flexibility
+   /// to append (or remove) more streams at
+   /// runtime
+   /// </summary>
+   }
+namespace bts.BlueOrb { 
+   [DataStreamUserRespresentation("UnionStream")]
+   public class MutableAppendStream : ViewStream {
+      private bool mySubscriptionActive = false;
+      private MutableAppendTable myTable;
+      private int myMovingIndex;
+      private Dictionary<int, Pair<ViewStream, ViewStream>> myStreamIndex = new Dictionary<int, Pair<ViewStream, ViewStream>>();
+      private bool _allign_col_orders = true;
+
+      public MutableAppendStream(params ViewStream[] streams)
+         : this(new List<ViewStream>(streams)) { }
+
+      public MutableAppendStream(bool allign_col_order, params ViewStream[] streams)
+         : this(allign_col_order, new List<ViewStream>(streams)) { }
+
+      public MutableAppendStream(ICollection<ViewStream> streams)
+         : this(true, streams) { }
+
+      public MutableAppendStream(bool allign_col_orders, ICollection<ViewStream> streams) {
+         _allign_col_orders = allign_col_orders;
+         foreach (ViewStream vs in streams) {
+            myMovingIndex++;
+            ViewStream pj = new ChangeTableNameStream(vs, myMovingIndex.ToString());
+            if (_allign_col_orders)
+               pj = new ProjectStream(pj, "!");
+            myStreamIndex.Add(myMovingIndex, new Pair<ViewStream, ViewStream>(vs, pj));
+         }
+      }
+
+      public void addStream(ViewStream vs) {
+         myMovingIndex++;
+         ///the change name is done to identify the callbacks, and the project to
+         ///make all the schemas col orders the same
+         ViewStream pj = new ChangeTableNameStream(vs, myMovingIndex.ToString());
+         if (_allign_col_orders)
+            pj = new ProjectStream(pj, "!");
+         myStreamIndex.Add(myMovingIndex, new Pair<ViewStream, ViewStream>(vs, pj));
+         if (mySubscriptionActive)
+            pj.onChange += new ChangeDelegate(myHandleStreamChanges);
+      }
+
+      public void removeStream(ViewStream vs) {
+         int index = -1;
+         foreach (KeyValuePair<int, Pair<ViewStream, ViewStream>> p in myStreamIndex) {
+            if (p.Value.first == vs) {
+               index = p.Key;
+               if (mySubscriptionActive) {
+                  p.Value.second.onChange -= new ChangeDelegate(myHandleStreamChanges);
+                  if (myTable != null) {
+                     IBlueRowCollection delRows = myTable.removeTable(p.Key);
+                     if (delRows != null)
+                        myFireEvent(new DataPacket(myTable, new BlueTable(myTable.schema), delRows));
+                  }
+               }
+               break;
+            }
+            if (index > 0)
+               myStreamIndex.Remove(index);
+         }
+      }
+
+      protected override void myOnFirstSubscriber() {
+         mySubscriptionActive = true;
+         foreach (KeyValuePair<int, Pair<ViewStream, ViewStream>> p in myStreamIndex)
+            p.Value.second.onChange += new ChangeDelegate(myHandleStreamChanges);
+      }
+
+      protected override void myOnLastSubscriber() {
+         mySubscriptionActive = false;
+         foreach (KeyValuePair<int, Pair<ViewStream, ViewStream>> p in myStreamIndex)
+            p.Value.second.onChange -= new ChangeDelegate(myHandleStreamChanges);
+         if (myTable != null)
+            myTable.clear();
+      }
+
+      private void myHandleStreamChanges(object newValue) {
+         DataPacket dp = newValue as DataPacket;
+         myTable = myTable ?? new MutableAppendTable(new BlueTableSchema("MutableAppend", dp.schema.columnInfo));
+         BlueTable adds = new BlueTable(myTable.schema),
+                     dels = new BlueTable(myTable.schema);
+         int index = Int32.Parse(dp.schema.tableName);
+         foreach (IBlueRow r in dp.deletedRows) {
+            IBlueRow dr = myTable.removeRow(r, index);
+            dels.addRow(dr);
+         }
+         foreach (IBlueRow r in dp.addedRows) {
+            IBlueRow nr = myTable.addRow(r, index);
+            adds.addRow(nr);
+         }
+         myFireEvent(new DataPacket(myTable, adds, dels));
+      }
+   }
+}
@@ -18,6 +18,7 @@

  <ItemGroup>
    <PackageReference Include="HtmlAgilityPack" Version="1.11.57" />
+    <PackageReference Include="IronPython" Version="3.4.2" />
    <PackageReference Include="Microsoft.Data.Analysis" Version="0.21.1" />
    <PackageReference Include="MongoDB.Bson" Version="2.23.1" />
    <PackageReference Include="MongoDB.Driver" Version="2.23.1" />
@@ -1,52 +1,47 @@
 using System;

-namespace BCG
-{
-   [DataStreamUserRespresentation ("count")]
-	public class CountStream : ViewStream
-	{
+namespace bts.BlueOrb {
+   using BCG;
+   [DataStreamUserRespresentation("CountStream")]
+   public class CountStream : ViewStream {
      private DataPacket myLastFiredDataPacket;
-      private IBlueTableSchema mySchema = new BlueTableSchema ("count", 
-         new ColumnInfo ("id", 0, typeof(int)),
-         new ColumnInfo ("count", 1, typeof(int)));
+      private IBlueTableSchema mySchema = new BlueTableSchema("count",
+         new ColumnInfo("id", 0, typeof(int)),
+         new ColumnInfo("count", 1, typeof(int)));
      private int myId = 0;
      private ViewStream myVS;

-      public CountStream (ViewStream vs)
-      {
+      public CountStream(ViewStream vs) {
         myVS = vs;
      }

-      public CountStream (string tableName)
-         : this (new TableStream (tableName))
-      {}
+      public CountStream(string tableName)
+         : this(new TableStream(tableName)) { }

-      protected override void myOnFirstSubscriber () {
+      protected override void myOnFirstSubscriber() {
         myVS.onChange += new ChangeDelegate(myHandleChanges);
      }
-      protected override void myOnLastSubscriber () {
+      protected override void myOnLastSubscriber() {
         myVS.onChange -= new ChangeDelegate(myHandleChanges);
      }


-      private void myHandleChanges(object newValue)
-      {
-         myFireEvent (
+      private void myHandleChanges(object newValue) {
+         myFireEvent(
            myLastFiredDataPacket =
               myLastFiredDataPacket == null ?
-                  new DataPacket (myGetRows (newValue, ++myId), myGetRows (newValue, myId), 
-                  new BlueTable (mySchema))
+                  new DataPacket(myGetRows(newValue, ++myId), myGetRows(newValue, myId),
+                  new BlueTable(mySchema))
                  :
-                  new DataPacket (myGetRows (newValue, ++myId), myGetRows (newValue, myId), 
+                  new DataPacket(myGetRows(newValue, ++myId), myGetRows(newValue, myId),
                  myLastFiredDataPacket.newRows)
            );
      }

-      private IBlueRowCollection myGetRows (object dp, int id)
-      {
-         BlueTable retVal = new BlueTable (mySchema);
+      private IBlueRowCollection myGetRows(object dp, int id) {
+         BlueTable retVal = new BlueTable(mySchema);
         retVal.addRow
-            (new BlueRow (mySchema, id, (dp as DataPacket).newRows.Count));
+            (new BlueRow(mySchema, id, (dp as DataPacket).newRows.Count));
         return retVal;
      }

@@ -5,25 +5,20 @@ using mbts.config;
 using System;
 using System.Text.RegularExpressions;

-namespace BCG.StreamWorld.Addition
-{
-   [DataStreamUserRespresentation (false)]
-   public class FilterStream : ViewStream
-   {
+namespace BCG.StreamWorld.Addition {
+   [DataStreamUserRespresentation(false)]
+   public class FilterStream : ViewStream {
      private ViewStream myBaseStream;
      private BlueTable myBaseTable, myStateTable;
      private IStreamFilter myFilter;

      public FilterStream(ViewStream vs)
-         : this(vs, "1=1")
-      { }
+         : this(vs, "1=1") { }

      public FilterStream(ViewStream vs, string filterExp)
-         : this(vs, new BooleanExpressionFilter(filterExp))
-      { }
+         : this(vs, new BooleanExpressionFilter(filterExp)) { }

-      public FilterStream(ViewStream vs, IStreamFilter filter)
-      {
+      public FilterStream(ViewStream vs, IStreamFilter filter) {
         myBaseStream = vs;
         myFilter = filter;
         if (filter is IDynamicStreamFilter)
@@ -31,43 +26,35 @@ namespace BCG.StreamWorld.Addition
      }


-      protected override void myOnFirstSubscriber()
-      {
+      protected override void myOnFirstSubscriber() {
         myBaseStream.onChange += new ChangeDelegate(myHandleChanges);
      }

-      protected override void myOnLastSubscriber()
-      {
+      protected override void myOnLastSubscriber() {
         myBaseStream.onChange -= new ChangeDelegate(myHandleChanges);
         myBaseTable.clear();
         myStateTable.clear();
      }

-      private void myHandleChanges(object newValue)
-      {
+      private void myHandleChanges(object newValue) {
         DataPacket dp = newValue as DataPacket;
-         if (myStateTable == null)
-         {
+         if (myStateTable == null) {
            bts.Log.assert(myBaseTable == null);
            myStateTable = new BlueTable(dp.schema);
            myBaseTable = new BlueTable(dp.schema);
         }
         BlueTable dels = new BlueTable(dp.schema),
                     adds = new BlueTable(dp.schema);
-         foreach (IBlueRow r in dp.deletedRows)
-         {
+         foreach (IBlueRow r in dp.deletedRows) {
            myBaseTable.deleteRow(r);
-            if (myStateTable.containsID((int)r["id"]))
-            {
+            if (myStateTable.containsID((int)r["id"])) {
               dels.addRow(r);
               myStateTable.deleteRow(r);
            }
         }
-         foreach (IBlueRow r in dp.addedRows)
-         {
+         foreach (IBlueRow r in dp.addedRows) {
            myBaseTable.addRow(r);
-            if (myFilter.filter(r))
-            {
+            if (myFilter.filter(r)) {
               adds.addRow(r);
               myStateTable.addRow(r);
            }
@@ -76,13 +63,10 @@ namespace BCG.StreamWorld.Addition
            myFireEvent(new DataPacket(myStateTable, adds, dels));
      }

-      public IStreamFilter filter
-      {
-         set
-         {
+      public IStreamFilter filter {
+         set {
            myFilter = value;
-            if (myBaseTable != null)
-            {
+            if (myBaseTable != null) {
               IBlueRowCollection oldState = new BlueTable(myStateTable);
               myStateTable.clear();
               foreach (IBlueRow r in myBaseTable)
@@ -5,7 +5,7 @@ using System.Collections.Generic;
 using System.Linq;

 namespace BCG {
-   public class GroupByStream : MethodDrivenStream {
+   internal class GroupByStream : MethodDrivenStream {
      private class MyFuncGroupByEvaluator : IGroupByEvaluator {
         private string[] _groupby_cols;
         private Func<DataPacket, IBlueRow> _groupby_func;
@@ -40,4 +40,4 @@ namespace BCG {
            myFireSwitch = myLastFiredPacket.addedRows.Count + myLastFiredPacket.deletedRows.Count == 0 ? false : true;
        }
    }
-}
+}
@@ -280,6 +280,28 @@ namespace BCG {

      #region json

+      public static void write_to_json_file(this IBlueRowCollection table, string file_path) { 
+         write_dict_to_json_file(file_path, table.to_namevalue_pairs());
+      }
+
+      public static void write_dict_to_json_file(string file_path, List<Dictionary<string, string>> data) {
+         var options = new JsonSerializerOptions { WriteIndented = true };
+         string json = System.Text.Json.JsonSerializer.Serialize(data, options);
+         File.WriteAllText(file_path, json);
+      }
+
+      public static List<Dictionary<string, string>> ReadFromFile(string filePath) {
+         string json = File.ReadAllText(filePath);
+         return System.Text.Json.JsonSerializer.Deserialize<List<Dictionary<string, string>>>(json);
+      }
+
+      public static void write_table_to_json(IBlueRowCollection table) {
+         List<Dictionary<string, string>> rows = new List<Dictionary<string, string>>();
+         foreach (IBlueRow row in table) {
+            row.to_json();
+         }
+      }
+
      public static Dictionary<string, string> convert_json_to_dictionary(string json_string) {
         Dictionary<string, string> dictionary = JsonConvert.DeserializeObject<Dictionary<string, string>>(json_string);
         return dictionary;
@@ -348,6 +370,14 @@ namespace BCG {
         }
         return rv;
      }
+
+      public static List<Dictionary<string,string>> to_namevalue_pairs(this IBlueRowCollection table) {
+         List<Dictionary<string, string>> rv = new List<Dictionary<string, string>>();
+         foreach (IBlueRow r in table) {
+            rv.Add(r.to_dict());
+         }
+         return rv;
+      }
      #region Schema types
      // these are the only types allowed in schemas
      private static IReadOnlyDictionary<TypeCode, Type> _typecode_to_type = new Dictionary<TypeCode, Type>
@@ -1225,6 +1255,20 @@ namespace BCG {
         return rv;
      }

+      public static Dictionary<string, string> to_dict(this IBlueRow r) {
+         Dictionary<string, string> map = new Dictionary<string, string>();
+         foreach (ColumnInfo c in r.schema.columnInfo) {
+            map.Add(c.colName, r[c.colName].ToString());
+         }
+         return map;
+      }
+
+      public static string to_json(this IBlueRow r) {
+         Dictionary<string, string> map = r.to_dict();
+         string json = System.Text.Json.JsonSerializer.Serialize(map);
+         return json;
+      }
+
      public static void compute_differences_fast(IBlueRowCollection new_rows, IBlueRowCollection old_rows,
                                                  out IBlueRowCollection out_d_rows, out IBlueRowCollection out_a_rows) {
         if (old_rows == null || old_rows.Count == 0) {
@@ -19,8 +19,8 @@
  </ItemGroup>

  <ItemGroup>
-    <Proto Include="../../lib/config/protos/data_stream_config.proto" CSName="DataStreamConfig">
-      <OutDir>$(ProtoOutputPath)/lib/config/protos</OutDir>
+    <Proto Include="../../lib_spark/spark/config/protos/data_stream_config.proto" CSName="DataStreamConfig">
+      <OutDir>$(ProtoOutputPath)/lib_spark/spark/config/protos</OutDir>
    </Proto>
  </ItemGroup>

@@ -47,10 +47,10 @@ namespace BCG {
      /// </summary>
      /// <param name="assemblyName"></param>
      public void loadAssembly(string assemblyName) {
-         Func<Type, bool> is_silverpine_only = (t) => {
-            if (!assemblyName.Contains("SharedStreams.dll"))
-               return true;
-            return t.Namespace == "bts.SilverPine";
+         Func<Type, bool> is_blue_orb = (t) => {
+            //if (!assemblyName.Contains("SharedStreams.dll"))
+            //   return true;
+            return t.Namespace == "bts.BlueOrb";// || t.Namespace == "BCG";
         };
         bts.Log.Gen.info($"PrimitiveManager.loadAssembly assemblyName: {assemblyName}");
         if (myAssemblyNames.Contains(assemblyName))
@@ -61,7 +61,7 @@ namespace BCG {
            Assembly.LoadFrom(assemblyName);

         foreach (Type type in loadedAssembly.GetTypes())
-            if (!type.IsInterface && is_silverpine_only(type)
+            if (!type.IsInterface && is_blue_orb(type)
               && !type.IsAbstract
               && type.IsSubclassOf(typeof(ViewStream))
               && type.IsPublic) {
@@ -77,7 +77,7 @@ namespace BCG {
                     break;
                  }
               if (isVisible) {
-                  Log.assert_d(myTypeLookupTable[userRep] == null, type.FullName);
+                 // Log.assert_d(myTypeLookupTable[userRep] == null, type.FullName);
                  myTypeLookupTable[userRep] = type;
               }
            }
@@ -1,4 +1,5 @@
-namespace bts.SilverPine {
+namespace bts.BlueOrb {
+   using bts.SilverPine;
   using BCG;

   using System.Runtime.CompilerServices;
@@ -45,7 +46,7 @@ namespace bts.SilverPine {
   }

   [DataStreamUserRespresentation("FuturesHistorical1DayBars")]
-   public class FuturesHistorical1DayBarsStream : ViewStream {
+   internal class FuturesHistorical1DayBarsStream : ViewStream {

      // this gets the raw bars for a symbol (perhaps make it into a raw bars stream?)
      public FuturesHistorical1DayBarsStream(string symbol) :
@@ -103,7 +104,7 @@ namespace bts.SilverPine {
   }

   [DataStreamUserRespresentation("FuturesHistorical1MinBars")]
-   public class FuturesHistorical1MinuteBarsStream : ViewStream {
+   internal class FuturesHistorical1MinuteBarsStream : ViewStream {
      // same as the above but for one min
      // this gets the raw bars for a symbol
      public FuturesHistorical1MinuteBarsStream(string symbol) :
@@ -187,46 +188,6 @@ namespace bts.SilverPine {
      }
   }

-   [DataStreamUserRespresentation("FuturePriceDiff")]
-   public class FutureHistoricalBarsPriceDiffStream : ViewStream {
-
-      public FutureHistoricalBarsPriceDiffStream(string future_contract, string dt1, string dt2)
-      :
-      this(new ContractUtils.FutureContract(future_contract), DateTime.Parse(dt1), DateTime.Parse(dt2), "close", ViewStreamDatabase.instance) {
-      }
-
-
-      public FutureHistoricalBarsPriceDiffStream(string future_contract, DateTime dt1, DateTime dt2)
-      :
-      this(new ContractUtils.FutureContract(future_contract), dt1, dt2, "close", ViewStreamDatabase.instance) {
-      }
-
-      public FutureHistoricalBarsPriceDiffStream(string future_contract, DateTime dt1, DateTime dt2, string col)
-      :
-      this(new ContractUtils.FutureContract(future_contract), dt1, dt2, col, ViewStreamDatabase.instance) {
-      }
-
-
-      public FutureHistoricalBarsPriceDiffStream(ContractUtils.FutureContract fc, DateTime dt1, DateTime dt2, string col, ViewStreamDatabase vsd) {
-         BarSize bsize = dt2 - dt1 >= TimeSpan.FromDays(1.0) ? BarSize.ONE_DAY : BarSize.ONE_MIN;
-         ViewStream rs = new FutureHistoricalBarsStream(fc, BarSize.ONE_DAY, new DateTimeRange(dt1, dt2));
-
-         rs = new ColumnDifferenceStream(rs, r => {
-            DateTime r_dt = r.get_datetime("ts_event");
-            var rv = dt1.Equals(r_dt);
-            return rv;
-         },
-         //r => dt2.Equals(r.get_datetime("ts_event")), col, col);
-         r => {
-            DateTime r_dt = r.get_datetime("ts_event");
-            var rv = dt2.Equals(r_dt);
-            return rv;
-         }, col, col);
-         rs = new NewProjectStream(rs, "diff !");
-         mySetRealSource(rs);
-      }
-   }
-
   // takes the contract name and expands it into multuiple columns corresponding to the contracts.
   // the contract columns are a union, of all properties with some of them empty
   public class ContractExpanderStream : ViewStream {
@@ -247,3 +208,29 @@ namespace bts.SilverPine {
   }

 }
+
+namespace bts.BlueOrb {
+   using BCG;
+
+   using bts.api;
+
+   using Newtonsoft.Json.Serialization;
+
+   [DataStreamUserRespresentation("FuturesContinousContract")]
+   public class FuturesContinousContractStream : ViewStream {
+      // the argument is symbol, like "ES" etc and not the contract itself
+      public FuturesContinousContractStream(string symbol) : this(symbol, ViewStreamDatabase.instance) { }
+      public FuturesContinousContractStream(string symbol, ViewStreamDatabase db) {
+         symbol = symbol.EndsWith("*") ? symbol : symbol + "*";
+         ViewStream r = new FuturesHistorical1DayBarsStream(symbol);
+         r = new ContractExpanderStream(r, "symbol");
+         r = new FilterStream(r, "contract_type='Future'");
+         // finding the max volume across symbols, if we need to do this for open_interest, then join here 
+         // with the open interest stream on symbol/contract
+         r = new NonIncGroupByStream(r, "ts_event", "volume:Max");
+         r = new NewProjectStream(r, "ts_event symbol !");
+         r = new SortStream(r, "-ts_event");
+         mySetRealSource(r);
+      }
+   }
+}
@@ -1,4 +1,5 @@
-namespace bts.SilverPine {
+namespace bts.BlueOrb {
+   using bts.SilverPine;
   using BCG;
   using System;

@@ -1,117 +1,118 @@
-// namespace bts.SilverPine {
-//    using bts;
-//    using BCG;
-//    using MongoDB.Driver.Linq;
-//    using System.ComponentModel.DataAnnotations;
-//    using ZstdSharp.Unsafe;
+// TODO: Remove
+//// namespace bts.SilverPine 
+////    using bts;
+////    using BCG;
+////    using MongoDB.Driver.Linq;
+////    using System.ComponentModel.DataAnnotations;
+////    using ZstdSharp.Unsafe;

-//    [DataStreamUserRespresentation("VWAP")]
-//    public class VWAPStream : ViewStream {
-//       // returns the VWAP stream for each contract 
-//       private static ViewStream build_vwap_stream(string symbol, DateTime begin_dt, DateTime end_dt) {
-//          var ohlcv_params = OHLCVStream.get_params(begin_dt, end_dt);
-//          ViewStream ohlc_str = new OHLCVStream(symbol, ohlcv_params);
-//          // stream is grouped per contract and then vwap is calculated per contract
-//          GlobalMapStream rs = new GlobalMapStream(ohlc_str, "ts_event", "contract",
-//          (rows, idx) => {
-//             Log.Gen.info($"VWAPStream.GlobalMapStream delegate n_rows: {rows.Count} idx: {idx}");
-//             IBlueRow ir = rows.ElementAt((int)idx);
-//             var sel_rows = rows.Skip((int)idx).Where(r => r.get_datetime("ts_event") >= end_dt && r.get_datetime("ts_event") <= begin_dt);
-//             var volumeprices = sel_rows.Select(r => Tuple.Create(r.get<int>("volume"), r.get<double>("close"))).ToList();
-//             double vwap = IndicatorCalc.calculate_VWAP(volumeprices);
-//             IBlueTableSchema schema = ir.schema.add_columns(Tuple.Create(typeof(double), "vwap"));
-//             MutableBlueRow nr = new MutableBlueRow(schema, ir);
-//             //vwap = -313.987;
-//             nr.setValue("vwap", vwap);
-//             return nr;
-//          });
-//          return rs;
-//       }
-//       // the args want dt1 > dt2, if its nto we, swap them
-//       static void swap_if_necessary(ref DateTime dt1, ref DateTime dt2) {
-//          if (dt1 < dt2) {
-//             DateTime aux = dt1;
-//             dt1 = dt2;
-//             dt2 = aux;
-//          }
-//       }
+////    [DataStreamUserRespresentation("VWAP")]
+////    public class VWAPStream : ViewStream {
+////       // returns the VWAP stream for each contract 
+////       private static ViewStream build_vwap_stream(string symbol, DateTime begin_dt, DateTime end_dt) {
+////          var ohlcv_params = OHLCVStream.get_params(begin_dt, end_dt);
+////          ViewStream ohlc_str = new OHLCVStream(symbol, ohlcv_params);
+////          // stream is grouped per contract and then vwap is calculated per contract
+////          GlobalMapStream rs = new GlobalMapStream(ohlc_str, "ts_event", "contract",
+////          (rows, idx) => {
+////             Log.Gen.info($"VWAPStream.GlobalMapStream delegate n_rows: {rows.Count} idx: {idx}");
+////             IBlueRow ir = rows.ElementAt((int)idx);
+////             var sel_rows = rows.Skip((int)idx).Where(r => r.get_datetime("ts_event") >= end_dt && r.get_datetime("ts_event") <= begin_dt);
+////             var volumeprices = sel_rows.Select(r => Tuple.Create(r.get<int>("volume"), r.get<double>("close"))).ToList();
+////             double vwap = IndicatorCalc.calculate_VWAP(volumeprices);
+////             IBlueTableSchema schema = ir.schema.add_columns(Tuple.Create(typeof(double), "vwap"));
+////             MutableBlueRow nr = new MutableBlueRow(schema, ir);
+////             //vwap = -313.987;
+////             nr.setValue("vwap", vwap);
+////             return nr;
+////          });
+////          return rs;
+////       }
+////       // the args want dt1 > dt2, if its nto we, swap them
+////       static void swap_if_necessary(ref DateTime dt1, ref DateTime dt2) {
+////          if (dt1 < dt2) {
+////             DateTime aux = dt1;
+////             dt1 = dt2;
+////             dt2 = aux;
+////          }
+////       }

-//       // this assumes the ohlc_str has diff contracts
-//       public VWAPStream(string symbol_or_contract, DateTime begin_dt, DateTime end_dt) {
-//          swap_if_necessary(ref begin_dt, ref end_dt);
-//          Tuple<ContractUtils.ContractCategory, string> ccat_sym_contr = ContractUtils.get_symbol_category(symbol_or_contract);
-//          string symbol = String.Empty;
-//          symbol = ccat_sym_contr.Item2;
-//          var ohlcv_params = OHLCVStream.get_params(begin_dt, end_dt);
-//          ViewStream vwap_str = build_vwap_stream(symbol, begin_dt, end_dt);
-//          // vwap_str now has vwap for all contracts. if the arg was a symbol, we need to filter it for the active contract,
-//          // whihc we do with FutureActivesContractStream. If the arg was a contract we'll simply filter it
-//          ViewStream rs = vwap_str;
-//          switch (ccat_sym_contr.Item1) {
-//             case ContractUtils.ContractCategory.FutureString:
-//             case ContractUtils.ContractCategory.SpreadString:
-//                rs = new FilterStream(vwap_str, r => r.get<string>("contract").CompareTo(ccat_sym_contr.Item2) == 0);
-//                break;
-//             case ContractUtils.ContractCategory.Symbol:
-//                rs = new FuturesActiveContractStream(vwap_str);
-//                break;
-//          }
-//          rs = new FilterStream(rs, r => r.get_datetime("ts_event").CompareTo(begin_dt) == 0);
-//          rs = new NewProjectStream(rs, "! vwap");
-//          mySetRealSource(rs);
-//       }
+////       // this assumes the ohlc_str has diff contracts
+////       public VWAPStream(string symbol_or_contract, DateTime begin_dt, DateTime end_dt) {
+////          swap_if_necessary(ref begin_dt, ref end_dt);
+////          Tuple<ContractUtils.ContractCategory, string> ccat_sym_contr = ContractUtils.get_symbol_category(symbol_or_contract);
+////          string symbol = String.Empty;
+////          symbol = ccat_sym_contr.Item2;
+////          var ohlcv_params = OHLCVStream.get_params(begin_dt, end_dt);
+////          ViewStream vwap_str = build_vwap_stream(symbol, begin_dt, end_dt);
+////          // vwap_str now has vwap for all contracts. if the arg was a symbol, we need to filter it for the active contract,
+////          // whihc we do with FutureActivesContractStream. If the arg was a contract we'll simply filter it
+////          ViewStream rs = vwap_str;
+////          switch (ccat_sym_contr.Item1) {
+////             case ContractUtils.ContractCategory.FutureString:
+////             case ContractUtils.ContractCategory.SpreadString:
+////                rs = new FilterStream(vwap_str, r => r.get<string>("contract").CompareTo(ccat_sym_contr.Item2) == 0);
+////                break;
+////             case ContractUtils.ContractCategory.Symbol:
+////                rs = new FuturesActiveContractStream(vwap_str);
+////                break;
+////          }
+////          rs = new FilterStream(rs, r => r.get_datetime("ts_event").CompareTo(begin_dt) == 0);
+////          rs = new NewProjectStream(rs, "! vwap");
+////          mySetRealSource(rs);
+////       }

-//       public VWAPStream(string symbol_or_contract, string begin_dt, string end_dt)
-//       :
-//       this(symbol_or_contract, DateTime.Parse(begin_dt), DateTime.Parse(end_dt)) { }
-//    }
-// }
+////       public VWAPStream(string symbol_or_contract, string begin_dt, string end_dt)
+////       :
+////       this(symbol_or_contract, DateTime.Parse(begin_dt), DateTime.Parse(end_dt)) { }
+////    }
+//// }

-using System;
+//using System;

-using BCG;
+//using BCG;

-using bts.api;
+//using bts.api;

-namespace bts.SilverPine {
-   /*
-      ----------------------------------------------------------------------
-                                  *Expression*
-      ----------------------------------------------------------------------
-          //nl("Find Settlement price of the SP500 future in the first week of 2023.")
+//namespace bts.SilverPine {
+//   /*
+//      ----------------------------------------------------------------------
+//                                  *Expression*
+//      ----------------------------------------------------------------------
+//          //nl("Find Settlement price of the SP500 future in the first week of 2023.")

-          r=FutureHistoricalBars("ES", "One_Day", "1/1/2024", "2/1/2024")
-          r=math(r, "vprice = volume * close")
-          r=aggregate(r, "vprice:Sum, volume:Sum")
-          r=math(r, "vwap = vprice_sum / volume_sum")
-          r=newProject(r, "vwap !")
+//          r=FutureHistoricalBars("ES", "One_Day", "1/1/2024", "2/1/2024")
+//          r=math(r, "vprice = volume * close")
+//          r=aggregate(r, "vprice:Sum, volume:Sum")
+//          r=math(r, "vwap = vprice_sum / volume_sum")
+//          r=newProject(r, "vwap !")

-      ----------------------------------------------------------------------
-                                    *Schema*
-      ----------------------------------------------------------------------
-          id                              System.Int32
-          vwap                            System.Double
-   */
+//      ----------------------------------------------------------------------
+//                                    *Schema*
+//      ----------------------------------------------------------------------
+//          id                              System.Int32
+//          vwap                            System.Double
+//   */

-   [DataStreamUserRespresentation("VWAP")]
-   public class VWAPStream : ViewStream {
-      public VWAPStream(string symbol_or_contract, DateTime beg_dt, DateTime end_dt)
-      :
-      this(symbol_or_contract, beg_dt, end_dt, ViewStreamDatabase.instance) { }
+//   [DataStreamUserRespresentation("VWAP")]
+//   public class VWAPStream : ViewStream {
+//      public VWAPStream(string symbol_or_contract, DateTime beg_dt, DateTime end_dt)
+//      :
+//      this(symbol_or_contract, beg_dt, end_dt, ViewStreamDatabase.instance) { }

-      public VWAPStream(string symbol_or_contract, DateTime beg_dt, DateTime end_dt, ViewStreamDatabase db) {
-         //var fc = new ContractUtils.FutureContract(symbol_or_contract);
-         ViewStream r = new FutureHistoricalBarsStream(symbol_or_contract, beg_dt.ToString(), end_dt.ToString());
-         // since this is already filtered, we can groupBy iit without a groupBy column
-         r = new MathStream(r,
-                 "vprice = volume * close");
-         r = new NonIncGroupByStream(r,
-                                      "vprice:Sum, volume:Sum");
-         r = new MathStream(r,
-                             "vwap = vprice_sum / volume_sum");
-         r = new NewProjectStream(r,
-                                   "vwap !");
-         mySetRealSource(r);
-      }
-   }
-}
+//      public VWAPStream(string symbol_or_contract, DateTime beg_dt, DateTime end_dt, ViewStreamDatabase db) {
+//         //var fc = new ContractUtils.FutureContract(symbol_or_contract);
+//         ViewStream r = new FutureHistoricalBarsStream(symbol_or_contract, beg_dt.ToString(), end_dt.ToString());
+//         // since this is already filtered, we can groupBy iit without a groupBy column
+//         r = new MathStream(r,
+//                 "vprice = volume * close");
+//         r = new NonIncGroupByStream(r,
+//                                      "vprice:Sum, volume:Sum");
+//         r = new MathStream(r,
+//                             "vwap = vprice_sum / volume_sum");
+//         r = new NewProjectStream(r,
+//                                   "vwap !");
+//         mySetRealSource(r);
+//      }
+//   }
+//}
@@ -1,53 +1,53 @@
-using BCG;
+//using BCG;

-namespace bts.SilverPine {
-   /*
-      ----------------------------------------------------------------------
-                                  *Expression*
-      ----------------------------------------------------------------------
-          //r=FutureHistoricalBars("ES*", "02/01/2024 09:30 AM", "02/01/2024 09:30 PM")
-          r=FutureHistoricalBars("ES*","One_Day", 02/01/2024, 02/01/2025)
-          r=contractExpander(r,"symbol")
-          r=filter(r,"contract_type='FutureSpread'")
-          r=aggregate(r,"symbol","volume:Sum","ts_event")
-          r=aggregate(r,"volume_sum:Max")
-          r=newProject(r,"volume_sum_max:volume symbol !")
+//namespace bts.SilverPine {
+//   /*
+//      ----------------------------------------------------------------------
+//                                  *Expression*
+//      ----------------------------------------------------------------------
+//          //r=FutureHistoricalBars("ES*", "02/01/2024 09:30 AM", "02/01/2024 09:30 PM")
+//          r=FutureHistoricalBars("ES*","One_Day", 02/01/2024, 02/01/2025)
+//          r=contractExpander(r,"symbol")
+//          r=filter(r,"contract_type='FutureSpread'")
+//          r=aggregate(r,"symbol","volume:Sum","ts_event")
+//          r=aggregate(r,"volume_sum:Max")
+//          r=newProject(r,"volume_sum_max:volume symbol !")

-      ----------------------------------------------------------------------
-                                    *Schema*
-      ----------------------------------------------------------------------
-          symbol                          System.String
-          id                              System.Int32
-          volume                          System.Double
-   */
+//      ----------------------------------------------------------------------
+//                                    *Schema*
+//      ----------------------------------------------------------------------
+//          symbol                          System.String
+//          id                              System.Int32
+//          volume                          System.Double
+//   */

-   [DataStreamUserRespresentation("VolumeAgg")]
-   public class VolumeAggregationStream : ViewStream {
-      public VolumeAggregationStream(string future_contract, string contract_type, DateTime range_beg, DateTime range_end)
-      : this(future_contract, contract_type, range_beg, range_end, ViewStreamDatabase.instance) { }
+//   [DataStreamUserRespresentation("VolumeAgg")]
+//   public class VolumeAggregationStream : ViewStream {
+//      public VolumeAggregationStream(string future_contract, string contract_type, DateTime range_beg, DateTime range_end)
+//      : this(future_contract, contract_type, range_beg, range_end, ViewStreamDatabase.instance) { }

-      public VolumeAggregationStream(string future_contract, string contract_type, string range_beg, string range_end)
-      :
-      this(future_contract, contract_type, DateTime.Parse(range_beg), DateTime.Parse(range_end), ViewStreamDatabase.instance) { }
+//      public VolumeAggregationStream(string future_contract, string contract_type, string range_beg, string range_end)
+//      :
+//      this(future_contract, contract_type, DateTime.Parse(range_beg), DateTime.Parse(range_end), ViewStreamDatabase.instance) { }

-      public VolumeAggregationStream(string future_contract, string contract_type, DateTime range_beg, DateTime range_end, ViewStreamDatabase db) {
-         future_contract = future_contract.EndsWith("*") ? future_contract : future_contract + "*";
-         ViewStream r;
-         r = new FutureHistoricalBarsStream(future_contract,
-                                             range_beg.ToString(), range_end.ToString());
-         r = new ContractExpanderStream(r, "symbol");
-         r = new FilterStream(r, $"contract_type='{contract_type}'");
-         r = new NonIncGroupByStream(r, "symbol", "volume:Sum", "ts_event");
-         r = new NonIncGroupByStream(r, "volume_sum:Max");
-         r = new NewProjectStream(r, "volume_sum_max:volume symbol !");
-         mySetRealSource(r);
-      }
-   }
+//      public VolumeAggregationStream(string future_contract, string contract_type, DateTime range_beg, DateTime range_end, ViewStreamDatabase db) {
+//         future_contract = future_contract.EndsWith("*") ? future_contract : future_contract + "*";
+//         ViewStream r;
+//         r = new FutureHistoricalBarsStream(future_contract,
+//                                             range_beg.ToString(), range_end.ToString());
+//         r = new ContractExpanderStream(r, "symbol");
+//         r = new FilterStream(r, $"contract_type='{contract_type}'");
+//         r = new NonIncGroupByStream(r, "symbol", "volume:Sum", "ts_event");
+//         r = new NonIncGroupByStream(r, "volume_sum:Max");
+//         r = new NewProjectStream(r, "volume_sum_max:volume symbol !");
+//         mySetRealSource(r);
+//      }
+//   }

-   [DataStreamUserRespresentation("PeriodicVolumeAgg")]
-   public class PeriodicVolumeAggregationStream : ViewStream {
-      public PeriodicVolumeAggregationStream(string future_contract, string contract_type, DateTime range_beg, DateTime range_end,
-                                             TimeSpan period, ViewStreamDatabase db) {
-      }
-   }
-}
+//   [DataStreamUserRespresentation("PeriodicVolumeAgg")]
+//   public class PeriodicVolumeAggregationStream : ViewStream {
+//      public PeriodicVolumeAggregationStream(string future_contract, string contract_type, DateTime range_beg, DateTime range_end,
+//                                             TimeSpan period, ViewStreamDatabase db) {
+//      }
+//   }
+//}
@@ -76,30 +76,3 @@ namespace bts.SilverPine {
      }
   }
 }
-
-
-namespace bts.SilverPine {
-   using BCG;
-
-   using bts.api;
-
-   using Newtonsoft.Json.Serialization;
-
-   [DataStreamUserRespresentation("FuturesContinousContract")]
-   public class FuturesContinousContractStream : ViewStream {
-      // the argument is symbol, like "ES" etc and not the contract itself
-      public FuturesContinousContractStream(string symbol) : this(symbol, ViewStreamDatabase.instance) { }
-      public FuturesContinousContractStream(string symbol, ViewStreamDatabase db) {
-         symbol = symbol.EndsWith("*") ? symbol : symbol + "*";
-         ViewStream r = new FuturesHistorical1DayBarsStream(symbol);
-         r = new ContractExpanderStream(r, "symbol");
-         r = new FilterStream(r, "contract_type='Future'");
-         // finding the max volume across symbols, if we need to do this for open_interest, then join here 
-         // with the open interest stream on symbol/contract
-         r = new NonIncGroupByStream(r, "ts_event", "volume:Max");
-         r = new NewProjectStream(r, "ts_event symbol !");
-         r = new SortStream(r, "-ts_event");
-         mySetRealSource(r);
-      }
-   }
-}
@@ -1,16 +1,18 @@
-namespace bts.SilverPine {
-   using bts;
-   using BCG;

-   // it uses the volume (better to use open interest perhap) to detect the rollover, ie when a new 
-   // future exp becomes the highest volume one
-   [DataStreamUserRespresentation("FuturesRoll")]
-   public class FuturesRollStream : ViewStream {
-      public FuturesRollStream(string future_symbol) {
-         ViewStream rs = new FuturesContinousContractStream(future_symbol);
-         rs = new ColumnEvolutionStream(rs, "ts_event", "symbol");
-         rs = new NewProjectStream(rs, "symbol ts_event:roll_date !");
-         mySetRealSource(rs);
-      }
-   }
-}
+//TODO: Remove
+//namespace bts.SilverPine {
+//   using bts;
+//   using BCG;
+
+//   // it uses the volume (better to use open interest perhap) to detect the rollover, ie when a new 
+//   // future exp becomes the highest volume one
+//   [DataStreamUserRespresentation("FuturesRoll")]
+//   public class FuturesRollStream : ViewStream {
+//      public FuturesRollStream(string future_symbol) {
+//         ViewStream rs = new FuturesContinousContractStream(future_symbol);
+//         rs = new ColumnEvolutionStream(rs, "ts_event", "symbol");
+//         rs = new NewProjectStream(rs, "symbol ts_event:roll_date !");
+//         mySetRealSource(rs);
+//      }
+//   }
+//}
@@ -1,26 +1,27 @@
-namespace bts.SilverPine {
-   using BCG;
+// TODO: Remove
+//namespace bts.SilverPine {
+//   using BCG;

-   using bts;
-   // used directly in blueorb
-   [DataStreamUserRespresentation("Settlement")]
-   public class SettlementPriceStream : ViewStream {
+//   using bts;
+//   // used directly in blueorb
+//   [DataStreamUserRespresentation("Settlement")]
+//   public class SettlementPriceStream : ViewStream {

-      // public SettlementPriceStream(string future_name, string date_txt)
-      // : this(future_name, date_txt, date_txt) { }
+//      // public SettlementPriceStream(string future_name, string date_txt)
+//      // : this(future_name, date_txt, date_txt) { }

-      // public SettlementPriceStream(string future_name, string begin_dt, string end_dt)
-      // : this(future_name, DateTime.Parse(begin_dt).Date, DateTime.Parse(end_dt).Date) { }
+//      // public SettlementPriceStream(string future_name, string begin_dt, string end_dt)
+//      // : this(future_name, DateTime.Parse(begin_dt).Date, DateTime.Parse(end_dt).Date) { }

-      public SettlementPriceStream(string future_contract, DateTime begin_dt)
-      : this(future_contract, begin_dt, begin_dt) { }
+//      public SettlementPriceStream(string future_contract, DateTime begin_dt)
+//      : this(future_contract, begin_dt, begin_dt) { }

-      public SettlementPriceStream(string future_contract, DateTime begin_dt, DateTime end_dt) {
-         ContractUtils.FutureContract fc = new ContractUtils.FutureContract(future_contract);
-         ViewStream rs = new FutureHistoricalBarsStream(fc, BarSize.ONE_DAY, new DateTimeRange(begin_dt, end_dt));
-         rs = new NewProjectStream(rs, "symbol ts_event:date close:settlement !");
-         mySetRealSource(rs);
-      }
+//      public SettlementPriceStream(string future_contract, DateTime begin_dt, DateTime end_dt) {
+//         ContractUtils.FutureContract fc = new ContractUtils.FutureContract(future_contract);
+//         ViewStream rs = new FutureHistoricalBarsStream(fc, BarSize.ONE_DAY, new DateTimeRange(begin_dt, end_dt));
+//         rs = new NewProjectStream(rs, "symbol ts_event:date close:settlement !");
+//         mySetRealSource(rs);
+//      }

-   }
-}
+//   }
+//}
@@ -1,276 +1,571 @@
-# **DSL Master Learning Prompt**
+# BlueOrbDSL Master Learning Prompt v1.1 (Optimized)

-This prompt is designed to teach the foundational concepts of the DSL for financial data streaming and transformation. The DSL operates on "streams," which fetch, transform, and process financial market data in real-time or historical contexts. Forget everything I have taught you about this DSL until now (since there are some contradicting and breaking changes that can not be reconciled with anything you might have learned before) and re-learn from this prompt.
+**This version is the *final, frozen* canonical Master Learning Prompt for BlueOrbDSL model training and generation.**  
+**No further edits are required. This is the single source of truth for Learning Mode.**  
+**Reviewed and validated as optimal on \[insert date here, e.g. June 9, 2025\] after full Meta Mode review.**

-# **Introduction to Streams**
+**Change Log — v1 → v1.1**  
+(June 7, 2025\)

-The DSL operates on "streams," its core functional units. Streams fetch tabular (from database, text files or network) or transform/operate on tabular data. They either act as **Datasources** (e.g., FutureHistoricalBars), that bootstrap the data or **Operators** (e.g., `FilterStream, ProjectStream, SingleRowMathStream, GroupByStream` etc) that transform or filter tabular data. Though the DSL can work on any tabular data, it is primarily written for streaming or static financial data
+✅ Applied full Meta Mode review recommendations:

-# **Streams**
+- Standardized **Arguments** section (bullet lists, first arg \= `parent_stream` for operators)  
+- Added **Adds Columns?** and **Modifies Schema?** rows for *all* streams  
+- Clarified **Special Notes** for all streams  
+- Clearly stated UnionStream schema requirements  
+- Explicitly documented JoinStream as inner join, with left/right prefix behavior  
+- Clarified behavior of **TimeBasedRollingFilterStream**  
+- Changed DateTimeExpanderStream new columns to a bullet list for clarity  
+- Standardized all **Example Usage** to fenced `dsl` code blocks  
+- Shortened **PythonDataframeStream** example; big one should go to appendix if needed  
+- Minor language and consistency cleanups across stream definitions

-## **FutureHistoricalBars (DataSource Stream)**
-
-The `FutureHistoricalBars` stream fetches OHLCV (Open, High, Low, Close, Volume) bars for financial futures contracts. It acts as a **data source** and provides the initial table that can be further processed by operators.
-
-#### **Stream arguments (parameters):**
-
-1. **`Future_contract` (string):** Specifies the contract to fetch data for. This can be one of the following 3 types  
-1. **Continuous contracts:** Use the base symbol (e.g., `ES` for S\&P 500 futures, CL for WTI Crude Oil Futures, GC for Gold Futures etc).  
-2. **Specific contracts:** Use a detailed symbol (e.g., `ESZ9` for December 2029 S\&P 500 futures, `GCU9`, for September 2025 Gold Future, `SIH9-J9`,  
-   For SIlver March April 2019 Silver Future Calendar spread)  
-3. **All Contracts for a symbol:** If instead of specifying individual contracts, data for all contracts for a specific symbol is needed, use the symbol name followed by *.* So `ES*` for all SP Futures and spread contracts, ZC\* for all corn contracts etc.  
-2. **`Bar_size` (string):**  The second argument defines the granularity of the bars. Accepted values:  
-   * `ONE_DAY` – Daily bars.  
-     * `ONE_MIN` – Minute bars.  
-3. **Optional:**  
-   * **`range_begin` (datetime):** Start of the desired time range.  
-   * **`range_end` (datetime):** End of the desired time range.
-
-#### **Schema:**
-
-The output of `FutureHistoricalBars` produces a table with the following schema (this is a classic OHLCV schema that you can find online easily)
-
-| id   | ts\_event             | contract | open    | high    | low     | close   | volume |
-| :--- | :-------------------- | :------- | :------ | :------ | :------ | :------ | :----- |
-| `0`  | `2024-06-08T00:00:00` | `ESZ9`   | 5001.50 | 5052.25 | 4990.75 | 5025.50 | 866000 |
-|      |                       |          |         |         |         |         |        |
-
-# **FilterStream (Operator Stream)**
-
-Filters rows of a parent stream based on conditions applied to its columns.
-
-#### **Stream arguments (parameters):**
-
-1. `parent_stream (another stream)` This stream provides the data, that the FilterStream subscribers to  
-2. `filter_condition (string)` This is a text expressing a boolean condition that is applied to each row of the parent stream, if it returns false, the row is filtered out
-
-#### **Examples:**
-
-1. **NL prompt:** Show me all the days when the open price for SP 500 Futures was greater than or equal to 5000:  
-   **DSL:** `r = FilterStream(FutureHistoricalBars("ES", "ONE_DAY"), "open >= 5000")`  
-2. **NL prompt:** Show me all the days when the close price for SP 500 Futures was less than 3000 and volume was greater than half a million   
-   **DSL:** `r = FilterStream(FutureHistoricalBars("ES", "ONE_MIN"), "close < 3000 and volume > 500000)`  
-3. **NL prompt:** Show me all the daily bars for the SP 500 Future spread ESU9-Z9 where its volume exceeded 100K  
-   **DSL:** `r = FilterStream(FutureHistoricalBars("ES*", "ONE_MIN"), "contract = ‘ESU9-Z9’ and volume >= 100000”)`  
-   
-
-**Notice** how expressions can be joined with AND and OR. These expressions can be nested with parenthesis.Also string needs to be in single quotes.
-
-# **ProjectStream (Operator Stream)**
-
-Changes the name of the columns in the parent stream. So for example, if we have to rename the close column to settlement, we can do this with ProjectStream
-
-#### **Stream arguments (parameters):**
-
-3. `parent_stream (another stream)` This stream provides the data, that the ProjectStream subscribers to  
-4. `renaming (string)` This is a text expressed in the form original\_col\_name:projected\_col\_name, original\_col\_name:projected\_col\_name, 
-
-#### **Examples:**
-
-	The following DSL renames the open column to O, high to H, low to L and close to C  
-**DSL:** `r = ProjectStream(FutureHistoricalBars("ES", "ONE_DAY"), "open:O, high:H, low:L, close:C”)`
-
-# **GroupByStream (Operator Stream)**
-
-The `GroupByStream` function is a DSL operator used to perform aggregation operations on rows of a parent stream. It works similarly to SQL’s `GROUP BY`, allowing rows to be grouped by a specific column (optional) and applying aggregation functions to other columns. 
+**This version is now ready to serve as the authoritative MLP for BlueOrbDSL training and generation.**  
+When instructed to "Switch to Learning Mode," this will become the model’s single source of truth.

 ---

-#### **Stream arguments (parameters):**
+# Table of Contents

-1. **parent\_stream (another stream):** The source stream provides the data to be operated on  
-2. **Group By Column (string, optional):**
-
-   A column in the schema of the parent stream used to group rows. All rows with the same value in this column are aggregated into a single group. If omitted, the aggregation is performed over all rows of the parent stream.
-
-3. **Operation Text (string):** A string defining the column(s) to be aggregated and the operation(s) to apply, in the format:  
-   `<column_name: operation>, <column_name: operation>, …` Each `<column_name: operation>` pair produces a new column in the output stream. The new column name is formatted as `<column_name>_<operation>` in lowercase.
+1. Preamble  
+2. Streams Overview  
+3. Data Sources  
+   3.1 FutureHistoricalBarsStream  
+   3.2 LiveBBOStream  
+   3.3 LiteralStream  
+4. Operators  
+   4.1 FilterStream  
+   4.2 ProjectStream  
+   4.3 GroupByStream  
+   4.4 SingleRowMathStream  
+   4.5 MultiRowRollingMathStream  
+   4.6 UnionStream  
+   4.7 JoinStream  
+   4.8 DateTimeExpanderStream  
+   4.9 ContractExpanderStream  
+   4.10 CountStream  
+   4.11 TimeSeriesStream  
+   4.12 TimeBasedRollingFilterStream  
+   4.13 PythonDataframeStream  
+5. Reserved Words & Enumerations  
+6. Common Pitfalls

 ---

-#### **Valid Aggregation Operations:**
+## 1 Preamble

-* `Mean`: Arithmetic mean of the values.  
-* `Median`: Median of the values.  
-* `Mode`: Most frequently occurring value.  
-* `Range`: Difference between the max and min.  
-* `StdDev`: Standard deviation of the values.  
-* `Max`: Maximum value.  
-* `Min`: Minimum value.  
-* `First`: First value in the group.  
-* `Last`: Last value in the group.  
-* `Sum`: Sum of the values.  
-* `EdgeDiff`: Difference between the edge values, meaning the difference between a certain col belonging to the first and last rows respectively  
-* `EdgeDiffPercentage`: Same as `EdgeDiff` but expressed as percentage instead
+BlueOrbDSL turns natural-language finance questions into **stream pipelines** that operate on tabular market data (historical or real-time).  
+Each *stream* is either a **Data Source** (fetches data) or an **Operator** (transforms data).  
+Streams compose: the output table of one becomes the input of the next.
+
+**Convention:** For all operators, the **first argument** is always the `parent_stream`. Subsequent arguments are constructor parameters.

 ---

-#### **Examples:**
+## 2 Streams Overview

-1. **NL:**   
-   Find the total SP 500 volume executed in the first week of 2021\.  
-   **DSL:**   
-   `bars = FutureHistoricalBars("ES", “ONE_DAY”, “2021-01-03T00:00:00”,“2021-01-11T00:00:00”)`   
-   `result = GroupByStream(bars,”volume_Sum”)`  
-2. **NL:**   
-   **Find the highest price and lowest price of the  WTI Crude Oil futures over the first half of 2024**  
-   **DSL:**   
-   `bars = FutureHistoricalBars("CL", “ONE_DAY”, “2021-01-03T00:00:00”,“2021-01-06T00:00:00”)`   
-   `result = GroupByStream(bars,”high:Max low:Min”)`  
-   `result = ProjectStream(result,”high_max:highest_price min_low: lowest_price”)`  
-3. **NL:**   
-   **Find the Average daily volume per contract in Natural Gas in 2023**  
-   **DSL:**   
-   `bars = FutureHistoricalBars("NG*", “ONE_DAY”, “2023-01-03T00:00:00”,“2023-31-12T23:59:59”)`   
-   `result = GroupByStream(bars, “contract”, ”volume:Mean”)`  
-   `result = ProjectStream(result,”mean_volume:average_daily_volume”)`
+All stream definitions follow this table layout:
+
+| Field                | Meaning                                                |
+| :------------------- | :----------------------------------------------------- |
+| **Type**             | DataSource / Operator                                  |
+| **Purpose**          | One-sentence description                               |
+| **Arguments**        | Bullet list; first arg \= parent\_stream (if Operator) |
+| **Output Schema**    | Columns created or removed                             |
+| **Adds Columns?**    | Yes / No                                               |
+| **Modifies Schema?** | Yes / No                                               |
+| **Example Usage**    | Fenced `dsl` block                                     |
+| **Special Notes**    | Constraints / caveats                                  |

 ---

-#### **Schema Impact:**
+## 3 Data Sources

-The `AggregateStream` modifies the schema of the parent stream by adding new columns for each aggregation operation. Existing columns are unaffected.
+### 3.1 FutureHistoricalBarsStream

-# **SingleRowMathStream (Operator Stream)**
+| Field                | Description                                                                                                                                                                                                |
+| :------------------- | :--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| **Type**             | DataSource                                                                                                                                                                                                 |
+| **Purpose**          | Fetch historical OHLCV bars for futures contracts                                                                                                                                                          |
+| **Arguments**        | • `contract` — continuous (`ES`), specific (`ESZ9`), or wildcard (`ES*`) • \``bar_size`\` — ONE\_DAY, ONE\_MIN • `` `range_begin` `` — ISO datetime, optional • `` `range_end` `` — ISO datetime, optional |
+| **Output Schema**    | `id, ts_event, date, contract, open, high, low, close, volume`                                                                                                                                             |
+| **Adds Columns?**    | Yes                                                                                                                                                                                                        |
+| **Modifies Schema?** | No                                                                                                                                                                                                         |
+| **Example Usage**    | `bars = FutureHistoricalBarsStream("ES", "ONE_DAY", "2023-01-01", "2023-12-31")`                                                                                                                           |
+| **Special Notes**    | Whereas the ts\_event column is the timestamp of the column, date is the date of the day.                                                                                                                  |

-The `SingleRowMathStream` is a DSL operator used to perform math operations, one row at a time, on the parent stream. 
+### 3.2 LiveBBOStream
+
+| Field                | Description                                                                                                                                                                |
+| :------------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| **Type**             | DataSource                                                                                                                                                                 |
+| **Purpose**          | Stream live best-bid/offer quotes                                                                                                                                          |
+| **Arguments**        | • `contract` — e.g. `"NGU5"` or `"CL*"`                                                                                                                                    |
+| **Output Schema**    | `ts_event, contract, bid_price, bid_size, ask_price, ask_size`                                                                                                             |
+| **Adds Columns?**    | Yes                                                                                                                                                                        |
+| **Modifies Schema?** | No                                                                                                                                                                         |
+| **Example Usage**    | `bbo = LiveBBOStream("NGU5")`                                                                                                                                              |
+| **Special Notes**    | LiveBBOStream produces the latest value of the bid/ask price and size. It does not maintain history. Each new row overwrites the old one. For history use TimeSeriesStream |
+
+### 3.3 LiteralStream
+
+| Field                | Description                                                                                                       |
+| :------------------- | :---------------------------------------------------------------------------------------------------------------- |
+| **Type**             | DataSource                                                                                                        |
+| **Purpose**          | Inject small static tables via JSON records                                                                       |
+| **Arguments**        | • `json_text` — string                                                                                            |
+| **Output Schema**    | Inferred from JSON keys                                                                                           |
+| **Adds Columns?**    | Yes                                                                                                               |
+| **Modifies Schema?** | No                                                                                                                |
+| **Example Usage**    | `meta = LiteralStream('[{"event":"OPEC","date":"2024-06-01"}]')`                                                  |
+| **Special Notes**    | If you download the data from the internet, and want to use it conjunction with the BlueOrbDSL this can be useful |

 ---

-#### **Stream arguments (parameters):**
+## 4 Operators

-1. **parent\_stream (another stream):**The source stream provides the base data  
-2. **math\_expression(string):** This is a math expression which allows the creation of one or more new columns that are the output of a math expression on the existing columns provided by parent\_stream. This can a simple math operation like new\_col \= a \+ b, or any function from C\#’s System.Math namespace, like new\_column \= Math.Log(Math.Pow(volume, 1.1))
+### 4.1 FilterStream

-#### **Examples:**
+| Field                | Description                                                                                                                  |
+| :------------------- | :--------------------------------------------------------------------------------------------------------------------------- |
+| **Type**             | Operator                                                                                                                     |
+| **Purpose**          | Keep rows where a boolean condition is true                                                                                  |
+| **Arguments**        | • `parent_stream` •`` `condition` `` — string                                                                                |
+| **Output Schema**    | Same as input                                                                                                                |
+| **Adds Columns?**    | No                                                                                                                           |
+| **Modifies Schema?** | No                                                                                                                           |
+| **Special Notes**    | Condition syntax: use ASCII single quotes (`contract = 'ESZ9'`). Supports `and`, `or`, parentheses, equality and inequality. |

-4. **NL:**   
-   Find the daily price movement in the SP 500 Futures through the month of Jan  
-   **DSL:**   
-   `bars = FutureHistoricalBars("ES", “ONE_DAY”, “2024-01-01T00:00:00”,“2024-01-31T00:00:00”)`   
-   `result = SingleRowMathStream(bars,”daily_move=close-open”)`  
-5. **NL:**   
-   Find the daily typical price of the  WTI Crude Oil futures over the first half of 2024  
-   **DSL:**   
-   `bars = FutureHistoricalBars("CL", “ONE_DAY”, “2021-01-03T00:00:00”,“2021-01-06T00:00:00”)`   
-   `result = SingleRowMathStream(bars, “typical_price = (high+low+close)/3.0”)`
+### 4.2 ProjectStream

-#### **Input Space (Scope):** The scope of this operation is a **single row.** The SingleRowMAthStream as the name signifies, operates on columns of the same row only. **No multi row operations are allowed.** For multi row operations using what follows.
-
-**Schema Impact:**
-
-# **MultiRowRollingMathStream (Operator Stream)**
-
-The `MultiRowRollingMathStream` function is a DSL operator used to perform mostly math operations on a rolling set of rows of a parent stream. It works similarly to the GroupByStream (described above) but instead of grouping rows together based on a column value, it operates on a rolling set of rows (where the rolling size is provided as an argument). 
+| Field                | Description                                                                         |
+| :------------------- | :---------------------------------------------------------------------------------- |
+| **Type**             | Operator                                                                            |
+| **Purpose**          | Rename or drop columns                                                              |
+| **Arguments**        | • `parent_stream` • `` `column_map ``\` — string                                    |
+| **Output Schema**    | Renamed subset                                                                      |
+| **Adds Columns?**    | No                                                                                  |
+| **Modifies Schema?** | Yes                                                                                 |
+| **Example Usage**    | `clean = ProjectStream(bars, "close:settlement, volume")`                           |
+| **Special Notes**    | **Cannot add new columns or constants.** Use LiteralStream \+ JoinStream if needed. |

 ---

-#### **Stream arguments (parameters):**
+### 4.3 GroupByStream

-1. **Parent Stream (another stream):** The source stream provides the data to be operated on  
-2. **Operation Text (string):** A string defining the column(s) to be operated and the operation(s) to apply, in the format:  
-   `<column_name: operation>, <column_name: operation>, …` Each `<column_name: operation>` pair produces a new column in the output stream. The new column name is formatted as `<column_name>_<operation>` in lowercase. This is like the GroupByStream’s operations string.   
-3. **Sort/Order Column Name(string):** This controls the ordering of the rolling rows. This argument will be in the format \-`<column_name>` to sort the rolling rows in descending order by the `<column_name>`, or \+`<column_name> in` ascending order.  
-4. **Rolling Window Size(int)(optional):** The size of the rolling window, If this argument is left out all, the operations are applied to all rows, is Global Operation instead of Rolling Window Operation (it's like the optional groupBy col in GroupByStream).
+| Field                | Description                                                                                            |
+| :------------------- | :----------------------------------------------------------------------------------------------------- |
+| **Type**             | Operator                                                                                               |
+| **Purpose**          | Aggregates rows by key (or globally)                                                                   |
+| **Arguments**        | • `parent_stream` • `` `group_by` `` — string (blank for global) • `` `aggregation_string` `` — string |
+| **Output Schema**    | Adds `<column>_<aggregation>` columns                                                                  |
+| **Adds Columns?**    | Yes                                                                                                    |
+| **Modifies Schema?** | Yes                                                                                                    |
+| **Example Usage**    | `daily = GroupByStream(bars, "date", "volume:Sum")`                                                    |
+| **Special Notes**    | None                                                                                                   |
+
+### 4.4 SingleRowMathStream
+
+| Field                | Description                                                                  |
+| :------------------- | :--------------------------------------------------------------------------- |
+| **Type**             | Operator                                                                     |
+| **Purpose**          | Add new columns from row-wise math expressions                               |
+| **Arguments**        | • `parent_stream` • `` `math_expression` `` — string                         |
+| **Output Schema**    | Adds new column(s)                                                           |
+| **Adds Columns?**    | Yes                                                                          |
+| **Modifies Schema?** | No                                                                           |
+| **Example Usage**    | `r = SingleRowMathStream(bars, "daily_return = close - open")`               |
+| **Special Notes**    | Operates on one row at a time only. Cannot reference values from other rows. |
+
+### 4.5 MultiRowRollingMathStream
+
+| Field                | Description                                                                                                                                 |
+| :------------------- | :------------------------------------------------------------------------------------------------------------------------------------------ |
+| **Type**             | Operator                                                                                                                                    |
+| **Purpose**          | Aggregates over rolling window of rows                                                                                                      |
+| **Arguments**        | • `parent_stream` • `` `aggregation_string` `` — string • `` `sort_column` `` — string with \+/- prefix • `` `rolling_window_size` `` — int |
+| **Output Schema**    | Adds `<column>_<aggregation>` columns                                                                                                       |
+| **Adds Columns?**    | Yes                                                                                                                                         |
+| **Modifies Schema?** | Yes                                                                                                                                         |
+| **Example Usage**    | `ma20 = MultiRowRollingMathStream(bars, "close:Mean", "-ts_event", 20)`                                                                     |
+| **Special Notes**    | sort\_column should be prefixed with \+ (ascending) or \- (descending). Example: "-ts\_event".                                              |
+
+### 4.6 UnionStream
+
+| Field                | Description                                              |
+| :------------------- | :------------------------------------------------------- |
+| **Type**             | Operator                                                 |
+| **Purpose**          | Append rows from two streams                             |
+| **Arguments**        | • `parent_stream1` • `parent_stream2`                    |
+| **Output Schema**    | Same as inputs                                           |
+| **Adds Columns?**    | No                                                       |
+| **Modifies Schema?** | No                                                       |
+| **Example Usage**    | `unioned = UnionStream(bars1, bars2)`                    |
+| **Special Notes**    | **Schemas must match exactly (column names AND types).** |
+
+### 4.7 JoinStream
+
+| Field                | Description                                                                                                                                                           |
+| :------------------- | :-------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| **Type**             | Operator                                                                                                                                                              |
+| **Purpose**          | Join two streams on equality of selected columns                                                                                                                      |
+| **Arguments**        | • `` `left_parent_stream` ``   • `` `left_join_cols` `` — comma-separated string   • `` `right_parent_stream` ``   • `` `right_join_cols` `` — comma-separated string |
+| **Output Schema**    | Combined columns from both streams                                                                                                                                    |
+| **Adds Columns?**    | Yes                                                                                                                                                                   |
+| **Modifies Schema?** | Yes                                                                                                                                                                   |
+| **Example Usage**    | `joined = JoinStream(a, "ts_event", b, "ts_event")`                                                                                                                   |
+| **Special Notes**    | Always performs an **inner join** on equality. If columns overlap, prefix applied (`left_`, `right_`). If join columns args left empty, does cartesian product        |
+
+### 4.8 DateTimeExpanderStream
+
+| Field                | Description                                                                                                                        |
+| :------------------- | :--------------------------------------------------------------------------------------------------------------------------------- |
+| **Type**             | Operator                                                                                                                           |
+| **Purpose**          | Adds columns with date/time components from a datetime column                                                                      |
+| **Arguments**        | • `parent_stream` • `datetime_column` — string                                                                                     |
+| **Output Schema**    | Adds: • \`\_date\` • \`\_day\_of\_week\` • \`\_day\_of\_month\` • \`\_month\` • \`\_hour\` • \`\_minute\` • \`\_seconds\_of\_day\` |
+| **Adds Columns?**    | Yes                                                                                                                                |
+| **Modifies Schema?** | Yes                                                                                                                                |
+| **Example Usage**    | `expanded = DateTimeExpanderStream(bars, "ts_event")`                                                                              |
+| **Special Notes**    | None                                                                                                                               |
+
+### 4.9 ContractExpanderStream
+
+| Field                | Description                                           |
+| :------------------- | :---------------------------------------------------- |
+| **Type**             | Operator                                              |
+| **Purpose**          | Adds metadata columns based on `contract`             |
+| **Arguments**        | • `parent_stream` • contract\_column — string         |
+| **Output Schema**    | Adds: `• contract_type`                               |
+| **Adds Columns?**    | Yes                                                   |
+| **Modifies Schema?** | Yes                                                   |
+| **Example Usage**    | `expanded = ContractExpanderStream(bars, "contract")` |
+| **Special Notes**    | None                                                  |
+
+### 4.10 CountStream
+
+| Field                | Description                       |
+| :------------------- | :-------------------------------- |
+| **Type**             | Operator                          |
+| **Purpose**          | Count number of rows              |
+| **Arguments**        | • `parent_stream`                 |
+| **Output Schema**    | `id, count`                       |
+| **Adds Columns?**    | Yes                               |
+| **Modifies Schema?** | Yes                               |
+| **Example Usage**    | `total_count = CountStream(bars)` |
+| **Special Notes**    | None                              |
+
+### 4.11 TimeSeriesStream
+
+| Field                | Description                                                          |
+| :------------------- | :------------------------------------------------------------------- |
+| **Type**             | Operator                                                             |
+| **Purpose**          | Append each update from parent stream into time series               |
+| **Arguments**        | • `parent_stream` • \``buffer_size`\` — int                          |
+| **Output Schema**    | Parent schema \+ `timestamp` (if not already present)                |
+| **Adds Columns?**    | Yes                                                                  |
+| **Modifies Schema?** | No                                                                   |
+| **Example Usage**    | `ts_bbo = TimeSeriesStream(bbo, 10000)`                              |
+| **Special Notes**    | Append-only stream, does not deduplicate or collapse repeated values |
+
+### 4.12 TimeBasedRollingFilterStream
+
+| Field                | Description                                                                  |
+| :------------------- | :--------------------------------------------------------------------------- |
+| **Type**             | Operator                                                                     |
+| **Purpose**          | Keep rows newer than rolling window of N seconds                             |
+| **Arguments**        | • `parent_stream` • \``no_of_seconds`\` — int                                |
+| **Output Schema**    | Same as input                                                                |
+| **Adds Columns?**    | No                                                                           |
+| **Modifies Schema?** | No                                                                           |
+| **Example Usage**    | `recent = TimeBasedRollingFilterStream(trades, 60)`                          |
+| **Special Notes**    | Drops rows older than `no_of_seconds` compared to current max timestamp seen |
+
+### 4.13 PythonDataframeStream
+
+| Field                | Description                                                                                                             |
+| :------------------- | :---------------------------------------------------------------------------------------------------------------------- |
+| **Type**             | Operator                                                                                                                |
+| **Purpose**          | Apply arbitrary Python transformation to DataFrame                                                                      |
+| **Arguments**        | • `parent_stream` • `` `python_function` `` — string                                                                    |
+| **Output Schema**    | Defined by function                                                                                                     |
+| **Adds Columns?**    | Yes/No — depends on function                                                                                            |
+| **Modifies Schema?** | Yes/No — depends on function                                                                                            |
+| **Example Usage**    | `r = PythonDataframeStream(bars, "def transform(df): return df[df.volume > 1e6]")`                                      |
+| **Special Notes**    | **Use only as last resort.** Function should be small and well-documented. Avoid large multi-step logic where possible. |

 ---

-#### **Valid Aggregation Operations:**
+## 5 Reserved Words & Enumerations

-* `Mean`: Arithmetic mean of the values.  
-* `Median`: Median of the values.  
-* `Mode`: Most frequently occurring value.  
-* `Range`: Difference between the max and min.  
-* `StdDev`: Standard deviation of the values.  
-* `Max`: Maximum value.  
-* `Min`: Minimum value.  
-* `First`: First value in the group.  
-* `Last`: Last value in the group.  
-* `Sum`: Sum of the values.  
-* `EdgeDiff`: Difference between the edge values, meaning the difference between a certain col belonging to the first and last rows respectively  
-* `EdgeDiffPercentage`: Same as `EdgeDiff` but expressed as percentage instead
+- **Bar\_size:** `ONE_DAY`, `ONE_MIN`  
+- **Aggregation Operations:** `Mean`, `Median`, `Mode`, `Sum`, `Max`, `Min`, `StdDev`, `Range`, `First`, `Last`, `EdgeDiff`, `EdgeDiffPercentage`

 ---

-#### **Examples:**
+## 6 Common Pitfalls

-1. **NL:**   
-   **Find the 20 d MA of SP 500 Futures close prices**  
-   `bars = FutureHistoricalBars("ES", “ONE_DAY”, “2021-01-03T00:00:00”,“2021-01-11T00:00:00”)`   
-   `result = MultirowRollingMathStream(bars, “close:Mean”, “-ts_event”, 20)`  
-2. **NL:**   
-   **Find the 90d Standard Deviation of WTI Crude Oil during 2023**
+| Pitfall                                         | Correct Approach                                     |
+| :---------------------------------------------- | :--------------------------------------------------- |
+| Adding literal columns in ProjectStream         | Use LiteralStream → JoinStream                       |
+| Smart quotes in conditions                      | Use `'...'` ASCII quotes                             |
+| Large functions in PythonDataframeStream        | Keep minimal or move to appendix                     |
+| Ambiguous TimeBasedRollingFilterStream behavior | Now clearly stated: drops rows older than N seconds. |

-   **DSL:**   
-   `bars = FutureHistoricalBars("CL", “ONE_DAY”, “2023-01-01T00:00:00”,“2023-12-31T00:00:00”)`  
-   `// calculating typical_price`
+## 7 Reinforced Learning DSL Examples

-   `result = SingleRowMathStream(bars, “typical_price = (high+low+close)/3.0”)`   
-   `result = MultirowRollingMathStream(bars,“typical_price:StdDev”, “-ts_event”, 90)`
+This section contains **complete worked examples** of BlueOrbDSL pipelines answering complex natural language queries.

-   `result=ProjectStream(result, “stddev_typical_price:90dStdDev”)`
+Each example shows:

-3. **NL:**   
-   **Find the Average daily volume of Natural Gas in 2023**  
-   **DSL:**   
-   `bars = FutureHistoricalBars("NG", “ONE_DAY”, “2023-01-03T00:00:00”,“2023-31-12T23:59:59”)`  
-   `result = MultirowRollingMathStream(bars, ”volume:Mean”, “-ts_event”)`  
-   `result = ProjectStream(result,”mean_volume:average_daily_volume”)`
+- The **NL Query** (natural language question)  
+- The **DSL Solution** in full  
+- Optional **Notes**

 ---

-#### **Schema Impact:**
+### Example 1 — On which dates did SP 500 Futures open 2% higher than the previous day's close?

-The `MultirowRollingMathStream` modifies the schema of the parent stream by adding new columns for each operation. Existing columns are unaffected.
+#### DSL Solution

-# **UnionStream (Operator Stream)**
+```
+bars = FutureHistoricalBarsStream("ES", "ONE_DAY", "1900-01-01T00:00:00", "2100-12-31T23:59:59")
+rolling_bars = MultiRowRollingMathStream(bars, "close:First, open:Last", "+ts_event", 2)
+percent_change = SingleRowMathStream(rolling_bars, "percent_change = 100 * (open_last - close_first) / close_first")
+filtered_bars = FilterStream(percent_change, "percent_change >= 2")
+result = ProjectStream(filtered_bars, "ts_event, percent_change")
+```

-Like the SQL Union appends all the rows from 2 streams together.
+---

-#### **Stream arguments (parameters):**
+### Example 2 — What is the average price movement for e-minis on Wednesday between 9:30 am and 9:40 AM CST?

-5. `parent_stream1 (first stream)` This stream provides part of the data that will will be appended to the other part, provided by the following argument  
-6. `parent_stream2 (second stream)` This stream provides the other part of the data that will will be appended to the first part, provided by the first argument
+#### DSL Solution

-#### **Examples:**
+```
+bars = FutureHistoricalBarsStream("ES", "ONE_MIN", "1900-01-01T00:00:00", "2100-12-31T23:59:59")
+expanded_bars = DateTimeExpanderStream(bars, "ts_event")
+filtered_bars = FilterStream(expanded_bars, "ts_event_day_of_week = 'Wednesday' and ts_event_hour = 9 and ts_event_minute >= 30 and ts_event_minute < 40")
+price_movement = SingleRowMathStream(filtered_bars, "movement = close - open")
+average_movement = GroupByStream(price_movement, "", "movement:Mean")
+result = ProjectStream(average_movement, "movement_mean:average_price_movement")
+```

-4. **NL prompt:** Show me all the bars across SP 500 Futures and Nasdaq Futures  
-   **DSL:** `r = UnionStream(FutureHistoricalBars("ES", "ONE_DAY"), FutureHistoricalBars("NQ", "ONE_DAY"))`  
-   
+---

-#### **Schema Impact:**
+### Example 3 — What did the volume and price of ZN in the 5 minutes following the last 10 FOMC rates?

-None
+#### DSL Solution

-# **DateTimeExpanderStream (Operator Stream)**
+```
+bars_1 = FutureHistoricalBarsStream("ZN", "ONE_MIN", "2023-11-01T14:00:00", "2023-11-01T14:05:00")
+bars_2 = FutureHistoricalBarsStream("ZN", "ONE_MIN", "2023-09-20T14:00:00", "2023-09-20T14:05:00")
+bars_3 = FutureHistoricalBarsStream("ZN", "ONE_MIN", "2023-07-26T14:00:00", "2023-07-26T14:05:00")
+bars_4 = FutureHistoricalBarsStream("ZN", "ONE_MIN", "2023-06-14T14:00:00", "2023-06-14T14:05:00")
+bars_5 = FutureHistoricalBarsStream("ZN", "ONE_MIN", "2023-05-03T14:00:00", "2023-05-03T14:05:00")
+bars_6 = FutureHistoricalBarsStream("ZN", "ONE_MIN", "2023-03-22T14:00:00", "2023-03-22T14:05:00")
+bars_7 = FutureHistoricalBarsStream("ZN", "ONE_MIN", "2023-02-01T14:00:00", "2023-02-01T14:05:00")
+bars_8 = FutureHistoricalBarsStream("ZN", "ONE_MIN", "2022-12-14T14:00:00", "2022-12-14T14:05:00")
+bars_9 = FutureHistoricalBarsStream("ZN", "ONE_MIN", "2022-11-02T14:00:00", "2022-11-02T14:05:00")
+bars_10 = FutureHistoricalBarsStream("ZN", "ONE_MIN", "2022-09-21T14:00:00", "2022-09-21T14:05:00")

-DateTimExpanderStream expands one datetime column into various datetime components, like seconds in day, day of week, date etc, so new columns are added for those. The existing columns remain as is.
+all_bars = UnionStream(bars_1, bars_2)
+all_bars = UnionStream(all_bars, bars_3)
+all_bars = UnionStream(all_bars, bars_4)
+all_bars = UnionStream(all_bars, bars_5)
+all_bars = UnionStream(all_bars, bars_6)
+all_bars = UnionStream(all_bars, bars_7)
+all_bars = UnionStream(all_bars, bars_8)
+all_bars = UnionStream(all_bars, bars_9)
+all_bars = UnionStream(all_bars, bars_10)

-#### **Stream arguments (parameters):**
+grouped_data = GroupByStream(all_bars, "date", "open:First, close:Last, high:Max, low:Min, volume:Sum")
+result = ProjectStream(grouped_data, "ts_event_date, open_first:5_min_open, close_last:5_min_close, high_max:5_min_high, low_min:5_min_low, volume_sum:5_min_volume")
+```

-1. `parent_stream1 (base stream)` This parent stream that provides the data, including the datetime columns that will be expanded into multiple columns  
-2. `Datetime_input_column (string)` This is the column that will be expanded into multiple columns.
+---

-**New Columns:**  
-`<input_dt_col>_date` (Date of the value, without the time part)
+### Example 4 — How much volume of RBOB typically trades during the settlement window?

-* `<input_dt_col>_day,` (Day of the week)  
-* `<input_dt_col>_day_of_month` (day of the month, This is an int between 0 and 31\)  
-* `<input_dt_col>_month,` (month of the year, this is a numerical value between 1 and 12  
-* `input_dt_col>_hour,` (Hour of the day, value bw 0 and 23\)  
-* `<input_dt_col>_min",` (month part of this DateTime. Value is integer between 1 and 12).  
-* `<input_dt_col>_seconds_of_day",` (Total seconds of the day) 
+#### DSL Solution

-#### **Examples:**
+```
+rbob_bars = FutureHistoricalBarsStream("RB", "ONE_MIN", "2023-01-01T00:00:00", "2023-12-31T23:59:59")
+expanded_bars = DateTimeExpanderStream(rbob_bars, "ts_event")
+settlement_window = FilterStream(expanded_bars, "(ts_event_hour = 14 and ts_event_minute >= 28) or (ts_event_hour = 14 and ts_event_minute < 30)")
+typical_settlement_volume = GroupByStream(settlement_window, "ts_event_date", "volume:Sum")
+result = ProjectStream(typical_settlement_volume, "ts_event_date, volume_sum:daily_settlement_volume")
+```

-**DSL:** `r = DateTimeExpanderStream(FutureHistoricalBars("ES", "ONE_DAY"), “ts_event”)`   
-**Input Schema:**
+---

-| id   | ts\_event | contract | open | high | low  | close | volume |
-| :--- | :-------- | :------- | :--- | :--- | :--- | :---- | :----- |
+### Example 5 — How much SP Futures during the settlement window, average per day, during the 2 week windows before future expiration, in 2023?

-**Output Schema:**
+#### DSL Solution

-| id   | ts\_event | contract | open | high | low  | close | volume |      |      |      |      |      |      |      |
-| :--- | :-------- | :------- | :--- | :--- | :--- | :---- | :----- | :--- | :--- | :--- | :--- | :--- | :--- | :--- |
+```
+sp_bars = FutureHistoricalBarsStream("ES", "ONE_MIN", "2023-01-01T00:00:00", "2023-12-31T23:59:59")
+expanded_bars = DateTimeExpanderStream(sp_bars, "ts_event")
+settlement_window = FilterStream(expanded_bars, "(ts_event_hour = 15 and ts_event_minute >= 58) or (ts_event_hour = 16 and ts_event_minute < 1)")
+filtered_bars = FilterStream(settlement_window, "(ts_event_date >= '2023-03-03' and ts_event_date < '2023-03-17') or (ts_event_date >= '2023-06-02' and ts_event_date < '2023-06-16') or (ts_event_date >= '2023-09-01' and ts_event_date < '2023-09-15') or (ts_event_date >= '2023-12-01' and ts_event_date < '2023-12-15')")
+daily_volume = GroupByStream(filtered_bars, "ts_event_date", "volume:Sum")
+average_daily_volume = GroupByStream(daily_volume, "", "volume_sum:Mean")
+result = ProjectStream(average_daily_volume, "volume_sum_mean:average_daily_volume")
+```

+---
+
+### Example 6 — Find the difference between the number of days in 2023 that SP 500 went up 2% and when it went down 2%
+
+#### DSL Solution
+
+```
+bars = FutureHistoricalBarsStream("ES", "ONE_DAY", "2023-01-01T00:00:00", "2023-12-31T23:59:59")
+percentage_change = SingleRowMathStream(bars, "percent_change = 100 * (close - open) / open")
+days_up_2_percent = FilterStream(percentage_change, "percent_change >= 2")
+days_up_count = CountStream(days_up_2_percent)
+days_down_2_percent = FilterStream(percentage_change, "percent_change <= -2")
+days_down_count = CountStream(days_down_2_percent)
+joined_counts = JoinStream(days_up_count, "", days_down_count, "")
+difference_result = SingleRowMathStream(joined_counts, "difference = days_up_count - days_down_count")
+result = ProjectStream(difference_result, "difference")
+```
+
+---
+
+### Example 7 — Monitor which CL contracts (CL) are trading with bid size \>= 1000, and record the last 1000 events\*
+
+#### DSL Solution
+
+```
+bbo = LiveBBOStream("CL*")
+large_bid = FilterStream(bbo, "bid_size >= 1000")
+large_bid_ts = TimeSeriesStream(large_bid, 1000)
+result = ProjectStream(large_bid_ts, "timestamp, ts_event, contract, bid_size, bid_price, ask_price")
+```
+
+---
+
+### Example 8 — Identify FutureSpreads in CrudeOil and monitor their spread width
+
+#### DSL Solution
+
+```
+bbo = LiveBBOStream("CL*")
+expanded_bbo = ContractExpanderStream(bbo, "contract")
+future_spreads = FilterStream(expanded_bbo, "contract_type = 'FutureSpread'")
+spread_stream = SingleRowMathStream(future_spreads, "spread = ask_price - bid_price")
+result = ProjectStream(spread_stream, "ts_event, contract, spread")
+```
+
+---
+
+### Example 9 — Track when bid prices in NaturalGas Futures cross above 3.00 and keep only the last 10 seconds of such events
+
+#### DSL Solution
+
+```
+bbo = LiveBBOStream("NG")
+high_bid = FilterStream(bbo, "bid_price > 3.00")
+high_bid_ts = TimeSeriesStream(high_bid, 10000)
+recent_high_bid = TimeBasedRollingFilterStream(high_bid_ts, 10)
+result = ProjectStream(recent_high_bid, "ts_event, contract, bid_price, bid_size, ask_price")
+```
+
+---
+
+### Example 11 — Detect sequences in NaturalGas future where bid\_price increased 3 times consecutively within a rolling 30-second window
+
+#### DSL Solution
+
+```
+bbo = LiveBBOStream("NG")
+bbo_ts = TimeSeriesStream(bbo, 10000)
+recent_bbo = TimeBasedRollingFilterStream(bbo, 30)  // 30-second window of updates
+
+result = PythonDataframeStream(recent_bbo, "
+def transform(df):
+    import pandas as pd
+    df = df.sort_values('ts_event')
+    df['bid_increase'] = df['bid_price'].diff() > 0
+    df['consec_increase'] = df['bid_increase'].rolling(window=3).sum()
+    # Find rows where we just reached 3 consecutive increases
+    df['trigger'] = (df['consec_increase'] == 3) & (df['consec_increase'].shift(1) < 3)
+    return df[df['trigger']][['ts_event', 'contract', 'bid_price', 'bid_size']]
+")
+```
+
+---
+
+### Example 12 — When we get a 5+ level bid sweep in SP 500, show me how the bid behaves in the 10 minutes after — mean, stddev, min/max bid\_price
+
+#### DSL Solution
+
+```
+bbo = LiveBBOStream("ES")
+bbo_ts = TimeSeriesStream(bbo, 10000)
+recent_bbo = TimeSeriesStream(bbo, 5000)  // keep large enough window for event tracking
+
+result = PythonDataframeStream(recent_bbo, "
+def transform(df):
+    import pandas as pd
+    df = df.sort_values('ts_event')
+    rows = []
+    in_measurement = False
+    measurement_start_time = None
+    measurement_end_time = None
+    bid_prices_in_window = []
+    min_bid_size_threshold = 100  # defines what we consider 'stable large bid'
+    min_levels_stable = 5
+    prior_bid_sizes = []
+
+    for i, row in df.iterrows():
+        bid_price = row['bid_price']
+        bid_size = row['bid_size']
+        ts = pd.to_datetime(row['ts_event'])
+
+        # Maintain rolling window of prior bid_sizes
+        prior_bid_sizes.append(bid_size)
+        if len(prior_bid_sizes) > min_levels_stable:
+            prior_bid_sizes.pop(0)
+
+        if not in_measurement:
+            # Detect sweep trigger
+            if all(x > min_bid_size_threshold for x in prior_bid_sizes[:-1]) and bid_size < min_bid_size_threshold:
+                # Sweep detected → start measurement window
+                measurement_start_time = ts
+                measurement_end_time = measurement_start_time + pd.Timedelta(minutes=10)
+                bid_prices_in_window = []
+                in_measurement = True
+        else:
+            # Inside measurement window
+            if ts <= measurement_end_time:
+                bid_prices_in_window.append(bid_price)
+            else:
+                # Window complete → emit stats
+                if bid_prices_in_window:
+                    mean_bid = pd.Series(bid_prices_in_window).mean()
+                    stddev_bid = pd.Series(bid_prices_in_window).std()
+                    min_bid = pd.Series(bid_prices_in_window).min()
+                    max_bid = pd.Series(bid_prices_in_window).max()
+                    rows.append({
+                        'sweep_time': measurement_start_time,
+                        'measurement_end': measurement_end_time,
+                        'mean_bid_price': mean_bid,
+                        'stddev_bid_price': stddev_bid,
+                        'min_bid_price': min_bid,
+                        'max_bid_price': max_bid
+                    })
+                # Reset state
+                in_measurement = False
+                measurement_start_time = None
+                measurement_end_time = None
+                bid_prices_in_window = []
+                prior_bid_sizes = []
+
+    return pd.DataFrame(rows)
+")
+
+```
+
+*End of Section 7 — Reinforced Learning DSL Examples*  
@@ -0,0 +1,27 @@
+**Enter Meta Mode**  
+ You are not learning the DSL yet.  
+ You are reviewing the following **BlueOrbDSL Master Learning Prompt** (MLP) as a *training document for yourself*.
+
+Your goal is to optimize it to make it easier for you — or any future LLM — to learn from it and generate DSL code with high accuracy.
+
+Please analyze the MLP and provide:
+
+1. **Clarity review** — is each stream definition clear, unambiguous, easy to parse?
+
+2. **Consistency review** — do all stream definitions follow the same structure and style?
+
+3. **Terminology check** — are key terms (parent stream, arguments, schema impact, example usage, etc.) used consistently?
+
+4. **Structural feedback** — is the overall document structured in a way that supports learning (headings, bold, tables, code blocks)?
+
+5. **Formatting feedback** — are important warnings / "cannot do X" constraints highlighted clearly?
+
+6. **Missing info** — is anything missing from any stream definition that would help you understand how to use it?
+
+7. **Generation risk spots** — any parts where you foresee yourself making common mistakes due to ambiguity?
+
+**Do not attempt to "learn" this DSL yet.** Stay in Meta Mode and provide feedback as an *editor and optimizer*.
+
+When I later say:  
+ → **Switch to Learning Mode — this is now the final BlueOrbDSL MLP** → you will forget your Meta comments and treat the MLP as the single authoritative source for DSL generation.
+
@@ -43,7 +43,7 @@
      Condition="Exists($(ComputerSpecificConfig))" />

  <PropertyGroup Label="ProtoBuf">
-    <ProtocPluginBase>$(SparkRoot)\src\lib\api\service_compiler</ProtocPluginBase>
+    <ProtocPluginBase>$(SparkRoot)\src\lib_spark\spark\api\service_compiler</ProtocPluginBase>
  </PropertyGroup>
  <Choose>
    <When Condition="'$(OS)' == 'Windows_NT'">
@@ -24,7 +24,7 @@
    <PropertyGroup>
      <GenProtosCmd>$(protoc) %(Proto.FullPath)</GenProtosCmd>
      <GenProtosCmd>$(GenProtosCmd) --error_format=msvs</GenProtosCmd>
-      <GenProtosCmd>$(GenProtosCmd) --proto_path=$(SourceRoot)</GenProtosCmd>
+      <GenProtosCmd>$(GenProtosCmd) --proto_path=$(SourceRoot)/lib_spark</GenProtosCmd>
      <GenProtosCmd>$(GenProtosCmd) --plugin=&quot;protoc-gen-spark-csharp=$(ProtocPlugin)&quot;</GenProtosCmd>
      <GenProtosCmd>$(GenProtosCmd) --spark-csharp_out=lang=c#,templates=immutable.pb.cs,strip_out_path=1:%(Proto.OutDir)</GenProtosCmd>
      <GenProtosCmd>$(GenProtosCmd) --csharp_out=%(Proto.OutDir)</GenProtosCmd>
@@ -65,7 +65,7 @@
    <PropertyGroup>
      <GenProtosCmd>$(protoc) %(ProtoService.FullPath)</GenProtosCmd>
      <GenProtosCmd>$(GenProtosCmd) --error_format=msvs</GenProtosCmd>
-      <GenProtosCmd>$(GenProtosCmd) --proto_path=$(SourceRoot)</GenProtosCmd>
+      <GenProtosCmd>$(GenProtosCmd) --proto_path=$(SourceRoot)/lib_spark</GenProtosCmd>
      <GenProtosCmd>$(GenProtosCmd) --plugin=&quot;protoc-gen-service=$(ProtocPlugin)&quot;</GenProtosCmd>
      <GenProtosCmd>$(GenProtosCmd) --service_out=lang=c#,strip_out_path=1:%(ProtoService.OutDir)</GenProtosCmd>
      <GenProtosCmd>$(GenProtosCmd) --csharp_out=%(ProtoService.OutDir)</GenProtosCmd>
@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 3.10)
+cmake_minimum_required(VERSION 3.5)
 project(Spark.Admin.WebApp)

 include(GNUInstallDirs)
@@ -10,11 +10,11 @@ include(${CMAKE_SOURCE_DIR}/cmake/ProtobufGenerateFs.cmake)
 file(
  GLOB_RECURSE ${PROJECT_NAME}_PROTO_SOURCES
  RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
-  ../lib/api/*.proto
-  ../lib/api/services/*.proto
-  ../lib/domain_model/*.proto
-  ../lib/sparkproto/*.proto
-  ../lib/md/protos/*.proto
+  ../lib_spark/spark/api/*.proto
+  ../lib_spark/spark/api/services/*.proto
+  ../lib_spark/spark/domain_model/*.proto
+  ../lib_spark/spark/sparkproto/*.proto
+  ../lib_spark/spark/md/protos/*.proto
  ../../externals/linux/include/google/protobuf/descriptor.proto
  ../../externals/linux/include/google/protobuf/timestamp.proto
 )
@@ -44,7 +44,7 @@ set(_build_admin_web_app_deps
  ${_build_admin_web_app_py}
  ${CMAKE_CURRENT_SOURCE_DIR}/src/Spark.Admin.WebApp.fsproj
  ${CMAKE_CURRENT_SOURCE_DIR}/webpack.config.js
-  ${CMAKE_SOURCE_DIR}/src/lib/api/service_compiler/templates/fs/fable.fs
+  ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/fs/fable.fs
  ${CMAKE_CURRENT_SOURCE_DIR}/package-lock.json
  ${${PROJECT_NAME}_PROTO_SOURCES}
  ${_api_ts_SOURCES}
@@ -73,7 +73,7 @@ add_custom_command(
 protobuf_generate_js(
  TARGET_NAME generate_js_protos
  PROTO_INCLUDE_DIRS
-    ..
+    ../lib_spark
    ../../externals/linux/include
  PROTO_SOURCES ${${PROJECT_NAME}_PROTO_SOURCES}
  OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/proto.js
@@ -82,7 +82,7 @@ protobuf_generate_js(
 protobuf_generate_fs(
  TARGET_NAME generate_fs_protos
  PROTO_INCLUDE_DIRS
-    ..
+    ../lib_spark
    ../../externals/linux/include
  PROTO_SOURCES ${${PROJECT_NAME}_PROTO_SOURCES}
  OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/fable
@@ -184,4 +184,3 @@ install(
  DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/public"
  DESTINATION "${CMAKE_INSTALL_BINDIR}/${PROJECT_NAME}"
 )
-
@@ -17,7 +17,7 @@ type Tab =
  | PositionManagementTab
  | MarginTab

-let venues = [| Venue.BINANCE; Venue.BinanceUs; Venue.BinanceUsdsFutures; Venue.CBOE; Venue.COINBASE; Venue.DERIBIT; Venue.OKX; Venue.GLBX; Venue.ICE; Venue.TAIFEX; Venue.TWSE; Venue.SIM; |]
+let venues = [| Venue.BINANCE; Venue.BinanceUs; Venue.BinanceUsdsFutures; Venue.BYBIT; Venue.CBOE; Venue.COINBASE; Venue.DERIBIT; Venue.GATEIO; Venue.OKX; Venue.GLBX; Venue.ICE; Venue.TAIFEX; Venue.TWSE; Venue.SIM; |]

 let securityTypes = [| SecurityType.SPOT; SecurityType.FUTURE; SecurityType.OPTION; SecurityType.SPREAD; |]

@@ -50,6 +50,11 @@ type BinanceUsdsFUTURESExchangeField =
  | Origin of CustomerOrFirm
  | ClearingAccount of string

+type BybitExchangeField =
+  | CTI of CTICode
+  | Origin of CustomerOrFirm
+  | ClearingAccount of string
+
 type COINBASEExchangeField =
  | CTI of CTICode
  | Origin of CustomerOrFirm
@@ -60,6 +65,11 @@ type DERIBITExchangeField =
  | Origin of CustomerOrFirm
  | ClearingAccount of string

+type GateioExchangeField =
+  | CTI of CTICode
+  | Origin of CustomerOrFirm
+  | ClearingAccount of string
+
 type ICEExchangeField =
  | CTI of CTICode
  | Origin of CustomerOrFirm
@@ -797,6 +797,8 @@ let venueToString (venue:bts.api.Venue) =
    | bts.api.Venue.BINANCE -> "BINANCE"
    | bts.api.Venue.BinanceUs -> "BinanceUs"
    | bts.api.Venue.BinanceUsdsFutures -> "BinanceUsdsFutures"
+    | bts.api.Venue.BYBIT -> "BYBIT"
+    | bts.api.Venue.GATEIO -> "GATEIO"
    | bts.api.Venue.FTX -> "FTX"
    | bts.api.Venue.OKX -> "OKX"
    | bts.api.Venue.TAIFEX -> "TAIFEX"
@@ -0,0 +1,172 @@
+cmake_minimum_required(VERSION 3.14)
+project(sparkd)
+
+include(FetchContent)
+include(${CMAKE_SOURCE_DIR}/cmake/safe_dep_glob.cmake)
+include(${CMAKE_SOURCE_DIR}/cmake/ProtobufGenerateCpp.cmake)
+include(${CMAKE_SOURCE_DIR}/cmake/InstallBasic.cmake)
+include(GNUInstallDirs)
+
+if (NOT DEFINED PROJECT_VERSION)
+  set(PROJECT_VERSION 1.0.0)
+endif()
+
+# dependencies
+file(
+  GLOB_RECURSE ${PROJECT_NAME}_PROTO_SOURCES
+  RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
+  *.proto
+)
+file(
+  GLOB_RECURSE ${PROJECT_NAME}_SOURCES
+  RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
+  *.cpp
+)
+update_deps_file("${${PROJECT_NAME}_PROTO_SOURCES};${${PROJECT_NAME}_SOURCES}")
+
+protobuf_generate_cpp(
+  CXX_SOURCES ${PROJECT_NAME}_PROTO_CXX_SOURCES
+  CXX_HEADERS ${PROJECT_NAME}_PROTO_CXX_HEADERS
+  CXX_OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/include
+  PROTO_INCLUDE_DIRS
+    ${EXTERNALS_INCLUDE}
+    ${CMAKE_SOURCE_DIR}/src/lib_spark
+    ${CMAKE_CURRENT_SOURCE_DIR}
+  PROTO_SOURCES
+    ${${PROJECT_NAME}_PROTO_SOURCES}
+)
+add_executable(sparkd ${${PROJECT_NAME}_SOURCES} ${${PROJECT_NAME}_PROTO_CXX_SOURCES})
+
+# include paths
+target_include_directories(
+  sparkd
+  PUBLIC
+    $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
+    $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>/include
+    $<BUILD_INTERFACE:${emumba-deribit-connector-library_INCLUDE_DIRS}>
+)
+
+find_package(PkgConfig REQUIRED)
+pkg_check_modules(emumba-deribit-connector-library REQUIRED emumba-deribit-connector)
+
+# This dir contains rapidjsonConfig.cmake
+list(APPEND CMAKE_PREFIX_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake_modules)
+find_package(rapidjson REQUIRED)
+target_include_directories(
+  ${PROJECT_NAME} SYSTEM
+  PUBLIC
+    $<BUILD_INTERFACE:${rapidjson_INCLUDE_DIR}> 
+)
+add_dependencies(${PROJECT_NAME} rapidjson_project)
+
+# databento
+FetchContent_Declare(
+  databento
+  GIT_REPOSITORY https://github.com/databento/databento-cpp
+  GIT_TAG v0.31.0
+)
+FetchContent_MakeAvailable(databento)
+
+# deribit-demo log level
+if(NOT DEFINED DERIBIT_DEMO_LOG_LEVEL)
+  add_definitions(-DSPDLOG_ACTIVE_LEVEL=0)
+  message("-- Building of DERIBIT_DEMO_LOG_LEVEL=TRACE")
+elseif(DERIBIT_DEMO_LOG_LEVEL STREQUAL "TRACE")
+  add_definitions(-DSPDLOG_ACTIVE_LEVEL=0)
+  message("-- Building of DERIBIT_DEMO_LOG_LEVEL=TRACE")
+elseif(DERIBIT_DEMO_LOG_LEVEL STREQUAL "DEBUG")
+  add_definitions(-DSPDLOG_ACTIVE_LEVEL=1)
+  message("-- Building of DERIBIT_DEMO_LOG_LEVEL=DEBUG")
+elseif(DERIBIT_DEMO_LOG_LEVEL STREQUAL "INFO")
+  add_definitions(-DSPDLOG_ACTIVE_LEVEL=2)
+  message("-- Building of DERIBIT_DEMO_LOG_LEVEL=INFO")
+elseif(DERIBIT_DEMO_LOG_LEVEL STREQUAL "WARN")
+  add_definitions(-DSPDLOG_ACTIVE_LEVEL=3)
+  message("-- Building of DERIBIT_DEMO_LOG_LEVEL=WARN")
+elseif(DERIBIT_DEMO_LOG_LEVEL STREQUAL "ERROR")
+  add_definitions(-DSPDLOG_ACTIVE_LEVEL=4)
+  message("-- Building of DERIBIT_DEMO_LOG_LEVEL=ERROR")
+elseif(DERIBIT_DEMO_LOG_LEVEL STREQUAL "CRITICAL")
+  add_definitions(-DSPDLOG_ACTIVE_LEVEL=5)
+  message("-- Building of DERIBIT_DEMO_LOG_LEVEL=CRITICAL")
+elseif(DERIBIT_DEMO_LOG_LEVEL STREQUAL "OFF")
+  add_definitions(-DSPDLOG_ACTIVE_LEVEL=6)
+  message("-- Building of DERIBIT_DEMO_LOG_LEVEL=OFF")
+else()
+  add_definitions(-DSPDLOG_ACTIVE_LEVEL=0)
+  message("-- Building of DERIBIT_DEMO_LOG_LEVEL=TRACE")
+endif()
+
+target_link_directories(
+  ${PROJECT_NAME} PUBLIC
+  $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
+  $<INSTALL_INTERFACE:${CMAKE_INSTALL_LIBDIR}>
+)
+
+# Set RPATH so sparkd knows where to find our shared libs (e.g., libspark).
+# The dirs are relative to sparkd's path, represented by "$ORIGIN".
+set_target_properties(
+  ${PROJECT_NAME} PROPERTIES
+  BUILD_RPATH "$ORIGIN/../lib_spark;$ORIGIN/../lib_python_interop"
+  INSTALL_RPATH "$ORIGIN/../${CMAKE_INSTALL_LIBDIR}"
+)
+
+include(${CMAKE_SOURCE_DIR}/cmake/SymLink.cmake)
+include(${CMAKE_SOURCE_DIR}/cmake/SymLinkPythonModule.cmake)
+# Py interop
+sym_link_python_module(
+  LIB_NAME python_interop
+  MODULE_NAME python_interop_module
+  PLUGIN_NAME python_interop_plugins
+)
+target_link_libraries(
+  sparkd
+  spark
+  binance_execution
+  okx_execution
+  python_interop
+  python_interop_module
+  databento::databento
+  ${emumba-deribit-connector-library_LIBRARIES}
+  tbb
+  quickfix
+  benchmark
+  Common++
+  Packet++
+  Pcap++
+  lttng-ust
+  -lpcap
+  -static-libgcc
+  -static-libstdc++
+  -pthread)
+
+# Onixs
+target_link_libraries(
+  ${PROJECT_NAME}
+  onixs-cmemdh
+  OnixS.IceImpactMulticastPriceFeedHandlerCpp
+  # gcc doesn't resolve onixs-fix-engine and ACE in runtime path unless we specify
+  # --no-as-needed. These are onixs deps that aren't called directly by our code.
+  # Note: this isn't an issue for clang
+  -Wl,--no-as-needed
+  onixs-fix-engine
+  ACE
+)
+
+# Catch2
+find_package(Catch2 CONFIG REQUIRED)
+target_link_libraries(${PROJECT_NAME} Catch2::Catch2)
+
+install_basic()
+install(
+  FILES
+    "${EXTERNALS_LIB}/libonixs-cmemdh.so"
+    "${EXTERNALS_LIB}/libonixs-cmemdh.so.4"
+    "${EXTERNALS_LIB}/libonixs-cmemdh.so.4.8.0"
+    "${EXTERNALS_LIB}/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.16.0.0"
+    "${EXTERNALS_LIB}/libonixs-fix-engine.so"
+    "${EXTERNALS_LIB}/libonixs-fix-engine.so.4"
+    "${EXTERNALS_LIB}/libonixs-fix-engine.so.4.10.1"
+    "${EXTERNALS_LIB}/libACE-5.7.5.so"
+  DESTINATION ${CMAKE_INSTALL_LIBDIR}
+)
@@ -0,0 +1,5 @@
+@PACKAGE_INIT@
+
+include("${CMAKE_CURRENT_LIST_DIR}/@PROJECT_NAME@Targets.cmake")
+
+check_required_components(@PROJECT_NAME@)
@@ -1,6 +1,7 @@
 ARG IMAGE="sparkd"
-ARG CONFIG_DIR="src/server/sparkd/configurations/dev"
+ARG CONFIG_DIR="src/app_sparkd/configurations/dev"
 ARG SOURCE_DIR="install/sparkd-release-gcc"
+ARG PYTHON_REQUIREMENTS="requirements.txt"

 # sparkd_base
 FROM bluetradingsystems/linux_dotnet7 AS sparkd_base
@@ -10,7 +11,11 @@ ARG SOURCE_DIR
 USER "root"
 RUN apt-get update && apt-get install -y liblttng-ust-dev
 RUN apt-get update && apt-get install -y libpcap0.8-dev
-RUN apt-get update && apt-get install -y python3.10-dev
+RUN apt-get update && apt-get install -y python3-dev
+RUN apt-get update && apt-get install -y python3-pip
+RUN python3 -m pip install --upgrade pip
+COPY "${PYTHON_REQUIREMENTS}" "/home/bts/bin/requirements.txt"
+RUN pip3 install -r "/home/bts/bin/requirements.txt"

 USER "bts"
 COPY --chown=bts:bts "${SOURCE_DIR}" "."
@@ -22,17 +27,22 @@ FROM sparkd_base AS sparkd

 # sparkd_md_replay
 FROM sparkd_base AS sparkd_md_replay
-ARG MD_RECORDING_DIR="src/server/sparkd/.md_recording"
+ARG MD_RECORDING_DIR="src/app_sparkd/.md_recording"
 COPY --chown=bts:bts "${MD_RECORDING_DIR}" "/home/bts/sparkdata/data/md_recording"

 # sparkd_symbols - like sparkd, but doesn't copy a config dir into the image
 FROM bluetradingsystems/linux_dotnet7 AS sparkd_symbols
 ARG SOURCE_DIR
+ARG PYTHON_REQUIREMENTS="requirements.txt"

 USER "root"
 RUN apt-get update && apt-get install -y liblttng-ust-dev
 RUN apt-get update && apt-get install -y libpcap0.8-dev
-RUN apt-get update && apt-get install -y python3.10-dev
+RUN apt-get update && apt-get install -y python3-dev
+RUN apt-get update && apt-get install -y python3-pip
+RUN python3 -m pip install --upgrade pip
+COPY "${PYTHON_REQUIREMENTS}" "/home/bts/bin/requirements.txt"
+RUN pip3 install -r "/home/bts/bin/requirements.txt"

 USER "bts"
 COPY --chown=bts:bts "${SOURCE_DIR}" "."
@@ -0,0 +1,4 @@
+SPARKD__CONFIG_DIR=src/app_sparkd/configurations/alphaworx_prod
+SPARKD__SHM_SIZE=8gb
+SPARKD__MEMORY=8gb
+SPARKD__RUNTIME_CONF_DIR=/home/ubuntu/8000/sparkdata/conf/
@@ -112,7 +112,7 @@ OkxExeCfg {
    secret_key: "240038D7A28D7F9461247CFCB8E43710"
    api_key: "dc188532-c74c-4d40-affe-3c6bf4d0ca9a"
    passphrase: ":,7i%tA%ivfj%SCbX?Ut"
-    ws_url: "wss://wsaws.okx.com:8443"
+    ws_url: "wss://ws.okx.com:8443"
    rest_url: "https://www.okx.com"
    ws_ping_timeout_seconds: 10 
    ws_ping_interval_seconds: 25
@@ -0,0 +1,6 @@
+SPARKD__CONFIG_DIR=src/app_sparkd/configurations/alphaworx_zen_hk_live-preview
+SPARKD__PYTHON_REQUIREMENTS=src/app_sparkd/configurations/alphaworx_zen_hk_live-preview/requirements.txt
+SPARKD__SHM_SIZE=8gb
+SPARKD__MEMORY=8gb
+SPARKD__IMAGE=sparkd_symbols
+SPARKD__RUNTIME_CONF_DIR=/home/bts/8000/sparkdata/conf
@@ -0,0 +1,6 @@
+* Connects to Binance, Okx, and Bybit PRODUCTION sites with small accounts designated for test trades
+* Api keys NOT included
+* Configured for Alphaworx and Bts testing in a production environment
+
+To retrieve api keys, run "python init_env.py". It should dump them to a file
+called  ".env.local".
@@ -0,0 +1,99 @@
+import botocore
+import json
+import os
+
+from boto3.session import Session
+from botocore.exceptions import ClientError
+from fileinput import FileInput
+
+
+def update_env_file(path, env_vars: dict):
+    """Update or add vars in a .env file in place."""
+    if not os.path.exists(path):
+        with open(path, "w") as file:
+            for k, v in env_vars.items():
+                file.write(f"{k}={v}\n")
+        return
+
+    with FileInput(path, inplace=True) as file:
+        for line in file:
+            if not line.startswith("#"):
+                stripped: str = line.lstrip()
+                key_len = stripped.find("=")
+                if key_len >= 0:
+                    k = stripped[0:key_len]
+                    if k in env_vars:
+                        print(f"{k}={env_vars[k]}")
+                        del env_vars[k]
+                        continue
+            print(line, end="")
+
+    if len(env_vars) > 0:
+        with open(path, "a") as file:
+            for k, v in env_vars.items():
+                file.write(f"{k}={v}\n")
+
+
+def get_secret():
+    secret_name = "alphaworx_api_keys"
+    region_name = "us-east-1"
+    profile_name = "bts-ap-northeast-1"
+    session = Session(profile_name=profile_name)
+    client = session.client(
+        service_name="secretsmanager",
+        region_name=region_name,
+    )
+
+    def on_token_error(e: Exception):
+        print(f"""
+Failed to get secret value. {e}
+Verify you are logged in to aws, then try again.
+
+  aws sso login --profile {profile_name}
+
+If this error persists, verify you have access to these api keys.
+For more help or information, talk to John.
+""")
+
+    try:
+        get_secret_value_response = client.get_secret_value(
+            SecretId=secret_name)
+    except ClientError as e:
+        print(f"Failed to get secret value.\n{e}")
+        raise e
+    except botocore.exceptions.SSOTokenLoadError as tl_err:
+        on_token_error(tl_err)
+        return None
+    except botocore.exceptions.TokenRetrievalError as tr_err:
+        on_token_error(tr_err)
+        return None
+    return get_secret_value_response["SecretString"]
+
+
+def _run():
+    this_dir = os.path.dirname(__file__)
+    secret_str = get_secret()
+    if secret_str is None:
+        exit(1)
+    secret = json.loads(secret_str)
+    env_vars = {}
+
+    def set_env_var(key_name: str, *json_path):
+        try:
+            val = secret[json_path[0]]
+            for p in json_path[1:]:
+                val = val[p]
+            env_vars[key_name] = val
+            print(f"Set {key_name}={val} ({'.'.join(json_path)})")
+        except KeyError:
+            print(f"Failed to set {key_name}= ({'.'.join(json_path)})")
+
+    set_env_var("BINANCE_API_KEY", "binance", "prod", "api_key")
+    set_env_var("BINANCE_SECRET_KEY", "binance", "prod", "secret_key")
+    set_env_var("BYBIT_API_KEY", "bybit", "prod_test", "api_key")
+    set_env_var("BYBIT_SECRET_KEY", "bybit", "prod_test", "secret_key")
+    update_env_file(f"{this_dir}/.env.local", env_vars)
+
+
+if __name__ == "__main__":
+    _run()
@@ -0,0 +1,15 @@
+{
+  "gateway_type": "bybit_execution_gateway.BybitExecutionGateway",
+  "account": "BYBIT_DEV",
+  "module_path": [
+    "python_interop_plugins"
+  ],
+  "venue": "VENUE_BYBIT",
+  "api_key": "${BYBIT_API_KEY}",
+  "secret_key": "${BYBIT_SECRET_KEY}",
+  "use_testnet": true,
+  "logging": {
+    "console_level": "DEBUG",
+    "file_level": "DEBUG"
+  }
+}
@@ -0,0 +1,17 @@
+{
+  "currencies": [
+    "USDT"
+  ],
+  "futures": [
+    "ANIME",
+    "BTC",
+    "ETH",
+    "MASK",
+    "RESOLV"
+  ],
+  "use_testnet": true,
+  "logging": {
+    "console_level": "WARNING",
+    "file_level": "INFO"
+  }
+}
@@ -0,0 +1,11 @@
+certifi==2025.4.26
+charset-normalizer==3.4.2
+dotenv==0.9.9
+idna==3.10
+numpy==2.2.6
+pybit==5.11.0
+pycryptodome==3.23.0
+python-dotenv==1.1.0
+requests==2.32.4
+urllib3==2.4.0
+websocket-client==1.8.0
@@ -0,0 +1,138 @@
+QuickbusCfg {
+  value {
+    node {
+      reconnect_period_secs: 1
+      hb_period_msecs: 15000
+      max_missed_hbs: 5
+      parent_host: ""
+      parent_port: 0
+      my_port: 0
+    }
+    client {
+      reconnect_period_secs: 1
+      hb_period_msecs: 15000
+      max_missed_hbs: 5
+      parent_host: @SystemConfig.config_ip
+      parent_port: @Ports.qbus_port
+      my_port: 0
+    }
+  }
+}
+
+JetFIX {
+  value {
+    # connection {
+    #   name: "binance_1"
+    #   venue: VENUE_BINANCE
+    #   max_order_msgs_per_window: 300
+    #   window_duration_msecs: 1000
+    # }
+    # connection {
+    #   name: "binance_usds_futures_1"
+    #   venue: VENUE_BINANCE_USDS_FUTURES
+    #   max_order_msgs_per_window: 300
+    #   window_duration_msecs: 1000
+    # }
+    connection {
+      name: "bybit_1"
+      venue: VENUE_BYBIT
+      max_order_msgs_per_window: 300
+      window_duration_msecs: 1000
+    }
+    # connection {
+    #   name: "okx_1"
+    #   venue: VENUE_OKX
+    #   max_order_msgs_per_window: 300
+    #   window_duration_msecs: 1000
+    # }
+  }
+}
+
+MarketDataConfig {
+  value {
+    # binance {
+    #   num_depth_levels: 5
+    #   url_rest_base: "https://api.binance.com"
+    #   url_ws_base: "wss://stream.binance.com:9443"
+    #   security_criteria {
+    #     clauses {
+    #       security_types: SECURITY_TYPE_SPOT
+    #       symbols: @SymbolLists.list[Binance].symbol
+    #       name_regexps: "USDC|USDT$"
+    #     }
+    #   }
+    # }
+    # binance_usds_futures {
+    #   num_depth_levels: 5
+    #   url_rest_base: "https://fapi.binance.com"
+    #   url_ws_base: "wss://fstream.binance.com"
+    #   security_criteria {
+    #     clauses {
+    #       security_types: SECURITY_TYPE_FUTURE
+    #       symbols: @SymbolLists.list[BinanceUsdsFutures].symbol
+    #     }
+    #   }
+    # }
+    # okx {
+    #   num_depth_levels: 5
+    #   url_rest_base: "https://www.okx.com"
+    #   url_ws_base: "wss://ws.okx.com:8443"
+    #   security_criteria {
+    #     clauses {
+    #       security_types: SECURITY_TYPE_SPOT
+    #       symbols: @SymbolLists.list[OkxSpots].symbol
+    #     }
+    #     clauses {
+    #       security_types: SECURITY_TYPE_FUTURE
+    #       symbols: @SymbolLists.list[OkxFutures].symbol
+    #     }
+    #   }
+    #   is_application_ping_pong_enabled: true
+    #   application_ping_interval_ms: 20000
+    #   application_pong_timeout_ms: 10000
+    #   is_protocol_ping_pong_enabled: true
+    #   protocol_ping_interval_ms: 60000
+    #   protocol_pong_timeout_ms: 30000
+    # }
+    python_md_providers {
+      venue: VENUE_BYBIT
+      module_path: "python_interop_plugins"
+      provider_type: "bybit_md_provider.BybitMdProvider"
+    }
+  }
+}
+
+NotSimExecution {
+  value {
+    # entries {
+    #   venue: VENUE_BINANCE
+    # }
+    # entries {
+    #   venue: VENUE_BINANCE_USDS_FUTURES
+    # }
+    entries {
+      venue: VENUE_BYBIT
+    }
+    # entries {
+    #   venue: VENUE_OKX
+    # }
+  }
+}
+
+# OkxExeCfg {
+#   value {
+#     account: "PROD"
+#     ops_id: "op_id_bts"
+#     secret_key: "240038D7A28D7F9461247CFCB8E43710"
+#     api_key: "dc188532-c74c-4d40-affe-3c6bf4d0ca9a"
+#     passphrase: ":,7i%tA%ivfj%SCbX?Ut"
+#     ws_url: "wss://ws.okx.com:8443"
+#     rest_url: "https://www.okx.com"
+#     ws_ping_timeout_seconds: 10 
+#     ws_ping_interval_seconds: 10
+#     ws_reconnect_interval_seconds: 5
+#     ws_reconnect_max_count: 5
+# 	  simulated_trading: false
+#     tdmode: "cross"
+#   }
+# }
@@ -0,0 +1,66 @@
+SymbolLists {
+  value {
+    list {
+      key: "Binance"
+      symbol: "ANIME"
+      symbol: "BTC"
+      symbol: "ETH"
+      symbol: "MASK"
+      symbol: "RESOLV"
+    }
+    list {
+      key: "BinanceUsdsFutures"
+      symbol: "ANIME"
+      symbol: "BTC"
+      symbol: "ETH"
+      symbol: "MASK"
+      symbol: "RESOLV"
+    }
+    list {
+      key: "OkxSpots"
+      symbol: "ANIME"
+      symbol: "BTC"
+      symbol: "ETH"
+      symbol: "MASK"
+      symbol: "RESOLV"
+    }
+    list {
+      key: "Bybit"
+      symbol: "ANIME"
+      symbol: "BTC"
+      symbol: "ETH"
+      symbol: "MASK"
+      symbol: "RESOLV"
+    }
+    list {
+      key: "OkxFutures"
+      symbol: "ANIME"
+      symbol: "BTC"
+      symbol: "ETH"
+      symbol: "MASK"
+      symbol: "RESOLV"
+    }
+  }
+}
+
+Symbols {
+  value {
+    # venues {
+    #   name: "VENUE_BINANCE"
+    #   spot: @SymbolLists.list[Binance].symbol
+    # }
+    # venues {
+    #   name: "VENUE_BINANCE_USDS_FUTURES"
+    #   future: @SymbolLists.list[BinanceUsdsFutures].symbol
+    # }
+    venues {
+      name: "VENUE_BYBIT"
+      future: @SymbolLists.list[Bybit].symbol
+    }
+    # venues {
+    #   name: "VENUE_OKX"
+    #   spot: @SymbolLists.list[OkxSpots].symbol
+    #   future: @SymbolLists.list[OkxFutures].symbol
+    # }
+  }
+}
@@ -1,4 +1,4 @@
-SPARKD__CONFIG_DIR=src/server/sparkd/configurations/alphaworx_zen_hk_live
+SPARKD__CONFIG_DIR=src/app_sparkd/configurations/alphaworx_zen_hk_live
 SPARKD__SHM_SIZE=8gb
 SPARKD__MEMORY=8gb
 SPARKD__IMAGE=sparkd_symbols
@@ -0,0 +1,9 @@
+{
+  "api_key": "${BINANCE_API_KEY}",
+  "secret_key": "${BINANCE_SECRET_KEY}",
+  "account": "PROD",
+  "rest_endpoint": "https://api.binance.com",
+  "ws_endpoint": "wss://ws-api.binance.com:443/ws-api/v3",
+  "ws_stream_endpoint": "wss://stream.binance.com:9443",
+  "use_cross_margin_api": true
+}
@@ -0,0 +1,9 @@
+{
+  "api_key": "${BINANCE_API_KEY}",
+  "secret_key": "${BINANCE_SECRET_KEY}",
+  "account": "PROD",
+  "rest_endpoint": "https://fapi.binance.com",
+  "ws_endpoint": "wss://ws-fapi.binance.com/ws-fapi/v1",
+  "ws_stream_endpoint": "wss://fstream.binance.com",
+  "use_cross_margin_api": false
+}
@@ -35,8 +35,8 @@ def update_env_file(path, env_vars: dict):


 def get_secret():
-    secret_name = "alphaworx-binance-api-keys"
-    region_name = "ap-northeast-1"
+    secret_name = "alphaworx_api_keys"
+    region_name = "us-east-1"
    profile_name = "bts-ap-northeast-1"
    session = Session(profile_name=profile_name)
    client = session.client(
@@ -77,10 +77,19 @@ def _run():
        exit(1)
    secret = json.loads(secret_str)
    env_vars = {}
-    if "api-key" in secret:
-        env_vars["BINANCE_API_KEY"] = secret["api-key"]
-    if "secret-key" in secret:
-        env_vars["BINANCE_SECRET_KEY"] = secret["secret-key"]
+
+    def set_env_var(key_name: str, *json_path):
+        try:
+            val = secret[json_path[0]]
+            for p in json_path[1:]:
+                val = val[p]
+            env_vars[key_name] = val
+            print(f"Set {key_name}={val} ({'.'.join(json_path)})")
+        except KeyError:
+            print(f"Failed to set {key_name}= ({'.'.join(json_path)})")
+
+    set_env_var("BINANCE_API_KEY", "binance", "prod", "api_key")
+    set_env_var("BINANCE_SECRET_KEY", "binance", "prod", "secret_key")
    update_env_file(f"{this_dir}/.env.local", env_vars)


@@ -112,7 +112,7 @@ OkxExeCfg {
    secret_key: "240038D7A28D7F9461247CFCB8E43710"
    api_key: "dc188532-c74c-4d40-affe-3c6bf4d0ca9a"
    passphrase: ":,7i%tA%ivfj%SCbX?Ut"
-    ws_url: "wss://wsaws.okx.com:8443"
+    ws_url: "wss://ws.okx.com:8443"
    rest_url: "https://www.okx.com"
    ws_ping_timeout_seconds: 10 
    ws_ping_interval_seconds: 10
@@ -0,0 +1,2 @@
+SPARKD__CONFIG_DIR=src/app_sparkd/configurations/binance_dev
+SPARKD__SHM_SIZE=8gb
@@ -43,10 +43,6 @@ Logging {
    }
    file {
      default_level: "Info"
-      subsystem {
-        name: "md"
-        level: "Diag"
-      }
      subsystem {
        name: "OrderMan"
        level: "Diag"
@@ -0,0 +1,2 @@
+SPARKD__CONFIG_DIR=src/app_sparkd/configurations/binance_py_dev
+SPARKD__SHM_SIZE=8gb
@@ -0,0 +1 @@
+* Connects to Binance test nets for market data and execution
@@ -0,0 +1,8 @@
+{
+  "api_key": "${BINANCE_API_KEY}",
+  "secret_key": "${HOME}/sparkdata/conf/.secrets/binance_1.pem",
+  "account": "BINANCE_DEV",
+  "rest_endpoint": "https://testnet.binance.vision",
+  "ws_endpoint": "wss://ws-api.testnet.binance.vision/ws-api/v3",
+  "ws_stream_endpoint": "wss://stream.testnet.binance.vision:9443"
+}
@@ -0,0 +1,8 @@
+{
+  "api_key": "${BINANCE_USDS_FUTURES_API_KEY}",
+  "secret_key": "${BINANCE_USDS_FUTURES_SECRET_KEY}",
+  "account": "BINANCE_DEV",
+  "rest_endpoint": "https://testnet.binancefuture.com",
+  "ws_endpoint": "wss://testnet.binancefuture.com/ws-fapi/v1",
+  "ws_stream_endpoint": "wss://stream.binancefuture.com"
+}
Author	SHA1	Message	Date
John Colvin	0cb46fd8ab	cleanup: Remove unnecessary py_module.cpp files We used to have separate modules for md and execution. I forgot to remove these after unifying them to a single module.	2025-06-16 14:31:28 +00:00
John Colvin	c098b5e442	diag: Reduce book log noise in md plugins They're too noisy for Info level.	2025-06-16 14:28:09 +00:00
John Colvin	dc8cd290b6	diag: Optimize logging in bybit exe gw plugin * Format params as json (not python pseudo json) * Format params lazily	2025-06-16 14:17:45 +00:00
John Colvin	17c77d9b31	fix: Use lock_guard instead of lock/unlock calls I just noticed it was technically possible for the "work thread" (in py plugin manager) to be started twice, due to improper lock/unlock calls.	2025-06-13 21:26:36 +00:00
John Colvin	2cf62d8d9a	Increase sleep time in bybit_md_provider keep-alive loop This thread's only job is to stay alive until "stop" is called. It doesn't need to be particularly fast/responsive. Longer sleep intervals reduce contention with the "hot" thread that receives market data.	2025-06-13 20:50:30 +00:00
John Colvin	0328356758	diag: Add security to assert msg when qty tick deduction fails	2025-06-13 20:50:30 +00:00
John Colvin	0e1be3f559	deploy: Add alphaworx_zen_hk_live-preview configuration	2025-06-13 20:50:30 +00:00
John Colvin	f2d1364c4b	fix: Assert that acc name length < max_size (not size) Since we changed the account field from std::array -> bts::core::SmallString	2025-06-13 20:50:30 +00:00
John Colvin	3d2043f926	deploy: Install python requirements in docker No point in using a virtual environment. These containers have one purpose: run sparkd with a constant, predetermined set of py plugins.	2025-06-13 20:50:30 +00:00
John Colvin	e525e740c1	chore: Update alphaworx_zen_hk_live init.py to query new secret I decided to put all alphaworx api keys in one secret. It's easier this way.	2025-06-13 20:50:30 +00:00
John Colvin	0ef72a45fe	fix: Fix config dir env var in bybit_dev profile	2025-06-13 20:50:30 +00:00
John Colvin	db5dbd3666	fix: Fix SmallString numeric parsing, make more robust * Fix by parsing numbers from left to right instead of right to left (previously, would parse "17" to 71) * Support leading +/- signs * Throw if unexpected char encountered * Unit tests	2025-06-13 20:50:30 +00:00
John Colvin	ca27dd3329	fix: Change thread_id type to size_t (not int64_t)	2025-06-13 20:50:30 +00:00
John Colvin	37b34ec6f7	cleanup: Remove unnecessary signal handler in bybit_md_provider.py We handle them properly in sparkd now	2025-06-13 20:50:30 +00:00
John Colvin	f869c97fbe	cleanup: Sort gateio symbols in symbols.conf	2025-06-13 20:50:30 +00:00
John Colvin	a27462b1a3	chore: Add config json for py plugins Support overrides per venue in a separate file, so symbols can be changed after deployment using arham's widget.	2025-06-13 20:50:30 +00:00
John Colvin	8e9abf8451	fix: Init shared_ptr w/ default obj (not null) Prob a typo	2025-06-13 20:50:30 +00:00
John Colvin	1b40382d56	chore: Fix path in .env file (server/sparkd -> app_sparkd)	2025-06-13 20:50:30 +00:00
John Colvin	402d9f5b84	fix build: Remove include of non-existent header I created the file locally, but ended up not using it and forgot to remove an include before I pushed.	2025-06-13 20:50:30 +00:00
John Colvin	20779387eb	fix build: Always return value in non-void fn	2025-06-13 20:50:30 +00:00
John Colvin	7a6bba0f19	fix: Don't compare size_t to -1 in SmallString num parsers	2025-06-13 20:50:30 +00:00
John Colvin	83dc93aeb1	refactor: Manage all py plugins (md and exe) in one class All calls to/from python are performed on one thread via a simple message queue (necessary because of the global interpreter lock).	2025-06-13 20:50:30 +00:00
John Colvin	6107bbdb5d	chore: Add pybit to py venv	2025-06-13 20:50:30 +00:00
John Colvin	71f95f8774	diag: Log incoming books at Info level in bybit_md_provider	2025-06-13 20:50:30 +00:00
John Colvin	4108ee087e	chore: Add bybit execution config to bybit_dev profile	2025-06-13 20:50:30 +00:00
John Colvin	e30bf5cdea	chore: Add numpy to py venv This is needed for md plugins	2025-06-13 20:50:30 +00:00
John Colvin	dce77049b1	fix: Include fill id in format str for ExecutionReport	2025-06-13 20:50:30 +00:00
John Colvin	72ef4f33b0	dev qol: Bind PySecurity __format__ to fmt::formatter impl	2025-06-13 20:50:30 +00:00
John Colvin	a7176f7851	refactor: Change floats to doubles in py bridge types Getting rounding errors with floats	2025-06-13 20:50:30 +00:00
John Colvin	1a0f3197ce	refactor: Use SmallString instead of std::array in ExchangeFillID	2025-06-13 20:50:30 +00:00
John Colvin	e34c2ffec7	Add py bybit exe config to bybit_dev profile	2025-06-13 20:50:30 +00:00
John Colvin	8a10b591ed	refactor: Consolidate all py interop into single lib It was just too hard trying to make it work with separate libs. The main issue was getting the python md and execution modules to recognize PySecurityList and other types defined in the core module.	2025-06-13 20:50:30 +00:00
John Colvin	efc2100177	refactor: Move tick fns into gen-purpose tick_utils.h	2025-06-13 20:50:30 +00:00
John Colvin	a4d57cc97c	Change bybit_dev symbols.conf to list bybit futs (not spots)	2025-06-13 20:50:30 +00:00
John Colvin	3426702d8f	fix: Actually use configured logger in bybit_md_provider.py	2025-06-13 20:50:30 +00:00
John Colvin	8e7a24c971	dev qol: Reduce noisy logs in PositionServicePNLHelper	2025-06-13 20:50:30 +00:00
John Colvin	475dda4e9e	build: Add cmake script to sym link py module and plugins We have several of these (python_core, python_md, python_execution), so the repetative boilerplate code was starting to pile up.	2025-06-13 20:50:30 +00:00
John Colvin	193f3380cb	deploy: Install/activate py venv in sparkd container	2025-06-13 20:50:29 +00:00
John Colvin	d6ac2c5134	dev qol: Add pyrightconfig.json for type hints in py/c++ bridge	2025-06-13 20:50:29 +00:00
John Colvin	076513e79b	Tweak py to get rid of false positive type hint warns	2025-06-13 20:50:29 +00:00
John Colvin	b227fc6254	feat: Support python execution plugins	2025-06-13 20:50:29 +00:00
John Colvin	ca82042669	refactor: Move py security (and related types) to python_core lib So the execution gw plugins can use it (as well as the md plugins).	2025-06-13 20:50:29 +00:00
John Colvin	ec7c0de946	fix: Add missing switch cases for bybit & gateio	2025-06-13 20:50:29 +00:00
John Colvin	391580b9da	fix: Initialize bybit secs as perp futs (not spots)	2025-06-13 20:50:29 +00:00
John Colvin	7e7e185968	dev qol: Add wait/notify functionality to ConcurrentQueue	2025-06-13 20:50:29 +00:00
John Colvin	517e64ceff	chore: Add CANCEL_ORDER op to binance.env	2025-06-13 20:50:29 +00:00
John Colvin	7df9a15d3b	dev qol: Add ConcurrentValueStore, with notify/wait functionality Uses std::condition_variable	2025-06-13 20:50:29 +00:00
John Colvin	b01f36d263	fix: Don't raise ex if var isn't defined in req path	2025-06-13 20:50:29 +00:00
John Colvin	36e555cd13	dev qol: Add gen-purpose validation fns for execution gws	2025-06-13 20:50:29 +00:00
John Colvin	3d0d1c6dba	build: Use new gen-purpose python cmake fns in py md proj Significantly simplifies CMakeLists.txt	2025-06-13 20:50:29 +00:00
John Colvin	f03a055c34	build: Add gen-purpose InitPythonModule, InitPythonPlugin cmake scripts	2025-06-13 20:50:29 +00:00
John Colvin	bfeb5e362a	dev qol: Add fn to make Stopwatch and start it at once	2025-06-13 20:50:29 +00:00
John Colvin	e48f2ec99d	chore: Use same log format as sparkd in py md plugins	2025-06-13 20:50:29 +00:00
John Colvin	cec62215e2	improvement: Set PyBook data in single call to reduce interop overhead	2025-06-13 20:50:29 +00:00
John Colvin	e361d405c9	Reduce log noise in py md providers	2025-06-13 20:50:29 +00:00
John Colvin	020cd44cf3	doc: Improve py md readme * Put examples in a table * Show command to create symlinks for intellisense	2025-06-13 20:50:29 +00:00
John Colvin	c02f3d152a	refactor: Redesign py md so it doesn't lock the global interpreter We can only embed one interpreter in the sparkd process, so all py plugins need to share it. * Move core py interop code to new lib_python_core. We're about to add a new lib for py execution, so it needs to be accesible from both execution and md libs * Fix typo for type hint in py_md_utils.py * Run py md providers on background thread * Rename MdSink -> MdBridge	2025-06-13 20:50:29 +00:00
John Colvin	b4b9e00f90	build: Add lib_spark subdir first (all others depend on it)	2025-06-13 20:50:29 +00:00
John Colvin	8f886008ef	dev qol: Add to_chrono fns to TimeSpan	2025-06-13 20:50:29 +00:00
John Colvin	23e4dbfa07	dev qol: Add -> operators to ValueWrapper for convenience So we can type "foo->" instead of "foo.value()" (much shorter!)	2025-06-13 20:50:29 +00:00
John Colvin	114ac2fdc9	build: Link against system installed ssl to support py interop Unless we want to build python packages ourselves, the underlying clibs they use link to the system installed ssl. Notably, the requests module.	2025-06-13 20:50:29 +00:00
John Colvin	3bdde53d15	chore: Add pybind11, pybind11-stubgen to py venv	2025-06-13 20:50:29 +00:00
John Colvin	bcc4d5fb3d	improvement: Generate py stubs for md providers Provides intellisense for the python_md_module	2025-06-13 20:50:29 +00:00
John Colvin	e93b5c901f	cleanup: Get rid of unecessary namespace prefixes	2025-06-13 20:50:29 +00:00
John Colvin	ccf8b671bf	cleanup: Get rid of "PyList" classes, bind to stl vector instead I wasn't aware this was possible until recently.	2025-06-13 20:50:29 +00:00
John Colvin	0462c97b76	refactor: Move SimpleOrderKeyMap from binance_execution to lib_spark It provides basic functionality that will be useful in other execution gateways.	2025-06-13 20:50:29 +00:00
John Colvin	3e23799772	chore: Add smart parse and pretty name fns for proto enums Mainly to ease the pain of representing enums as strings in python md and execution plugins	2025-06-13 20:50:29 +00:00
John Colvin	2b4119a023	refactor: Move ErrorList from binance_execution to lib_spark It provides basic functionality that will be useful elsewhere	2025-06-13 20:50:29 +00:00
John Colvin	da469270e7	refactor: Use SmallString std::array in AccountName	2025-06-13 20:50:29 +00:00
John Colvin	c9e9be0772	refactor: Use SmallString instead of std::array in ExchangeOrderID	2025-06-13 20:50:29 +00:00
John Colvin	2870fefc15	Add our own "SmallString" class for fixed cap strings This will replace the std::array<char, N> we use in ExecutionReport, etc. among other things	2025-06-13 20:50:29 +00:00
John Colvin	cd1a5946b3	refactor: Move ExecutionReportStore to lib_spark From lib_binance_execution	2025-06-13 20:50:29 +00:00
Adeel Khan	2cf60d74cc	CLEANUP: BYBIT Symbols.conf.	2025-06-13 20:50:29 +00:00
Adeel Khan	10c018e995	ADD: BYBIT MD Working, NaN's also Added.	2025-06-13 20:50:29 +00:00
Adeel Khan	25b937c096	ADD: BYBIT Securities showing up on frontend. Default Contract Spec	2025-06-13 20:50:29 +00:00
Adeel Khan	516b9c0462	ADD: GATEIO md_provider with dynamic contract specs using Batch REST call	2025-06-13 20:50:29 +00:00
Adeel Khan	347ec1a7ea	ADD: GATEIO md_provider with dynamic contract specs 1 REST call per Contract	2025-06-13 20:50:29 +00:00
Adeel Khan	80f4b4bc1c	ADD: GATEIO md_provider with default contract specs	2025-06-13 20:50:29 +00:00
John Colvin	c470984bd2	Add gateio_dev config profile for Adeel	2025-06-13 20:50:29 +00:00
John Colvin	f560eca4d7	cleanup: Remove unused spark.conf file Originally, this spark.conf was hardcoded into compose.yaml, and copied into the sparkd container on each deploy. It hasn't been used since config profiles were introduced.	2025-06-13 20:50:29 +00:00
John Colvin	f9b28dcb7a	Add py_dummy_md config profile to test py md plugin	2025-06-13 20:50:29 +00:00
John Colvin	8275423b89	deploy: Move configurations to new stutter dir "app_sparkd"	2025-06-13 20:50:29 +00:00
John Colvin	86a725227b	Add protos for gateio	2025-06-13 20:50:29 +00:00
John Colvin	43ec58dfe3	Add protos for bybit	2025-06-13 20:50:29 +00:00
John Colvin	922924c31d	build: Move onix fix adapter to its own lib We should keep order execution dependencies out of lib_spark.	2025-06-13 20:50:29 +00:00
John Colvin	2b7dbb6ad3	fix build: Move OnixsUtils to from lib_spark to app_sparkd They're only needed MDP3Main.cpp and ICEMain.cpp. We're one step closer to removing lib_spark's dependency on onixs.	2025-06-13 20:50:29 +00:00
John Colvin	1c431f542e	cleanup: Remove unnecessary onixs include from TrafixOrderStateHandler.h	2025-06-13 20:50:29 +00:00
John Colvin	7fa8e4ca47	feat: Support python md plugins	2025-06-13 20:50:29 +00:00
John Colvin	8579c9fdc5	Add "clear" fn to Book & BookSide Prelude to python market data bindings	2025-06-13 20:50:28 +00:00
John Colvin	ebcb070310	Add "smart" parse fns for a couple proto enums Prelude to python market data bindings. Unlike the normal parse fns, the "smart" variants aren't case sensitive, and can match strings with or without TYPE_NAME_ prefixes.	2025-06-13 20:50:28 +00:00
John Colvin	f5171d8636	build: Move okx execution to its own lib	2025-06-13 20:50:28 +00:00
John Colvin	8af399ca0f	cleanup: Remove unnecessary onixs include from IDGenerator.h	2025-06-13 20:50:28 +00:00
John Colvin	ea48dbabe3	cleanup: Remove unnecessary includes from MDP3Listener.h	2025-06-13 20:50:28 +00:00
John Colvin	9c7f712e11	refactor: Organize libspark proj w/ stuttering include This new structure provides several benefits: * Allows us to install multiple CMake projects in the same destination dir (without include path conflicts). - This makes shipping the trading algo sdk a trivial excercise. Simply copy the lib and include dirs from the install dest. That's it! * Puts us in a position to break up the monolithic libspark into smaller modules, which are easier to manage. Windows specific notes: * Get rid of unnecessary script vs_filters.cmake * Define WebsocketConnectionGateway directly in cs (instead of proto) - The proto would no longer be in the include path passed to protoc. Moving it to c# because there wasn't a compelling reason for it to be a proto in the first place. * Set spark output prefix to "lib" on windows - So we get "libspark.dll" instead of "spark.dll", which conflicts with the csharp lib, "Spark.dll". * Copy libspark.dll to net8.0 & net8.0-windows in a single cmake build target fix build: Copy libspark to both dirs net8.0, net8.0-windows	2025-06-13 20:50:28 +00:00
John Colvin	e4df7215b0	cleanup: Move TradeSummarySignalDispatcher to libspark It is needed by TradeSummarySignal, which is in libspark. This prevents a circular dependency from libspark -> sparkd.	2025-06-13 20:50:28 +00:00
John Colvin	4f404eec24	build: Clean up protobuf cmake scripts Make them more flexible, reusable. This is a precursor to porting over python md provider code from another branch.	2025-06-13 20:50:28 +00:00
John Colvin	633aa362ff	build: Rename external path vars w/ "EXTERNAL_" prefix in cmake The old "SPARK_" prefix felt like a misnomer.	2025-06-13 20:50:28 +00:00
John Colvin	bfd837764a	Add a few arithmetic ops to TimeSpan	2025-06-13 20:50:28 +00:00
John Colvin	99f3a845b8	diag: Log time it takes to start md sources So we can detect bottlenecks	2025-06-13 20:50:28 +00:00
John Colvin	49f9f50fdb	refactor: Split MarketDataGateway up into providers/sources MarketDataGateway still exists, but is greatly simplified. It creates several `IMarketDataProvider`s, each of which generates a list of `IMarketDataSource`s. The market data sources provide market data to an `IMarketDataListener`. * EmumbaMarketDataProvider * CCApiMarketDataProvider * PyMarketDataProvider * OnixsMarketDataProvider * ReplayMarketDataProvider As a side effect, individual py market data sources are now explicitly limited to a single venue (it probably should have been this way all along, since the shared memory writers do not expect updates to involve multiple venues).	2025-06-13 20:50:28 +00:00
John Colvin	bb25eab078	cleanup: Remove dependency on quickfix for string to_lower We shouldn't depend on external libs for trivial things like converting strings to lower case.	2025-06-13 20:50:28 +00:00
Taha	70fae06a30	BlueOrb streams and MLP	2025-06-08 21:52:38 -04:00
John Colvin	07b59559a2	dev qol: Reduce md log level in config profile	2025-06-03 02:23:59 +00:00
John Colvin	cd1709563b	fix: Generate reject exe reports from ack at binance This only applies to "new" order acks. In all other cases, exe reports are generated from the order update stream. New order rejects are the exception because they're not visible in the stream. Presumably, they're never actually entered into binance's order system (they're "rejected" before that happens).	2025-06-03 02:21:18 +00:00
arham	94f22ddf40	Update: changed url for okx ws exe in config	2025-05-30 12:59:26 +00:00
arham	7f6c08f956	Fix: positions now go through dispatch incoming	2025-05-29 16:26:05 +00:00
arham	1d737cdf82	Fix: change time to 5s for pos update okx	2025-05-23 16:12:12 +00:00
arham	b645192355	Add: OKX position turned on	2025-05-23 15:33:11 +00:00
				`@@ -0,0 +1 @@`
				`* Connects to Binance test nets for market data and execution`