fix (ui): Change qty to double in Manual Fills Import dialog

fix (ui): Show decimal places in Manual Fills Qty column
refactor: Expose price/qty display settings as Security ext fns
2025-08-18 12:11:22 -05:00 · 2025-08-18 12:11:22 -05:00 · 2025-08-18 12:11:13 -05:00 · 2025-08-18 15:54:22 +00:00 · 2025-08-18 15:54:22 +00:00 · 2025-08-18 15:51:11 +00:00
2365 changed files with 32234 additions and 13258 deletions
@@ -22,6 +22,7 @@
      "extensions": [
        "eamodio.gitlens",
        "llvm-vs-code-extensions.vscode-clangd",
+        "ms-python.python",
        "ms-vscode.cmake-tools",
        "ms-vscode.cpptools",
        "xaver.clang-format"
@@ -34,7 +34,7 @@
 #   src/server/sparkd/Dockerfile
 SPARKD__IMAGE=sparkd
 # The config dir for this service to use
-SPARKD__CONFIG_DIR=src/server/sparkd/configurations/dev
+SPARKD__CONFIG_DIR=src/app_sparkd/configurations/dev
 # Mount a specific volume or dir to "~/sparkdata/data" inside the docker.
 # Defaults to a volume named "sparkd_<config_port>" (e.g., sparkd_8000).
 # SPARKD__DATA_DIR=/home/bts/sparkdata/data
@@ -44,5 +44,6 @@ SPARKD__CONFIG_PORT=8000
 SPARKD__QUICKBUS_PORT=8001
 SPARKD__SIM_FIX_PORT=8002
 SPARKD__SHM_SIZE=4gb
+SPARKD__PYTHON_REQUIREMENTS=requirements.txt

 # TODO: add config for other services (pubsub, md-publisher, webapp, api-server, etc.)
@@ -11,6 +11,7 @@ cmake-build-debug/
 **.vscode/*
 !.vscode/launch.json
 !.vscode/tasks.json
+!.vscode/*.env
 .vs
 obj/
 *.csproj.user
@@ -199,24 +199,27 @@
      "request": "launch",
      "program": "",
      "linux": {
-        "preLaunchTask": "Build Sparkd (Debug)",
-        "program": "${workspaceFolder}/build/sparkd-debug-clang/server/sparkd",
-        "args": [
-          "run"
-        ],
-        "stopAtEntry": false,
-        "cwd": "${workspaceFolder}/build/sparkd-debug-clang/server",
-        "environment": [],
-        "externalConsole": false,
-        "MIMode": "gdb",
-        "setupCommands": [
-          {
-            "description": "Enable pretty-printing for gdb",
-            "text": "-enable-pretty-printing",
-            "ignoreFailures": true
-          }
-        ]
-      }
+        "program": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd/sparkd",
+      },
+      "args": [
+        "run"
+      ],
+      "cwd": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd",
+      "stopAtEntry": false,
+      "externalConsole": false,
+      "MIMode": "gdb",
+      "setupCommands": [
+        {
+          "description": "Enable pretty-printing for gdb",
+          "text": "-enable-pretty-printing",
+          "ignoreFailures": true
+        },
+        {
+          "description": "Set Disassembly Flavor to Intel",
+          "text": "-gdb-set disassembly-flavor intel",
+          "ignoreFailures": true
+        }
+      ]
    },
    {
      "name": "Sparkd (Skip Build)",
@@ -224,74 +227,27 @@
      "request": "launch",
      "program": "",
      "linux": {
-        "program": "${workspaceFolder}/build/sparkd-debug-clang/server/sparkd",
-        "args": [
-          "run"
-        ],
-        "stopAtEntry": false,
-        "cwd": "${workspaceFolder}/build/sparkd-debug-clang/server",
-        "environment": [],
-        "externalConsole": false,
-        "MIMode": "gdb",
-        "setupCommands": [
-          {
-            "description": "Enable pretty-printing for gdb",
-            "text": "-enable-pretty-printing",
-            "ignoreFailures": true
-          }
-        ]
-      }
-    },
-    {
-      "name": "Sparkd Test",
-      "type": "cppdbg",
-      "request": "launch",
-      "program": "",
-      "linux": {
-        "preLaunchTask": "Build Sparkd (Debug)",
-        "program": "${workspaceFolder}/build/sparkd-debug-clang/server/sparkd",
-        "args": [
-          "test",
-          "${input:sparkdTest}",
-        ],
-        "stopAtEntry": false,
-        "cwd": "${workspaceFolder}/build/sparkd-debug-clang/server",
-        "environment": [],
-        "externalConsole": false,
-        "MIMode": "gdb",
-        "setupCommands": [
-          {
-            "description": "Enable pretty-printing for gdb",
-            "text": "-enable-pretty-printing",
-            "ignoreFailures": true
-          }
-        ]
-      }
-    },
-    {
-      "name": "Sparkd Test (Skip Build)",
-      "type": "cppdbg",
-      "request": "launch",
-      "program": "",
-      "linux": {
-        "program": "${workspaceFolder}/build/sparkd-debug-clang/server/sparkd",
-        "args": [
-          "test",
-          "${input:sparkdTest}",
-        ],
-        "stopAtEntry": false,
-        "cwd": "${workspaceFolder}/build/sparkd-debug-clang/server",
-        "environment": [],
-        "externalConsole": false,
-        "MIMode": "gdb",
-        "setupCommands": [
-          {
-            "description": "Enable pretty-printing for gdb",
-            "text": "-enable-pretty-printing",
-            "ignoreFailures": true
-          }
-        ]
-      }
+        "program": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd/sparkd",
+      },
+      "args": [
+        "run"
+      ],
+      "cwd": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd",
+      "stopAtEntry": false,
+      "externalConsole": false,
+      "MIMode": "gdb",
+      "setupCommands": [
+        {
+          "description": "Enable pretty-printing for gdb",
+          "text": "-enable-pretty-printing",
+          "ignoreFailures": true
+        },
+        {
+          "description": "Set Disassembly Flavor to Intel",
+          "text": "-gdb-set disassembly-flavor intel",
+          "ignoreFailures": true
+        }
+      ]
    },
  ],
  "inputs": [
@@ -0,0 +1 @@
+PYTHONPATH=/workspaces/spark/build/sparkd-debug-clang/src/lib_python_interop;/workspaces/spark/src/lib_python_interop/python_interop_plugins
@@ -363,7 +363,7 @@
          "${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
        ],
        "options": {
-          "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0-windows"
+          "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0-windows"
        },
      },
      "icon": {
@@ -383,7 +383,7 @@
          "${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
        ],
        "options": {
-          "cwd": "${workspaceFolder}/build/Debug/bin/net7.0-windows"
+          "cwd": "${workspaceFolder}/build/Debug/bin/net8.0-windows"
        },
      },
      "icon": {
@@ -403,7 +403,7 @@
          "${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
        ],
        "options": {
-          "cwd": "${workspaceFolder}/build/Release/bin/net7.0-windows"
+          "cwd": "${workspaceFolder}/build/Release/bin/net8.0-windows"
        },
      },
      "icon": {
@@ -427,7 +427,7 @@
        "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
      ],
      "options": {
-        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0"
+        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0"
      },
      "icon": {
        "id": "run",
@@ -450,7 +450,7 @@
        "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
      ],
      "options": {
-        "cwd": "${workspaceFolder}/build/Debug/bin/net7.0"
+        "cwd": "${workspaceFolder}/build/Debug/bin/net8.0"
      },
      "icon": {
        "id": "run",
@@ -473,7 +473,7 @@
        "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
      ],
      "options": {
-        "cwd": "${workspaceFolder}/build/Release/bin/net7.0"
+        "cwd": "${workspaceFolder}/build/Release/bin/net8.0"
      },
      "icon": {
        "id": "run",
@@ -496,7 +496,7 @@
        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
      ],
      "options": {
-        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0"
+        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0"
      },
      "icon": {
        "id": "run",
@@ -519,7 +519,7 @@
        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
      ],
      "options": {
-        "cwd": "${workspaceFolder}/build/Debug/bin/net7.0"
+        "cwd": "${workspaceFolder}/build/Debug/bin/net8.0"
      },
      "icon": {
        "id": "run",
@@ -541,7 +541,7 @@
        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
      ],
      "options": {
-        "cwd": "${workspaceFolder}/build/Debug/bin/net7.0"
+        "cwd": "${workspaceFolder}/build/Debug/bin/net8.0"
      },
      "icon": {
        "id": "run",
@@ -564,7 +564,7 @@
        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
      ],
      "options": {
-        "cwd": "${workspaceFolder}/build/Release/bin/net7.0"
+        "cwd": "${workspaceFolder}/build/Release/bin/net8.0"
      },
      "icon": {
        "id": "run",
@@ -586,7 +586,7 @@
        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
      ],
      "options": {
-        "cwd": "${workspaceFolder}/build/Release/bin/net7.0"
+        "cwd": "${workspaceFolder}/build/Release/bin/net8.0"
      },
      "icon": {
        "id": "run",
@@ -601,7 +601,7 @@
      "windows": {
        "command": "Spark.exe",
        "options": {
-          "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0-windows"
+          "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0-windows"
        },
      },
      "icon": {
@@ -617,7 +617,7 @@
      "windows": {
        "command": "Spark.exe",
        "options": {
-          "cwd": "${workspaceFolder}/build/Debug/bin/net7.0-windows"
+          "cwd": "${workspaceFolder}/build/Debug/bin/net8.0-windows"
        },
      },
      "icon": {
@@ -633,7 +633,7 @@
      "windows": {
        "command": "Spark.exe",
        "options": {
-          "cwd": "${workspaceFolder}/build/Release/bin/net7.0-windows"
+          "cwd": "${workspaceFolder}/build/Release/bin/net8.0-windows"
        },
      },
      "icon": {
@@ -648,7 +648,7 @@
      "windows": {
        "command": "Spark.exe",
        "options": {
-          "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0-windows"
+          "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0-windows"
        },
      },
      "icon": {
@@ -673,7 +673,7 @@
        "${input:sparkConfigQuery}",
      ],
      "options": {
-        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0"
+        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0"
      },
      "icon": {
        "id": "run",
@@ -754,7 +754,7 @@
          "${input:sparkTest}"
        ],
        "options": {
-          "cwd": "${workspaceFolder}/build/Debug/bin/net7.0",
+          "cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
        }
      },
    },
@@ -22,9 +22,9 @@ set(
 # can run simultaneously without fighting over the same shm files (e.g., sparkd,
 # api server, etc.).
 set(SHARED_MEMORY_PREFIX "bts" CACHE STRING "Prefix to use when creating shared memory files")
-set(SPARK_BIN "${EXTERNALS_ROOT}/bin")
-set(SPARK_INCLUDE "${EXTERNALS_ROOT}/include")
-set(SPARK_LIB "${EXTERNALS_ROOT}/lib/$<CONFIG>")
+set(EXTERNALS_BIN "${EXTERNALS_ROOT}/bin")
+set(EXTERNALS_INCLUDE "${EXTERNALS_ROOT}/include")
+set(EXTERNALS_LIB "${EXTERNALS_ROOT}/lib/$<CONFIG>")
 set(BUILD_DOCS OFF CACHE BOOL "Build documentation")
 set(BUILD_SDK OFF CACHE BOOL "Build the sdk")
 set(EXPORT_MD_CONSUMER OFF CACHE BOOL "Export (extern) C functions for MD consumer interop")
@@ -36,8 +36,8 @@ else()
  set(BUILD_ADMIN_WEB_APP ON CACHE BOOL "Build the Admin WebApp")
 endif()

-include_directories(BEFORE SYSTEM ${SPARK_INCLUDE} ${EXTERNALS_ROOT}/include/zlib)
-link_directories(${SPARK_LIB})
+include_directories(BEFORE SYSTEM ${EXTERNALS_INCLUDE} ${EXTERNALS_ROOT}/include/zlib)
+link_directories(${EXTERNALS_LIB})

 add_compile_definitions(SHARED_MEMORY_PREFIX="${SHARED_MEMORY_PREFIX}")
 add_compile_definitions(APP_DATA_DIR="${APP_DATA_DIR}")
@@ -47,12 +47,8 @@ endif()
 add_compile_definitions(EXPORT_MD_CONSUMER="${EXPORT_MD_CONSUMER}")
 if(WIN32)
  # subdirs
-  add_subdirectory(src/lib)
+  add_subdirectory(src/lib_spark)
 else()
-  # make sparkd look for libspark.so in its directory
-  set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE)
-  set(CMAKE_INSTALL_RPATH ".")
-
  if(BUILD_SPARKD)
    # Note: All macros that begin w/ 'CCAPI' are ccapi's macros (defined and/or
    # read by their code). We override some below.
@@ -117,12 +113,15 @@ else()
  endif()

  # subdirs
+  add_subdirectory(src/lib_spark)
  if (BUILD_SPARKD)
-    add_subdirectory(src/lib_binance_execution)
-    add_subdirectory(src/server/sparkd)
+    add_subdirectory(src/lib_python_interop)
+    add_subdirectory(src/lib_onixs_fix)
+    add_subdirectory(src/lib_binance)
+    add_subdirectory(src/lib_okx_execution)
+    add_subdirectory(src/app_sparkd)
    add_subdirectory(src/data_files)
  endif()
-  add_subdirectory(src/lib)
  if (BUILD_ADMIN_WEB_APP)
    add_subdirectory(src/Spark.Admin.WebApp)
  endif()
@@ -132,37 +131,4 @@ else()
  if(BUILD_SDK)
    add_subdirectory(src/sdk)
  endif()
-
-  if(BUILD_SPARKD)
-    add_custom_command(
-      OUTPUT ${CMAKE_BINARY_DIR}/server
-      COMMAND mkdir -p ${CMAKE_BINARY_DIR}/server
-      COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/src/server/sparkd/sparkd ${CMAKE_BINARY_DIR}/server
-      COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/data_files ${CMAKE_BINARY_DIR}/server
-      COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/src/lib/libspark.so ${CMAKE_BINARY_DIR}/server
-      COMMAND ln ARGS -sf ${EXTERNALS_ROOT}/lib/${CMAKE_BUILD_TYPE}/libonixs-fix-engine.so.4 ${CMAKE_BINARY_DIR}/server
-      
-      COMMAND
-        ln ARGS -sf
-        ${EXTERNALS_ROOT}/lib/Release/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.16.0.0
-        ${CMAKE_BINARY_DIR}/server
-      COMMAND ln ARGS -sf ${EXTERNALS_ROOT}/lib/Release/libACE-5.7.5.so ${CMAKE_BINARY_DIR}/server
-      COMMAND mkdir ARGS -p ${CMAKE_BINARY_DIR}/server/plugins
-      COMMENT "Creating server runtime layout"
-      DEPENDS sparkd data_files_dir)
-
-    add_custom_target(server_layout ALL DEPENDS ${CMAKE_BINARY_DIR}/server)
-
-    install(
-      DIRECTORY "${CMAKE_BINARY_DIR}/server/plugins"
-      DESTINATION "${CMAKE_INSTALL_BINDIR}")
-    install(
-      FILES
-      "${SPARK_LIB}/libACE-5.7.5.so"
-      "${SPARK_LIB}/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.16.0.0"
-      "${SPARK_LIB}/libonixs-fix-engine.so.4.10.1"
-      "${SPARK_LIB}/libonixs-fix-engine.so.4"
-      "${SPARK_LIB}/libonixs-fix-engine.so"
-      DESTINATION "${CMAKE_INSTALL_BINDIR}")
-  endif()
 endif()
@@ -31,9 +31,8 @@
        "CMAKE_SHARED_LINKER_FLAGS_RELEASE": "/OPT:REF /OPT:ICF /LTCG",
        "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
        "EXTERNALS_ROOT": "${sourceDir}/externals/windows",
-        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0",
-        "DOTNET_WINDOWS_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0-windows",
+        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf",
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0;${sourceDir}/build/$<CONFIG>/bin/net8.0-windows",
        "EXPORT_MD_CONSUMER": {
          "type": "BOOL",
          "value": "ON"
@@ -58,11 +57,12 @@
        "APP_DATA_DIR_OVERRIDE": "${dollar}{SPARKD_DATA_DIR}",
        "CMAKE_BUILD_WITH_INSTALL_RPATH": {
          "type": "BOOL",
-          "value": "ON"
+          "value": "OFF"
        },
        "CMAKE_CXX_STANDARD": "20",
        "CMAKE_INSTALL_RPATH": ".",
        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
+        "CMAKE_PREFIX_PATH": "${sourceDir}/cmake/;${sourceDir}/.venv/lib/python3.10/site-packages/pybind11/share/cmake/pybind11",
        "CMAKE_EXE_LINKER_FLAGS": "-Wno-undef",
        "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
        "DATABENTO_USE_EXTERNAL_JSON": {
@@ -93,7 +93,7 @@
          "value": "ON"
        },
        "CCAPI_LOG_LEVEL": "Info",
-        "CONFIG_DEFAULTS_PATH": "config/profiles/md_dev/*.conf;config/profiles/default/*.conf"
+        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/md_dev/*.conf;spark/config/profiles/default/*.conf"
      }
    },

@@ -104,7 +104,7 @@
      "cacheVariables": {
        "CMAKE_BUILD_TYPE": "Release",
        "CCAPI_LOG_LEVEL": "Warn",
-        "CONFIG_DEFAULTS_PATH": "config/profiles/md_prod/*.conf;config/profiles/default/*.conf"
+        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/md_prod/*.conf;spark/config/profiles/default/*.conf"
      }
    },

@@ -186,7 +186,11 @@
        "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
-        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
+        "CMAKE_EXPORT_COMPILE_COMMANDS": {
+          "type": "BOOL",
+          "value": "ON"
+        },
+        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf",
        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0",
        "EXPORT_MD_CONSUMER": {
          "type": "BOOL",
@@ -227,7 +231,7 @@
      "cacheVariables": {
        "APP_DATA_DIR": "${dollar}{APPDATA}/bts/DataStream",
        "APP_DATA_DIR_OVERRIDE": "${dollar}{DATA_STREAM_DATA_DIR}",
-        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf",
+        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf;spark/config/profiles/data_stream/*.conf",
        "SHARED_MEMORY_PREFIX": "data_stream"
      }
    },
@@ -262,8 +266,7 @@
        "CMAKE_CXX_COMPILER": "clang++",
        "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
-        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
-        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf",
+        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf;spark/config/profiles/data_stream/*.conf",
        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
        "EXPORT_MD_CONSUMER": {
          "type": "BOOL",
@@ -0,0 +1,7 @@
+include_guard(GLOBAL)
+
+macro(export_to_parent)
+  foreach(var IN ITEMS ${ARGN})
+    set(${var} ${${var}} PARENT_SCOPE)
+  endforeach()
+endmacro()
@@ -0,0 +1,142 @@
+include_guard(DIRECTORY)
+include(${CMAKE_SOURCE_DIR}/cmake/ExportToParent.cmake)
+include(${CMAKE_SOURCE_DIR}/cmake/InstallBasic.cmake)
+include(${CMAKE_SOURCE_DIR}/cmake/ProtobufGenerateCpp.cmake)
+include(${CMAKE_SOURCE_DIR}/cmake/SetFirstDefined.cmake)
+include(${CMAKE_SOURCE_DIR}/cmake/safe_dep_glob.cmake)
+
+function(init_library)
+  set(_options SHARED STATIC)
+  set(_one_value_args
+    NAME
+    SOURCE_DIR
+    INCLUDE_DIR
+  )
+  set(_multi_value_args)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+  if (_ARGS_SHARED AND _ARGS_STATIC)
+    message(FATAL_ERROR "Invalid args passed to init_library: cannot specify STATIC and SHARED")
+  endif()
+  set_first_defined(_name _ARGS_NAME PROJECT_NAME)
+  set_first_defined(_source_dir _ARGS_SOURCE_DIR _name)
+  set_first_defined(_include_dir _ARGS_INCLUDE_DIR _source_dir)
+
+  # Cpp sources
+  file(
+    GLOB_RECURSE ${_name}_SOURCES
+    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
+    ${_source_dir}/*.cpp
+  )
+  # Note: cpp files inside 'tests' dir(s) are considered tests.
+  # Remove tests from main sources, and add them to test sources
+  foreach(item ${${_name}_SOURCES})
+    if(${item} MATCHES "tests/.*\.cpp$")
+      list(APPEND ${_name}_TEST_SOURCES ${item})
+    endif()
+  endforeach()
+  foreach(item ${${_name}_TEST_SOURCES})
+    list(REMOVE_ITEM ${_name}_SOURCES ${item})
+  endforeach()
+
+  # Proto sources 
+  file(
+    GLOB_RECURSE ${_name}_PROTO_SOURCES
+    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
+    ${_source_dir}/protobufs/*.proto
+  )
+  update_deps_file("${${_name}_PROTO_SOURCES};")
+  protobuf_generate_cpp(
+    CXX_SOURCES ${_name}_CXX_PROTO_SOURCES
+    CXX_HEADERS ${_name}_CXX_PROTO_HEADERS
+    CXX_OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}
+    PROTO_INCLUDE_DIRS
+      ${EXTERNALS_INCLUDE}
+      ${CMAKE_CURRENT_SOURCE_DIR}
+    PROTO_SOURCES
+      ${${_name}_PROTO_SOURCES}
+  )
+
+  export_to_parent(
+    ${_name}_SOURCES
+    ${_name}_TEST_SOURCES
+    ${_name}_PROTO_SOURCES
+    ${_name}_CXX_PROTO_SOURCES
+    ${_name}_CXX_PROTO_HEADERS
+  )
+
+  # Add main lib
+  if (_ARGS_STATIC)
+    add_library(${_name} STATIC ${${_name}_SOURCES} ${${_name}_CXX_PROTO_SOURCES})
+  elseif(_ARGS_SHARED)
+    add_library(${_name} SHARED ${${_name}_SOURCES} ${${_name}_CXX_PROTO_SOURCES})
+  else()
+    add_library(${_name} ${${_name}_SOURCES} ${${_name}_CXX_PROTO_SOURCES})
+  endif()
+  target_include_directories(
+    ${_name}
+    SYSTEM
+    BEFORE
+    PUBLIC
+      $<BUILD_INTERFACE:${EXTERNALS_ROOT}/include>
+      $<BUILD_INTERFACE:${EXTERNALS_ROOT}/include/zlib>
+  )
+  target_include_directories(
+    ${_name}
+    PUBLIC
+      $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
+      $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>
+  )
+  # Include protos (if any)
+  if (${_name}_PROTO_SOURCES)
+    target_include_directories(
+      ${_name}
+      PUBLIC
+        $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
+    )
+  endif()
+  target_link_directories(
+    ${_name}
+    PUBLIC
+      $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
+      $<BUILD_INTERFACE:${EXTERNALS_ROOT}/lib/$<CONFIG>>
+      $<INSTALL_INTERFACE:${CMAKE_INSTALL_LIBDIR}>
+  )
+  # Install main lib
+  foreach(_hdr ${${_name}_CXX_PROTO_HEADERS})
+    file(RELATIVE_PATH _relative_path ${CMAKE_CURRENT_BINARY_DIR} ${_hdr})
+    install(FILES ${_hdr} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/${_relative_path})
+  endforeach()
+  install_basic(
+    NAME ${_name}
+    INCLUDE_DIR ${_include_dir}
+  )
+
+  # Add tests executable (if there are any)
+  if (${_name}_TEST_SOURCES AND NOT WIN32)
+    add_executable(${_name}-tests ${${_name}_TEST_SOURCES})
+
+    get_target_property(_lib_type ${_name} TYPE)
+    if (_lib_type STREQUAL "SHARED_LIBRARY")
+      # Set RPATH so the tests exe knows where to find the shared lib.
+      set_target_properties(
+        ${_name}-tests
+        PROPERTIES
+          BUILD_RPATH "$ORIGIN"
+          INSTALL_RPATH "$ORIGIN/../${CMAKE_INSTALL_LIBDIR}"
+      )
+    endif()
+
+    # Link main lib
+    target_link_directories(
+      ${_name}-tests
+      PRIVATE
+        $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
+        $<INSTALL_INTERFACE:${CMAKE_INSTALL_LIBDIR}>
+    )
+    target_link_libraries(${_name}-tests PRIVATE ${_name})
+
+    # Catch2
+    find_package(Catch2 CONFIG REQUIRED COMPONENTS Catch2WithMain)
+    target_link_libraries(${_name}-tests PRIVATE Catch2::Catch2WithMain)
+  endif()
+endfunction()
@@ -0,0 +1,71 @@
+include_guard(DIRECTORY)
+include(GNUInstallDirs)
+include(${CMAKE_SOURCE_DIR}/cmake/ExportToParent.cmake)
+include(${CMAKE_SOURCE_DIR}/cmake/SetFirstDefined.cmake)
+
+function(init_python_module)
+  set(_options)
+  set(_one_value_args
+    NAME
+  )
+  set(_multi_value_args
+    SOURCES
+  )
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+  set_first_defined(_name _ARGS_NAME PROJECT_NAME)
+  set_first_defined(_sources _ARGS_SOURCES ${_name}_SOURCES)
+
+  set(module_proj ${_name}_module)
+  set(${module_proj}_SOURCES ${_sources})
+  pybind11_add_module(${module_proj} SHARED ${_sources})
+  target_link_libraries(
+    ${module_proj}
+    PUBLIC
+      pybind11::embed
+      spark
+      ${_name}
+  )
+  target_include_directories(
+    ${module_proj}
+    PRIVATE
+      $<BUILD_INTERFACE:${pybind11_INCLUDE_DIRS}>
+  )
+  target_include_directories(
+    ${module_proj}
+    PUBLIC
+      ${Python3_INCLUDE_DIRS}
+      $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
+      $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>
+  )
+  add_custom_command(
+    OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/stubs/${_name}.pyi
+    DEPENDS ${_name} ${module_proj}
+    COMMAND PYTHONPATH=${CMAKE_CURRENT_BINARY_DIR} pybind11-stubgen --numpy-array-use-type-var --ignore-invalid-expressions '.*' ${_name}_module
+    COMMENT "Generating py stubs"
+  )
+  add_custom_target(
+    "${_name}_py_stubs"
+    ALL
+    DEPENDS
+      ${CMAKE_CURRENT_BINARY_DIR}/stubs/${_name}.pyi
+  )
+  install(
+    TARGETS ${module_proj}
+    EXPORT ${module_proj}Targets
+    RUNTIME
+      DESTINATION ${CMAKE_INSTALL_BINDIR}
+    LIBRARY
+      DESTINATION ${CMAKE_INSTALL_LIBDIR}
+    ARCHIVE
+      DESTINATION ${CMAKE_INSTALL_LIBDIR}
+  )
+  foreach(dir ${pybind11_INCLUDE_DIRS})
+    install(DIRECTORY ${dir} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
+  endforeach()
+  install(
+    EXPORT ${module_proj}Targets
+    FILE ${module_proj}Targets.cmake
+    NAMESPACE ${_name}::
+    DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name}
+  )
+endfunction()
@@ -0,0 +1,65 @@
+include_guard(DIRECTORY)
+include(GNUInstallDirs)
+
+function(init_python_plugin)
+  set(_options)
+  set(_one_value_args NAME PLUGIN_NAME)
+  set(_multi_value_args SOURCES)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+  set_first_defined(_name _ARGS_NAME PROJECT_NAME)
+  set(_plugin_name ${_ARGS_PLUGIN_NAME})
+  if (NOT DEFINED _ARGS_PLUGIN_NAME)
+    set(_plugin_name ${_name}_plugins)
+  endif()
+  set(${_plugin_name}_SOURCES ${_ARGS_SOURCES})
+  if (NOT DEFINED _ARGS_SOURCES)
+    file(
+      GLOB_RECURSE ${_plugin_name}_SOURCES
+      RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
+      ${_plugin_name}/*.py)
+  endif()
+  set(src_outs)
+  foreach(_src ${${_plugin_name}_SOURCES})
+    set(_src_in ${CMAKE_CURRENT_SOURCE_DIR}/${_src})
+    set(_src_out ${CMAKE_CURRENT_BINARY_DIR}/${_src})
+    add_custom_command(
+      OUTPUT ${_src_out}
+      COMMAND ${CMAKE_COMMAND} -E copy ${_src_in} ${_src_out}
+      DEPENDS ${_src_in}
+      COMMENT "Copying ${_plugin_name} plugin ${_src} to ${_src_out}"
+    )
+    list(APPEND src_outs ${_src_out})
+  endforeach()
+  
+  add_custom_target(copy_${_plugin_name} ALL DEPENDS ${src_outs})
+  
+  add_library(${_plugin_name} INTERFACE)
+  add_library(${_name}::${_plugin_name} ALIAS ${_plugin_name})
+  
+  set_target_properties(
+    ${_plugin_name}
+    PROPERTIES
+      INTERFACE_PYTHON_SCRIPTS_DIR
+        $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/${_plugin_name}>$<INSTALL_INTERFACE:${CMAKE_INSTALL_BINDIR}/${_plugin_name}>
+  )
+  
+  set_target_properties(
+    ${_plugin_name}
+    PROPERTIES
+    INTERFACE_PYTHON_SCRIPTS $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/${${_plugin_name}_SOURCES}>$<INSTALL_INTERFACE:${CMAKE_INSTALL_BINDIR}/${${_plugin_name}_SOURCES}>
+  )
+  
+  install(
+    TARGETS ${_plugin_name}
+    EXPORT ${_plugin_name}Targets
+    DESTINATION ${CMAKE_INSTALL_BINDIR}
+  )
+  
+  install(
+    EXPORT ${_plugin_name}Targets
+    FILE ${_plugin_name}Targets.cmake
+    NAMESPACE ${_name}::
+    DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name}
+  )
+  install(DIRECTORY ${_plugin_name} DESTINATION ${CMAKE_INSTALL_BINDIR})
+endfunction()
@@ -0,0 +1,39 @@
+include_guard(DIRECTORY)
+include(GNUInstallDirs)
+include(${CMAKE_SOURCE_DIR}/cmake/PackBasic.cmake)
+include(${CMAKE_SOURCE_DIR}/cmake/SetFirstDefined.cmake)
+
+function(install_basic)
+  set(_options)
+  set(_one_value_args NAME INCLUDE_DIR)
+  set(_multi_value_args)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+  set_first_defined(_name _ARGS_NAME PROJECT_NAME)
+  set_first_defined(_include_dir _ARGS_INCLUDE_DIR _name)
+
+  file(
+    GLOB_RECURSE ${_name}_HEADERS
+    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
+    ${_include_dir}/*.h)
+
+  target_sources(${_name}
+    PUBLIC
+      FILE_SET HEADERS FILES ${${_name}_HEADERS})
+
+  install(
+    TARGETS ${_name}
+    EXPORT ${_name}Targets
+    RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
+    LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
+    ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
+    FILE_SET HEADERS DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}
+  )
+
+  install(
+    EXPORT ${_name}Targets
+    FILE ${_name}Targets.cmake
+    NAMESPACE ${_name}::
+    DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name})
+
+  pack_basic()
+endfunction()
@@ -0,0 +1,44 @@
+include_guard(DIRECTORY)
+include(CMakePackageConfigHelpers)
+include(${CMAKE_SOURCE_DIR}/cmake/SetFirstDefined.cmake)
+
+function(pack_basic)
+  set(_options)
+  set(_one_value_args NAME)
+  set(_multi_value_args)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+  set_first_defined(_name _ARGS_NAME PROJECT_NAME)
+
+  find_package(Git QUIET)
+
+  if(GIT_FOUND)
+    execute_process(
+      COMMAND ${GIT_EXECUTABLE} describe --tags --always --dirty
+      WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
+      OUTPUT_VARIABLE PROJECT_VERSION
+      OUTPUT_STRIP_TRAILING_WHITESPACE
+    )
+  endif()
+
+  if(NOT GIT_FOUND OR PROJECT_VERSION STREQUAL "")
+    set(PROJECT_VERSION "0.0.0")
+  endif()
+
+  write_basic_package_version_file(
+    "${CMAKE_CURRENT_BINARY_DIR}/${_name}ConfigVersion.cmake"
+    VERSION ${PROJECT_VERSION}
+    COMPATIBILITY AnyNewerVersion
+  )
+
+  configure_package_config_file(
+    "${CMAKE_CURRENT_SOURCE_DIR}/Config.cmake.in"
+    "${CMAKE_CURRENT_BINARY_DIR}/${_name}Config.cmake"
+    INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name}
+  )
+
+  install(
+    FILES
+      "${CMAKE_CURRENT_BINARY_DIR}/${_name}Config.cmake"
+      "${CMAKE_CURRENT_BINARY_DIR}/${_name}ConfigVersion.cmake"
+    DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name})
+endfunction()
@@ -1,12 +1,23 @@
-set(Protobuf_PROTOC_EXECUTABLE "${SPARK_BIN}/protoc")
-set(PROTOBUF_INCLUDE_DIRS "${SPARK_INCLUDE}")
+set(Protobuf_PROTOC_EXECUTABLE "${EXTERNALS_BIN}/protoc")
+set(PROTOBUF_INCLUDE_DIRS "${EXTERNALS_INCLUDE}")
 set(PROTOBUF_STATIC_LIB "libprotobuf")
+set(_protobuf_src_dir ${CMAKE_SOURCE_DIR}/src/lib_spark)
+set(_protobuf_bin_dir ${CMAKE_BINARY_DIR}/src/lib_spark)

-function(Protobuf_Generate_Cpp SRCS HDRS)
-  set(${SRCS})
-  set(${HDRS})
+function(Protobuf_Generate_Cpp)
+  set(_options)
+  set(_one_value_args CXX_SOURCES CXX_HEADERS CXX_OUT_DIR)
+  set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+  set(${_ARGS_CXX_SOURCES})
+  set(${_ARGS_CXX_HEADERS})

-  foreach(FIL ${ARGN})
+  set(_proto_include_cli_args)
+  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
+    list(APPEND _proto_include_cli_args -I ${_dir})
+  endforeach()
+
+  foreach(FIL ${_ARGS_PROTO_SOURCES})
    get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
    get_filename_component(FIL_WE ${FIL} NAME_WE)
    get_filename_component(FIL_DIR ${FIL} DIRECTORY)
@@ -14,48 +25,56 @@ function(Protobuf_Generate_Cpp SRCS HDRS)
      set(FIL_WE "${FIL_DIR}/${FIL_WE}")
    endif()

-    list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc")
-    list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h")
+    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc")
+    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h")

    add_custom_command(
-      OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc"
-             "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h"
+      OUTPUT "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
+             "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
      COMMAND
-        ${Protobuf_PROTOC_EXECUTABLE} ARGS --cpp_out=${CMAKE_BINARY_DIR}/src -I
-        ${PROTOBUF_INCLUDE_DIRS} -I ${CMAKE_SOURCE_DIR}/src ${ABS_FIL}
+        ${Protobuf_PROTOC_EXECUTABLE} ARGS --cpp_out=${_ARGS_CXX_OUT_DIR} ${_proto_include_cli_args} ${ABS_FIL}
      DEPENDS ${ABS_FIL}
      COMMENT "Running C++ protocol buffer compiler on ${FIL}"
      VERBATIM)
  endforeach()

-  set_source_files_properties(${${SRCS}} ${${HDRS}} PROPERTIES GENERATED TRUE)
+  set_source_files_properties(${${_ARGS_CXX_SOURCES}} ${${_ARGS_CXX_HEADERS}} PROPERTIES GENERATED TRUE)

  # remove some warnings from protobuf generated files
  if(WIN32)
-    set_source_files_properties(${${SRCS}} PROPERTIES COMPILE_FLAGS
+    set_source_files_properties(${${_ARGS_CXX_SOURCES}} PROPERTIES COMPILE_FLAGS
                                                      "/wd4309 /wd4838")
  else()
    set_source_files_properties(
-      ${${SRCS}}
+      ${${_ARGS_CXX_SOURCES}}
      PROPERTIES
        COMPILE_FLAGS
        "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
    )
  endif()

-  set(${SRCS}
-      ${${SRCS}}
+  set(${_ARGS_CXX_SOURCES}
+      ${${_ARGS_CXX_SOURCES}}
      PARENT_SCOPE)
-  set(${HDRS}
-      ${${HDRS}}
+  set(${_ARGS_CXX_HEADERS}
+      ${${_ARGS_CXX_HEADERS}}
      PARENT_SCOPE)
 endfunction()

-function(Service_Compiler_Generate_Cpp SRCS HDRS)
-  set(${SRCS})
-  set(${HDRS})
+function(Service_Compiler_Generate_Cpp)
+  set(_options)
+  set(_one_value_args CXX_SOURCES CXX_HEADERS CXX_OUT_DIR)
+  set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+  set(${_ARGS_CXX_SOURCES})
+  set(${_ARGS_CXX_HEADERS})

-  foreach(FIL ${ARGN})
+  set(_proto_include_cli_args)
+  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
+    list(APPEND _proto_include_cli_args -I ${_dir})
+  endforeach()
+
+  foreach(FIL ${_ARGS_PROTO_SOURCES})
    get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
    get_filename_component(FIL_WE ${FIL} NAME_WE)
    get_filename_component(FIL_DIR ${FIL} DIRECTORY)
@@ -63,58 +82,58 @@ function(Service_Compiler_Generate_Cpp SRCS HDRS)
      set(FIL_WE "${FIL_DIR}/${FIL_WE}")
    endif()

-    list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc")
-    list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h")
+    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc")
+    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h")

-    list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.cc")
-    list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.h")
+    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc")
+    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h")

-    list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.cc")
-    list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.h")
+    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc")
+    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h")

    set(TEMPLATES
-        ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/templates/cpp/client.pb.cc
-        ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/templates/cpp/client.pb.h
-        ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/templates/cpp/server.pb.cc
-        ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/templates/cpp/server.pb.h
+        ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/client.pb.cc
+        ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/client.pb.h
+        ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/server.pb.cc
+        ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/server.pb.h
    )

+    set(_plugin_py ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/plugin.py)
    if(WIN32)
+      set(_plugin_cmd ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/plugin.cmd)
      add_custom_command(
-        OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc"
-               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h"
-               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.cc"
-               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.h"
-               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.cc"
-               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.h"
+        OUTPUT "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h"
        COMMAND SET PYTHONPATH=${SPARK_PYTHON}
        COMMAND ${CMAKE_SOURCE_DIR}/.venv/Scripts/activate.bat
        COMMAND
-          ${Protobuf_PROTOC_EXECUTABLE} ARGS --cpp_out=${CMAKE_BINARY_DIR}/src
-          --plugin=protoc-gen-service=${CMAKE_SOURCE_DIR}/src/lib/api/service_compiler/plugin.cmd
-          --service_out=lang=cpp:${CMAKE_BINARY_DIR}/src -I
-          ${PROTOBUF_INCLUDE_DIRS} -I ${CMAKE_SOURCE_DIR}/src ${ABS_FIL}
-        DEPENDS ${ABS_FIL} ${TEMPLATES}
-                ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/plugin.py
+          ${Protobuf_PROTOC_EXECUTABLE} ARGS --cpp_out=${_ARGS_CXX_OUT_DIR}
+          --plugin=protoc-gen-service=${_plugin_cmd}
+          --service_out=lang=cpp:${_ARGS_CXX_OUT_DIR} -I
+          ${PROTOBUF_INCLUDE_DIRS} -I ${_protobuf_src_dir} ${ABS_FIL}
+        DEPENDS ${ABS_FIL} ${TEMPLATES} ${_plugin_cmd} ${_plugin_py}
        COMMENT "Running C++ service compiler on ${FIL}"
        VERBATIM)
    else()
      add_custom_command(
-        OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc"
-               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h"
-               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.cc"
-               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.h"
-               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.cc"
-               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.h"
+        OUTPUT "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h"
        COMMAND ${CMAKE_SOURCE_DIR}/tools/venv_exec.sh
        ARGS
          ${Protobuf_PROTOC_EXECUTABLE}
-          --cpp_out=${CMAKE_BINARY_DIR}/src
-          --plugin=protoc-gen-service=${CMAKE_SOURCE_DIR}/src/lib/api/service_compiler/plugin.py
-          --service_out=lang=cpp,make_file_name=${ABS_FIL}:${CMAKE_BINARY_DIR}/src
-          -I ${PROTOBUF_INCLUDE_DIRS} -I ${CMAKE_SOURCE_DIR}/src ${ABS_FIL}
-        DEPENDS ${ABS_FIL} ${TEMPLATES}
-                ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/plugin.py
+          --cpp_out=${_ARGS_CXX_OUT_DIR}
+          --plugin=protoc-gen-service=${_plugin_py}
+          --service_out=lang=cpp,make_file_name=${ABS_FIL}:${_ARGS_CXX_OUT_DIR}
+          ${_proto_include_cli_args} ${ABS_FIL}
+        DEPENDS ${ABS_FIL} ${TEMPLATES} ${_plugin_py}
        COMMENT "Running C++ service compiler on ${FIL}"
        VERBATIM)
    endif()
@@ -123,22 +142,22 @@ function(Service_Compiler_Generate_Cpp SRCS HDRS)

  # remove some warnings from protobuf generated files
  if(WIN32)
-    set_source_files_properties(${${SRCS}} PROPERTIES COMPILE_FLAGS
+    set_source_files_properties(${${_ARGS_CXX_SOURCES}} PROPERTIES COMPILE_FLAGS
                                                      "/wd4309 /wd4838")
  else()
    set_source_files_properties(
-      ${${SRCS}}
+      ${${_ARGS_CXX_SOURCES}}
      PROPERTIES
        COMPILE_FLAGS
        "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
    )
  endif()

-  set(${SRCS}
-      ${${SRCS}}
+  set(${_ARGS_CXX_SOURCES}
+      ${${_ARGS_CXX_SOURCES}}
      PARENT_SCOPE)
-  set(${HDRS}
-      ${${HDRS}}
+  set(${_ARGS_CXX_HEADERS}
+      ${${_ARGS_CXX_HEADERS}}
      PARENT_SCOPE)

 endfunction()
@@ -0,0 +1,83 @@
+include_guard(GLOBAL)
+
+function(protobuf_generate_cpp)
+  set(_options)
+  set(_one_value_args
+    CXX_SOURCES
+    CXX_HEADERS
+    CXX_OUT_DIR
+    PROTOC
+  )
+  set(_multi_value_args
+    PROTO_INCLUDE_DIRS
+    PROTO_SOURCES
+  )
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+
+  if (NOT DEFINED _ARGS_PROTOC)
+    if (WIN32)
+      set(_ARGS_PROTOC "${EXTERNALS_BIN}/protoc.exe")
+    else()
+      set(_ARGS_PROTOC "${EXTERNALS_BIN}/protoc")
+    endif()
+  endif()
+  set(${_ARGS_CXX_SOURCES})
+  set(${_ARGS_CXX_HEADERS})
+
+  set(_proto_include_cli_args)
+  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
+    list(APPEND _proto_include_cli_args -I ${_dir})
+  endforeach()
+
+  foreach(_fil ${_ARGS_PROTO_SOURCES})
+    get_filename_component(_abs_fil ${_fil} ABSOLUTE)
+    get_filename_component(_fil_we ${_fil} NAME_WE)
+    get_filename_component(_fil_dir ${_fil} DIRECTORY)
+    if(_fil_dir)
+      set(_fil_we "${_fil_dir}/${_fil_we}")
+    endif()
+    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.cc")
+    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.h")
+    message(STATUS "add_custom_command for ${_fil} -> ${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.h")
+    # TODO: Pass proto sources by relative path (so this fn can be called from
+    # arbitrary subdirs).
+    add_custom_command(
+      OUTPUT
+        "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.cc"
+        "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.h"
+      COMMAND
+        ${_ARGS_PROTOC}
+      ARGS
+        --cpp_out=${_ARGS_CXX_OUT_DIR} ${_proto_include_cli_args} ${_abs_fil}
+      DEPENDS ${_abs_fil}
+      COMMENT "Running C++ protocol buffer compiler on ${_fil}"
+      VERBATIM
+    )
+  endforeach()
+
+  set_source_files_properties(
+    ${${_ARGS_CXX_SOURCES}}
+    ${${_ARGS_CXX_HEADERS}}
+    PROPERTIES
+      GENERATED TRUE
+  )
+
+  # remove some warnings from protobuf generated files
+  if(WIN32)
+    set_source_files_properties(
+      ${${_ARGS_CXX_SOURCES}}
+      PROPERTIES
+        COMPILE_FLAGS "/wd4309 /wd4838"
+    )
+  else()
+    set_source_files_properties(
+      ${${_ARGS_CXX_SOURCES}}
+      PROPERTIES
+        COMPILE_FLAGS "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
+    )
+  endif()
+
+  set(${_ARGS_CXX_SOURCES} ${${_ARGS_CXX_SOURCES}} PARENT_SCOPE)
+  set(${_ARGS_CXX_HEADERS} ${${_ARGS_CXX_HEADERS}} PARENT_SCOPE)
+endfunction()
+
@@ -0,0 +1,61 @@
+include_guard(GLOBAL)
+
+function(protobuf_generate_cpp_config)
+  set(_options)
+  set(_one_value_args
+    PROTO_SOURCES_HEADER
+    PROTO_DEFAULTS_HEADER
+  )
+  set(_multi_value_args
+    PROTO_SOURCES
+    PROTO_DEFAULTS
+  )
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+
+  # Configure the 'config_proto_files' header. It provides an array of strings populated with the
+  # files in PROTO_SOURCES. At runtime, we use this array to generate a special 'config' proto
+  # message with fields for each message defined in said files. A global instance of this 'config'
+  # proto is then constructed, and populated with [compile-time] defaults, and [runtime] overrides
+  # (i.e., in spark.conf). Calls to `bts::config::get<T>()` retrieve the value from the field
+  # associated with type `T` in this global instance.
+  message(STATUS "PROTO_SOURCES: ${_ARGS_PROTO_SOURCES}")
+  message(STATUS "PROTO_SOURCES_HEADER: ${_ARGS_PROTO_SOURCES_HEADER}")
+  message(STATUS "PROTO_DEFAULTS_HEADER: ${_ARGS_PROTO_DEFAULTS_HEADER}")
+  if (_ARGS_PROTO_SOURCES_HEADER)
+    set(PROTO_SOURCES)
+    set(PROTO_SOURCES_CXX_ARRAY_ITEMS "")
+    foreach(_proto_path ${_ARGS_PROTO_SOURCES})
+      list(APPEND PROTO_SOURCES ${_proto_path})
+      string(APPEND PROTO_SOURCES_CXX_ARRAY_ITEMS "\"${_proto_path}\",\n")
+    endforeach()
+    list(LENGTH PROTO_SOURCES PROTO_SOURCES_LENGTH)
+    message(STATUS "CONFIGURING PROTO_SOURCES_HEADER): ${_ARGS_PROTO_SOURCES_HEADER}")
+    configure_file(${CMAKE_SOURCE_DIR}/cmake/config_proto_files.h.in ${_ARGS_PROTO_SOURCES_HEADER})
+  endif()
+
+  # Generate static defaults. Ideally, this would be a single string, but we use an array because
+  # the compiler complains if the string is longer than 65k.
+  set(_cfg_defaults_str "inline const char * config_static_defaults[] = { ")
+
+  foreach(_cfg_file_path ${_ARGS_PROTO_DEFAULTS})
+    file(READ ${_cfg_file_path} _cfg_file_content)
+    string(REPLACE "\"" "\\\"" _cfg_file_content_escaped ${_cfg_file_content})
+    string(REPLACE "\n" ";" _cfg_lines ${_cfg_file_content_escaped})
+
+    foreach(_line ${_cfg_lines})
+      string(APPEND _cfg_defaults_str "\n  \"${_line}\",")
+    endforeach(_line)
+  endforeach(_cfg_file_path)
+
+  string(APPEND _cfg_defaults_str "};")
+  set(_old_cfg_defaults_str "")
+
+  if(EXISTS ${_ARGS_PROTO_DEFAULTS_HEADER})
+    file(READ ${_ARGS_PROTO_DEFAULTS_HEADER} _old_cfg_defaults_str)
+  endif()
+
+  if(NOT _cfg_defaults_str STREQUAL _old_cfg_defaults_str)
+    file(WRITE "${_ARGS_PROTO_DEFAULTS_HEADER}" "${_cfg_defaults_str}")
+  endif()
+endfunction()
+
@@ -10,7 +10,7 @@ function(protobuf_generate_fs)
    set(_ARGS_TARGET_NAME "${PROJECT_NAME}_generate_fs_protos")
  endif()
  if (NOT DEFINED _ARGS_PROTOC)
-    set(_ARGS_PROTOC "${SPARK_BIN}/protoc")
+    set(_ARGS_PROTOC "${EXTERNALS_BIN}/protoc")
  endif()

  set(_proto_include_cli_args)
@@ -27,8 +27,8 @@ function(protobuf_generate_fs)
  endforeach()

  set(_venv_exec ${CMAKE_SOURCE_DIR}/tools/venv_exec.sh)
-  set(_plugin_py ${CMAKE_SOURCE_DIR}/src/lib/api/service_compiler/plugin.py)
-  set(_template_fs ${CMAKE_SOURCE_DIR}/src/lib/api/service_compiler/templates/fs/fable.fs)
+  set(_plugin_py ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/plugin.py)
+  set(_template_fs ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/fs/fable.fs)
  add_custom_command(
    OUTPUT ${_outputs}
    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
@@ -0,0 +1,10 @@
+include_guard(GLOBAL)
+
+macro(set_first_defined RESULT)
+  foreach(_arg ${ARGN})
+    if (DEFINED ${_arg})
+      set(${RESULT} ${${_arg}})
+      break()
+    endif()
+  endforeach()
+endmacro()
@@ -0,0 +1,39 @@
+include_guard(DIRECTORY)
+include(${CMAKE_SOURCE_DIR}/cmake/SymLink.cmake)
+
+function(sym_link_python_module)
+  set(_options)
+  set(_one_value_args LIB_NAME MODULE_NAME PLUGIN_NAME)
+  set(_multi_value_args)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}"
+                        "${_multi_value_args}" ${ARGN})
+  if(NOT DEFINED _ARGS_LIB_NAME)
+    message(
+      FATAL_ERROR "LIB_NAME must be specified for sym_link_python_module()")
+  endif()
+  set(_lib_name ${_ARGS_LIB_NAME})
+  set(_module_name ${_ARGS_MODULE_NAME})
+  if(NOT DEFINED _ARGS_MODULE_NAME)
+    set(_module_name ${_lib_name}_module)
+  endif()
+  set(_plugin_name ${_ARGS_PLUGIN_NAME})
+  set(_deps ${_lib_name} ${_module_name})
+  if(DEFINED _ARGS_PLUGIN_NAME)
+    list(APPEND _deps ${_plugin_name} copy_${_plugin_name})
+  endif()
+  add_dependencies(${PROJECT_NAME} ${_deps})
+  sym_link_local_lib(${_lib_name})
+  set(_module_suffix cpython-310-x86_64-linux-gnu.so)
+  sym_link_file(
+    SOURCE
+    ${CMAKE_BINARY_DIR}/src/lib_${_lib_name}/${_module_name}.${_module_suffix}
+    DEST
+    ${CMAKE_CURRENT_BINARY_DIR}/${_module_name}.${_module_suffix}
+    INSTALL_DEST
+    ${CMAKE_INSTALL_BINDIR}/)
+  if(DEFINED _ARGS_PLUGIN_NAME)
+    get_target_property(plugins ${_plugin_name} INTERFACE_PYTHON_SCRIPTS_DIR)
+    sym_link_dir(TARGET_NAME ${PROJECT_NAME}_link_${_plugin_name} SOURCE
+                 ${plugins} DEST ${CMAKE_CURRENT_BINARY_DIR}/${_plugin_name})
+  endif()
+endfunction()
@@ -0,0 +1,11 @@
+#pragma once
+
+#include <array>
+
+namespace bts::@PROJECT_NAME@::protobufs {
+
+inline std::array<const char *, @PROTO_SOURCES_LENGTH@> config_proto_files = {
+@PROTO_SOURCES_CXX_ARRAY_ITEMS@
+};
+
+} // namespace bts::@PROJECT_NAME@::protobufs
@@ -1,24 +0,0 @@
-cmake_minimum_required(VERSION 3.5)
-
-# Explicitly set policy to convert relative paths to absolute (although this is a default policy,
-# setting it explicitly prevents a compile-warning)
-cmake_policy(SET CMP0076 NEW)
-
-# add protos to the library to facilitate source_group
-target_sources(libspark PUBLIC ${Config_Protos} ${NonConfig_Protos} ${Service_Protos})
-
-function(assign_source_group)
-  foreach(_source IN ITEMS ${ARGN})
-    if(IS_ABSOLUTE "${_source}")
-      file(RELATIVE_PATH _source_rel "${CMAKE_CURRENT_SOURCE_DIR}" "${_source}")
-    else()
-      set(_source_rel "${_source}")
-    endif()
-    get_filename_component(_source_path "${_source_rel}" PATH)
-    string(REPLACE "/" "\\" _source_path_msvc "${_source_path}")
-    source_group("${_source_path_msvc}" FILES "${_source}")
-    set(SOURCE_FILES "${_source}")
-  endforeach()
-endfunction(assign_source_group)
-
-assign_source_group(${SparkLib_Sources} ${Config_Protos} ${NonConfig_Protos} ${Service_Protos})
@@ -3,11 +3,12 @@ services:
    volumes:
      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/sparkdata/data
    build:
-      dockerfile: src/server/sparkd/Dockerfile
+      dockerfile: src/app_sparkd/Dockerfile
      args:
        - IMAGE=${SPARKD__IMAGE}
        - SOURCE_DIR=${SPARKD__SOURCE_DIR}
        - CONFIG_DIR=${SPARKD__CONFIG_DIR}
+        - PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
    shm_size: "${SPARKD__SHM_SIZE:-512mb}"
    command: >
      sh -c "
@@ -21,6 +22,10 @@ services:
          . /home/bts/sparkdata/conf/.env.local
          set +a
        fi
+        if [ -d /home/bts/sparkdata/conf/plugins ]; then
+          echo 'Copying /home/bts/sparkdata/conf/plugins to runtime dir (/home/bts/bin)'
+          cp -r /home/bts/sparkdata/conf/plugins /home/bts/bin/
+        fi
        ./sparkd run"
    env_file:
      - .env
@@ -39,11 +44,12 @@ services:
      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/sparkdata/data
      - ${SPARKD__RUNTIME_CONF_DIR:-sparkd_conf}:/home/bts/sparkdata/conf
    build:
-      dockerfile: src/server/sparkd/Dockerfile
+      dockerfile: src/app_sparkd/Dockerfile
      args:
        - IMAGE=${SPARKD__IMAGE}
        - SOURCE_DIR=${SPARKD__SOURCE_DIR}
        - CONFIG_DIR=${SPARKD__CONFIG_DIR}
+        - PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
    shm_size: "${SPARKD__SHM_SIZE:-512mb}"
    command: >
      sh -c "
@@ -59,6 +65,10 @@ services:
          . /home/bts/sparkdata/conf/.env.local
          set +a
        fi
+        if [ -d /home/bts/sparkdata/conf/plugins ]; then
+          echo 'Copying /home/bts/sparkdata/conf/plugins to runtime dir (/home/bts/bin)'
+          cp -r /home/bts/sparkdata/conf/plugins /home/bts/bin/
+        fi
        ./sparkd run"
    env_file:
      - .env
@@ -124,8 +124,8 @@ service OrderService {
 ```
 ### C++ Example
 ```cpp
-#include "lib/api/public/ServiceContext.h"
-#include "lib/api/services/OrderService.client.pb.h"
+#include "spark/api/public/ServiceContext.h"
+#include "spark/api/services/OrderService.client.pb.h"
 using namespace bts::api;
 ServiceContext ctx;
 auto orders = ctx.locate<orderman::OrderService>();
@@ -29,9 +29,9 @@ namespace sample {
 ```

 ```cpp
-#include "lib/api/public/ServiceContext.h"
-#include "spark/lib/domain_model/Security.h"
-#include "lib/api/services/MarketDataService.client.pb.h"
+#include "spark/api/public/ServiceContext.h"
+#include "spark/domain_model/Security.h"
+#include "spark/api/services/MarketDataService.client.pb.h"
 #include <thread>

 using namespace bts::api;
@@ -16,11 +16,16 @@ jmespath==1.0.1
 lazy-object-proxy==1.6.0
 MarkupSafe==3.0.2
 mccabe==0.6.1
+numpy==2.2.6
 platformdirs==2.3.0
 protobuf==4.21.12
 pybars3==0.9.7
+pybind11==2.13.6
+pybind11-stubgen==2.5.3
+pybit==5.11.0
 pycodestyle==2.7.0
 pycparser==2.20
+pycryptodome==3.23.0
 pylint==2.10.2
 PyMeta3==0.5.1
 python-dateutil==2.9.0.post0
@@ -31,6 +36,7 @@ six==1.17.0
 termcolor==1.1.0
 toml==0.10.2
 urllib3==2.0.3
+websocket-client==1.8.0
 Werkzeug==3.1.3
 wrapt==1.12.1
 yapf==0.31.0
@@ -18,6 +18,14 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_websocket", "src\clie
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_api_server", "src\client\spark_api_server\spark_api_server.csproj", "{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamCore", "src\DataStreamSuite\DataStreamCore\DataStreamCore.csproj", "{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStream", "src\DataStreamSuite\DataStream\DataStream.csproj", "{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SharedStreams", "src\DataStreamSuite\SharedStreams\SharedStreams.csproj", "{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PubSub", "src\DataStreamSuite\PubSub\PubSub.csproj", "{51C40BA9-F226-4A35-939D-1ED6F5FF7299}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|x64 = Debug|x64
@@ -90,6 +98,38 @@ Global
 		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x64.Build.0 = Release|x64
 		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x64.ActiveCfg = Release|x64
 		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x64.Build.0 = Release|x64
+		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Debug|x64.Build.0 = Debug|Any CPU
+		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Release|x64.ActiveCfg = Release|Any CPU
+		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Release|x64.Build.0 = Release|Any CPU
+		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Debug|x64.Build.0 = Debug|Any CPU
+		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Release|x64.ActiveCfg = Release|Any CPU
+		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Release|x64.Build.0 = Release|Any CPU
+		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Debug|x64.Build.0 = Debug|Any CPU
+		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Release|x64.ActiveCfg = Release|Any CPU
+		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Release|x64.Build.0 = Release|Any CPU
+		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Debug|x64.Build.0 = Debug|Any CPU
+		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Release|x64.ActiveCfg = Release|Any CPU
+		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Release|x64.Build.0 = Release|Any CPU
+		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -0,0 +1,11 @@
+namespace bts.BlueOrb {
+   using BCG;
+
+   [DataStreamUserRespresentation("AddColumnsStream")]
+   public class AddColumnsStream: ViewStream {
+      public AddColumnsStream(ViewStream parent_str, string add_col_expr) {
+         ViewStream rs = new BCG.AddColumnsStream(parent_str, add_col_expr);
+         mySetRealSource(rs);
+      }
+   }
+}
@@ -1,7 +1,7 @@
 using System;

 namespace BCG {
-   [DataStreamUserRespresentation("count")]
+   [DataStreamUserRespresentation("CountStream")]
   public class CountStream : ViewStream {
      private DataPacket myLastFiredDataPacket;
      private IBlueTableSchema mySchema = new BlueTableSchema("count",
@@ -3,36 +3,34 @@ namespace bts.BlueOrb {
   using System;
   using System.Text.RegularExpressions;

-
-   public class RegExpFilter : IStreamFilter {
-      private Regex[] myRegExps;
-      private string[] myCols;
-
-      public RegExpFilter(string col, string regExp)
-         : this(new string[] { col }, new string[] { regExp }) { }
-
-      public RegExpFilter(string[] cols, string[] regExps) {
-         myCols = cols;
-         myRegExps = new Regex[regExps.Length];
-         for (int i = 0; i < regExps.Length; i++)
-            myRegExps[i] = new Regex(regExps[i]);
-      }
-
-      #region IStreamFilter Members
-      public bool filter(IBlueRow dr) {
-         bool rv = true;
-         for (int i = 0; i < myCols.Length; i++) {
-            rv &= myRegExps[i].IsMatch(dr[myCols[i]] is DBNull ? "" : dr[myCols[i]].ToString());
-            if (!rv)
-               return rv;
-         }
-         return rv;
-      }
-      #endregion
-   }
-
-   //[DataStreamUserReprsentation("FilterStream")]
+   [DataStreamUserRespresentation("FilterStream")]
   public class FilterStream : ViewStream {
+      public class RegExpFilter : IStreamFilter {
+         private Regex[] myRegExps;
+         private string[] myCols;
+
+         public RegExpFilter(string col, string regExp)
+            : this(new string[] { col }, new string[] { regExp }) { }
+
+         public RegExpFilter(string[] cols, string[] regExps) {
+            myCols = cols;
+            myRegExps = new Regex[regExps.Length];
+            for (int i = 0; i < regExps.Length; i++)
+               myRegExps[i] = new Regex(regExps[i]);
+         }
+
+         #region IStreamFilter Members
+         public bool filter(IBlueRow dr) {
+            bool rv = true;
+            for (int i = 0; i < myCols.Length; i++) {
+               rv &= myRegExps[i].IsMatch(dr[myCols[i]] is DBNull ? "" : dr[myCols[i]].ToString());
+               if (!rv)
+                  return rv;
+            }
+            return rv;
+         }
+         #endregion
+      }
      private BlueTable myLastFiredRows;
      private IBlueRowCollection myLastReceivedRows;
      private IStreamFilter myFilter;
@@ -1,17 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-using BCG;
-
-namespace bts.BlueOrb {   
-   public interface INonIncJoinInlineMapper
-   {      
-      IBlueTableSchema get_output_schema(IBlueTableSchema schema);
-
-      IBlueRowCollection map_rows(IBlueRowCollection rows);
-      
-      event MapperChangeDelegate onMapperChange;
-   }
-}
@@ -1,152 +0,0 @@
-using bts;
-using bts.collections;
-
-using mbts;
-
-using System;
-using System.Collections.Generic;
-using System.Diagnostics;
-using System.Linq;
-using System.Security.Cryptography;
-using BCG;
-
-namespace bts.BlueOrb {   
-   internal class NonIncJoinBookkeeper : IDisposable {
-      public Action on_new_data;
-      private readonly NonIncJoinSpec _join_spec;
-      private readonly int _join_position;
-      private readonly int _start_index_in_output;
-      private readonly BlueRowHasher _row_hasher;
-      private readonly bool _is_left_most;
-      private readonly bool _is_tag_along;
-      private int _n_total_fires = 0, _n_expensive_fires = 0;
-
-      private IBlueTableSchema _schema = null;
-      private IDictionary<int, IBlueRow> _row_map = new Dictionary<int, IBlueRow>();
-      private List<Pair<int, IBlueRow>> _row_list = new List<Pair<int, IBlueRow>>();
-      private IBlueRow _solitary_row = null;
-
-      private int[] _join_col_indices = null,
-                     _data_col_indices = null;
-
-      public NonIncJoinBookkeeper(NonIncJoinSpec the_join_spec, int join_position, int start_index_in_output, BlueRowHasher row_hasher) {
-         _join_spec = the_join_spec;
-         _join_position = join_position;
-         _start_index_in_output = start_index_in_output;
-         _row_hasher = row_hasher;
-         _is_left_most = _join_position == 0;
-         _is_tag_along = _join_spec.join_col_names.Length == 0;
-      }
-
-      private bool _filled_missing_data = false;
-      private void _fill_missing_data(DataPacket dp) {
-         if (_filled_missing_data)
-            return;
-         Func<string[], bool> null_or_empty = (str_arr) => {
-            if (str_arr == null || str_arr.Length == 0)
-               return true;
-            foreach (string s in str_arr) {
-               if (!String.IsNullOrEmpty(s)) {
-                  return false;
-               }
-            }
-            return true;
-         };
-         Func<string[], string[]> remove_bad_cols = (col_names) => {
-            HashSet<string> schema_col_map = new HashSet<string>(dp.schema.columnNames);
-            List<string> cols_list = new List<string>(col_names);
-            cols_list.RemoveWhere(c => !schema_col_map.Contains(c));
-            return cols_list.ToArray();
-         };
-         if (null_or_empty(_join_spec.data_col_names)) {
-            _join_spec.data_col_names = dp.schema.columnNames;
-         }
-         _join_spec.data_col_names = remove_bad_cols(_join_spec.data_col_names);
-         if (null_or_empty(_join_spec.data_col_names)) {
-            _join_spec.data_col_names = dp.schema.columnNames;
-         }
-         if (null_or_empty(_join_spec.projected_col_names)) {
-            _join_spec.projected_col_names = _join_spec.data_col_names;
-         }
-      }
-
-      public void init() {
-         _join_spec.stream.onChange += _on_stream_cb;
-      }
-
-      public void _on_stream_cb(object dp_obj) {
-         //Log.Gen.info($"NonIncJoinBookkeeper._on_stream_cb dp_obj: {dp_obj}");
-         var timer = Stopwatch.StartNew();
-         DataPacket dp = (DataPacket)dp_obj;
-         _fill_missing_data(dp);
-         if (_schema == null) {
-            _schema = dp.schema;
-            _join_col_indices = BlueDataUtility.get_column_indices(_schema, _join_spec.join_col_names);
-            _data_col_indices = BlueDataUtility.get_column_indices(_schema, _join_spec.data_col_names);
-         }
-
-         if (_is_left_most) {
-            _row_list.Clear();
-            foreach (IBlueRow r in dp.newRows)
-               _row_list.Add(new Pair<int, IBlueRow>(_row_hasher.get_hash(r, _join_col_indices), r));
-         } else {
-            if (_is_tag_along)
-               _solitary_row = BlueDataUtility.extractRow(dp.newRows, 0);
-            else {
-               _row_map.Clear();
-               foreach (IBlueRow r in dp.newRows)
-                  _row_map[_row_hasher.get_hash(r, _join_col_indices)] = r;
-            }
-         }
-         timer.Stop();
-         _n_total_fires++;
-         if (timer.ElapsedMilliseconds >= 50) {
-            _n_expensive_fires++;
-            Log.Gen.info($"NonIncJoinBookkeepingData._on_stream_cb {dp.newRows.Count} rows {_join_col_indices.Length} "
-               + $"key cols {_join_position} join pos time {timer.Elapsed} ratio {_n_expensive_fires}/{_n_total_fires}");
-         }
-         if (on_new_data != null)
-            on_new_data();
-      }
-
-      public bool is_ready { get { return _schema != null; } }
-
-      public void constribute_cols_to_schema(List<ColumnInfo> cis) {
-         for (int i = 0; i < _join_spec.data_col_names.Length; i++)
-            cis.Add(new ColumnInfo(_join_spec.projected_col_names[i], cis.Count, _schema[_join_spec.data_col_names[i]].colType));
-      }
-
-      public List<Pair<int, IBlueRow>> get_list() { return _row_list; }
-
-      public void copy_master_row_to_output_row(int row_index, object[] output_row) {
-         IBlueRow this_row = _row_list[row_index].second;
-         BlueDataUtility.copy_to_row(this_row, _data_col_indices, output_row, _start_index_in_output);
-      }
-
-      public void copy_matching_row_to_output_row(int row_hash, object[] output_row) {
-         if (_is_tag_along) {
-            if (_solitary_row != null)
-               BlueDataUtility.copy_to_row(_solitary_row, _data_col_indices, output_row, _start_index_in_output);
-            else
-               copy_dbnull_to_output_row(output_row);
-         } else {
-            IBlueRow this_row = null;
-            if (_row_map.TryGetValue(row_hash, out this_row))
-               BlueDataUtility.copy_to_row(this_row, _data_col_indices, output_row, _start_index_in_output);
-            else
-               copy_dbnull_to_output_row(output_row);
-         }
-      }
-
-      private void copy_dbnull_to_output_row(object[] output_row) {
-         for (int i = 0; i < _data_col_indices.Length; i++)
-            output_row[i + _start_index_in_output] = DBNull.Value;
-      }
-
-      #region IDisposable
-
-      public void Dispose() { _join_spec.stream.onChange -= _on_stream_cb; }
-
-      #endregion
-   }
-}
@@ -1,67 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-
-using BCG;
-
-using mbts;
-
-namespace bts.BlueOrb {   
-   // this a collection of spec, theres no joining logic in here
-   public class NonIncJoinSpec {
-      public ViewStream stream;
-      // join col names are the names on which the streams are joined (for equality),
-      // data col names are the names in the original stream
-      // projected col names are their new names, ie if its x:l_x, then x is in the 
-      // data_col_names and l_x in projected cols
-      public string[] join_col_names, data_col_names, projected_col_names;
-
-      public NonIncJoinSpec() { }
-
-      public NonIncJoinSpec(ViewStream the_stream, string[] the_join_col_names, string[] the_data_col_names)
-         : this(the_stream, the_join_col_names, the_data_col_names, the_data_col_names) { }
-
-      public NonIncJoinSpec(ViewStream the_stream, string[] the_join_col_names, string[] the_data_col_names, string[] the_projected_col_names) {
-         stream = the_stream;
-         join_col_names = the_join_col_names;
-         data_col_names = the_data_col_names;
-         projected_col_names = the_projected_col_names;
-      }
-
-      public static NonIncJoinSpec[] from_triplet(params Triplet<ViewStream, string, string>[] join_specs) {
-         List<NonIncJoinSpec> rv = new List<NonIncJoinSpec>();
-         foreach (Triplet<ViewStream, string, string> r in join_specs) {
-            string[] keys = String.IsNullOrEmpty(r.second) ? new string[] { } : r.second.Split(' ');
-            string[] visible_cols = r.third.Split(' ');
-            List<string> data_col_names = new List<string>(), mapped_col_names = new List<string>();
-            foreach (string s in visible_cols) {
-               string[] toks = s.Split(':');
-               if (toks.Length == 1) {
-                  data_col_names.Add(toks[0]);
-                  mapped_col_names.Add(toks[0]);
-               } else if (toks.Length == 2) {
-                  data_col_names.Add(toks[0]);
-                  mapped_col_names.Add(toks[1]);
-               }
-            }
-            rv.Add(new NonIncJoinSpec(r.first, keys, data_col_names.ToArray(), mapped_col_names.ToArray()));
-         }
-         return rv.ToArray();
-      }
-
-      public override string ToString() {
-         Func<string[], string> arr_txt = (a) => {
-            string rv = "";
-            foreach (string s in a) {
-               rv += "{s} ";
-            }
-            rv = rv.Trim();
-            return rv;
-         };
-         string rv = $"stream: {stream} join_col_names: {arr_txt(join_col_names)} data_col_names: {arr_txt(data_col_names)} projected_col_names: {arr_txt(projected_col_names)}";
-         return rv;
-      }
-   }
-}
@@ -1,165 +0,0 @@
-using bts;
-
-using mbts;
-
-using System;
-using System.Collections.Generic;
-using System.Diagnostics;
-using BCG;
-
-namespace bts.BlueOrb {   
-   [DataStreamUserRespresentation("JoinStream")]
-   public class NonIncJoinStream : ViewStream, BCG.Utilities.General.IWorker {
-      private readonly NonIncJoinSpec[] _join_specs;
-      private readonly INonIncJoinInlineMapper _output_mapper;
-      private readonly BlueRowHasher _row_hasher;
-      private readonly TimeSpan _throttle_period;
-      private readonly ViewStreamDatabase _vsd;
-
-      private List<NonIncJoinBookkeeper> _book_keeping_data = new List<NonIncJoinBookkeeper>();
-      private IBlueTableSchema _output_schema = null;
-      private DateTime _last_fire_time = DateTime.MinValue;
-      private bool _is_dirty = false;
-
-      public NonIncJoinStream(ViewStream left, string left_join_cols, string left_proj,
-                              ViewStream right, string right_join_cols, string right_proj)
-         : this(new Triplet<ViewStream, string, string>[]
-               {
-                  new Triplet<ViewStream, string, string> (left, left_join_cols, left_proj),
-                  new Triplet<ViewStream, string, string> (right, right_join_cols, right_proj)
-               }) { }
-
-      public NonIncJoinStream(params Triplet<ViewStream, string, string>[] join_specs)
-         : this(NonIncJoinSpec.from_triplet(join_specs), null, 0) { }
-
-      public NonIncJoinStream(ViewStreamDatabase vsd, INonIncJoinInlineMapper output_mapper, int throttles_ms,
-                              params Triplet<ViewStream, string, string>[] join_specs)
-         : this(vsd, NonIncJoinSpec.from_triplet(join_specs), output_mapper, throttles_ms) { 
-      }
-
-      public NonIncJoinStream(NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper output_mapper, int throttles_ms = 0)
-         : this(ViewStreamDatabase.instance, join_specs, output_mapper, throttles_ms) { }
-
-      public NonIncJoinStream(ViewStreamDatabase vsd, NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper output_mapper, int throttles_ms = 0) {
-         _join_specs = join_specs;
-         _output_mapper = output_mapper;
-         _row_hasher = new BlueRowHasher(_join_specs[0].join_col_names.Length);
-         _throttle_period = throttles_ms == 0 ? TimeSpan.Zero : new TimeSpan(0, 0, 0, 0, throttles_ms);
-         _vsd = vsd;
-      }
-
-      protected override void myOnFirstSubscriber() {
-         int data_col_no = 1;
-         for (int jp = 0; jp < _join_specs.Length; jp++) {
-            NonIncJoinBookkeeper s = new NonIncJoinBookkeeper(_join_specs[jp], jp, data_col_no, _row_hasher);
-            //Log.Gen.info($"NonIncJoinStream.myOnFirstSubscriber jp: {jp} spec: {_join_specs[jp]}");
-            data_col_no += _join_specs[jp].data_col_names.Length;
-            s.on_new_data += _stream_callback;
-            _book_keeping_data.Add(s);
-         }
-         if (_output_mapper != null)
-            _output_mapper.onMapperChange += _on_mapper_change;
-         _vsd.addWorker(this);
-         foreach (var bkd in _book_keeping_data)
-            bkd.init();
-      }
-
-      protected override void myOnLastSubscriber() {
-         foreach (NonIncJoinBookkeeper d in _book_keeping_data)
-            d.Dispose();
-         if (_output_mapper != null)
-            _output_mapper.onMapperChange -= _on_mapper_change;
-         _vsd.removeWorker(this);
-      }
-
-      private void _stream_callback() {
-         //Log.Gen.info("NonIncJoinStream._stream_callback received callback");
-         _is_dirty = true;
-         doWork();
-      }
-
-      private void _on_mapper_change() {
-         _is_dirty = true;
-         doWork();
-      }
-
-      private bool _is_ready() {
-         bool rv = true;
-         foreach (NonIncJoinBookkeeper d in _book_keeping_data)
-            rv &= d.is_ready;
-         return rv;
-      }
-
-      private void _construct_schema() {
-         if (_output_schema == null) {
-            List<ColumnInfo> cis = new List<ColumnInfo>(new ColumnInfo[] { new ColumnInfo("id", 0, typeof(int)) });
-            foreach (NonIncJoinBookkeeper d in _book_keeping_data)
-               d.constribute_cols_to_schema(cis);
-            _output_schema = new BlueTableSchema("NonIncJoinStream", cis.ToArray());
-         }
-      }
-
-      private int _n_total_fires = 0, _n_expensive_fires = 0;
-      private void _form_and_fire() {
-         //Log.Gen.info($"NonIncJoinStream._form_and_fire _is_ready(): {_is_ready()}");
-         if (_is_ready()) {
-            _construct_schema();
-            var timer = Stopwatch.StartNew();
-            IList<Pair<int, IBlueRow>> left_row_list = _book_keeping_data[0].get_list();
-            List<object[]> output_rows = new List<object[]>();
-
-            for (int i = 0; i < left_row_list.Count; i++) {
-               object[] or = new object[_output_schema.columnInfo.Length];
-               or[0] = i;
-               output_rows.Add(or);
-            }
-
-            for (int ji = 0; ji < _book_keeping_data.Count; ji++) {
-               NonIncJoinBookkeeper bk_data = _book_keeping_data[ji];
-               for (int i = 0; i < left_row_list.Count; i++) {
-                  if (ji == 0)
-                     bk_data.copy_master_row_to_output_row(i, output_rows[i]);
-                  else
-                     bk_data.copy_matching_row_to_output_row(left_row_list[i].first, output_rows[i]);
-               }
-            }
-            IBlueRowCollection dp_output_rows = _apply_map(new SequentialIDBlueTable(_output_schema, output_rows));
-            timer.Stop();
-            _n_total_fires++;
-            //if (timer.ElapsedMilliseconds >= 250) {
-            _n_expensive_fires++;
-            //string log_msg = $"Join and map time for {_book_keeping_data.Count} "
-            //   + $"tables {timer.Elapsed.TotalMilliseconds} msecs, ratio {_n_expensive_fires}/{_n_total_fires}";
-            //Log.Gen.info(log_msg);
-            //}
-            var dp_out = BlueDataUtility.build_dp_inc(dp_output_rows, myLastValue as DataPacket, true);
-            myFireEvent(dp_out);
-            //myFireEvent(new DataPacket(dp_output_rows, dp_output_rows, myLastValue == null ? new BlueTable(_output_schema) : (myLastValue as DataPacket).newRows));
-         }
-      }
-
-      private IBlueRowCollection _apply_map(IBlueRowCollection unmapped_rows) {
-         if (_output_mapper != null) {
-            IBlueRowCollection output_rows;
-            var timer = Stopwatch.StartNew();
-            output_rows = _output_mapper.map_rows(unmapped_rows);
-            timer.Stop();
-            if (timer.ElapsedMilliseconds >= 50) {
-               Log.Gen.info($"Time to map {unmapped_rows.Count} rows into {output_rows.Count} rows {timer.Elapsed}");
-            }
-            return output_rows;
-         } else {
-            return unmapped_rows;
-         }
-      }
-
-      public void doWork() {
-         //Log.Gen.info($"NonIncJoinStream.doWork _last_fire_time: {_last_fire_time} _is_dirty: {_is_dirty}");
-         if (DateTime.Now - _last_fire_time >= _throttle_period && _is_dirty) {
-            _is_dirty = false;
-            _last_fire_time = DateTime.Now;
-            _form_and_fire();
-         }
-      }
-   }
-}
@@ -0,0 +1,187 @@
+namespace bts.BlueOrb {
+   using BCG;
+   using RowHash = System.Int32;
+   using System;
+   using System.Collections.Generic;
+   using System.Linq;
+   using System.Text;
+   using System.Threading.Tasks;
+   using System.Collections;
+   using System.Data.Entity.Core.Mapping;
+
+   [DataStreamUserRespresentation("JoinStream")]
+   public class SimpleJoinStream : ViewStream {
+      public class JoinBookkeepingData {
+         // initialized in cons
+         private readonly List<string> _join_col_names;
+         private readonly bool _is_left;
+         private readonly BlueRowHasher _hasher;
+         // initialized on first udpate         
+         private IBlueTableSchema _schema;
+         private int[] _join_cols;
+         private IDictionary<RowHash, List<IBlueRow>> _hash_row_map = new Dictionary<RowHash, List<IBlueRow>>();
+         
+         public JoinBookkeepingData(ViewStream vs, List<string> join_cols, bool is_left, BlueRowHasher row_hasher) {
+            _join_col_names = join_cols.Where (c => !String.IsNullOrEmpty(c)).ToList ();
+            _is_left = is_left;
+            _hasher = row_hasher;
+         }
+
+         public IBlueTableSchema schema => _schema;
+
+         public IDictionary<RowHash, List<IBlueRow>> rows => _hash_row_map;
+
+         public List<IBlueRow> get_matching_rows (int hash) {
+            List<IBlueRow> rv = null;
+            _hash_row_map.TryGetValue(hash, out rv);
+            return rv;
+         }
+
+         private RowHash _get_hash(IBlueRow r) {
+            if (_join_cols == null || _join_cols.Length == 0) { 
+               return 0;
+            }
+            return _hasher.get_hash(r, _join_cols);
+         }
+
+         public void update_data(DataPacket dp) {
+            if (!ready) {               
+               _schema = dp.newRows.schema;
+               _join_cols = _join_col_names.Count == 0 ? new int[] { }: _join_col_names.Select(n => dp.schema[n].colNo).ToArray();
+            }
+            _hash_row_map.Clear();
+            foreach (IBlueRow r in dp.newRows) {
+               int hash = _get_hash(r);
+               List<IBlueRow> hash_rows = null;
+               if (!_hash_row_map.TryGetValue(hash, out hash_rows)) {
+                  hash_rows = new List<IBlueRow>();
+                  _hash_row_map.Add(hash, hash_rows);
+               }
+               hash_rows.Add(r);
+            }
+         }
+
+         public bool ready {
+            get {
+               return _schema != null;
+            }
+         }
+
+      }
+
+      private ViewStream _ls, _rs;
+      private JoinBookkeepingData _ldata, _rdata;
+      private ViewStreamDatabase _vsd;
+      private IBlueTableSchema _schema = null;
+
+      public SimpleJoinStream(ViewStream ls, string ljc, ViewStream rs, string rjc) :
+         this(ViewStreamDatabase.instance, ls, ljc, rs, rjc) { }
+
+      public SimpleJoinStream(ViewStreamDatabase vsd, ViewStream ls, string ljc, ViewStream rs, string rjc) {
+         _ls = ls;
+         _rs = rs;
+         List<string> lj_col_list = ljc.Split(',').Where(x => !String.IsNullOrEmpty(x)).ToList();
+         List<string> rj_col_list = rjc.Split(',').Where(x => !String.IsNullOrEmpty(x)).ToList();
+         Log.assert(lj_col_list.Count ==  rj_col_list.Count);
+         int n_cols = lj_col_list.Count;
+         BlueRowHasher hasher = new BlueRowHasher(n_cols);
+         _ldata = new JoinBookkeepingData(ls, lj_col_list, true, hasher);
+         _rdata = new JoinBookkeepingData(rs, rj_col_list, false, hasher);
+      }
+
+      protected override void myOnFirstSubscriber() {
+         _ls.onChange += _ls_onchange;
+         _rs.onChange += _rs_onchange;
+      }
+
+      private void _ls_onchange(object dp_obj) {
+         DataPacket dp = dp_obj as DataPacket;
+         _ldata.update_data(dp);
+         _form_and_fire();
+      }
+
+      private void _rs_onchange(object dp_obj) {
+         DataPacket dp = dp_obj as DataPacket ;
+         _rdata.update_data(dp);
+         _form_and_fire();
+      }
+
+      protected override void myOnLastSubscriber() {
+         _ls.onChange -= _ls_onchange;
+         _rs.onChange -= _rs_onchange;
+      }
+
+      private void _build_schema() {
+         if (_schema != null)
+            return;
+         List<ColumnInfo> cis = new List<ColumnInfo>();
+         cis.Add(new ColumnInfo("id", 0, TypeCode.Int32));
+         foreach (ColumnInfo ci in _ldata.schema.columnInfo) {
+            string col_name = _rdata.schema.contains_column(ci.colName) ? $"left_{ci.colName}" : ci.colName;
+            cis.Add(new ColumnInfo(col_name, cis.Count, ci.colType));
+         }
+         foreach (ColumnInfo ci in _rdata.schema.columnInfo) {
+            string col_name = _ldata.schema.contains_column(ci.colName) ? $"right_{ci.colName}" : ci.colName;
+            cis.Add(new ColumnInfo(col_name, cis.Count, ci.colType));
+         }
+         _schema = new BlueTableSchema($"{_ldata.schema.tableName}_join_{_rdata.schema.tableName}", cis.ToArray());
+      }
+
+      private void _form_and_fire() {
+         if (!_ldata.ready||!_rdata.ready)
+            return;
+         _build_schema();
+         List<object[]> res_rows = new List<object[]>();
+         
+         Action<object[], IBlueRow, bool> copy_row = (rval, r, is_left) => {
+            int offset = 1;
+            JoinBookkeepingData sdata = _ldata;
+            if (is_left) {
+               offset = 1;
+               sdata = _ldata;
+            } else {
+               sdata = _rdata;
+               offset = 1 + _ldata.schema.columnInfo.Length;
+            }
+            for (int i = 0; i < sdata.schema.columnInfo.Length; i++) {
+               rval[offset + i] = r[i];
+            }
+         };
+         Func<int, IBlueRow, object[]> build_row_array = (id, l_row) => {
+            object[] rv = new object[_schema.columnNames.Length];            
+            for (int i = 0; i < _schema.columnNames.Length; i++) {
+               rv[i] = DBNull.Value;
+            }
+            rv[0] = id;
+            copy_row(rv, l_row, true);
+            return rv;
+         };
+         // this is where the join happens
+         foreach (var kv in _ldata.rows) {
+            RowHash hash = kv.Key;
+            List<IBlueRow> l_rows = kv.Value;
+            List<IBlueRow> r_rows = _rdata.get_matching_rows(hash);
+
+            foreach (IBlueRow lr in l_rows) {                              
+               if (r_rows != null) {
+                  foreach (IBlueRow r in r_rows) {
+                     object[] rowvals = build_row_array(res_rows.Count, lr);
+                     copy_row(rowvals, r, false);
+                     res_rows.Add(rowvals);
+                  }
+               } else {
+                  object[] rowvals = build_row_array(res_rows.Count, lr);
+                  res_rows.Add(rowvals);
+               }
+            }
+         }
+         BlueTable rv = new BlueTable(_schema);
+         foreach (object[] rval in res_rows) {
+            rv.addRow(new BlueRow(_schema, rval));
+         }
+         var dp = BlueDataUtility.build_dp(rv, myLastValue as DataPacket);
+         myFireEvent(dp);
+      }
+
+   }
+}
@@ -49,14 +49,14 @@ namespace bts.BlueOrb {
         }
      }
   }
-   public class MSecondStream : TimeUnitStream {
+   internal class MSecondStream : TimeUnitStream {
      public MSecondStream(int intervalInms, int size)
         : base(DateTime.Now, new TimeSpan(0, 0, 0, 0, intervalInms), size, ViewStreamDatabase.instance) { }

      protected override DateTime myRoundTime(DateTime dt) { return dt; }
   }

-   public class DisplayableMillisecondStream : ViewStream, IStreamMapper {
+   internal class DisplayableMillisecondStream : ViewStream, IStreamMapper {
      private IBlueTableSchema mySchema = new BlueTableSchema("ms",
                                                               new ColumnInfo("id", 0, typeof(int)),
                                                               new ColumnInfo("time", 1, typeof(string)));
@@ -20,13 +20,14 @@ namespace bts.BlueOrb {
      Sum,
      EdgeDiff,
      EdgeDiffPercentage,
-      //IsEdgeDiff,
   }
   public class MultiRowOpHelper {
+      // input + op
      public readonly List<Tuple<string, MultiRowOp>> operations;

      public static MultiRowOpHelper parse_ops(string text) {
         // text is inp_col: op, inp_col: op, inp_col:op
+         // ex: "open:First, close:Last, high:Max, low:Min, volume:Sum")
         List<Tuple<string, MultiRowOp>> rv = new List<Tuple<string, MultiRowOp>>();
         string[] calcs = text.Split(',');
         foreach (string c in calcs) {
@@ -71,6 +72,35 @@ namespace bts.BlueOrb {
         return or;
      }

+      // aggregates multiple rows into one, used by MultiRowRollingMathStream
+      public static IBlueRow group_by_compute(List<IBlueRow> rows, int start_idx, int n_rows, MultiRowOpHelper parameters) {
+         if (rows.Count() == 0) {
+            return null;
+         }
+         IBlueRow ir = rows.ElementAt(start_idx);
+         IBlueTableSchema in_schema = ir.schema;
+         IBlueTableSchema out_schema = in_schema;
+         // out_col vs inp_cal + type
+         IDictionary<string, Tuple<string, MultiRowOp>> out_col_map =
+            new Dictionary<string, Tuple<string, MultiRowOp>>();
+         foreach (var t in parameters.operations) {
+            string out_col_name = $"{t.Item1}_{t.Item2}".ToLower();
+            out_schema = out_schema.add_column(TypeCode.Double, out_col_name);
+            out_col_map[out_col_name] = t;
+         }
+         MutableBlueRow or = new MutableBlueRow(out_schema, ir);
+         foreach (var kv in out_col_map) {
+            string out_col_name = kv.Key;
+            string in_col_name = kv.Value.Item1;
+            MultiRowOp op = kv.Value.Item2;
+            IEnumerable<double> col_vals = BlueDataUtility.get_colvalues<double>(rows, start_idx, n_rows, in_col_name);
+            double comp_res = compute_op(col_vals, op);
+            or.setValue(out_col_name, comp_res);
+            or.setValue("id", ir["id"]);
+         }
+         return or;
+      }
+
      // rolling_rows are the rolling window of rows. the 0th row is the one thats gettin
      // calculated. rows 1 - n-1 are its window rows. the schema is based on the operations.

@@ -107,9 +137,13 @@ namespace bts.BlueOrb {
            case MultiRowOp.Min:
               return col_vals.Min<double>();
            case MultiRowOp.Mean:
-               return col_vals.Sum() / col_vals.Count();
+               return MathNet.Numerics.Statistics.Statistics.Mean(col_vals);
            case MultiRowOp.Sum:
               return col_vals.Sum();
+            case MultiRowOp.Median:
+               return MathNet.Numerics.Statistics.Statistics.Median(col_vals);
+            //case MultiRowOp.Mode:
+            //   return MathNet.Numerics.Statistics.Statistics.(col_vals);
            case MultiRowOp.First:
               return col_vals.First();
            case MultiRowOp.Last:
@@ -0,0 +1,40 @@
+namespace bts.BlueOrb {
+   using BCG;
+
+   using System.Collections.Generic;
+   using System.Net;
+
+   [DataStreamUserRespresentation("MultiRowRollingMathStream")]
+   public class MultiRowRollingMathStream : ViewStream {
+      private ViewStream _sorted_parent_str;
+      private MultiRowOpHelper _op;
+      private int _rolling_window_sz;
+      public MultiRowRollingMathStream(ViewStream parent_str, string op_txt, string sort_exp, int rolling_window_sz) {
+         _sorted_parent_str = new SortStream(parent_str, sort_exp);
+         _op = MultiRowOpHelper.parse_ops(op_txt);
+         _rolling_window_sz = rolling_window_sz;
+      }
+
+      protected override void myOnFirstSubscriber() {
+         _sorted_parent_str.onChange += _parent_str_onchange;
+      }      
+
+      protected override void myOnLastSubscriber() {
+         _sorted_parent_str.onChange -= _parent_str_onchange;
+      }
+
+      private void _parent_str_onchange(object dp_obj) {
+         DataPacket dp = dp_obj as DataPacket;
+         var rows = dp.newRows.to_list();
+         BlueTable out_table = null;
+         for (int i = 0; i < rows.Count; i++) {
+            var out_r = MultiRowOpHelper.group_by_compute(rows, i, _rolling_window_sz, _op);
+            if (out_table == null)
+               out_table = new BlueTable(out_r.schema);
+            out_table.addRow(out_r);
+         }            
+         var dp_out = BlueDataUtility.build_dp_inc(out_table, myLastValue as DataPacket, true);
+         myFireEvent(dp_out);
+      }
+   }
+}
@@ -1,8 +1,5 @@
 namespace bts.BlueOrb {
   using BCG;
-
-   using ProtoBuf;
-
   using System;
   using System.Collections;
   using System.Collections.Generic;
@@ -26,7 +23,7 @@ namespace bts.BlueOrb {
   #endregion


-#region Row with mapipings build in
+#region Row with mappings build in
   internal class NewProjectRow : IBlueRow {
      #region IBlueRow members
      private ProjectData _project_data;
@@ -135,38 +132,30 @@ namespace bts.BlueOrb {
         return new NewProjectRow(_project_data, row);
      }

+      // this is where most of the logic is, in terms of translating cols
      private static ProjectData _create_project_data(IBlueTableSchema input_schema,
                                                      string project_expression) {
         project_expression = project_expression.Trim();
-         string[] project_toks = project_expression.Split(' ');
+         string[] project_toks = project_expression.Split(' ', ',');
         HashSet<string> removed_cols = new HashSet<string>();
-         Dictionary<string, string> mapped_to_orig_map = new Dictionary<string, string>();
-         Dictionary<string, string> orig_to_mapped = new Dictionary<string, string>();
+         Dictionary<string, string> orig_to_mapped = new Dictionary<string, string>();      
         // by default every col will show up, so if we have the projection as (a:a1)
         // it will convert a to a1, 
+         // modifications (6/30/2025):
+         // only the columns represented here, either mapped
+         // or not, will show
         // examples: if a schema has cols a b c
-         // project(parent_str, "a:aa ") -> aa b c
-         // project(parent_str, "a:aa !c") -> aa b
-         // project(parent_str, "a:aa !") -> aa
-         // project(parent_str, "a:aa b")) -> aa b
-         bool hide_all = false;
-         foreach (string p_tok in project_toks) {
-            if (p_tok.StartsWith("!")) {
-               if (p_tok.Length == 1) {
-                  // if it is a "blanket "!"=> hide all cols except the one that
-                  // are stated explicitly
-                  hide_all = true;
-               } else {
-                  string removed_col = p_tok.Substring(1);
-                  removed_cols.Add(removed_col);
-               }
-            } else if (p_tok.Contains(":")) {
+         // project(parent_str, "a:aa b") -> aa b
+         foreach (string o_p_tok in project_toks) {
+            string p_tok = o_p_tok.Trim();
+            if (p_tok.Contains(":")) {
+               // explicit mapping
               string[] map_toks = p_tok.Split(":");
               string orig_col = map_toks[0];
               string mapped_col = map_toks[1];
-               mapped_to_orig_map.Add(mapped_col, orig_col);
-               orig_to_mapped.Add(orig_col, mapped_col);
+               orig_to_mapped.Add(orig_col, mapped_col);  
            } else {
+               // this is a col_name explicitely stated
               if (input_schema.columnNames.Contains(p_tok)) {
                  string orig_col = p_tok;
                  string mapped_col = orig_col;
@@ -181,27 +170,19 @@ namespace bts.BlueOrb {
            if (ci.colName == "id") {
               ColumnInfo nc = new ColumnInfo(ci.colName, new_cols.Count, ci.colType);
               new_cols.Add(nc);
-               continue;
-            }
-            if (removed_cols.Contains(ci.colName)) {
-               continue;
+               continue;           
            } else if (orig_to_mapped.ContainsKey(ci.colName)) {
               string mapped_name = orig_to_mapped[ci.colName];
               ColumnInfo nc = new ColumnInfo(mapped_name, new_cols.Count, ci.colType);
               new_cols.Add(nc);
-            } else {
-               if (hide_all)
-                  continue;
-               //if (ci.colName == "id" || removed_cols.Count > 0) {
-               // unless columns are  explicitely hidden, all columns show up
-               ColumnInfo nc = new ColumnInfo(ci.colName, new_cols.Count, ci.colType);
-               new_cols.Add(nc);
-               //}
            }
         }
         IBlueTableSchema new_schema = new BlueTableSchema(input_schema.tableName, new_cols.ToArray());
+         var mapped_to_orig_map = orig_to_mapped.ToDictionary(kv => kv.Value, kv => kv.Key);
         return new ProjectData(input_schema, new_schema, mapped_to_orig_map);
      }

+   
+
   }
 }
@@ -0,0 +1,127 @@
+namespace bts.BlueOrb {
+   using BCG;
+   using System.IO;
+   using System.Diagnostics;
+   using System;
+
+   [DataStreamUserRespresentation("PythonDataframeStream")]
+   public class PythonDataframeStream : ViewStream {
+      private ViewStream _parent_str;
+      private readonly string _in_csv_file_name;
+      private readonly string _out_csv_filename;
+      private readonly string _transform_function;
+      public PythonDataframeStream(ViewStream parent_str, string transform_function) {
+         var guid = System.Guid.NewGuid();
+         _in_csv_file_name = $"pdfs_in_{guid}.csv";
+         _out_csv_filename = $"pdfs_out_{guid}.csv";
+         _parent_str = new CSVSaverStream(parent_str, _in_csv_file_name);
+         _transform_function = transform_function;
+      }
+
+      protected override void myOnFirstSubscriber() {
+         _parent_str.onChange += _parent_str_onchange;
+      }
+
+      protected override void myOnLastSubscriber() {
+         _parent_str.onChange -= _parent_str_onchange;
+      }
+
+      private void _parent_str_onchange(object dp_obj) {
+         Log.assert(dp_obj != null);
+         // the file _in_csv_file_name is now populated.
+         // call py_func_wrapper_main and pass this file name as well
+         // as the output file name
+         // py_func_wrapper_main reads _in_csv_file_name into the df and calls 
+         // transform function, which returns the tranformed df. py_func_wrapper_main then
+         // writes it into the _out_csv_filename file.
+         // now this function should read _out_csv_filename into a dp
+         // and fire it
+         _compute_and_write();
+         ViewStream rs = new SchemaGuesserStream(new CSVReaderStream(_out_csv_filename));
+         IBlueRowCollection nrows = null;
+         rs.onChange += (dp_obj) => {
+            nrows = (dp_obj as DataPacket).newRows;
+         };
+         var dp = BlueDataUtility.build_dp(nrows, myLastValue as DataPacket);
+         myFireEvent(dp);
+      }
+
+      private void _compute_and_write() {
+         File.WriteAllText("user_code.py", _transform_function);
+         // wrapper function
+         string py_func_wrapper_main = @"
+import sys
+from user_code import transform
+import pandas as pd
+
+if __name__ == '__main__':
+   input_file_name = sys.argv[1]
+   output_file_name = sys.argv[2]
+# 1. load input_file_name into df
+# 2. do some test calc
+# 3. write the output of 2 into output_file_name
+   df = pd.read_csv(input_file_name)   
+   df_out = transform(df)
+   df_out.to_csv(output_file_name, index=False)";
+
+
+         File.WriteAllText("runner.py", py_func_wrapper_main);
+
+         ProcessStartInfo psi = new ProcessStartInfo {
+            FileName = "python",
+            Arguments = $"runner.py {_in_csv_file_name} {_out_csv_filename}",
+            RedirectStandardOutput = false,
+            UseShellExecute = false
+         };
+         var proc = Process.Start(psi);
+         proc.WaitForExit();
+
+      }
+
+      #region Unused test code
+      private void _simple_test() {
+
+         // actual function that needs to be called ////////////////////////
+         string py_real_function = @"
+def test(x, out_path):
+   rv = x * 10
+   with open(out_path, 'w') as f:
+      f.write(str(rv))
+   return rv";
+         // the function is in a string object.
+         File.WriteAllText("user_code.py", py_real_function);
+
+
+         // runner app (top level main function) //////////////////////////////
+         string py_func_wrapper_main = @"
+import sys
+from user_code import test
+
+if __name__ == '__main__':
+   arg = int(sys.argv[1])
+   out_path = sys.argv[2]
+   rv = test(arg, out_path)
+   rv = rv + 12
+   with open(out_path, 'w') as f:to \
+      f.write(str(rv))";
+   
+         File.WriteAllText("runner.py", py_func_wrapper_main);
+
+         string outputPath = "py_out.txt";
+         ProcessStartInfo psi = new ProcessStartInfo {
+            FileName = "python",
+            Arguments = $"runner.py 4 {outputPath}",
+            RedirectStandardOutput = false,
+            UseShellExecute = false
+         };
+         var proc = Process.Start(psi);
+         proc.WaitForExit();
+
+         // Now read the output written by the function
+         string result = File.ReadAllText(outputPath);
+         Console.WriteLine(result); // Should print "6"
+
+      }
+      #endregion
+   }
+}
@@ -0,0 +1,17 @@
+namespace bts.BlueOrb {
+   using BCG;
+
+   using System;
+   using System.Collections.Generic;
+   using System.Linq;
+   using System.Text;
+   using System.Threading.Tasks;
+
+   [DataStreamUserRespresentation("SortStream")]
+   public class SortStream: ViewStream {
+      public SortStream (ViewStream parent_stream, string sort_expr) {
+         ViewStream rs = new BCG.SortStream(parent_stream, sort_expr);
+         mySetRealSource(rs);
+      }
+   }
+}
@@ -0,0 +1,16 @@
+namespace bts.BlueOrb {
+   using BCG;
+
+   using System;
+   using System.Collections.Generic;
+   using System.Linq;
+   using System.Text;
+   using System.Threading.Tasks;
+   [DataStreamUserRespresentation("TimeBasedRollingFilterStream")]
+   public class TimeBasedRollingFilterStream : ViewStream {
+      public TimeBasedRollingFilterStream(ViewStream parent_str, string time_col, double n_seconds) {
+         ViewStream rs = new BCG.TimeBasedFilterStream(ViewStreamDatabase.instance, parent_str, time_col, TimeSpan.FromSeconds(n_seconds));
+         mySetRealSource(rs);
+      }
+   }
+}
@@ -2,7 +2,7 @@ using System;

 using BCG;

-namespace bts.SilverPine {
+namespace bts.BlueOrb {
   // Keeps adding changing rows to the stream, hence preserving the history
   // and creating a time series, for example this can be used to make bbo updates
   // into a time series of prices/qty 
@@ -1,261 +1,16 @@
-using System;
-using System.Collections.Generic;
-
-using mbts;
-using BCG;
-
 namespace bts.BlueOrb {
-   internal class MutableAppendTable : IBlueRowCollection {
-      private class MyRow : IBlueRow {
-         private IBlueRow myRow;
-         private int myID;
+   using System;
+   using System.Collections.Generic;

-         public MyRow(IBlueRow r, int id) {
-            myRow = r;
-            myID = id;
-         }
+   using mbts;
+   using BCG;

-         #region IBlueRow Members
-
-         public IBlueTableSchema schema { get { return myRow.schema; } }
-
-         public object this[int index] {
-            get {
-               if (myRow.schema[index].colName == "id")
-                  return myID;
-               else
-                  return myRow[index];
-            }
-         }
-
-         public object this[string colName] { get { return colName == "id" ? (object)myID : myRow[colName]; } }
-
-         #endregion
-
-         #region IDictionary Members
-
-         public void Add(object key, object value) { throw new Exception("Row is read only."); }
-
-         public void Clear() { throw new Exception("Row is read only."); }
-
-         public bool Contains(object key) { return myRow.Contains(key); }
-
-         public System.Collections.IDictionaryEnumerator GetEnumerator() { return new BlueRowDictionaryEnumerator(this); }
-
-         public bool IsFixedSize { get { return true; } }
-
-         public bool IsReadOnly { get { return true; } }
-
-         public System.Collections.ICollection Keys { get { return myRow.Keys; } }
-
-         public void Remove(object key) { throw new Exception("The method or operation is not implemented."); }
-
-         public System.Collections.ICollection Values {
-            get {
-               object[] vals = new object[myRow.Count];
-               CopyTo(vals, 0);
-               return vals;
-            }
-         }
-
-         public object this[object key] {
-            get { return key is string ? this[key as string] : this[(int)key]; }
-            set { throw new Exception("The method or operation is not implemented."); }
-         }
-
-         #endregion
-
-         #region ICollection Members
-
-         public void CopyTo(Array array, int index) {
-            myRow.CopyTo(array, index);
-            array.SetValue(myID, index + myRow.schema["id"].colNo);
-         }
-
-         public int Count { get { return myRow.Count; } }
-
-         public bool IsSynchronized { get { return myRow.IsSynchronized; } }
-
-         public object SyncRoot { get { return myRow.SyncRoot; } }
-
-         #endregion
-
-         #region IEnumerable Members
-
-         System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() { return new BlueRowEnumerator(this); }
-
-         #endregion
-      }
-
-      private BlueTable myRows;
-      private int myID = 0;
-      /// <summary>
-      /// maps the table no to a dictionary containing the
-      /// local IDs to a row in the final big table
-      /// </summary>
-      private Dictionary<int, Dictionary<int, IBlueRow>> myIDMap = new Dictionary<int, Dictionary<int, IBlueRow>>();
-
-      public MutableAppendTable(IBlueTableSchema schema) { myRows = new BlueTable(schema); }
-
-      public IBlueRow addRow(IBlueRow r, int tableNo) {
-         if (!myIDMap.ContainsKey(tableNo))
-            myIDMap.Add(tableNo, new Dictionary<int, IBlueRow>());
-         IBlueRow nr = new MyRow(r, myID++); // BlueDataUtility.getNewReplacedRow(r, myID++, "id");
-         myRows.addRow(nr);
-         myIDMap[tableNo].Add((int)r["id"], nr);
-         return nr;
-      }
-
-      public IBlueRow removeRow(IBlueRow r, int tableNo) {
-         IBlueRow dr = myIDMap[tableNo][(int)r["id"]];
-         myRows.deleteRow(dr);
-         myIDMap[tableNo].Remove((int)r["id"]);
-         return dr;
-      }
-
-      public IBlueRowCollection removeTable(int tableNo) {
-         if (!myIDMap.ContainsKey(tableNo))
-            return null;
-         else {
-            BlueTable rv = new BlueTable(myRows.schema);
-            foreach (IBlueRow r in myIDMap[tableNo].Values)
-               rv.addRow(r);
-            myRows.deleteRows(rv);
-            myIDMap.Remove(tableNo);
-            return rv;
-         }
-      }
-
-      public void clear() {
-         myRows.clear();
-         myIDMap.Clear();
-      }
-
-      #region IBlueRowCollection Members
-
-      public IBlueRow this[int id] { get { return myRows[id]; } }
-
-      public IBlueTableSchema schema { get { return myRows.schema; } }
-
-      public bool containsID(int id) { return myRows.containsID(id); }
-
-      #endregion
-
-      #region ICollection Members
-
-      public void CopyTo(Array array, int index) { myRows.CopyTo(array, index); }
-
-      public int Count { get { return myRows.Count; } }
-
-      public bool IsSynchronized { get { return myRows.IsSynchronized; } }
-
-      public object SyncRoot { get { return myRows.SyncRoot; } }
-
-      #endregion
-
-      #region IEnumerable Members
-
-      public System.Collections.IEnumerator GetEnumerator() { return myRows.GetEnumerator(); }
-
-      #endregion
-   }
-   /// <summary>
-   /// MutableStream can append multiple streams
-   /// as well as give the user the flexibility
-   /// to append (or remove) more streams at
-   /// runtime
-   /// </summary>
-   }
-namespace bts.BlueOrb { 
   [DataStreamUserRespresentation("UnionStream")]
-   public class MutableAppendStream : ViewStream {
-      private bool mySubscriptionActive = false;
-      private MutableAppendTable myTable;
-      private int myMovingIndex;
-      private Dictionary<int, Pair<ViewStream, ViewStream>> myStreamIndex = new Dictionary<int, Pair<ViewStream, ViewStream>>();
-      private bool _allign_col_orders = true;
-
-      public MutableAppendStream(params ViewStream[] streams)
-         : this(new List<ViewStream>(streams)) { }
-
-      public MutableAppendStream(bool allign_col_order, params ViewStream[] streams)
-         : this(allign_col_order, new List<ViewStream>(streams)) { }
-
-      public MutableAppendStream(ICollection<ViewStream> streams)
-         : this(true, streams) { }
-
-      public MutableAppendStream(bool allign_col_orders, ICollection<ViewStream> streams) {
-         _allign_col_orders = allign_col_orders;
-         foreach (ViewStream vs in streams) {
-            myMovingIndex++;
-            ViewStream pj = new ChangeTableNameStream(vs, myMovingIndex.ToString());
-            if (_allign_col_orders)
-               pj = new ProjectStream(pj, "!");
-            myStreamIndex.Add(myMovingIndex, new Pair<ViewStream, ViewStream>(vs, pj));
-         }
-      }
-
-      public void addStream(ViewStream vs) {
-         myMovingIndex++;
-         ///the change name is done to identify the callbacks, and the project to
-         ///make all the schemas col orders the same
-         ViewStream pj = new ChangeTableNameStream(vs, myMovingIndex.ToString());
-         if (_allign_col_orders)
-            pj = new ProjectStream(pj, "!");
-         myStreamIndex.Add(myMovingIndex, new Pair<ViewStream, ViewStream>(vs, pj));
-         if (mySubscriptionActive)
-            pj.onChange += new ChangeDelegate(myHandleStreamChanges);
-      }
-
-      public void removeStream(ViewStream vs) {
-         int index = -1;
-         foreach (KeyValuePair<int, Pair<ViewStream, ViewStream>> p in myStreamIndex) {
-            if (p.Value.first == vs) {
-               index = p.Key;
-               if (mySubscriptionActive) {
-                  p.Value.second.onChange -= new ChangeDelegate(myHandleStreamChanges);
-                  if (myTable != null) {
-                     IBlueRowCollection delRows = myTable.removeTable(p.Key);
-                     if (delRows != null)
-                        myFireEvent(new DataPacket(myTable, new BlueTable(myTable.schema), delRows));
-                  }
-               }
-               break;
-            }
-            if (index > 0)
-               myStreamIndex.Remove(index);
-         }
-      }
-
-      protected override void myOnFirstSubscriber() {
-         mySubscriptionActive = true;
-         foreach (KeyValuePair<int, Pair<ViewStream, ViewStream>> p in myStreamIndex)
-            p.Value.second.onChange += new ChangeDelegate(myHandleStreamChanges);
-      }
-
-      protected override void myOnLastSubscriber() {
-         mySubscriptionActive = false;
-         foreach (KeyValuePair<int, Pair<ViewStream, ViewStream>> p in myStreamIndex)
-            p.Value.second.onChange -= new ChangeDelegate(myHandleStreamChanges);
-         if (myTable != null)
-            myTable.clear();
-      }
-
-      private void myHandleStreamChanges(object newValue) {
-         DataPacket dp = newValue as DataPacket;
-         myTable = myTable ?? new MutableAppendTable(new BlueTableSchema("MutableAppend", dp.schema.columnInfo));
-         BlueTable adds = new BlueTable(myTable.schema),
-                     dels = new BlueTable(myTable.schema);
-         int index = Int32.Parse(dp.schema.tableName);
-         foreach (IBlueRow r in dp.deletedRows) {
-            IBlueRow dr = myTable.removeRow(r, index);
-            dels.addRow(dr);
-         }
-         foreach (IBlueRow r in dp.addedRows) {
-            IBlueRow nr = myTable.addRow(r, index);
-            adds.addRow(nr);
-         }
-         myFireEvent(new DataPacket(myTable, adds, dels));
+   public class UnionStream : ViewStream {
+      public UnionStream(ViewStream left_stream, ViewStream right_stream) {
+         ViewStream rs = new BCG.MutableAppendStream(left_stream, right_stream);
+         mySetRealSource(rs);
      }
   }
+
 }
@@ -2,15 +2,27 @@ namespace Bts.DataStream;

 using BCG;

+using bts.api.core;
+
+using Bts.DataStreamCore;
+
+using Microsoft.Extensions.Logging;
+
 using Spark.Core.Disposables;

 using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Linq;
 using System.Reactive.Linq;

 public static class DataStreamExtensions {
   public static ViewStream AsViewStream(this IObservable<DataPacket> source) =>
      source as ViewStream ?? new ObservableViewStream(source);

+   public static ViewStream AsViewStream<T>(this IObservable<IDataPacket<T>> source, string? name = null, ILogger? logger = null) where T : IBlueRow =>
+      source as ViewStream ?? new ObservableViewStream(source.Select(dp => new DataPacket(dp.newRows, dp.addedRows, dp.deletedRows)), name, logger);
+
   public static IObservable<DataPacket> AsObservable(this ViewStream source) {
      return source as IObservable<DataPacket> ?? Observable.Create<DataPacket>(Subscribe);

@@ -23,4 +35,100 @@ public static class DataStreamExtensions {
               $"Expected DataPacket, got {newValue?.GetType()}", nameof(newValue)));
      }
   }
+
+   public static IObservable<IDataPacket<T>> SelectDataPacket<T>(this IObservable<IDataPacket> source, Func<IBlueRow, T> selector) where T : IBlueRow =>
+      source.Select(dp => {
+         dp.get_new_added_deleted(out var newRows, out var addedRows, out var deletedRows);
+         return new DataPacket<T>(CastRows(newRows), CastRows(addedRows), CastRows(deletedRows));
+
+         IBlueRowCollection<T> CastRows(ICollection rows) =>
+            new BlueTableSelectorWrapper<T>(
+               rows as IBlueRowCollection ?? new BlueRowCollection(rows, dp.Schema), selector);
+      });
+
+   public static IObservable<TableUpdate<IBlueRow, int>> AsTables(this IObservable<IDataPacket> source) =>
+      source.AsTables<IBlueRow, int>(r => r, r => r.Id());
+
+   public static IObservable<TableUpdate<T, int>> AsTables<T>(this IObservable<IDataPacket> source, Func<IBlueRow, T> valueSelector) where T : IBlueRow =>
+      source.AsTables<T, int>(valueSelector, r => r.Id());
+
+   public static IObservable<TableUpdate<T, int>> AsTables<T>(
+      this IObservable<IDataPacket> source, Func<IBlueRow, T> valueSelector, Func<T, int> keySelector) =>
+      source.AsTables<T, int>(valueSelector, keySelector);
+
+   public static IObservable<TableUpdate<T, TKey>> AsTables<T, TKey>(
+      this IObservable<IDataPacket> source, Func<IBlueRow, T> valueSelector, Func<T, TKey> keySelector, Func<IBlueRow, TKey>? rowKeySelector = null) where TKey : IEquatable<TKey> {
+      rowKeySelector ??= r => keySelector(valueSelector(r));
+      return source.Scan(TableUpdate<T, TKey>.mk_empty(keySelector), (res, dp) => {
+         var bd = res.to_builder();
+         dp.get_new_added_deleted(out var newRows, out var addedRows, out var deletedRows);
+         foreach (IBlueRow r in addedRows) {
+            bd.add_or_update(valueSelector(r));
+         }
+         foreach (IBlueRow r in deletedRows) {
+            bd.remove(rowKeySelector(r));
+         }
+         return bd.to_table();
+      });
+   }
+
+   public static IObservable<IDataPacket<T>> AsDataPackets<T>(this IObservable<TableUpdate<T, int>> source, IBlueTableSchema schema) where T : IBlueRow =>
+      source.Scan(
+         (Table: new BlueTableImmutable<T>(schema), Dp: (IDataPacket<T>)DataPacket<T>.Empty(schema)),
+         (res, table) => {
+            var bd = res.Table.ToDataPacketBuilder();
+            foreach (var r in table.removed) {
+               bd.Remove(r.Id());
+            }
+            foreach (var r in table.added) {
+               bd.AddOrUpdate(r);
+            }
+            var resDp = bd.ToDataPacketT(out var newRows, out var addedRows, out var deletedRows);
+            return (newRows, resDp);
+         })
+         .Where(t => t.Dp.addedRows.Any() || t.Dp.deletedRows.Any())
+         .Select(t => t.Dp);
+
+   public static ViewStream AsViewStream<T>(this IObservable<TableUpdate<T, int>> source, IBlueTableSchema schema) where T : IBlueRow =>
+      source.AsDataPackets(schema).AsViewStream();
+
+   public static int ResolveIndices(this IBlueTableSchema schema, string name) {
+      var indices = ResolveIndices(schema, new[] { name });
+      return indices[0];
+   }
+
+   public static (int, int) ResolveIndices(this IBlueTableSchema schema, string name1, string name2) {
+      var indices = ResolveIndices(schema, new[] { name1, name2 });
+      return (indices[0], indices[1]);
+   }
+
+   public static (int, int, int) ResolveIndices(this IBlueTableSchema schema, string name1, string name2, string name3) {
+      var indices = ResolveIndices(schema, new[] { name1, name2, name3 });
+      return (indices[0], indices[1], indices[2]);
+   }
+
+   public static (int, int, int, int) ResolveIndices(this IBlueTableSchema schema, string name1, string name2, string name3, string name4) {
+      var indices = ResolveIndices(schema, new[] { name1, name2, name3, name4 });
+      return (indices[0], indices[1], indices[2], indices[3]);
+   }
+
+   public static int[] ResolveIndices(this IBlueTableSchema schema, string[] columnNames) {
+      var columns = schema.columnInfo;
+      var indices = new int[columnNames.Length];
+      int nResolved = 0;
+      for (int i = 0; i < columnNames.Length; i++) {
+         int origNResolved = nResolved;
+         for (int j = 0; j < columns.Length; j++) {
+            if (columns[j].colName == columnNames[i]) {
+               indices[i] = j;
+               nResolved++;
+               break;
+            }
+         }
+         if (nResolved == origNResolved) {
+            throw new ArgumentException($"No column named '{columnNames[i]}'", nameof(schema));
+         }
+      }
+      return indices;
+   }
 }
@@ -4,14 +4,21 @@ using BCG;

 using bts;

+using Microsoft.Extensions.Logging;
+
 using System;

 public class ObservableViewStream : ViewStream, IObservable<DataPacket>, IDisposable {
   readonly IObservable<DataPacket> source;
+   readonly string? name;
+   readonly ILogger? logger;
   IDisposable? subscription;

-   public ObservableViewStream(IObservable<DataPacket> source) =>
+   public ObservableViewStream(IObservable<DataPacket> source, string? name = null, ILogger? logger = null) {
      this.source = source;
+      this.name = name;
+      this.logger = logger;
+   }

   public void Dispose() {
      GC.SuppressFinalize(this);
@@ -23,7 +30,13 @@ public class ObservableViewStream : ViewStream, IObservable<DataPacket>, IDispos

   protected override void myOnFirstSubscriber() {
      Log.isnull(subscription);
-      subscription = source.Subscribe(myFireEvent);
+      subscription = source.Subscribe(myFireEvent, OnError, OnCompleted);
+
+      void OnError(Exception exception) =>
+         logger?.LogError("Error in ObservableViewStream. name: {Name}, error: {Exception}", name, exception);
+
+      void OnCompleted() =>
+         logger?.LogInformation("ObservableViewStream completed. name: {Name}", name);
   }

   protected override void myOnLastSubscriber() {
@@ -1,33 +1,51 @@
 using System;
+using System.Collections.Generic;

-namespace BCG {
+namespace BCG {   
   public class AddColumnsStream : ViewStream {
-      public static Tuple<string, Type, object> parse_exp(string exp) {
+      public static Tuple<string, Type, object> parse_single_tok(string exp) {
         string[] toks = exp.Split(':');
-         string col_name = toks[0];
-         string col_value = toks[1];
+         string col_name = toks[0].Trim();
+         string col_value = toks[1].Trim();
         Tuple<Type, object> tup = BlueDataUtility.guess_type(col_value);
         return Tuple.Create(col_name, tup.Item1, tup.Item2);
      }

+      public static List<Tuple<string, Type, object>> parse_exp(string exp) {
+         string[] toks = exp.Split(",");
+         List<Tuple<string, Type, object>> rv = new List<Tuple<string, Type, object>>();
+         foreach (string s in toks) {
+            var tup = parse_single_tok(s.Trim());
+            rv.Add(tup);
+         }
+         return rv;
+      }
+
      private Tuple<string, Type, object> _new_col;
+      private List<Tuple<string, Type, object>> _new_col_map;
      private IBlueTableSchema _out_schema;

      public AddColumnsStream(ViewStream vs, string col_def) {
-         _new_col = parse_exp(col_def);
+         _new_col_map = parse_exp(col_def);
         ViewStream rs = new MapStream(vs, (IBlueTableSchema s) => _get_schema(s), (IBlueRow r) => _map(r));
         mySetRealSource(rs);
      }

      private IBlueTableSchema _get_schema(IBlueTableSchema input_schema) {
-         if (_out_schema == null)
-            _out_schema = BlueDataUtility.add_column(input_schema, _new_col.Item1, _new_col.Item2);
+         if (_out_schema == null) {
+            _out_schema = input_schema;
+            foreach (var m in _new_col_map) {
+               _out_schema = BlueDataUtility.add_column(_out_schema, m.Item1, m.Item2);
+            }
+         }
         return _out_schema;
      }

      private IBlueRow _map(IBlueRow r) {
         MutableBlueRow rv = new MutableBlueRow(_get_schema(r.schema), r);
-         rv.setValue(_new_col.Item1, _new_col.Item3);
+         foreach (var m in _new_col_map) {
+            rv.setValue(m.Item1, m.Item3);
+         }
         return rv;
      }
   }
@@ -52,14 +52,14 @@ namespace BCG {
      }

      public TimeBasedFilter(ViewStreamDatabase vsd, string rowTimeColumnName, TimeSpan rowTimeToLive)
-      : this(vsd, rowTimeColumnName, rowTimeToLive, new TimeSpan(0, 1, 0) /* 1 minute */) {
+      : this(vsd, rowTimeColumnName, rowTimeToLive, rowTimeToLive) {
      }

-      public TimeBasedFilter(ViewStreamDatabase vsd, string rowTimeColumnName, TimeSpan rowTimeToLive, TimeSpan filterFrequency) {
+      public TimeBasedFilter(ViewStreamDatabase vsd, string rowTimeColumnName, TimeSpan rowTimeToLive, TimeSpan filter_period) {
         myVsd = vsd;
         _time_col = rowTimeColumnName;
         _ttl = rowTimeToLive;
-         myFilterFrequency = filterFrequency;
+         _filter_period = filter_period;
         myNextFilter = DateTime.MinValue;

         // add worker
@@ -92,7 +92,7 @@ namespace BCG {
            // fire event
            onFilterChange();

-            myNextFilter = now.Add(myFilterFrequency);
+            myNextFilter = now.Add(_filter_period);
         }
      }
      #endregion
@@ -108,7 +108,7 @@ namespace BCG {
      private ViewStreamDatabase myVsd;
      private string _time_col;
      private TimeSpan _ttl;
-      private TimeSpan myFilterFrequency;
+      private TimeSpan _filter_period;
      private DateTime myNextFilter;
      #endregion
   }
@@ -8,7 +8,7 @@ using bts;
 using bts.SilverPine;

 namespace BCG {
-   [DataStreamUserRespresentation("AggregateStream")]
+   //[DataStreamUserRespresentation("AggregateStream")]
   public class NonIncGroupByStream : ViewStream {
      private class DelNonIncGroupByEvaluator : INonIncGroupByEvaluator {
         Func<string[]> _groupby_cols_func;
@@ -1,31 +1,31 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
+//using System;
+//using System.Collections.Generic;
+//using System.Linq;
+//using System.Text;

-namespace BCG
-{
-   public class FullJoinStream : ViewStream
-   {
-      private ViewStream myMainStream;
-      public FullJoinStream(ViewStream v1, string joinCol1, string cmExp1,
-         ViewStream v2, string joinCol2, string cmExp2, string empty_col)
-      {
-         myMainStream = new LeftJoinStream(v1, joinCol1, cmExp1, v2, joinCol2, cmExp2);
-         ViewStream filtered = new LeftJoinStream(v2, joinCol2, cmExp2, v1, joinCol1, cmExp1);
-         filtered = new FilterStream(filtered, String.Format("{0}=null", empty_col));
-         myMainStream = new AppendStream(myMainStream, filtered);
-      }
+//namespace BCG
+//{
+//   public class FullJoinStream : ViewStream
+//   {
+//      private ViewStream myMainStream;
+//      public FullJoinStream(ViewStream v1, string joinCol1, string cmExp1,
+//         ViewStream v2, string joinCol2, string cmExp2, string empty_col)
+//      {
+//         myMainStream = new LeftJoinStream(v1, joinCol1, cmExp1, v2, joinCol2, cmExp2);
+//         ViewStream filtered = new LeftJoinStream(v2, joinCol2, cmExp2, v1, joinCol1, cmExp1);
+//         filtered = new FilterStream(filtered, String.Format("{0}=null", empty_col));
+//         myMainStream = new AppendStream(myMainStream, filtered);
+//      }

-      protected override void myOnFirstSubscriber()
-      {
-         myMainStream.onChange += myHandleMainStreamChange;
-      }
+//      protected override void myOnFirstSubscriber()
+//      {
+//         myMainStream.onChange += myHandleMainStreamChange;
+//      }

-      private void myHandleMainStreamChange(object newData)
-      {
-         IBlueRowCollection result = (newData as DataPacket).newRows;
-         myFireEvent(new DataPacket(result, result, myLastValue == null ? new BlueTable(result.schema) : (myLastValue as DataPacket).newRows));
-      }
-   }
-}
+//      private void myHandleMainStreamChange(object newData)
+//      {
+//         IBlueRowCollection result = (newData as DataPacket).newRows;
+//         myFireEvent(new DataPacket(result, result, myLastValue == null ? new BlueTable(result.schema) : (myLastValue as DataPacket).newRows));
+//      }
+//   }
+//}
@@ -12,7 +12,7 @@ using System.Collections.Generic;
 using System.Data.SQLite;


-namespace bts.SilverPine {
+namespace bts.BlueOrb{

   public static class SQLiteHelper {
      public static string location = "C:\\Users\\m_tah\\development\\data\\";
@@ -1,14 +1,12 @@
-namespace bts.SilverPine {
+namespace bts.BlueOrb {
   using System;
   using System.Data;
   using System.Data.SQLite;
   using System.Diagnostics;
   using System.Collections;
-
-
   using BCG;

-   [DataStreamUserRespresentation("SQLite")]
+   [DataStreamUserRespresentation("SQLiteStream")]
   public class SQLiteStream : ViewStream {
      private readonly string _connection_string, _query;
      private Stopwatch _stopwatch = new Stopwatch();
@@ -70,4 +68,5 @@ namespace bts.SilverPine {

   }

-}
+}
+
@@ -1,5 +1,6 @@
 using System;

+using BCG;
 using BCG.Utilities.General;

 using bts;
@@ -87,3 +88,17 @@ namespace BCG {
      }
   }
 }
+
+//namespace bts.BlueOrb {
+//   [DataStreamUserRespresentation("MillisecondStream")]
+//   public class MillisecondStream : ViewStream {
+//      public MillisecondStream(ViewStreamDatabase vsd, int interval_msecs, int size) {
+//         ViewStream rs = new BCG.MillisecondStream(vsd, interval_msecs, size);
+//         mySetRealSource(rs);
+//      }
+
+//      public MillisecondStream(int interval_msecs, int size)
+//         : this(interval_msecs, size) {
+//      }
+//   }
+//}
@@ -1,124 +1,124 @@
-namespace bts.SilverPine {
-   using BCG;
+//namespace bts.SilverPine {
+//   using BCG;

-   using bts;
-   using bts.collections;
+//   using bts;
+//   //using bts.collections;

-   using System;
-   using System.CodeDom;
-   using System.Collections.Generic;
-   using System.Linq;
+//   using System;
+//   using System.CodeDom;
+//   using System.Collections.Generic;
+//   using System.Linq;

-   // Given 2 filters, and 2 columns, finds the different in the value of the 2 columns from the 2 rows
-   // so:
-   // f1(base_stream.rows)[col1] - f1(base_stream.rows)[col2]
-   [DataStreamUserRespresentation("ColDiff")]
-   public class ColumnDifferenceStream : ViewStream {
-      private readonly ViewStream _base_stream;
-      private readonly FilterStream.OnFilter _f1, _f2;
-      private readonly string _c1, _c2;
+//   // Given 2 filters, and 2 columns, finds the different in the value of the 2 columns from the 2 rows
+//   // so:
+//   // f1(base_stream.rows)[col1] - f1(base_stream.rows)[col2]
+//   [DataStreamUserRespresentation("ColDiff")]
+//   public class ColumnDifferenceStream : ViewStream {
+//      private readonly ViewStream _base_stream;
+//      private readonly FilterStream.OnFilter _f1, _f2;
+//      private readonly string _c1, _c2;

-      public ColumnDifferenceStream(ViewStream base_stream, FilterStream.OnFilter f1, FilterStream.OnFilter f2, string col1, string col2) {
-         _base_stream = base_stream;
-         _f1 = f1;
-         _f2 = f2;
-         _c1 = col1;
-         _c2 = col2;
-      }
+//      public ColumnDifferenceStream(ViewStream base_stream, FilterStream.OnFilter f1, FilterStream.OnFilter f2, string col1, string col2) {
+//         _base_stream = base_stream;
+//         _f1 = f1;
+//         _f2 = f2;
+//         _c1 = col1;
+//         _c2 = col2;
+//      }

-      public ColumnDifferenceStream(ViewStream base_stream, string f1, string f2, string col1, string col2) {
-         FilterStream.OnFilter f1_del = r => {
-            BooleanExpressionFilter rv = new BooleanExpressionFilter(f1);
-            return rv.filter(r);
-         };
-         FilterStream.OnFilter f2_del = r => {
-            BooleanExpressionFilter rv = new BooleanExpressionFilter(f2);
-            return rv.filter(r);
-         };
-         ViewStream rs = new ColumnDifferenceStream(base_stream, f1_del, f2_del, col1, col2);
-         mySetRealSource(rs);
-      }
+//      public ColumnDifferenceStream(ViewStream base_stream, string f1, string f2, string col1, string col2) {
+//         FilterStream.OnFilter f1_del = r => {
+//            BooleanExpressionFilter rv = new BooleanExpressionFilter(f1);
+//            return rv.filter(r);
+//         };
+//         FilterStream.OnFilter f2_del = r => {
+//            BooleanExpressionFilter rv = new BooleanExpressionFilter(f2);
+//            return rv.filter(r);
+//         };
+//         ViewStream rs = new ColumnDifferenceStream(base_stream, f1_del, f2_del, col1, col2);
+//         mySetRealSource(rs);
+//      }

-      // for each row, does a row[new_col_name] = row[col1] - rows[row_ffset][col2]
-      // we should generlize this for any math operation
-      public ColumnDifferenceStream(ViewStream parent_stream, string sort_col_name, int row_offset, string col1, string col2, string diff_col_name, double optional_perc_diff_filter = 0.0) {
-         // use GlobalMapStream
-         string diff_col_perc_name = $"{diff_col_name}_percentage";
-         System.Func<System.Collections.Generic.List<BCG.IBlueRow>, uint, BCG.IBlueRow> g_map_func = (rows, idx) => {
-            int minuend_idx = Convert.ToInt32(idx);
-            var inp_schema = rows[0].schema;
-            var out_schema = inp_schema.add_column(TypeCode.Double, diff_col_name);
-            out_schema = out_schema.add_column(TypeCode.Double, diff_col_perc_name);
-            // minuend - subtrahend
-            IBlueRow minuend_row = rows.ElementAt(minuend_idx);
-            MutableBlueRow rv = new MutableBlueRow(out_schema, minuend_row);
-            int subtrahend_index = minuend_idx + row_offset;
-            double diff_val = Double.NaN, diff_val_perc = Double.NaN;
-            if (subtrahend_index >= 0) {
-               try {
-                  IBlueRow subtrahend_row = rows.ElementAt(subtrahend_index);
-                  diff_val = minuend_row.get<double>(col1) - subtrahend_row.get<double>(col2);
-                  diff_val_perc = 100.0 * diff_val / subtrahend_row.get<double>(col1);
-               } catch (Exception ex) {
-                  Log.assert(false, ex.Message);
-               }
-            }
-            rv.setValue(diff_col_name, diff_val);
-            rv.setValue(diff_col_perc_name, diff_val_perc);
-            return rv;
-         };
-         ViewStream rs = new GlobalMapStream(parent_stream, sort_col_name, g_map_func);
-         if (optional_perc_diff_filter != 0) {
-            rs = new FilterStream(rs, r => {
-               double diff_val_perc = r.get<double>(diff_col_perc_name);
-               double abs_diff_val_perc = Math.Abs(diff_val_perc);
-               return abs_diff_val_perc >= optional_perc_diff_filter;
-            });
-         }
-         mySetRealSource(rs);
-      }
+//      // for each row, does a row[new_col_name] = row[col1] - rows[row_ffset][col2]
+//      // we should generlize this for any math operation
+//      public ColumnDifferenceStream(ViewStream parent_stream, string sort_col_name, int row_offset, string col1, string col2, string diff_col_name, double optional_perc_diff_filter = 0.0) {
+//         // use GlobalMapStream
+//         string diff_col_perc_name = $"{diff_col_name}_percentage";
+//         System.Func<System.Collections.Generic.List<BCG.IBlueRow>, uint, BCG.IBlueRow> g_map_func = (rows, idx) => {
+//            int minuend_idx = Convert.ToInt32(idx);
+//            var inp_schema = rows[0].schema;
+//            var out_schema = inp_schema.add_column(TypeCode.Double, diff_col_name);
+//            out_schema = out_schema.add_column(TypeCode.Double, diff_col_perc_name);
+//            // minuend - subtrahend
+//            IBlueRow minuend_row = rows.ElementAt(minuend_idx);
+//            MutableBlueRow rv = new MutableBlueRow(out_schema, minuend_row);
+//            int subtrahend_index = minuend_idx + row_offset;
+//            double diff_val = Double.NaN, diff_val_perc = Double.NaN;
+//            if (subtrahend_index >= 0) {
+//               try {
+//                  IBlueRow subtrahend_row = rows.ElementAt(subtrahend_index);
+//                  diff_val = minuend_row.get<double>(col1) - subtrahend_row.get<double>(col2);
+//                  diff_val_perc = 100.0 * diff_val / subtrahend_row.get<double>(col1);
+//               } catch (Exception ex) {
+//                  Log.assert(false, ex.Message);
+//               }
+//            }
+//            rv.setValue(diff_col_name, diff_val);
+//            rv.setValue(diff_col_perc_name, diff_val_perc);
+//            return rv;
+//         };
+//         ViewStream rs = new GlobalMapStream(parent_stream, sort_col_name, g_map_func);
+//         if (optional_perc_diff_filter != 0) {
+//            rs = new FilterStream(rs, r => {
+//               double diff_val_perc = r.get<double>(diff_col_perc_name);
+//               double abs_diff_val_perc = Math.Abs(diff_val_perc);
+//               return abs_diff_val_perc >= optional_perc_diff_filter;
+//            });
+//         }
+//         mySetRealSource(rs);
+//      }

-      protected override void myOnFirstSubscriber() {
-         _base_stream.onChange += _handle_base_stream_update;
-      }
+//      protected override void myOnFirstSubscriber() {
+//         _base_stream.onChange += _handle_base_stream_update;
+//      }

-      protected override void myOnLastSubscriber() {
-         _base_stream.onChange -= _handle_base_stream_update;
-      }
+//      protected override void myOnLastSubscriber() {
+//         _base_stream.onChange -= _handle_base_stream_update;
+//      }

-      private void _handle_base_stream_update(object dp_obj) {
-         // could have been done with filter and cartesian prod, and then math
-         DataPacket dp = dp_obj as DataPacket;
-         List<IBlueRow> rows = dp.newRows.to_list();
-         var f1_rows = rows.Where(r => _f1(r));
-         //Log.assert(f1_rows.Count() == 1);
-         var f2_rows = rows.Where(r => _f2(r));
-         //Log.assert(f2_rows.Count() == 1);
-         if (f1_rows != null && f1_rows.Count() > 0 && f2_rows != null && f2_rows.Count() > 0) {
-            IBlueRow f1_r = f1_rows.ElementAt(0);
-            IBlueRow f2_r = f2_rows.ElementAt(0);
-            double diff = f1_r.get<double>(_c1) - f2_r.get<double>(_c2);
-            IBlueTableSchema out_schema = f1_r.schema.add_columns(Tuple.Create(typeof(double), "diff"));
-            BlueTable out_table = new BlueTable(out_schema);
-            MutableBlueRow out_row = new MutableBlueRow(out_schema, f1_r);
-            out_row.setValue("diff", diff);
-            out_table.addRow(out_row);
-            DataPacket dp_out = BlueDataUtility.build_dp(out_table, myLastValue as DataPacket);
-            myFireEvent(dp_out);
-         } else {
-            IBlueRow f1_r = rows.ElementAt(0);
-            IBlueRow f2_r = rows.ElementAt(0);
-            double diff = Double.NaN;
-            IBlueTableSchema out_schema = f1_r.schema.add_columns(Tuple.Create(typeof(double), "diff"));
-            BlueTable out_table = new BlueTable(out_schema);
-            MutableBlueRow out_row = new MutableBlueRow(out_schema, f1_r);
-            out_row.setValue("diff", diff);
-            out_table.addRow(out_row);
-            DataPacket dp_out = BlueDataUtility.build_dp(out_table, myLastValue as DataPacket);
-            myFireEvent(dp_out);
-         }
+//      private void _handle_base_stream_update(object dp_obj) {
+//         // could have been done with filter and cartesian prod, and then math
+//         DataPacket dp = dp_obj as DataPacket;
+//         List<IBlueRow> rows = dp.newRows.to_list();
+//         var f1_rows = rows.Where(r => _f1(r));
+//         //Log.assert(f1_rows.Count() == 1);
+//         var f2_rows = rows.Where(r => _f2(r));
+//         //Log.assert(f2_rows.Count() == 1);
+//         if (f1_rows != null && f1_rows.Count() > 0 && f2_rows != null && f2_rows.Count() > 0) {
+//            IBlueRow f1_r = f1_rows.ElementAt(0);
+//            IBlueRow f2_r = f2_rows.ElementAt(0);
+//            double diff = f1_r.get<double>(_c1) - f2_r.get<double>(_c2);
+//            IBlueTableSchema out_schema = f1_r.schema.add_columns(Tuple.Create(typeof(double), "diff"));
+//            BlueTable out_table = new BlueTable(out_schema);
+//            MutableBlueRow out_row = new MutableBlueRow(out_schema, f1_r);
+//            out_row.setValue("diff", diff);
+//            out_table.addRow(out_row);
+//            DataPacket dp_out = BlueDataUtility.build_dp(out_table, myLastValue as DataPacket);
+//            myFireEvent(dp_out);
+//         } else {
+//            IBlueRow f1_r = rows.ElementAt(0);
+//            IBlueRow f2_r = rows.ElementAt(0);
+//            double diff = Double.NaN;
+//            IBlueTableSchema out_schema = f1_r.schema.add_columns(Tuple.Create(typeof(double), "diff"));
+//            BlueTable out_table = new BlueTable(out_schema);
+//            MutableBlueRow out_row = new MutableBlueRow(out_schema, f1_r);
+//            out_row.setValue("diff", diff);
+//            out_table.addRow(out_row);
+//            DataPacket dp_out = BlueDataUtility.build_dp(out_table, myLastValue as DataPacket);
+//            myFireEvent(dp_out);
+//         }

-      }
-   }
+//      }
+//   }

-}
+//}
@@ -1,118 +1,118 @@
-using System;
-using System.Data;
-using System.Collections;
-using BCG.Utilities;
-using BCG.Utilities.General;
-namespace BCG
-{
-   /// <summary>
-   /// AppendStream appends 2 tables with each other, with a generated id having no bearing to the original id.
-   /// The column ordering the 2 tables need not be the same
-   /// </summary>
-   public class AppendStream : UnionStream
-   {
-      private bool myUseNewAppend = DataStreamLoadFileConfig.instance.useNewAppend;
-      public AppendStream(ViewStream v1, ViewStream v2)
-      {
-         if (myUseNewAppend)
-            mySetRealSource(new MutableAppendStream(v1, v2));
-         else
-         {
-            myLeftVs = v1;
-            myRightVs = v2;
-         }
-      }
+//using System;
+//using System.Data;
+//using System.Collections;
+//using BCG.Utilities;
+//using BCG.Utilities.General;
+//namespace BCG
+//{
+//   /// <summary>
+//   /// AppendStream appends 2 tables with each other, with a generated id having no bearing to the original id.
+//   /// The column ordering the 2 tables need not be the same
+//   /// </summary>
+//   public class AppendStream : UnionStream
+//   {
+//      private bool myUseNewAppend = DataStreamLoadFileConfig.instance.useNewAppend;
+//      public AppendStream(ViewStream v1, ViewStream v2)
+//      {
+//         if (myUseNewAppend)
+//            mySetRealSource(new MutableAppendStream(v1, v2));
+//         else
+//         {
+//            myLeftVs = v1;
+//            myRightVs = v2;
+//         }
+//      }

-      public AppendStream (params ViewStream[] vss)
-      {
+//      public AppendStream (params ViewStream[] vss)
+//      {

-         if (myUseNewAppend)
-            mySetRealSource(new MutableAppendStream(vss));
-         else
-         {
-            if (vss.Length == 1)
-               mySetRealSource(vss[0]);
-            else if (vss.Length > 1) 
-            {
-               AppendStream rv = null;
-               foreach (ViewStream s in vss)
-                  rv = rv == null ? new AppendStream(new ViewStream[] { s }) : new AppendStream(rv, s);
-               mySetRealSource(rv);
-            }
-            else
-               //don't do anything if they didnt' supply any viewstreams
-               mySetRealSource(null);
-         }
+//         if (myUseNewAppend)
+//            mySetRealSource(new MutableAppendStream(vss));
+//         else
+//         {
+//            if (vss.Length == 1)
+//               mySetRealSource(vss[0]);
+//            else if (vss.Length > 1) 
+//            {
+//               AppendStream rv = null;
+//               foreach (ViewStream s in vss)
+//                  rv = rv == null ? new AppendStream(new ViewStream[] { s }) : new AppendStream(rv, s);
+//               mySetRealSource(rv);
+//            }
+//            else
+//               //don't do anything if they didnt' supply any viewstreams
+//               mySetRealSource(null);
+//         }
          
-      }
+//      }

-      public override void add (IBlueRowCollection rows, bool isLeft) 
-      {
-         IDictionary index = isLeft ? myLeftIndex : myRightIndex;
-         int[] indexToIndexMap = isLeft ? myLeftIndexToIndexMap : myRightIndexToIndexMap;
-         foreach (IBlueRow row in rows)
-         {
-            UnionRow newRow = new UnionRow (myLastFiredPacket.newRows.schema, row, 
-               myIdCounter ++, myIdIndex, indexToIndexMap);
-            ((BlueTable)myLastFiredPacket.newRows).addRow (newRow);
-            index[row["id"]] =  newRow;
-            ((BlueTable)(myLastFiredPacket.addedRows)).addRow (newRow);
-         }
-      }
+//      public override void add (IBlueRowCollection rows, bool isLeft) 
+//      {
+//         IDictionary index = isLeft ? myLeftIndex : myRightIndex;
+//         int[] indexToIndexMap = isLeft ? myLeftIndexToIndexMap : myRightIndexToIndexMap;
+//         foreach (IBlueRow row in rows)
+//         {
+//            UnionRow newRow = new UnionRow (myLastFiredPacket.newRows.schema, row, 
+//               myIdCounter ++, myIdIndex, indexToIndexMap);
+//            ((BlueTable)myLastFiredPacket.newRows).addRow (newRow);
+//            index[row["id"]] =  newRow;
+//            ((BlueTable)(myLastFiredPacket.addedRows)).addRow (newRow);
+//         }
+//      }

-      public override void delete (IBlueRowCollection rows, bool isLeft) 
-      {
-         IDictionary index = isLeft ? myLeftIndex : myRightIndex;
-         foreach (IBlueRow axedRow in rows)
-         {
-            int rowId = (int)axedRow["id"];
-            IBlueRow row = (IBlueRow)index[rowId];
-            index.Remove (rowId);
-            ((BlueTable)myLastFiredPacket.newRows).deleteRow (row);
-            ((BlueTable)(myLastFiredPacket.deletedRows)).addRow (row);
-         }
-      }
+//      public override void delete (IBlueRowCollection rows, bool isLeft) 
+//      {
+//         IDictionary index = isLeft ? myLeftIndex : myRightIndex;
+//         foreach (IBlueRow axedRow in rows)
+//         {
+//            int rowId = (int)axedRow["id"];
+//            IBlueRow row = (IBlueRow)index[rowId];
+//            index.Remove (rowId);
+//            ((BlueTable)myLastFiredPacket.newRows).deleteRow (row);
+//            ((BlueTable)(myLastFiredPacket.deletedRows)).addRow (row);
+//         }
+//      }

-      public override void mergeChanges (IBlueRowCollection dels, IBlueRowCollection adds, bool isLeft)
-      {
-         IDictionary temp = new Hashtable ();
-         IDictionary index = isLeft ? myLeftIndex : myRightIndex;
-         int[] indexToIndexMap = isLeft ? myLeftIndexToIndexMap : myRightIndexToIndexMap;
-         IBlueRow corrRow;
-         foreach (IBlueRow dRow in dels)
-         {
-            corrRow = index[dRow["id"]] as IBlueRow;
-            (myLastFiredPacket.newRows as BlueTable).deleteRow (corrRow);
-            (myLastFiredPacket.deletedRows as BlueTable).addRow (corrRow);
-            index.Remove (dRow["id"]);
-            if (adds.containsID ((int)dRow["id"]))
-            {
-               IBlueRow r = adds[(int)dRow["id"]];
-               corrRow = new UnionRow (myLastFiredPacket.schema, r, (int)corrRow["id"], myIdIndex, indexToIndexMap);
-               (myLastFiredPacket.newRows as BlueTable).addRow (corrRow);
-               (myLastFiredPacket.addedRows as BlueTable).addRow (corrRow);
-               index.Add (r["id"], corrRow);
-               temp.Add (r["id"], true);
-            }
-         }
-         foreach (IBlueRow aRow in adds)
-         {
-            if (!temp.Contains (aRow["id"]))
-            {
-               corrRow = new UnionRow (myLastFiredPacket.schema, aRow, myIdCounter++, myIdIndex, indexToIndexMap);
-               (myLastFiredPacket.newRows as BlueTable).addRow (corrRow);
-               (myLastFiredPacket.addedRows as BlueTable).addRow (corrRow);
-               index.Add (aRow["id"], corrRow);
-            }
-         }
-      }
-   }
+//      public override void mergeChanges (IBlueRowCollection dels, IBlueRowCollection adds, bool isLeft)
+//      {
+//         IDictionary temp = new Hashtable ();
+//         IDictionary index = isLeft ? myLeftIndex : myRightIndex;
+//         int[] indexToIndexMap = isLeft ? myLeftIndexToIndexMap : myRightIndexToIndexMap;
+//         IBlueRow corrRow;
+//         foreach (IBlueRow dRow in dels)
+//         {
+//            corrRow = index[dRow["id"]] as IBlueRow;
+//            (myLastFiredPacket.newRows as BlueTable).deleteRow (corrRow);
+//            (myLastFiredPacket.deletedRows as BlueTable).addRow (corrRow);
+//            index.Remove (dRow["id"]);
+//            if (adds.containsID ((int)dRow["id"]))
+//            {
+//               IBlueRow r = adds[(int)dRow["id"]];
+//               corrRow = new UnionRow (myLastFiredPacket.schema, r, (int)corrRow["id"], myIdIndex, indexToIndexMap);
+//               (myLastFiredPacket.newRows as BlueTable).addRow (corrRow);
+//               (myLastFiredPacket.addedRows as BlueTable).addRow (corrRow);
+//               index.Add (r["id"], corrRow);
+//               temp.Add (r["id"], true);
+//            }
+//         }
+//         foreach (IBlueRow aRow in adds)
+//         {
+//            if (!temp.Contains (aRow["id"]))
+//            {
+//               corrRow = new UnionRow (myLastFiredPacket.schema, aRow, myIdCounter++, myIdIndex, indexToIndexMap);
+//               (myLastFiredPacket.newRows as BlueTable).addRow (corrRow);
+//               (myLastFiredPacket.addedRows as BlueTable).addRow (corrRow);
+//               index.Add (aRow["id"], corrRow);
+//            }
+//         }
+//      }
+//   }

-   public class MergeStream : ViewStream
-   {
-      public MergeStream (ViewStream v1, ViewStream v2, string[] mergeCols)
-      {
-         mySetRealSource (new DistinctRowStream (new AppendStream (v1, v2), mergeCols));
-      }
-   }
-}
+//   public class MergeStream : ViewStream
+//   {
+//      public MergeStream (ViewStream v1, ViewStream v2, string[] mergeCols)
+//      {
+//         mySetRealSource (new DistinctRowStream (new AppendStream (v1, v2), mergeCols));
+//      }
+//   }
+//}
@@ -2,170 +2,171 @@ using System;
 using System.Collections.Generic;

 using mbts;
+//using BCG;

 namespace BCG {
-   internal class MutableAppendTable : IBlueRowCollection {
-      private class MyRow : IBlueRow {
-         private IBlueRow myRow;
-         private int myID;
-
-         public MyRow(IBlueRow r, int id) {
-            myRow = r;
-            myID = id;
-         }
-
-         #region IBlueRow Members
-
-         public IBlueTableSchema schema { get { return myRow.schema; } }
-
-         public object this[int index] {
-            get {
-               if (myRow.schema[index].colName == "id")
-                  return myID;
-               else
-                  return myRow[index];
-            }
-         }
-
-         public object this[string colName] { get { return colName == "id" ? (object)myID : myRow[colName]; } }
-
-         #endregion
-
-         #region IDictionary Members
-
-         public void Add(object key, object value) { throw new Exception("Row is read only."); }
-
-         public void Clear() { throw new Exception("Row is read only."); }
-
-         public bool Contains(object key) { return myRow.Contains(key); }
-
-         public System.Collections.IDictionaryEnumerator GetEnumerator() { return new BlueRowDictionaryEnumerator(this); }
-
-         public bool IsFixedSize { get { return true; } }
-
-         public bool IsReadOnly { get { return true; } }
-
-         public System.Collections.ICollection Keys { get { return myRow.Keys; } }
-
-         public void Remove(object key) { throw new Exception("The method or operation is not implemented."); }
-
-         public System.Collections.ICollection Values {
-            get {
-               object[] vals = new object[myRow.Count];
-               CopyTo(vals, 0);
-               return vals;
-            }
-         }
-
-         public object this[object key] {
-            get { return key is string ? this[key as string] : this[(int)key]; }
-            set { throw new Exception("The method or operation is not implemented."); }
-         }
-
-         #endregion
-
-         #region ICollection Members
-
-         public void CopyTo(Array array, int index) {
-            myRow.CopyTo(array, index);
-            array.SetValue(myID, index + myRow.schema["id"].colNo);
-         }
-
-         public int Count { get { return myRow.Count; } }
-
-         public bool IsSynchronized { get { return myRow.IsSynchronized; } }
-
-         public object SyncRoot { get { return myRow.SyncRoot; } }
-
-         #endregion
-
-         #region IEnumerable Members
-
-         System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() { return new BlueRowEnumerator(this); }
-
-         #endregion
-      }
-
-      private BlueTable myRows;
-      private int myID = 0;
-      /// <summary>
-      /// maps the table no to a dictionary containing the
-      /// local IDs to a row in the final big table
-      /// </summary>
-      private Dictionary<int, Dictionary<int, IBlueRow>> myIDMap = new Dictionary<int, Dictionary<int, IBlueRow>>();
-
-      public MutableAppendTable(IBlueTableSchema schema) { myRows = new BlueTable(schema); }
-
-      public IBlueRow addRow(IBlueRow r, int tableNo) {
-         if (!myIDMap.ContainsKey(tableNo))
-            myIDMap.Add(tableNo, new Dictionary<int, IBlueRow>());
-         IBlueRow nr = new MyRow(r, myID++); // BlueDataUtility.getNewReplacedRow(r, myID++, "id");
-         myRows.addRow(nr);
-         myIDMap[tableNo].Add((int)r["id"], nr);
-         return nr;
-      }
-
-      public IBlueRow removeRow(IBlueRow r, int tableNo) {
-         IBlueRow dr = myIDMap[tableNo][(int)r["id"]];
-         myRows.deleteRow(dr);
-         myIDMap[tableNo].Remove((int)r["id"]);
-         return dr;
-      }
-
-      public IBlueRowCollection removeTable(int tableNo) {
-         if (!myIDMap.ContainsKey(tableNo))
-            return null;
-         else {
-            BlueTable rv = new BlueTable(myRows.schema);
-            foreach (IBlueRow r in myIDMap[tableNo].Values)
-               rv.addRow(r);
-            myRows.deleteRows(rv);
-            myIDMap.Remove(tableNo);
-            return rv;
-         }
-      }
-
-      public void clear() {
-         myRows.clear();
-         myIDMap.Clear();
-      }
-
-      #region IBlueRowCollection Members
-
-      public IBlueRow this[int id] { get { return myRows[id]; } }
-
-      public IBlueTableSchema schema { get { return myRows.schema; } }
-
-      public bool containsID(int id) { return myRows.containsID(id); }
-
-      #endregion
-
-      #region ICollection Members
-
-      public void CopyTo(Array array, int index) { myRows.CopyTo(array, index); }
-
-      public int Count { get { return myRows.Count; } }
-
-      public bool IsSynchronized { get { return myRows.IsSynchronized; } }
-
-      public object SyncRoot { get { return myRows.SyncRoot; } }
-
-      #endregion
-
-      #region IEnumerable Members
-
-      public System.Collections.IEnumerator GetEnumerator() { return myRows.GetEnumerator(); }
-
-      #endregion
-   }
-   /// <summary>
+    /// <summary>
   /// MutableStream can append multiple streams
   /// as well as give the user the flexibility
   /// to append (or remove) more streams at
   /// runtime
   /// </summary>
-   [DataStreamUserRespresentation("UnionStream")]
+   //[DataStreamUserRespresentation("UnionStream")]
   public class MutableAppendStream : ViewStream {
+      internal class MutableAppendTable : IBlueRowCollection {
+         private class MyRow : IBlueRow {
+            private IBlueRow myRow;
+            private int myID;
+
+            public MyRow(IBlueRow r, int id) {
+               myRow = r;
+               myID = id;
+            }
+
+            #region IBlueRow Members
+
+            public IBlueTableSchema schema { get { return myRow.schema; } }
+
+            public object this[int index] {
+               get {
+                  if (myRow.schema[index].colName == "id")
+                     return myID;
+                  else
+                     return myRow[index];
+               }
+            }
+
+            public object this[string colName] { get { return colName == "id" ? (object)myID : myRow[colName]; } }
+
+            #endregion
+
+            #region IDictionary Members
+
+            public void Add(object key, object value) { throw new Exception("Row is read only."); }
+
+            public void Clear() { throw new Exception("Row is read only."); }
+
+            public bool Contains(object key) { return myRow.Contains(key); }
+
+            public System.Collections.IDictionaryEnumerator GetEnumerator() { return new BlueRowDictionaryEnumerator(this); }
+
+            public bool IsFixedSize { get { return true; } }
+
+            public bool IsReadOnly { get { return true; } }
+
+            public System.Collections.ICollection Keys { get { return myRow.Keys; } }
+
+            public void Remove(object key) { throw new Exception("The method or operation is not implemented."); }
+
+            public System.Collections.ICollection Values {
+               get {
+                  object[] vals = new object[myRow.Count];
+                  CopyTo(vals, 0);
+                  return vals;
+               }
+            }
+
+            public object this[object key] {
+               get { return key is string ? this[key as string] : this[(int)key]; }
+               set { throw new Exception("The method or operation is not implemented."); }
+            }
+
+            #endregion
+
+            #region ICollection Members
+
+            public void CopyTo(Array array, int index) {
+               myRow.CopyTo(array, index);
+               array.SetValue(myID, index + myRow.schema["id"].colNo);
+            }
+
+            public int Count { get { return myRow.Count; } }
+
+            public bool IsSynchronized { get { return myRow.IsSynchronized; } }
+
+            public object SyncRoot { get { return myRow.SyncRoot; } }
+
+            #endregion
+
+            #region IEnumerable Members
+
+            System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() { return new BlueRowEnumerator(this); }
+
+            #endregion
+         }
+
+         private BlueTable myRows;
+         private int myID = 0;
+         /// <summary>
+         /// maps the table no to a dictionary containing the
+         /// local IDs to a row in the final big table
+         /// </summary>
+         private Dictionary<int, Dictionary<int, IBlueRow>> myIDMap = new Dictionary<int, Dictionary<int, IBlueRow>>();
+
+         public MutableAppendTable(IBlueTableSchema schema) { myRows = new BlueTable(schema); }
+
+         public IBlueRow addRow(IBlueRow r, int tableNo) {
+            if (!myIDMap.ContainsKey(tableNo))
+               myIDMap.Add(tableNo, new Dictionary<int, IBlueRow>());
+            IBlueRow nr = new MyRow(r, myID++); // BlueDataUtility.getNewReplacedRow(r, myID++, "id");
+            myRows.addRow(nr);
+            myIDMap[tableNo].Add((int)r["id"], nr);
+            return nr;
+         }
+
+         public IBlueRow removeRow(IBlueRow r, int tableNo) {
+            IBlueRow dr = myIDMap[tableNo][(int)r["id"]];
+            myRows.deleteRow(dr);
+            myIDMap[tableNo].Remove((int)r["id"]);
+            return dr;
+         }
+
+         public IBlueRowCollection removeTable(int tableNo) {
+            if (!myIDMap.ContainsKey(tableNo))
+               return null;
+            else {
+               BlueTable rv = new BlueTable(myRows.schema);
+               foreach (IBlueRow r in myIDMap[tableNo].Values)
+                  rv.addRow(r);
+               myRows.deleteRows(rv);
+               myIDMap.Remove(tableNo);
+               return rv;
+            }
+         }
+
+         public void clear() {
+            myRows.clear();
+            myIDMap.Clear();
+         }
+
+         #region IBlueRowCollection Members
+
+         public IBlueRow this[int id] { get { return myRows[id]; } }
+
+         public IBlueTableSchema schema { get { return myRows.schema; } }
+
+         public bool containsID(int id) { return myRows.containsID(id); }
+
+         #endregion
+
+         #region ICollection Members
+
+         public void CopyTo(Array array, int index) { myRows.CopyTo(array, index); }
+
+         public int Count { get { return myRows.Count; } }
+
+         public bool IsSynchronized { get { return myRows.IsSynchronized; } }
+
+         public object SyncRoot { get { return myRows.SyncRoot; } }
+
+         #endregion
+
+         #region IEnumerable Members
+
+         public System.Collections.IEnumerator GetEnumerator() { return myRows.GetEnumerator(); }
+
+         #endregion
+      }
      private bool mySubscriptionActive = false;
      private MutableAppendTable myTable;
      private int myMovingIndex;
@@ -173,7 +174,8 @@ namespace BCG {
      private bool _allign_col_orders = true;

      public MutableAppendStream(params ViewStream[] streams)
-         : this(new List<ViewStream>(streams)) { }
+         : this(new List<ViewStream>(streams)) {
+      }

      public MutableAppendStream(bool allign_col_order, params ViewStream[] streams)
         : this(allign_col_order, new List<ViewStream>(streams)) { }
@@ -256,3 +258,13 @@ namespace BCG {
      }
   }
 }
+
+
+//namespace bts.BlueOrb {
+//   public class UnionStream : ViewStream {
+//      public UnionStream(ViewStream left_stream, ViewStream right_stream) {
+//         ViewStream rs = new BCG.MutableAppendStream(left_stream, right_stream);
+//         mySetRealSource(rs);
+//      }
+//   }
+//}
@@ -1,63 +1,63 @@
-using System;
+//using System;

-namespace BCG
-{
-   /// <summary>
-   /// Respresents a Union Operand. Hooks up to the Stream delegate, and alerts the UnionStream
-   /// on receiving updates
-   /// </summary>
-   public class UnionOperand 
-   {
-      private bool myIsLeft;
-      private DataPacket myLastArrivedPacket;
-      private UnionStream myMaster;
+//namespace BCG
+//{
+//   /// <summary>
+//   /// Respresents a Union Operand. Hooks up to the Stream delegate, and alerts the UnionStream
+//   /// on receiving updates
+//   /// </summary>
+//   public class UnionOperand 
+//   {
+//      private bool myIsLeft;
+//      private DataPacket myLastArrivedPacket;
+//      private UnionStream myMaster;
       
-      public UnionOperand (bool isLeft, UnionStream master)
-      {
-         myIsLeft = isLeft;
-         myMaster = master;
-      }
+//      public UnionOperand (bool isLeft, UnionStream master)
+//      {
+//         myIsLeft = isLeft;
+//         myMaster = master;
+//      }

-      internal void myChangeHandler (object o)
-      {
-         if (myLastArrivedPacket != null)
-            mySubsequentChangeHandler ((DataPacket)o);
-         else
-         {
-            myLastArrivedPacket = (DataPacket)o;
-            myFirstChangeHandler ((DataPacket)o);
-         }
-      }
+//      internal void myChangeHandler (object o)
+//      {
+//         if (myLastArrivedPacket != null)
+//            mySubsequentChangeHandler ((DataPacket)o);
+//         else
+//         {
+//            myLastArrivedPacket = (DataPacket)o;
+//            myFirstChangeHandler ((DataPacket)o);
+//         }
+//      }

-      private void myFirstChangeHandler(DataPacket dp)
-      {
-         myMaster.startSession ();
-         myMaster.add (dp.newRows, myIsLeft);
-         myMaster.endSession ();
-      }
+//      private void myFirstChangeHandler(DataPacket dp)
+//      {
+//         myMaster.startSession ();
+//         myMaster.add (dp.newRows, myIsLeft);
+//         myMaster.endSession ();
+//      }

-      private void mySubsequentChangeHandler (DataPacket dp)
-      {
-         myMaster.startSession ();  
-         //myMaster.delete (dp.deletedRows, myIsLeft);
-         //myMaster.add (dp.addedRows, myIsLeft);
-         myMaster.mergeChanges (dp.deletedRows, dp.addedRows, myIsLeft);
-         myMaster.endSession ();
-      }
+//      private void mySubsequentChangeHandler (DataPacket dp)
+//      {
+//         myMaster.startSession ();  
+//         //myMaster.delete (dp.deletedRows, myIsLeft);
+//         //myMaster.add (dp.addedRows, myIsLeft);
+//         myMaster.mergeChanges (dp.deletedRows, dp.addedRows, myIsLeft);
+//         myMaster.endSession ();
+//      }

-      public IBlueTableSchema opSchema
-      {
-         get
-         {
-            if (myLastArrivedPacket == null  || myLastArrivedPacket.newRows == null)
-               return null;
-            return myLastArrivedPacket.newRows.schema;
-         }
-      }
+//      public IBlueTableSchema opSchema
+//      {
+//         get
+//         {
+//            if (myLastArrivedPacket == null  || myLastArrivedPacket.newRows == null)
+//               return null;
+//            return myLastArrivedPacket.newRows.schema;
+//         }
+//      }

-      public void init ()
-      {
-         myLastArrivedPacket = null;
-      }
-   }
-}
+//      public void init ()
+//      {
+//         myLastArrivedPacket = null;
+//      }
+//   }
+//}
@@ -1,119 +1,119 @@
-using System;
-using System.Data;
-using System.Collections;
-using BCG.Utilities;
-using BCG.Utilities.General;
-namespace BCG
-{
-   /// <summary>
-   /// Provides a live stream of Union of 2 streams
-   /// </summary>
-   public abstract class UnionStream : ViewStream 
-   {
-      protected UnionOperand myLeftOp, myRightOp;
-      protected ViewStream myLeftVs, myRightVs;
-      protected DataPacket myLastFiredPacket;
-      protected IDictionary myLeftIndex = new Hashtable (), myRightIndex = new Hashtable ();
-      protected int[] myLeftIndexToIndexMap, myRightIndexToIndexMap;
-      protected int myIdCounter, myIdIndex;
+//using System;
+//using System.Data;
+//using System.Collections;
+//using BCG.Utilities;
+//using BCG.Utilities.General;
+//namespace BCG
+//{
+//   /// <summary>
+//   /// Provides a live stream of Union of 2 streams
+//   /// </summary>
+//   public abstract class UnionStream : ViewStream 
+//   {
+//      protected UnionOperand myLeftOp, myRightOp;
+//      protected ViewStream myLeftVs, myRightVs;
+//      protected DataPacket myLastFiredPacket;
+//      protected IDictionary myLeftIndex = new Hashtable (), myRightIndex = new Hashtable ();
+//      protected int[] myLeftIndexToIndexMap, myRightIndexToIndexMap;
+//      protected int myIdCounter, myIdIndex;

-      protected override void myOnFirstSubscriber() 
-      {
-         myLeftOp = new UnionOperand (true, this);
-         myRightOp = new UnionOperand (false, this);
-         myLeftVs.onChange += new DataStream.ChangeDelegate (myLeftOp.myChangeHandler);
-         myRightVs.onChange += new DataStream.ChangeDelegate (myRightOp.myChangeHandler);
-      }
+//      protected override void myOnFirstSubscriber() 
+//      {
+//         myLeftOp = new UnionOperand (true, this);
+//         myRightOp = new UnionOperand (false, this);
+//         myLeftVs.onChange += new DataStream.ChangeDelegate (myLeftOp.myChangeHandler);
+//         myRightVs.onChange += new DataStream.ChangeDelegate (myRightOp.myChangeHandler);
+//      }

-      protected override void myOnLastSubscriber () 
-      {
-         myInit ();
-         myLeftVs.onChange -= new DataStream.ChangeDelegate (myLeftOp.myChangeHandler);
-         myRightVs.onChange -= new DataStream.ChangeDelegate (myRightOp.myChangeHandler);
-      }
+//      protected override void myOnLastSubscriber () 
+//      {
+//         myInit ();
+//         myLeftVs.onChange -= new DataStream.ChangeDelegate (myLeftOp.myChangeHandler);
+//         myRightVs.onChange -= new DataStream.ChangeDelegate (myRightOp.myChangeHandler);
+//      }

-      public void startSession ()
-      {
-         if (myLastFiredPacket == null)
-         {
-            myLastFiredPacket = new DataPacket ();
-            if (myLeftOp.opSchema != null)//ie call back initiated by left operand
-            {
-               myLeftIndexToIndexMap = myInitIndexToIndex (myLeftOp.opSchema, myLeftOp.opSchema);
-               myLastFiredPacket.newRows = new BlueTable (myLeftOp.opSchema);
-               myIdIndex = myLeftOp.opSchema["id"].colNo;
-            }
-            else
-            {
-               myRightIndexToIndexMap = myInitIndexToIndex (myRightOp.opSchema, myRightOp.opSchema);
-               myLastFiredPacket.newRows = new BlueTable (myRightOp.opSchema);
-               myIdIndex = myRightOp.opSchema["id"].colNo;
-            }
-         }
-         else if (myRightIndexToIndexMap == null && myRightOp.opSchema != null)
-            myRightIndexToIndexMap = myInitIndexToIndex (myLeftOp.opSchema, myRightOp.opSchema);
-         else if (myLeftIndexToIndexMap == null && myLeftOp.opSchema != null)
-            myLeftIndexToIndexMap = myInitIndexToIndex (myRightOp.opSchema, myLeftOp.opSchema);
-         myLastFiredPacket.addedRows = new BlueTable (myLastFiredPacket.newRows.schema);
-         myLastFiredPacket.deletedRows = new BlueTable (myLastFiredPacket.newRows.schema);
-      }
+//      public void startSession ()
+//      {
+//         if (myLastFiredPacket == null)
+//         {
+//            myLastFiredPacket = new DataPacket ();
+//            if (myLeftOp.opSchema != null)//ie call back initiated by left operand
+//            {
+//               myLeftIndexToIndexMap = myInitIndexToIndex (myLeftOp.opSchema, myLeftOp.opSchema);
+//               myLastFiredPacket.newRows = new BlueTable (myLeftOp.opSchema);
+//               myIdIndex = myLeftOp.opSchema["id"].colNo;
+//            }
+//            else
+//            {
+//               myRightIndexToIndexMap = myInitIndexToIndex (myRightOp.opSchema, myRightOp.opSchema);
+//               myLastFiredPacket.newRows = new BlueTable (myRightOp.opSchema);
+//               myIdIndex = myRightOp.opSchema["id"].colNo;
+//            }
+//         }
+//         else if (myRightIndexToIndexMap == null && myRightOp.opSchema != null)
+//            myRightIndexToIndexMap = myInitIndexToIndex (myLeftOp.opSchema, myRightOp.opSchema);
+//         else if (myLeftIndexToIndexMap == null && myLeftOp.opSchema != null)
+//            myLeftIndexToIndexMap = myInitIndexToIndex (myRightOp.opSchema, myLeftOp.opSchema);
+//         myLastFiredPacket.addedRows = new BlueTable (myLastFiredPacket.newRows.schema);
+//         myLastFiredPacket.deletedRows = new BlueTable (myLastFiredPacket.newRows.schema);
+//      }

-      private int[] myInitIndexToIndex (IBlueTableSchema sch1, IBlueTableSchema sch2)
-      {
-         //BlueEventLog.logAssert (sch1.columnNames.Length.Equals (sch2.columnNames.Length));
-         //foreach (string colName in sch1.columnNames)
-         //   BlueEventLog.logAssert (sch2[colName] != null);
-         if(!sch1.columnNames.Length.Equals(sch2.columnNames.Length))
-            throw new BlueDataException("Union stream: different schemas");
-         foreach(string colName in sch1.columnNames)
-         {
-            if(sch2[colName] == null)
-               throw new BlueDataException("Union stream: can't find " + colName);
-         }
-         int[] retVal = new int[sch1.columnNames.Length];
-         for (int i = 0; i < retVal.Length; i++)
-            retVal[i] = sch2[sch1[i].colName].colNo;
-         return retVal;
-      }
+//      private int[] myInitIndexToIndex (IBlueTableSchema sch1, IBlueTableSchema sch2)
+//      {
+//         //BlueEventLog.logAssert (sch1.columnNames.Length.Equals (sch2.columnNames.Length));
+//         //foreach (string colName in sch1.columnNames)
+//         //   BlueEventLog.logAssert (sch2[colName] != null);
+//         if(!sch1.columnNames.Length.Equals(sch2.columnNames.Length))
+//            throw new BlueDataException("Union stream: different schemas");
+//         foreach(string colName in sch1.columnNames)
+//         {
+//            if(sch2[colName] == null)
+//               throw new BlueDataException("Union stream: can't find " + colName);
+//         }
+//         int[] retVal = new int[sch1.columnNames.Length];
+//         for (int i = 0; i < retVal.Length; i++)
+//            retVal[i] = sch2[sch1[i].colName].colNo;
+//         return retVal;
+//      }

-      public virtual void endSession ()
-      {
-         myFireEvent (myLastFiredPacket.Clone ());
-      }
+//      public virtual void endSession ()
+//      {
+//         myFireEvent (myLastFiredPacket.Clone ());
+//      }

-      public abstract void add (IBlueRowCollection rows, bool isLeft);
+//      public abstract void add (IBlueRowCollection rows, bool isLeft);

-      public abstract void delete (IBlueRowCollection rows, bool isLeft);
+//      public abstract void delete (IBlueRowCollection rows, bool isLeft);

-      public abstract void mergeChanges (IBlueRowCollection dels, IBlueRowCollection adds, bool isLeft);
+//      public abstract void mergeChanges (IBlueRowCollection dels, IBlueRowCollection adds, bool isLeft);
      
-      private void myInit ()
-      {
-         myLastFiredPacket = null;
-         myLeftOp.init ();
-         myRightOp.init ();
-         myLeftIndex.Clear ();
-         myRightIndex.Clear ();
-         myIdCounter = myIdIndex = 0;
-         myLeftIndexToIndexMap = myRightIndexToIndexMap = null;
-      }
+//      private void myInit ()
+//      {
+//         myLastFiredPacket = null;
+//         myLeftOp.init ();
+//         myRightOp.init ();
+//         myLeftIndex.Clear ();
+//         myRightIndex.Clear ();
+//         myIdCounter = myIdIndex = 0;
+//         myLeftIndexToIndexMap = myRightIndexToIndexMap = null;
+//      }

-      /// <summary>
-      /// Used for test purposes only. Users of this class specifically, should ignore it 
-      /// (too bad, the NUnit cant see the private methods)
-      /// </summary>
-      /// <param name="dp"></param>
-      /// <param name="isLeft"></param>
-      internal void receiveStream (DataPacket dp, bool isLeft)
-      {
-         if (isLeft)
-            myLeftOp.myChangeHandler (dp);
-         else
-            myRightOp.myChangeHandler (dp);
-      }
-      public void receiveStreamTest(DataPacket dp, bool isLeft)
-      {
-          receiveStream(dp, isLeft);
-      }
-   }
-}
+//      /// <summary>
+//      /// Used for test purposes only. Users of this class specifically, should ignore it 
+//      /// (too bad, the NUnit cant see the private methods)
+//      /// </summary>
+//      /// <param name="dp"></param>
+//      /// <param name="isLeft"></param>
+//      internal void receiveStream (DataPacket dp, bool isLeft)
+//      {
+//         if (isLeft)
+//            myLeftOp.myChangeHandler (dp);
+//         else
+//            myRightOp.myChangeHandler (dp);
+//      }
+//      public void receiveStreamTest(DataPacket dp, bool isLeft)
+//      {
+//          receiveStream(dp, isLeft);
+//      }
+//   }
+//}
@@ -0,0 +1,73 @@
+namespace Bts.DataStream;
+
+using BCG;
+
+using bts.api.core;
+
+using Bts.DataStreamCore;
+
+using System;
+using System.Collections;
+using System.Collections.Generic;
+
+
+public class TableUpdateDataPacket<T> : IDataPacket<T> where T : IBlueRow {
+   readonly TableUpdate<T, int> table;
+   readonly IBlueTableSchema schema;
+
+   public TableUpdateDataPacket(TableUpdate<T, int> table, IBlueTableSchema schema) {
+      this.table = table;
+      this.schema = schema;
+   }
+
+   public IBlueRowCollection<T> newRows {
+      get {
+         var rv = new BlueTableImmutable<T>(schema);
+         return rv;
+      }
+   }
+
+   public IBlueRowCollection<T> addedRows => throw new NotImplementedException();
+
+   public IBlueRowCollection<T> deletedRows => throw new NotImplementedException();
+
+   class MyRowCollection : IBlueRowCollection<T> {
+      readonly TableUpdate<T, int> table;
+      readonly IEnumerable<T> items;
+      Lazy<SortedList<int, T>> itemList;
+
+      public MyRowCollection(IEnumerable<T> items, IBlueTableSchema schema) {
+         this.items = items;
+         this.schema = schema;
+         itemList = new(() => {
+            var rv = new SortedList<int, T>();
+            foreach (var item in this.items) {
+               rv.Add(item.Id(), item);
+            }
+            return rv;
+         });
+      }
+
+      public int Count => itemList.Value.Count;
+      public bool IsSynchronized => false;
+      public object SyncRoot => this;
+
+      public T this[int id] => itemList.Value[id];
+
+      public IBlueTableSchema schema { get; }
+
+      public bool containsID(int id) =>
+         itemList.Value.ContainsKey(id);
+
+      public void CopyTo(Array array, int index) {
+      }
+
+      public IEnumerator<T> GetEnumerator() {
+         throw new NotImplementedException();
+      }
+
+      IEnumerator IEnumerable.GetEnumerator() {
+         throw new NotImplementedException();
+      }
+   }
+}
@@ -114,6 +114,15 @@ namespace BCG {
            rv.Add(r.get<T>(col));
         return rv;
      }
+      public static IEnumerable<T> get_colvalues<T>(IEnumerable<IBlueRow> rows, int start_idx, int n_items,string col) {
+         List<T> rv = new List<T>();
+         int end_idx = Math.Min(start_idx + n_items, rows.Count());
+         for (int i = start_idx; i < end_idx; i++) {
+            var r = rows.ElementAt(i);
+            rv.Add(r.get<T>(col));
+         }
+         return rv;
+      }
      public static IEnumerable<T> get_colvalues<T>(IEnumerable<IBlueRow> rows, string col, int start_index, int count) {
         List<T> rv = new List<T>();
         int i = 0;
@@ -0,0 +1,56 @@
+#nullable enable
+namespace Bts.DataStreamCore;
+
+using BCG;
+
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Linq;
+
+public class BlueRowCollection : IBlueRowCollection {
+   readonly ICollection source;
+   readonly Func<int, IBlueRow> findRow;
+   readonly Func<int, bool> containsKey;
+
+   public BlueRowCollection(ICollection source, IBlueTableSchema schema, Func<int, IBlueRow>? findRow = null, Func<int, bool>? containsKey = null) {
+      this.source = source;
+      this.schema = schema;
+      if (findRow is null || containsKey is null) {
+         if (source is IBlueRowCollection brc) {
+            findRow ??= id => brc[id];
+            containsKey ??= brc.containsID;
+         } else if (source is IDictionary dict) {
+            findRow ??= id => dict[id] as IBlueRow ?? throw new KeyNotFoundException();
+            containsKey ??= id => dict.Contains(id);
+         } else if (source is IDictionary<int, IBlueRow> genDict) {
+            findRow ??= id => genDict[id];
+            containsKey ??= genDict.ContainsKey;
+         } else if (source is IReadOnlyDictionary<int, IBlueRow> roGenDict) {
+            findRow ??= id => roGenDict[id];
+            containsKey = roGenDict.ContainsKey;
+         } else {
+            var cachedRows = new Lazy<Dictionary<int, IBlueRow>>(() =>
+               source.Cast<IBlueRow>().ToDictionary(r => r.Id()));
+            findRow ??= id => cachedRows.Value[id];
+            // Don't remove this lambda expression. Intellisense may suggest it, but that would
+            // instantiate 'cachedRows' immediately and defeat the purpose of using Lazy<T>.
+            containsKey ??= id => cachedRows.Value.ContainsKey(id);
+         }
+      }
+      this.findRow = findRow;
+      this.containsKey = containsKey;
+   }
+
+   public IBlueRow this[int id] => findRow(id);
+
+   public IBlueTableSchema schema { get; }
+
+   public int Count => source.Count;
+
+   public bool containsID(int id) =>
+      containsKey(id);
+
+   public IEnumerator GetEnumerator() =>
+      source.GetEnumerator();
+}
@@ -96,3 +96,125 @@ public class BlueTableImmutable : IBlueRowCollection, IImmutableRowCollection {
         new(schema, table.ToImmutable());
   }
 }
+
+public class BlueTableImmutable<T> : IBlueRowCollection<T> where T : IBlueRow {
+   readonly ImmutableDictionary<int, T> rows;
+
+   public BlueTableImmutable(IBlueTableSchema schema)
+      : this(schema, ImmutableDictionary<int, T>.Empty) { }
+
+   public BlueTableImmutable(IBlueTableSchema schema, ImmutableDictionary<int, T> rows) {
+      this.schema = schema;
+      this.rows = rows;
+   }
+
+   public int Count => rows.Count;
+   public bool IsSynchronized => false;
+   public IBlueTableSchema schema { get; }
+   public object SyncRoot => this;
+
+   public T this[int id] => rows[id];
+
+   public bool containsID(int id) =>
+      rows.ContainsKey(id);
+
+   public void CopyTo(Array array, int index) {
+      foreach (var r in rows.Values) {
+         array.SetValue(r, index++);
+      }
+   }
+
+   public IEnumerator<T> GetEnumerator() =>
+      rows.Values.GetEnumerator();
+
+   IEnumerator IEnumerable.GetEnumerator() =>
+      GetEnumerator();
+
+   public DataPacketBuilder ToDataPacketBuilder() =>
+      new(this);
+
+   public class DataPacketBuilder {
+      readonly IBlueTableSchema schema;
+      readonly ImmutableDictionary<int, T>.Builder table;
+      readonly ImmutableDictionary<int, T>.Builder added =
+         ImmutableDictionary<int, T>.Empty.ToBuilder();
+      readonly ImmutableDictionary<int, T>.Builder removed =
+         ImmutableDictionary<int, T>.Empty.ToBuilder();
+
+      public DataPacketBuilder(BlueTableImmutable<T> table) {
+         schema = table.schema;
+         this.table = table.rows.ToBuilder();
+      }
+
+      public void AddOrUpdate(T row, IEqualityComparer<IBlueRow>? comparer = null) {
+         var id = row.Id();
+         if (table.TryGetValue(id, out var orig)) {
+            if (comparer?.Equals(row, orig) ?? false) {
+               // No op: This row was already in the table (according to 'comparer').
+               // Remove it from 'added' and 'removed'.
+               added.Remove(id);
+               removed.Remove(id);
+               return;
+            }
+            // Change op: A row with this id was already in the table,
+            // but its values have changed (according to 'comparer').
+            removed[id] = orig;
+         }
+         added[id] = row;
+      }
+
+      public void Remove(int id) {
+         added.Remove(id);
+         if (table.TryGetValue(id, out var orig)) {
+            removed[id] = orig;
+         }
+      }
+
+      public DataPacket ToDataPacket(bool reset = true) {
+         table.RemoveRange(removed.Keys);
+         table.AddRange(added);
+         var newRows = new BlueTableImmutable<T>(schema, table.ToImmutable());
+         var addedRows = new BlueTableImmutable<T>(schema, added.ToImmutable());
+         var removedRows = new BlueTableImmutable<T>(schema, removed.ToImmutable());
+         if (reset) {
+            added.Clear();
+            removed.Clear();
+         }
+         return new(newRows, addedRows, removedRows);
+      }
+
+      public IDataPacket<T> ToDataPacketT(bool reset = true) {
+         table.RemoveRange(removed.Keys);
+         table.AddRange(added);
+         var newRows = new BlueTableImmutable<T>(schema, table.ToImmutable());
+         var addedRows = new BlueTableImmutable<T>(schema, added.ToImmutable());
+         var removedRows = new BlueTableImmutable<T>(schema, removed.ToImmutable());
+         if (reset) {
+            added.Clear();
+            removed.Clear();
+         }
+         return new DataPacket<T>(newRows, addedRows, removedRows);
+      }
+
+      public IDataPacket<T> ToDataPacketT(
+         out BlueTableImmutable<T> newRows,
+         out BlueTableImmutable<T> addedRows,
+         out BlueTableImmutable<T> removedRows,
+         bool reset = true
+      ) {
+         table.RemoveRange(removed.Keys);
+         table.AddRange(added);
+         newRows = new BlueTableImmutable<T>(schema, table.ToImmutable());
+         addedRows = new BlueTableImmutable<T>(schema, added.ToImmutable());
+         removedRows = new BlueTableImmutable<T>(schema, removed.ToImmutable());
+         if (reset) {
+            added.Clear();
+            removed.Clear();
+         }
+         return new DataPacket<T>(newRows, addedRows, removedRows);
+      }
+
+      public BlueTableImmutable<T> ToImmutable() =>
+         new(schema, table.ToImmutable());
+   }
+}
@@ -0,0 +1,29 @@
+#nullable enable
+namespace Bts.DataStreamCore;
+
+using BCG;
+
+using System;
+using System.Collections.Generic;
+
+public class BlueTableSchemaBuilder {
+   readonly string? tableName;
+   readonly List<(string Name, Type Type)> colDefs = new();
+
+   public BlueTableSchemaBuilder(string? tableName = null) =>
+      this.tableName = tableName;
+
+   public void Add<T>(string Name) =>
+      colDefs.Add((Name, typeof(T)));
+
+   public void Add(string Name, Type Type) =>
+      colDefs.Add((Name, Type));
+
+   public IBlueTableSchema ToSchema(string? tableName = null) {
+      var colInfos = new ColumnInfo[colDefs.Count];
+      for (int i = 0; i < colInfos.Length; i++) {
+         colInfos[i] = new(colDefs[i].Name, i, colDefs[i].Type);
+      }
+      return new BlueTableSchema(colInfos, tableName ?? this.tableName);
+   }
+}
@@ -0,0 +1,16 @@
+#nullable enable
+namespace Bts.DataStreamCore;
+
+using BCG;
+
+using System;
+
+public class BlueTableSchemaFactory {
+   public static IBlueTableSchema Create(string tableName, params (string Name, Type Type)[] columns) {
+      var colInfos = new ColumnInfo[columns.Length];
+      for (int i = 0; i < columns.Length; i++) {
+         colInfos[i] = new(columns[i].Name, i, columns[i].Type);
+      }
+      return new BlueTableSchema(colInfos, tableName);
+   }
+}
@@ -0,0 +1,39 @@
+#nullable enable
+namespace Bts.DataStreamCore;
+
+using BCG;
+
+using System;
+using System.Collections.Generic;
+
+public static class BlueTableSelectorWrapper {
+   public static BlueTableSelectorWrapper<T> Create<T>(
+      IBlueRowCollection source, Func<IBlueRow, T> selector, IBlueTableSchema? schema = null) where T : IBlueRow =>
+      new(source, selector, schema);
+}
+
+public class BlueTableSelectorWrapper<T> : IBlueRowCollection<T> where T : IBlueRow {
+   readonly IBlueRowCollection source;
+   readonly Func<IBlueRow, T> selector;
+
+   public BlueTableSelectorWrapper(IBlueRowCollection source, Func<IBlueRow, T> selector, IBlueTableSchema? schema = null) {
+      this.source = source;
+      this.selector = selector;
+      this.schema = schema ?? source.schema;
+   }
+
+   public T this[int id] => selector(source[id]);
+
+   public IBlueTableSchema schema { get; }
+
+   public int Count => source.Count;
+
+   public bool containsID(int id) =>
+      source.containsID(id);
+
+   public IEnumerator<T> GetEnumerator() {
+      foreach (IBlueRow r in source) {
+         yield return selector(r);
+      }
+   }
+}
@@ -1,4 +1,5 @@
 using System;
+using System.Text.Json.Serialization;

 namespace BCG {
   /// <summary>
@@ -56,9 +57,16 @@ namespace BCG {
      private Type myColType;
      private object myColValue;

+      [JsonPropertyName("colNo")]
      public int colNo { get { return myColNo; } }
+
+      [JsonPropertyName("colName")]
      public string colName { get { return myColName; } }
+
+      [JsonPropertyName("colType")]
      public Type colType { get { return myColType; } }
+
+      [JsonIgnore]
      public object colValue {get{return myColValue;}}
   }
 }
@@ -1,5 +1,6 @@
 using System;
 using System.Collections;
+using System.Collections.Generic;

 namespace BCG
 {
@@ -15,5 +16,23 @@ namespace BCG
      IBlueRow this[int id]{get;}
      IBlueTableSchema schema {get;}
      bool containsID (int id);
+
+      bool ICollection.IsSynchronized => false;
+      object ICollection.SyncRoot => this;
+
+      void ICollection.CopyTo(Array array, int index) {
+         foreach (IBlueRow r in this) {
+            array.SetValue(r, index++);
+         }
+      }
+   }
+
+   public interface IBlueRowCollection<T> : IBlueRowCollection, IReadOnlyCollection<T> where T : IBlueRow {
+      new T this[int id] { get; }
+
+      IBlueRow IBlueRowCollection.this[int id] => this[id];
+
+      IEnumerator IEnumerable.GetEnumerator() =>
+         GetEnumerator();
   }
 }
@@ -3,6 +3,8 @@ using System.Collections;
 using System.Data;
 using BCG.Utilities.General;

+using Bts.DataStreamCore;
+
 namespace BCG
 {
 	[Serializable]
@@ -183,6 +185,7 @@ namespace BCG
            return retVal;
         }
      }
+      IBlueTableSchema IDataPacket.Schema => schema;

      /*          
       status  D total  A total  D this   A This   D' total A' Total 
@@ -314,6 +317,24 @@ namespace BCG
      }
   }

+   public class DataPacket<T> : IDataPacket<T> where T : IBlueRow {
+      public DataPacket(IBlueRowCollection<T> newRows, IBlueRowCollection<T> addedRows, IBlueRowCollection<T> deletedRows) {
+         this.newRows = newRows;
+         this.addedRows = addedRows;
+         this.deletedRows = deletedRows;
+      }
+
+      public IBlueRowCollection<T> newRows { get; }
+      public IBlueRowCollection<T> addedRows { get; }
+      public IBlueRowCollection<T> deletedRows { get; }
+
+      public static DataPacket<T> Empty(IBlueTableSchema schema) =>
+         new(
+            new BlueTableImmutable<T>(schema),
+            new BlueTableImmutable<T>(schema),
+            new BlueTableImmutable<T>(schema));
+   }
+
   public static class DataPacketExtensions {
      public static bool HasChanges(this DataPacket dataPacket) =>
         dataPacket.addedRows.Count > 0 || dataPacket.deletedRows.Count > 0;
@@ -19,8 +19,8 @@
  </ItemGroup>

  <ItemGroup>
-    <Proto Include="../../lib/config/protos/data_stream_config.proto" CSName="DataStreamConfig">
-      <OutDir>$(ProtoOutputPath)/lib/config/protos</OutDir>
+    <Proto Include="../../lib_spark/spark/config/protos/data_stream_config.proto" CSName="DataStreamConfig">
+      <OutDir>$(ProtoOutputPath)/lib_spark/spark/config/protos</OutDir>
    </Proto>
  </ItemGroup>

@@ -9,7 +9,22 @@ namespace BCG
 {
   public interface IDataPacket
   {
+      IBlueTableSchema Schema { get; }
      //this is an ICollection of IDictionary
      void get_new_added_deleted(out ICollection new_rows, out ICollection added_rows, out ICollection deleted_rows);
   }
+
+   public interface IDataPacket<T> : IDataPacket where T : IBlueRow {
+      IBlueRowCollection<T> newRows { get; }
+      IBlueRowCollection<T> addedRows { get; }
+      IBlueRowCollection<T> deletedRows { get; }
+
+      IBlueTableSchema IDataPacket.Schema => newRows.schema;
+
+      void IDataPacket.get_new_added_deleted(out ICollection new_rows, out ICollection added_rows, out ICollection deleted_rows) {
+         new_rows = newRows;
+         added_rows = addedRows;
+         deleted_rows = deletedRows;
+      }
+   }
 }
@@ -147,7 +147,7 @@ namespace bts.BlueOrb {
      }
   }

-   [DataStreamUserRespresentation("FutureHistoricalBars")]
+   [DataStreamUserRespresentation("FutureHistoricalBarsStream")]
   public class FutureHistoricalBarsStream : ViewStream {
      public FutureHistoricalBarsStream(string fut_contract, string range_beg, string range_end) {
         DateTime dt_range_beg = DateTime.Parse(range_beg);
@@ -1,8 +1,8 @@
 using bts.SilverPine;

-namespace bts.SilverPine {
-   using bts.api;
-   using bts.spark;
+namespace bts.BlueOrb {
+   //using bts.api;
+   //using bts.spark;

   using System;

@@ -16,7 +16,7 @@ namespace bts.SilverPine {
         public string contract_text;
      }
      public class FutureSpreadContract {
-         public List<Tuple<double, bts.SilverPine.ContractUtils.FutureTradableContract>> legs;
+         public List<Tuple<double, ContractUtils.FutureTradableContract>> legs;
         public string symbol {
            get {
               return legs == null || legs.Count == 0 ? String.Empty : legs.ElementAt(0).Item2.symbol;
@@ -206,7 +206,7 @@ namespace bts.SilverPine {
         }
         // tehre are some non calendar spreads we cant parse
         //Log.assert(false);
-         return bts.SilverPine.ContractUtils.ContractType.FutureSpread;
+         return ContractUtils.ContractType.FutureSpread;
      }

      public static ContractCategory get_category(string symbol) {
@@ -1,23 +1,23 @@
-namespace bts.SilverPine {
-   using BCG;
+//namespace bts.BlueOrb {
+//   using BCG;

-   // Finds the regions around specified events 
-   [DataStreamUserRespresentation("EventsPriceAction")]
-   public class EventsPriceActionStream : ViewStream {
-      public EventsPriceActionStream(string symbol, string event_type, string pre_event_period,
-            string post_event_period, int n_events) :
-         this(symbol, event_type, TimeBasedFilter.parse_duration(pre_event_period), TimeBasedFilter.parse_duration(post_event_period),
-         n_events, ViewStreamDatabase.instance) { }
+//   // Finds the regions around specified events 
+//   [DataStreamUserRespresentation("EventsPriceAction")]
+//   public class EventsPriceActionStream : ViewStream {
+//      public EventsPriceActionStream(string symbol, string event_type, string pre_event_period,
+//            string post_event_period, int n_events) :
+//         this(symbol, event_type, TimeBasedFilter.parse_duration(pre_event_period), TimeBasedFilter.parse_duration(post_event_period),
+//         n_events, ViewStreamDatabase.instance) { }

-      public EventsPriceActionStream(string symbol, string event_type, TimeSpan pre_event_period,
-      TimeSpan post_event_period, int n_events, ViewStreamDatabase vsd) {
-         // TODO: figure out which bars to use, 1 min or 1 day or live, based on pre_event_period / post_event_period
-         FuturesActiveContractStream olhcv_str = new FuturesActiveContractStream(symbol);
-         ViewStream events_str = new EventsStream(vsd, event_type);
-         // todo
-         //events_str = new TopStream(events_str, n_events, "-ts");
-         ViewStream rs = new TimeProximityJoinStream(olhcv_str, "ts_event", events_str, "ts", pre_event_period, post_event_period);
-         mySetRealSource(rs);
-      }
-   }
-}
+//      public EventsPriceActionStream(string symbol, string event_type, TimeSpan pre_event_period,
+//      TimeSpan post_event_period, int n_events, ViewStreamDatabase vsd) {
+//         // TODO: figure out which bars to use, 1 min or 1 day or live, based on pre_event_period / post_event_period
+//         FuturesActiveContractStream olhcv_str = new FuturesActiveContractStream(symbol);
+//         ViewStream events_str = new EventsStream(vsd, event_type);
+//         // todo
+//         //events_str = new TopStream(events_str, n_events, "-ts");
+//         ViewStream rs = new TimeProximityJoinStream(olhcv_str, "ts_event", events_str, "ts", pre_event_period, post_event_period);
+//         mySetRealSource(rs);
+//      }
+//   }
+//}
@@ -1,20 +1,20 @@
-namespace bts.SilverPine {
-   using BCG;
+//namespace bts.SilverPine {
+//   using BCG;

-   public class IndicatorStream : ViewStream {
-      enum Indicator {
-         StandardDeviation = 0
-      }
-      public class IndicatorInfo {
-         Indicator indicator;
-         IDictionary<string, object> parameters;
-      }
-      public IndicatorStream(string symbol_name, string indicator_name, string indicator_params_json) {
+//   public class IndicatorStream : ViewStream {
+//      enum Indicator {
+//         StandardDeviation = 0
+//      }
+//      public class IndicatorInfo {
+//         Indicator indicator;
+//         IDictionary<string, object> parameters;
+//      }
+//      public IndicatorStream(string symbol_name, string indicator_name, string indicator_params_json) {

-      }
-      public IndicatorStream(string symbol_name, IndicatorInfo info) {
+//      }
+//      public IndicatorStream(string symbol_name, IndicatorInfo info) {

-      }
+//      }

-   }
-}
+//   }
+//}
@@ -1,57 +1,57 @@
-namespace bts.SilverPine {
-   using BCG;
+//namespace bts.SilverPine {
+//   using BCG;

-   using ZstdSharp.Unsafe;
+//   using ZstdSharp.Unsafe;

-   [DataStreamUserRespresentation("AbsVolume")]
-   // Calculates the volume within a time window (this does not do rates, like highest daily volume etc,
-   // but with some thought can perhaps be extended)
-   public class AbsoluteVolumeStream : ViewStream {
-      // TODO, abstract this out
-      // adds contract_type to OHLCV stream
-      private static ViewStream _get_ohlcv_steam(string symbol, DateTime d1, DateTime d2) {
-         var ohlcv_params = OHLCVStream.get_params(d1, d2);
-         ViewStream rs = new OHLCVStream(symbol, ohlcv_params);
-         Func<IBlueTableSchema, IBlueTableSchema> create_schema =
-         s => BlueDataUtility.add_columns(s, Tuple.Create(typeof(string), "contract_type"));
-         rs = new SuperMapStream(rs,
-         s => create_schema(s),
-         r => {
-            IBlueTableSchema out_schema = create_schema(r.schema);
-            string contract_txt = r.get<string>("contract");
-            var contr_type = ContractUtils.get_type(contract_txt);
-            MutableBlueRow out_row = new MutableBlueRow(out_schema, r);
-            out_row.setValue("contract_type", contr_type.ToString());
-            return new IBlueRow[] { out_row };
-         });
-         return rs;
-      }
+//   [DataStreamUserRespresentation("AbsVolume")]
+//   // Calculates the volume within a time window (this does not do rates, like highest daily volume etc,
+//   // but with some thought can perhaps be extended)
+//   public class AbsoluteVolumeStream : ViewStream {
+//      // TODO, abstract this out
+//      // adds contract_type to OHLCV stream
+//      private static ViewStream _get_ohlcv_steam(string symbol, DateTime d1, DateTime d2) {
+//         var ohlcv_params = OHLCVStream.get_params(d1, d2);
+//         ViewStream rs = new OHLCVStream(symbol, ohlcv_params);
+//         Func<IBlueTableSchema, IBlueTableSchema> create_schema =
+//         s => BlueDataUtility.add_columns(s, Tuple.Create(typeof(string), "contract_type"));
+//         rs = new SuperMapStream(rs,
+//         s => create_schema(s),
+//         r => {
+//            IBlueTableSchema out_schema = create_schema(r.schema);
+//            string contract_txt = r.get<string>("contract");
+//            var contr_type = ContractUtils.get_type(contract_txt);
+//            MutableBlueRow out_row = new MutableBlueRow(out_schema, r);
+//            out_row.setValue("contract_type", contr_type.ToString());
+//            return new IBlueRow[] { out_row };
+//         });
+//         return rs;
+//      }

-      public AbsoluteVolumeStream(string symbol, string ctype, DateTime d1)
-      : this(symbol, ctype, d1, d1) { }
+//      public AbsoluteVolumeStream(string symbol, string ctype, DateTime d1)
+//      : this(symbol, ctype, d1, d1) { }

-      public AbsoluteVolumeStream(string symbol, string ctype, DateTime d1, DateTime d2)
-      : this(symbol, Enum.Parse<ContractUtils.ContractType>(ctype), d1, d2) {
-      }
+//      public AbsoluteVolumeStream(string symbol, string ctype, DateTime d1, DateTime d2)
+//      : this(symbol, Enum.Parse<ContractUtils.ContractType>(ctype), d1, d2) {
+//      }

-      static void swap_if_necessary(ref DateTime dt1, ref DateTime dt2) {
-         if (dt1 < dt2) {
-            DateTime aux = dt1;
-            dt1 = dt2;
-            dt2 = aux;
-         }
-      }
+//      static void swap_if_necessary(ref DateTime dt1, ref DateTime dt2) {
+//         if (dt1 < dt2) {
+//            DateTime aux = dt1;
+//            dt1 = dt2;
+//            dt2 = aux;
+//         }
+//      }

-      private AbsoluteVolumeStream(string symbol, ContractUtils.ContractType ctype, DateTime d1, DateTime d2) {
-         swap_if_necessary(ref d1, ref d2);
-         ViewStream ohlcv_str = _get_ohlcv_steam(symbol, d1, d2);
-         ohlcv_str = new FilterStream(ohlcv_str,
-            r => r.get_datetime("ts_event") <= d1 && r.get_datetime("ts_event") >= d2);
-         ohlcv_str = new FilterStream(ohlcv_str, r => String.Compare(ctype.ToString(), r.get<string>("contract_type"), true) == 0);
-         ViewStream rs = new MaxStream(ohlcv_str, "volume");
-         rs = new NewProjectStream(rs, "volume:total_volume contract !");
-         mySetRealSource(rs);
-      }
+//      private AbsoluteVolumeStream(string symbol, ContractUtils.ContractType ctype, DateTime d1, DateTime d2) {
+//         swap_if_necessary(ref d1, ref d2);
+//         ViewStream ohlcv_str = _get_ohlcv_steam(symbol, d1, d2);
+//         ohlcv_str = new FilterStream(ohlcv_str,
+//            r => r.get_datetime("ts_event") <= d1 && r.get_datetime("ts_event") >= d2);
+//         ohlcv_str = new FilterStream(ohlcv_str, r => String.Compare(ctype.ToString(), r.get<string>("contract_type"), true) == 0);
+//         ViewStream rs = new MaxStream(ohlcv_str, "volume");
+//         rs = new NewProjectStream(rs, "volume:total_volume contract !");
+//         mySetRealSource(rs);
+//      }

-   }
-}
+//   }
+//}
@@ -1,4 +1,4 @@
-namespace bts.SilverPine {
+namespace bts.BlueOrb {
   using BCG;

   using SharpCompress.Common.Rar;
@@ -1,4 +1,4 @@
-namespace bts.SilverPine {
+namespace bts.BlueOrb {
   using BCG;

   using bts.api;
@@ -1,4 +1,4 @@
-namespace bts.SilverPine {
+namespace bts.BlueOrb {
   using BCG;

   using System.Diagnostics;
@@ -1,4 +1,4 @@
-namespace bts.SilverPine{
+namespace bts.BlueOrb{
   using BCG;
   public class SpreadStatArbsWeightLinearRegressionStream : ViewStream {
      private IBlueTableSchema _schema;
@@ -1,4 +1,4 @@
-namespace bts.SilverPine {
+namespace bts.BlueOrb {
   using BCG;

   using MathNet.Numerics.LinearAlgebra.Factorization;
@@ -2,7 +2,7 @@
 using System.Collections.Generic;
 using MathNet.Numerics.LinearRegression;

-namespace bts.SilverPine {
+namespace bts.BlueOrb {

   public static class SpreadWeightCalculator {
      /// <summary>
@@ -1,40 +1,40 @@
-namespace bts.SilverPine {
-   using BCG;
+//namespace bts.SilverPine {
+//   using BCG;

-   using MathNet.Numerics.Statistics;
+//   using MathNet.Numerics.Statistics;

-   // calculates the reazlied standard deviation of a series of prices
-   public class StandardDeviationStream : ViewStream {
-      public StandardDeviationStream(string symbol, string std_dev_period)
-      :
-      this(symbol, TimeBasedFilter.parse_duration(std_dev_period)) { }
+//   // calculates the reazlied standard deviation of a series of prices
+//   public class StandardDeviationStream : ViewStream {
+//      public StandardDeviationStream(string symbol, string std_dev_period)
+//      :
+//      this(symbol, TimeBasedFilter.parse_duration(std_dev_period)) { }

-      public StandardDeviationStream(string symbol, TimeSpan std_dev_period) {
-         // since this is a 1_day bar str, 90 rows => 90 days, so no need to filter
-         int n_days = std_dev_period.Days;
-         ViewStream ohlcv_str = new FuturesActiveContractStream(symbol);
-         IBlueTableSchema out_schema = null;
-         int id = 0;
-         Func<List<IBlueRow>, uint, IBlueRow> find_std_dev = (rows, idx) => {
-            int iidx = Convert.ToInt32(idx);
-            if (rows == null || rows.Count == 0)
-               return null;
-            IBlueRow ir = rows.ElementAt(iidx);
-            out_schema = out_schema ?? ir.schema.add_column(TypeCode.Double, "std_dev");
-            MutableBlueRow rv = new MutableBlueRow(out_schema, ir);
-            rv.setValue("std_dev", Double.NaN);
-            IEnumerable<double> last_n_close_prices = rows.Skip(iidx).Take(n_days).Select(r => r.get<double>("close"));
-            if (last_n_close_prices.Count() >= n_days) {
-               double std_dev = MathNet.Numerics.Statistics.Statistics.StandardDeviation(last_n_close_prices);
-               rv.setValue("std_dev", std_dev);
-            }
-            //rv.setValue("id", id++);
-            return rv;
-         };
-         ViewStream rs = new GlobalMapStream(ohlcv_str, "ts_event", find_std_dev);
-         rs = new NullFilterStream(rs, "std_dev", true);
-         mySetRealSource(rs);
-      }
+//      public StandardDeviationStream(string symbol, TimeSpan std_dev_period) {
+//         // since this is a 1_day bar str, 90 rows => 90 days, so no need to filter
+//         int n_days = std_dev_period.Days;
+//         ViewStream ohlcv_str = new FuturesActiveContractStream(symbol);
+//         IBlueTableSchema out_schema = null;
+//         int id = 0;
+//         Func<List<IBlueRow>, uint, IBlueRow> find_std_dev = (rows, idx) => {
+//            int iidx = Convert.ToInt32(idx);
+//            if (rows == null || rows.Count == 0)
+//               return null;
+//            IBlueRow ir = rows.ElementAt(iidx);
+//            out_schema = out_schema ?? ir.schema.add_column(TypeCode.Double, "std_dev");
+//            MutableBlueRow rv = new MutableBlueRow(out_schema, ir);
+//            rv.setValue("std_dev", Double.NaN);
+//            IEnumerable<double> last_n_close_prices = rows.Skip(iidx).Take(n_days).Select(r => r.get<double>("close"));
+//            if (last_n_close_prices.Count() >= n_days) {
+//               double std_dev = MathNet.Numerics.Statistics.Statistics.StandardDeviation(last_n_close_prices);
+//               rv.setValue("std_dev", std_dev);
+//            }
+//            //rv.setValue("id", id++);
+//            return rv;
+//         };
+//         ViewStream rs = new GlobalMapStream(ohlcv_str, "ts_event", find_std_dev);
+//         rs = new NullFilterStream(rs, "std_dev", true);
+//         mySetRealSource(rs);
+//      }

-   }
-}
+//   }
+//}
@@ -1,4 +1,4 @@
-namespace bts.SilverPine {
+namespace bts.BlueOrb {
   using BCG.Utilities.General;

   using bts.md;
@@ -1,5 +1,6 @@
-namespace BCG {
+namespace bts.BlueOrb{
   using BCG.Utilities.General;
+   using BCG;

   using bts.md;

@@ -1,4 +1,4 @@
-namespace bts.SilverPine {
+namespace bts.BlueOrb {
   using BCG;

   [DataStreamUserRespresentation("Events")]
@@ -15,4 +15,4 @@ namespace bts.SilverPine {
         mySetRealSource(rs);
      }
   }
-}
+}
@@ -1,4 +1,4 @@
-namespace bts.SilverPine {
+namespace bts.BlueOrb{
   using BCG;

   using bts;
@@ -1,4 +1,4 @@
-namespace bts.SilverPine {
+namespace bts.BlueOrb {
   using BCG.Utilities.General;

   using bts.api;
@@ -1,37 +1,37 @@
-namespace bts.SilverPine {
-   using BCG;
+//namespace bts.SilverPine {
+//   using BCG;

-   [DataStreamUserRespresentation("OHLCVPriceDiff")]
-   public class OHLCVPriceDiffStream : ViewStream {
-      // TODO, abstract this out
-      private static ViewStream _get_ohlcv_steam(string symbol_or_contract, DateTime d1, DateTime d2) {
-         var tup = ContractUtils.get_symbol_category(symbol_or_contract);
-         var ohlcv_params = OHLCVStream.get_params(d1, d2);
-         var cont_cat = tup.Item1;
-         string symbol = tup.Item2;
-         ViewStream rs = new OHLCVStream(symbol, ohlcv_params);
-         switch (cont_cat) {
-            case ContractUtils.ContractCategory.FutureString:
-            case ContractUtils.ContractCategory.SpreadString:
-               rs = new FilterStream(rs, $"contract = '{symbol_or_contract}'");
-               break;
-            case ContractUtils.ContractCategory.Symbol:
-               rs = new FuturesActiveContractStream(rs);
-               break;
-         }
-         return rs;
-      }
+//   [DataStreamUserRespresentation("OHLCVPriceDiff")]
+//   public class OHLCVPriceDiffStream : ViewStream {
+//      // TODO, abstract this out
+//      private static ViewStream _get_ohlcv_steam(string symbol_or_contract, DateTime d1, DateTime d2) {
+//         var tup = ContractUtils.get_symbol_category(symbol_or_contract);
+//         var ohlcv_params = OHLCVStream.get_params(d1, d2);
+//         var cont_cat = tup.Item1;
+//         string symbol = tup.Item2;
+//         ViewStream rs = new OHLCVStream(symbol, ohlcv_params);
+//         switch (cont_cat) {
+//            case ContractUtils.ContractCategory.FutureString:
+//            case ContractUtils.ContractCategory.SpreadString:
+//               rs = new FilterStream(rs, $"contract = '{symbol_or_contract}'");
+//               break;
+//            case ContractUtils.ContractCategory.Symbol:
+//               rs = new FuturesActiveContractStream(rs);
+//               break;
+//         }
+//         return rs;
+//      }

-      public OHLCVPriceDiffStream(string symbol_or_contract, DateTime d1, DateTime d2)
-      : this(symbol_or_contract, d1, d2, "close") { }
+//      public OHLCVPriceDiffStream(string symbol_or_contract, DateTime d1, DateTime d2)
+//      : this(symbol_or_contract, d1, d2, "close") { }

-      public OHLCVPriceDiffStream(string symbol_or_contract, DateTime d1, DateTime d2, string col) {
-         ViewStream rs = _get_ohlcv_steam(symbol_or_contract, d1, d2);
-         rs = new ColumnDifferenceStream(rs, r => d1.Equals(r.get_datetime("ts_event")),
-                                              r => d2.Equals(r.get_datetime("ts_event")), col, col);
-         rs = new NewProjectStream(rs, "diff !");
-         mySetRealSource(rs);
-      }
+//      public OHLCVPriceDiffStream(string symbol_or_contract, DateTime d1, DateTime d2, string col) {
+//         ViewStream rs = _get_ohlcv_steam(symbol_or_contract, d1, d2);
+//         rs = new ColumnDifferenceStream(rs, r => d1.Equals(r.get_datetime("ts_event")),
+//                                              r => d2.Equals(r.get_datetime("ts_event")), col, col);
+//         rs = new NewProjectStream(rs, "diff !");
+//         mySetRealSource(rs);
+//      }

-   }
-}
+//   }
+//}
@@ -1,4 +1,4 @@
-namespace bts.SilverPine {
+namespace bts.BlueOrb {
   using bts;
   using BCG;
   using MongoDB.Driver;
@@ -1,8 +1,7 @@
 using System;
 using BCG;

-namespace bts.SilverPine
-{
+namespace bts.BlueOrb {
 /*
   ----------------------------------------------------------------------
                               *Expression*
@@ -1,32 +1,32 @@
-namespace bts.SilverPine {
-   using BCG;
+//namespace bts.SilverPine {
+//   using BCG;

-   // this is an internal stream. its job is to take an OHLCV stream
-   // and filter it for 
-   // 1. symbol (table_name and active contract) and 
-   // 2. time  (can be a day of the week, month, and a time of the day period)
-   public class BarsSearcherStream : ViewStream {
-      public BarsSearcherStream(string symbol, OHLCVStream.OHLCVBarSize bar_size,
-                                 string day_of_week, TimeSpan tod_begin, TimeSpan tod_end) {
-         ViewStream rs = new OHLCVStream(symbol, bar_size);
-         rs = new DateTimeExpanstionStream(rs, "ts_event");
-         rs = new FilterStream(rs, r => {
-            DateTime ts = r.get_datetime("ts_event");
-            return r.get<string>("ts_event_day") == day_of_week.ToString() &&
-                  ts.TimeOfDay >= tod_begin && ts.TimeOfDay <= tod_end;
-         });
-         // look at symbol and either filter ir ot do the active contracts
-         // assuming its symbol
-         rs = new NonIncJoinStream(rs, "ts_event_date", "open high low close volume contract ts_event",
-                                 new FuturesActiveContractStream(symbol), "ts_event", "ts_event:date contract:active_contract");
-         // filter it for the active contract only
-         rs = new FilterStream(rs, r => String.Compare(r.get<string>("contract"), r.get<string>("active_contract")) == 0);
-         mySetRealSource(rs);
-      }
-      public BarsSearcherStream(string symbol, string bar_size,
-                                 string day_of_week, string tod_begin, string tod_end)
-                                 :
-                                 this(symbol, Enum.Parse<OHLCVStream.OHLCVBarSize>(bar_size), day_of_week,
-                                       TimeSpan.Parse(tod_begin), TimeSpan.Parse(tod_end)) { }
-   }
-}
+//   // this is an internal stream. its job is to take an OHLCV stream
+//   // and filter it for 
+//   // 1. symbol (table_name and active contract) and 
+//   // 2. time  (can be a day of the week, month, and a time of the day period)
+//   public class BarsSearcherStream : ViewStream {
+//      public BarsSearcherStream(string symbol, OHLCVStream.OHLCVBarSize bar_size,
+//                                 string day_of_week, TimeSpan tod_begin, TimeSpan tod_end) {
+//         ViewStream rs = new OHLCVStream(symbol, bar_size);
+//         rs = new DateTimeExpanstionStream(rs, "ts_event");
+//         rs = new FilterStream(rs, r => {
+//            DateTime ts = r.get_datetime("ts_event");
+//            return r.get<string>("ts_event_day") == day_of_week.ToString() &&
+//                  ts.TimeOfDay >= tod_begin && ts.TimeOfDay <= tod_end;
+//         });
+//         // look at symbol and either filter ir ot do the active contracts
+//         // assuming its symbol
+//         rs = new NonIncJoinStream(rs, "ts_event_date", "open high low close volume contract ts_event",
+//                                 new FuturesActiveContractStream(symbol), "ts_event", "ts_event:date contract:active_contract");
+//         // filter it for the active contract only
+//         rs = new FilterStream(rs, r => String.Compare(r.get<string>("contract"), r.get<string>("active_contract")) == 0);
+//         mySetRealSource(rs);
+//      }
+//      public BarsSearcherStream(string symbol, string bar_size,
+//                                 string day_of_week, string tod_begin, string tod_end)
+//                                 :
+//                                 this(symbol, Enum.Parse<OHLCVStream.OHLCVBarSize>(bar_size), day_of_week,
+//                                       TimeSpan.Parse(tod_begin), TimeSpan.Parse(tod_end)) { }
+//   }
+//}
@@ -2,34 +2,38 @@ namespace bts.SilverPine {
   using BCG;

   using bts;
-   using bts.SilverPine;
+   //using bts.SilverPine;

-   [DataStreamUserRespresentation("nl")]
-   public class NLStream : ViewStream {
-      private readonly string _nl_prompt;
-      private ViewStream _nl_to_dsl_stream, _dsl_stream;
-      public NLStream(string nl_prompt) {
-         _nl_prompt = nl_prompt;
-      }
+   namespace bts.BlueOrb {

-      protected override void myOnFirstSubscriber() {
-         _nl_to_dsl_stream = new NLToDSLStream(_nl_prompt);
-         _nl_to_dsl_stream.onChange += _handle_dsl_callback;
-      }
+      [DataStreamUserRespresentation("nl")]
+      public class NLStream : ViewStream {
+         private readonly string _nl_prompt;
+         private ViewStream _nl_to_dsl_stream, _dsl_stream;
+         public NLStream(string nl_prompt) {
+            _nl_prompt = nl_prompt;
+         }

-      private void _handle_dsl_callback(object dp_obj) {
-         DataPacket dp = dp_obj as DataPacket;
-         var r = dp.newRows.to_list().ElementAt(0);
-         string dsl = r.get<string>("response");
-         Log.Gen.info($"NLStream._handle_dsl_callback dsl: {dsl}");
-         _dsl_stream = new DSLStream(dsl);
-         _dsl_stream.onChange += _handle_data_cb;
-      }
+         protected override void myOnFirstSubscriber() {
+            _nl_to_dsl_stream = new NLToDSLStream(_nl_prompt);
+            _nl_to_dsl_stream.onChange += _handle_dsl_callback;
+         }

-      private void _handle_data_cb(object dp_obj) {
-         var dp = dp_obj as DataPacket;
-         DataPacket dp_out = BlueDataUtility.build_dp_inc(dp.newRows, myLastValue as DataPacket);
-         myFireEvent(dp_out);
+         private void _handle_dsl_callback(object dp_obj) {
+            DataPacket dp = dp_obj as DataPacket;
+            var r = dp.newRows.to_list().ElementAt(0);
+            string dsl = r.get<string>("response");
+            Log.Gen.info($"NLStream._handle_dsl_callback dsl: {dsl}");
+            _dsl_stream = new DSLStream(dsl);
+            _dsl_stream.onChange += _handle_data_cb;
+         }
+
+         private void _handle_data_cb(object dp_obj) {
+            var dp = dp_obj as DataPacket;
+            DataPacket dp_out = BlueDataUtility.build_dp_inc(dp.newRows, myLastValue as DataPacket);
+            myFireEvent(dp_out);
+         }
      }
   }
-}
+
+}
@@ -43,7 +43,7 @@
      Condition="Exists($(ComputerSpecificConfig))" />

  <PropertyGroup Label="ProtoBuf">
-    <ProtocPluginBase>$(SparkRoot)\src\lib\api\service_compiler</ProtocPluginBase>
+    <ProtocPluginBase>$(SparkRoot)\src\lib_spark\spark\api\service_compiler</ProtocPluginBase>
  </PropertyGroup>
  <Choose>
    <When Condition="'$(OS)' == 'Windows_NT'">
@@ -24,7 +24,7 @@
    <PropertyGroup>
      <GenProtosCmd>$(protoc) %(Proto.FullPath)</GenProtosCmd>
      <GenProtosCmd>$(GenProtosCmd) --error_format=msvs</GenProtosCmd>
-      <GenProtosCmd>$(GenProtosCmd) --proto_path=$(SourceRoot)</GenProtosCmd>
+      <GenProtosCmd>$(GenProtosCmd) --proto_path=$(SourceRoot)/lib_spark</GenProtosCmd>
      <GenProtosCmd>$(GenProtosCmd) --plugin=&quot;protoc-gen-spark-csharp=$(ProtocPlugin)&quot;</GenProtosCmd>
      <GenProtosCmd>$(GenProtosCmd) --spark-csharp_out=lang=c#,templates=immutable.pb.cs,strip_out_path=1:%(Proto.OutDir)</GenProtosCmd>
      <GenProtosCmd>$(GenProtosCmd) --csharp_out=%(Proto.OutDir)</GenProtosCmd>
@@ -65,7 +65,7 @@
    <PropertyGroup>
      <GenProtosCmd>$(protoc) %(ProtoService.FullPath)</GenProtosCmd>
      <GenProtosCmd>$(GenProtosCmd) --error_format=msvs</GenProtosCmd>
-      <GenProtosCmd>$(GenProtosCmd) --proto_path=$(SourceRoot)</GenProtosCmd>
+      <GenProtosCmd>$(GenProtosCmd) --proto_path=$(SourceRoot)/lib_spark</GenProtosCmd>
      <GenProtosCmd>$(GenProtosCmd) --plugin=&quot;protoc-gen-service=$(ProtocPlugin)&quot;</GenProtosCmd>
      <GenProtosCmd>$(GenProtosCmd) --service_out=lang=c#,strip_out_path=1:%(ProtoService.OutDir)</GenProtosCmd>
      <GenProtosCmd>$(GenProtosCmd) --csharp_out=%(ProtoService.OutDir)</GenProtosCmd>
@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 3.10)
+cmake_minimum_required(VERSION 3.5)
 project(Spark.Admin.WebApp)

 include(GNUInstallDirs)
@@ -10,11 +10,11 @@ include(${CMAKE_SOURCE_DIR}/cmake/ProtobufGenerateFs.cmake)
 file(
  GLOB_RECURSE ${PROJECT_NAME}_PROTO_SOURCES
  RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
-  ../lib/api/*.proto
-  ../lib/api/services/*.proto
-  ../lib/domain_model/*.proto
-  ../lib/sparkproto/*.proto
-  ../lib/md/protos/*.proto
+  ../lib_spark/spark/api/*.proto
+  ../lib_spark/spark/api/services/*.proto
+  ../lib_spark/spark/domain_model/*.proto
+  ../lib_spark/spark/sparkproto/*.proto
+  ../lib_spark/spark/md/protos/*.proto
  ../../externals/linux/include/google/protobuf/descriptor.proto
  ../../externals/linux/include/google/protobuf/timestamp.proto
 )
@@ -44,7 +44,7 @@ set(_build_admin_web_app_deps
  ${_build_admin_web_app_py}
  ${CMAKE_CURRENT_SOURCE_DIR}/src/Spark.Admin.WebApp.fsproj
  ${CMAKE_CURRENT_SOURCE_DIR}/webpack.config.js
-  ${CMAKE_SOURCE_DIR}/src/lib/api/service_compiler/templates/fs/fable.fs
+  ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/fs/fable.fs
  ${CMAKE_CURRENT_SOURCE_DIR}/package-lock.json
  ${${PROJECT_NAME}_PROTO_SOURCES}
  ${_api_ts_SOURCES}
@@ -73,7 +73,7 @@ add_custom_command(
 protobuf_generate_js(
  TARGET_NAME generate_js_protos
  PROTO_INCLUDE_DIRS
-    ..
+    ../lib_spark
    ../../externals/linux/include
  PROTO_SOURCES ${${PROJECT_NAME}_PROTO_SOURCES}
  OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/proto.js
@@ -82,7 +82,7 @@ protobuf_generate_js(
 protobuf_generate_fs(
  TARGET_NAME generate_fs_protos
  PROTO_INCLUDE_DIRS
-    ..
+    ../lib_spark
    ../../externals/linux/include
  PROTO_SOURCES ${${PROJECT_NAME}_PROTO_SOURCES}
  OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/fable
@@ -184,4 +184,3 @@ install(
  DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/public"
  DESTINATION "${CMAKE_INSTALL_BINDIR}/${PROJECT_NAME}"
 )
-
@@ -17,7 +17,7 @@ type Tab =
  | PositionManagementTab
  | MarginTab

-let venues = [| Venue.BINANCE; Venue.BinanceUs; Venue.BinanceUsdsFutures; Venue.CBOE; Venue.COINBASE; Venue.DERIBIT; Venue.OKX; Venue.GLBX; Venue.ICE; Venue.TAIFEX; Venue.TWSE; Venue.SIM; |]
+let venues = [| Venue.BINANCE; Venue.BinanceUs; Venue.BinanceUsdsFutures; Venue.BYBIT; Venue.CBOE; Venue.COINBASE; Venue.DERIBIT; Venue.GATEIO; Venue.OKX; Venue.GLBX; Venue.ICE; Venue.TAIFEX; Venue.TWSE; Venue.SIM; |]

 let securityTypes = [| SecurityType.SPOT; SecurityType.FUTURE; SecurityType.OPTION; SecurityType.SPREAD; |]

@@ -50,6 +50,11 @@ type BinanceUsdsFUTURESExchangeField =
  | Origin of CustomerOrFirm
  | ClearingAccount of string

+type BybitExchangeField =
+  | CTI of CTICode
+  | Origin of CustomerOrFirm
+  | ClearingAccount of string
+
 type COINBASEExchangeField =
  | CTI of CTICode
  | Origin of CustomerOrFirm
@@ -60,6 +65,11 @@ type DERIBITExchangeField =
  | Origin of CustomerOrFirm
  | ClearingAccount of string

+type GateioExchangeField =
+  | CTI of CTICode
+  | Origin of CustomerOrFirm
+  | ClearingAccount of string
+
 type ICEExchangeField =
  | CTI of CTICode
  | Origin of CustomerOrFirm
@@ -797,6 +797,8 @@ let venueToString (venue:bts.api.Venue) =
    | bts.api.Venue.BINANCE -> "BINANCE"
    | bts.api.Venue.BinanceUs -> "BinanceUs"
    | bts.api.Venue.BinanceUsdsFutures -> "BinanceUsdsFutures"
+    | bts.api.Venue.BYBIT -> "BYBIT"
+    | bts.api.Venue.GATEIO -> "GATEIO"
    | bts.api.Venue.FTX -> "FTX"
    | bts.api.Venue.OKX -> "OKX"
    | bts.api.Venue.TAIFEX -> "TAIFEX"
@@ -0,0 +1,172 @@
+cmake_minimum_required(VERSION 3.14)
+project(sparkd)
+
+include(FetchContent)
+include(${CMAKE_SOURCE_DIR}/cmake/safe_dep_glob.cmake)
+include(${CMAKE_SOURCE_DIR}/cmake/ProtobufGenerateCpp.cmake)
+include(${CMAKE_SOURCE_DIR}/cmake/InstallBasic.cmake)
+include(GNUInstallDirs)
+
+if (NOT DEFINED PROJECT_VERSION)
+  set(PROJECT_VERSION 1.0.0)
+endif()
+
+# dependencies
+file(
+  GLOB_RECURSE ${PROJECT_NAME}_PROTO_SOURCES
+  RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
+  *.proto
+)
+file(
+  GLOB_RECURSE ${PROJECT_NAME}_SOURCES
+  RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
+  *.cpp
+)
+update_deps_file("${${PROJECT_NAME}_PROTO_SOURCES};${${PROJECT_NAME}_SOURCES}")
+
+protobuf_generate_cpp(
+  CXX_SOURCES ${PROJECT_NAME}_PROTO_CXX_SOURCES
+  CXX_HEADERS ${PROJECT_NAME}_PROTO_CXX_HEADERS
+  CXX_OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/include
+  PROTO_INCLUDE_DIRS
+    ${EXTERNALS_INCLUDE}
+    ${CMAKE_SOURCE_DIR}/src/lib_spark
+    ${CMAKE_CURRENT_SOURCE_DIR}
+  PROTO_SOURCES
+    ${${PROJECT_NAME}_PROTO_SOURCES}
+)
+add_executable(sparkd ${${PROJECT_NAME}_SOURCES} ${${PROJECT_NAME}_PROTO_CXX_SOURCES})
+
+# include paths
+target_include_directories(
+  sparkd
+  PUBLIC
+    $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
+    $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>/include
+    $<BUILD_INTERFACE:${emumba-deribit-connector-library_INCLUDE_DIRS}>
+)
+
+find_package(PkgConfig REQUIRED)
+pkg_check_modules(emumba-deribit-connector-library REQUIRED emumba-deribit-connector)
+
+# This dir contains rapidjsonConfig.cmake
+list(APPEND CMAKE_PREFIX_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake_modules)
+find_package(rapidjson REQUIRED)
+target_include_directories(
+  ${PROJECT_NAME} SYSTEM
+  PUBLIC
+    $<BUILD_INTERFACE:${rapidjson_INCLUDE_DIR}> 
+)
+add_dependencies(${PROJECT_NAME} rapidjson_project)
+
+# databento
+FetchContent_Declare(
+  databento
+  GIT_REPOSITORY https://github.com/databento/databento-cpp
+  GIT_TAG v0.31.0
+)
+FetchContent_MakeAvailable(databento)
+
+# deribit-demo log level
+if(NOT DEFINED DERIBIT_DEMO_LOG_LEVEL)
+  add_definitions(-DSPDLOG_ACTIVE_LEVEL=0)
+  message("-- Building of DERIBIT_DEMO_LOG_LEVEL=TRACE")
+elseif(DERIBIT_DEMO_LOG_LEVEL STREQUAL "TRACE")
+  add_definitions(-DSPDLOG_ACTIVE_LEVEL=0)
+  message("-- Building of DERIBIT_DEMO_LOG_LEVEL=TRACE")
+elseif(DERIBIT_DEMO_LOG_LEVEL STREQUAL "DEBUG")
+  add_definitions(-DSPDLOG_ACTIVE_LEVEL=1)
+  message("-- Building of DERIBIT_DEMO_LOG_LEVEL=DEBUG")
+elseif(DERIBIT_DEMO_LOG_LEVEL STREQUAL "INFO")
+  add_definitions(-DSPDLOG_ACTIVE_LEVEL=2)
+  message("-- Building of DERIBIT_DEMO_LOG_LEVEL=INFO")
+elseif(DERIBIT_DEMO_LOG_LEVEL STREQUAL "WARN")
+  add_definitions(-DSPDLOG_ACTIVE_LEVEL=3)
+  message("-- Building of DERIBIT_DEMO_LOG_LEVEL=WARN")
+elseif(DERIBIT_DEMO_LOG_LEVEL STREQUAL "ERROR")
+  add_definitions(-DSPDLOG_ACTIVE_LEVEL=4)
+  message("-- Building of DERIBIT_DEMO_LOG_LEVEL=ERROR")
+elseif(DERIBIT_DEMO_LOG_LEVEL STREQUAL "CRITICAL")
+  add_definitions(-DSPDLOG_ACTIVE_LEVEL=5)
+  message("-- Building of DERIBIT_DEMO_LOG_LEVEL=CRITICAL")
+elseif(DERIBIT_DEMO_LOG_LEVEL STREQUAL "OFF")
+  add_definitions(-DSPDLOG_ACTIVE_LEVEL=6)
+  message("-- Building of DERIBIT_DEMO_LOG_LEVEL=OFF")
+else()
+  add_definitions(-DSPDLOG_ACTIVE_LEVEL=0)
+  message("-- Building of DERIBIT_DEMO_LOG_LEVEL=TRACE")
+endif()
+
+target_link_directories(
+  ${PROJECT_NAME} PUBLIC
+  $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
+  $<INSTALL_INTERFACE:${CMAKE_INSTALL_LIBDIR}>
+)
+
+# Set RPATH so sparkd knows where to find our shared libs (e.g., libspark).
+# The dirs are relative to sparkd's path, represented by "$ORIGIN".
+set_target_properties(
+  ${PROJECT_NAME} PROPERTIES
+  BUILD_RPATH "$ORIGIN/../lib_spark;$ORIGIN/../lib_python_interop"
+  INSTALL_RPATH "$ORIGIN/../${CMAKE_INSTALL_LIBDIR}"
+)
+
+include(${CMAKE_SOURCE_DIR}/cmake/SymLink.cmake)
+include(${CMAKE_SOURCE_DIR}/cmake/SymLinkPythonModule.cmake)
+# Py interop
+sym_link_python_module(
+  LIB_NAME python_interop
+  MODULE_NAME python_interop_module
+  PLUGIN_NAME python_interop_plugins
+)
+target_link_libraries(
+  sparkd
+  spark
+  binance
+  okx_execution
+  python_interop
+  python_interop_module
+  databento::databento
+  ${emumba-deribit-connector-library_LIBRARIES}
+  tbb
+  quickfix
+  benchmark
+  Common++
+  Packet++
+  Pcap++
+  lttng-ust
+  -lpcap
+  -static-libgcc
+  -static-libstdc++
+  -pthread)
+
+# Onixs
+target_link_libraries(
+  ${PROJECT_NAME}
+  onixs-cmemdh
+  OnixS.IceImpactMulticastPriceFeedHandlerCpp
+  # gcc doesn't resolve onixs-fix-engine and ACE in runtime path unless we specify
+  # --no-as-needed. These are onixs deps that aren't called directly by our code.
+  # Note: this isn't an issue for clang
+  -Wl,--no-as-needed
+  onixs-fix-engine
+  ACE
+)
+
+# Catch2
+find_package(Catch2 CONFIG REQUIRED)
+target_link_libraries(${PROJECT_NAME} Catch2::Catch2)
+
+install_basic()
+install(
+  FILES
+    "${EXTERNALS_LIB}/libonixs-cmemdh.so"
+    "${EXTERNALS_LIB}/libonixs-cmemdh.so.4"
+    "${EXTERNALS_LIB}/libonixs-cmemdh.so.4.8.0"
+    "${EXTERNALS_LIB}/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.16.0.0"
+    "${EXTERNALS_LIB}/libonixs-fix-engine.so"
+    "${EXTERNALS_LIB}/libonixs-fix-engine.so.4"
+    "${EXTERNALS_LIB}/libonixs-fix-engine.so.4.10.1"
+    "${EXTERNALS_LIB}/libACE-5.7.5.so"
+  DESTINATION ${CMAKE_INSTALL_LIBDIR}
+)
				`@@ -0,0 +1 @@`
				`PYTHONPATH=/workspaces/spark/build/sparkd-debug-clang/src/lib_python_interop;/workspaces/spark/src/lib_python_interop/python_interop_plugins`