cleanup: Remove unnecessary py_module.cpp files

We used to have separate modules for md and execution. I forgot to remove these after unifying them to a single module.
diag: Reduce book log noise in md plugins
2025-06-16 14:31:28 +00:00 · 2025-06-16 14:28:09 +00:00 · 2025-06-16 14:17:45 +00:00 · 2025-06-13 21:26:36 +00:00 · 2025-06-13 20:50:30 +00:00 · 2025-06-13 20:50:30 +00:00
2170 changed files with 20600 additions and 9944 deletions
@@ -34,7 +34,7 @@
 #   src/server/sparkd/Dockerfile
 SPARKD__IMAGE=sparkd
 # The config dir for this service to use
-SPARKD__CONFIG_DIR=src/server/sparkd/configurations/dev
+SPARKD__CONFIG_DIR=src/app_sparkd/configurations/dev
 # Mount a specific volume or dir to "~/sparkdata/data" inside the docker.
 # Defaults to a volume named "sparkd_<config_port>" (e.g., sparkd_8000).
 # SPARKD__DATA_DIR=/home/bts/sparkdata/data
@@ -44,5 +44,6 @@ SPARKD__CONFIG_PORT=8000
 SPARKD__QUICKBUS_PORT=8001
 SPARKD__SIM_FIX_PORT=8002
 SPARKD__SHM_SIZE=4gb
+SPARKD__PYTHON_REQUIREMENTS=requirements.txt

 # TODO: add config for other services (pubsub, md-publisher, webapp, api-server, etc.)
@@ -199,24 +199,27 @@
      "request": "launch",
      "program": "",
      "linux": {
-        "preLaunchTask": "Build Sparkd (Debug)",
-        "program": "${workspaceFolder}/build/sparkd-debug-clang/server/sparkd",
-        "args": [
-          "run"
-        ],
-        "stopAtEntry": false,
-        "cwd": "${workspaceFolder}/build/sparkd-debug-clang/server",
-        "environment": [],
-        "externalConsole": false,
-        "MIMode": "gdb",
-        "setupCommands": [
-          {
-            "description": "Enable pretty-printing for gdb",
-            "text": "-enable-pretty-printing",
-            "ignoreFailures": true
-          }
-        ]
-      }
+        "program": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd/sparkd",
+      },
+      "args": [
+        "run"
+      ],
+      "cwd": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd",
+      "stopAtEntry": false,
+      "externalConsole": false,
+      "MIMode": "gdb",
+      "setupCommands": [
+        {
+          "description": "Enable pretty-printing for gdb",
+          "text": "-enable-pretty-printing",
+          "ignoreFailures": true
+        },
+        {
+          "description": "Set Disassembly Flavor to Intel",
+          "text": "-gdb-set disassembly-flavor intel",
+          "ignoreFailures": true
+        }
+      ]
    },
    {
      "name": "Sparkd (Skip Build)",
@@ -224,74 +227,27 @@
      "request": "launch",
      "program": "",
      "linux": {
-        "program": "${workspaceFolder}/build/sparkd-debug-clang/server/sparkd",
-        "args": [
-          "run"
-        ],
-        "stopAtEntry": false,
-        "cwd": "${workspaceFolder}/build/sparkd-debug-clang/server",
-        "environment": [],
-        "externalConsole": false,
-        "MIMode": "gdb",
-        "setupCommands": [
-          {
-            "description": "Enable pretty-printing for gdb",
-            "text": "-enable-pretty-printing",
-            "ignoreFailures": true
-          }
-        ]
-      }
-    },
-    {
-      "name": "Sparkd Test",
-      "type": "cppdbg",
-      "request": "launch",
-      "program": "",
-      "linux": {
-        "preLaunchTask": "Build Sparkd (Debug)",
-        "program": "${workspaceFolder}/build/sparkd-debug-clang/server/sparkd",
-        "args": [
-          "test",
-          "${input:sparkdTest}",
-        ],
-        "stopAtEntry": false,
-        "cwd": "${workspaceFolder}/build/sparkd-debug-clang/server",
-        "environment": [],
-        "externalConsole": false,
-        "MIMode": "gdb",
-        "setupCommands": [
-          {
-            "description": "Enable pretty-printing for gdb",
-            "text": "-enable-pretty-printing",
-            "ignoreFailures": true
-          }
-        ]
-      }
-    },
-    {
-      "name": "Sparkd Test (Skip Build)",
-      "type": "cppdbg",
-      "request": "launch",
-      "program": "",
-      "linux": {
-        "program": "${workspaceFolder}/build/sparkd-debug-clang/server/sparkd",
-        "args": [
-          "test",
-          "${input:sparkdTest}",
-        ],
-        "stopAtEntry": false,
-        "cwd": "${workspaceFolder}/build/sparkd-debug-clang/server",
-        "environment": [],
-        "externalConsole": false,
-        "MIMode": "gdb",
-        "setupCommands": [
-          {
-            "description": "Enable pretty-printing for gdb",
-            "text": "-enable-pretty-printing",
-            "ignoreFailures": true
-          }
-        ]
-      }
+        "program": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd/sparkd",
+      },
+      "args": [
+        "run"
+      ],
+      "cwd": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd",
+      "stopAtEntry": false,
+      "externalConsole": false,
+      "MIMode": "gdb",
+      "setupCommands": [
+        {
+          "description": "Enable pretty-printing for gdb",
+          "text": "-enable-pretty-printing",
+          "ignoreFailures": true
+        },
+        {
+          "description": "Set Disassembly Flavor to Intel",
+          "text": "-gdb-set disassembly-flavor intel",
+          "ignoreFailures": true
+        }
+      ]
    },
  ],
  "inputs": [
@@ -22,9 +22,9 @@ set(
 # can run simultaneously without fighting over the same shm files (e.g., sparkd,
 # api server, etc.).
 set(SHARED_MEMORY_PREFIX "bts" CACHE STRING "Prefix to use when creating shared memory files")
-set(SPARK_BIN "${EXTERNALS_ROOT}/bin")
-set(SPARK_INCLUDE "${EXTERNALS_ROOT}/include")
-set(SPARK_LIB "${EXTERNALS_ROOT}/lib/$<CONFIG>")
+set(EXTERNALS_BIN "${EXTERNALS_ROOT}/bin")
+set(EXTERNALS_INCLUDE "${EXTERNALS_ROOT}/include")
+set(EXTERNALS_LIB "${EXTERNALS_ROOT}/lib/$<CONFIG>")
 set(BUILD_DOCS OFF CACHE BOOL "Build documentation")
 set(BUILD_SDK OFF CACHE BOOL "Build the sdk")
 set(EXPORT_MD_CONSUMER OFF CACHE BOOL "Export (extern) C functions for MD consumer interop")
@@ -36,8 +36,8 @@ else()
  set(BUILD_ADMIN_WEB_APP ON CACHE BOOL "Build the Admin WebApp")
 endif()

-include_directories(BEFORE SYSTEM ${SPARK_INCLUDE} ${EXTERNALS_ROOT}/include/zlib)
-link_directories(${SPARK_LIB})
+include_directories(BEFORE SYSTEM ${EXTERNALS_INCLUDE} ${EXTERNALS_ROOT}/include/zlib)
+link_directories(${EXTERNALS_LIB})

 add_compile_definitions(SHARED_MEMORY_PREFIX="${SHARED_MEMORY_PREFIX}")
 add_compile_definitions(APP_DATA_DIR="${APP_DATA_DIR}")
@@ -47,12 +47,8 @@ endif()
 add_compile_definitions(EXPORT_MD_CONSUMER="${EXPORT_MD_CONSUMER}")
 if(WIN32)
  # subdirs
-  add_subdirectory(src/lib)
+  add_subdirectory(src/lib_spark)
 else()
-  # make sparkd look for libspark.so in its directory
-  set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE)
-  set(CMAKE_INSTALL_RPATH ".")
-
  if(BUILD_SPARKD)
    # Note: All macros that begin w/ 'CCAPI' are ccapi's macros (defined and/or
    # read by their code). We override some below.
@@ -117,12 +113,15 @@ else()
  endif()

  # subdirs
+  add_subdirectory(src/lib_spark)
  if (BUILD_SPARKD)
+    add_subdirectory(src/lib_python_interop)
+    add_subdirectory(src/lib_onixs_fix)
    add_subdirectory(src/lib_binance_execution)
-    add_subdirectory(src/server/sparkd)
+    add_subdirectory(src/lib_okx_execution)
+    add_subdirectory(src/app_sparkd)
    add_subdirectory(src/data_files)
  endif()
-  add_subdirectory(src/lib)
  if (BUILD_ADMIN_WEB_APP)
    add_subdirectory(src/Spark.Admin.WebApp)
  endif()
@@ -132,37 +131,4 @@ else()
  if(BUILD_SDK)
    add_subdirectory(src/sdk)
  endif()
-
-  if(BUILD_SPARKD)
-    add_custom_command(
-      OUTPUT ${CMAKE_BINARY_DIR}/server
-      COMMAND mkdir -p ${CMAKE_BINARY_DIR}/server
-      COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/src/server/sparkd/sparkd ${CMAKE_BINARY_DIR}/server
-      COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/data_files ${CMAKE_BINARY_DIR}/server
-      COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/src/lib/libspark.so ${CMAKE_BINARY_DIR}/server
-      COMMAND ln ARGS -sf ${EXTERNALS_ROOT}/lib/${CMAKE_BUILD_TYPE}/libonixs-fix-engine.so.4 ${CMAKE_BINARY_DIR}/server
-      
-      COMMAND
-        ln ARGS -sf
-        ${EXTERNALS_ROOT}/lib/Release/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.16.0.0
-        ${CMAKE_BINARY_DIR}/server
-      COMMAND ln ARGS -sf ${EXTERNALS_ROOT}/lib/Release/libACE-5.7.5.so ${CMAKE_BINARY_DIR}/server
-      COMMAND mkdir ARGS -p ${CMAKE_BINARY_DIR}/server/plugins
-      COMMENT "Creating server runtime layout"
-      DEPENDS sparkd data_files_dir)
-
-    add_custom_target(server_layout ALL DEPENDS ${CMAKE_BINARY_DIR}/server)
-
-    install(
-      DIRECTORY "${CMAKE_BINARY_DIR}/server/plugins"
-      DESTINATION "${CMAKE_INSTALL_BINDIR}")
-    install(
-      FILES
-      "${SPARK_LIB}/libACE-5.7.5.so"
-      "${SPARK_LIB}/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.16.0.0"
-      "${SPARK_LIB}/libonixs-fix-engine.so.4.10.1"
-      "${SPARK_LIB}/libonixs-fix-engine.so.4"
-      "${SPARK_LIB}/libonixs-fix-engine.so"
-      DESTINATION "${CMAKE_INSTALL_BINDIR}")
-  endif()
 endif()
@@ -31,9 +31,8 @@
        "CMAKE_SHARED_LINKER_FLAGS_RELEASE": "/OPT:REF /OPT:ICF /LTCG",
        "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
        "EXTERNALS_ROOT": "${sourceDir}/externals/windows",
-        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0",
-        "DOTNET_WINDOWS_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0-windows",
+        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf",
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0;${sourceDir}/build/$<CONFIG>/bin/net8.0-windows",
        "EXPORT_MD_CONSUMER": {
          "type": "BOOL",
          "value": "ON"
@@ -58,11 +57,12 @@
        "APP_DATA_DIR_OVERRIDE": "${dollar}{SPARKD_DATA_DIR}",
        "CMAKE_BUILD_WITH_INSTALL_RPATH": {
          "type": "BOOL",
-          "value": "ON"
+          "value": "OFF"
        },
        "CMAKE_CXX_STANDARD": "20",
        "CMAKE_INSTALL_RPATH": ".",
        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
+        "CMAKE_PREFIX_PATH": "${sourceDir}/cmake/;${sourceDir}/.venv/lib/python3.10/site-packages/pybind11/share/cmake/pybind11",
        "CMAKE_EXE_LINKER_FLAGS": "-Wno-undef",
        "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
        "DATABENTO_USE_EXTERNAL_JSON": {
@@ -93,7 +93,7 @@
          "value": "ON"
        },
        "CCAPI_LOG_LEVEL": "Info",
-        "CONFIG_DEFAULTS_PATH": "config/profiles/md_dev/*.conf;config/profiles/default/*.conf"
+        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/md_dev/*.conf;spark/config/profiles/default/*.conf"
      }
    },

@@ -104,7 +104,7 @@
      "cacheVariables": {
        "CMAKE_BUILD_TYPE": "Release",
        "CCAPI_LOG_LEVEL": "Warn",
-        "CONFIG_DEFAULTS_PATH": "config/profiles/md_prod/*.conf;config/profiles/default/*.conf"
+        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/md_prod/*.conf;spark/config/profiles/default/*.conf"
      }
    },

@@ -186,7 +186,11 @@
        "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
-        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
+        "CMAKE_EXPORT_COMPILE_COMMANDS": {
+          "type": "BOOL",
+          "value": "ON"
+        },
+        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf",
        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0",
        "EXPORT_MD_CONSUMER": {
          "type": "BOOL",
@@ -227,7 +231,7 @@
      "cacheVariables": {
        "APP_DATA_DIR": "${dollar}{APPDATA}/bts/DataStream",
        "APP_DATA_DIR_OVERRIDE": "${dollar}{DATA_STREAM_DATA_DIR}",
-        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf",
+        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf;spark/config/profiles/data_stream/*.conf",
        "SHARED_MEMORY_PREFIX": "data_stream"
      }
    },
@@ -262,8 +266,7 @@
        "CMAKE_CXX_COMPILER": "clang++",
        "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
-        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
-        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf",
+        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf;spark/config/profiles/data_stream/*.conf",
        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
        "EXPORT_MD_CONSUMER": {
          "type": "BOOL",
@@ -0,0 +1,7 @@
+include_guard(GLOBAL)
+
+macro(export_to_parent)
+  foreach(var IN ITEMS ${ARGN})
+    set(${var} ${${var}} PARENT_SCOPE)
+  endforeach()
+endmacro()
@@ -0,0 +1,142 @@
+include_guard(DIRECTORY)
+include(${CMAKE_SOURCE_DIR}/cmake/ExportToParent.cmake)
+include(${CMAKE_SOURCE_DIR}/cmake/InstallBasic.cmake)
+include(${CMAKE_SOURCE_DIR}/cmake/ProtobufGenerateCpp.cmake)
+include(${CMAKE_SOURCE_DIR}/cmake/SetFirstDefined.cmake)
+include(${CMAKE_SOURCE_DIR}/cmake/safe_dep_glob.cmake)
+
+function(init_library)
+  set(_options SHARED STATIC)
+  set(_one_value_args
+    NAME
+    SOURCE_DIR
+    INCLUDE_DIR
+  )
+  set(_multi_value_args)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+  if (_ARGS_SHARED AND _ARGS_STATIC)
+    message(FATAL_ERROR "Invalid args passed to init_library: cannot specify STATIC and SHARED")
+  endif()
+  set_first_defined(_name _ARGS_NAME PROJECT_NAME)
+  set_first_defined(_source_dir _ARGS_SOURCE_DIR _name)
+  set_first_defined(_include_dir _ARGS_INCLUDE_DIR _source_dir)
+
+  # Cpp sources
+  file(
+    GLOB_RECURSE ${_name}_SOURCES
+    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
+    ${_source_dir}/*.cpp
+  )
+  # Note: cpp files inside 'tests' dir(s) are considered tests.
+  # Remove tests from main sources, and add them to test sources
+  foreach(item ${${_name}_SOURCES})
+    if(${item} MATCHES "tests/.*\.cpp$")
+      list(APPEND ${_name}_TEST_SOURCES ${item})
+    endif()
+  endforeach()
+  foreach(item ${${_name}_TEST_SOURCES})
+    list(REMOVE_ITEM ${_name}_SOURCES ${item})
+  endforeach()
+
+  # Proto sources 
+  file(
+    GLOB_RECURSE ${_name}_PROTO_SOURCES
+    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
+    ${_source_dir}/protobufs/*.proto
+  )
+  update_deps_file("${${_name}_PROTO_SOURCES};")
+  protobuf_generate_cpp(
+    CXX_SOURCES ${_name}_CXX_PROTO_SOURCES
+    CXX_HEADERS ${_name}_CXX_PROTO_HEADERS
+    CXX_OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}
+    PROTO_INCLUDE_DIRS
+      ${EXTERNALS_INCLUDE}
+      ${CMAKE_CURRENT_SOURCE_DIR}
+    PROTO_SOURCES
+      ${${_name}_PROTO_SOURCES}
+  )
+
+  export_to_parent(
+    ${_name}_SOURCES
+    ${_name}_TEST_SOURCES
+    ${_name}_PROTO_SOURCES
+    ${_name}_CXX_PROTO_SOURCES
+    ${_name}_CXX_PROTO_HEADERS
+  )
+
+  # Add main lib
+  if (_ARGS_STATIC)
+    add_library(${_name} STATIC ${${_name}_SOURCES} ${${_name}_CXX_PROTO_SOURCES})
+  elseif(_ARGS_SHARED)
+    add_library(${_name} SHARED ${${_name}_SOURCES} ${${_name}_CXX_PROTO_SOURCES})
+  else()
+    add_library(${_name} ${${_name}_SOURCES} ${${_name}_CXX_PROTO_SOURCES})
+  endif()
+  target_include_directories(
+    ${_name}
+    SYSTEM
+    BEFORE
+    PUBLIC
+      $<BUILD_INTERFACE:${EXTERNALS_ROOT}/include>
+      $<BUILD_INTERFACE:${EXTERNALS_ROOT}/include/zlib>
+  )
+  target_include_directories(
+    ${_name}
+    PUBLIC
+      $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
+      $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>
+  )
+  # Include protos (if any)
+  if (${_name}_PROTO_SOURCES)
+    target_include_directories(
+      ${_name}
+      PUBLIC
+        $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
+    )
+  endif()
+  target_link_directories(
+    ${_name}
+    PUBLIC
+      $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
+      $<BUILD_INTERFACE:${EXTERNALS_ROOT}/lib/$<CONFIG>>
+      $<INSTALL_INTERFACE:${CMAKE_INSTALL_LIBDIR}>
+  )
+  # Install main lib
+  foreach(_hdr ${${_name}_CXX_PROTO_HEADERS})
+    file(RELATIVE_PATH _relative_path ${CMAKE_CURRENT_BINARY_DIR} ${_hdr})
+    install(FILES ${_hdr} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/${_relative_path})
+  endforeach()
+  install_basic(
+    NAME ${_name}
+    INCLUDE_DIR ${_include_dir}
+  )
+
+  # Add tests executable (if there are any)
+  if (${_name}_TEST_SOURCES AND NOT WIN32)
+    add_executable(${_name}-tests ${${_name}_TEST_SOURCES})
+
+    get_target_property(_lib_type ${_name} TYPE)
+    if (_lib_type STREQUAL "SHARED_LIBRARY")
+      # Set RPATH so the tests exe knows where to find the shared lib.
+      set_target_properties(
+        ${_name}-tests
+        PROPERTIES
+          BUILD_RPATH "$ORIGIN"
+          INSTALL_RPATH "$ORIGIN/../${CMAKE_INSTALL_LIBDIR}"
+      )
+    endif()
+
+    # Link main lib
+    target_link_directories(
+      ${_name}-tests
+      PRIVATE
+        $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
+        $<INSTALL_INTERFACE:${CMAKE_INSTALL_LIBDIR}>
+    )
+    target_link_libraries(${_name}-tests PRIVATE ${_name})
+
+    # Catch2
+    find_package(Catch2 CONFIG REQUIRED COMPONENTS Catch2WithMain)
+    target_link_libraries(${_name}-tests PRIVATE Catch2::Catch2WithMain)
+  endif()
+endfunction()
@@ -0,0 +1,71 @@
+include_guard(DIRECTORY)
+include(GNUInstallDirs)
+include(${CMAKE_SOURCE_DIR}/cmake/ExportToParent.cmake)
+include(${CMAKE_SOURCE_DIR}/cmake/SetFirstDefined.cmake)
+
+function(init_python_module)
+  set(_options)
+  set(_one_value_args
+    NAME
+  )
+  set(_multi_value_args
+    SOURCES
+  )
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+  set_first_defined(_name _ARGS_NAME PROJECT_NAME)
+  set_first_defined(_sources _ARGS_SOURCES ${_name}_SOURCES)
+
+  set(module_proj ${_name}_module)
+  set(${module_proj}_SOURCES ${_sources})
+  pybind11_add_module(${module_proj} SHARED ${_sources})
+  target_link_libraries(
+    ${module_proj}
+    PUBLIC
+      pybind11::embed
+      spark
+      ${_name}
+  )
+  target_include_directories(
+    ${module_proj}
+    PRIVATE
+      $<BUILD_INTERFACE:${pybind11_INCLUDE_DIRS}>
+  )
+  target_include_directories(
+    ${module_proj}
+    PUBLIC
+      ${Python3_INCLUDE_DIRS}
+      $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
+      $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>
+  )
+  add_custom_command(
+    OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/stubs/${_name}.pyi
+    DEPENDS ${_name} ${module_proj}
+    COMMAND PYTHONPATH=${CMAKE_CURRENT_BINARY_DIR} pybind11-stubgen --numpy-array-use-type-var --ignore-invalid-expressions '.*' ${_name}_module
+    COMMENT "Generating py stubs"
+  )
+  add_custom_target(
+    "${_name}_py_stubs"
+    ALL
+    DEPENDS
+      ${CMAKE_CURRENT_BINARY_DIR}/stubs/${_name}.pyi
+  )
+  install(
+    TARGETS ${module_proj}
+    EXPORT ${module_proj}Targets
+    RUNTIME
+      DESTINATION ${CMAKE_INSTALL_BINDIR}
+    LIBRARY
+      DESTINATION ${CMAKE_INSTALL_LIBDIR}
+    ARCHIVE
+      DESTINATION ${CMAKE_INSTALL_LIBDIR}
+  )
+  foreach(dir ${pybind11_INCLUDE_DIRS})
+    install(DIRECTORY ${dir} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
+  endforeach()
+  install(
+    EXPORT ${module_proj}Targets
+    FILE ${module_proj}Targets.cmake
+    NAMESPACE ${_name}::
+    DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name}
+  )
+endfunction()
@@ -0,0 +1,65 @@
+include_guard(DIRECTORY)
+include(GNUInstallDirs)
+
+function(init_python_plugin)
+  set(_options)
+  set(_one_value_args NAME PLUGIN_NAME)
+  set(_multi_value_args SOURCES)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+  set_first_defined(_name _ARGS_NAME PROJECT_NAME)
+  set(_plugin_name ${_ARGS_PLUGIN_NAME})
+  if (NOT DEFINED _ARGS_PLUGIN_NAME)
+    set(_plugin_name ${_name}_plugins)
+  endif()
+  set(${_plugin_name}_SOURCES ${_ARGS_SOURCES})
+  if (NOT DEFINED _ARGS_SOURCES)
+    file(
+      GLOB_RECURSE ${_plugin_name}_SOURCES
+      RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
+      ${_plugin_name}/*.py)
+  endif()
+  set(src_outs)
+  foreach(_src ${${_plugin_name}_SOURCES})
+    set(_src_in ${CMAKE_CURRENT_SOURCE_DIR}/${_src})
+    set(_src_out ${CMAKE_CURRENT_BINARY_DIR}/${_src})
+    add_custom_command(
+      OUTPUT ${_src_out}
+      COMMAND ${CMAKE_COMMAND} -E copy ${_src_in} ${_src_out}
+      DEPENDS ${_src_in}
+      COMMENT "Copying ${_plugin_name} plugin ${_src} to ${_src_out}"
+    )
+    list(APPEND src_outs ${_src_out})
+  endforeach()
+  
+  add_custom_target(copy_${_plugin_name} ALL DEPENDS ${src_outs})
+  
+  add_library(${_plugin_name} INTERFACE)
+  add_library(${_name}::${_plugin_name} ALIAS ${_plugin_name})
+  
+  set_target_properties(
+    ${_plugin_name}
+    PROPERTIES
+      INTERFACE_PYTHON_SCRIPTS_DIR
+        $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/${_plugin_name}>$<INSTALL_INTERFACE:${CMAKE_INSTALL_BINDIR}/${_plugin_name}>
+  )
+  
+  set_target_properties(
+    ${_plugin_name}
+    PROPERTIES
+    INTERFACE_PYTHON_SCRIPTS $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/${${_plugin_name}_SOURCES}>$<INSTALL_INTERFACE:${CMAKE_INSTALL_BINDIR}/${${_plugin_name}_SOURCES}>
+  )
+  
+  install(
+    TARGETS ${_plugin_name}
+    EXPORT ${_plugin_name}Targets
+    DESTINATION ${CMAKE_INSTALL_BINDIR}
+  )
+  
+  install(
+    EXPORT ${_plugin_name}Targets
+    FILE ${_plugin_name}Targets.cmake
+    NAMESPACE ${_name}::
+    DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name}
+  )
+  install(DIRECTORY ${_plugin_name} DESTINATION ${CMAKE_INSTALL_BINDIR})
+endfunction()
@@ -0,0 +1,39 @@
+include_guard(DIRECTORY)
+include(GNUInstallDirs)
+include(${CMAKE_SOURCE_DIR}/cmake/PackBasic.cmake)
+include(${CMAKE_SOURCE_DIR}/cmake/SetFirstDefined.cmake)
+
+function(install_basic)
+  set(_options)
+  set(_one_value_args NAME INCLUDE_DIR)
+  set(_multi_value_args)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+  set_first_defined(_name _ARGS_NAME PROJECT_NAME)
+  set_first_defined(_include_dir _ARGS_INCLUDE_DIR _name)
+
+  file(
+    GLOB_RECURSE ${_name}_HEADERS
+    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
+    ${_include_dir}/*.h)
+
+  target_sources(${_name}
+    PUBLIC
+      FILE_SET HEADERS FILES ${${_name}_HEADERS})
+
+  install(
+    TARGETS ${_name}
+    EXPORT ${_name}Targets
+    RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
+    LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
+    ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
+    FILE_SET HEADERS DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}
+  )
+
+  install(
+    EXPORT ${_name}Targets
+    FILE ${_name}Targets.cmake
+    NAMESPACE ${_name}::
+    DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name})
+
+  pack_basic()
+endfunction()
@@ -0,0 +1,44 @@
+include_guard(DIRECTORY)
+include(CMakePackageConfigHelpers)
+include(${CMAKE_SOURCE_DIR}/cmake/SetFirstDefined.cmake)
+
+function(pack_basic)
+  set(_options)
+  set(_one_value_args NAME)
+  set(_multi_value_args)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+  set_first_defined(_name _ARGS_NAME PROJECT_NAME)
+
+  find_package(Git QUIET)
+
+  if(GIT_FOUND)
+    execute_process(
+      COMMAND ${GIT_EXECUTABLE} describe --tags --always --dirty
+      WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
+      OUTPUT_VARIABLE PROJECT_VERSION
+      OUTPUT_STRIP_TRAILING_WHITESPACE
+    )
+  endif()
+
+  if(NOT GIT_FOUND OR PROJECT_VERSION STREQUAL "")
+    set(PROJECT_VERSION "0.0.0")
+  endif()
+
+  write_basic_package_version_file(
+    "${CMAKE_CURRENT_BINARY_DIR}/${_name}ConfigVersion.cmake"
+    VERSION ${PROJECT_VERSION}
+    COMPATIBILITY AnyNewerVersion
+  )
+
+  configure_package_config_file(
+    "${CMAKE_CURRENT_SOURCE_DIR}/Config.cmake.in"
+    "${CMAKE_CURRENT_BINARY_DIR}/${_name}Config.cmake"
+    INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name}
+  )
+
+  install(
+    FILES
+      "${CMAKE_CURRENT_BINARY_DIR}/${_name}Config.cmake"
+      "${CMAKE_CURRENT_BINARY_DIR}/${_name}ConfigVersion.cmake"
+    DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name})
+endfunction()
@@ -1,12 +1,23 @@
-set(Protobuf_PROTOC_EXECUTABLE "${SPARK_BIN}/protoc")
-set(PROTOBUF_INCLUDE_DIRS "${SPARK_INCLUDE}")
+set(Protobuf_PROTOC_EXECUTABLE "${EXTERNALS_BIN}/protoc")
+set(PROTOBUF_INCLUDE_DIRS "${EXTERNALS_INCLUDE}")
 set(PROTOBUF_STATIC_LIB "libprotobuf")
+set(_protobuf_src_dir ${CMAKE_SOURCE_DIR}/src/lib_spark)
+set(_protobuf_bin_dir ${CMAKE_BINARY_DIR}/src/lib_spark)

-function(Protobuf_Generate_Cpp SRCS HDRS)
-  set(${SRCS})
-  set(${HDRS})
+function(Protobuf_Generate_Cpp)
+  set(_options)
+  set(_one_value_args CXX_SOURCES CXX_HEADERS CXX_OUT_DIR)
+  set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+  set(${_ARGS_CXX_SOURCES})
+  set(${_ARGS_CXX_HEADERS})

-  foreach(FIL ${ARGN})
+  set(_proto_include_cli_args)
+  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
+    list(APPEND _proto_include_cli_args -I ${_dir})
+  endforeach()
+
+  foreach(FIL ${_ARGS_PROTO_SOURCES})
    get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
    get_filename_component(FIL_WE ${FIL} NAME_WE)
    get_filename_component(FIL_DIR ${FIL} DIRECTORY)
@@ -14,48 +25,56 @@ function(Protobuf_Generate_Cpp SRCS HDRS)
      set(FIL_WE "${FIL_DIR}/${FIL_WE}")
    endif()

-    list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc")
-    list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h")
+    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc")
+    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h")

    add_custom_command(
-      OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc"
-             "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h"
+      OUTPUT "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
+             "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
      COMMAND
-        ${Protobuf_PROTOC_EXECUTABLE} ARGS --cpp_out=${CMAKE_BINARY_DIR}/src -I
-        ${PROTOBUF_INCLUDE_DIRS} -I ${CMAKE_SOURCE_DIR}/src ${ABS_FIL}
+        ${Protobuf_PROTOC_EXECUTABLE} ARGS --cpp_out=${_ARGS_CXX_OUT_DIR} ${_proto_include_cli_args} ${ABS_FIL}
      DEPENDS ${ABS_FIL}
      COMMENT "Running C++ protocol buffer compiler on ${FIL}"
      VERBATIM)
  endforeach()

-  set_source_files_properties(${${SRCS}} ${${HDRS}} PROPERTIES GENERATED TRUE)
+  set_source_files_properties(${${_ARGS_CXX_SOURCES}} ${${_ARGS_CXX_HEADERS}} PROPERTIES GENERATED TRUE)

  # remove some warnings from protobuf generated files
  if(WIN32)
-    set_source_files_properties(${${SRCS}} PROPERTIES COMPILE_FLAGS
+    set_source_files_properties(${${_ARGS_CXX_SOURCES}} PROPERTIES COMPILE_FLAGS
                                                      "/wd4309 /wd4838")
  else()
    set_source_files_properties(
-      ${${SRCS}}
+      ${${_ARGS_CXX_SOURCES}}
      PROPERTIES
        COMPILE_FLAGS
        "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
    )
  endif()

-  set(${SRCS}
-      ${${SRCS}}
+  set(${_ARGS_CXX_SOURCES}
+      ${${_ARGS_CXX_SOURCES}}
      PARENT_SCOPE)
-  set(${HDRS}
-      ${${HDRS}}
+  set(${_ARGS_CXX_HEADERS}
+      ${${_ARGS_CXX_HEADERS}}
      PARENT_SCOPE)
 endfunction()

-function(Service_Compiler_Generate_Cpp SRCS HDRS)
-  set(${SRCS})
-  set(${HDRS})
+function(Service_Compiler_Generate_Cpp)
+  set(_options)
+  set(_one_value_args CXX_SOURCES CXX_HEADERS CXX_OUT_DIR)
+  set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+  set(${_ARGS_CXX_SOURCES})
+  set(${_ARGS_CXX_HEADERS})

-  foreach(FIL ${ARGN})
+  set(_proto_include_cli_args)
+  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
+    list(APPEND _proto_include_cli_args -I ${_dir})
+  endforeach()
+
+  foreach(FIL ${_ARGS_PROTO_SOURCES})
    get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
    get_filename_component(FIL_WE ${FIL} NAME_WE)
    get_filename_component(FIL_DIR ${FIL} DIRECTORY)
@@ -63,58 +82,58 @@ function(Service_Compiler_Generate_Cpp SRCS HDRS)
      set(FIL_WE "${FIL_DIR}/${FIL_WE}")
    endif()

-    list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc")
-    list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h")
+    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc")
+    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h")

-    list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.cc")
-    list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.h")
+    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc")
+    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h")

-    list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.cc")
-    list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.h")
+    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc")
+    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h")

    set(TEMPLATES
-        ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/templates/cpp/client.pb.cc
-        ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/templates/cpp/client.pb.h
-        ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/templates/cpp/server.pb.cc
-        ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/templates/cpp/server.pb.h
+        ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/client.pb.cc
+        ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/client.pb.h
+        ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/server.pb.cc
+        ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/server.pb.h
    )

+    set(_plugin_py ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/plugin.py)
    if(WIN32)
+      set(_plugin_cmd ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/plugin.cmd)
      add_custom_command(
-        OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc"
-               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h"
-               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.cc"
-               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.h"
-               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.cc"
-               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.h"
+        OUTPUT "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h"
        COMMAND SET PYTHONPATH=${SPARK_PYTHON}
        COMMAND ${CMAKE_SOURCE_DIR}/.venv/Scripts/activate.bat
        COMMAND
-          ${Protobuf_PROTOC_EXECUTABLE} ARGS --cpp_out=${CMAKE_BINARY_DIR}/src
-          --plugin=protoc-gen-service=${CMAKE_SOURCE_DIR}/src/lib/api/service_compiler/plugin.cmd
-          --service_out=lang=cpp:${CMAKE_BINARY_DIR}/src -I
-          ${PROTOBUF_INCLUDE_DIRS} -I ${CMAKE_SOURCE_DIR}/src ${ABS_FIL}
-        DEPENDS ${ABS_FIL} ${TEMPLATES}
-                ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/plugin.py
+          ${Protobuf_PROTOC_EXECUTABLE} ARGS --cpp_out=${_ARGS_CXX_OUT_DIR}
+          --plugin=protoc-gen-service=${_plugin_cmd}
+          --service_out=lang=cpp:${_ARGS_CXX_OUT_DIR} -I
+          ${PROTOBUF_INCLUDE_DIRS} -I ${_protobuf_src_dir} ${ABS_FIL}
+        DEPENDS ${ABS_FIL} ${TEMPLATES} ${_plugin_cmd} ${_plugin_py}
        COMMENT "Running C++ service compiler on ${FIL}"
        VERBATIM)
    else()
      add_custom_command(
-        OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc"
-               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h"
-               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.cc"
-               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.h"
-               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.cc"
-               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.h"
+        OUTPUT "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h"
        COMMAND ${CMAKE_SOURCE_DIR}/tools/venv_exec.sh
        ARGS
          ${Protobuf_PROTOC_EXECUTABLE}
-          --cpp_out=${CMAKE_BINARY_DIR}/src
-          --plugin=protoc-gen-service=${CMAKE_SOURCE_DIR}/src/lib/api/service_compiler/plugin.py
-          --service_out=lang=cpp,make_file_name=${ABS_FIL}:${CMAKE_BINARY_DIR}/src
-          -I ${PROTOBUF_INCLUDE_DIRS} -I ${CMAKE_SOURCE_DIR}/src ${ABS_FIL}
-        DEPENDS ${ABS_FIL} ${TEMPLATES}
-                ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/plugin.py
+          --cpp_out=${_ARGS_CXX_OUT_DIR}
+          --plugin=protoc-gen-service=${_plugin_py}
+          --service_out=lang=cpp,make_file_name=${ABS_FIL}:${_ARGS_CXX_OUT_DIR}
+          ${_proto_include_cli_args} ${ABS_FIL}
+        DEPENDS ${ABS_FIL} ${TEMPLATES} ${_plugin_py}
        COMMENT "Running C++ service compiler on ${FIL}"
        VERBATIM)
    endif()
@@ -123,22 +142,22 @@ function(Service_Compiler_Generate_Cpp SRCS HDRS)

  # remove some warnings from protobuf generated files
  if(WIN32)
-    set_source_files_properties(${${SRCS}} PROPERTIES COMPILE_FLAGS
+    set_source_files_properties(${${_ARGS_CXX_SOURCES}} PROPERTIES COMPILE_FLAGS
                                                      "/wd4309 /wd4838")
  else()
    set_source_files_properties(
-      ${${SRCS}}
+      ${${_ARGS_CXX_SOURCES}}
      PROPERTIES
        COMPILE_FLAGS
        "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
    )
  endif()

-  set(${SRCS}
-      ${${SRCS}}
+  set(${_ARGS_CXX_SOURCES}
+      ${${_ARGS_CXX_SOURCES}}
      PARENT_SCOPE)
-  set(${HDRS}
-      ${${HDRS}}
+  set(${_ARGS_CXX_HEADERS}
+      ${${_ARGS_CXX_HEADERS}}
      PARENT_SCOPE)

 endfunction()
@@ -0,0 +1,83 @@
+include_guard(GLOBAL)
+
+function(protobuf_generate_cpp)
+  set(_options)
+  set(_one_value_args
+    CXX_SOURCES
+    CXX_HEADERS
+    CXX_OUT_DIR
+    PROTOC
+  )
+  set(_multi_value_args
+    PROTO_INCLUDE_DIRS
+    PROTO_SOURCES
+  )
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+
+  if (NOT DEFINED _ARGS_PROTOC)
+    if (WIN32)
+      set(_ARGS_PROTOC "${EXTERNALS_BIN}/protoc.exe")
+    else()
+      set(_ARGS_PROTOC "${EXTERNALS_BIN}/protoc")
+    endif()
+  endif()
+  set(${_ARGS_CXX_SOURCES})
+  set(${_ARGS_CXX_HEADERS})
+
+  set(_proto_include_cli_args)
+  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
+    list(APPEND _proto_include_cli_args -I ${_dir})
+  endforeach()
+
+  foreach(_fil ${_ARGS_PROTO_SOURCES})
+    get_filename_component(_abs_fil ${_fil} ABSOLUTE)
+    get_filename_component(_fil_we ${_fil} NAME_WE)
+    get_filename_component(_fil_dir ${_fil} DIRECTORY)
+    if(_fil_dir)
+      set(_fil_we "${_fil_dir}/${_fil_we}")
+    endif()
+    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.cc")
+    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.h")
+    message(STATUS "add_custom_command for ${_fil} -> ${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.h")
+    # TODO: Pass proto sources by relative path (so this fn can be called from
+    # arbitrary subdirs).
+    add_custom_command(
+      OUTPUT
+        "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.cc"
+        "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.h"
+      COMMAND
+        ${_ARGS_PROTOC}
+      ARGS
+        --cpp_out=${_ARGS_CXX_OUT_DIR} ${_proto_include_cli_args} ${_abs_fil}
+      DEPENDS ${_abs_fil}
+      COMMENT "Running C++ protocol buffer compiler on ${_fil}"
+      VERBATIM
+    )
+  endforeach()
+
+  set_source_files_properties(
+    ${${_ARGS_CXX_SOURCES}}
+    ${${_ARGS_CXX_HEADERS}}
+    PROPERTIES
+      GENERATED TRUE
+  )
+
+  # remove some warnings from protobuf generated files
+  if(WIN32)
+    set_source_files_properties(
+      ${${_ARGS_CXX_SOURCES}}
+      PROPERTIES
+        COMPILE_FLAGS "/wd4309 /wd4838"
+    )
+  else()
+    set_source_files_properties(
+      ${${_ARGS_CXX_SOURCES}}
+      PROPERTIES
+        COMPILE_FLAGS "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
+    )
+  endif()
+
+  set(${_ARGS_CXX_SOURCES} ${${_ARGS_CXX_SOURCES}} PARENT_SCOPE)
+  set(${_ARGS_CXX_HEADERS} ${${_ARGS_CXX_HEADERS}} PARENT_SCOPE)
+endfunction()
+
@@ -0,0 +1,61 @@
+include_guard(GLOBAL)
+
+function(protobuf_generate_cpp_config)
+  set(_options)
+  set(_one_value_args
+    PROTO_SOURCES_HEADER
+    PROTO_DEFAULTS_HEADER
+  )
+  set(_multi_value_args
+    PROTO_SOURCES
+    PROTO_DEFAULTS
+  )
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+
+  # Configure the 'config_proto_files' header. It provides an array of strings populated with the
+  # files in PROTO_SOURCES. At runtime, we use this array to generate a special 'config' proto
+  # message with fields for each message defined in said files. A global instance of this 'config'
+  # proto is then constructed, and populated with [compile-time] defaults, and [runtime] overrides
+  # (i.e., in spark.conf). Calls to `bts::config::get<T>()` retrieve the value from the field
+  # associated with type `T` in this global instance.
+  message(STATUS "PROTO_SOURCES: ${_ARGS_PROTO_SOURCES}")
+  message(STATUS "PROTO_SOURCES_HEADER: ${_ARGS_PROTO_SOURCES_HEADER}")
+  message(STATUS "PROTO_DEFAULTS_HEADER: ${_ARGS_PROTO_DEFAULTS_HEADER}")
+  if (_ARGS_PROTO_SOURCES_HEADER)
+    set(PROTO_SOURCES)
+    set(PROTO_SOURCES_CXX_ARRAY_ITEMS "")
+    foreach(_proto_path ${_ARGS_PROTO_SOURCES})
+      list(APPEND PROTO_SOURCES ${_proto_path})
+      string(APPEND PROTO_SOURCES_CXX_ARRAY_ITEMS "\"${_proto_path}\",\n")
+    endforeach()
+    list(LENGTH PROTO_SOURCES PROTO_SOURCES_LENGTH)
+    message(STATUS "CONFIGURING PROTO_SOURCES_HEADER): ${_ARGS_PROTO_SOURCES_HEADER}")
+    configure_file(${CMAKE_SOURCE_DIR}/cmake/config_proto_files.h.in ${_ARGS_PROTO_SOURCES_HEADER})
+  endif()
+
+  # Generate static defaults. Ideally, this would be a single string, but we use an array because
+  # the compiler complains if the string is longer than 65k.
+  set(_cfg_defaults_str "inline const char * config_static_defaults[] = { ")
+
+  foreach(_cfg_file_path ${_ARGS_PROTO_DEFAULTS})
+    file(READ ${_cfg_file_path} _cfg_file_content)
+    string(REPLACE "\"" "\\\"" _cfg_file_content_escaped ${_cfg_file_content})
+    string(REPLACE "\n" ";" _cfg_lines ${_cfg_file_content_escaped})
+
+    foreach(_line ${_cfg_lines})
+      string(APPEND _cfg_defaults_str "\n  \"${_line}\",")
+    endforeach(_line)
+  endforeach(_cfg_file_path)
+
+  string(APPEND _cfg_defaults_str "};")
+  set(_old_cfg_defaults_str "")
+
+  if(EXISTS ${_ARGS_PROTO_DEFAULTS_HEADER})
+    file(READ ${_ARGS_PROTO_DEFAULTS_HEADER} _old_cfg_defaults_str)
+  endif()
+
+  if(NOT _cfg_defaults_str STREQUAL _old_cfg_defaults_str)
+    file(WRITE "${_ARGS_PROTO_DEFAULTS_HEADER}" "${_cfg_defaults_str}")
+  endif()
+endfunction()
+
@@ -10,7 +10,7 @@ function(protobuf_generate_fs)
    set(_ARGS_TARGET_NAME "${PROJECT_NAME}_generate_fs_protos")
  endif()
  if (NOT DEFINED _ARGS_PROTOC)
-    set(_ARGS_PROTOC "${SPARK_BIN}/protoc")
+    set(_ARGS_PROTOC "${EXTERNALS_BIN}/protoc")
  endif()

  set(_proto_include_cli_args)
@@ -27,8 +27,8 @@ function(protobuf_generate_fs)
  endforeach()

  set(_venv_exec ${CMAKE_SOURCE_DIR}/tools/venv_exec.sh)
-  set(_plugin_py ${CMAKE_SOURCE_DIR}/src/lib/api/service_compiler/plugin.py)
-  set(_template_fs ${CMAKE_SOURCE_DIR}/src/lib/api/service_compiler/templates/fs/fable.fs)
+  set(_plugin_py ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/plugin.py)
+  set(_template_fs ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/fs/fable.fs)
  add_custom_command(
    OUTPUT ${_outputs}
    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
@@ -0,0 +1,10 @@
+include_guard(GLOBAL)
+
+macro(set_first_defined RESULT)
+  foreach(_arg ${ARGN})
+    if (DEFINED ${_arg})
+      set(${RESULT} ${${_arg}})
+      break()
+    endif()
+  endforeach()
+endmacro()
@@ -0,0 +1,39 @@
+include_guard(DIRECTORY)
+include(${CMAKE_SOURCE_DIR}/cmake/SymLink.cmake)
+
+function(sym_link_python_module)
+  set(_options)
+  set(_one_value_args LIB_NAME MODULE_NAME PLUGIN_NAME)
+  set(_multi_value_args)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}"
+                        "${_multi_value_args}" ${ARGN})
+  if(NOT DEFINED _ARGS_LIB_NAME)
+    message(
+      FATAL_ERROR "LIB_NAME must be specified for sym_link_python_module()")
+  endif()
+  set(_lib_name ${_ARGS_LIB_NAME})
+  set(_module_name ${_ARGS_MODULE_NAME})
+  if(NOT DEFINED _ARGS_MODULE_NAME)
+    set(_module_name ${_lib_name}_module)
+  endif()
+  set(_plugin_name ${_ARGS_PLUGIN_NAME})
+  set(_deps ${_lib_name} ${_module_name})
+  if(DEFINED _ARGS_PLUGIN_NAME)
+    list(APPEND _deps ${_plugin_name} copy_${_plugin_name})
+  endif()
+  add_dependencies(${PROJECT_NAME} ${_deps})
+  sym_link_local_lib(${_lib_name})
+  set(_module_suffix cpython-310-x86_64-linux-gnu.so)
+  sym_link_file(
+    SOURCE
+    ${CMAKE_BINARY_DIR}/src/lib_${_lib_name}/${_module_name}.${_module_suffix}
+    DEST
+    ${CMAKE_CURRENT_BINARY_DIR}/${_module_name}.${_module_suffix}
+    INSTALL_DEST
+    ${CMAKE_INSTALL_BINDIR}/)
+  if(DEFINED _ARGS_PLUGIN_NAME)
+    get_target_property(plugins ${_plugin_name} INTERFACE_PYTHON_SCRIPTS_DIR)
+    sym_link_dir(TARGET_NAME ${PROJECT_NAME}_link_${_plugin_name} SOURCE
+                 ${plugins} DEST ${CMAKE_CURRENT_BINARY_DIR}/${_plugin_name})
+  endif()
+endfunction()
@@ -0,0 +1,11 @@
+#pragma once
+
+#include <array>
+
+namespace bts::@PROJECT_NAME@::protobufs {
+
+inline std::array<const char *, @PROTO_SOURCES_LENGTH@> config_proto_files = {
+@PROTO_SOURCES_CXX_ARRAY_ITEMS@
+};
+
+} // namespace bts::@PROJECT_NAME@::protobufs
@@ -1,24 +0,0 @@
-cmake_minimum_required(VERSION 3.5)
-
-# Explicitly set policy to convert relative paths to absolute (although this is a default policy,
-# setting it explicitly prevents a compile-warning)
-cmake_policy(SET CMP0076 NEW)
-
-# add protos to the library to facilitate source_group
-target_sources(libspark PUBLIC ${Config_Protos} ${NonConfig_Protos} ${Service_Protos})
-
-function(assign_source_group)
-  foreach(_source IN ITEMS ${ARGN})
-    if(IS_ABSOLUTE "${_source}")
-      file(RELATIVE_PATH _source_rel "${CMAKE_CURRENT_SOURCE_DIR}" "${_source}")
-    else()
-      set(_source_rel "${_source}")
-    endif()
-    get_filename_component(_source_path "${_source_rel}" PATH)
-    string(REPLACE "/" "\\" _source_path_msvc "${_source_path}")
-    source_group("${_source_path_msvc}" FILES "${_source}")
-    set(SOURCE_FILES "${_source}")
-  endforeach()
-endfunction(assign_source_group)
-
-assign_source_group(${SparkLib_Sources} ${Config_Protos} ${NonConfig_Protos} ${Service_Protos})
@@ -3,11 +3,12 @@ services:
    volumes:
      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/sparkdata/data
    build:
-      dockerfile: src/server/sparkd/Dockerfile
+      dockerfile: src/app_sparkd/Dockerfile
      args:
        - IMAGE=${SPARKD__IMAGE}
        - SOURCE_DIR=${SPARKD__SOURCE_DIR}
        - CONFIG_DIR=${SPARKD__CONFIG_DIR}
+        - PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
    shm_size: "${SPARKD__SHM_SIZE:-512mb}"
    command: >
      sh -c "
@@ -39,11 +40,12 @@ services:
      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/sparkdata/data
      - ${SPARKD__RUNTIME_CONF_DIR:-sparkd_conf}:/home/bts/sparkdata/conf
    build:
-      dockerfile: src/server/sparkd/Dockerfile
+      dockerfile: src/app_sparkd/Dockerfile
      args:
        - IMAGE=${SPARKD__IMAGE}
        - SOURCE_DIR=${SPARKD__SOURCE_DIR}
        - CONFIG_DIR=${SPARKD__CONFIG_DIR}
+        - PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
    shm_size: "${SPARKD__SHM_SIZE:-512mb}"
    command: >
      sh -c "
@@ -124,8 +124,8 @@ service OrderService {
 ```
 ### C++ Example
 ```cpp
-#include "lib/api/public/ServiceContext.h"
-#include "lib/api/services/OrderService.client.pb.h"
+#include "spark/api/public/ServiceContext.h"
+#include "spark/api/services/OrderService.client.pb.h"
 using namespace bts::api;
 ServiceContext ctx;
 auto orders = ctx.locate<orderman::OrderService>();
@@ -29,9 +29,9 @@ namespace sample {
 ```

 ```cpp
-#include "lib/api/public/ServiceContext.h"
-#include "spark/lib/domain_model/Security.h"
-#include "lib/api/services/MarketDataService.client.pb.h"
+#include "spark/api/public/ServiceContext.h"
+#include "spark/domain_model/Security.h"
+#include "spark/api/services/MarketDataService.client.pb.h"
 #include <thread>

 using namespace bts::api;
@@ -16,11 +16,16 @@ jmespath==1.0.1
 lazy-object-proxy==1.6.0
 MarkupSafe==3.0.2
 mccabe==0.6.1
+numpy==2.2.6
 platformdirs==2.3.0
 protobuf==4.21.12
 pybars3==0.9.7
+pybind11==2.13.6
+pybind11-stubgen==2.5.3
+pybit==5.11.0
 pycodestyle==2.7.0
 pycparser==2.20
+pycryptodome==3.23.0
 pylint==2.10.2
 PyMeta3==0.5.1
 python-dateutil==2.9.0.post0
@@ -31,6 +36,7 @@ six==1.17.0
 termcolor==1.1.0
 toml==0.10.2
 urllib3==2.0.3
+websocket-client==1.8.0
 Werkzeug==3.1.3
 wrapt==1.12.1
 yapf==0.31.0
@@ -19,8 +19,8 @@
  </ItemGroup>

  <ItemGroup>
-    <Proto Include="../../lib/config/protos/data_stream_config.proto" CSName="DataStreamConfig">
-      <OutDir>$(ProtoOutputPath)/lib/config/protos</OutDir>
+    <Proto Include="../../lib_spark/spark/config/protos/data_stream_config.proto" CSName="DataStreamConfig">
+      <OutDir>$(ProtoOutputPath)/lib_spark/spark/config/protos</OutDir>
    </Proto>
  </ItemGroup>

@@ -43,7 +43,7 @@
      Condition="Exists($(ComputerSpecificConfig))" />

  <PropertyGroup Label="ProtoBuf">
-    <ProtocPluginBase>$(SparkRoot)\src\lib\api\service_compiler</ProtocPluginBase>
+    <ProtocPluginBase>$(SparkRoot)\src\lib_spark\spark\api\service_compiler</ProtocPluginBase>
  </PropertyGroup>
  <Choose>
    <When Condition="'$(OS)' == 'Windows_NT'">
@@ -24,7 +24,7 @@
    <PropertyGroup>
      <GenProtosCmd>$(protoc) %(Proto.FullPath)</GenProtosCmd>
      <GenProtosCmd>$(GenProtosCmd) --error_format=msvs</GenProtosCmd>
-      <GenProtosCmd>$(GenProtosCmd) --proto_path=$(SourceRoot)</GenProtosCmd>
+      <GenProtosCmd>$(GenProtosCmd) --proto_path=$(SourceRoot)/lib_spark</GenProtosCmd>
      <GenProtosCmd>$(GenProtosCmd) --plugin=&quot;protoc-gen-spark-csharp=$(ProtocPlugin)&quot;</GenProtosCmd>
      <GenProtosCmd>$(GenProtosCmd) --spark-csharp_out=lang=c#,templates=immutable.pb.cs,strip_out_path=1:%(Proto.OutDir)</GenProtosCmd>
      <GenProtosCmd>$(GenProtosCmd) --csharp_out=%(Proto.OutDir)</GenProtosCmd>
@@ -65,7 +65,7 @@
    <PropertyGroup>
      <GenProtosCmd>$(protoc) %(ProtoService.FullPath)</GenProtosCmd>
      <GenProtosCmd>$(GenProtosCmd) --error_format=msvs</GenProtosCmd>
-      <GenProtosCmd>$(GenProtosCmd) --proto_path=$(SourceRoot)</GenProtosCmd>
+      <GenProtosCmd>$(GenProtosCmd) --proto_path=$(SourceRoot)/lib_spark</GenProtosCmd>
      <GenProtosCmd>$(GenProtosCmd) --plugin=&quot;protoc-gen-service=$(ProtocPlugin)&quot;</GenProtosCmd>
      <GenProtosCmd>$(GenProtosCmd) --service_out=lang=c#,strip_out_path=1:%(ProtoService.OutDir)</GenProtosCmd>
      <GenProtosCmd>$(GenProtosCmd) --csharp_out=%(ProtoService.OutDir)</GenProtosCmd>
@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 3.10)
+cmake_minimum_required(VERSION 3.5)
 project(Spark.Admin.WebApp)

 include(GNUInstallDirs)
@@ -10,11 +10,11 @@ include(${CMAKE_SOURCE_DIR}/cmake/ProtobufGenerateFs.cmake)
 file(
  GLOB_RECURSE ${PROJECT_NAME}_PROTO_SOURCES
  RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
-  ../lib/api/*.proto
-  ../lib/api/services/*.proto
-  ../lib/domain_model/*.proto
-  ../lib/sparkproto/*.proto
-  ../lib/md/protos/*.proto
+  ../lib_spark/spark/api/*.proto
+  ../lib_spark/spark/api/services/*.proto
+  ../lib_spark/spark/domain_model/*.proto
+  ../lib_spark/spark/sparkproto/*.proto
+  ../lib_spark/spark/md/protos/*.proto
  ../../externals/linux/include/google/protobuf/descriptor.proto
  ../../externals/linux/include/google/protobuf/timestamp.proto
 )
@@ -44,7 +44,7 @@ set(_build_admin_web_app_deps
  ${_build_admin_web_app_py}
  ${CMAKE_CURRENT_SOURCE_DIR}/src/Spark.Admin.WebApp.fsproj
  ${CMAKE_CURRENT_SOURCE_DIR}/webpack.config.js
-  ${CMAKE_SOURCE_DIR}/src/lib/api/service_compiler/templates/fs/fable.fs
+  ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/fs/fable.fs
  ${CMAKE_CURRENT_SOURCE_DIR}/package-lock.json
  ${${PROJECT_NAME}_PROTO_SOURCES}
  ${_api_ts_SOURCES}
@@ -73,7 +73,7 @@ add_custom_command(
 protobuf_generate_js(
  TARGET_NAME generate_js_protos
  PROTO_INCLUDE_DIRS
-    ..
+    ../lib_spark
    ../../externals/linux/include
  PROTO_SOURCES ${${PROJECT_NAME}_PROTO_SOURCES}
  OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/proto.js
@@ -82,7 +82,7 @@ protobuf_generate_js(
 protobuf_generate_fs(
  TARGET_NAME generate_fs_protos
  PROTO_INCLUDE_DIRS
-    ..
+    ../lib_spark
    ../../externals/linux/include
  PROTO_SOURCES ${${PROJECT_NAME}_PROTO_SOURCES}
  OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/fable
@@ -184,4 +184,3 @@ install(
  DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/public"
  DESTINATION "${CMAKE_INSTALL_BINDIR}/${PROJECT_NAME}"
 )
-
@@ -17,7 +17,7 @@ type Tab =
  | PositionManagementTab
  | MarginTab

-let venues = [| Venue.BINANCE; Venue.BinanceUs; Venue.BinanceUsdsFutures; Venue.CBOE; Venue.COINBASE; Venue.DERIBIT; Venue.OKX; Venue.GLBX; Venue.ICE; Venue.TAIFEX; Venue.TWSE; Venue.SIM; |]
+let venues = [| Venue.BINANCE; Venue.BinanceUs; Venue.BinanceUsdsFutures; Venue.BYBIT; Venue.CBOE; Venue.COINBASE; Venue.DERIBIT; Venue.GATEIO; Venue.OKX; Venue.GLBX; Venue.ICE; Venue.TAIFEX; Venue.TWSE; Venue.SIM; |]

 let securityTypes = [| SecurityType.SPOT; SecurityType.FUTURE; SecurityType.OPTION; SecurityType.SPREAD; |]

@@ -50,6 +50,11 @@ type BinanceUsdsFUTURESExchangeField =
  | Origin of CustomerOrFirm
  | ClearingAccount of string

+type BybitExchangeField =
+  | CTI of CTICode
+  | Origin of CustomerOrFirm
+  | ClearingAccount of string
+
 type COINBASEExchangeField =
  | CTI of CTICode
  | Origin of CustomerOrFirm
@@ -60,6 +65,11 @@ type DERIBITExchangeField =
  | Origin of CustomerOrFirm
  | ClearingAccount of string

+type GateioExchangeField =
+  | CTI of CTICode
+  | Origin of CustomerOrFirm
+  | ClearingAccount of string
+
 type ICEExchangeField =
  | CTI of CTICode
  | Origin of CustomerOrFirm
@@ -797,6 +797,8 @@ let venueToString (venue:bts.api.Venue) =
    | bts.api.Venue.BINANCE -> "BINANCE"
    | bts.api.Venue.BinanceUs -> "BinanceUs"
    | bts.api.Venue.BinanceUsdsFutures -> "BinanceUsdsFutures"
+    | bts.api.Venue.BYBIT -> "BYBIT"
+    | bts.api.Venue.GATEIO -> "GATEIO"
    | bts.api.Venue.FTX -> "FTX"
    | bts.api.Venue.OKX -> "OKX"
    | bts.api.Venue.TAIFEX -> "TAIFEX"
@@ -0,0 +1,172 @@
+cmake_minimum_required(VERSION 3.14)
+project(sparkd)
+
+include(FetchContent)
+include(${CMAKE_SOURCE_DIR}/cmake/safe_dep_glob.cmake)
+include(${CMAKE_SOURCE_DIR}/cmake/ProtobufGenerateCpp.cmake)
+include(${CMAKE_SOURCE_DIR}/cmake/InstallBasic.cmake)
+include(GNUInstallDirs)
+
+if (NOT DEFINED PROJECT_VERSION)
+  set(PROJECT_VERSION 1.0.0)
+endif()
+
+# dependencies
+file(
+  GLOB_RECURSE ${PROJECT_NAME}_PROTO_SOURCES
+  RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
+  *.proto
+)
+file(
+  GLOB_RECURSE ${PROJECT_NAME}_SOURCES
+  RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
+  *.cpp
+)
+update_deps_file("${${PROJECT_NAME}_PROTO_SOURCES};${${PROJECT_NAME}_SOURCES}")
+
+protobuf_generate_cpp(
+  CXX_SOURCES ${PROJECT_NAME}_PROTO_CXX_SOURCES
+  CXX_HEADERS ${PROJECT_NAME}_PROTO_CXX_HEADERS
+  CXX_OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/include
+  PROTO_INCLUDE_DIRS
+    ${EXTERNALS_INCLUDE}
+    ${CMAKE_SOURCE_DIR}/src/lib_spark
+    ${CMAKE_CURRENT_SOURCE_DIR}
+  PROTO_SOURCES
+    ${${PROJECT_NAME}_PROTO_SOURCES}
+)
+add_executable(sparkd ${${PROJECT_NAME}_SOURCES} ${${PROJECT_NAME}_PROTO_CXX_SOURCES})
+
+# include paths
+target_include_directories(
+  sparkd
+  PUBLIC
+    $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
+    $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>/include
+    $<BUILD_INTERFACE:${emumba-deribit-connector-library_INCLUDE_DIRS}>
+)
+
+find_package(PkgConfig REQUIRED)
+pkg_check_modules(emumba-deribit-connector-library REQUIRED emumba-deribit-connector)
+
+# This dir contains rapidjsonConfig.cmake
+list(APPEND CMAKE_PREFIX_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake_modules)
+find_package(rapidjson REQUIRED)
+target_include_directories(
+  ${PROJECT_NAME} SYSTEM
+  PUBLIC
+    $<BUILD_INTERFACE:${rapidjson_INCLUDE_DIR}> 
+)
+add_dependencies(${PROJECT_NAME} rapidjson_project)
+
+# databento
+FetchContent_Declare(
+  databento
+  GIT_REPOSITORY https://github.com/databento/databento-cpp
+  GIT_TAG v0.31.0
+)
+FetchContent_MakeAvailable(databento)
+
+# deribit-demo log level
+if(NOT DEFINED DERIBIT_DEMO_LOG_LEVEL)
+  add_definitions(-DSPDLOG_ACTIVE_LEVEL=0)
+  message("-- Building of DERIBIT_DEMO_LOG_LEVEL=TRACE")
+elseif(DERIBIT_DEMO_LOG_LEVEL STREQUAL "TRACE")
+  add_definitions(-DSPDLOG_ACTIVE_LEVEL=0)
+  message("-- Building of DERIBIT_DEMO_LOG_LEVEL=TRACE")
+elseif(DERIBIT_DEMO_LOG_LEVEL STREQUAL "DEBUG")
+  add_definitions(-DSPDLOG_ACTIVE_LEVEL=1)
+  message("-- Building of DERIBIT_DEMO_LOG_LEVEL=DEBUG")
+elseif(DERIBIT_DEMO_LOG_LEVEL STREQUAL "INFO")
+  add_definitions(-DSPDLOG_ACTIVE_LEVEL=2)
+  message("-- Building of DERIBIT_DEMO_LOG_LEVEL=INFO")
+elseif(DERIBIT_DEMO_LOG_LEVEL STREQUAL "WARN")
+  add_definitions(-DSPDLOG_ACTIVE_LEVEL=3)
+  message("-- Building of DERIBIT_DEMO_LOG_LEVEL=WARN")
+elseif(DERIBIT_DEMO_LOG_LEVEL STREQUAL "ERROR")
+  add_definitions(-DSPDLOG_ACTIVE_LEVEL=4)
+  message("-- Building of DERIBIT_DEMO_LOG_LEVEL=ERROR")
+elseif(DERIBIT_DEMO_LOG_LEVEL STREQUAL "CRITICAL")
+  add_definitions(-DSPDLOG_ACTIVE_LEVEL=5)
+  message("-- Building of DERIBIT_DEMO_LOG_LEVEL=CRITICAL")
+elseif(DERIBIT_DEMO_LOG_LEVEL STREQUAL "OFF")
+  add_definitions(-DSPDLOG_ACTIVE_LEVEL=6)
+  message("-- Building of DERIBIT_DEMO_LOG_LEVEL=OFF")
+else()
+  add_definitions(-DSPDLOG_ACTIVE_LEVEL=0)
+  message("-- Building of DERIBIT_DEMO_LOG_LEVEL=TRACE")
+endif()
+
+target_link_directories(
+  ${PROJECT_NAME} PUBLIC
+  $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
+  $<INSTALL_INTERFACE:${CMAKE_INSTALL_LIBDIR}>
+)
+
+# Set RPATH so sparkd knows where to find our shared libs (e.g., libspark).
+# The dirs are relative to sparkd's path, represented by "$ORIGIN".
+set_target_properties(
+  ${PROJECT_NAME} PROPERTIES
+  BUILD_RPATH "$ORIGIN/../lib_spark;$ORIGIN/../lib_python_interop"
+  INSTALL_RPATH "$ORIGIN/../${CMAKE_INSTALL_LIBDIR}"
+)
+
+include(${CMAKE_SOURCE_DIR}/cmake/SymLink.cmake)
+include(${CMAKE_SOURCE_DIR}/cmake/SymLinkPythonModule.cmake)
+# Py interop
+sym_link_python_module(
+  LIB_NAME python_interop
+  MODULE_NAME python_interop_module
+  PLUGIN_NAME python_interop_plugins
+)
+target_link_libraries(
+  sparkd
+  spark
+  binance_execution
+  okx_execution
+  python_interop
+  python_interop_module
+  databento::databento
+  ${emumba-deribit-connector-library_LIBRARIES}
+  tbb
+  quickfix
+  benchmark
+  Common++
+  Packet++
+  Pcap++
+  lttng-ust
+  -lpcap
+  -static-libgcc
+  -static-libstdc++
+  -pthread)
+
+# Onixs
+target_link_libraries(
+  ${PROJECT_NAME}
+  onixs-cmemdh
+  OnixS.IceImpactMulticastPriceFeedHandlerCpp
+  # gcc doesn't resolve onixs-fix-engine and ACE in runtime path unless we specify
+  # --no-as-needed. These are onixs deps that aren't called directly by our code.
+  # Note: this isn't an issue for clang
+  -Wl,--no-as-needed
+  onixs-fix-engine
+  ACE
+)
+
+# Catch2
+find_package(Catch2 CONFIG REQUIRED)
+target_link_libraries(${PROJECT_NAME} Catch2::Catch2)
+
+install_basic()
+install(
+  FILES
+    "${EXTERNALS_LIB}/libonixs-cmemdh.so"
+    "${EXTERNALS_LIB}/libonixs-cmemdh.so.4"
+    "${EXTERNALS_LIB}/libonixs-cmemdh.so.4.8.0"
+    "${EXTERNALS_LIB}/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.16.0.0"
+    "${EXTERNALS_LIB}/libonixs-fix-engine.so"
+    "${EXTERNALS_LIB}/libonixs-fix-engine.so.4"
+    "${EXTERNALS_LIB}/libonixs-fix-engine.so.4.10.1"
+    "${EXTERNALS_LIB}/libACE-5.7.5.so"
+  DESTINATION ${CMAKE_INSTALL_LIBDIR}
+)
@@ -0,0 +1,5 @@
+@PACKAGE_INIT@
+
+include("${CMAKE_CURRENT_LIST_DIR}/@PROJECT_NAME@Targets.cmake")
+
+check_required_components(@PROJECT_NAME@)
@@ -1,6 +1,7 @@
 ARG IMAGE="sparkd"
-ARG CONFIG_DIR="src/server/sparkd/configurations/dev"
+ARG CONFIG_DIR="src/app_sparkd/configurations/dev"
 ARG SOURCE_DIR="install/sparkd-release-gcc"
+ARG PYTHON_REQUIREMENTS="requirements.txt"

 # sparkd_base
 FROM bluetradingsystems/linux_dotnet7 AS sparkd_base
@@ -10,7 +11,11 @@ ARG SOURCE_DIR
 USER "root"
 RUN apt-get update && apt-get install -y liblttng-ust-dev
 RUN apt-get update && apt-get install -y libpcap0.8-dev
-RUN apt-get update && apt-get install -y python3.10-dev
+RUN apt-get update && apt-get install -y python3-dev
+RUN apt-get update && apt-get install -y python3-pip
+RUN python3 -m pip install --upgrade pip
+COPY "${PYTHON_REQUIREMENTS}" "/home/bts/bin/requirements.txt"
+RUN pip3 install -r "/home/bts/bin/requirements.txt"

 USER "bts"
 COPY --chown=bts:bts "${SOURCE_DIR}" "."
@@ -22,17 +27,22 @@ FROM sparkd_base AS sparkd

 # sparkd_md_replay
 FROM sparkd_base AS sparkd_md_replay
-ARG MD_RECORDING_DIR="src/server/sparkd/.md_recording"
+ARG MD_RECORDING_DIR="src/app_sparkd/.md_recording"
 COPY --chown=bts:bts "${MD_RECORDING_DIR}" "/home/bts/sparkdata/data/md_recording"

 # sparkd_symbols - like sparkd, but doesn't copy a config dir into the image
 FROM bluetradingsystems/linux_dotnet7 AS sparkd_symbols
 ARG SOURCE_DIR
+ARG PYTHON_REQUIREMENTS="requirements.txt"

 USER "root"
 RUN apt-get update && apt-get install -y liblttng-ust-dev
 RUN apt-get update && apt-get install -y libpcap0.8-dev
-RUN apt-get update && apt-get install -y python3.10-dev
+RUN apt-get update && apt-get install -y python3-dev
+RUN apt-get update && apt-get install -y python3-pip
+RUN python3 -m pip install --upgrade pip
+COPY "${PYTHON_REQUIREMENTS}" "/home/bts/bin/requirements.txt"
+RUN pip3 install -r "/home/bts/bin/requirements.txt"

 USER "bts"
 COPY --chown=bts:bts "${SOURCE_DIR}" "."
@@ -0,0 +1,4 @@
+SPARKD__CONFIG_DIR=src/app_sparkd/configurations/alphaworx_prod
+SPARKD__SHM_SIZE=8gb
+SPARKD__MEMORY=8gb
+SPARKD__RUNTIME_CONF_DIR=/home/ubuntu/8000/sparkdata/conf/
@@ -0,0 +1,6 @@
+SPARKD__CONFIG_DIR=src/app_sparkd/configurations/alphaworx_zen_hk_live-preview
+SPARKD__PYTHON_REQUIREMENTS=src/app_sparkd/configurations/alphaworx_zen_hk_live-preview/requirements.txt
+SPARKD__SHM_SIZE=8gb
+SPARKD__MEMORY=8gb
+SPARKD__IMAGE=sparkd_symbols
+SPARKD__RUNTIME_CONF_DIR=/home/bts/8000/sparkdata/conf
@@ -0,0 +1,6 @@
+* Connects to Binance, Okx, and Bybit PRODUCTION sites with small accounts designated for test trades
+* Api keys NOT included
+* Configured for Alphaworx and Bts testing in a production environment
+
+To retrieve api keys, run "python init_env.py". It should dump them to a file
+called  ".env.local".
@@ -0,0 +1,99 @@
+import botocore
+import json
+import os
+
+from boto3.session import Session
+from botocore.exceptions import ClientError
+from fileinput import FileInput
+
+
+def update_env_file(path, env_vars: dict):
+    """Update or add vars in a .env file in place."""
+    if not os.path.exists(path):
+        with open(path, "w") as file:
+            for k, v in env_vars.items():
+                file.write(f"{k}={v}\n")
+        return
+
+    with FileInput(path, inplace=True) as file:
+        for line in file:
+            if not line.startswith("#"):
+                stripped: str = line.lstrip()
+                key_len = stripped.find("=")
+                if key_len >= 0:
+                    k = stripped[0:key_len]
+                    if k in env_vars:
+                        print(f"{k}={env_vars[k]}")
+                        del env_vars[k]
+                        continue
+            print(line, end="")
+
+    if len(env_vars) > 0:
+        with open(path, "a") as file:
+            for k, v in env_vars.items():
+                file.write(f"{k}={v}\n")
+
+
+def get_secret():
+    secret_name = "alphaworx_api_keys"
+    region_name = "us-east-1"
+    profile_name = "bts-ap-northeast-1"
+    session = Session(profile_name=profile_name)
+    client = session.client(
+        service_name="secretsmanager",
+        region_name=region_name,
+    )
+
+    def on_token_error(e: Exception):
+        print(f"""
+Failed to get secret value. {e}
+Verify you are logged in to aws, then try again.
+
+  aws sso login --profile {profile_name}
+
+If this error persists, verify you have access to these api keys.
+For more help or information, talk to John.
+""")
+
+    try:
+        get_secret_value_response = client.get_secret_value(
+            SecretId=secret_name)
+    except ClientError as e:
+        print(f"Failed to get secret value.\n{e}")
+        raise e
+    except botocore.exceptions.SSOTokenLoadError as tl_err:
+        on_token_error(tl_err)
+        return None
+    except botocore.exceptions.TokenRetrievalError as tr_err:
+        on_token_error(tr_err)
+        return None
+    return get_secret_value_response["SecretString"]
+
+
+def _run():
+    this_dir = os.path.dirname(__file__)
+    secret_str = get_secret()
+    if secret_str is None:
+        exit(1)
+    secret = json.loads(secret_str)
+    env_vars = {}
+
+    def set_env_var(key_name: str, *json_path):
+        try:
+            val = secret[json_path[0]]
+            for p in json_path[1:]:
+                val = val[p]
+            env_vars[key_name] = val
+            print(f"Set {key_name}={val} ({'.'.join(json_path)})")
+        except KeyError:
+            print(f"Failed to set {key_name}= ({'.'.join(json_path)})")
+
+    set_env_var("BINANCE_API_KEY", "binance", "prod", "api_key")
+    set_env_var("BINANCE_SECRET_KEY", "binance", "prod", "secret_key")
+    set_env_var("BYBIT_API_KEY", "bybit", "prod_test", "api_key")
+    set_env_var("BYBIT_SECRET_KEY", "bybit", "prod_test", "secret_key")
+    update_env_file(f"{this_dir}/.env.local", env_vars)
+
+
+if __name__ == "__main__":
+    _run()
@@ -0,0 +1,15 @@
+{
+  "gateway_type": "bybit_execution_gateway.BybitExecutionGateway",
+  "account": "BYBIT_DEV",
+  "module_path": [
+    "python_interop_plugins"
+  ],
+  "venue": "VENUE_BYBIT",
+  "api_key": "${BYBIT_API_KEY}",
+  "secret_key": "${BYBIT_SECRET_KEY}",
+  "use_testnet": true,
+  "logging": {
+    "console_level": "DEBUG",
+    "file_level": "DEBUG"
+  }
+}
@@ -0,0 +1,17 @@
+{
+  "currencies": [
+    "USDT"
+  ],
+  "futures": [
+    "ANIME",
+    "BTC",
+    "ETH",
+    "MASK",
+    "RESOLV"
+  ],
+  "use_testnet": true,
+  "logging": {
+    "console_level": "WARNING",
+    "file_level": "INFO"
+  }
+}
@@ -0,0 +1,11 @@
+certifi==2025.4.26
+charset-normalizer==3.4.2
+dotenv==0.9.9
+idna==3.10
+numpy==2.2.6
+pybit==5.11.0
+pycryptodome==3.23.0
+python-dotenv==1.1.0
+requests==2.32.4
+urllib3==2.4.0
+websocket-client==1.8.0
@@ -0,0 +1,138 @@
+QuickbusCfg {
+  value {
+    node {
+      reconnect_period_secs: 1
+      hb_period_msecs: 15000
+      max_missed_hbs: 5
+      parent_host: ""
+      parent_port: 0
+      my_port: 0
+    }
+    client {
+      reconnect_period_secs: 1
+      hb_period_msecs: 15000
+      max_missed_hbs: 5
+      parent_host: @SystemConfig.config_ip
+      parent_port: @Ports.qbus_port
+      my_port: 0
+    }
+  }
+}
+
+JetFIX {
+  value {
+    # connection {
+    #   name: "binance_1"
+    #   venue: VENUE_BINANCE
+    #   max_order_msgs_per_window: 300
+    #   window_duration_msecs: 1000
+    # }
+    # connection {
+    #   name: "binance_usds_futures_1"
+    #   venue: VENUE_BINANCE_USDS_FUTURES
+    #   max_order_msgs_per_window: 300
+    #   window_duration_msecs: 1000
+    # }
+    connection {
+      name: "bybit_1"
+      venue: VENUE_BYBIT
+      max_order_msgs_per_window: 300
+      window_duration_msecs: 1000
+    }
+    # connection {
+    #   name: "okx_1"
+    #   venue: VENUE_OKX
+    #   max_order_msgs_per_window: 300
+    #   window_duration_msecs: 1000
+    # }
+  }
+}
+
+MarketDataConfig {
+  value {
+    # binance {
+    #   num_depth_levels: 5
+    #   url_rest_base: "https://api.binance.com"
+    #   url_ws_base: "wss://stream.binance.com:9443"
+    #   security_criteria {
+    #     clauses {
+    #       security_types: SECURITY_TYPE_SPOT
+    #       symbols: @SymbolLists.list[Binance].symbol
+    #       name_regexps: "USDC|USDT$"
+    #     }
+    #   }
+    # }
+    # binance_usds_futures {
+    #   num_depth_levels: 5
+    #   url_rest_base: "https://fapi.binance.com"
+    #   url_ws_base: "wss://fstream.binance.com"
+    #   security_criteria {
+    #     clauses {
+    #       security_types: SECURITY_TYPE_FUTURE
+    #       symbols: @SymbolLists.list[BinanceUsdsFutures].symbol
+    #     }
+    #   }
+    # }
+    # okx {
+    #   num_depth_levels: 5
+    #   url_rest_base: "https://www.okx.com"
+    #   url_ws_base: "wss://ws.okx.com:8443"
+    #   security_criteria {
+    #     clauses {
+    #       security_types: SECURITY_TYPE_SPOT
+    #       symbols: @SymbolLists.list[OkxSpots].symbol
+    #     }
+    #     clauses {
+    #       security_types: SECURITY_TYPE_FUTURE
+    #       symbols: @SymbolLists.list[OkxFutures].symbol
+    #     }
+    #   }
+    #   is_application_ping_pong_enabled: true
+    #   application_ping_interval_ms: 20000
+    #   application_pong_timeout_ms: 10000
+    #   is_protocol_ping_pong_enabled: true
+    #   protocol_ping_interval_ms: 60000
+    #   protocol_pong_timeout_ms: 30000
+    # }
+    python_md_providers {
+      venue: VENUE_BYBIT
+      module_path: "python_interop_plugins"
+      provider_type: "bybit_md_provider.BybitMdProvider"
+    }
+  }
+}
+
+NotSimExecution {
+  value {
+    # entries {
+    #   venue: VENUE_BINANCE
+    # }
+    # entries {
+    #   venue: VENUE_BINANCE_USDS_FUTURES
+    # }
+    entries {
+      venue: VENUE_BYBIT
+    }
+    # entries {
+    #   venue: VENUE_OKX
+    # }
+  }
+}
+
+# OkxExeCfg {
+#   value {
+#     account: "PROD"
+#     ops_id: "op_id_bts"
+#     secret_key: "240038D7A28D7F9461247CFCB8E43710"
+#     api_key: "dc188532-c74c-4d40-affe-3c6bf4d0ca9a"
+#     passphrase: ":,7i%tA%ivfj%SCbX?Ut"
+#     ws_url: "wss://ws.okx.com:8443"
+#     rest_url: "https://www.okx.com"
+#     ws_ping_timeout_seconds: 10 
+#     ws_ping_interval_seconds: 10
+#     ws_reconnect_interval_seconds: 5
+#     ws_reconnect_max_count: 5
+# 	  simulated_trading: false
+#     tdmode: "cross"
+#   }
+# }
@@ -0,0 +1,66 @@
+SymbolLists {
+  value {
+    list {
+      key: "Binance"
+      symbol: "ANIME"
+      symbol: "BTC"
+      symbol: "ETH"
+      symbol: "MASK"
+      symbol: "RESOLV"
+    }
+    list {
+      key: "BinanceUsdsFutures"
+      symbol: "ANIME"
+      symbol: "BTC"
+      symbol: "ETH"
+      symbol: "MASK"
+      symbol: "RESOLV"
+    }
+    list {
+      key: "OkxSpots"
+      symbol: "ANIME"
+      symbol: "BTC"
+      symbol: "ETH"
+      symbol: "MASK"
+      symbol: "RESOLV"
+    }
+    list {
+      key: "Bybit"
+      symbol: "ANIME"
+      symbol: "BTC"
+      symbol: "ETH"
+      symbol: "MASK"
+      symbol: "RESOLV"
+    }
+    list {
+      key: "OkxFutures"
+      symbol: "ANIME"
+      symbol: "BTC"
+      symbol: "ETH"
+      symbol: "MASK"
+      symbol: "RESOLV"
+    }
+  }
+}
+
+Symbols {
+  value {
+    # venues {
+    #   name: "VENUE_BINANCE"
+    #   spot: @SymbolLists.list[Binance].symbol
+    # }
+    # venues {
+    #   name: "VENUE_BINANCE_USDS_FUTURES"
+    #   future: @SymbolLists.list[BinanceUsdsFutures].symbol
+    # }
+    venues {
+      name: "VENUE_BYBIT"
+      future: @SymbolLists.list[Bybit].symbol
+    }
+    # venues {
+    #   name: "VENUE_OKX"
+    #   spot: @SymbolLists.list[OkxSpots].symbol
+    #   future: @SymbolLists.list[OkxFutures].symbol
+    # }
+  }
+}
@@ -1,4 +1,4 @@
-SPARKD__CONFIG_DIR=src/server/sparkd/configurations/alphaworx_zen_hk_live
+SPARKD__CONFIG_DIR=src/app_sparkd/configurations/alphaworx_zen_hk_live
 SPARKD__SHM_SIZE=8gb
 SPARKD__MEMORY=8gb
 SPARKD__IMAGE=sparkd_symbols
@@ -0,0 +1,9 @@
+{
+  "api_key": "${BINANCE_API_KEY}",
+  "secret_key": "${BINANCE_SECRET_KEY}",
+  "account": "PROD",
+  "rest_endpoint": "https://api.binance.com",
+  "ws_endpoint": "wss://ws-api.binance.com:443/ws-api/v3",
+  "ws_stream_endpoint": "wss://stream.binance.com:9443",
+  "use_cross_margin_api": true
+}
@@ -0,0 +1,9 @@
+{
+  "api_key": "${BINANCE_API_KEY}",
+  "secret_key": "${BINANCE_SECRET_KEY}",
+  "account": "PROD",
+  "rest_endpoint": "https://fapi.binance.com",
+  "ws_endpoint": "wss://ws-fapi.binance.com/ws-fapi/v1",
+  "ws_stream_endpoint": "wss://fstream.binance.com",
+  "use_cross_margin_api": false
+}
@@ -35,8 +35,8 @@ def update_env_file(path, env_vars: dict):


 def get_secret():
-    secret_name = "alphaworx-binance-api-keys"
-    region_name = "ap-northeast-1"
+    secret_name = "alphaworx_api_keys"
+    region_name = "us-east-1"
    profile_name = "bts-ap-northeast-1"
    session = Session(profile_name=profile_name)
    client = session.client(
@@ -77,10 +77,19 @@ def _run():
        exit(1)
    secret = json.loads(secret_str)
    env_vars = {}
-    if "api-key" in secret:
-        env_vars["BINANCE_API_KEY"] = secret["api-key"]
-    if "secret-key" in secret:
-        env_vars["BINANCE_SECRET_KEY"] = secret["secret-key"]
+
+    def set_env_var(key_name: str, *json_path):
+        try:
+            val = secret[json_path[0]]
+            for p in json_path[1:]:
+                val = val[p]
+            env_vars[key_name] = val
+            print(f"Set {key_name}={val} ({'.'.join(json_path)})")
+        except KeyError:
+            print(f"Failed to set {key_name}= ({'.'.join(json_path)})")
+
+    set_env_var("BINANCE_API_KEY", "binance", "prod", "api_key")
+    set_env_var("BINANCE_SECRET_KEY", "binance", "prod", "secret_key")
    update_env_file(f"{this_dir}/.env.local", env_vars)


@@ -0,0 +1,2 @@
+SPARKD__CONFIG_DIR=src/app_sparkd/configurations/binance_dev
+SPARKD__SHM_SIZE=8gb
@@ -0,0 +1,2 @@
+SPARKD__CONFIG_DIR=src/app_sparkd/configurations/binance_py_dev
+SPARKD__SHM_SIZE=8gb
@@ -0,0 +1 @@
+* Connects to Binance test nets for market data and execution
@@ -0,0 +1,8 @@
+{
+  "api_key": "${BINANCE_API_KEY}",
+  "secret_key": "${HOME}/sparkdata/conf/.secrets/binance_1.pem",
+  "account": "BINANCE_DEV",
+  "rest_endpoint": "https://testnet.binance.vision",
+  "ws_endpoint": "wss://ws-api.testnet.binance.vision/ws-api/v3",
+  "ws_stream_endpoint": "wss://stream.testnet.binance.vision:9443"
+}
@@ -0,0 +1,8 @@
+{
+  "api_key": "${BINANCE_USDS_FUTURES_API_KEY}",
+  "secret_key": "${BINANCE_USDS_FUTURES_SECRET_KEY}",
+  "account": "BINANCE_DEV",
+  "rest_endpoint": "https://testnet.binancefuture.com",
+  "ws_endpoint": "wss://testnet.binancefuture.com/ws-fapi/v1",
+  "ws_stream_endpoint": "wss://stream.binancefuture.com"
+}
@@ -0,0 +1,124 @@
+import traceback
+import botocore
+import json
+import os
+
+from boto3.session import Session
+from botocore.exceptions import ClientError
+from fileinput import FileInput
+
+
+def update_env_file(path, env_vars: dict):
+    """Update or add vars in a .env file in place."""
+    if not env_vars:
+        # Nothing to update
+        return
+    if not os.path.exists(path):
+        with open(path, "w") as file:
+            for k, v in env_vars.items():
+                file.write(f"{k}={v}\n")
+        return
+
+    with FileInput(path, inplace=True) as file:
+        for line in file:
+            if not line.startswith("#"):
+                stripped: str = line.lstrip()
+                key_len = stripped.find("=")
+                if key_len >= 0:
+                    k = stripped[0:key_len]
+                    if k in env_vars:
+                        print(f"{k}={env_vars[k]}")
+                        del env_vars[k]
+                        continue
+            print(line, end="")
+
+    if len(env_vars) > 0:
+        with open(path, "a") as file:
+            for k, v in env_vars.items():
+                file.write(f"{k}={v}\n")
+
+
+def write_pem_file(file_path: str, text: str):
+    try:
+        dir_path = os.path.dirname(file_path)
+        os.makedirs(dir_path, exist_ok=True)
+        # rw-------
+        mode = 0o600
+        fd = os.open(file_path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, mode)
+        with os.fdopen(fd, "x") as f:
+            f.write(text)
+        return True
+    except Exception:
+        print(f"Error writing pem file '{file_path}'.")
+        traceback.print_exc()
+        return False
+
+
+def get_secret(secret_name: str, profile_name: str):
+    session = Session(profile_name=profile_name)
+    client = session.client(service_name="secretsmanager")
+
+    def on_token_error(e: Exception):
+        print(f"""
+Failed to get secret value '{secret_name}'. {e}
+Verify you are logged in to aws, then try again.
+
+  aws sso login --profile {profile_name}
+
+If this error persists, verify you have access to these api keys.
+For more help or information, talk to John.
+""")
+
+    try:
+        get_secret_value_response = client.get_secret_value(
+            SecretId=secret_name)
+    except ClientError as e:
+        print(f"Failed to get secret value.\n{e}")
+        raise e
+    except botocore.exceptions.SSOTokenLoadError as tl_err:
+        on_token_error(tl_err)
+        return None
+    except botocore.exceptions.TokenRetrievalError as tr_err:
+        on_token_error(tr_err)
+        return None
+    return get_secret_value_response["SecretString"]
+
+
+def _init_binance_1():
+    this_dir = os.path.dirname(__file__)
+    secret_str = get_secret("dev/binance/john", "bts-us-east-1")
+    if secret_str is None:
+        exit(1)
+    secret = json.loads(secret_str)
+    env_vars = {}
+    if "api-key" in secret:
+        env_vars["BINANCE_API_KEY"] = secret["api-key"]
+    if "secret-key-pem" in secret:
+        sk_pem = secret["secret-key-pem"]
+        sk_pem = "\n".join(sk_pem)
+        write_pem_file(f"{this_dir}/.secrets/binance_1.pem", sk_pem)
+    update_env_file(f"{this_dir}/.env.local", env_vars)
+
+
+def _init_binance_usds_futures_1():
+    this_dir = os.path.dirname(__file__)
+    secret_str = get_secret("dev/binance_usds_futures/john.hmac",
+                            "bts-us-east-1")
+    if secret_str is None:
+        exit(1)
+    secret = json.loads(secret_str)
+    env_vars = {}
+    if "api-key" in secret:
+        env_vars["BINANCE_USDS_FUTURES_API_KEY"] = secret["api-key"]
+    if "secret-key" in secret:
+        env_vars["BINANCE_USDS_FUTURES_SECRET_KEY"] = secret["secret-key"]
+    update_env_file(f"{this_dir}/.env.local", env_vars)
+
+
+def _run():
+    _init_binance_1()
+    _init_binance_usds_futures_1()
+
+
+if __name__ == "__main__":
+    _run()
@@ -0,0 +1,8 @@
+{
+  "gateway_type": "dummy_execution_gateway.DummyExecutionGateway",
+  "account": "BINANCE_DEV",
+  "module_path": [
+    "python_execution_gateways"
+  ],
+  "venue": "VENUE_BINANCE_USDS_FUTURES"
+}
@@ -0,0 +1,119 @@
+JetFIX {
+  value {
+    # connection {
+    #   name: "binance_1"
+    #   venue: VENUE_BINANCE
+    #   max_order_msgs_per_window: 300
+    #   window_duration_msecs: 1000
+    # }
+    connection {
+      name: "binance_usds_futures_1"
+      venue: VENUE_BINANCE_USDS_FUTURES
+      max_order_msgs_per_window: 300
+      window_duration_msecs: 1000
+    }
+  }
+}
+
+PublisherOptions {
+  value {
+    network_interface: "0.0.0.0"
+    heartbeat_ms: 5000
+    heartbeat_boot_seconds: 60
+    skip_options: true
+    hot_initial: 1024
+    hot_growth: 1024
+    cold_initial: 8192
+    cold_growth: 2048
+    channel_prefix: ""
+    disable_heartbeat: true
+  }
+}
+
+Logging {
+  value {
+    initial_queue_size: 120001
+    max_queue_growth: 4
+    console {
+      default_level: "Info"
+      subsystem {
+        name: "OrderMan"
+        level: "Diag"
+      }
+    }
+    file {
+      default_level: "Info"
+      # subsystem {
+      #   name: "md"
+      #   level: "Diag"
+      # }
+      subsystem {
+        name: "OrderMan"
+        level: "Diag"
+      }
+      subsystem {
+        name: "Http"
+        level: "Diag"
+      }
+    }
+  }
+}
+
+MarketDataConfig {
+  value {
+    # binance {
+    #   num_depth_levels: 5
+    #   # url_rest_base: "wss://ws-api.binance.com:9443"
+    #   # According to Binance, the "apiN" urls offer performance at the cost of stability.
+    #   # https://developers.binance.com/docs/binance-spot-api-docs/rest-api/general-api-information
+    #   # url_rest_base: "https://api1.binance.com:9443"
+    #   # url_rest_base: "https://api.binance.com"
+    #   url_rest_base: "https://testnet.binance.vision"
+    #   # url_ws_base: "wss://ws-api.binance.com"
+    #   url_ws_base: "wss://testnet.binance.vision"
+
+    #   security_criteria {
+    #     clauses {
+    #       security_types: SECURITY_TYPE_SPOT
+    #       symbols: @SymbolLists.list[Binance].symbol
+    #       name_regexps: "USDC|USDT$"
+    #     }
+    #   }
+    # }
+
+    binance_usds_futures {
+      num_depth_levels: 5
+      # url_rest_base: "https://fapi.binance.com"
+      # url_ws_base: "wss://fstream.binance.com"
+      url_rest_base: "https://testnet.binancefuture.com"
+      url_ws_base: "wss://stream.binancefuture.com"
+
+      security_criteria {
+        clauses {
+          security_types: SECURITY_TYPE_FUTURE
+          symbols: @SymbolLists.list[BinanceUsdsFutures].symbol
+        }
+      }
+    }
+  }
+}
+
+NotSimExecution {
+  value {
+    # entries {
+    #   venue: VENUE_BINANCE
+    # }
+    entries {
+      venue: VENUE_BINANCE_USDS_FUTURES
+    }
+    # entries {
+    #   venue: VENUE_GLBX
+    # }
+    # entries {
+    #   venue: VENUE_OKX
+    # }
+    # entries {
+    #   venue: VENUE_TAIFEX
+    # }
+  }
+}
@@ -0,0 +1,37 @@
+SymbolLists {
+  value {
+    list {
+      key: "Binance"
+      symbol: "AAVE"
+      symbol: "AVAX"
+      symbol: "BNB"
+      symbol: "BTC"
+      symbol: "DOGE"
+      symbol: "EIGEN"
+      symbol: "ETH"
+    }
+    list {
+      key: "BinanceUsdsFutures"
+      symbol: "AAVE"
+      symbol: "AVAX"
+      symbol: "BNB"
+      symbol: "BTC"
+      symbol: "DOGE"
+      symbol: "EIGEN"
+      symbol: "ETH"
+    }
+  }
+}
+
+Symbols {
+  value {
+    venues {
+      name: "VENUE_BINANCE"
+      spot: @SymbolLists.list[Binance].symbol
+    }
+    venues {
+      name: "VENUE_BINANCE_USDS_FUTURES"
+      future: @SymbolLists.list[BinanceUsdsFutures].symbol
+    }
+  }
+}
@@ -0,0 +1 @@
+SPARKD__CONFIG_DIR=src/app_sparkd/configurations/bybit_dev
@@ -0,0 +1,15 @@
+{
+  "gateway_type": "bybit_execution_gateway.BybitExecutionGateway",
+  "account": "BYBIT_DEV",
+  "module_path": [
+    "python_interop_plugins"
+  ],
+  "venue": "VENUE_BYBIT",
+  "api_key": "${BYBIT_API_KEY}",
+  "secret_key": "${BYBIT_SECRET_KEY}",
+  "use_testnet": true,
+  "logging": {
+    "console_level": "WARNING",
+    "file_level": "INFO"
+  }
+}
@@ -0,0 +1,18 @@
+{
+  "currencies": [
+    "USDT"
+  ],
+  "futures": [
+    "ADA",
+    "BNB",
+    "BTC",
+    "ETH",
+    "PRIME",
+    "XRP"
+  ],
+  "use_testnet": true,
+  "logging": {
+    "console_level": "WARNING",
+    "file_level": "INFO"
+  }
+}
@@ -0,0 +1,61 @@
+JetFIX {
+  value {
+    connection {
+      name: "bybit_1"
+      venue: VENUE_BYBIT
+      max_order_msgs_per_window: 300
+      window_duration_msecs: 1000
+    }
+  }
+}
+
+PublisherOptions {
+  value {
+    network_interface: "0.0.0.0"
+    heartbeat_ms: 5000
+    heartbeat_boot_seconds: 60
+    skip_options: true
+    hot_initial: 1024
+    hot_growth: 1024
+    cold_initial: 8192
+    cold_growth: 2048
+    channel_prefix: ""
+    disable_heartbeat: true
+  }
+}
+
+Logging {
+  value {
+    initial_queue_size: 120001
+    max_queue_growth: 4
+    file {
+      default_level: "Info"
+      subsystem {
+        name: "QBus"
+        level: "Warn"
+      }
+      subsystem {
+        name: "md"
+        level: "Diag"
+      }
+    }
+  }
+}
+
+MarketDataConfig {
+  value {
+    python_md_providers {
+      venue: VENUE_BYBIT
+      module_path: "python_interop_plugins"
+      provider_type: "bybit_md_provider.BybitMdProvider"
+    }
+  }
+}
+
+NotSimExecution {
+  value {
+    entries {
+      venue: VENUE_BYBIT
+    }
+  }
+}
@@ -0,0 +1,22 @@
+SymbolLists {
+  value {
+    list {
+      key: "Bybit"
+      symbol: "ADA"
+      symbol: "BNB"
+      symbol: "BTC"
+      symbol: "ETH"    
+      symbol: "PRIME"
+      symbol: "XRP"
+    }
+  }
+}
+
+Symbols {
+  value {
+    venues {
+      name: "VENUE_BYBIT"
+      future: @SymbolLists.list[Bybit].symbol
+    }
+  }
+}
@@ -0,0 +1,2 @@
+SPARKD__CONFIG_DIR=src/app_sparkd/configurations/dev
+SPARKD__SHM_SIZE=8gb
@@ -0,0 +1 @@
+SPARKD__CONFIG_DIR=src/app_sparkd/configurations/py_dummy_md
@@ -0,0 +1,45 @@
+{
+  "api_key": "e4a0c6b198549042b13faa7645640d60",
+  "secret_key": "527c37877efc247bb0b6158cf3d5d18b84db720da894307f1e2ab88df3aaf0ee",
+  "ws_url": "wss://fx-ws-testnet.gateio.ws/v4/ws/usdt",
+  "currencies": [
+    "USDT"
+  ],
+  "spots": [
+    "AMP",
+    "AR",
+    "ARB",
+    "ATOM",
+    "BTC",
+    "CAKE",
+    "CELO",
+    "CRV",
+    "ETH",
+    "FIL",
+    "GMT",
+    "GMX",
+    "HIGH",
+    "ICP",
+    "IMX",
+    "IOTX",
+    "LRC",
+    "MANA",
+    "MINA",
+    "NEAR",
+    "NOT",
+    "OMG",
+    "OP",
+    "OXY",
+    "PEOPLE",
+    "TURBO",
+    "VET",
+    "WAVES",
+    "YFI",
+    "ZIL",
+    "ZRX"
+  ],
+  "logging": {
+    "console_level": "WARNING",
+    "file_level": "INFO"
+  }
+}
@@ -0,0 +1,38 @@
+PublisherOptions {
+  value {
+    network_interface: "0.0.0.0"
+    heartbeat_ms: 5000
+    heartbeat_boot_seconds: 60
+    skip_options: true
+    hot_initial: 1024
+    hot_growth: 1024
+    cold_initial: 8192
+    cold_growth: 2048
+    channel_prefix: ""
+    disable_heartbeat: true
+  }
+}
+
+Logging {
+  value {
+    initial_queue_size: 120001
+    max_queue_growth: 4
+    file {
+      default_level: "Info"
+      subsystem {
+        name: "md"
+        level: "Diag"
+      }
+    }
+  }
+}
+
+MarketDataConfig {
+  value {
+    python_md_providers {
+      venue: VENUE_GATEIO
+      module_path: "python_md_providers"
+      provider_type: "gateio_md_provider.GateIoMdProvider"
+    }
+  }
+}
@@ -0,0 +1,47 @@
+  SymbolLists {
+  value {
+    list {
+      key: "Gateio"
+      symbol: "AMP"
+      symbol: "AR"
+      symbol: "ARB"
+      symbol: "ATOM"
+      symbol: "BTC"
+      symbol: "CAKE"
+      symbol: "CELO"
+      symbol: "CRV"
+      symbol: "ETH"
+      symbol: "FIL"
+      symbol: "GMT"
+      symbol: "GMX"
+      symbol: "HIGH"
+      symbol: "ICP"
+      symbol: "IMX"
+      symbol: "IOTX"
+      symbol: "LRC"
+      symbol: "MANA"
+      symbol: "MINA"
+      symbol: "NEAR"
+      symbol: "NOT"
+      symbol: "OMG"
+      symbol: "OP"
+      symbol: "OXY"
+      symbol: "PEOPLE"
+      symbol: "TURBO"
+      symbol: "VET"
+      symbol: "WAVES"
+      symbol: "YFI"
+      symbol: "ZIL"
+      symbol: "ZRX"
+    }
+  }
+}
+
+Symbols {
+  value {
+    venues {
+      name: "VENUE_GATEIO"
+      spot: @SymbolLists.list[Gateio].symbol
+    }
+  }
+}
@@ -0,0 +1 @@
+SPARKD__CONFIG_DIR=src/app_sparkd/configurations/py_dummy_md
@@ -0,0 +1,38 @@
+PublisherOptions {
+  value {
+    network_interface: "0.0.0.0"
+    heartbeat_ms: 5000
+    heartbeat_boot_seconds: 60
+    skip_options: true
+    hot_initial: 1024
+    hot_growth: 1024
+    cold_initial: 8192
+    cold_growth: 2048
+    channel_prefix: ""
+    disable_heartbeat: true
+  }
+}
+
+Logging {
+  value {
+    initial_queue_size: 120001
+    max_queue_growth: 4
+    file {
+      default_level: "Info"
+      subsystem {
+        name: "md"
+        level: "Diag"
+      }
+    }
+  }
+}
+
+MarketDataConfig {
+  value {
+    python_md_providers {
+      venue: VENUE_GLBX
+      module_path: "python_md_providers"
+      provider_type: "dummy_md_provider.DummyMdProvider"
+    }
+  }
+}
@@ -0,0 +1,17 @@
+SymbolLists {
+  value {
+    list {
+      key: "Glbx"
+      symbol: "ES"
+    }
+  }
+}
+
+Symbols {
+  value {
+    venues {
+      name: "VENUE_GLBX"
+      future: @SymbolLists.list[Glbx].symbol
+    }
+  }
+}
@@ -0,0 +1,2 @@
+SPARKD__CONFIG_DIR=src/app_sparkd/configurations/sim_dev
+SPARKD__SHM_SIZE=8gb
@@ -1,5 +1,5 @@
 #include "AlgoDefinitionServiceServer.h"
-#include "lib/utils/PBUtils.h"
+#include "spark/utils/PBUtils.h"

 namespace bts {
 namespace api {
@@ -1,9 +1,17 @@
 #pragma once

-#include "lib/api/public/ServiceContext.h"
-#include "lib/api/services/AlgoDefinitionService.server.pb.h"
-#include "lib/persistence/IStore.h"
-#include "lib/persistence/SqliteStore.h"
+#include "spark/api/public/ServiceContext.h"
+#include "spark/api/services/AlgoDefinitionService.server.pb.h"
+#include "spark/api/services/OrderService.client.pb.h"
+#include "spark/api/services/OrderService.pb.h"
+#include "spark/logging/logging.h"
+#include "spark/persistence/IStore.h"
+#include "spark/persistence/SqliteStore.h"
+#include "spark/utils/PBUtils.h"
+#include "spark/utils/TimeStamp.h"
+#include "spark/utils/bts_assert.h"
+#include "spark/utils/bts_no_copy.h"
+#include <memory>

 #include <boost/tokenizer.hpp>
 #include <fmt/format.h>
@@ -1,14 +1,14 @@
 #include "AnalyticsServiceServer.h"
-#include "lib/api/apicore.pb.h"
-#include "lib/api/services/AnalyticsService.pb.h"
-#include "lib/api/services/AnalyticsService.server.pb.h"
-#include "lib/logging/logging.h"
-#include "lib/persistence/IStore.h"
-#include "lib/persistence/SqliteStore.h"
-#include "lib/utils/PBUtils.h"
-#include "lib/utils/TimeStamp.h"
-#include "lib/utils/bts_assert.h"
-#include "server/sparkd/algo/AlgoDefinitionServiceServer.h"
+#include "spark/api/apicore.pb.h"
+#include "spark/api/services/AnalyticsService.pb.h"
+#include "spark/api/services/AnalyticsService.server.pb.h"
+#include "spark/logging/logging.h"
+#include "spark/persistence/IStore.h"
+#include "spark/persistence/SqliteStore.h"
+#include "spark/utils/PBUtils.h"
+#include "spark/utils/TimeStamp.h"
+#include "spark/utils/bts_assert.h"
+#include "sparkd/algo/AlgoDefinitionServiceServer.h"

 #include <boost/algorithm/string.hpp>
 #include <boost/tokenizer.hpp>
@@ -1,6 +1,7 @@
 #pragma once

-#include "lib/api/services/AnalyticsService.server.pb.h"
+#include "spark/api/services/AnalyticsService.server.pb.h"
+#include "spark/utils/bts_no_copy.h"

 namespace bts {
 namespace api {
@@ -1,13 +1,13 @@
 #include "AdminReport.h"

-#include "lib/api/public/ServiceContext.h"
-#include "lib/config/config.h"
-#include "lib/config/protos/logging.pb.h"
-#include "lib/logging/logging.h"
-#include "lib/quickbus/quickbus.h"
+#include "spark/api/public/ServiceContext.h"
+#include "spark/config/config.h"
+#include "spark/config/protos/logging.pb.h"
+#include "spark/logging/logging.h"
+#include "spark/quickbus/quickbus.h"

-#include "lib/api/service_helpers/AdminServiceUtils.h"
-#include "lib/api/services/AdminService.client.pb.h"
+#include "spark/api/service_helpers/AdminServiceUtils.h"
+#include "spark/api/services/AdminService.client.pb.h"

 #include <algorithm>
 #include <boost/algorithm/string.hpp>
@@ -1,12 +1,12 @@
 #include "AutoCert.h"
-#include "lib/utils/PBUtils.h"
+#include "spark/utils/PBUtils.h"

-#include "lib/jetfix/common/InMsgBuffer.h"
-#include "lib/utils/bts_chrono.h"
-#include "lib/utils/datetime.h"
+#include "spark/jetfix/common/InMsgBuffer.h"
+#include "spark/utils/bts_chrono.h"
+#include "spark/utils/datetime.h"

-#include "lib/jetfix/codec/i32_toa.h"
-#include "lib/utils/SecureLogon.h"
+#include "spark/jetfix/codec/i32_toa.h"
+#include "spark/utils/SecureLogon.h"
 #include <boost/array.hpp>
 #include <boost/asio.hpp>
 #include <boost/filesystem.hpp>
Author	SHA1	Message	Date
John Colvin	0cb46fd8ab	cleanup: Remove unnecessary py_module.cpp files We used to have separate modules for md and execution. I forgot to remove these after unifying them to a single module.	2025-06-16 14:31:28 +00:00
John Colvin	c098b5e442	diag: Reduce book log noise in md plugins They're too noisy for Info level.	2025-06-16 14:28:09 +00:00
John Colvin	dc8cd290b6	diag: Optimize logging in bybit exe gw plugin * Format params as json (not python pseudo json) * Format params lazily	2025-06-16 14:17:45 +00:00
John Colvin	17c77d9b31	fix: Use lock_guard instead of lock/unlock calls I just noticed it was technically possible for the "work thread" (in py plugin manager) to be started twice, due to improper lock/unlock calls.	2025-06-13 21:26:36 +00:00
John Colvin	2cf62d8d9a	Increase sleep time in bybit_md_provider keep-alive loop This thread's only job is to stay alive until "stop" is called. It doesn't need to be particularly fast/responsive. Longer sleep intervals reduce contention with the "hot" thread that receives market data.	2025-06-13 20:50:30 +00:00
John Colvin	0328356758	diag: Add security to assert msg when qty tick deduction fails	2025-06-13 20:50:30 +00:00
John Colvin	0e1be3f559	deploy: Add alphaworx_zen_hk_live-preview configuration	2025-06-13 20:50:30 +00:00
John Colvin	f2d1364c4b	fix: Assert that acc name length < max_size (not size) Since we changed the account field from std::array -> bts::core::SmallString	2025-06-13 20:50:30 +00:00
John Colvin	3d2043f926	deploy: Install python requirements in docker No point in using a virtual environment. These containers have one purpose: run sparkd with a constant, predetermined set of py plugins.	2025-06-13 20:50:30 +00:00
John Colvin	e525e740c1	chore: Update alphaworx_zen_hk_live init.py to query new secret I decided to put all alphaworx api keys in one secret. It's easier this way.	2025-06-13 20:50:30 +00:00
John Colvin	0ef72a45fe	fix: Fix config dir env var in bybit_dev profile	2025-06-13 20:50:30 +00:00
John Colvin	db5dbd3666	fix: Fix SmallString numeric parsing, make more robust * Fix by parsing numbers from left to right instead of right to left (previously, would parse "17" to 71) * Support leading +/- signs * Throw if unexpected char encountered * Unit tests	2025-06-13 20:50:30 +00:00
John Colvin	ca27dd3329	fix: Change thread_id type to size_t (not int64_t)	2025-06-13 20:50:30 +00:00
John Colvin	37b34ec6f7	cleanup: Remove unnecessary signal handler in bybit_md_provider.py We handle them properly in sparkd now	2025-06-13 20:50:30 +00:00
John Colvin	f869c97fbe	cleanup: Sort gateio symbols in symbols.conf	2025-06-13 20:50:30 +00:00
John Colvin	a27462b1a3	chore: Add config json for py plugins Support overrides per venue in a separate file, so symbols can be changed after deployment using arham's widget.	2025-06-13 20:50:30 +00:00
John Colvin	8e9abf8451	fix: Init shared_ptr w/ default obj (not null) Prob a typo	2025-06-13 20:50:30 +00:00
John Colvin	1b40382d56	chore: Fix path in .env file (server/sparkd -> app_sparkd)	2025-06-13 20:50:30 +00:00
John Colvin	402d9f5b84	fix build: Remove include of non-existent header I created the file locally, but ended up not using it and forgot to remove an include before I pushed.	2025-06-13 20:50:30 +00:00
John Colvin	20779387eb	fix build: Always return value in non-void fn	2025-06-13 20:50:30 +00:00
John Colvin	7a6bba0f19	fix: Don't compare size_t to -1 in SmallString num parsers	2025-06-13 20:50:30 +00:00
John Colvin	83dc93aeb1	refactor: Manage all py plugins (md and exe) in one class All calls to/from python are performed on one thread via a simple message queue (necessary because of the global interpreter lock).	2025-06-13 20:50:30 +00:00
John Colvin	6107bbdb5d	chore: Add pybit to py venv	2025-06-13 20:50:30 +00:00
John Colvin	71f95f8774	diag: Log incoming books at Info level in bybit_md_provider	2025-06-13 20:50:30 +00:00
John Colvin	4108ee087e	chore: Add bybit execution config to bybit_dev profile	2025-06-13 20:50:30 +00:00
John Colvin	e30bf5cdea	chore: Add numpy to py venv This is needed for md plugins	2025-06-13 20:50:30 +00:00
John Colvin	dce77049b1	fix: Include fill id in format str for ExecutionReport	2025-06-13 20:50:30 +00:00
John Colvin	72ef4f33b0	dev qol: Bind PySecurity __format__ to fmt::formatter impl	2025-06-13 20:50:30 +00:00
John Colvin	a7176f7851	refactor: Change floats to doubles in py bridge types Getting rounding errors with floats	2025-06-13 20:50:30 +00:00
John Colvin	1a0f3197ce	refactor: Use SmallString instead of std::array in ExchangeFillID	2025-06-13 20:50:30 +00:00
John Colvin	e34c2ffec7	Add py bybit exe config to bybit_dev profile	2025-06-13 20:50:30 +00:00
John Colvin	8a10b591ed	refactor: Consolidate all py interop into single lib It was just too hard trying to make it work with separate libs. The main issue was getting the python md and execution modules to recognize PySecurityList and other types defined in the core module.	2025-06-13 20:50:30 +00:00
John Colvin	efc2100177	refactor: Move tick fns into gen-purpose tick_utils.h	2025-06-13 20:50:30 +00:00
John Colvin	a4d57cc97c	Change bybit_dev symbols.conf to list bybit futs (not spots)	2025-06-13 20:50:30 +00:00
John Colvin	3426702d8f	fix: Actually use configured logger in bybit_md_provider.py	2025-06-13 20:50:30 +00:00
John Colvin	8e7a24c971	dev qol: Reduce noisy logs in PositionServicePNLHelper	2025-06-13 20:50:30 +00:00
John Colvin	475dda4e9e	build: Add cmake script to sym link py module and plugins We have several of these (python_core, python_md, python_execution), so the repetative boilerplate code was starting to pile up.	2025-06-13 20:50:30 +00:00
John Colvin	193f3380cb	deploy: Install/activate py venv in sparkd container	2025-06-13 20:50:29 +00:00
John Colvin	d6ac2c5134	dev qol: Add pyrightconfig.json for type hints in py/c++ bridge	2025-06-13 20:50:29 +00:00
John Colvin	076513e79b	Tweak py to get rid of false positive type hint warns	2025-06-13 20:50:29 +00:00
John Colvin	b227fc6254	feat: Support python execution plugins	2025-06-13 20:50:29 +00:00
John Colvin	ca82042669	refactor: Move py security (and related types) to python_core lib So the execution gw plugins can use it (as well as the md plugins).	2025-06-13 20:50:29 +00:00
John Colvin	ec7c0de946	fix: Add missing switch cases for bybit & gateio	2025-06-13 20:50:29 +00:00
John Colvin	391580b9da	fix: Initialize bybit secs as perp futs (not spots)	2025-06-13 20:50:29 +00:00
John Colvin	7e7e185968	dev qol: Add wait/notify functionality to ConcurrentQueue	2025-06-13 20:50:29 +00:00
John Colvin	517e64ceff	chore: Add CANCEL_ORDER op to binance.env	2025-06-13 20:50:29 +00:00
John Colvin	7df9a15d3b	dev qol: Add ConcurrentValueStore, with notify/wait functionality Uses std::condition_variable	2025-06-13 20:50:29 +00:00
John Colvin	b01f36d263	fix: Don't raise ex if var isn't defined in req path	2025-06-13 20:50:29 +00:00
John Colvin	36e555cd13	dev qol: Add gen-purpose validation fns for execution gws	2025-06-13 20:50:29 +00:00
John Colvin	3d0d1c6dba	build: Use new gen-purpose python cmake fns in py md proj Significantly simplifies CMakeLists.txt	2025-06-13 20:50:29 +00:00
John Colvin	f03a055c34	build: Add gen-purpose InitPythonModule, InitPythonPlugin cmake scripts	2025-06-13 20:50:29 +00:00
John Colvin	bfeb5e362a	dev qol: Add fn to make Stopwatch and start it at once	2025-06-13 20:50:29 +00:00
John Colvin	e48f2ec99d	chore: Use same log format as sparkd in py md plugins	2025-06-13 20:50:29 +00:00
John Colvin	cec62215e2	improvement: Set PyBook data in single call to reduce interop overhead	2025-06-13 20:50:29 +00:00
John Colvin	e361d405c9	Reduce log noise in py md providers	2025-06-13 20:50:29 +00:00
John Colvin	020cd44cf3	doc: Improve py md readme * Put examples in a table * Show command to create symlinks for intellisense	2025-06-13 20:50:29 +00:00
John Colvin	c02f3d152a	refactor: Redesign py md so it doesn't lock the global interpreter We can only embed one interpreter in the sparkd process, so all py plugins need to share it. * Move core py interop code to new lib_python_core. We're about to add a new lib for py execution, so it needs to be accesible from both execution and md libs * Fix typo for type hint in py_md_utils.py * Run py md providers on background thread * Rename MdSink -> MdBridge	2025-06-13 20:50:29 +00:00
John Colvin	b4b9e00f90	build: Add lib_spark subdir first (all others depend on it)	2025-06-13 20:50:29 +00:00
John Colvin	8f886008ef	dev qol: Add to_chrono fns to TimeSpan	2025-06-13 20:50:29 +00:00
John Colvin	23e4dbfa07	dev qol: Add -> operators to ValueWrapper for convenience So we can type "foo->" instead of "foo.value()" (much shorter!)	2025-06-13 20:50:29 +00:00
John Colvin	114ac2fdc9	build: Link against system installed ssl to support py interop Unless we want to build python packages ourselves, the underlying clibs they use link to the system installed ssl. Notably, the requests module.	2025-06-13 20:50:29 +00:00
John Colvin	3bdde53d15	chore: Add pybind11, pybind11-stubgen to py venv	2025-06-13 20:50:29 +00:00
John Colvin	bcc4d5fb3d	improvement: Generate py stubs for md providers Provides intellisense for the python_md_module	2025-06-13 20:50:29 +00:00
John Colvin	e93b5c901f	cleanup: Get rid of unecessary namespace prefixes	2025-06-13 20:50:29 +00:00
John Colvin	ccf8b671bf	cleanup: Get rid of "PyList" classes, bind to stl vector instead I wasn't aware this was possible until recently.	2025-06-13 20:50:29 +00:00
John Colvin	0462c97b76	refactor: Move SimpleOrderKeyMap from binance_execution to lib_spark It provides basic functionality that will be useful in other execution gateways.	2025-06-13 20:50:29 +00:00
John Colvin	3e23799772	chore: Add smart parse and pretty name fns for proto enums Mainly to ease the pain of representing enums as strings in python md and execution plugins	2025-06-13 20:50:29 +00:00
John Colvin	2b4119a023	refactor: Move ErrorList from binance_execution to lib_spark It provides basic functionality that will be useful elsewhere	2025-06-13 20:50:29 +00:00
John Colvin	da469270e7	refactor: Use SmallString std::array in AccountName	2025-06-13 20:50:29 +00:00
John Colvin	c9e9be0772	refactor: Use SmallString instead of std::array in ExchangeOrderID	2025-06-13 20:50:29 +00:00
John Colvin	2870fefc15	Add our own "SmallString" class for fixed cap strings This will replace the std::array<char, N> we use in ExecutionReport, etc. among other things	2025-06-13 20:50:29 +00:00
John Colvin	cd1a5946b3	refactor: Move ExecutionReportStore to lib_spark From lib_binance_execution	2025-06-13 20:50:29 +00:00
Adeel Khan	2cf60d74cc	CLEANUP: BYBIT Symbols.conf.	2025-06-13 20:50:29 +00:00
Adeel Khan	10c018e995	ADD: BYBIT MD Working, NaN's also Added.	2025-06-13 20:50:29 +00:00
Adeel Khan	25b937c096	ADD: BYBIT Securities showing up on frontend. Default Contract Spec	2025-06-13 20:50:29 +00:00
Adeel Khan	516b9c0462	ADD: GATEIO md_provider with dynamic contract specs using Batch REST call	2025-06-13 20:50:29 +00:00
Adeel Khan	347ec1a7ea	ADD: GATEIO md_provider with dynamic contract specs 1 REST call per Contract	2025-06-13 20:50:29 +00:00
Adeel Khan	80f4b4bc1c	ADD: GATEIO md_provider with default contract specs	2025-06-13 20:50:29 +00:00
John Colvin	c470984bd2	Add gateio_dev config profile for Adeel	2025-06-13 20:50:29 +00:00
John Colvin	f560eca4d7	cleanup: Remove unused spark.conf file Originally, this spark.conf was hardcoded into compose.yaml, and copied into the sparkd container on each deploy. It hasn't been used since config profiles were introduced.	2025-06-13 20:50:29 +00:00
John Colvin	f9b28dcb7a	Add py_dummy_md config profile to test py md plugin	2025-06-13 20:50:29 +00:00
John Colvin	8275423b89	deploy: Move configurations to new stutter dir "app_sparkd"	2025-06-13 20:50:29 +00:00
John Colvin	86a725227b	Add protos for gateio	2025-06-13 20:50:29 +00:00
John Colvin	43ec58dfe3	Add protos for bybit	2025-06-13 20:50:29 +00:00
John Colvin	922924c31d	build: Move onix fix adapter to its own lib We should keep order execution dependencies out of lib_spark.	2025-06-13 20:50:29 +00:00
John Colvin	2b7dbb6ad3	fix build: Move OnixsUtils to from lib_spark to app_sparkd They're only needed MDP3Main.cpp and ICEMain.cpp. We're one step closer to removing lib_spark's dependency on onixs.	2025-06-13 20:50:29 +00:00
John Colvin	1c431f542e	cleanup: Remove unnecessary onixs include from TrafixOrderStateHandler.h	2025-06-13 20:50:29 +00:00
John Colvin	7fa8e4ca47	feat: Support python md plugins	2025-06-13 20:50:29 +00:00
John Colvin	8579c9fdc5	Add "clear" fn to Book & BookSide Prelude to python market data bindings	2025-06-13 20:50:28 +00:00
John Colvin	ebcb070310	Add "smart" parse fns for a couple proto enums Prelude to python market data bindings. Unlike the normal parse fns, the "smart" variants aren't case sensitive, and can match strings with or without TYPE_NAME_ prefixes.	2025-06-13 20:50:28 +00:00
John Colvin	f5171d8636	build: Move okx execution to its own lib	2025-06-13 20:50:28 +00:00
John Colvin	8af399ca0f	cleanup: Remove unnecessary onixs include from IDGenerator.h	2025-06-13 20:50:28 +00:00
John Colvin	ea48dbabe3	cleanup: Remove unnecessary includes from MDP3Listener.h	2025-06-13 20:50:28 +00:00
John Colvin	9c7f712e11	refactor: Organize libspark proj w/ stuttering include This new structure provides several benefits: * Allows us to install multiple CMake projects in the same destination dir (without include path conflicts). - This makes shipping the trading algo sdk a trivial excercise. Simply copy the lib and include dirs from the install dest. That's it! * Puts us in a position to break up the monolithic libspark into smaller modules, which are easier to manage. Windows specific notes: * Get rid of unnecessary script vs_filters.cmake * Define WebsocketConnectionGateway directly in cs (instead of proto) - The proto would no longer be in the include path passed to protoc. Moving it to c# because there wasn't a compelling reason for it to be a proto in the first place. * Set spark output prefix to "lib" on windows - So we get "libspark.dll" instead of "spark.dll", which conflicts with the csharp lib, "Spark.dll". * Copy libspark.dll to net8.0 & net8.0-windows in a single cmake build target fix build: Copy libspark to both dirs net8.0, net8.0-windows	2025-06-13 20:50:28 +00:00
John Colvin	e4df7215b0	cleanup: Move TradeSummarySignalDispatcher to libspark It is needed by TradeSummarySignal, which is in libspark. This prevents a circular dependency from libspark -> sparkd.	2025-06-13 20:50:28 +00:00
John Colvin	4f404eec24	build: Clean up protobuf cmake scripts Make them more flexible, reusable. This is a precursor to porting over python md provider code from another branch.	2025-06-13 20:50:28 +00:00
John Colvin	633aa362ff	build: Rename external path vars w/ "EXTERNAL_" prefix in cmake The old "SPARK_" prefix felt like a misnomer.	2025-06-13 20:50:28 +00:00
John Colvin	bfd837764a	Add a few arithmetic ops to TimeSpan	2025-06-13 20:50:28 +00:00
John Colvin	99f3a845b8	diag: Log time it takes to start md sources So we can detect bottlenecks	2025-06-13 20:50:28 +00:00
John Colvin	49f9f50fdb	refactor: Split MarketDataGateway up into providers/sources MarketDataGateway still exists, but is greatly simplified. It creates several `IMarketDataProvider`s, each of which generates a list of `IMarketDataSource`s. The market data sources provide market data to an `IMarketDataListener`. * EmumbaMarketDataProvider * CCApiMarketDataProvider * PyMarketDataProvider * OnixsMarketDataProvider * ReplayMarketDataProvider As a side effect, individual py market data sources are now explicitly limited to a single venue (it probably should have been this way all along, since the shared memory writers do not expect updates to involve multiple venues).	2025-06-13 20:50:28 +00:00
John Colvin	bb25eab078	cleanup: Remove dependency on quickfix for string to_lower We shouldn't depend on external libs for trivial things like converting strings to lower case.	2025-06-13 20:50:28 +00:00
				`@@ -0,0 +1 @@`
				`* Connects to Binance test nets for market data and execution`
				`@@ -0,0 +1 @@`
				`SPARKD__CONFIG_DIR=src/app_sparkd/configurations/bybit_dev`