fix build: Upgrade IronPython to 3.4.2

All csprojs need to be on same version (Spark.csproj was on 3.4.1, while DataStream.csproj was on 3.4.2)
2025-06-16 11:08:50 -04:00
2170 changed files with 9945 additions and 20601 deletions
@@ -34,7 +34,7 @@
 #   src/server/sparkd/Dockerfile
 SPARKD__IMAGE=sparkd
 # The config dir for this service to use
-SPARKD__CONFIG_DIR=src/app_sparkd/configurations/dev
+SPARKD__CONFIG_DIR=src/server/sparkd/configurations/dev
 # Mount a specific volume or dir to "~/sparkdata/data" inside the docker.
 # Defaults to a volume named "sparkd_<config_port>" (e.g., sparkd_8000).
 # SPARKD__DATA_DIR=/home/bts/sparkdata/data
@@ -44,6 +44,5 @@ SPARKD__CONFIG_PORT=8000
 SPARKD__QUICKBUS_PORT=8001
 SPARKD__SIM_FIX_PORT=8002
 SPARKD__SHM_SIZE=4gb
-SPARKD__PYTHON_REQUIREMENTS=requirements.txt

 # TODO: add config for other services (pubsub, md-publisher, webapp, api-server, etc.)
@@ -199,27 +199,24 @@
      "request": "launch",
      "program": "",
      "linux": {
-        "program": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd/sparkd",
-      },
-      "args": [
-        "run"
-      ],
-      "cwd": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd",
-      "stopAtEntry": false,
-      "externalConsole": false,
-      "MIMode": "gdb",
-      "setupCommands": [
-        {
-          "description": "Enable pretty-printing for gdb",
-          "text": "-enable-pretty-printing",
-          "ignoreFailures": true
-        },
-        {
-          "description": "Set Disassembly Flavor to Intel",
-          "text": "-gdb-set disassembly-flavor intel",
-          "ignoreFailures": true
-        }
-      ]
+        "preLaunchTask": "Build Sparkd (Debug)",
+        "program": "${workspaceFolder}/build/sparkd-debug-clang/server/sparkd",
+        "args": [
+          "run"
+        ],
+        "stopAtEntry": false,
+        "cwd": "${workspaceFolder}/build/sparkd-debug-clang/server",
+        "environment": [],
+        "externalConsole": false,
+        "MIMode": "gdb",
+        "setupCommands": [
+          {
+            "description": "Enable pretty-printing for gdb",
+            "text": "-enable-pretty-printing",
+            "ignoreFailures": true
+          }
+        ]
+      }
    },
    {
      "name": "Sparkd (Skip Build)",
@@ -227,27 +224,74 @@
      "request": "launch",
      "program": "",
      "linux": {
-        "program": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd/sparkd",
-      },
-      "args": [
-        "run"
-      ],
-      "cwd": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd",
-      "stopAtEntry": false,
-      "externalConsole": false,
-      "MIMode": "gdb",
-      "setupCommands": [
-        {
-          "description": "Enable pretty-printing for gdb",
-          "text": "-enable-pretty-printing",
-          "ignoreFailures": true
-        },
-        {
-          "description": "Set Disassembly Flavor to Intel",
-          "text": "-gdb-set disassembly-flavor intel",
-          "ignoreFailures": true
-        }
-      ]
+        "program": "${workspaceFolder}/build/sparkd-debug-clang/server/sparkd",
+        "args": [
+          "run"
+        ],
+        "stopAtEntry": false,
+        "cwd": "${workspaceFolder}/build/sparkd-debug-clang/server",
+        "environment": [],
+        "externalConsole": false,
+        "MIMode": "gdb",
+        "setupCommands": [
+          {
+            "description": "Enable pretty-printing for gdb",
+            "text": "-enable-pretty-printing",
+            "ignoreFailures": true
+          }
+        ]
+      }
+    },
+    {
+      "name": "Sparkd Test",
+      "type": "cppdbg",
+      "request": "launch",
+      "program": "",
+      "linux": {
+        "preLaunchTask": "Build Sparkd (Debug)",
+        "program": "${workspaceFolder}/build/sparkd-debug-clang/server/sparkd",
+        "args": [
+          "test",
+          "${input:sparkdTest}",
+        ],
+        "stopAtEntry": false,
+        "cwd": "${workspaceFolder}/build/sparkd-debug-clang/server",
+        "environment": [],
+        "externalConsole": false,
+        "MIMode": "gdb",
+        "setupCommands": [
+          {
+            "description": "Enable pretty-printing for gdb",
+            "text": "-enable-pretty-printing",
+            "ignoreFailures": true
+          }
+        ]
+      }
+    },
+    {
+      "name": "Sparkd Test (Skip Build)",
+      "type": "cppdbg",
+      "request": "launch",
+      "program": "",
+      "linux": {
+        "program": "${workspaceFolder}/build/sparkd-debug-clang/server/sparkd",
+        "args": [
+          "test",
+          "${input:sparkdTest}",
+        ],
+        "stopAtEntry": false,
+        "cwd": "${workspaceFolder}/build/sparkd-debug-clang/server",
+        "environment": [],
+        "externalConsole": false,
+        "MIMode": "gdb",
+        "setupCommands": [
+          {
+            "description": "Enable pretty-printing for gdb",
+            "text": "-enable-pretty-printing",
+            "ignoreFailures": true
+          }
+        ]
+      }
    },
  ],
  "inputs": [
@@ -22,9 +22,9 @@ set(
 # can run simultaneously without fighting over the same shm files (e.g., sparkd,
 # api server, etc.).
 set(SHARED_MEMORY_PREFIX "bts" CACHE STRING "Prefix to use when creating shared memory files")
-set(EXTERNALS_BIN "${EXTERNALS_ROOT}/bin")
-set(EXTERNALS_INCLUDE "${EXTERNALS_ROOT}/include")
-set(EXTERNALS_LIB "${EXTERNALS_ROOT}/lib/$<CONFIG>")
+set(SPARK_BIN "${EXTERNALS_ROOT}/bin")
+set(SPARK_INCLUDE "${EXTERNALS_ROOT}/include")
+set(SPARK_LIB "${EXTERNALS_ROOT}/lib/$<CONFIG>")
 set(BUILD_DOCS OFF CACHE BOOL "Build documentation")
 set(BUILD_SDK OFF CACHE BOOL "Build the sdk")
 set(EXPORT_MD_CONSUMER OFF CACHE BOOL "Export (extern) C functions for MD consumer interop")
@@ -36,8 +36,8 @@ else()
  set(BUILD_ADMIN_WEB_APP ON CACHE BOOL "Build the Admin WebApp")
 endif()

-include_directories(BEFORE SYSTEM ${EXTERNALS_INCLUDE} ${EXTERNALS_ROOT}/include/zlib)
-link_directories(${EXTERNALS_LIB})
+include_directories(BEFORE SYSTEM ${SPARK_INCLUDE} ${EXTERNALS_ROOT}/include/zlib)
+link_directories(${SPARK_LIB})

 add_compile_definitions(SHARED_MEMORY_PREFIX="${SHARED_MEMORY_PREFIX}")
 add_compile_definitions(APP_DATA_DIR="${APP_DATA_DIR}")
@@ -47,8 +47,12 @@ endif()
 add_compile_definitions(EXPORT_MD_CONSUMER="${EXPORT_MD_CONSUMER}")
 if(WIN32)
  # subdirs
-  add_subdirectory(src/lib_spark)
+  add_subdirectory(src/lib)
 else()
+  # make sparkd look for libspark.so in its directory
+  set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE)
+  set(CMAKE_INSTALL_RPATH ".")
+
  if(BUILD_SPARKD)
    # Note: All macros that begin w/ 'CCAPI' are ccapi's macros (defined and/or
    # read by their code). We override some below.
@@ -113,15 +117,12 @@ else()
  endif()

  # subdirs
-  add_subdirectory(src/lib_spark)
  if (BUILD_SPARKD)
-    add_subdirectory(src/lib_python_interop)
-    add_subdirectory(src/lib_onixs_fix)
    add_subdirectory(src/lib_binance_execution)
-    add_subdirectory(src/lib_okx_execution)
-    add_subdirectory(src/app_sparkd)
+    add_subdirectory(src/server/sparkd)
    add_subdirectory(src/data_files)
  endif()
+  add_subdirectory(src/lib)
  if (BUILD_ADMIN_WEB_APP)
    add_subdirectory(src/Spark.Admin.WebApp)
  endif()
@@ -131,4 +132,37 @@ else()
  if(BUILD_SDK)
    add_subdirectory(src/sdk)
  endif()
+
+  if(BUILD_SPARKD)
+    add_custom_command(
+      OUTPUT ${CMAKE_BINARY_DIR}/server
+      COMMAND mkdir -p ${CMAKE_BINARY_DIR}/server
+      COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/src/server/sparkd/sparkd ${CMAKE_BINARY_DIR}/server
+      COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/data_files ${CMAKE_BINARY_DIR}/server
+      COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/src/lib/libspark.so ${CMAKE_BINARY_DIR}/server
+      COMMAND ln ARGS -sf ${EXTERNALS_ROOT}/lib/${CMAKE_BUILD_TYPE}/libonixs-fix-engine.so.4 ${CMAKE_BINARY_DIR}/server
+      
+      COMMAND
+        ln ARGS -sf
+        ${EXTERNALS_ROOT}/lib/Release/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.16.0.0
+        ${CMAKE_BINARY_DIR}/server
+      COMMAND ln ARGS -sf ${EXTERNALS_ROOT}/lib/Release/libACE-5.7.5.so ${CMAKE_BINARY_DIR}/server
+      COMMAND mkdir ARGS -p ${CMAKE_BINARY_DIR}/server/plugins
+      COMMENT "Creating server runtime layout"
+      DEPENDS sparkd data_files_dir)
+
+    add_custom_target(server_layout ALL DEPENDS ${CMAKE_BINARY_DIR}/server)
+
+    install(
+      DIRECTORY "${CMAKE_BINARY_DIR}/server/plugins"
+      DESTINATION "${CMAKE_INSTALL_BINDIR}")
+    install(
+      FILES
+      "${SPARK_LIB}/libACE-5.7.5.so"
+      "${SPARK_LIB}/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.16.0.0"
+      "${SPARK_LIB}/libonixs-fix-engine.so.4.10.1"
+      "${SPARK_LIB}/libonixs-fix-engine.so.4"
+      "${SPARK_LIB}/libonixs-fix-engine.so"
+      DESTINATION "${CMAKE_INSTALL_BINDIR}")
+  endif()
 endif()
@@ -31,8 +31,9 @@
        "CMAKE_SHARED_LINKER_FLAGS_RELEASE": "/OPT:REF /OPT:ICF /LTCG",
        "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
        "EXTERNALS_ROOT": "${sourceDir}/externals/windows",
-        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0;${sourceDir}/build/$<CONFIG>/bin/net8.0-windows",
+        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0",
+        "DOTNET_WINDOWS_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0-windows",
        "EXPORT_MD_CONSUMER": {
          "type": "BOOL",
          "value": "ON"
@@ -57,12 +58,11 @@
        "APP_DATA_DIR_OVERRIDE": "${dollar}{SPARKD_DATA_DIR}",
        "CMAKE_BUILD_WITH_INSTALL_RPATH": {
          "type": "BOOL",
-          "value": "OFF"
+          "value": "ON"
        },
        "CMAKE_CXX_STANDARD": "20",
        "CMAKE_INSTALL_RPATH": ".",
        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
-        "CMAKE_PREFIX_PATH": "${sourceDir}/cmake/;${sourceDir}/.venv/lib/python3.10/site-packages/pybind11/share/cmake/pybind11",
        "CMAKE_EXE_LINKER_FLAGS": "-Wno-undef",
        "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
        "DATABENTO_USE_EXTERNAL_JSON": {
@@ -93,7 +93,7 @@
          "value": "ON"
        },
        "CCAPI_LOG_LEVEL": "Info",
-        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/md_dev/*.conf;spark/config/profiles/default/*.conf"
+        "CONFIG_DEFAULTS_PATH": "config/profiles/md_dev/*.conf;config/profiles/default/*.conf"
      }
    },

@@ -104,7 +104,7 @@
      "cacheVariables": {
        "CMAKE_BUILD_TYPE": "Release",
        "CCAPI_LOG_LEVEL": "Warn",
-        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/md_prod/*.conf;spark/config/profiles/default/*.conf"
+        "CONFIG_DEFAULTS_PATH": "config/profiles/md_prod/*.conf;config/profiles/default/*.conf"
      }
    },

@@ -186,11 +186,7 @@
        "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
-        "CMAKE_EXPORT_COMPILE_COMMANDS": {
-          "type": "BOOL",
-          "value": "ON"
-        },
-        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf",
+        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0",
        "EXPORT_MD_CONSUMER": {
          "type": "BOOL",
@@ -231,7 +227,7 @@
      "cacheVariables": {
        "APP_DATA_DIR": "${dollar}{APPDATA}/bts/DataStream",
        "APP_DATA_DIR_OVERRIDE": "${dollar}{DATA_STREAM_DATA_DIR}",
-        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf;spark/config/profiles/data_stream/*.conf",
+        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf",
        "SHARED_MEMORY_PREFIX": "data_stream"
      }
    },
@@ -266,7 +262,8 @@
        "CMAKE_CXX_COMPILER": "clang++",
        "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
-        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf;spark/config/profiles/data_stream/*.conf",
+        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
+        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf",
        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
        "EXPORT_MD_CONSUMER": {
          "type": "BOOL",
@@ -1,7 +0,0 @@
-include_guard(GLOBAL)
-
-macro(export_to_parent)
-  foreach(var IN ITEMS ${ARGN})
-    set(${var} ${${var}} PARENT_SCOPE)
-  endforeach()
-endmacro()
@@ -1,142 +0,0 @@
-include_guard(DIRECTORY)
-include(${CMAKE_SOURCE_DIR}/cmake/ExportToParent.cmake)
-include(${CMAKE_SOURCE_DIR}/cmake/InstallBasic.cmake)
-include(${CMAKE_SOURCE_DIR}/cmake/ProtobufGenerateCpp.cmake)
-include(${CMAKE_SOURCE_DIR}/cmake/SetFirstDefined.cmake)
-include(${CMAKE_SOURCE_DIR}/cmake/safe_dep_glob.cmake)
-
-function(init_library)
-  set(_options SHARED STATIC)
-  set(_one_value_args
-    NAME
-    SOURCE_DIR
-    INCLUDE_DIR
-  )
-  set(_multi_value_args)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-  if (_ARGS_SHARED AND _ARGS_STATIC)
-    message(FATAL_ERROR "Invalid args passed to init_library: cannot specify STATIC and SHARED")
-  endif()
-  set_first_defined(_name _ARGS_NAME PROJECT_NAME)
-  set_first_defined(_source_dir _ARGS_SOURCE_DIR _name)
-  set_first_defined(_include_dir _ARGS_INCLUDE_DIR _source_dir)
-
-  # Cpp sources
-  file(
-    GLOB_RECURSE ${_name}_SOURCES
-    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
-    ${_source_dir}/*.cpp
-  )
-  # Note: cpp files inside 'tests' dir(s) are considered tests.
-  # Remove tests from main sources, and add them to test sources
-  foreach(item ${${_name}_SOURCES})
-    if(${item} MATCHES "tests/.*\.cpp$")
-      list(APPEND ${_name}_TEST_SOURCES ${item})
-    endif()
-  endforeach()
-  foreach(item ${${_name}_TEST_SOURCES})
-    list(REMOVE_ITEM ${_name}_SOURCES ${item})
-  endforeach()
-
-  # Proto sources 
-  file(
-    GLOB_RECURSE ${_name}_PROTO_SOURCES
-    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
-    ${_source_dir}/protobufs/*.proto
-  )
-  update_deps_file("${${_name}_PROTO_SOURCES};")
-  protobuf_generate_cpp(
-    CXX_SOURCES ${_name}_CXX_PROTO_SOURCES
-    CXX_HEADERS ${_name}_CXX_PROTO_HEADERS
-    CXX_OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}
-    PROTO_INCLUDE_DIRS
-      ${EXTERNALS_INCLUDE}
-      ${CMAKE_CURRENT_SOURCE_DIR}
-    PROTO_SOURCES
-      ${${_name}_PROTO_SOURCES}
-  )
-
-  export_to_parent(
-    ${_name}_SOURCES
-    ${_name}_TEST_SOURCES
-    ${_name}_PROTO_SOURCES
-    ${_name}_CXX_PROTO_SOURCES
-    ${_name}_CXX_PROTO_HEADERS
-  )
-
-  # Add main lib
-  if (_ARGS_STATIC)
-    add_library(${_name} STATIC ${${_name}_SOURCES} ${${_name}_CXX_PROTO_SOURCES})
-  elseif(_ARGS_SHARED)
-    add_library(${_name} SHARED ${${_name}_SOURCES} ${${_name}_CXX_PROTO_SOURCES})
-  else()
-    add_library(${_name} ${${_name}_SOURCES} ${${_name}_CXX_PROTO_SOURCES})
-  endif()
-  target_include_directories(
-    ${_name}
-    SYSTEM
-    BEFORE
-    PUBLIC
-      $<BUILD_INTERFACE:${EXTERNALS_ROOT}/include>
-      $<BUILD_INTERFACE:${EXTERNALS_ROOT}/include/zlib>
-  )
-  target_include_directories(
-    ${_name}
-    PUBLIC
-      $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
-      $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>
-  )
-  # Include protos (if any)
-  if (${_name}_PROTO_SOURCES)
-    target_include_directories(
-      ${_name}
-      PUBLIC
-        $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
-    )
-  endif()
-  target_link_directories(
-    ${_name}
-    PUBLIC
-      $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
-      $<BUILD_INTERFACE:${EXTERNALS_ROOT}/lib/$<CONFIG>>
-      $<INSTALL_INTERFACE:${CMAKE_INSTALL_LIBDIR}>
-  )
-  # Install main lib
-  foreach(_hdr ${${_name}_CXX_PROTO_HEADERS})
-    file(RELATIVE_PATH _relative_path ${CMAKE_CURRENT_BINARY_DIR} ${_hdr})
-    install(FILES ${_hdr} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/${_relative_path})
-  endforeach()
-  install_basic(
-    NAME ${_name}
-    INCLUDE_DIR ${_include_dir}
-  )
-
-  # Add tests executable (if there are any)
-  if (${_name}_TEST_SOURCES AND NOT WIN32)
-    add_executable(${_name}-tests ${${_name}_TEST_SOURCES})
-
-    get_target_property(_lib_type ${_name} TYPE)
-    if (_lib_type STREQUAL "SHARED_LIBRARY")
-      # Set RPATH so the tests exe knows where to find the shared lib.
-      set_target_properties(
-        ${_name}-tests
-        PROPERTIES
-          BUILD_RPATH "$ORIGIN"
-          INSTALL_RPATH "$ORIGIN/../${CMAKE_INSTALL_LIBDIR}"
-      )
-    endif()
-
-    # Link main lib
-    target_link_directories(
-      ${_name}-tests
-      PRIVATE
-        $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
-        $<INSTALL_INTERFACE:${CMAKE_INSTALL_LIBDIR}>
-    )
-    target_link_libraries(${_name}-tests PRIVATE ${_name})
-
-    # Catch2
-    find_package(Catch2 CONFIG REQUIRED COMPONENTS Catch2WithMain)
-    target_link_libraries(${_name}-tests PRIVATE Catch2::Catch2WithMain)
-  endif()
-endfunction()
@@ -1,71 +0,0 @@
-include_guard(DIRECTORY)
-include(GNUInstallDirs)
-include(${CMAKE_SOURCE_DIR}/cmake/ExportToParent.cmake)
-include(${CMAKE_SOURCE_DIR}/cmake/SetFirstDefined.cmake)
-
-function(init_python_module)
-  set(_options)
-  set(_one_value_args
-    NAME
-  )
-  set(_multi_value_args
-    SOURCES
-  )
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-  set_first_defined(_name _ARGS_NAME PROJECT_NAME)
-  set_first_defined(_sources _ARGS_SOURCES ${_name}_SOURCES)
-
-  set(module_proj ${_name}_module)
-  set(${module_proj}_SOURCES ${_sources})
-  pybind11_add_module(${module_proj} SHARED ${_sources})
-  target_link_libraries(
-    ${module_proj}
-    PUBLIC
-      pybind11::embed
-      spark
-      ${_name}
-  )
-  target_include_directories(
-    ${module_proj}
-    PRIVATE
-      $<BUILD_INTERFACE:${pybind11_INCLUDE_DIRS}>
-  )
-  target_include_directories(
-    ${module_proj}
-    PUBLIC
-      ${Python3_INCLUDE_DIRS}
-      $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
-      $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>
-  )
-  add_custom_command(
-    OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/stubs/${_name}.pyi
-    DEPENDS ${_name} ${module_proj}
-    COMMAND PYTHONPATH=${CMAKE_CURRENT_BINARY_DIR} pybind11-stubgen --numpy-array-use-type-var --ignore-invalid-expressions '.*' ${_name}_module
-    COMMENT "Generating py stubs"
-  )
-  add_custom_target(
-    "${_name}_py_stubs"
-    ALL
-    DEPENDS
-      ${CMAKE_CURRENT_BINARY_DIR}/stubs/${_name}.pyi
-  )
-  install(
-    TARGETS ${module_proj}
-    EXPORT ${module_proj}Targets
-    RUNTIME
-      DESTINATION ${CMAKE_INSTALL_BINDIR}
-    LIBRARY
-      DESTINATION ${CMAKE_INSTALL_LIBDIR}
-    ARCHIVE
-      DESTINATION ${CMAKE_INSTALL_LIBDIR}
-  )
-  foreach(dir ${pybind11_INCLUDE_DIRS})
-    install(DIRECTORY ${dir} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
-  endforeach()
-  install(
-    EXPORT ${module_proj}Targets
-    FILE ${module_proj}Targets.cmake
-    NAMESPACE ${_name}::
-    DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name}
-  )
-endfunction()
@@ -1,65 +0,0 @@
-include_guard(DIRECTORY)
-include(GNUInstallDirs)
-
-function(init_python_plugin)
-  set(_options)
-  set(_one_value_args NAME PLUGIN_NAME)
-  set(_multi_value_args SOURCES)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-  set_first_defined(_name _ARGS_NAME PROJECT_NAME)
-  set(_plugin_name ${_ARGS_PLUGIN_NAME})
-  if (NOT DEFINED _ARGS_PLUGIN_NAME)
-    set(_plugin_name ${_name}_plugins)
-  endif()
-  set(${_plugin_name}_SOURCES ${_ARGS_SOURCES})
-  if (NOT DEFINED _ARGS_SOURCES)
-    file(
-      GLOB_RECURSE ${_plugin_name}_SOURCES
-      RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
-      ${_plugin_name}/*.py)
-  endif()
-  set(src_outs)
-  foreach(_src ${${_plugin_name}_SOURCES})
-    set(_src_in ${CMAKE_CURRENT_SOURCE_DIR}/${_src})
-    set(_src_out ${CMAKE_CURRENT_BINARY_DIR}/${_src})
-    add_custom_command(
-      OUTPUT ${_src_out}
-      COMMAND ${CMAKE_COMMAND} -E copy ${_src_in} ${_src_out}
-      DEPENDS ${_src_in}
-      COMMENT "Copying ${_plugin_name} plugin ${_src} to ${_src_out}"
-    )
-    list(APPEND src_outs ${_src_out})
-  endforeach()
-  
-  add_custom_target(copy_${_plugin_name} ALL DEPENDS ${src_outs})
-  
-  add_library(${_plugin_name} INTERFACE)
-  add_library(${_name}::${_plugin_name} ALIAS ${_plugin_name})
-  
-  set_target_properties(
-    ${_plugin_name}
-    PROPERTIES
-      INTERFACE_PYTHON_SCRIPTS_DIR
-        $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/${_plugin_name}>$<INSTALL_INTERFACE:${CMAKE_INSTALL_BINDIR}/${_plugin_name}>
-  )
-  
-  set_target_properties(
-    ${_plugin_name}
-    PROPERTIES
-    INTERFACE_PYTHON_SCRIPTS $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/${${_plugin_name}_SOURCES}>$<INSTALL_INTERFACE:${CMAKE_INSTALL_BINDIR}/${${_plugin_name}_SOURCES}>
-  )
-  
-  install(
-    TARGETS ${_plugin_name}
-    EXPORT ${_plugin_name}Targets
-    DESTINATION ${CMAKE_INSTALL_BINDIR}
-  )
-  
-  install(
-    EXPORT ${_plugin_name}Targets
-    FILE ${_plugin_name}Targets.cmake
-    NAMESPACE ${_name}::
-    DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name}
-  )
-  install(DIRECTORY ${_plugin_name} DESTINATION ${CMAKE_INSTALL_BINDIR})
-endfunction()
@@ -1,39 +0,0 @@
-include_guard(DIRECTORY)
-include(GNUInstallDirs)
-include(${CMAKE_SOURCE_DIR}/cmake/PackBasic.cmake)
-include(${CMAKE_SOURCE_DIR}/cmake/SetFirstDefined.cmake)
-
-function(install_basic)
-  set(_options)
-  set(_one_value_args NAME INCLUDE_DIR)
-  set(_multi_value_args)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-  set_first_defined(_name _ARGS_NAME PROJECT_NAME)
-  set_first_defined(_include_dir _ARGS_INCLUDE_DIR _name)
-
-  file(
-    GLOB_RECURSE ${_name}_HEADERS
-    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
-    ${_include_dir}/*.h)
-
-  target_sources(${_name}
-    PUBLIC
-      FILE_SET HEADERS FILES ${${_name}_HEADERS})
-
-  install(
-    TARGETS ${_name}
-    EXPORT ${_name}Targets
-    RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
-    LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
-    ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
-    FILE_SET HEADERS DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}
-  )
-
-  install(
-    EXPORT ${_name}Targets
-    FILE ${_name}Targets.cmake
-    NAMESPACE ${_name}::
-    DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name})
-
-  pack_basic()
-endfunction()
@@ -1,44 +0,0 @@
-include_guard(DIRECTORY)
-include(CMakePackageConfigHelpers)
-include(${CMAKE_SOURCE_DIR}/cmake/SetFirstDefined.cmake)
-
-function(pack_basic)
-  set(_options)
-  set(_one_value_args NAME)
-  set(_multi_value_args)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-  set_first_defined(_name _ARGS_NAME PROJECT_NAME)
-
-  find_package(Git QUIET)
-
-  if(GIT_FOUND)
-    execute_process(
-      COMMAND ${GIT_EXECUTABLE} describe --tags --always --dirty
-      WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
-      OUTPUT_VARIABLE PROJECT_VERSION
-      OUTPUT_STRIP_TRAILING_WHITESPACE
-    )
-  endif()
-
-  if(NOT GIT_FOUND OR PROJECT_VERSION STREQUAL "")
-    set(PROJECT_VERSION "0.0.0")
-  endif()
-
-  write_basic_package_version_file(
-    "${CMAKE_CURRENT_BINARY_DIR}/${_name}ConfigVersion.cmake"
-    VERSION ${PROJECT_VERSION}
-    COMPATIBILITY AnyNewerVersion
-  )
-
-  configure_package_config_file(
-    "${CMAKE_CURRENT_SOURCE_DIR}/Config.cmake.in"
-    "${CMAKE_CURRENT_BINARY_DIR}/${_name}Config.cmake"
-    INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name}
-  )
-
-  install(
-    FILES
-      "${CMAKE_CURRENT_BINARY_DIR}/${_name}Config.cmake"
-      "${CMAKE_CURRENT_BINARY_DIR}/${_name}ConfigVersion.cmake"
-    DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name})
-endfunction()
@@ -1,23 +1,12 @@
-set(Protobuf_PROTOC_EXECUTABLE "${EXTERNALS_BIN}/protoc")
-set(PROTOBUF_INCLUDE_DIRS "${EXTERNALS_INCLUDE}")
+set(Protobuf_PROTOC_EXECUTABLE "${SPARK_BIN}/protoc")
+set(PROTOBUF_INCLUDE_DIRS "${SPARK_INCLUDE}")
 set(PROTOBUF_STATIC_LIB "libprotobuf")
-set(_protobuf_src_dir ${CMAKE_SOURCE_DIR}/src/lib_spark)
-set(_protobuf_bin_dir ${CMAKE_BINARY_DIR}/src/lib_spark)

-function(Protobuf_Generate_Cpp)
-  set(_options)
-  set(_one_value_args CXX_SOURCES CXX_HEADERS CXX_OUT_DIR)
-  set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-  set(${_ARGS_CXX_SOURCES})
-  set(${_ARGS_CXX_HEADERS})
+function(Protobuf_Generate_Cpp SRCS HDRS)
+  set(${SRCS})
+  set(${HDRS})

-  set(_proto_include_cli_args)
-  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
-    list(APPEND _proto_include_cli_args -I ${_dir})
-  endforeach()
-
-  foreach(FIL ${_ARGS_PROTO_SOURCES})
+  foreach(FIL ${ARGN})
    get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
    get_filename_component(FIL_WE ${FIL} NAME_WE)
    get_filename_component(FIL_DIR ${FIL} DIRECTORY)
@@ -25,56 +14,48 @@ function(Protobuf_Generate_Cpp)
      set(FIL_WE "${FIL_DIR}/${FIL_WE}")
    endif()

-    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc")
-    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h")
+    list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc")
+    list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h")

    add_custom_command(
-      OUTPUT "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
-             "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
+      OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc"
+             "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h"
      COMMAND
-        ${Protobuf_PROTOC_EXECUTABLE} ARGS --cpp_out=${_ARGS_CXX_OUT_DIR} ${_proto_include_cli_args} ${ABS_FIL}
+        ${Protobuf_PROTOC_EXECUTABLE} ARGS --cpp_out=${CMAKE_BINARY_DIR}/src -I
+        ${PROTOBUF_INCLUDE_DIRS} -I ${CMAKE_SOURCE_DIR}/src ${ABS_FIL}
      DEPENDS ${ABS_FIL}
      COMMENT "Running C++ protocol buffer compiler on ${FIL}"
      VERBATIM)
  endforeach()

-  set_source_files_properties(${${_ARGS_CXX_SOURCES}} ${${_ARGS_CXX_HEADERS}} PROPERTIES GENERATED TRUE)
+  set_source_files_properties(${${SRCS}} ${${HDRS}} PROPERTIES GENERATED TRUE)

  # remove some warnings from protobuf generated files
  if(WIN32)
-    set_source_files_properties(${${_ARGS_CXX_SOURCES}} PROPERTIES COMPILE_FLAGS
+    set_source_files_properties(${${SRCS}} PROPERTIES COMPILE_FLAGS
                                                      "/wd4309 /wd4838")
  else()
    set_source_files_properties(
-      ${${_ARGS_CXX_SOURCES}}
+      ${${SRCS}}
      PROPERTIES
        COMPILE_FLAGS
        "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
    )
  endif()

-  set(${_ARGS_CXX_SOURCES}
-      ${${_ARGS_CXX_SOURCES}}
+  set(${SRCS}
+      ${${SRCS}}
      PARENT_SCOPE)
-  set(${_ARGS_CXX_HEADERS}
-      ${${_ARGS_CXX_HEADERS}}
+  set(${HDRS}
+      ${${HDRS}}
      PARENT_SCOPE)
 endfunction()

-function(Service_Compiler_Generate_Cpp)
-  set(_options)
-  set(_one_value_args CXX_SOURCES CXX_HEADERS CXX_OUT_DIR)
-  set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-  set(${_ARGS_CXX_SOURCES})
-  set(${_ARGS_CXX_HEADERS})
+function(Service_Compiler_Generate_Cpp SRCS HDRS)
+  set(${SRCS})
+  set(${HDRS})

-  set(_proto_include_cli_args)
-  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
-    list(APPEND _proto_include_cli_args -I ${_dir})
-  endforeach()
-
-  foreach(FIL ${_ARGS_PROTO_SOURCES})
+  foreach(FIL ${ARGN})
    get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
    get_filename_component(FIL_WE ${FIL} NAME_WE)
    get_filename_component(FIL_DIR ${FIL} DIRECTORY)
@@ -82,58 +63,58 @@ function(Service_Compiler_Generate_Cpp)
      set(FIL_WE "${FIL_DIR}/${FIL_WE}")
    endif()

-    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc")
-    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h")
+    list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc")
+    list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h")

-    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc")
-    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h")
+    list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.cc")
+    list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.h")

-    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc")
-    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h")
+    list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.cc")
+    list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.h")

    set(TEMPLATES
-        ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/client.pb.cc
-        ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/client.pb.h
-        ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/server.pb.cc
-        ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/server.pb.h
+        ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/templates/cpp/client.pb.cc
+        ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/templates/cpp/client.pb.h
+        ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/templates/cpp/server.pb.cc
+        ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/templates/cpp/server.pb.h
    )

-    set(_plugin_py ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/plugin.py)
    if(WIN32)
-      set(_plugin_cmd ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/plugin.cmd)
      add_custom_command(
-        OUTPUT "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
-               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
-               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc"
-               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h"
-               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc"
-               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h"
+        OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc"
+               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h"
+               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.cc"
+               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.h"
+               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.cc"
+               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.h"
        COMMAND SET PYTHONPATH=${SPARK_PYTHON}
        COMMAND ${CMAKE_SOURCE_DIR}/.venv/Scripts/activate.bat
        COMMAND
-          ${Protobuf_PROTOC_EXECUTABLE} ARGS --cpp_out=${_ARGS_CXX_OUT_DIR}
-          --plugin=protoc-gen-service=${_plugin_cmd}
-          --service_out=lang=cpp:${_ARGS_CXX_OUT_DIR} -I
-          ${PROTOBUF_INCLUDE_DIRS} -I ${_protobuf_src_dir} ${ABS_FIL}
-        DEPENDS ${ABS_FIL} ${TEMPLATES} ${_plugin_cmd} ${_plugin_py}
+          ${Protobuf_PROTOC_EXECUTABLE} ARGS --cpp_out=${CMAKE_BINARY_DIR}/src
+          --plugin=protoc-gen-service=${CMAKE_SOURCE_DIR}/src/lib/api/service_compiler/plugin.cmd
+          --service_out=lang=cpp:${CMAKE_BINARY_DIR}/src -I
+          ${PROTOBUF_INCLUDE_DIRS} -I ${CMAKE_SOURCE_DIR}/src ${ABS_FIL}
+        DEPENDS ${ABS_FIL} ${TEMPLATES}
+                ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/plugin.py
        COMMENT "Running C++ service compiler on ${FIL}"
        VERBATIM)
    else()
      add_custom_command(
-        OUTPUT "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
-               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
-               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc"
-               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h"
-               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc"
-               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h"
+        OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc"
+               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h"
+               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.cc"
+               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.h"
+               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.cc"
+               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.h"
        COMMAND ${CMAKE_SOURCE_DIR}/tools/venv_exec.sh
        ARGS
          ${Protobuf_PROTOC_EXECUTABLE}
-          --cpp_out=${_ARGS_CXX_OUT_DIR}
-          --plugin=protoc-gen-service=${_plugin_py}
-          --service_out=lang=cpp,make_file_name=${ABS_FIL}:${_ARGS_CXX_OUT_DIR}
-          ${_proto_include_cli_args} ${ABS_FIL}
-        DEPENDS ${ABS_FIL} ${TEMPLATES} ${_plugin_py}
+          --cpp_out=${CMAKE_BINARY_DIR}/src
+          --plugin=protoc-gen-service=${CMAKE_SOURCE_DIR}/src/lib/api/service_compiler/plugin.py
+          --service_out=lang=cpp,make_file_name=${ABS_FIL}:${CMAKE_BINARY_DIR}/src
+          -I ${PROTOBUF_INCLUDE_DIRS} -I ${CMAKE_SOURCE_DIR}/src ${ABS_FIL}
+        DEPENDS ${ABS_FIL} ${TEMPLATES}
+                ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/plugin.py
        COMMENT "Running C++ service compiler on ${FIL}"
        VERBATIM)
    endif()
@@ -142,22 +123,22 @@ function(Service_Compiler_Generate_Cpp)

  # remove some warnings from protobuf generated files
  if(WIN32)
-    set_source_files_properties(${${_ARGS_CXX_SOURCES}} PROPERTIES COMPILE_FLAGS
+    set_source_files_properties(${${SRCS}} PROPERTIES COMPILE_FLAGS
                                                      "/wd4309 /wd4838")
  else()
    set_source_files_properties(
-      ${${_ARGS_CXX_SOURCES}}
+      ${${SRCS}}
      PROPERTIES
        COMPILE_FLAGS
        "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
    )
  endif()

-  set(${_ARGS_CXX_SOURCES}
-      ${${_ARGS_CXX_SOURCES}}
+  set(${SRCS}
+      ${${SRCS}}
      PARENT_SCOPE)
-  set(${_ARGS_CXX_HEADERS}
-      ${${_ARGS_CXX_HEADERS}}
+  set(${HDRS}
+      ${${HDRS}}
      PARENT_SCOPE)

 endfunction()
@@ -1,83 +0,0 @@
-include_guard(GLOBAL)
-
-function(protobuf_generate_cpp)
-  set(_options)
-  set(_one_value_args
-    CXX_SOURCES
-    CXX_HEADERS
-    CXX_OUT_DIR
-    PROTOC
-  )
-  set(_multi_value_args
-    PROTO_INCLUDE_DIRS
-    PROTO_SOURCES
-  )
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-
-  if (NOT DEFINED _ARGS_PROTOC)
-    if (WIN32)
-      set(_ARGS_PROTOC "${EXTERNALS_BIN}/protoc.exe")
-    else()
-      set(_ARGS_PROTOC "${EXTERNALS_BIN}/protoc")
-    endif()
-  endif()
-  set(${_ARGS_CXX_SOURCES})
-  set(${_ARGS_CXX_HEADERS})
-
-  set(_proto_include_cli_args)
-  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
-    list(APPEND _proto_include_cli_args -I ${_dir})
-  endforeach()
-
-  foreach(_fil ${_ARGS_PROTO_SOURCES})
-    get_filename_component(_abs_fil ${_fil} ABSOLUTE)
-    get_filename_component(_fil_we ${_fil} NAME_WE)
-    get_filename_component(_fil_dir ${_fil} DIRECTORY)
-    if(_fil_dir)
-      set(_fil_we "${_fil_dir}/${_fil_we}")
-    endif()
-    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.cc")
-    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.h")
-    message(STATUS "add_custom_command for ${_fil} -> ${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.h")
-    # TODO: Pass proto sources by relative path (so this fn can be called from
-    # arbitrary subdirs).
-    add_custom_command(
-      OUTPUT
-        "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.cc"
-        "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.h"
-      COMMAND
-        ${_ARGS_PROTOC}
-      ARGS
-        --cpp_out=${_ARGS_CXX_OUT_DIR} ${_proto_include_cli_args} ${_abs_fil}
-      DEPENDS ${_abs_fil}
-      COMMENT "Running C++ protocol buffer compiler on ${_fil}"
-      VERBATIM
-    )
-  endforeach()
-
-  set_source_files_properties(
-    ${${_ARGS_CXX_SOURCES}}
-    ${${_ARGS_CXX_HEADERS}}
-    PROPERTIES
-      GENERATED TRUE
-  )
-
-  # remove some warnings from protobuf generated files
-  if(WIN32)
-    set_source_files_properties(
-      ${${_ARGS_CXX_SOURCES}}
-      PROPERTIES
-        COMPILE_FLAGS "/wd4309 /wd4838"
-    )
-  else()
-    set_source_files_properties(
-      ${${_ARGS_CXX_SOURCES}}
-      PROPERTIES
-        COMPILE_FLAGS "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
-    )
-  endif()
-
-  set(${_ARGS_CXX_SOURCES} ${${_ARGS_CXX_SOURCES}} PARENT_SCOPE)
-  set(${_ARGS_CXX_HEADERS} ${${_ARGS_CXX_HEADERS}} PARENT_SCOPE)
-endfunction()
-
@@ -1,61 +0,0 @@
-include_guard(GLOBAL)
-
-function(protobuf_generate_cpp_config)
-  set(_options)
-  set(_one_value_args
-    PROTO_SOURCES_HEADER
-    PROTO_DEFAULTS_HEADER
-  )
-  set(_multi_value_args
-    PROTO_SOURCES
-    PROTO_DEFAULTS
-  )
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-
-  # Configure the 'config_proto_files' header. It provides an array of strings populated with the
-  # files in PROTO_SOURCES. At runtime, we use this array to generate a special 'config' proto
-  # message with fields for each message defined in said files. A global instance of this 'config'
-  # proto is then constructed, and populated with [compile-time] defaults, and [runtime] overrides
-  # (i.e., in spark.conf). Calls to `bts::config::get<T>()` retrieve the value from the field
-  # associated with type `T` in this global instance.
-  message(STATUS "PROTO_SOURCES: ${_ARGS_PROTO_SOURCES}")
-  message(STATUS "PROTO_SOURCES_HEADER: ${_ARGS_PROTO_SOURCES_HEADER}")
-  message(STATUS "PROTO_DEFAULTS_HEADER: ${_ARGS_PROTO_DEFAULTS_HEADER}")
-  if (_ARGS_PROTO_SOURCES_HEADER)
-    set(PROTO_SOURCES)
-    set(PROTO_SOURCES_CXX_ARRAY_ITEMS "")
-    foreach(_proto_path ${_ARGS_PROTO_SOURCES})
-      list(APPEND PROTO_SOURCES ${_proto_path})
-      string(APPEND PROTO_SOURCES_CXX_ARRAY_ITEMS "\"${_proto_path}\",\n")
-    endforeach()
-    list(LENGTH PROTO_SOURCES PROTO_SOURCES_LENGTH)
-    message(STATUS "CONFIGURING PROTO_SOURCES_HEADER): ${_ARGS_PROTO_SOURCES_HEADER}")
-    configure_file(${CMAKE_SOURCE_DIR}/cmake/config_proto_files.h.in ${_ARGS_PROTO_SOURCES_HEADER})
-  endif()
-
-  # Generate static defaults. Ideally, this would be a single string, but we use an array because
-  # the compiler complains if the string is longer than 65k.
-  set(_cfg_defaults_str "inline const char * config_static_defaults[] = { ")
-
-  foreach(_cfg_file_path ${_ARGS_PROTO_DEFAULTS})
-    file(READ ${_cfg_file_path} _cfg_file_content)
-    string(REPLACE "\"" "\\\"" _cfg_file_content_escaped ${_cfg_file_content})
-    string(REPLACE "\n" ";" _cfg_lines ${_cfg_file_content_escaped})
-
-    foreach(_line ${_cfg_lines})
-      string(APPEND _cfg_defaults_str "\n  \"${_line}\",")
-    endforeach(_line)
-  endforeach(_cfg_file_path)
-
-  string(APPEND _cfg_defaults_str "};")
-  set(_old_cfg_defaults_str "")
-
-  if(EXISTS ${_ARGS_PROTO_DEFAULTS_HEADER})
-    file(READ ${_ARGS_PROTO_DEFAULTS_HEADER} _old_cfg_defaults_str)
-  endif()
-
-  if(NOT _cfg_defaults_str STREQUAL _old_cfg_defaults_str)
-    file(WRITE "${_ARGS_PROTO_DEFAULTS_HEADER}" "${_cfg_defaults_str}")
-  endif()
-endfunction()
-
@@ -10,7 +10,7 @@ function(protobuf_generate_fs)
    set(_ARGS_TARGET_NAME "${PROJECT_NAME}_generate_fs_protos")
  endif()
  if (NOT DEFINED _ARGS_PROTOC)
-    set(_ARGS_PROTOC "${EXTERNALS_BIN}/protoc")
+    set(_ARGS_PROTOC "${SPARK_BIN}/protoc")
  endif()

  set(_proto_include_cli_args)
@@ -27,8 +27,8 @@ function(protobuf_generate_fs)
  endforeach()

  set(_venv_exec ${CMAKE_SOURCE_DIR}/tools/venv_exec.sh)
-  set(_plugin_py ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/plugin.py)
-  set(_template_fs ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/fs/fable.fs)
+  set(_plugin_py ${CMAKE_SOURCE_DIR}/src/lib/api/service_compiler/plugin.py)
+  set(_template_fs ${CMAKE_SOURCE_DIR}/src/lib/api/service_compiler/templates/fs/fable.fs)
  add_custom_command(
    OUTPUT ${_outputs}
    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
@@ -1,10 +0,0 @@
-include_guard(GLOBAL)
-
-macro(set_first_defined RESULT)
-  foreach(_arg ${ARGN})
-    if (DEFINED ${_arg})
-      set(${RESULT} ${${_arg}})
-      break()
-    endif()
-  endforeach()
-endmacro()
@@ -1,39 +0,0 @@
-include_guard(DIRECTORY)
-include(${CMAKE_SOURCE_DIR}/cmake/SymLink.cmake)
-
-function(sym_link_python_module)
-  set(_options)
-  set(_one_value_args LIB_NAME MODULE_NAME PLUGIN_NAME)
-  set(_multi_value_args)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}"
-                        "${_multi_value_args}" ${ARGN})
-  if(NOT DEFINED _ARGS_LIB_NAME)
-    message(
-      FATAL_ERROR "LIB_NAME must be specified for sym_link_python_module()")
-  endif()
-  set(_lib_name ${_ARGS_LIB_NAME})
-  set(_module_name ${_ARGS_MODULE_NAME})
-  if(NOT DEFINED _ARGS_MODULE_NAME)
-    set(_module_name ${_lib_name}_module)
-  endif()
-  set(_plugin_name ${_ARGS_PLUGIN_NAME})
-  set(_deps ${_lib_name} ${_module_name})
-  if(DEFINED _ARGS_PLUGIN_NAME)
-    list(APPEND _deps ${_plugin_name} copy_${_plugin_name})
-  endif()
-  add_dependencies(${PROJECT_NAME} ${_deps})
-  sym_link_local_lib(${_lib_name})
-  set(_module_suffix cpython-310-x86_64-linux-gnu.so)
-  sym_link_file(
-    SOURCE
-    ${CMAKE_BINARY_DIR}/src/lib_${_lib_name}/${_module_name}.${_module_suffix}
-    DEST
-    ${CMAKE_CURRENT_BINARY_DIR}/${_module_name}.${_module_suffix}
-    INSTALL_DEST
-    ${CMAKE_INSTALL_BINDIR}/)
-  if(DEFINED _ARGS_PLUGIN_NAME)
-    get_target_property(plugins ${_plugin_name} INTERFACE_PYTHON_SCRIPTS_DIR)
-    sym_link_dir(TARGET_NAME ${PROJECT_NAME}_link_${_plugin_name} SOURCE
-                 ${plugins} DEST ${CMAKE_CURRENT_BINARY_DIR}/${_plugin_name})
-  endif()
-endfunction()
@@ -1,11 +0,0 @@
-#pragma once
-
-#include <array>
-
-namespace bts::@PROJECT_NAME@::protobufs {
-
-inline std::array<const char *, @PROTO_SOURCES_LENGTH@> config_proto_files = {
-@PROTO_SOURCES_CXX_ARRAY_ITEMS@
-};
-
-} // namespace bts::@PROJECT_NAME@::protobufs
@@ -0,0 +1,24 @@
+cmake_minimum_required(VERSION 3.5)
+
+# Explicitly set policy to convert relative paths to absolute (although this is a default policy,
+# setting it explicitly prevents a compile-warning)
+cmake_policy(SET CMP0076 NEW)
+
+# add protos to the library to facilitate source_group
+target_sources(libspark PUBLIC ${Config_Protos} ${NonConfig_Protos} ${Service_Protos})
+
+function(assign_source_group)
+  foreach(_source IN ITEMS ${ARGN})
+    if(IS_ABSOLUTE "${_source}")
+      file(RELATIVE_PATH _source_rel "${CMAKE_CURRENT_SOURCE_DIR}" "${_source}")
+    else()
+      set(_source_rel "${_source}")
+    endif()
+    get_filename_component(_source_path "${_source_rel}" PATH)
+    string(REPLACE "/" "\\" _source_path_msvc "${_source_path}")
+    source_group("${_source_path_msvc}" FILES "${_source}")
+    set(SOURCE_FILES "${_source}")
+  endforeach()
+endfunction(assign_source_group)
+
+assign_source_group(${SparkLib_Sources} ${Config_Protos} ${NonConfig_Protos} ${Service_Protos})
@@ -3,12 +3,11 @@ services:
    volumes:
      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/sparkdata/data
    build:
-      dockerfile: src/app_sparkd/Dockerfile
+      dockerfile: src/server/sparkd/Dockerfile
      args:
        - IMAGE=${SPARKD__IMAGE}
        - SOURCE_DIR=${SPARKD__SOURCE_DIR}
        - CONFIG_DIR=${SPARKD__CONFIG_DIR}
-        - PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
    shm_size: "${SPARKD__SHM_SIZE:-512mb}"
    command: >
      sh -c "
@@ -40,12 +39,11 @@ services:
      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/sparkdata/data
      - ${SPARKD__RUNTIME_CONF_DIR:-sparkd_conf}:/home/bts/sparkdata/conf
    build:
-      dockerfile: src/app_sparkd/Dockerfile
+      dockerfile: src/server/sparkd/Dockerfile
      args:
        - IMAGE=${SPARKD__IMAGE}
        - SOURCE_DIR=${SPARKD__SOURCE_DIR}
        - CONFIG_DIR=${SPARKD__CONFIG_DIR}
-        - PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
    shm_size: "${SPARKD__SHM_SIZE:-512mb}"
    command: >
      sh -c "
@@ -124,8 +124,8 @@ service OrderService {
 ```
 ### C++ Example
 ```cpp
-#include "spark/api/public/ServiceContext.h"
-#include "spark/api/services/OrderService.client.pb.h"
+#include "lib/api/public/ServiceContext.h"
+#include "lib/api/services/OrderService.client.pb.h"
 using namespace bts::api;
 ServiceContext ctx;
 auto orders = ctx.locate<orderman::OrderService>();
@@ -29,9 +29,9 @@ namespace sample {
 ```

 ```cpp
-#include "spark/api/public/ServiceContext.h"
-#include "spark/domain_model/Security.h"
-#include "spark/api/services/MarketDataService.client.pb.h"
+#include "lib/api/public/ServiceContext.h"
+#include "spark/lib/domain_model/Security.h"
+#include "lib/api/services/MarketDataService.client.pb.h"
 #include <thread>

 using namespace bts::api;
@@ -16,16 +16,11 @@ jmespath==1.0.1
 lazy-object-proxy==1.6.0
 MarkupSafe==3.0.2
 mccabe==0.6.1
-numpy==2.2.6
 platformdirs==2.3.0
 protobuf==4.21.12
 pybars3==0.9.7
-pybind11==2.13.6
-pybind11-stubgen==2.5.3
-pybit==5.11.0
 pycodestyle==2.7.0
 pycparser==2.20
-pycryptodome==3.23.0
 pylint==2.10.2
 PyMeta3==0.5.1
 python-dateutil==2.9.0.post0
@@ -36,7 +31,6 @@ six==1.17.0
 termcolor==1.1.0
 toml==0.10.2
 urllib3==2.0.3
-websocket-client==1.8.0
 Werkzeug==3.1.3
 wrapt==1.12.1
 yapf==0.31.0
@@ -19,8 +19,8 @@
  </ItemGroup>

  <ItemGroup>
-    <Proto Include="../../lib_spark/spark/config/protos/data_stream_config.proto" CSName="DataStreamConfig">
-      <OutDir>$(ProtoOutputPath)/lib_spark/spark/config/protos</OutDir>
+    <Proto Include="../../lib/config/protos/data_stream_config.proto" CSName="DataStreamConfig">
+      <OutDir>$(ProtoOutputPath)/lib/config/protos</OutDir>
    </Proto>
  </ItemGroup>

@@ -43,7 +43,7 @@
      Condition="Exists($(ComputerSpecificConfig))" />

  <PropertyGroup Label="ProtoBuf">
-    <ProtocPluginBase>$(SparkRoot)\src\lib_spark\spark\api\service_compiler</ProtocPluginBase>
+    <ProtocPluginBase>$(SparkRoot)\src\lib\api\service_compiler</ProtocPluginBase>
  </PropertyGroup>
  <Choose>
    <When Condition="'$(OS)' == 'Windows_NT'">
@@ -24,7 +24,7 @@
    <PropertyGroup>
      <GenProtosCmd>$(protoc) %(Proto.FullPath)</GenProtosCmd>
      <GenProtosCmd>$(GenProtosCmd) --error_format=msvs</GenProtosCmd>
-      <GenProtosCmd>$(GenProtosCmd) --proto_path=$(SourceRoot)/lib_spark</GenProtosCmd>
+      <GenProtosCmd>$(GenProtosCmd) --proto_path=$(SourceRoot)</GenProtosCmd>
      <GenProtosCmd>$(GenProtosCmd) --plugin=&quot;protoc-gen-spark-csharp=$(ProtocPlugin)&quot;</GenProtosCmd>
      <GenProtosCmd>$(GenProtosCmd) --spark-csharp_out=lang=c#,templates=immutable.pb.cs,strip_out_path=1:%(Proto.OutDir)</GenProtosCmd>
      <GenProtosCmd>$(GenProtosCmd) --csharp_out=%(Proto.OutDir)</GenProtosCmd>
@@ -65,7 +65,7 @@
    <PropertyGroup>
      <GenProtosCmd>$(protoc) %(ProtoService.FullPath)</GenProtosCmd>
      <GenProtosCmd>$(GenProtosCmd) --error_format=msvs</GenProtosCmd>
-      <GenProtosCmd>$(GenProtosCmd) --proto_path=$(SourceRoot)/lib_spark</GenProtosCmd>
+      <GenProtosCmd>$(GenProtosCmd) --proto_path=$(SourceRoot)</GenProtosCmd>
      <GenProtosCmd>$(GenProtosCmd) --plugin=&quot;protoc-gen-service=$(ProtocPlugin)&quot;</GenProtosCmd>
      <GenProtosCmd>$(GenProtosCmd) --service_out=lang=c#,strip_out_path=1:%(ProtoService.OutDir)</GenProtosCmd>
      <GenProtosCmd>$(GenProtosCmd) --csharp_out=%(ProtoService.OutDir)</GenProtosCmd>
@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 3.5)
+cmake_minimum_required(VERSION 3.10)
 project(Spark.Admin.WebApp)

 include(GNUInstallDirs)
@@ -10,11 +10,11 @@ include(${CMAKE_SOURCE_DIR}/cmake/ProtobufGenerateFs.cmake)
 file(
  GLOB_RECURSE ${PROJECT_NAME}_PROTO_SOURCES
  RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
-  ../lib_spark/spark/api/*.proto
-  ../lib_spark/spark/api/services/*.proto
-  ../lib_spark/spark/domain_model/*.proto
-  ../lib_spark/spark/sparkproto/*.proto
-  ../lib_spark/spark/md/protos/*.proto
+  ../lib/api/*.proto
+  ../lib/api/services/*.proto
+  ../lib/domain_model/*.proto
+  ../lib/sparkproto/*.proto
+  ../lib/md/protos/*.proto
  ../../externals/linux/include/google/protobuf/descriptor.proto
  ../../externals/linux/include/google/protobuf/timestamp.proto
 )
@@ -44,7 +44,7 @@ set(_build_admin_web_app_deps
  ${_build_admin_web_app_py}
  ${CMAKE_CURRENT_SOURCE_DIR}/src/Spark.Admin.WebApp.fsproj
  ${CMAKE_CURRENT_SOURCE_DIR}/webpack.config.js
-  ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/fs/fable.fs
+  ${CMAKE_SOURCE_DIR}/src/lib/api/service_compiler/templates/fs/fable.fs
  ${CMAKE_CURRENT_SOURCE_DIR}/package-lock.json
  ${${PROJECT_NAME}_PROTO_SOURCES}
  ${_api_ts_SOURCES}
@@ -73,7 +73,7 @@ add_custom_command(
 protobuf_generate_js(
  TARGET_NAME generate_js_protos
  PROTO_INCLUDE_DIRS
-    ../lib_spark
+    ..
    ../../externals/linux/include
  PROTO_SOURCES ${${PROJECT_NAME}_PROTO_SOURCES}
  OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/proto.js
@@ -82,7 +82,7 @@ protobuf_generate_js(
 protobuf_generate_fs(
  TARGET_NAME generate_fs_protos
  PROTO_INCLUDE_DIRS
-    ../lib_spark
+    ..
    ../../externals/linux/include
  PROTO_SOURCES ${${PROJECT_NAME}_PROTO_SOURCES}
  OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/fable
@@ -184,3 +184,4 @@ install(
  DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/public"
  DESTINATION "${CMAKE_INSTALL_BINDIR}/${PROJECT_NAME}"
 )
+
@@ -17,7 +17,7 @@ type Tab =
  | PositionManagementTab
  | MarginTab

-let venues = [| Venue.BINANCE; Venue.BinanceUs; Venue.BinanceUsdsFutures; Venue.BYBIT; Venue.CBOE; Venue.COINBASE; Venue.DERIBIT; Venue.GATEIO; Venue.OKX; Venue.GLBX; Venue.ICE; Venue.TAIFEX; Venue.TWSE; Venue.SIM; |]
+let venues = [| Venue.BINANCE; Venue.BinanceUs; Venue.BinanceUsdsFutures; Venue.CBOE; Venue.COINBASE; Venue.DERIBIT; Venue.OKX; Venue.GLBX; Venue.ICE; Venue.TAIFEX; Venue.TWSE; Venue.SIM; |]

 let securityTypes = [| SecurityType.SPOT; SecurityType.FUTURE; SecurityType.OPTION; SecurityType.SPREAD; |]

@@ -50,11 +50,6 @@ type BinanceUsdsFUTURESExchangeField =
  | Origin of CustomerOrFirm
  | ClearingAccount of string

-type BybitExchangeField =
-  | CTI of CTICode
-  | Origin of CustomerOrFirm
-  | ClearingAccount of string
-
 type COINBASEExchangeField =
  | CTI of CTICode
  | Origin of CustomerOrFirm
@@ -65,11 +60,6 @@ type DERIBITExchangeField =
  | Origin of CustomerOrFirm
  | ClearingAccount of string

-type GateioExchangeField =
-  | CTI of CTICode
-  | Origin of CustomerOrFirm
-  | ClearingAccount of string
-
 type ICEExchangeField =
  | CTI of CTICode
  | Origin of CustomerOrFirm
@@ -797,8 +797,6 @@ let venueToString (venue:bts.api.Venue) =
    | bts.api.Venue.BINANCE -> "BINANCE"
    | bts.api.Venue.BinanceUs -> "BinanceUs"
    | bts.api.Venue.BinanceUsdsFutures -> "BinanceUsdsFutures"
-    | bts.api.Venue.BYBIT -> "BYBIT"
-    | bts.api.Venue.GATEIO -> "GATEIO"
    | bts.api.Venue.FTX -> "FTX"
    | bts.api.Venue.OKX -> "OKX"
    | bts.api.Venue.TAIFEX -> "TAIFEX"
@@ -1,172 +0,0 @@
-cmake_minimum_required(VERSION 3.14)
-project(sparkd)
-
-include(FetchContent)
-include(${CMAKE_SOURCE_DIR}/cmake/safe_dep_glob.cmake)
-include(${CMAKE_SOURCE_DIR}/cmake/ProtobufGenerateCpp.cmake)
-include(${CMAKE_SOURCE_DIR}/cmake/InstallBasic.cmake)
-include(GNUInstallDirs)
-
-if (NOT DEFINED PROJECT_VERSION)
-  set(PROJECT_VERSION 1.0.0)
-endif()
-
-# dependencies
-file(
-  GLOB_RECURSE ${PROJECT_NAME}_PROTO_SOURCES
-  RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
-  *.proto
-)
-file(
-  GLOB_RECURSE ${PROJECT_NAME}_SOURCES
-  RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
-  *.cpp
-)
-update_deps_file("${${PROJECT_NAME}_PROTO_SOURCES};${${PROJECT_NAME}_SOURCES}")
-
-protobuf_generate_cpp(
-  CXX_SOURCES ${PROJECT_NAME}_PROTO_CXX_SOURCES
-  CXX_HEADERS ${PROJECT_NAME}_PROTO_CXX_HEADERS
-  CXX_OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/include
-  PROTO_INCLUDE_DIRS
-    ${EXTERNALS_INCLUDE}
-    ${CMAKE_SOURCE_DIR}/src/lib_spark
-    ${CMAKE_CURRENT_SOURCE_DIR}
-  PROTO_SOURCES
-    ${${PROJECT_NAME}_PROTO_SOURCES}
-)
-add_executable(sparkd ${${PROJECT_NAME}_SOURCES} ${${PROJECT_NAME}_PROTO_CXX_SOURCES})
-
-# include paths
-target_include_directories(
-  sparkd
-  PUBLIC
-    $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
-    $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>/include
-    $<BUILD_INTERFACE:${emumba-deribit-connector-library_INCLUDE_DIRS}>
-)
-
-find_package(PkgConfig REQUIRED)
-pkg_check_modules(emumba-deribit-connector-library REQUIRED emumba-deribit-connector)
-
-# This dir contains rapidjsonConfig.cmake
-list(APPEND CMAKE_PREFIX_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake_modules)
-find_package(rapidjson REQUIRED)
-target_include_directories(
-  ${PROJECT_NAME} SYSTEM
-  PUBLIC
-    $<BUILD_INTERFACE:${rapidjson_INCLUDE_DIR}> 
-)
-add_dependencies(${PROJECT_NAME} rapidjson_project)
-
-# databento
-FetchContent_Declare(
-  databento
-  GIT_REPOSITORY https://github.com/databento/databento-cpp
-  GIT_TAG v0.31.0
-)
-FetchContent_MakeAvailable(databento)
-
-# deribit-demo log level
-if(NOT DEFINED DERIBIT_DEMO_LOG_LEVEL)
-  add_definitions(-DSPDLOG_ACTIVE_LEVEL=0)
-  message("-- Building of DERIBIT_DEMO_LOG_LEVEL=TRACE")
-elseif(DERIBIT_DEMO_LOG_LEVEL STREQUAL "TRACE")
-  add_definitions(-DSPDLOG_ACTIVE_LEVEL=0)
-  message("-- Building of DERIBIT_DEMO_LOG_LEVEL=TRACE")
-elseif(DERIBIT_DEMO_LOG_LEVEL STREQUAL "DEBUG")
-  add_definitions(-DSPDLOG_ACTIVE_LEVEL=1)
-  message("-- Building of DERIBIT_DEMO_LOG_LEVEL=DEBUG")
-elseif(DERIBIT_DEMO_LOG_LEVEL STREQUAL "INFO")
-  add_definitions(-DSPDLOG_ACTIVE_LEVEL=2)
-  message("-- Building of DERIBIT_DEMO_LOG_LEVEL=INFO")
-elseif(DERIBIT_DEMO_LOG_LEVEL STREQUAL "WARN")
-  add_definitions(-DSPDLOG_ACTIVE_LEVEL=3)
-  message("-- Building of DERIBIT_DEMO_LOG_LEVEL=WARN")
-elseif(DERIBIT_DEMO_LOG_LEVEL STREQUAL "ERROR")
-  add_definitions(-DSPDLOG_ACTIVE_LEVEL=4)
-  message("-- Building of DERIBIT_DEMO_LOG_LEVEL=ERROR")
-elseif(DERIBIT_DEMO_LOG_LEVEL STREQUAL "CRITICAL")
-  add_definitions(-DSPDLOG_ACTIVE_LEVEL=5)
-  message("-- Building of DERIBIT_DEMO_LOG_LEVEL=CRITICAL")
-elseif(DERIBIT_DEMO_LOG_LEVEL STREQUAL "OFF")
-  add_definitions(-DSPDLOG_ACTIVE_LEVEL=6)
-  message("-- Building of DERIBIT_DEMO_LOG_LEVEL=OFF")
-else()
-  add_definitions(-DSPDLOG_ACTIVE_LEVEL=0)
-  message("-- Building of DERIBIT_DEMO_LOG_LEVEL=TRACE")
-endif()
-
-target_link_directories(
-  ${PROJECT_NAME} PUBLIC
-  $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
-  $<INSTALL_INTERFACE:${CMAKE_INSTALL_LIBDIR}>
-)
-
-# Set RPATH so sparkd knows where to find our shared libs (e.g., libspark).
-# The dirs are relative to sparkd's path, represented by "$ORIGIN".
-set_target_properties(
-  ${PROJECT_NAME} PROPERTIES
-  BUILD_RPATH "$ORIGIN/../lib_spark;$ORIGIN/../lib_python_interop"
-  INSTALL_RPATH "$ORIGIN/../${CMAKE_INSTALL_LIBDIR}"
-)
-
-include(${CMAKE_SOURCE_DIR}/cmake/SymLink.cmake)
-include(${CMAKE_SOURCE_DIR}/cmake/SymLinkPythonModule.cmake)
-# Py interop
-sym_link_python_module(
-  LIB_NAME python_interop
-  MODULE_NAME python_interop_module
-  PLUGIN_NAME python_interop_plugins
-)
-target_link_libraries(
-  sparkd
-  spark
-  binance_execution
-  okx_execution
-  python_interop
-  python_interop_module
-  databento::databento
-  ${emumba-deribit-connector-library_LIBRARIES}
-  tbb
-  quickfix
-  benchmark
-  Common++
-  Packet++
-  Pcap++
-  lttng-ust
-  -lpcap
-  -static-libgcc
-  -static-libstdc++
-  -pthread)
-
-# Onixs
-target_link_libraries(
-  ${PROJECT_NAME}
-  onixs-cmemdh
-  OnixS.IceImpactMulticastPriceFeedHandlerCpp
-  # gcc doesn't resolve onixs-fix-engine and ACE in runtime path unless we specify
-  # --no-as-needed. These are onixs deps that aren't called directly by our code.
-  # Note: this isn't an issue for clang
-  -Wl,--no-as-needed
-  onixs-fix-engine
-  ACE
-)
-
-# Catch2
-find_package(Catch2 CONFIG REQUIRED)
-target_link_libraries(${PROJECT_NAME} Catch2::Catch2)
-
-install_basic()
-install(
-  FILES
-    "${EXTERNALS_LIB}/libonixs-cmemdh.so"
-    "${EXTERNALS_LIB}/libonixs-cmemdh.so.4"
-    "${EXTERNALS_LIB}/libonixs-cmemdh.so.4.8.0"
-    "${EXTERNALS_LIB}/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.16.0.0"
-    "${EXTERNALS_LIB}/libonixs-fix-engine.so"
-    "${EXTERNALS_LIB}/libonixs-fix-engine.so.4"
-    "${EXTERNALS_LIB}/libonixs-fix-engine.so.4.10.1"
-    "${EXTERNALS_LIB}/libACE-5.7.5.so"
-  DESTINATION ${CMAKE_INSTALL_LIBDIR}
-)
@@ -1,5 +0,0 @@
-@PACKAGE_INIT@
-
-include("${CMAKE_CURRENT_LIST_DIR}/@PROJECT_NAME@Targets.cmake")
-
-check_required_components(@PROJECT_NAME@)
@@ -1,4 +0,0 @@
-SPARKD__CONFIG_DIR=src/app_sparkd/configurations/alphaworx_prod
-SPARKD__SHM_SIZE=8gb
-SPARKD__MEMORY=8gb
-SPARKD__RUNTIME_CONF_DIR=/home/ubuntu/8000/sparkdata/conf/
@@ -1,6 +0,0 @@
-SPARKD__CONFIG_DIR=src/app_sparkd/configurations/alphaworx_zen_hk_live-preview
-SPARKD__PYTHON_REQUIREMENTS=src/app_sparkd/configurations/alphaworx_zen_hk_live-preview/requirements.txt
-SPARKD__SHM_SIZE=8gb
-SPARKD__MEMORY=8gb
-SPARKD__IMAGE=sparkd_symbols
-SPARKD__RUNTIME_CONF_DIR=/home/bts/8000/sparkdata/conf
@@ -1,6 +0,0 @@
-* Connects to Binance, Okx, and Bybit PRODUCTION sites with small accounts designated for test trades
-* Api keys NOT included
-* Configured for Alphaworx and Bts testing in a production environment
-
-To retrieve api keys, run "python init_env.py". It should dump them to a file
-called  ".env.local".
@@ -1,99 +0,0 @@
-import botocore
-import json
-import os
-
-from boto3.session import Session
-from botocore.exceptions import ClientError
-from fileinput import FileInput
-
-
-def update_env_file(path, env_vars: dict):
-    """Update or add vars in a .env file in place."""
-    if not os.path.exists(path):
-        with open(path, "w") as file:
-            for k, v in env_vars.items():
-                file.write(f"{k}={v}\n")
-        return
-
-    with FileInput(path, inplace=True) as file:
-        for line in file:
-            if not line.startswith("#"):
-                stripped: str = line.lstrip()
-                key_len = stripped.find("=")
-                if key_len >= 0:
-                    k = stripped[0:key_len]
-                    if k in env_vars:
-                        print(f"{k}={env_vars[k]}")
-                        del env_vars[k]
-                        continue
-            print(line, end="")
-
-    if len(env_vars) > 0:
-        with open(path, "a") as file:
-            for k, v in env_vars.items():
-                file.write(f"{k}={v}\n")
-
-
-def get_secret():
-    secret_name = "alphaworx_api_keys"
-    region_name = "us-east-1"
-    profile_name = "bts-ap-northeast-1"
-    session = Session(profile_name=profile_name)
-    client = session.client(
-        service_name="secretsmanager",
-        region_name=region_name,
-    )
-
-    def on_token_error(e: Exception):
-        print(f"""
-Failed to get secret value. {e}
-Verify you are logged in to aws, then try again.
-
-  aws sso login --profile {profile_name}
-
-If this error persists, verify you have access to these api keys.
-For more help or information, talk to John.
-""")
-
-    try:
-        get_secret_value_response = client.get_secret_value(
-            SecretId=secret_name)
-    except ClientError as e:
-        print(f"Failed to get secret value.\n{e}")
-        raise e
-    except botocore.exceptions.SSOTokenLoadError as tl_err:
-        on_token_error(tl_err)
-        return None
-    except botocore.exceptions.TokenRetrievalError as tr_err:
-        on_token_error(tr_err)
-        return None
-    return get_secret_value_response["SecretString"]
-
-
-def _run():
-    this_dir = os.path.dirname(__file__)
-    secret_str = get_secret()
-    if secret_str is None:
-        exit(1)
-    secret = json.loads(secret_str)
-    env_vars = {}
-
-    def set_env_var(key_name: str, *json_path):
-        try:
-            val = secret[json_path[0]]
-            for p in json_path[1:]:
-                val = val[p]
-            env_vars[key_name] = val
-            print(f"Set {key_name}={val} ({'.'.join(json_path)})")
-        except KeyError:
-            print(f"Failed to set {key_name}= ({'.'.join(json_path)})")
-
-    set_env_var("BINANCE_API_KEY", "binance", "prod", "api_key")
-    set_env_var("BINANCE_SECRET_KEY", "binance", "prod", "secret_key")
-    set_env_var("BYBIT_API_KEY", "bybit", "prod_test", "api_key")
-    set_env_var("BYBIT_SECRET_KEY", "bybit", "prod_test", "secret_key")
-    update_env_file(f"{this_dir}/.env.local", env_vars)
-
-
-if __name__ == "__main__":
-    _run()
@@ -1,15 +0,0 @@
-{
-  "gateway_type": "bybit_execution_gateway.BybitExecutionGateway",
-  "account": "BYBIT_DEV",
-  "module_path": [
-    "python_interop_plugins"
-  ],
-  "venue": "VENUE_BYBIT",
-  "api_key": "${BYBIT_API_KEY}",
-  "secret_key": "${BYBIT_SECRET_KEY}",
-  "use_testnet": true,
-  "logging": {
-    "console_level": "DEBUG",
-    "file_level": "DEBUG"
-  }
-}
@@ -1,17 +0,0 @@
-{
-  "currencies": [
-    "USDT"
-  ],
-  "futures": [
-    "ANIME",
-    "BTC",
-    "ETH",
-    "MASK",
-    "RESOLV"
-  ],
-  "use_testnet": true,
-  "logging": {
-    "console_level": "WARNING",
-    "file_level": "INFO"
-  }
-}
@@ -1,11 +0,0 @@
-certifi==2025.4.26
-charset-normalizer==3.4.2
-dotenv==0.9.9
-idna==3.10
-numpy==2.2.6
-pybit==5.11.0
-pycryptodome==3.23.0
-python-dotenv==1.1.0
-requests==2.32.4
-urllib3==2.4.0
-websocket-client==1.8.0
@@ -1,138 +0,0 @@
-QuickbusCfg {
-  value {
-    node {
-      reconnect_period_secs: 1
-      hb_period_msecs: 15000
-      max_missed_hbs: 5
-      parent_host: ""
-      parent_port: 0
-      my_port: 0
-    }
-    client {
-      reconnect_period_secs: 1
-      hb_period_msecs: 15000
-      max_missed_hbs: 5
-      parent_host: @SystemConfig.config_ip
-      parent_port: @Ports.qbus_port
-      my_port: 0
-    }
-  }
-}
-
-JetFIX {
-  value {
-    # connection {
-    #   name: "binance_1"
-    #   venue: VENUE_BINANCE
-    #   max_order_msgs_per_window: 300
-    #   window_duration_msecs: 1000
-    # }
-    # connection {
-    #   name: "binance_usds_futures_1"
-    #   venue: VENUE_BINANCE_USDS_FUTURES
-    #   max_order_msgs_per_window: 300
-    #   window_duration_msecs: 1000
-    # }
-    connection {
-      name: "bybit_1"
-      venue: VENUE_BYBIT
-      max_order_msgs_per_window: 300
-      window_duration_msecs: 1000
-    }
-    # connection {
-    #   name: "okx_1"
-    #   venue: VENUE_OKX
-    #   max_order_msgs_per_window: 300
-    #   window_duration_msecs: 1000
-    # }
-  }
-}
-
-MarketDataConfig {
-  value {
-    # binance {
-    #   num_depth_levels: 5
-    #   url_rest_base: "https://api.binance.com"
-    #   url_ws_base: "wss://stream.binance.com:9443"
-    #   security_criteria {
-    #     clauses {
-    #       security_types: SECURITY_TYPE_SPOT
-    #       symbols: @SymbolLists.list[Binance].symbol
-    #       name_regexps: "USDC|USDT$"
-    #     }
-    #   }
-    # }
-    # binance_usds_futures {
-    #   num_depth_levels: 5
-    #   url_rest_base: "https://fapi.binance.com"
-    #   url_ws_base: "wss://fstream.binance.com"
-    #   security_criteria {
-    #     clauses {
-    #       security_types: SECURITY_TYPE_FUTURE
-    #       symbols: @SymbolLists.list[BinanceUsdsFutures].symbol
-    #     }
-    #   }
-    # }
-    # okx {
-    #   num_depth_levels: 5
-    #   url_rest_base: "https://www.okx.com"
-    #   url_ws_base: "wss://ws.okx.com:8443"
-    #   security_criteria {
-    #     clauses {
-    #       security_types: SECURITY_TYPE_SPOT
-    #       symbols: @SymbolLists.list[OkxSpots].symbol
-    #     }
-    #     clauses {
-    #       security_types: SECURITY_TYPE_FUTURE
-    #       symbols: @SymbolLists.list[OkxFutures].symbol
-    #     }
-    #   }
-    #   is_application_ping_pong_enabled: true
-    #   application_ping_interval_ms: 20000
-    #   application_pong_timeout_ms: 10000
-    #   is_protocol_ping_pong_enabled: true
-    #   protocol_ping_interval_ms: 60000
-    #   protocol_pong_timeout_ms: 30000
-    # }
-    python_md_providers {
-      venue: VENUE_BYBIT
-      module_path: "python_interop_plugins"
-      provider_type: "bybit_md_provider.BybitMdProvider"
-    }
-  }
-}
-
-NotSimExecution {
-  value {
-    # entries {
-    #   venue: VENUE_BINANCE
-    # }
-    # entries {
-    #   venue: VENUE_BINANCE_USDS_FUTURES
-    # }
-    entries {
-      venue: VENUE_BYBIT
-    }
-    # entries {
-    #   venue: VENUE_OKX
-    # }
-  }
-}
-
-# OkxExeCfg {
-#   value {
-#     account: "PROD"
-#     ops_id: "op_id_bts"
-#     secret_key: "240038D7A28D7F9461247CFCB8E43710"
-#     api_key: "dc188532-c74c-4d40-affe-3c6bf4d0ca9a"
-#     passphrase: ":,7i%tA%ivfj%SCbX?Ut"
-#     ws_url: "wss://ws.okx.com:8443"
-#     rest_url: "https://www.okx.com"
-#     ws_ping_timeout_seconds: 10 
-#     ws_ping_interval_seconds: 10
-#     ws_reconnect_interval_seconds: 5
-#     ws_reconnect_max_count: 5
-# 	  simulated_trading: false
-#     tdmode: "cross"
-#   }
-# }
@@ -1,66 +0,0 @@
-SymbolLists {
-  value {
-    list {
-      key: "Binance"
-      symbol: "ANIME"
-      symbol: "BTC"
-      symbol: "ETH"
-      symbol: "MASK"
-      symbol: "RESOLV"
-    }
-    list {
-      key: "BinanceUsdsFutures"
-      symbol: "ANIME"
-      symbol: "BTC"
-      symbol: "ETH"
-      symbol: "MASK"
-      symbol: "RESOLV"
-    }
-    list {
-      key: "OkxSpots"
-      symbol: "ANIME"
-      symbol: "BTC"
-      symbol: "ETH"
-      symbol: "MASK"
-      symbol: "RESOLV"
-    }
-    list {
-      key: "Bybit"
-      symbol: "ANIME"
-      symbol: "BTC"
-      symbol: "ETH"
-      symbol: "MASK"
-      symbol: "RESOLV"
-    }
-    list {
-      key: "OkxFutures"
-      symbol: "ANIME"
-      symbol: "BTC"
-      symbol: "ETH"
-      symbol: "MASK"
-      symbol: "RESOLV"
-    }
-  }
-}
-
-Symbols {
-  value {
-    # venues {
-    #   name: "VENUE_BINANCE"
-    #   spot: @SymbolLists.list[Binance].symbol
-    # }
-    # venues {
-    #   name: "VENUE_BINANCE_USDS_FUTURES"
-    #   future: @SymbolLists.list[BinanceUsdsFutures].symbol
-    # }
-    venues {
-      name: "VENUE_BYBIT"
-      future: @SymbolLists.list[Bybit].symbol
-    }
-    # venues {
-    #   name: "VENUE_OKX"
-    #   spot: @SymbolLists.list[OkxSpots].symbol
-    #   future: @SymbolLists.list[OkxFutures].symbol
-    # }
-  }
-}
@@ -1,9 +0,0 @@
-{
-  "api_key": "${BINANCE_API_KEY}",
-  "secret_key": "${BINANCE_SECRET_KEY}",
-  "account": "PROD",
-  "rest_endpoint": "https://api.binance.com",
-  "ws_endpoint": "wss://ws-api.binance.com:443/ws-api/v3",
-  "ws_stream_endpoint": "wss://stream.binance.com:9443",
-  "use_cross_margin_api": true
-}
@@ -1,9 +0,0 @@
-{
-  "api_key": "${BINANCE_API_KEY}",
-  "secret_key": "${BINANCE_SECRET_KEY}",
-  "account": "PROD",
-  "rest_endpoint": "https://fapi.binance.com",
-  "ws_endpoint": "wss://ws-fapi.binance.com/ws-fapi/v1",
-  "ws_stream_endpoint": "wss://fstream.binance.com",
-  "use_cross_margin_api": false
-}
@@ -1,2 +0,0 @@
-SPARKD__CONFIG_DIR=src/app_sparkd/configurations/binance_dev
-SPARKD__SHM_SIZE=8gb
@@ -1,2 +0,0 @@
-SPARKD__CONFIG_DIR=src/app_sparkd/configurations/binance_py_dev
-SPARKD__SHM_SIZE=8gb
@@ -1 +0,0 @@
-* Connects to Binance test nets for market data and execution
@@ -1,8 +0,0 @@
-{
-  "api_key": "${BINANCE_API_KEY}",
-  "secret_key": "${HOME}/sparkdata/conf/.secrets/binance_1.pem",
-  "account": "BINANCE_DEV",
-  "rest_endpoint": "https://testnet.binance.vision",
-  "ws_endpoint": "wss://ws-api.testnet.binance.vision/ws-api/v3",
-  "ws_stream_endpoint": "wss://stream.testnet.binance.vision:9443"
-}
@@ -1,8 +0,0 @@
-{
-  "api_key": "${BINANCE_USDS_FUTURES_API_KEY}",
-  "secret_key": "${BINANCE_USDS_FUTURES_SECRET_KEY}",
-  "account": "BINANCE_DEV",
-  "rest_endpoint": "https://testnet.binancefuture.com",
-  "ws_endpoint": "wss://testnet.binancefuture.com/ws-fapi/v1",
-  "ws_stream_endpoint": "wss://stream.binancefuture.com"
-}
@@ -1,124 +0,0 @@
-import traceback
-import botocore
-import json
-import os
-
-from boto3.session import Session
-from botocore.exceptions import ClientError
-from fileinput import FileInput
-
-
-def update_env_file(path, env_vars: dict):
-    """Update or add vars in a .env file in place."""
-    if not env_vars:
-        # Nothing to update
-        return
-    if not os.path.exists(path):
-        with open(path, "w") as file:
-            for k, v in env_vars.items():
-                file.write(f"{k}={v}\n")
-        return
-
-    with FileInput(path, inplace=True) as file:
-        for line in file:
-            if not line.startswith("#"):
-                stripped: str = line.lstrip()
-                key_len = stripped.find("=")
-                if key_len >= 0:
-                    k = stripped[0:key_len]
-                    if k in env_vars:
-                        print(f"{k}={env_vars[k]}")
-                        del env_vars[k]
-                        continue
-            print(line, end="")
-
-    if len(env_vars) > 0:
-        with open(path, "a") as file:
-            for k, v in env_vars.items():
-                file.write(f"{k}={v}\n")
-
-
-def write_pem_file(file_path: str, text: str):
-    try:
-        dir_path = os.path.dirname(file_path)
-        os.makedirs(dir_path, exist_ok=True)
-        # rw-------
-        mode = 0o600
-        fd = os.open(file_path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, mode)
-        with os.fdopen(fd, "x") as f:
-            f.write(text)
-        return True
-    except Exception:
-        print(f"Error writing pem file '{file_path}'.")
-        traceback.print_exc()
-        return False
-
-
-def get_secret(secret_name: str, profile_name: str):
-    session = Session(profile_name=profile_name)
-    client = session.client(service_name="secretsmanager")
-
-    def on_token_error(e: Exception):
-        print(f"""
-Failed to get secret value '{secret_name}'. {e}
-Verify you are logged in to aws, then try again.
-
-  aws sso login --profile {profile_name}
-
-If this error persists, verify you have access to these api keys.
-For more help or information, talk to John.
-""")
-
-    try:
-        get_secret_value_response = client.get_secret_value(
-            SecretId=secret_name)
-    except ClientError as e:
-        print(f"Failed to get secret value.\n{e}")
-        raise e
-    except botocore.exceptions.SSOTokenLoadError as tl_err:
-        on_token_error(tl_err)
-        return None
-    except botocore.exceptions.TokenRetrievalError as tr_err:
-        on_token_error(tr_err)
-        return None
-    return get_secret_value_response["SecretString"]
-
-
-def _init_binance_1():
-    this_dir = os.path.dirname(__file__)
-    secret_str = get_secret("dev/binance/john", "bts-us-east-1")
-    if secret_str is None:
-        exit(1)
-    secret = json.loads(secret_str)
-    env_vars = {}
-    if "api-key" in secret:
-        env_vars["BINANCE_API_KEY"] = secret["api-key"]
-    if "secret-key-pem" in secret:
-        sk_pem = secret["secret-key-pem"]
-        sk_pem = "\n".join(sk_pem)
-        write_pem_file(f"{this_dir}/.secrets/binance_1.pem", sk_pem)
-    update_env_file(f"{this_dir}/.env.local", env_vars)
-
-
-def _init_binance_usds_futures_1():
-    this_dir = os.path.dirname(__file__)
-    secret_str = get_secret("dev/binance_usds_futures/john.hmac",
-                            "bts-us-east-1")
-    if secret_str is None:
-        exit(1)
-    secret = json.loads(secret_str)
-    env_vars = {}
-    if "api-key" in secret:
-        env_vars["BINANCE_USDS_FUTURES_API_KEY"] = secret["api-key"]
-    if "secret-key" in secret:
-        env_vars["BINANCE_USDS_FUTURES_SECRET_KEY"] = secret["secret-key"]
-    update_env_file(f"{this_dir}/.env.local", env_vars)
-
-
-def _run():
-    _init_binance_1()
-    _init_binance_usds_futures_1()
-
-
-if __name__ == "__main__":
-    _run()
@@ -1,8 +0,0 @@
-{
-  "gateway_type": "dummy_execution_gateway.DummyExecutionGateway",
-  "account": "BINANCE_DEV",
-  "module_path": [
-    "python_execution_gateways"
-  ],
-  "venue": "VENUE_BINANCE_USDS_FUTURES"
-}
@@ -1,119 +0,0 @@
-JetFIX {
-  value {
-    # connection {
-    #   name: "binance_1"
-    #   venue: VENUE_BINANCE
-    #   max_order_msgs_per_window: 300
-    #   window_duration_msecs: 1000
-    # }
-    connection {
-      name: "binance_usds_futures_1"
-      venue: VENUE_BINANCE_USDS_FUTURES
-      max_order_msgs_per_window: 300
-      window_duration_msecs: 1000
-    }
-  }
-}
-
-PublisherOptions {
-  value {
-    network_interface: "0.0.0.0"
-    heartbeat_ms: 5000
-    heartbeat_boot_seconds: 60
-    skip_options: true
-    hot_initial: 1024
-    hot_growth: 1024
-    cold_initial: 8192
-    cold_growth: 2048
-    channel_prefix: ""
-    disable_heartbeat: true
-  }
-}
-
-Logging {
-  value {
-    initial_queue_size: 120001
-    max_queue_growth: 4
-    console {
-      default_level: "Info"
-      subsystem {
-        name: "OrderMan"
-        level: "Diag"
-      }
-    }
-    file {
-      default_level: "Info"
-      # subsystem {
-      #   name: "md"
-      #   level: "Diag"
-      # }
-      subsystem {
-        name: "OrderMan"
-        level: "Diag"
-      }
-      subsystem {
-        name: "Http"
-        level: "Diag"
-      }
-    }
-  }
-}
-
-MarketDataConfig {
-  value {
-    # binance {
-    #   num_depth_levels: 5
-    #   # url_rest_base: "wss://ws-api.binance.com:9443"
-    #   # According to Binance, the "apiN" urls offer performance at the cost of stability.
-    #   # https://developers.binance.com/docs/binance-spot-api-docs/rest-api/general-api-information
-    #   # url_rest_base: "https://api1.binance.com:9443"
-    #   # url_rest_base: "https://api.binance.com"
-    #   url_rest_base: "https://testnet.binance.vision"
-    #   # url_ws_base: "wss://ws-api.binance.com"
-    #   url_ws_base: "wss://testnet.binance.vision"
-
-    #   security_criteria {
-    #     clauses {
-    #       security_types: SECURITY_TYPE_SPOT
-    #       symbols: @SymbolLists.list[Binance].symbol
-    #       name_regexps: "USDC|USDT$"
-    #     }
-    #   }
-    # }
-
-    binance_usds_futures {
-      num_depth_levels: 5
-      # url_rest_base: "https://fapi.binance.com"
-      # url_ws_base: "wss://fstream.binance.com"
-      url_rest_base: "https://testnet.binancefuture.com"
-      url_ws_base: "wss://stream.binancefuture.com"
-
-      security_criteria {
-        clauses {
-          security_types: SECURITY_TYPE_FUTURE
-          symbols: @SymbolLists.list[BinanceUsdsFutures].symbol
-        }
-      }
-    }
-  }
-}
-
-NotSimExecution {
-  value {
-    # entries {
-    #   venue: VENUE_BINANCE
-    # }
-    entries {
-      venue: VENUE_BINANCE_USDS_FUTURES
-    }
-    # entries {
-    #   venue: VENUE_GLBX
-    # }
-    # entries {
-    #   venue: VENUE_OKX
-    # }
-    # entries {
-    #   venue: VENUE_TAIFEX
-    # }
-  }
-}
@@ -1,37 +0,0 @@
-SymbolLists {
-  value {
-    list {
-      key: "Binance"
-      symbol: "AAVE"
-      symbol: "AVAX"
-      symbol: "BNB"
-      symbol: "BTC"
-      symbol: "DOGE"
-      symbol: "EIGEN"
-      symbol: "ETH"
-    }
-    list {
-      key: "BinanceUsdsFutures"
-      symbol: "AAVE"
-      symbol: "AVAX"
-      symbol: "BNB"
-      symbol: "BTC"
-      symbol: "DOGE"
-      symbol: "EIGEN"
-      symbol: "ETH"
-    }
-  }
-}
-
-Symbols {
-  value {
-    venues {
-      name: "VENUE_BINANCE"
-      spot: @SymbolLists.list[Binance].symbol
-    }
-    venues {
-      name: "VENUE_BINANCE_USDS_FUTURES"
-      future: @SymbolLists.list[BinanceUsdsFutures].symbol
-    }
-  }
-}
@@ -1 +0,0 @@
-SPARKD__CONFIG_DIR=src/app_sparkd/configurations/bybit_dev
@@ -1,15 +0,0 @@
-{
-  "gateway_type": "bybit_execution_gateway.BybitExecutionGateway",
-  "account": "BYBIT_DEV",
-  "module_path": [
-    "python_interop_plugins"
-  ],
-  "venue": "VENUE_BYBIT",
-  "api_key": "${BYBIT_API_KEY}",
-  "secret_key": "${BYBIT_SECRET_KEY}",
-  "use_testnet": true,
-  "logging": {
-    "console_level": "WARNING",
-    "file_level": "INFO"
-  }
-}
@@ -1,18 +0,0 @@
-{
-  "currencies": [
-    "USDT"
-  ],
-  "futures": [
-    "ADA",
-    "BNB",
-    "BTC",
-    "ETH",
-    "PRIME",
-    "XRP"
-  ],
-  "use_testnet": true,
-  "logging": {
-    "console_level": "WARNING",
-    "file_level": "INFO"
-  }
-}
@@ -1,61 +0,0 @@
-JetFIX {
-  value {
-    connection {
-      name: "bybit_1"
-      venue: VENUE_BYBIT
-      max_order_msgs_per_window: 300
-      window_duration_msecs: 1000
-    }
-  }
-}
-
-PublisherOptions {
-  value {
-    network_interface: "0.0.0.0"
-    heartbeat_ms: 5000
-    heartbeat_boot_seconds: 60
-    skip_options: true
-    hot_initial: 1024
-    hot_growth: 1024
-    cold_initial: 8192
-    cold_growth: 2048
-    channel_prefix: ""
-    disable_heartbeat: true
-  }
-}
-
-Logging {
-  value {
-    initial_queue_size: 120001
-    max_queue_growth: 4
-    file {
-      default_level: "Info"
-      subsystem {
-        name: "QBus"
-        level: "Warn"
-      }
-      subsystem {
-        name: "md"
-        level: "Diag"
-      }
-    }
-  }
-}
-
-MarketDataConfig {
-  value {
-    python_md_providers {
-      venue: VENUE_BYBIT
-      module_path: "python_interop_plugins"
-      provider_type: "bybit_md_provider.BybitMdProvider"
-    }
-  }
-}
-
-NotSimExecution {
-  value {
-    entries {
-      venue: VENUE_BYBIT
-    }
-  }
-}
@@ -1,22 +0,0 @@
-SymbolLists {
-  value {
-    list {
-      key: "Bybit"
-      symbol: "ADA"
-      symbol: "BNB"
-      symbol: "BTC"
-      symbol: "ETH"    
-      symbol: "PRIME"
-      symbol: "XRP"
-    }
-  }
-}
-
-Symbols {
-  value {
-    venues {
-      name: "VENUE_BYBIT"
-      future: @SymbolLists.list[Bybit].symbol
-    }
-  }
-}
@@ -1,2 +0,0 @@
-SPARKD__CONFIG_DIR=src/app_sparkd/configurations/dev
-SPARKD__SHM_SIZE=8gb
@@ -1 +0,0 @@
-SPARKD__CONFIG_DIR=src/app_sparkd/configurations/py_dummy_md
@@ -1,45 +0,0 @@
-{
-  "api_key": "e4a0c6b198549042b13faa7645640d60",
-  "secret_key": "527c37877efc247bb0b6158cf3d5d18b84db720da894307f1e2ab88df3aaf0ee",
-  "ws_url": "wss://fx-ws-testnet.gateio.ws/v4/ws/usdt",
-  "currencies": [
-    "USDT"
-  ],
-  "spots": [
-    "AMP",
-    "AR",
-    "ARB",
-    "ATOM",
-    "BTC",
-    "CAKE",
-    "CELO",
-    "CRV",
-    "ETH",
-    "FIL",
-    "GMT",
-    "GMX",
-    "HIGH",
-    "ICP",
-    "IMX",
-    "IOTX",
-    "LRC",
-    "MANA",
-    "MINA",
-    "NEAR",
-    "NOT",
-    "OMG",
-    "OP",
-    "OXY",
-    "PEOPLE",
-    "TURBO",
-    "VET",
-    "WAVES",
-    "YFI",
-    "ZIL",
-    "ZRX"
-  ],
-  "logging": {
-    "console_level": "WARNING",
-    "file_level": "INFO"
-  }
-}
@@ -1,38 +0,0 @@
-PublisherOptions {
-  value {
-    network_interface: "0.0.0.0"
-    heartbeat_ms: 5000
-    heartbeat_boot_seconds: 60
-    skip_options: true
-    hot_initial: 1024
-    hot_growth: 1024
-    cold_initial: 8192
-    cold_growth: 2048
-    channel_prefix: ""
-    disable_heartbeat: true
-  }
-}
-
-Logging {
-  value {
-    initial_queue_size: 120001
-    max_queue_growth: 4
-    file {
-      default_level: "Info"
-      subsystem {
-        name: "md"
-        level: "Diag"
-      }
-    }
-  }
-}
-
-MarketDataConfig {
-  value {
-    python_md_providers {
-      venue: VENUE_GATEIO
-      module_path: "python_md_providers"
-      provider_type: "gateio_md_provider.GateIoMdProvider"
-    }
-  }
-}
@@ -1,47 +0,0 @@
-  SymbolLists {
-  value {
-    list {
-      key: "Gateio"
-      symbol: "AMP"
-      symbol: "AR"
-      symbol: "ARB"
-      symbol: "ATOM"
-      symbol: "BTC"
-      symbol: "CAKE"
-      symbol: "CELO"
-      symbol: "CRV"
-      symbol: "ETH"
-      symbol: "FIL"
-      symbol: "GMT"
-      symbol: "GMX"
-      symbol: "HIGH"
-      symbol: "ICP"
-      symbol: "IMX"
-      symbol: "IOTX"
-      symbol: "LRC"
-      symbol: "MANA"
-      symbol: "MINA"
-      symbol: "NEAR"
-      symbol: "NOT"
-      symbol: "OMG"
-      symbol: "OP"
-      symbol: "OXY"
-      symbol: "PEOPLE"
-      symbol: "TURBO"
-      symbol: "VET"
-      symbol: "WAVES"
-      symbol: "YFI"
-      symbol: "ZIL"
-      symbol: "ZRX"
-    }
-  }
-}
-
-Symbols {
-  value {
-    venues {
-      name: "VENUE_GATEIO"
-      spot: @SymbolLists.list[Gateio].symbol
-    }
-  }
-}
@@ -1 +0,0 @@
-SPARKD__CONFIG_DIR=src/app_sparkd/configurations/py_dummy_md
@@ -1,38 +0,0 @@
-PublisherOptions {
-  value {
-    network_interface: "0.0.0.0"
-    heartbeat_ms: 5000
-    heartbeat_boot_seconds: 60
-    skip_options: true
-    hot_initial: 1024
-    hot_growth: 1024
-    cold_initial: 8192
-    cold_growth: 2048
-    channel_prefix: ""
-    disable_heartbeat: true
-  }
-}
-
-Logging {
-  value {
-    initial_queue_size: 120001
-    max_queue_growth: 4
-    file {
-      default_level: "Info"
-      subsystem {
-        name: "md"
-        level: "Diag"
-      }
-    }
-  }
-}
-
-MarketDataConfig {
-  value {
-    python_md_providers {
-      venue: VENUE_GLBX
-      module_path: "python_md_providers"
-      provider_type: "dummy_md_provider.DummyMdProvider"
-    }
-  }
-}
@@ -1,17 +0,0 @@
-SymbolLists {
-  value {
-    list {
-      key: "Glbx"
-      symbol: "ES"
-    }
-  }
-}
-
-Symbols {
-  value {
-    venues {
-      name: "VENUE_GLBX"
-      future: @SymbolLists.list[Glbx].symbol
-    }
-  }
-}
@@ -1,2 +0,0 @@
-SPARKD__CONFIG_DIR=src/app_sparkd/configurations/sim_dev
-SPARKD__SHM_SIZE=8gb
@@ -1,168 +0,0 @@
-#include "Run.h"
-#include "python_interop/plugin_manager.h"
-#include "spark/api/public/ServiceContext.h"
-#include "spark/config/config.h"
-#include "spark/config/protos/ports.pb.h"
-#include "spark/config/protos/quickbus_cfg.pb.h"
-#include "spark/io/http/curl_session.h"
-#include "spark/logging/logging.h"
-#include "spark/md/MBOProcessor.h"
-#include "spark/md/MBOServiceServer.h"
-#include "spark/md/MDClient.h"
-#include "spark/md/SyntheticSecurityService.h"
-#include "spark/md/relay/QueryServer.h"
-#include "spark/md/service/MarketDataServiceServer.h"
-#include "spark/metrics/metrics.h"
-#include "spark/metrics/system_metrics.h"
-#include "spark/orderman/OrderServiceServer.h"
-#include "spark/orderman/events/TradeSummarySignalDispatcher.h"
-#include "spark/orderman/order_factory/DynamicOrderRegistry.h"
-#include "spark/os/os.h"
-#include "spark/os/spark_data_directory.h"
-#include "spark/persistence/PersistenceServiceServer.h"
-#include "spark/plugin/SparkServices.h"
-#include "spark/plugin/plugin.h"
-#include "spark/quickbus/core/Quickbus.h"
-#include "spark/tracing/PerfLog.h"
-#include "spark/utils/Dispatcher.h"
-#include "spark/version.h"
-#include "sparkd/algo/AlgoDefinitionServiceServer.h"
-#include "sparkd/analytics/AnalyticsServiceServer.h"
-#include "sparkd/cmds/md/cmds/Run.h"
-#include "sparkd/execution/execution_gateway_provider.h"
-#include "sparkd/http/WebServer.h"
-#include "sparkd/md/MarketDataAuxiliaryServiceServer.h"
-#include "sparkd/md/recording/MDRecorder.h"
-#include "sparkd/position/MarginServiceServer.h"
-#include "sparkd/position/PositionServiceServer.h"
-#include "sparkd/position/import/BasePositionServiceServer.h"
-#include "sparkd/position/import/MoneyFileServiceServer.h"
-#include "sparkd/subprocess/SubProcessRunner.h"
-#include "sparkd/support/ErrorReportService.h"
-#include "sparkd/support/ShellLatencyServer.h"
-#include "sparkd/user/AdminServiceServer.h"
-#include "sparkd/user/AuthServiceServer.h"
-
-namespace bts::cmds {
-
-using bts::api::ServiceContext;
-using bts::md::MDClient;
-using bts::md::TradeSummarySignalDispatcher;
-using bts::metrics::MetricsContainer;
-using bts::orderman::DynamicOrderRegistry;
-using bts::orderman::ExecutionGatewayProvider;
-using bts::orderman::IExecutionGatewayProvider;
-using bts::orderman::OrderServiceServer;
-using bts::trace::PerfLog;
-using std::shared_ptr;
-using std::unique_ptr;
-using std::vector;
-
-static OrderServiceServer create_order_service_server(
-    ServiceContext &sc,
-    MDClient &mdclient,
-    DynamicOrderRegistry &dyn_order_registry,
-    MetricsContainer &stats,
-    PerfLog &perf_log,
-    TradeSummarySignalDispatcher &tss_dispatcher,
-    shared_ptr<python_interop::PluginManager> py_plugin_man) {
-   vector<unique_ptr<IExecutionGatewayProvider>> gw_pros;
-   gw_pros.push_back(std::make_unique<ExecutionGatewayProvider>(
-       &perf_log, mdclient.oracle(), config::get<config::JetFIX>(),
-       [](const std::string &name) {
-          const spark::SparkDataDirectory data_dir;
-          return data_dir.load_config_json(name);
-       },
-       std::move(py_plugin_man)));
-   return OrderServiceServer(
-       sc, mdclient, dyn_order_registry, std::move(gw_pros), stats, perf_log,
-       tss_dispatcher);
-}
-
-static void execute() {
-   using namespace bts;
-   LOG(Gen, Warn,
-       "------------------------------------ Sparkd " BTS_SPARK_VERSION_FULL
-       " launched "
-       "--------------------------------");
-   // Initializes curl for the application, and eventual cleanup in destructor
-   http::CurlSession global_curl_session;
-   auto qb_cfg = config::get<config::QuickbusCfg>().node();
-   auto &ports = config::get<config::Ports>();
-   qb_cfg.set_my_port(ports.qbus_port());
-   qbus::Quickbus quickbus(qb_cfg, qbus::QuickbusThreadOptions::DEFERRED_START);
-   api::ServiceContext sc(&quickbus);
-
-   trace::PerfLog perf_log;
-   metrics::MetricsContainer stats;
-   metrics::publish_system_metrics(stats);
-   http::SparkWebServer webserver(sc, stats, perf_log, ports.config_port());
-
-   persistence::PersistenceServiceServer settings_saver(sc, "settings");
-   user::AuthServiceServer login_server(sc);
-
-   bts::md::MBOProcessor mbo_processor;
-   bts::md::TradeSummarySignalDispatcher tss_dispatcher;
-   // The py plugin manager is shared by md and execution, so it lives in this scope.
-   auto py_plugin_man = std::make_shared<python_interop::PluginManager>();
-   // start work loop in background
-   py_plugin_man->start_work_thread();
-   md::cmds::Run::run(py_plugin_man);
-
-   api::synthetic::SyntheticSecurityService synthetic_service(sc);
-   bts::md::MDClient mdclient(
-       &utils::Dispatcher::background(), &synthetic_service,
-       &mbo_processor.reader());
-
-   bts::md::QueryServerPtr md_query_server(
-       bts::md::run_query_server(sc, mdclient, stats));
-
-   bts::md::MBOServiceServer mboserver(
-       sc, utils::Dispatcher::background(), mdclient);
-   bts::md::MarketDataService mdserver(mdclient, sc, stats);
-
-   // initialize order plugins
-   orderman::DynamicOrderRegistry dyn_order_registry;
-   SparkServices plugin_services(dyn_order_registry);
-   plugin::init_plugins(plugin_services);
-   // finished initializing order plugins
-
-   auto order_service_server = create_order_service_server(
-       sc, mdclient, dyn_order_registry, stats, perf_log, tss_dispatcher,
-       py_plugin_man);
-   support::ErrorReportServer error_report_service(sc);
-   admin::AdminServiceServer admin_service(sc);
-   bts::md::auxiliary::MarketDataAuxiliaryServiceServer md_aux_server(sc);
-   bts::api::orderman::algo::AlgoDefinitionServiceServer algo_definition_server(
-       sc);
-   bts::analytics::AnalyticsServiceServer analytics_server(sc);
-   position::BasePositionServiceServer base_pos_server(sc);
-   position::MoneyFileServiceServer money_file_server(sc);
-   position::PositionServiceServer position_service(sc, mdclient);
-   position::MarginServiceServer margin_service(sc, mdclient);
-   metrics::ShellLatencyServer shell_latency_service(
-       sc, stats, utils::Dispatcher::background());
-   bts::md::recording::MDRecorder::start_recording(mdclient);
-
-   auto bg_health =
-       std::make_shared<metrics::IntCounterMetric>("spark_background_thread");
-   utils::DispatcherTimer timer(utils::Dispatcher::background(), 5000, [=]() {
-      bg_health->increment();
-   });
-   timer.start();
-
-   utils::DispatcherTimer timer2(utils::Dispatcher::background(), 1000, [&]() {
-      perf_log.store_log_entries();
-   });
-   timer2.start();
-   stats.publish(bg_health);
-   subprocess::run_all();
-
-   quickbus.run();
-}
-
-Run::Run() : CLI::App("Run default sparkd services", "run") {
-   callback([]() { execute(); });
-}
-
-} // namespace bts::cmds
@@ -1,32 +0,0 @@
-#include "LiveLadderDemo.h"
-#include "python_interop/plugin_manager.h"
-#include "spark/api/md/MarketDataConfig.pb.h"
-#include "spark/persistence/IDGenerator.h"
-#include "sparkd/md/MarketDataGateway.h"
-#include "sparkd/md/listeners/LiveLadderPrinter.h"
-
-#include <thread>
-
-namespace bts::cmds {
-
-using bts::api::md::MarketDataConfig;
-using bts::persistence::IDGenerator;
-
-LiveLadderDemo::LiveLadderDemo(const MarketDataConfig &config) {
-   char ssl_cert_file[] = "SSL_CERT_FILE=/etc/ssl/cert/ca-certificates.crt";
-   putenv(ssl_cert_file);
-
-   auto py_plugin_man = std::make_shared<python_interop::PluginManager>();
-   py_plugin_man->start_work_thread();
-   md::MarketDataGateway gw{config, IDGenerator::instance_ptr(), py_plugin_man};
-   md::listeners::LiveLadderPrinter ladder_printer;
-   gw.start();
-
-   const std::size_t n_loops = 16;
-   for (std::size_t n = 0; n < n_loops; n++) {
-      std::cout << "Main loop iter " << n + 1;
-      std::this_thread::sleep_for(std::chrono::seconds(5));
-   }
-}
-
-} // namespace bts::cmds
@@ -1,184 +0,0 @@
-#include "execution_gateway_provider.h"
-#include "binance_execution/cross_margin/cross_margin_gateway.h"
-#include "binance_execution/gateway_config.h"
-#include "binance_execution/spots/spots_gateway.h"
-#include "binance_execution/usds_futures/usds_futures_gateway.h"
-#include "okx_execution/okx_execution_gateway.h"
-#include "onixs_fix/OnixFIXAdapter.h"
-#include "python_interop/plugin_manager.h"
-#include "spark/api/domain/Venue.h"
-#include "spark/domain/DomainSecurityUtils.h"
-#include "spark/domain_model/Security.pb.h"
-#include "spark/domain_model/Venue.h"
-#include "spark/io/JsonReader.h"
-#include "spark/orderman/gateways/CCAPIExecutionGateway.h"
-#include "spark/orderman/gateways/FakeExecutionGateway.h"
-#include "spark/orderman/gateways/GLBXExecutionGateway.h"
-#include "spark/orderman/gateways/GLBXiLink3ExecutionGateway.h"
-#include "spark/orderman/gateways/GeneralExecutionGateway.h"
-#include "spark/orderman/gateways/ICEExecutionGateway.h"
-#include "spark/orderman/gateways/IExecutionGenericAdapter.h"
-#include "spark/orderman/gateways/sim/SimExecutionGatewayFactory.h"
-#include "spark/orderman/gateways/sim/SimExecutionGatewayUniqueWrapper.h"
-#include "spark/utils/PBUtils.h"
-#include "spark/utils/bts_assert.h"
-#include "spark/utils/type_name.h"
-#include "sparkd/execution/py_execution_gateway_adapter.h"
-#include "sparkd/execution/py_execution_gateway_config.h"
-
-namespace bts::orderman {
-
-#define _LOG_FN(level, fn, ...)                                                \
-   LOG(OrderMan, level, "ExecutionGatewayProvider::" fn ". " __VA_ARGS__)
-#define INFO_FN(fn, ...) _LOG_FN(Info, fn, __VA_ARGS__)
-#define WARN_FN(fn, ...) _LOG_FN(Warn, fn, __VA_ARGS__)
-#define ERROR_FN(fn, ...) _LOG_FN(Error, fn, __VA_ARGS__)
-
-using bts::io::JsonReader;
-using nlohmann::json;
-using std::function;
-using std::optional;
-using std::shared_ptr;
-using std::string;
-using std::vector;
-
-template <typename T, typename... Args>
-static std::unique_ptr<T> make_gw(
-    const config::JetFIX::Connection &conn, Args &&...args) {
-   LOG(OrderMan, Info,
-       R"([create_exe_gw.json]){{ "name": "{}", "venue": "{}", "type": "{}", "ts": {} }})",
-       conn.name(), bts::api::venue_name_pretty(conn.venue()),
-       std::string(bts::utils::cached_type_name<T>()),
-       bts::core::TimeStamp::now_ns());
-   return std::make_unique<T>(std::forward<Args>(args)...);
-}
-
-struct ExecutionGatewayProvider::PImpl {
-   PImpl(
-       trace::PerfLog *l,
-       md::SecurityOracle o,
-       config::JetFIX config,
-       function<optional<json>(const string &)> load_config_json,
-       shared_ptr<python_interop::PluginManager> py_plugin_man)
-       : l(l), o(std::move(o)), config(std::move(config)),
-         load_config_json(std::move(load_config_json)),
-         py_plugin_man(std::move(py_plugin_man)) {}
-
-   trace::PerfLog *l;
-   md::SecurityOracle o;
-   config::JetFIX config;
-   function<optional<json>(const string &)> load_config_json;
-   shared_ptr<python_interop::PluginManager> py_plugin_man;
-
-   template <class T>
-   T parse_config_json(const string &name) {
-      auto j = load_config_json(name);
-      bts_assert_msgf(
-          j, "Failed to load execution gateway config. name: {}", name);
-      JsonReader jr{&*j};
-      T res;
-      bts_assert_msgf(
-          jr.read(res), "Failed to parse execution gateway config. name: {}",
-          name);
-      return res;
-   }
-
-   std::unique_ptr<IExecutionGateway> create_gateway(
-       const config::JetFIX::Connection &conn) {
-      INFO_FN("create_gateway", "conn: {}", utils::pb::to_string(conn));
-      // *************************************
-      // Simple hack to integrate python execution gateways:
-      //   If a json file exists with the same name as the jetfix connection,
-      //   but with a "py_" prefix, assume that file defines python gateway
-      //   config for that connection.
-      // *************************************
-      if (SimExecutionGatewayFactory::is_sim_execution_on(conn.venue())) {
-         return make_gw<SimExecutionGatewayUniqueWrapper>(conn, conn.venue());
-      }
-      auto py_cfg_j = load_config_json(fmt::format("py_{}", conn.name()));
-      if (py_cfg_j) {
-         auto py_cfg = py_cfg_j->get<PyExecutionGatewayConfig>();
-         bts_assert(
-             domain::DomainSecurityUtils::convert(conn.venue()) ==
-             py_cfg.venue);
-         return make_gw<PyExecutionGatewayAdapter>(
-             conn, o, std::move(py_cfg), conn, py_plugin_man);
-      }
-      switch (conn.venue()) {
-      case api::VENUE_ICE:
-      case api::VENUE_SIMICE:
-         return make_gw<ICEExecutionGateway>(conn, conn, *l, o);
-      case api::VENUE_GLBX:
-         if (conn.use_ilink3()) {
-            make_gw<ilink3::GLBXiLink3ExecutionGateway>(conn, conn, *l, o);
-         }
-         return make_gw<GLBXExecutionGateway>(conn, conn, *l, o);
-      case api::VENUE_DERIBIT:
-         return make_gw<CCAPIExecutionGateway>(conn, conn.venue(), o);
-      case api::VENUE_CBOE:
-         return make_gw<GeneralExecutionGateway>(
-             conn, conn.venue(), std::make_unique<OnixFIXAdapter>(o), o);
-      case api::VENUE_OKX:
-         return make_gw<OkxExecutionGateway>(conn, o);
-      case api::VENUE_BINANCE: {
-         auto cfg =
-             parse_config_json<binance_execution::GatewayConfig>(conn.name());
-         cfg.expand_env();
-         cfg.deduce_key_type();
-         if (cfg.use_cross_margin_api) {
-            return make_gw<binance_execution::cross_margin::CrossMarginGateway>(
-                conn, o, std::move(cfg), conn);
-         }
-         return make_gw<binance_execution::spots::SpotsGateway>(
-             conn, o, std::move(cfg), conn);
-      }
-      case api::VENUE_BINANCE_USDS_FUTURES: {
-         auto cfg =
-             parse_config_json<binance_execution::GatewayConfig>(conn.name());
-         cfg.expand_env();
-         if (cfg.use_cross_margin_api) {
-            WARN_FN(
-                "create_gateway(BinanceUsdsFutures)",
-                "Config has 'use_cross_margin_api' set to true, but this is "
-                "only supported for spots (using regular api).");
-         }
-         return make_gw<binance_execution::usds_futures::UsdsFuturesGateway>(
-             conn, o, std::move(cfg), conn);
-      }
-      default:
-         return make_gw<FakeExecutionGateway>(conn, o);
-      }
-   }
-
-   std::vector<OrderGatewayDefinition> create() {
-      std::vector<OrderGatewayDefinition> gateways;
-      for (auto c : config.connection()) {
-         auto &gw = gateways.emplace_back();
-         gw.connection = std::move(c);
-         gw.gateway = create_gateway(gw.connection);
-      }
-      return gateways;
-   }
-};
-
-ExecutionGatewayProvider::ExecutionGatewayProvider(
-    trace::PerfLog *l,
-    md::SecurityOracle o,
-    config::JetFIX config,
-    std::function<std::optional<nlohmann::json>(const std::string &)>
-        load_config_json,
-    shared_ptr<python_interop::PluginManager> py_plugin_man)
-    : me_(std::make_unique<PImpl>(
-          l,
-          std::move(o),
-          std::move(config),
-          std::move(load_config_json),
-          std::move(py_plugin_man))) {}
-
-ExecutionGatewayProvider::~ExecutionGatewayProvider() = default;
-
-std::vector<OrderGatewayDefinition> ExecutionGatewayProvider::create() {
-   return me_->create();
-}
-
-} // namespace bts::orderman
@@ -1,637 +0,0 @@
-#include "py_execution_gateway_adapter.h"
-#include "python_interop/execution/execution_gateway.h"
-#include "python_interop/execution/execution_gateway_bridge.h"
-#include "python_interop/execution/execution_gateway_bridge_impl.h"
-#include "python_interop/execution/py_cancel_order_request.h"
-#include "python_interop/execution/py_cancel_replace_order_request.h"
-#include "python_interop/execution/py_create_order_request.h"
-#include "python_interop/execution/py_execution_gateway.h"
-#include "python_interop/execution/py_execution_report.h"
-#include "python_interop/plugin_manager.h"
-#include "python_interop/py_contract_price_info.h"
-#include "python_interop/py_contract_quantity_info.h"
-#include "python_interop/py_contract_specification.h"
-#include "python_interop/py_security.h"
-#include "python_interop/python_object.h"
-#include "spark/api/domain/Venue.h"
-#include "spark/api/service_helpers/AdminServiceUtils.h"
-#include "spark/api/services/AdminService.pb.h"
-#include "spark/api/services/OrderService.h"
-#include "spark/api/services/exec_type.h"
-#include "spark/api/services/native_order_status.h"
-#include "spark/api/services/native_order_type.h"
-#include "spark/api/services/time_in_force.h"
-#include "spark/config/protos/jetfix.pb.h"
-#include "spark/core/TimeStamp.h"
-#include "spark/core/start_stop_state_machine.h"
-#include "spark/domain/tick_utils.h"
-#include "spark/domain_model/Security.h"
-#include "spark/domain_model/Security.pb.h"
-#include "spark/domain_model/SecurityType.h"
-#include "spark/domain_model/Side.h"
-#include "spark/domain_model/Venue.h"
-#include "spark/io/json_formatter.h"
-#include "spark/logging/logging.h"
-#include "spark/logging/trace_log.h"
-#include "spark/md/SecurityOracle.h"
-#include "spark/mercury/events/Update.h"
-#include "spark/orderman/CancelOrderMsg.h"
-#include "spark/orderman/CancelReplaceOrderMsg.h"
-#include "spark/orderman/CreateOrderMsg.h"
-#include "spark/orderman/RFQMsg.h"
-#include "spark/orderman/events/NewAdminDataEvent.h"
-#include "spark/orderman/events/NewSecuritiesEvent.h"
-#include "spark/orderman/gateways/IExecutionGateway.h"
-#include "spark/orderman/log_messages/cancel_order_log.h"
-#include "spark/orderman/log_messages/create_order_log.h"
-#include "spark/orderman/log_messages/execution_report_log.h"
-#include "spark/orderman/log_messages/modify_order_log.h"
-#include "spark/orderman/orderman_utils/ExchangePosition.h"
-#include "spark/orderman/orderman_utils/ExecutionReport.h"
-#include "spark/orderman/orderman_utils/execution_report_store.h"
-#include "spark/orderman/orderman_utils/orderman_common.h"
-#include "spark/orderman/orderman_utils/simple_validation.h"
-#include "spark/utils/PBUtils.h"
-#include "spark/utils/bts_lock_guard.h"
-#include "spark/utils/overloaded.h"
-#include "spark/utils/stopwatch.h"
-#include "spark/utils/string_utils.h"
-#include "sparkd/execution/py_execution_gateway_config.h"
-
-#include <fmt/format.h>
-#include <future>
-#include <nlohmann/json.hpp>
-#include <string>
-#include <variant>
-#include <vector>
-
-namespace bts::orderman {
-
-#define ASSERT_FN(test, fn, ...)                                               \
-   bts_assert_msgf(test, "PyExecutionGatewayAdapter::" fn ". " __VA_ARGS__)
-#define LOG_FN(level, fn, ...)                                                 \
-   LOG(OrderMan, level, "PyExecutionGatewayAdapter::" fn ". " __VA_ARGS__)
-#define DIAG_FN(fn, ...) LOG_FN(bts::Diag, fn, __VA_ARGS__)
-#define INFO_FN(fn, ...) LOG_FN(bts::Info, fn, __VA_ARGS__)
-#define WARN_FN(fn, ...) LOG_FN(bts::Warn, fn, __VA_ARGS__)
-#define ERROR_FN(fn, ...) LOG_FN(bts::Error, fn, __VA_ARGS__)
-
-#define _LOG(level, ...) LOG(OrderMan, level, __VA_ARGS__)
-#define INFO(...) _LOG(bts::Info, __VA_ARGS__)
-#define ERROR(...) _LOG(bts::Error, __VA_ARGS__)
-
-#define _LOG_TRACE(level, name)                                                \
-   LOG_TRACE(OrderMan, level, std::string("py_exe_gw_adapt"), std::string(name))
-#define INFO_TRACE(name) _LOG_TRACE(bts::Info, name)
-#define WARN_TRACE(name) _LOG_TRACE(bts::Warn, name)
-#define ERROR_TRACE(name) _LOG_TRACE(bts::Error, name)
-
-#define _LOG_TRACE_PARAMS(level, name, params)                                 \
-   LOG_TRACE_PARAMS(                                                           \
-       OrderMan, level, std::string("py_exe_gw_adapt"), std::string(name),     \
-       params)
-#define DIAG_TRACE_PARAMS(name, params)                                        \
-   _LOG_TRACE_PARAMS(bts::Diag, name, params)
-#define ERROR_TRACE_PARAMS(name, params)                                       \
-   _LOG_TRACE_PARAMS(bts::Error, name, params)
-
-#define INFO_TRACE_PARAMS(name, params)                                        \
-   LOG_TRACE_PARAMS(                                                           \
-       OrderMan, bts::Info, std::string("py_exe_gw_adapt"), std::string(name), \
-       params)
-
-using bts::Quantity;
-using bts::api::security_type_parse_smart;
-using bts::api::SecurityType;
-using bts::api::Venue;
-using bts::api::venue_parse_smart;
-using bts::api::admin::BTSAccount;
-using bts::api::orderman::exec_type_parse_smart;
-using bts::api::orderman::native_order_status_parse_smart;
-using bts::api::orderman::time_in_force_parse_smart;
-using bts::core::Delegate;
-using bts::core::StartStopStateMachine;
-using bts::core::TimeStamp;
-using bts::md::SecurityOracle;
-using bts::orderman::CancelOrderMsg;
-using bts::orderman::CancelReplaceOrderMsg;
-using bts::orderman::CreateOrderMsg;
-using bts::orderman::ExchangeOrderID64;
-using bts::orderman::ExchangePosition;
-using bts::orderman::ExecutionReport;
-using bts::orderman::ExecutionReportStore;
-using bts::orderman::NewAdminDataEvent;
-using bts::orderman::NewSecuritiesEvent;
-using bts::orderman::OrderKey;
-using bts::orderman::OrderMsg;
-using bts::orderman::RFQMsg;
-using bts::system::string_utils::parse_uint;
-using nlohmann::json;
-using python_execution::ExecutionGateway;
-using python_execution::ExecutionGatewayBridgeImpl;
-using python_execution::PyCancelOrderRequest;
-using python_execution::PyCancelReplaceOrderRequest;
-using python_execution::PyCreateOrderRequest;
-using python_interop::PluginManager;
-using python_interop::PythonObjectT;
-using std::int64_t;
-using std::optional;
-using std::shared_ptr;
-using std::size_t;
-using std::string;
-using std::thread;
-using std::uint32_t;
-using std::uint64_t;
-using std::vector;
-
-struct ExecutionGatewayCallbacks {
-   bts::orderman::OnExeReport exe_report;
-   bts::orderman::OnGwStatusChange gw_status_change;
-   bts::orderman::OnCancelReject cancel_reject;
-   bts::orderman::OnExchangePositionState exchange_position_state;
-   bts::orderman::OnExchangeOpenOrdersState exchange_open_orders;
-};
-
-struct PyExecutionGatewayAdapter::PImpl {
-   explicit PImpl(
-       bts::md::SecurityOracle sec_oracle,
-       PyExecutionGatewayConfig config,
-       bts::config::JetFIX::Connection jetfix_connection,
-       shared_ptr<PluginManager> plugin_man)
-       : sec_oracle(std::move(sec_oracle)), config(std::move(config)),
-         name(fmt::format("{}_py_gw", string{jetfix_connection.name()})),
-         bridge_ptr(std::make_shared<ExecutionGatewayBridgeImpl>(
-             jetfix_connection.name())),
-         bridge(*bridge_ptr), plugin_man(std::move(plugin_man)),
-         jetfix_connection(std::make_shared<bts::config::JetFIX_Connection>(
-             std::move(jetfix_connection))),
-         orders(this->config.account, this->jetfix_connection) {
-      bts_assert(this->jetfix_connection->venue() != api::VENUE_UNSET);
-      bts_assert(!this->jetfix_connection->name().empty());
-      pos.exchange = this->jetfix_connection->venue();
-      sm.set_name(name);
-      sm.set_subsystem(bts::log::OrderMan);
-      sm.start_core_impl.bind<&PImpl::start_impl>(*this);
-      sm.stop_core_impl.bind<&PImpl::stop_impl>(*this);
-   }
-
-   ~PImpl() { stop(); }
-
-   enum RequestType {
-      ORDER,
-      POSITION,
-   };
-
-   SecurityOracle sec_oracle;
-   PyExecutionGatewayConfig config;
-   std::string name;
-   std::shared_ptr<ExecutionGatewayBridgeImpl> bridge_ptr;
-   ExecutionGatewayBridgeImpl &bridge;
-   shared_ptr<PluginManager> plugin_man;
-   shared_ptr<bts::config::JetFIX_Connection> jetfix_connection;
-   optional<thread> pos_snapshot_thread;
-   optional<thread> py_thread;
-   bool is_ready = false;
-   bool is_ready_dirty = false;
-   volatile bool py_keep_going = false;
-   uint32_t n_active = 0;
-
-   StartStopStateMachine sm;
-   // SimpleOrderKeyMap key_map;
-
-   ExecutionReportStore orders;
-   ExecutionGatewayCallbacks callbacks;
-   ExchangePosition pos;
-   BTSAccount account;
-   OnExeReport exe_report_callback;
-
-   void to_exe_report(
-       bts::orderman::ExecutionReport &er,
-       const python_execution::PyExecutionReport &pyer,
-       std::function<const bts::api::Security *(
-           bts::api::Venue, bts::api::SecurityType, const std::string &)>
-           find_sec) {
-      *er.order_key = pyer.key();
-      er.account_name->copy_from(pyer.account());
-      if (std::holds_alternative<ExchangeOrderID>(er.exchange_order_id)) {
-         std::get<ExchangeOrderID>(er.exchange_order_id)
-             ->copy_from(pyer.exchange_order_id());
-      } else {
-         er.exchange_order_id = ExchangeOrderID{pyer.exchange_order_id()};
-      }
-      native_order_type_parse_smart(pyer.order_type(), &er.order_type);
-      native_order_status_parse_smart(pyer.order_status(), &er.order_status);
-      time_in_force_parse_smart(pyer.time_in_force(), &er.time_in_force);
-      exec_type_parse_smart(pyer.exec_type(), &er.exec_type);
-      Venue venue;
-      bts_assert(venue_parse_smart(pyer.venue(), &venue));
-      SecurityType sec_type;
-      bts_assert(security_type_parse_smart(pyer.security_type(), &sec_type));
-      er.security = find_sec(venue, sec_type, pyer.exchange_security_id());
-      er.side = pyer.is_buy() ? bts::api::SIDE_BUY : bts::api::SIDE_SELL;
-      *er.price = pyer.price();
-      *er.stop_price = std::numeric_limits<double>::quiet_NaN();
-      er.quantity = Quantity::create_new(pyer.quantity());
-      *er.avg_price = pyer.avg_filled_price();
-      er.filled_quantity = Quantity::create_new(pyer.filled_quantity());
-      *er.last_price = pyer.last_filled_price();
-      er.last_shares = Quantity::create_new(pyer.last_filled_quantity());
-      er.transact_time =
-          ExchangeTimeStamp64{static_cast<uint64_t>(pyer.transact_time())};
-   }
-
-   void start() {
-      INFO_TRACE("start");
-      sm.start();
-   }
-
-   // should not be called directly (use the state machine)
-   void start_impl() {
-      INFO_TRACE("start_impl");
-      bts_assert(!py_thread);
-      py_keep_going = true;
-      py_thread.emplace([&]() {
-         auto orig_signal_handler = std::signal(SIGINT, SIG_DFL);
-         std::future<shared_ptr<PythonObjectT<ExecutionGateway>>> gw_fut =
-             plugin_man->start_execution_gateway(
-                 name, config.gateway_type, bridge_ptr);
-         gw_fut.wait();
-         std::signal(SIGINT, orig_signal_handler);
-         bridge.run_py(&py_keep_going, gw_fut.get(), [&](auto callback) {
-            plugin_man->invoke(callback);
-         });
-      });
-   }
-
-   void stop() {
-      INFO_TRACE("stop");
-      sm.stop();
-   }
-
-   void stop_impl() {
-      INFO_TRACE("stop_impl");
-      py_keep_going = false;
-      if (py_thread) {
-         py_thread->join();
-         py_thread.reset();
-      }
-   }
-
-   void create_order(const CreateOrderMsg &m) {
-      LOG_CREATE_ORDER(Info, m);
-      PyCreateOrderRequest py_req;
-      to_py(py_req, m);
-      bridge.on_order_msg(
-          std::make_shared<PyCreateOrderRequest>(std::move(py_req)));
-   }
-
-   void update_order(const CancelReplaceOrderMsg &m) {
-      LOG_MODIFY_ORDER(Info, m);
-      PyCancelReplaceOrderRequest py_req;
-      to_py(py_req, m);
-      bridge.on_order_msg(
-          std::make_shared<PyCancelReplaceOrderRequest>(std::move(py_req)));
-   }
-
-   void cancel_order(const CancelOrderMsg &m) {
-      LOG_CANCEL_ORDER(Info, m);
-      PyCancelOrderRequest req;
-      to_py(req, m);
-      bridge.on_order_msg(
-          std::make_shared<PyCancelOrderRequest>(std::move(req)));
-   }
-
-   void on_securities(const std::vector<api::Security const *> &secs) {
-      std::vector<std::shared_ptr<bts::md::py_interop::PySecurity>> py_secs;
-      for (const auto *sec : secs) {
-         bts::md::py_interop::PySecurity py_sec;
-         to_py(py_sec, *sec);
-         py_secs.push_back(std::make_shared<bts::md::py_interop::PySecurity>(
-             std::move(py_sec)));
-      }
-      bridge.on_securities(std::move(py_secs));
-   }
-
-   void get_position_info() {
-      sm.invoke_if_active([&](auto) {
-         //
-      });
-   }
-
-   void dispatch_is_ready() {
-      BTS_LOCK_GUARD(sm.mutex());
-      if (is_ready_dirty) {
-         is_ready_dirty = false;
-         callbacks.gw_status_change(is_ready);
-      }
-   }
-
-   void from_py(
-       const python_execution::PyExecutionReport &py_er,
-       ExecutionReport &er) const {
-      *er.order_key = py_er.key();
-      bts_assert(venue_parse_smart(py_er.venue(), &er.exchange));
-      SecurityType sec_type;
-      bts_assert(security_type_parse_smart(py_er.security_type(), &sec_type));
-      er.transact_time =
-          ExchangeTimeStamp64{static_cast<uint64_t>(py_er.transact_time())};
-      uint64_t exch_oid;
-      if (parse_uint(py_er.exchange_order_id(), exch_oid)) {
-         er.exchange_order_id.emplace<ExchangeOrderID64>().value() = exch_oid;
-      } else {
-         er.exchange_order_id.emplace<ExchangeOrderID>().value() =
-             py_er.exchange_order_id();
-      }
-      er.security = sec_oracle.get_security(
-          er.exchange, sec_type, py_er.exchange_security_id());
-      er.leg_security = nullptr;
-      er.side = py_er.is_buy() ? api::SIDE_BUY : api::SIDE_SELL;
-      time_in_force_parse_smart(py_er.time_in_force(), &er.time_in_force);
-      native_order_type_parse_smart(py_er.order_type(), &er.order_type);
-      exec_type_parse_smart(py_er.exec_type(), &er.exec_type);
-      native_order_status_parse_smart(py_er.order_status(), &er.order_status);
-      er.manual_order_indicator = bts::api::orderman::MANUAL_ORDER_INDICATOR_NO;
-      er.price.value() = py_er.price();
-      er.quantity = py_er.quantity();
-      er.filled_quantity = py_er.filled_quantity();
-      er.last_shares = py_er.last_filled_quantity();
-      er.last_price.value() = py_er.last_filled_price();
-      er.avg_price.value() = py_er.avg_filled_price();
-      er.stop_price.value() = std::numeric_limits<double>::quiet_NaN();
-      const string &tx_id = py_er.transact_id();
-      bts_assert(tx_id.size() < er.exchange_fill_id.value().max_size());
-      bts_assert(!py_er.transact_id().empty());
-      er.exchange_fill_id->copy_from(py_er.transact_id());
-   }
-
-   void to_py(
-       bts::md::py_interop::PySecurity &py_sec, const bts::api::Security &sec) {
-      py_sec.set_id(sec.id());
-      py_sec.set_asset(sec.symbol());
-      py_sec.set_venue(bts::api::venue_name_pretty(sec.exchange()));
-      py_sec.set_currency(sec.currency());
-      // Exchange ID and name
-      std::string name = bts::securities::descriptor(sec);
-      py_sec.set_exchange_id(name);
-      py_sec.set_name(std::move(name));
-      // Price ticks
-      const bts::api::ContractSpecification &spec = sec.contract_spec();
-      md::py_interop::PyContractSpecification py_spec;
-      md::py_interop::PyContractPriceInfo &py_price_info =
-          py_spec.mutable_price_infos().emplace_back();
-      float tick_unit;
-      float tick_base;
-      if (!domain::get_tick_factors(spec, tick_unit, tick_base)) {
-         WARN_FN(
-             "on_securities", "Failed to get price tick from {} {}. spec: {}. ",
-             sec.exchange(), name, utils::pb::to_string(sec.contract_spec()));
-         tick_unit = 0.01f;
-         tick_base = 0.01f;
-      }
-      py_price_info.set_tick_unit(tick_unit);
-      py_price_info.set_tick_base(tick_base);
-      // Quantity ticks
-      md::py_interop::PyContractQuantityInfo &py_qty_info =
-          py_spec.mutable_quantity_infos().emplace_back();
-      if (spec.quantity_infos().empty()) {
-         tick_unit = 1.0f;
-         tick_base = 1.0f;
-         WARN_FN(
-             "on_securities", "Failed to get qty tick from {} {}. spec: {}. ",
-             sec.exchange(), name, utils::pb::to_string(sec.contract_spec()));
-      } else {
-         tick_unit = spec.quantity_infos(0).tick_unit();
-         tick_base = spec.quantity_infos(0).tick_base();
-      }
-      py_qty_info.set_tick_unit(tick_unit);
-      py_qty_info.set_tick_base(tick_base);
-      py_sec.set_contract_spec(std::move(py_spec));
-      // Spot/Future/Option
-      switch (sec.Contract_case()) {
-      case bts::api::Security::kSpot: {
-         py_sec.set_spot(bts::md::py_interop::PySpot{});
-         break;
-      }
-      case bts::api::Security::kFuture: {
-         const bts::api::FutureContract &fut = sec.future();
-         bts::md::py_interop::PyFuture py_fut;
-         py_fut.set_expiration(fmt::format(
-             "{}-{:02}-{:02}", fut.expiration_year(), fut.expiration_month(),
-             fut.expiration_day()));
-         py_sec.set_future(std::move(py_fut));
-         break;
-      }
-      case bts::api::Security::kOption: {
-         const bts::api::OptionContract &opt = sec.option();
-         bts::md::py_interop::PyOption py_opt;
-         py_opt.set_expiration(fmt::format(
-             "{}-{:02}-{:02}", opt.expiration_year(), opt.expiration_month(),
-             opt.expiration_day()));
-         py_opt.set_strike(opt.strike());
-         py_opt.set_is_call(opt.call_put() == bts::api::PUTORCALL_CALL);
-         py_sec.set_option(std::move(py_opt));
-         break;
-      }
-      default:
-         bts_assert_msg(false, "Only spots, futures, and options supported");
-         break;
-      }
-   }
-
-   void to_py(PyCreateOrderRequest &py_req, const CreateOrderMsg &msg) {
-      py_req.set_key(msg.key.value());
-      py_req.set_exchange_security_id(msg.security_name());
-      py_req.set_venue(bts::api::venue_name_pretty(msg.venue()));
-      py_req.set_is_buy(msg.side == bts::api::SIDE_BUY);
-      py_req.set_price(msg.price.value());
-      py_req.set_quantity(msg.quantity.full_value_float());
-      py_req.set_order_type(
-          bts::api::orderman::native_order_type_name_pretty(msg.order_type));
-      py_req.set_time_in_force(
-          bts::api::orderman::time_in_force_name_pretty(msg.time_in_force));
-   }
-
-   void to_py(
-       PyCancelReplaceOrderRequest &py_req,
-       const bts::orderman::CancelReplaceOrderMsg &msg) {
-      auto &cf = msg.create_fields;
-      py_req.set_key(cf.key.value());
-      py_req.set_exchange_order_id(
-          std::to_string(msg.exchange_order_id.value()));
-      py_req.set_exchange_security_id(cf.security_name());
-      py_req.set_venue(bts::api::venue_name_pretty(cf.venue()));
-      py_req.set_is_buy(cf.side == bts::api::SIDE_BUY);
-      py_req.set_price(cf.price.value());
-      py_req.set_quantity(cf.quantity.full_value_float());
-   }
-
-   void to_py(PyCancelOrderRequest &py_req, const CancelOrderMsg &msg) {
-      py_req.set_key(msg.key.value());
-      py_req.set_exchange_order_id(
-          std::to_string(msg.exchange_order_id.value()));
-      py_req.set_exchange_security_id(msg.security_name());
-      py_req.set_venue(bts::api::venue_name_pretty(msg.venue()));
-      py_req.set_is_buy(msg.side == bts::api::SIDE_BUY);
-   }
-
-   void dispatch_order_updates() {
-      BTS_LOCK_GUARD(sm.mutex());
-      std::shared_ptr<const python_execution::PyExecutionReport> py_er_p;
-      utils::Stopwatch sw;
-      uint32_t n_dequed = 0;
-      uint32_t n_invalid = 0;
-      sw.start();
-      while (bridge.pop_exe_report(py_er_p)) {
-         n_dequed++;
-         const python_execution::PyExecutionReport &py_er = *py_er_p;
-         shared_ptr<const ExecutionReport> er_p =
-             orders.update(OrderKey{py_er.key()}, [&](auto &er_p, bool) {
-                from_py(py_er, *er_p);
-             });
-         const ExecutionReport &er = *er_p;
-         string error;
-         if (!validate(er, error)) {
-            LOG_EXECUTION_REPORT(Error, er_p, "INVALID_STATE", error);
-            n_invalid++;
-            continue;
-         }
-         LOG_EXECUTION_REPORT(Info, er_p);
-         switch (er.order_status) {
-         case api::orderman::NATIVE_ORDER_STATUS_FILLED:
-         case api::orderman::NATIVE_ORDER_STATUS_CANCELED:
-         case api::orderman::NATIVE_ORDER_STATUS_REJECTED: {
-            orders.erase(er.order_key);
-            INFO_FN(
-                "dispatch_order_updates",
-                "Order reached terminal state {} (id={})", er.order_status,
-                er.exchange_order_id);
-            break;
-         }
-         default:
-            break;
-         }
-         INFO_FN(
-             "dispatch_order_updates",
-             "Firing exe report callback. {} {} {} @ {} {} (filled {}/{}, "
-             "id={}). {} active orders",
-             er.side, er.security_name(), er.quantity, er.price,
-             er.order_status, er.filled_quantity, er.quantity, er.order_key,
-             orders.size());
-         callbacks.exe_report(er_p, {});
-      }
-      sw.stop();
-      if (n_dequed > 0) {
-         INFO(
-             R"([py_exe_gw_disp_ord_stats.json]{{ "n_active": {}, "n_dequed": {}, "n_invalid": {}, "dur": {}, "ts": {} }})",
-             orders.size(), n_dequed, n_invalid, sw.total_elapsed().nanos(),
-             TimeStamp::now_ns());
-      }
-   }
-};
-
-PyExecutionGatewayAdapter::PyExecutionGatewayAdapter(
-    SecurityOracle sec_oracle,
-    PyExecutionGatewayConfig session_cfg,
-    bts::config::JetFIX::Connection jetfix_connection,
-    shared_ptr<PluginManager> plugin_man)
-    : me(std::make_unique<PImpl>(
-          std::move(sec_oracle),
-          std::move(session_cfg),
-          std::move(jetfix_connection),
-          std::move(plugin_man))) {
-   bts_assert(
-       me->config.account.size() <=
-       ExecutionReport{}.account_name.value().max_size());
-}
-
-PyExecutionGatewayAdapter::~PyExecutionGatewayAdapter() = default;
-
-void PyExecutionGatewayAdapter::start() { me->start(); }
-
-void PyExecutionGatewayAdapter::on_securities(
-    const bts::mercury::Update<NewSecuritiesEvent> & /*update*/) {
-   INFO_FN(
-       "on_securities", "t_id: {} t_name: {}", bts::os::get_thread_id(),
-       bts::os::get_thread_name());
-   std::vector<api::Security const *> secs =
-       me->sec_oracle.get_securities([&](const bts::api::Security &sec) {
-          return sec.exchange() == me->jetfix_connection->venue();
-       });
-   me->on_securities(secs);
-}
-
-void PyExecutionGatewayAdapter::on_admin_data(
-    const bts::mercury::Update<NewAdminDataEvent> &update) {
-   INFO_FN(
-       "on_admin_data", "t_id: {} t_name: {}", bts::os::get_thread_id(),
-       bts::os::get_thread_name());
-   if (update.events == nullptr) {
-      return;
-   }
-   std::lock_guard<std::recursive_mutex> lock{me->sm.mutex()};
-   const auto *account = bts::admin::AdminServiceUtils::find_account(
-       *update.events->back().admin_data, me->config.account);
-   if (account == nullptr) {
-      WARN_FN(
-          "on_admin_data",
-          "Gateway account not found. Cannot execute orders. name: '{}'",
-          me->config.account);
-      return;
-   }
-   if (me->account.name() == account->name() &&
-       me->account.id() == account->id()) {
-      return;
-   }
-   me->account.CopyFrom(*account);
-   INFO_FN("on_admin_data", "Resolved gateway account '{}'", account->name());
-}
-
-void PyExecutionGatewayAdapter::send_msgs(
-    vector<OrderMsg> const &msg_vec, bts::trace::CorID) {
-   INFO_FN("send_msgs", "count: {}", msg_vec.size());
-   for (const auto &msg : msg_vec) {
-      std::visit(
-          bts::overloaded{
-              [&](const CreateOrderMsg &m) { me->create_order(m); },
-              [&](const CancelReplaceOrderMsg &m) { me->update_order(m); },
-              [&](const CancelOrderMsg &m) { me->cancel_order(m); },
-              [&](const RFQMsg &) {
-                 bts_assert_msg(
-                     false, "PyExecutionGatewayAdapter RFQ not implemented");
-              },
-          },
-          msg);
-   }
-}
-
-void PyExecutionGatewayAdapter::send_fix(
-    const bts::jetfix::OutFIXPtr &, bts::trace::CorID) {
-   bts_assert_msg(false, "ExecutionGateway does not support fix");
-}
-
-void PyExecutionGatewayAdapter::dispatch_incoming_fix_messages() {
-   me->dispatch_is_ready();
-   me->dispatch_order_updates();
-}
-
-// Callbacks
-bts::orderman::OnExeReport &PyExecutionGatewayAdapter::on_exe_report() {
-   return me->callbacks.exe_report;
-}
-bts::orderman::OnGwStatusChange &
-PyExecutionGatewayAdapter::on_gw_status_change() {
-   return me->callbacks.gw_status_change;
-}
-bts::orderman::OnCancelReject &PyExecutionGatewayAdapter::on_cancel_reject() {
-   return me->callbacks.cancel_reject;
-}
-bts::orderman::OnExchangePositionState &
-PyExecutionGatewayAdapter::on_exchange_position_state() {
-   return me->callbacks.exchange_position_state;
-}
-bts::orderman::OnExchangeOpenOrdersState &
-PyExecutionGatewayAdapter::on_exchange_open_orders() {
-   return me->callbacks.exchange_open_orders;
-}
-
-} // namespace bts::orderman
@@ -1,50 +0,0 @@
-#pragma once
-
-#include "spark/config/protos/jetfix.pb.h"
-#include "spark/md/SecurityOracle.h"
-#include "spark/orderman/gateways/IExecutionGateway.h"
-
-#include <memory>
-
-namespace python_interop {
-class PluginManager;
-}
-
-namespace bts::orderman {
-
-struct PyExecutionGatewayConfig;
-
-class PyExecutionGatewayAdapter : public bts::orderman::IExecutionGateway {
- public:
-   explicit PyExecutionGatewayAdapter(
-       bts::md::SecurityOracle sec_oracle,
-       PyExecutionGatewayConfig session_cfg,
-       bts::config::JetFIX::Connection jetfix_connection,
-       std::shared_ptr<python_interop::PluginManager> plugin_man);
-   ~PyExecutionGatewayAdapter() override;
-
-   void start() override;
-   void dispatch_incoming_fix_messages() override;
-   void on_securities(
-       const bts::mercury::Update<bts::orderman::NewSecuritiesEvent> &)
-       override;
-   void on_admin_data(
-       const bts::mercury::Update<bts::orderman::NewAdminDataEvent> &) override;
-   void send_fix(const bts::jetfix::OutFIXPtr &, bts::trace::CorID) override;
-   void send_msgs(
-       std::vector<bts::orderman::OrderMsg> const &,
-       bts::trace::CorID) override;
-
-   bts::orderman::OnExeReport &on_exe_report() override;
-   bts::orderman::OnGwStatusChange &on_gw_status_change() override;
-   bts::orderman::OnCancelReject &on_cancel_reject() override;
-   bts::orderman::OnExchangePositionState &
-   on_exchange_position_state() override;
-   bts::orderman::OnExchangeOpenOrdersState &on_exchange_open_orders() override;
-
- private:
-   struct PImpl;
-   std::unique_ptr<PImpl> me;
-};
-
-} // namespace bts::orderman
@@ -1,44 +0,0 @@
-#pragma once
-
-#include "spark/api/domain/Venue.pb.h"
-
-#include <nlohmann/json.hpp>
-#include <string>
-#include <vector>
-
-namespace bts::orderman {
-
-struct PyExecutionGatewayConfig {
-   std::string gateway_type;
-   std::string account;
-   std::vector<std::string> module_path;
-   api::domain::Venue venue;
-};
-
-inline void to_json(nlohmann::json &j, const PyExecutionGatewayConfig &cfg) {
-   j["gateway_type"] = cfg.gateway_type;
-   j["account"] = cfg.account;
-   j["module_path"] = cfg.module_path;
-   j["venue"] = cfg.venue;
-}
-
-inline void from_json(const nlohmann::json &j, PyExecutionGatewayConfig &cfg) {
-   auto it = j.find("gateway_type");
-   if (it != j.end()) {
-      it->get_to(cfg.gateway_type);
-   }
-   it = j.find("account");
-   if (it != j.end()) {
-      it->get_to(cfg.account);
-   }
-   it = j.find("module_path");
-   if (it != j.end()) {
-      it->get_to(cfg.module_path);
-   }
-   it = j.find("venue");
-   if (it != j.end()) {
-      it->get_to(cfg.venue);
-   }
-}
-
-} // namespace bts::orderman
@@ -1,145 +0,0 @@
-#include "MarketDataGateway.h"
-#include "python_interop/plugin_manager.h"
-#include "spark/api/domain/Venue.h"
-#include "spark/api/md/MarketDataConfig.pb.h"
-#include "spark/api/md/MarketDataConfigUtils.h"
-#include "spark/core/FMT.h"
-#include "spark/core/TimeSpan.h"
-#include "spark/domain/DomainSecurityUtils.h"
-#include "spark/domain/SecurityID.h"
-#include "spark/domain_model/Venue.h"
-#include "spark/logging/logging.h"
-#include "spark/md/GeneralMarketDataListener.h"
-#include "spark/md/GeneralSecurityIndex.h"
-#include "spark/md/IMarketDataListener.h"
-#include "spark/md/IMarketDataProvider.h"
-#include "spark/md/IMarketDataSource.h"
-#include "spark/md/MDProducer.h"
-#include "spark/os/os.h"
-#include "spark/persistence/IDGenerator.h"
-#include "spark/utils/bts_fatal.h"
-#include "spark/utils/stopwatch.h"
-#include "sparkd/md/OnixsMarketDataProvider.h"
-#include "sparkd/md/PyMarketDataProvider.h"
-#include "sparkd/md/ReplayMarketDataProvider.h"
-#include "sparkd/md/ccapi/CCApiMarketDataProvider.h"
-#include "sparkd/md/databento/DatabentoMarketDataProvider.h"
-#include "sparkd/md/emumba/EmumbaMarketDataProvider.h"
-#include "sparkd/md/taifex/TaifexMarketDataProvider.h"
-
-#include <bits/stdint-uintn.h>
-#include <memory>
-#include <vector>
-
-namespace bts::md {
-
-using bts::api::md::MarketDataConfig;
-using bts::domain::DomainSecurityUtils;
-using bts::domain::SecurityID;
-using bts::persistence::IDGenerator;
-using std::shared_ptr;
-using std::string;
-using std::unique_ptr;
-using std::vector;
-
-static bool validate(const MarketDataConfig &cfg, vector<string> &errs) {
-   auto orig_n_errs = errs.size();
-   if (!cfg.deribit().empty() && !cfg.emumba_deribit().empty()) {
-      errs.emplace_back("Both 'deribit' (ccapi) and 'emumba_deribit' are "
-                        "enabled. Remove one of them from MarketDataConfig");
-   }
-   return errs.size() == orig_n_errs;
-}
-
-struct MarketDataGateway::PImpl {
-   struct MdSink {
-      GeneralSecurityIndex sec_index;
-      MDProducer md_producer;
-      shared_ptr<IMarketDataListener> listener;
-
-      MdSink(api::Venue venue, IDGenerator *sec_id_store)
-          : sec_index(), listener(std::make_shared<GeneralMarketDataListener>(
-                             sec_index, md_producer, venue, sec_id_store)) {}
-   };
-
-   MarketDataConfig config;
-   shared_ptr<IDGenerator> sec_id_store;
-   vector<unique_ptr<IMarketDataProvider>> providers;
-   vector<unique_ptr<MdSink>> sinks;
-   vector<shared_ptr<IMarketDataSource>> sources;
-
-   PImpl(
-       MarketDataConfig config,
-       shared_ptr<IDGenerator> sec_id_store,
-       shared_ptr<python_interop::PluginManager> py_plugin_man)
-       : config(std::move(config)), sec_id_store(std::move(sec_id_store)) {
-      providers.push_back(std::make_unique<CCApiMarketDataProvider>(
-          this->config, this->sec_id_store));
-      providers.push_back(
-          std::make_unique<EmumbaMarketDataProvider>(this->config));
-      providers.push_back(
-          std::make_unique<OnixsMarketDataProvider>(this->config));
-      providers.push_back(std::make_unique<DatabentoMarketDataProvider>(
-          this->config, this->sec_id_store));
-      providers.push_back(std::make_unique<TaifexMarketDataProvider>(
-          this->config, this->sec_id_store));
-      providers.push_back(std::make_unique<PyMarketDataProvider>(
-          this->config, this->sec_id_store, std::move(py_plugin_man)));
-      providers.push_back(std::make_unique<ReplayMarketDataProvider>(
-          this->config, this->sec_id_store));
-   }
-
-   void start() {
-      bts_assert(sources.empty());
-      vector<string> cfg_errs;
-      if (!validate(config, cfg_errs)) {
-         BTS_FATAL_MSGF("Invalid MarketDataConfig:", FMT_JOIN(cfg_errs, "\n"));
-         return;
-      }
-      LOG(md, Info, "MarketDataGateway::run starting t_id:{} t_name:{}",
-          os::get_thread_id(), os::get_thread_name());
-      const auto slow_start_thresh = core::TimeSpan::from_millis(100);
-      utils::Stopwatch sw;
-      for (auto &provider : providers) {
-         for (auto &[venue, source] : provider->get_sources()) {
-            sw.start();
-            sources.push_back(source);
-            auto &sink = sinks.emplace_back(std::make_unique<PImpl::MdSink>(
-                DomainSecurityUtils::convert(venue), sec_id_store.get()));
-            LOG(md, Info, "MarketDataGateway::run. Starting {} md", venue);
-            source->start(sink->listener);
-            const auto lap = sw.stop().lap().last_lap();
-            if (lap.elapsed() > slow_start_thresh) {
-               LOG(md, Warn,
-                   "MarketDataGateway::run. Took {:.2f}ms to start {} md",
-                   lap.elapsed().total_millis(), venue);
-            }
-         }
-      }
-      const core::TimeSpan total_elapsed = sw.total_elapsed();
-      LOG(md, total_elapsed > slow_start_thresh * sources.size() ? Warn : Info,
-          "MarketDataGateway::run. Took {:.2f}ms to start all {} md sources",
-          total_elapsed.total_millis(), sources.size());
-   }
-
-   void stop() {
-      for (auto &source : sources) {
-         source->stop();
-      }
-   }
-};
-
-MarketDataGateway::MarketDataGateway(
-    MarketDataConfig config,
-    shared_ptr<IDGenerator> sec_id_store,
-    std::shared_ptr<python_interop::PluginManager> py_plugin_man)
-    : me_(std::make_unique<MarketDataGateway::PImpl>(
-          std::move(config),
-          std::move(sec_id_store),
-          std::move(py_plugin_man))) {}
-
-MarketDataGateway::~MarketDataGateway() = default;
-
-void MarketDataGateway::start() { me_->start(); }
-
-} // namespace bts::md
@@ -1,38 +0,0 @@
-#pragma once
-
-#include <memory>
-
-namespace bts::api::md {
-class MarketDataConfig;
-}
-
-namespace bts::persistence {
-class IDGenerator;
-}
-
-namespace python_interop {
-class PluginManager;
-}
-
-namespace bts::md {
-
-/**
- * @brief The `MarketDataGateway` is responsible for starting all configured market data sources.
- * It connects each source to a special `IMarketDataListener` that writes the data to shared memory.
- */
-class MarketDataGateway {
- public:
-   explicit MarketDataGateway(
-       api::md::MarketDataConfig config,
-       std::shared_ptr<persistence::IDGenerator> sec_id_store,
-       std::shared_ptr<python_interop::PluginManager> py_plugin_man);
-   ~MarketDataGateway();
-
-   void start();
-
- private:
-   struct PImpl;
-   std::unique_ptr<PImpl> me_;
-};
-
-} // namespace bts::md
@@ -1,44 +0,0 @@
-#include "OnixsMarketDataProvider.h"
-#include "spark/api/domain/Venue.pb.h"
-#include "spark/persistence/IDGenerator.h"
-#include "sparkd/md/glbx/OnixsGlbxSubscription.h"
-
-#include <memory>
-#include <vector>
-
-namespace bts::md {
-
-using bts::api::domain::Venue;
-using bts::api::md::MarketDataConfig;
-using bts::md::OnixsMarketDataProvider;
-using bts::persistence::IDGenerator;
-using std::pair;
-using std::shared_ptr;
-using std::string;
-using std::vector;
-
-struct OnixsMarketDataProvider::PImpl {
-   MarketDataConfig config;
-   std::shared_ptr<IDGenerator> sec_id_store;
-   vector<shared_ptr<OnixsGlbxSubscription>> sources;
-};
-
-OnixsMarketDataProvider::OnixsMarketDataProvider(MarketDataConfig config)
-    : me_(std::make_unique<PImpl>()) {
-   me_->config = std::move(config);
-}
-
-OnixsMarketDataProvider::~OnixsMarketDataProvider() = default;
-
-vector<pair<Venue, shared_ptr<IMarketDataSource>>>
-OnixsMarketDataProvider::get_sources() {
-   vector<pair<Venue, shared_ptr<IMarketDataSource>>> rv;
-   for (const auto &glbx : me_->config.glbx()) {
-      me_->sources.push_back(
-          std::make_shared<OnixsGlbxSubscription>(me_->sec_id_store, glbx));
-      rv.push_back({Venue::VENUE_GLBX, me_->sources.back()});
-   }
-   return rv;
-}
-
-} // namespace bts::md
@@ -1,26 +0,0 @@
-#pragma once
-
-#include "spark/api/md/MarketDataConfig.pb.h"
-#include "spark/md/IMarketDataProvider.h"
-#include "spark/md/IMarketDataSource.h"
-
-#include <memory>
-
-namespace bts::md {
-
-class OnixsMarketDataProvider : public IMarketDataProvider {
- public:
-   OnixsMarketDataProvider();
-   explicit OnixsMarketDataProvider(api::md::MarketDataConfig config);
-   ~OnixsMarketDataProvider();
-
-   std::vector<
-       std::pair<bts::api::domain::Venue, std::shared_ptr<IMarketDataSource>>>
-   get_sources() override;
-
- private:
-   struct PImpl;
-   std::unique_ptr<PImpl> me_;
-};
-
-} // namespace bts::md
@@ -1,61 +0,0 @@
-#include "PyMarketDataProvider.h"
-#include "python_interop/plugin_manager.h"
-#include "spark/api/md/MarketDataConfig.pb.h"
-#include "spark/md/IMarketDataSource.h"
-#include "spark/persistence/IDGenerator.h"
-#include "spark/utils/bts_assert.h"
-#include "sparkd/md/PyMarketDataSubscription.h"
-
-namespace bts::md {
-
-using bts::api::domain::Venue;
-using bts::api::md::MarketDataConfig;
-using bts::persistence::IDGenerator;
-using std::pair;
-using std::shared_ptr;
-using std::string;
-using std::vector;
-using Path = boost::filesystem::path;
-
-struct PyMarketDataProvider::PImpl {
-   MarketDataConfig config;
-   shared_ptr<IDGenerator> sec_id_store;
-   shared_ptr<python_interop::PluginManager> py_worker;
-   vector<shared_ptr<PyMarketDataSubscription>> py_md_subs;
-
-   PImpl(
-       MarketDataConfig config,
-       shared_ptr<IDGenerator> sec_id_store,
-       shared_ptr<python_interop::PluginManager> py_worker)
-       : config(std::move(config)), sec_id_store(std::move(sec_id_store)),
-         py_worker(std::move(py_worker)) {}
-};
-
-PyMarketDataProvider::PyMarketDataProvider(
-    MarketDataConfig config,
-    shared_ptr<IDGenerator> sec_id_store,
-    shared_ptr<python_interop::PluginManager> py_worker)
-    : me_(std::make_unique<PImpl>(
-          std::move(config), std::move(sec_id_store), std::move(py_worker))) {}
-
-PyMarketDataProvider::~PyMarketDataProvider() = default;
-
-vector<pair<Venue, shared_ptr<IMarketDataSource>>>
-PyMarketDataProvider::get_sources() {
-   vector<pair<Venue, shared_ptr<IMarketDataSource>>> rv;
-   for (const auto &py_plugin : me_->config.python_md_providers()) {
-      try {
-         auto source = std::make_shared<PyMarketDataSubscription>(
-             py_plugin, me_->sec_id_store.get(), me_->py_worker);
-         me_->py_md_subs.push_back(source);
-         rv.emplace_back(py_plugin.venue(), std::move(source));
-      } catch (const std::exception &ex) {
-         bts_assert_msgf(
-             false, "PyMarketDataSubscription threw: {}",
-             std::string{ex.what()});
-      }
-   }
-   return rv;
-}
-
-} // namespace bts::md
@@ -1,33 +0,0 @@
-#pragma once
-
-#include "spark/api/domain/Venue.pb.h"
-#include "spark/api/md/MarketDataConfig.pb.h"
-#include "spark/md/IMarketDataProvider.h"
-#include "spark/persistence/IDGenerator.h"
-
-#include <memory>
-
-namespace python_interop {
-class PluginManager;
-}
-
-namespace bts::md {
-
-class PyMarketDataProvider : public IMarketDataProvider {
- public:
-   explicit PyMarketDataProvider(
-       api::md::MarketDataConfig config,
-       std::shared_ptr<persistence::IDGenerator> sec_id_store,
-       std::shared_ptr<python_interop::PluginManager> py_worker);
-   ~PyMarketDataProvider();
-
-   std::vector<
-       std::pair<api::domain::Venue, std::shared_ptr<IMarketDataSource>>>
-   get_sources() override;
-
- private:
-   struct PImpl;
-   std::unique_ptr<PImpl> me_;
-};
-
-} // namespace bts::md
@@ -1,420 +0,0 @@
-#include "PyMarketDataSubscription.h"
-#include "python_interop/md/md_bridge_impl.h"
-#include "python_interop/md/py_book.h"
-#include "python_interop/md/py_trade.h"
-#include "python_interop/plugin_manager.h"
-#include "python_interop/py_contract_specification.h"
-#include "python_interop/py_security.h"
-#include "python_interop/python_object.h"
-#include "spark/api/domain/CallPut.pb.h"
-#include "spark/api/domain/SecurityType.h"
-#include "spark/api/domain/Venue.h"
-#include "spark/api/md/PyMarketDataProviderConfig.pb.h"
-#include "spark/core/TimeSpan.h"
-#include "spark/core/TimeStamp.h"
-#include "spark/domain/AggressorSide.h"
-#include "spark/domain/FutureSecurity.h"
-#include "spark/domain/Security.h"
-#include "spark/domain/SecurityID.h"
-#include "spark/domain/SpotSecurity.h"
-#include "spark/domain/TradeRecord.h"
-#include "spark/domain_model/SecurityType.h"
-#include "spark/io/json_formatter.h"
-#include "spark/logging/logging.h"
-#include "spark/logging/trace_log.h"
-#include "spark/md/IMarketDataListener.h"
-#include "spark/os/os.h"
-#include "spark/persistence/IDGenerator.h"
-#include "spark/utils/bts_assert.h"
-#include "spark/utils/stopwatch.h"
-
-#include <future>
-#include <memory>
-#include <optional>
-#include <pybind11/embed.h>
-#include <pybind11/pybind11.h>
-#include <utility>
-#include <vector>
-
-namespace bts::md {
-
-#define _LOG(level, ...) LOG(md, level, __VA_ARGS__)
-#define INFO(...) _LOG(bts::Info, __VA_ARGS__)
-#define ERROR(...) _LOG(bts::Error, __VA_ARGS__)
-
-#define _LOG_TRACE(level, name)                                                \
-   LOG_TRACE(md, level, std::string("py_md_sub"), std::string(name))
-#define INFO_TRACE(name) _LOG_TRACE(bts::Info, name)
-#define WARN_TRACE(name) _LOG_TRACE(bts::Warn, name)
-#define ERROR_TRACE(name) _LOG_TRACE(bts::Error, name)
-
-#define INFO_TRACE_PARAMS(name, params)                                        \
-   LOG_TRACE_PARAMS(                                                           \
-       md, bts::Info, std::string("py_md_sub"), std::string(name), params)
-
-#define LOG_DISPATCH_STATS(level, typ, n, dur)                                 \
-   _LOG(                                                                       \
-       level,                                                                  \
-       R"([py_md_dispatch.json]{{ "typ": ")" typ                               \
-       R"(", "n_recs": {}, "dur": "{:.2f}ms", "ts": {} }})",                   \
-       n, dur.total_millis(), bts::core::TimeStamp::now_ns())
-
-#define LOG_SECURITY(level, py_sec)                                            \
-   LOG(md, level, R"([py_md_sec.json]{{ "sec": {}, "ts": {}}})",               \
-       json_formatter(py_sec), bts::core::TimeStamp::now_ns())
-
-#define LOG_BOOK(level, py_book)                                               \
-   LOG(md, level, R"([py_md_book.json]{{ "bk": {}, "ts": {}}})",               \
-       json_formatter(py_book), bts::core::TimeStamp::now_ns())
-
-#define LOG_TRADE(level, py_trade)                                             \
-   LOG(md, level, R"([py_md_trade.json]{{ "trd": {}, "ts": {}}})",             \
-       json_formatter(py_trade), bts::core::TimeStamp::now_ns())
-
-using bts::api::domain::CallPut;
-using bts::api::domain::SecurityType;
-using bts::api::domain::Venue;
-using bts::api::md::PyMarketDataProviderConfig;
-using bts::core::TimeSpan;
-using bts::core::TimeStamp;
-using bts::domain::AggressorSide;
-using bts::domain::Book;
-using bts::domain::FutureSecurity;
-using bts::domain::OptionSecurity;
-using bts::domain::Price;
-using bts::domain::Security;
-using bts::domain::SecurityID;
-using bts::domain::SpotSecurity;
-using bts::domain::TradeRecord;
-using bts::io::json_formatter;
-using bts::md::py_interop::PyBook;
-using bts::md::py_interop::PyBookSide;
-using bts::md::py_interop::PySecurity;
-using bts::md::py_interop::PyTrade;
-using bts::persistence::IDGenerator;
-using std::optional;
-using std::shared_ptr;
-using std::size_t;
-using std::string;
-using std::thread;
-using std::vector;
-
-using Path = boost::filesystem::path;
-
-struct TradeTuple {
-   PyTrade py_trade;
-   TradeRecord trade;
-   Venue venue;
-};
-
-struct PyMarketDataSubscription::PImpl {
-   PImpl(
-       PyMarketDataProviderConfig config,
-       IDGenerator *sec_id_store,
-       shared_ptr<python_interop::PluginManager> py_plugin_man)
-       : config(std::move(config)),
-         name(fmt::format("{}_md_pro", string{this->config.provider_type()})),
-         bridge_ptr(std::make_shared<py_interop::MdBridgeImpl>()),
-         bridge(*bridge_ptr), py_plugin_man(std::move(py_plugin_man)),
-         sec_id_store_(sec_id_store) {}
-
-   size_t dispatch_securities() {
-      // `securities_` and `security_ptrs_` should already be empty, but no harm in clearing them
-      // again. We keep them as member variables to avoid (re)allocating memory in each callback.
-      securities_.clear();
-      security_ptrs_.clear();
-      std::shared_ptr<const PySecurity> py_sec_p;
-      while (bridge.pop_security(py_sec_p)) {
-         const auto &py_sec = *py_sec_p;
-         LOG_SECURITY(Info, py_sec);
-         auto sec = to_bts_security_(py_sec);
-         if (!sec) {
-            LOG(md, Error,
-                "Failed to convert security {} to bts::domain::Security",
-                py_sec.name());
-            LOG(md, Error,
-                R"([py_md_sec_parse_error.json]{{ "sec": {}, "ts": {}}})",
-                py_sec);
-            continue;
-         }
-         securities_.push_back(std::move(*sec));
-      }
-      const size_t n = securities_.size();
-      if (!securities_.empty()) {
-         // Populate vectors of security ptrs per each venue.
-         for (auto &sec : securities_) {
-            security_ptrs_.push_back(&sec);
-         }
-         // Pass securities to the sinks (per venue).
-         listener_->on_securities(security_ptrs_);
-         securities_.clear();
-         security_ptrs_.clear();
-      }
-      return n;
-   }
-
-   size_t dispatch_books() {
-      size_t n = 0;
-      std::shared_ptr<const PyBook> py_book_p;
-      while (bridge.pop_book(py_book_p)) {
-         const auto &py_book = *py_book_p;
-         LOG_BOOK(Diag, py_book);
-         bts_assert(book_.count_populated() == 0);
-         to_book_(py_book, book_);
-         listener_->on_book(book_);
-         book_.clear();
-         n++;
-      }
-      return n;
-   }
-
-   size_t dispatch_trades() {
-      // `trades_` and `trade_ptrs_` should already be empty, but no harm in clearing them again.
-      // We keep them as member variables to avoid (re)allocating memory in each callback.
-      trades_.clear();
-      trade_ptrs_.clear();
-      std::shared_ptr<const PyTrade> py_trade_p;
-      while (bridge.pop_trade(py_trade_p)) {
-         const auto &py_trade = *py_trade_p;
-         LOG_TRADE(Diag, py_trade);
-         to_bts_trade_(py_trade, trades_.emplace_back());
-      }
-      const size_t n = trades_.size();
-      for (const auto &t : trades_) {
-         trade_ptrs_.push_back(&t);
-      }
-      listener_->on_trades(trade_ptrs_);
-      trades_.clear();
-      trade_ptrs_.clear();
-      return n;
-   }
-
-   void start(shared_ptr<IMarketDataListener> listener) {
-      bts_assert(listener != nullptr);
-      this->listener_ = std::move(listener);
-      LOG(md, Info, "PyMarketDataSubscription::start t_id:{} t_name:{}",
-          os::get_thread_id(), os::get_thread_name());
-      py_keep_going = true;
-      py_thread.emplace([&]() {
-         INFO_TRACE_PARAMS(
-             "run_py", json_formatter(std::map<string, string>{
-                           {"name", name}, {"ev", "STARTING_PY_MD_PRO"}}));
-         utils::Stopwatch sw = utils::Stopwatch::start_new();
-         std::future<std::shared_ptr<
-             python_interop::PythonObjectT<bts::md::py_interop::MdProvider>>>
-             md_pro_fut = py_plugin_man->start_md_provider(
-                 name, config.provider_type(), bridge_ptr);
-         md_pro_fut.wait();
-         INFO_TRACE_PARAMS(
-             "run_py",
-             json_formatter(std::map<string, string>{
-                 {"name", name},
-                 {"ev", "STARTED_PY_MD_PRO"},
-                 {"dur",
-                  fmt::format("{:.3f}ms", sw.elapsed().total_millis())}}));
-      });
-      cpp_keep_going = true;
-      cpp_thread.emplace([&]() {
-         static const TimeSpan sec_warn_thresh = TimeSpan::from_millis(100);
-         static const TimeSpan bk_warn_thresh = TimeSpan::from_millis(30);
-         static const TimeSpan trd_warn_thresh = TimeSpan::from_millis(30);
-         utils::Stopwatch sw;
-         TimeSpan dur;
-         size_t n;
-         while (cpp_keep_going) {
-            // Dispatch securities
-            sw.restart();
-            n = dispatch_securities();
-            dur = sw.elapsed();
-            if (n > 0) {
-               LOG_DISPATCH_STATS(
-                   dur > sec_warn_thresh ? Warn : Info, "sec", n, dur);
-            }
-            // Dispatch books
-            sw.restart();
-            n = dispatch_books();
-            dur = sw.elapsed();
-            if (n > 0) {
-               LOG_DISPATCH_STATS(
-                   dur > bk_warn_thresh ? Warn : Diag, "bk", n, dur);
-            }
-            // Dispatch trades
-            sw.restart();
-            n = dispatch_trades();
-            dur = sw.elapsed();
-            if (n > 0) {
-               LOG_DISPATCH_STATS(
-                   dur > trd_warn_thresh ? Warn : Diag, "trd", n, dur);
-            }
-         }
-      });
-   }
-
-   void stop() {
-      py_keep_going = false;
-      if (py_thread) {
-         py_thread->join();
-         py_thread.reset();
-      }
-      cpp_keep_going = false;
-      if (cpp_thread) {
-         cpp_thread->join();
-         cpp_thread.reset();
-      }
-   }
-
- private:
-   PyMarketDataProviderConfig config;
-   std::string name;
-   std::shared_ptr<py_interop::MdBridgeImpl> bridge_ptr;
-   py_interop::MdBridgeImpl &bridge;
-   optional<thread> py_thread;
-   std::shared_ptr<python_interop::PluginManager> py_plugin_man;
-   volatile bool py_keep_going = false;
-   optional<thread> cpp_thread;
-   volatile bool cpp_keep_going = false;
-   shared_ptr<IMarketDataListener> listener_;
-
-   shared_ptr<IDGenerator> sec_id_store_;
-   // The main store for securities.
-   vector<Security> securities_;
-   vector<const Security *> security_ptrs_;
-   Book book_;
-   // Since the python md provider isn't tethered to a single venue, we
-   // must know the venue for each individual trade.
-   vector<TradeRecord> trades_;
-   vector<const TradeRecord *> trade_ptrs_;
-
-   SecurityID get_sec_id_(
-       Venue venue, SecurityType sec_type, const string &sec_name) {
-      auto sec_id = sec_id_store_->get_id(venue, sec_type, sec_name, true);
-      bts_assert_msgf(
-          sec_id,
-          "Failed to get security id for py_book. venue: {}, type: {}, name: "
-          "{}",
-          venue, sec_type, sec_name);
-      return SecurityID{*sec_id};
-   }
-
-   optional<Security> to_bts_security_(const PySecurity &py_sec) {
-      // We can't let python provider assign the id, because it is unaware of other
-      // md providers, so can't possibly guarentee a unique id.
-      bts_assert(py_sec.id() == 0);
-      Security sec;
-      auto &sec_comm = sec.security_common;
-      if (!venue_parse_smart(py_sec.venue(), &sec_comm.venue)) {
-         LOG(md, Error, "Failed to parse venue from '{}'", py_sec.venue());
-         return std::nullopt;
-      }
-      sec_comm.exchange_sec_id = py_sec.exchange_id();
-      sec_comm.currency = py_sec.currency();
-      sec_comm.asset = py_sec.asset();
-      sec_comm.group = py_sec.group();
-      sec_comm.symbol = py_sec.asset();
-      if (sec_comm.symbol.empty()) {
-         sec_comm.symbol = py_sec.group();
-      }
-      const py_interop::PyContractSpecification &spec = py_sec.contract_spec();
-      if (spec.price_infos().size() > 0) {
-         bts_assert(spec.price_infos().size() == 1);
-         const py_interop::PyContractPriceInfo &price_info =
-             spec.price_infos()[0];
-         bts_assert(price_info.tick_unit() > 0 && price_info.tick_base() > 0);
-         sec_comm.min_price_increment =
-             price_info.tick_unit() / price_info.tick_base();
-         if (price_info.cash_per_point() > 0) {
-            sec_comm.cash_per_point = price_info.cash_per_point();
-         }
-      }
-      if (spec.quantity_infos().size() > 0) {
-         bts_assert(spec.quantity_infos().size() == 1);
-         const py_interop::PyContractQuantityInfo &qty_info =
-             spec.quantity_infos()[0];
-         bts_assert(qty_info.tick_unit() > 0 && qty_info.tick_base() > 0);
-         sec_comm.min_quantity_increment =
-             qty_info.tick_unit() / qty_info.tick_base();
-         sec_comm.min_quantity =
-             std::max(qty_info.min_value(), sec_comm.min_quantity_increment);
-      }
-      sec_comm.desc = py_sec.name();
-      if (py_sec.is_spot()) {
-         sec.security = SpotSecurity{};
-      } else if (py_sec.is_future()) {
-         FutureSecurity fut;
-         fut.expiration =
-             TimeStamp::from_iso_string(py_sec.future().expiration());
-         sec.security = std::move(fut);
-      } else if (py_sec.is_option()) {
-         OptionSecurity opt;
-         const auto &py_opt = py_sec.option();
-         opt.expiration = TimeStamp::from_iso_string(py_opt.expiration());
-         opt.call_put = py_opt.is_call() ? CallPut::CALL : CallPut::PUT;
-         opt.strike = Price::from_double(py_opt.strike());
-         sec.security = std::move(opt);
-      } else {
-         LOG(md, Error,
-             "PySecurity has incomplete definition (not spot, future, or "
-             "option)");
-         return std::nullopt;
-      }
-      sec_comm.id =
-          get_sec_id_(sec_comm.venue, sec.type(), sec_comm.exchange_sec_id);
-      return sec;
-   }
-
-   void to_bts_trade_(const PyTrade &source, domain::TradeRecord &dest) {
-      Venue venue;
-      bts_assert(venue_parse_smart(source.venue(), &venue));
-      SecurityType sec_type;
-      bts_assert(security_type_parse_smart(source.security_type(), &sec_type));
-      dest.sec_id = get_sec_id_(venue, sec_type, source.exchange_security_id());
-      dest.time_stamp = core::TimeStamp::from_nanos(source.timestamp());
-      const string &agg_str = source.aggressor_side();
-      dest.agg_side = agg_str.starts_with('B') || agg_str.starts_with('b')
-                          ? AggressorSide::Buy
-                          : AggressorSide::Sell;
-      dest.trade_price = Price::from_double(source.price());
-      *dest.trade_quantity = source.quantity();
-   }
-
-   void to_book_side_(const PyBookSide &source, domain::BookSide &dest) {
-      for (int i = 0; i < source.size(); i++) {
-         auto py_level = source.get_level(i);
-         const auto price = py_level.price();
-         const auto quantity = py_level.quantity();
-         bts_assert(!std::isnan(price));
-         bts_assert(quantity > 0);
-         *dest[i].total_quantity = quantity;
-         *dest[i].direct_quantity = quantity;
-         dest[i].price = domain::Price::from_double(price);
-      }
-   }
-
-   void to_book_(const PyBook &source, domain::Book &dest) {
-      to_book_side_(source.bids(), dest.bids());
-      to_book_side_(source.asks(), dest.asks());
-      Venue venue;
-      bts_assert(venue_parse_smart(source.venue(), &venue));
-      SecurityType type;
-      bts_assert(security_type_parse_smart(source.security_type(), &type));
-      dest.set_security_id(
-          get_sec_id_(venue, type, source.exchange_security_id()));
-   }
-};
-
-PyMarketDataSubscription::PyMarketDataSubscription(
-    PyMarketDataProviderConfig config,
-    IDGenerator *sec_id_store,
-    std::shared_ptr<python_interop::PluginManager> py_plugin_man)
-    : me(std::make_unique<PyMarketDataSubscription::PImpl>(
-          std::move(config), sec_id_store, std::move(py_plugin_man))) {}
-
-PyMarketDataSubscription::~PyMarketDataSubscription() { me->stop(); }
-
-void PyMarketDataSubscription::start(shared_ptr<IMarketDataListener> listener) {
-   me->start(std::move(listener));
-}
-
-void PyMarketDataSubscription::stop() { me->stop(); }
-
-} // namespace bts::md
@@ -1,34 +0,0 @@
-#pragma once
-
-#include "python_interop/plugin_manager.h"
-#include "spark/md/IMarketDataSource.h"
-
-#include <memory>
-
-namespace bts::api::md {
-class PyMarketDataProviderConfig;
-}
-
-namespace bts::persistence {
-class IDGenerator;
-}
-
-namespace bts::md {
-
-class PyMarketDataSubscription : public IMarketDataSource {
- public:
-   explicit PyMarketDataSubscription(
-       api::md::PyMarketDataProviderConfig config,
-       persistence::IDGenerator *sec_id_store,
-      std::shared_ptr<python_interop::PluginManager> py_worker);
-   ~PyMarketDataSubscription();
-
-   void start(std::shared_ptr<IMarketDataListener> listener) override;
-   void stop() override;
-
- private:
-   struct PImpl;
-   std::unique_ptr<PImpl> me;
-};
-
-} // namespace bts::md
@@ -1,52 +0,0 @@
-#include "ReplayMarketDataProvider.h"
-#include "spark/api/md/MarketDataConfig.pb.h"
-#include "spark/api/md/MarketDataConfigUtils.h"
-#include "spark/config/config.h"
-#include "spark/domain/DomainSecurityUtils.h"
-#include "spark/md/IMarketDataSource.h"
-#include "spark/persistence/IDGenerator.h"
-#include "sparkd/md/recording/MDReplayer.h"
-
-namespace bts::md {
-
-using bts::api::md::MarketDataConfig;
-using bts::domain::DomainSecurityUtils;
-using bts::md::recording::MDReplayer;
-using bts::persistence::IDGenerator;
-using std::pair;
-using std::shared_ptr;
-using std::string;
-using std::vector;
-
-struct ReplayMarketDataProvider::PImpl {
-   MarketDataConfig config;
-   shared_ptr<IDGenerator> sec_id_store;
-   vector<pair<api::domain::Venue, shared_ptr<IMarketDataSource>>> sources;
-
-   PImpl(MarketDataConfig config, shared_ptr<IDGenerator> sec_id_store)
-       : config(std::move(config)), sec_id_store(std::move(sec_id_store)) {}
-};
-
-ReplayMarketDataProvider::ReplayMarketDataProvider()
-    : ReplayMarketDataProvider(
-          config::get<MarketDataConfig>(), IDGenerator::instance_ptr()) {}
-
-ReplayMarketDataProvider::ReplayMarketDataProvider(
-    MarketDataConfig config, shared_ptr<IDGenerator> sec_id_store)
-    : me_(std::make_unique<PImpl>(std::move(config), std::move(sec_id_store))) {
-}
-
-ReplayMarketDataProvider::~ReplayMarketDataProvider() = default;
-
-vector<pair<api::domain::Venue, shared_ptr<IMarketDataSource>>>
-ReplayMarketDataProvider::get_sources() {
-   const auto replay_cfgs =
-       MarketDataConfigUtils::get_effective_replay_config(me_->config);
-   for (const auto &r : replay_cfgs) {
-      const auto v = DomainSecurityUtils::convert(r.venue());
-      me_->sources.push_back({v, std::make_shared<MDReplayer>(r.venue())});
-   }
-   return me_->sources;
-}
-
-} // namespace bts::md
@@ -1,28 +0,0 @@
-#pragma once
-
-#include "spark/api/md/MarketDataConfig.pb.h"
-#include "spark/md/IMarketDataProvider.h"
-#include "spark/persistence/IDGenerator.h"
-
-#include <memory>
-
-namespace bts::md {
-
-class ReplayMarketDataProvider : public IMarketDataProvider {
- public:
-   ReplayMarketDataProvider();
-   explicit ReplayMarketDataProvider(
-       api::md::MarketDataConfig config,
-       std::shared_ptr<persistence::IDGenerator> sec_id_store);
-   ~ReplayMarketDataProvider();
-
-   std::vector<
-       std::pair<bts::api::domain::Venue, std::shared_ptr<IMarketDataSource>>>
-   get_sources() override;
-
- private:
-   struct PImpl;
-   std::unique_ptr<PImpl> me_;
-};
-
-} // namespace bts::md
@@ -1,47 +0,0 @@
-#include "ReplayMarketDataSubscription.h"
-#include "spark/api/md/MarketDataReplayConfig.pb.h"
-#include "spark/logging/logging.h"
-#include "spark/md/GeneralMarketDataListener.h"
-#include "spark/md/IMarketDataListener.h"
-#include "spark/md/MDProducer.h"
-#include "spark/utils/thread_manager.h"
-#include "sparkd/md/recording/MDReplayer.h"
-
-#include <ccapi_cpp/ccapi_event_dispatcher.h>
-#include <optional>
-
-namespace bts::md {
-
-using bts::api::md::MarketDataReplayConfig;
-using bts::md::recording::MDReplayer;
-using std::string;
-
-struct ReplayMarketDataSource::PImpl {
-   MarketDataReplayConfig config;
-
-   PImpl(const MarketDataReplayConfig &config) : config(config) {}
-};
-
-ReplayMarketDataSource::ReplayMarketDataSource(MarketDataReplayConfig config)
-    : me_(std::make_unique<PImpl>(config)) {
-   me_->config = std::move(config);
-}
-
-ReplayMarketDataSource::~ReplayMarketDataSource() = default;
-
-void ReplayMarketDataSource::start(
-    std::shared_ptr<IMarketDataListener> listener) {
-   LOG(md, Info, "MarketDataGateway::_start_replays starting t_id:{} t_name:{}",
-       os::get_thread_id(), os::get_thread_name());
-   auto md_replayer = std::make_shared<MDReplayer>(me_->config.venue());
-   std::string tid_name = "MDReplayer_" + Venue_Name(me_->config.venue());
-   bts::threading::make_thread(tid_name, [&, listener, md_replayer]() {
-      md_replayer->start(listener);
-   }).detach();
-   LOG(md, Info, "MarketDataGateway::_start_replays ending t_id:{} t_name:{}",
-       os::get_thread_id(), os::get_thread_name());
-}
-
-void ReplayMarketDataSource::stop() {}
-
-} // namespace bts::md
@@ -1,25 +0,0 @@
-#pragma once
-
-#include "spark/api/md/MarketDataReplayConfig.pb.h"
-#include "spark/md/IMarketDataListener.h"
-#include "spark/md/IMarketDataSource.h"
-
-#include <memory>
-
-namespace bts::md {
-
-class ReplayMarketDataSource : public IMarketDataSource {
- public:
-   ReplayMarketDataSource();
-   explicit ReplayMarketDataSource(api::md::MarketDataReplayConfig config);
-   ~ReplayMarketDataSource();
-
-   void start(std::shared_ptr<IMarketDataListener> listener) override;
-   void stop() override;
-
- private:
-   struct PImpl;
-   std::unique_ptr<PImpl> me_;
-};
-
-} // namespace bts::md
@@ -1,139 +0,0 @@
-#include "CCApiMarketDataProvider.h"
-#include "spark/api/domain/Venue.pb.h"
-#include "spark/api/md/MarketDataConfig.pb.h"
-#include "spark/config/config.h"
-#include "spark/io/ProtoPrinter.h"
-#include "spark/logging/logging.h"
-#include "spark/md/GeneralMarketDataListener.h"
-#include "spark/md/GeneralSecurityIndex.h"
-#include "spark/md/MDProducer.h"
-#include "spark/persistence/IDGenerator.h"
-#include "sparkd/md/binance/BinanceSubscription.h"
-#include "sparkd/md/binance_us/BinanceUsSubscription.h"
-#include "sparkd/md/binance_usds_futures/BinanceUsdsFuturesSubscription.h"
-#include "sparkd/md/coinbase/CoinbaseSubscription.h"
-#include "sparkd/md/deribit/DeribitSubscription.h"
-#include "sparkd/md/ftx/FTXSubscription.h"
-#include "sparkd/md/okx/OkxSubscription.h"
-
-#include <ccapi_cpp/ccapi_event_dispatcher.h>
-
-namespace bts::md {
-
-using bts::api::md::MarketDataConfig;
-using bts::persistence::IDGenerator;
-using std::pair;
-using std::shared_ptr;
-using std::string;
-using std::unique_ptr;
-using std::unordered_map;
-using std::vector;
-
-static bool validate(
-    const MarketDataConfig & /*cfg*/, vector<string> & /*errs*/) {
-   return true;
-}
-
-struct CCApiMarketDataProvider::PImpl {
-   struct VenueSink {
-      GeneralSecurityIndex sec_index;
-      MDProducer md_producer;
-      GeneralMarketDataListener listener;
-
-      VenueSink(api::Venue venue, shared_ptr<IDGenerator> sec_id_store)
-          : sec_index(),
-            listener(sec_index, md_producer, venue, sec_id_store.get()) {}
-   };
-
-   MarketDataConfig config;
-   ccapi::EventDispatcher dispatcher;
-   shared_ptr<IDGenerator> sec_id_store;
-   unordered_map<api::Venue, unique_ptr<VenueSink>> venue_sinks;
-   vector<std::pair<api::domain::Venue, std::shared_ptr<IMarketDataSource>>>
-       sources;
-
-   PImpl(MarketDataConfig config, shared_ptr<IDGenerator> sec_id_store)
-       : config(std::move(config)), sec_id_store(std::move(sec_id_store)) {}
-
-   VenueSink *get_venue_sink(api::Venue venue) {
-      auto it = venue_sinks.find(venue);
-      if (it != venue_sinks.end()) {
-         return it->second.get();
-      }
-      return venue_sinks
-          .try_emplace(venue, std::make_unique<VenueSink>(venue, sec_id_store))
-          .first->second.get();
-   }
-};
-
-CCApiMarketDataProvider::CCApiMarketDataProvider(
-    MarketDataConfig config, shared_ptr<IDGenerator> sec_id_store)
-    : me_(std::make_unique<PImpl>(std::move(config), std::move(sec_id_store))) {
-}
-
-CCApiMarketDataProvider::~CCApiMarketDataProvider() = default;
-
-vector<pair<api::domain::Venue, shared_ptr<IMarketDataSource>>>
-CCApiMarketDataProvider::get_sources() {
-   auto &cfg = me_->config;
-   vector<string> cfg_errs;
-   if (!validate(cfg, cfg_errs)) {
-      LOG(md, Error,
-          "CCApiMarketDataProvider::get_sources failed. Invalid config: {}",
-          fmt::to_string(fmt::join(cfg_errs, "\n")));
-   }
-
-   // Binance
-   for (const auto &binance : cfg.binance()) {
-      me_->sources.emplace_back(
-          api::domain::VENUE_BINANCE,
-          std::make_shared<binance::BinanceSubscription>(
-              &me_->dispatcher, binance, me_->sec_id_store.get()));
-   }
-   // Binance US
-   for (const auto &binance_us : cfg.binance_us()) {
-      me_->sources.emplace_back(
-          api::domain::VENUE_BINANCE_US,
-          std::make_shared<binance_us::BinanceUsSubscription>(
-              &me_->dispatcher, binance_us, me_->sec_id_store.get()));
-   }
-   // Binance USDS Futures
-   for (const auto &binance_ufs : cfg.binance_usds_futures()) {
-      me_->sources.emplace_back(
-          api::domain::VENUE_BINANCE_USDS_FUTURES,
-          std::make_shared<
-              binance_usds_futures::BinanceUsdsFuturesSubscription>(
-              &me_->dispatcher, binance_ufs, me_->sec_id_store.get()));
-   }
-   // Coinbase
-   for (const auto &coinbase : cfg.coinbase()) {
-      me_->sources.emplace_back(
-          api::domain::VENUE_COINBASE,
-          std::make_shared<coinbase::CoinbaseSubscription>(
-              &me_->dispatcher, coinbase, me_->sec_id_store.get()));
-   }
-   // Deribit
-   for (const auto &deribit : cfg.deribit()) {
-      me_->sources.emplace_back(
-          api::domain::VENUE_DERIBIT,
-          std::make_shared<deribit::DeribitSubscription>(
-              &me_->dispatcher, deribit, me_->sec_id_store.get()));
-   }
-   // FTX
-   for (const auto &ftx : cfg.ftx()) {
-      me_->sources.emplace_back(
-          api::domain::VENUE_FTX,
-          std::make_shared<ftx::FTXSubscription>(
-              &me_->dispatcher, ftx, me_->sec_id_store.get()));
-   }
-   // Okx
-   for (const auto &okx : cfg.okx()) {
-      me_->sources.emplace_back(
-          api::domain::VENUE_OKX,
-          std::make_shared<okx::OkxSubscription>(
-              &me_->dispatcher, okx, me_->sec_id_store.get()));
-   }
-   return me_->sources;
-}
-
-} // namespace bts::md
@@ -1,27 +0,0 @@
-#pragma once
-
-#include "spark/api/md/MarketDataConfig.pb.h"
-#include "spark/md/IMarketDataProvider.h"
-#include "spark/persistence/IDGenerator.h"
-
-#include <memory>
-
-namespace bts::md {
-
-class CCApiMarketDataProvider : public IMarketDataProvider {
- public:
-   explicit CCApiMarketDataProvider(
-       api::md::MarketDataConfig config,
-       std::shared_ptr<persistence::IDGenerator> sec_id_store);
-   ~CCApiMarketDataProvider();
-
-   std::vector<
-       std::pair<api::domain::Venue, std::shared_ptr<IMarketDataSource>>>
-   get_sources() override;
-
- private:
-   struct PImpl;
-   std::unique_ptr<PImpl> me_;
-};
-
-} // namespace bts::md
@@ -1,52 +0,0 @@
-#include "DatabentoMarketDataProvider.h"
-#include "spark/api/domain/Venue.pb.h"
-#include "spark/api/md/MarketDataConfig.pb.h"
-#include "spark/md/IMarketDataSource.h"
-#include "spark/persistence/IDGenerator.h"
-#include "sparkd/md/databento/DatabentoDataHandler.h"
-#include "sparkd/md/glbx/DatabentoGlbxSubscription.h"
-
-#include <memory>
-#include <vector>
-
-namespace bts::md {
-
-using bts::api::md::MarketDataConfig;
-using bts::persistence::IDGenerator;
-using std::pair;
-using std::shared_ptr;
-using std::string;
-using std::vector;
-
-struct DatabentoMarketDataProvider::PImpl {
-   MarketDataConfig config;
-   shared_ptr<IDGenerator> sec_id_store;
-   vector<shared_ptr<IMarketDataSource>> glbx_sources;
-
-   PImpl(MarketDataConfig config, shared_ptr<IDGenerator> sec_id_store)
-       : config(std::move(config)), sec_id_store(std::move(sec_id_store)) {}
-};
-
-DatabentoMarketDataProvider::DatabentoMarketDataProvider(
-    MarketDataConfig config, shared_ptr<IDGenerator> sec_id_store)
-    : me_(std::make_unique<PImpl>(std::move(config), std::move(sec_id_store))) {
-   me_->config = std::move(config);
-}
-
-DatabentoMarketDataProvider::~DatabentoMarketDataProvider() = default;
-
-vector<pair<api::domain::Venue, shared_ptr<IMarketDataSource>>>
-DatabentoMarketDataProvider::get_sources() {
-   vector<pair<api::domain::Venue, shared_ptr<IMarketDataSource>>> rv;
-   for (const auto &glbx : me_->config.glbx()) {
-      if (glbx.use_databento()) {
-         me_->glbx_sources.push_back(
-             std::make_shared<DatabentoGlbxSubscription>(
-                 glbx, me_->sec_id_store));
-         rv.push_back({api::domain::VENUE_GLBX, me_->glbx_sources.back()});
-      }
-   }
-   return rv;
-}
-
-} // namespace bts::md
@@ -1,33 +0,0 @@
-#pragma once
-
-#include "spark/md/IMarketDataProvider.h"
-
-#include <memory>
-
-namespace bts::persistence {
-class IDGenerator;
-}
-
-namespace bts::api::md {
-class MarketDataConfig;
-}
-
-namespace bts::md {
-
-class DatabentoMarketDataProvider : public IMarketDataProvider {
- public:
-   explicit DatabentoMarketDataProvider(
-       api::md::MarketDataConfig config,
-       std::shared_ptr<persistence::IDGenerator> sec_id_store);
-   ~DatabentoMarketDataProvider();
-
-   std::vector<
-       std::pair<api::domain::Venue, std::shared_ptr<IMarketDataSource>>>
-   get_sources() override;
-
- private:
-   struct PImpl;
-   std::unique_ptr<PImpl> me_;
-};
-
-} // namespace bts::md
@@ -1,78 +0,0 @@
-#include "EmumbaDeribitSubscription.h"
-#include "spark/api/md/EmumbaDeribitMarketDataConfig.pb.h"
-#include "spark/config/protos/system_config.pb.h"
-#include "spark/utils/thread_manager.h"
-#include "sparkd/md/emumba/EmumbaDeribitData.h"
-
-namespace bts::md {
-
-using bts::config::SystemConfig;
-
-using Path = boost::filesystem::path;
-
-struct EmumbaDeribitSubscription::PImpl {
-   api::md::EmumbaDeribitMarketDataConfig config;
-};
-
-EmumbaDeribitSubscription::EmumbaDeribitSubscription(
-    api::md::EmumbaDeribitMarketDataConfig config)
-    : me_(std::make_unique<PImpl>()) {
-   me_->config = std::move(config);
-}
-
-EmumbaDeribitSubscription::~EmumbaDeribitSubscription() = default;
-
-void EmumbaDeribitSubscription::start(
-    std::shared_ptr<IMarketDataListener> /*listener*/) {
-   auto &cfg = me_->config;
-   LOG(md, Info, "EmumbaMarketDataProvider::start t_id:{} t_name:{}. cfg:\n{}",
-       os::get_thread_id(), os::get_thread_name(), cfg.DebugString());
-   auto start_deribit_thr = [=]() {
-      const auto config_path = Path{config::get<SystemConfig>().config_path()} /
-                               cfg.json_conf_file_name();
-      const auto conf =
-          bts::cmds::DeribitDemoConfiguration::read_file(config_path.string());
-      bts_assert_msg(conf.has_value(), "Failed to parse deribit config");
-      // log
-      for (const auto &s : conf.value().symbol_config_list) {
-         LOG(md, Info, "sub.symbol: {}", s.symbol_name);
-         LOG(md, Info, "sub.futs: {} sub.options: {}", s.interested_in_future,
-             s.interested_in_option);
-         for (const auto &i : s.interested_instruments) {
-            LOG(md, Info, "sub.instrument: {}", i);
-         }
-      }
-      EmumbaDeribitData ddata_obj;
-      EmumbaDeribitData *ddata = &ddata_obj;
-      // end log
-      bts_assert(conf.has_value());
-      LOG(md, Info, "Deribit conf: {}", conf.value());
-      ddata->real_handler =
-          std::make_shared<emumba::deribit_connector::deribit_handler>(
-              conf.value().symbol_config_list, conf.value().trading_config,
-              conf.value().credential_map, conf.value().logging_configuration,
-              conf.value().deribit_url, &ddata->listener);
-      // since the trading interface is housed inside the EmMDListener, we need to
-      // create and send emumba's trading_session_interface
-      for (const auto &p : conf.value().credential_map) {
-         const auto &sess = p.second;
-         auto ts_ptr = std::make_shared<
-             emumba::deribit_connector::trading_session_interface>(
-             ddata->real_handler->create_new_trading_session(sess));
-         ddata->trading_session_vec.push_back({sess, ts_ptr});
-      }
-      bts_assert(!ddata->trading_session_vec.empty());
-      ddata->listener.set_handler(
-          ddata->real_handler.get(), ddata->trading_session_vec);
-      ddata->real_handler->run();
-   };
-   threading::make_thread("Emumba", [&]() { start_deribit_thr(); }).detach();
-   LOG(md, Info, "MarketDataGateway::_start_deribit ending t_id:{} t_name:{}",
-       os::get_thread_id(), os::get_thread_name());
-}
-
-void EmumbaDeribitSubscription::stop() {
-   //
-}
-
-} // namespace bts::md
@@ -1,24 +0,0 @@
-#pragma once
-
-#include "spark/api/md/EmumbaDeribitMarketDataConfig.pb.h"
-#include "spark/md/IMarketDataSource.h"
-
-#include <memory>
-
-namespace bts::md {
-
-class EmumbaDeribitSubscription : public IMarketDataSource {
- public:
-   explicit EmumbaDeribitSubscription(
-       api::md::EmumbaDeribitMarketDataConfig config);
-   ~EmumbaDeribitSubscription() override;
-
-   void start(std::shared_ptr<IMarketDataListener> listener) override;
-   void stop() override;
-
- private:
-   struct PImpl;
-   std::unique_ptr<PImpl> me_;
-};
-
-} // namespace bts::md
@@ -1,40 +0,0 @@
-#include "EmumbaMarketDataProvider.h"
-#include "spark/api/md/MarketDataConfig.pb.h"
-#include "spark/api/md/MarketDataConfigUtils.h"
-#include "spark/md/IMarketDataSource.h"
-#include "sparkd/md/emumba/EmumbaDeribitSubscription.h"
-
-#include <memory>
-#include <vector>
-
-namespace bts::md {
-
-using bts::api::md::MarketDataConfig;
-using std::pair;
-using std::shared_ptr;
-using std::string;
-using std::vector;
-
-struct EmumbaMarketDataProvider::PImpl {
-   MarketDataConfig config;
-   vector<pair<api::domain::Venue, shared_ptr<IMarketDataSource>>> deribit_subs;
-
-   PImpl(MarketDataConfig config) : config(std::move(config)) {}
-};
-
-EmumbaMarketDataProvider::EmumbaMarketDataProvider(MarketDataConfig config)
-    : me_(std::make_unique<PImpl>(std::move(config))) {}
-
-EmumbaMarketDataProvider::~EmumbaMarketDataProvider() = default;
-
-vector<pair<api::domain::Venue, shared_ptr<IMarketDataSource>>>
-EmumbaMarketDataProvider::get_sources() {
-   for (const auto &cfg : me_->config.emumba_deribit()) {
-      me_->deribit_subs.push_back(
-          {api::domain::VENUE_DERIBIT,
-           std::make_shared<EmumbaDeribitSubscription>(cfg)});
-   }
-   return me_->deribit_subs;
-}
-
-} // namespace bts::md
@@ -1,25 +0,0 @@
-#pragma once
-
-#include "spark/api/md/MarketDataConfig.pb.h"
-#include "spark/md/IMarketDataProvider.h"
-
-#include <memory>
-
-namespace bts::md {
-
-class EmumbaMarketDataProvider : public IMarketDataProvider {
- public:
-   EmumbaMarketDataProvider();
-   explicit EmumbaMarketDataProvider(api::md::MarketDataConfig config);
-   ~EmumbaMarketDataProvider();
-
-   std::vector<
-       std::pair<bts::api::domain::Venue, std::shared_ptr<IMarketDataSource>>>
-   get_sources() override;
-
- private:
-   struct PImpl;
-   std::unique_ptr<PImpl> me_;
-};
-
-} // namespace bts::md
@@ -1,41 +0,0 @@
-#include "DatabentoGlbxSubscription.h"
-#include "spark/api/md/GlbxMarketDataConfig.pb.h"
-#include "spark/utils/thread_manager.h"
-#include "sparkd/md/databento/DatabentoDataHandler.h"
-#include "sparkd/md/glbx/GlbxMarketDataHandler.h"
-
-namespace bts::md {
-
-using bts::api::md::GlbxMarketDataConfig;
-using bts::persistence::IDGenerator;
-using std::shared_ptr;
-
-struct DatabentoGlbxSubscription::PImpl {
-   GlbxMarketDataConfig config;
-   shared_ptr<IDGenerator> sec_id_store;
-
-   PImpl(GlbxMarketDataConfig config, shared_ptr<IDGenerator> sec_id_store)
-       : config(std::move(config)), sec_id_store(std::move(sec_id_store)) {}
-};
-
-DatabentoGlbxSubscription::DatabentoGlbxSubscription(
-    GlbxMarketDataConfig config, shared_ptr<IDGenerator> sec_id_store)
-    : me_(std::make_unique<PImpl>(std::move(config), std::move(sec_id_store))) {
-}
-
-DatabentoGlbxSubscription::~DatabentoGlbxSubscription() = default;
-
-void DatabentoGlbxSubscription::start(
-    std::shared_ptr<IMarketDataListener> listener) {
-   bts::threading::make_thread(
-       "glbx",
-       [config = me_->config, sec_id_store = me_->sec_id_store,
-        listener = std::move(listener)]() {
-          api::md::glbx::GlbxMarketDatahandler::run(
-              std::move(listener), std::move(sec_id_store), config);
-       })
-       .detach();
-   // TODO: implement 'stop' (signal the thread above to terminate, then join it).
-}
-
-} // namespace bts::md
@@ -1,47 +0,0 @@
-#pragma once
-
-#include "spark/md/IMarketDataSource.h"
-
-#include <memory>
-
-namespace bts::api::md {
-class GlbxMarketDataConfig;
-}
-
-namespace bts::persistence {
-class IDGenerator;
-}
-
-namespace bts::md {
-
-/**
- * @brief Subscribes to market data at Glbx via Databento
- *
- * @a https://www.Databento.biz/
- */
-class DatabentoGlbxSubscription : public IMarketDataSource {
- public:
-   DatabentoGlbxSubscription(
-       api::md::GlbxMarketDataConfig config,
-       std::shared_ptr<persistence::IDGenerator> sec_id_store);
-   DatabentoGlbxSubscription(const DatabentoGlbxSubscription &) noexcept =
-       delete;
-   DatabentoGlbxSubscription(DatabentoGlbxSubscription &&) noexcept;
-   DatabentoGlbxSubscription &operator=(
-       const DatabentoGlbxSubscription &) noexcept = delete;
-   DatabentoGlbxSubscription &operator=(DatabentoGlbxSubscription &&) noexcept;
-   ~DatabentoGlbxSubscription() override;
-
-   /**
-    * Starts market data subscription on a background thread.
-    *
-    * @note Does not block.
-    */
-   void start(std::shared_ptr<IMarketDataListener> listener) override;
-
- private:
-   struct PImpl;
-   std::unique_ptr<PImpl> me_;
-};
-
-} // namespace bts::md
@@ -1,118 +0,0 @@
-#include "OnixsGlbxSubscription.h"
-#include "spark/api/md/GlbxMarketDataConfig.pb.h"
-#include "spark/core/LOCK_GUARD.h"
-#include "spark/logging/logging.h"
-#include "spark/md/IMarketDataListener.h"
-#include "spark/md/SymbolLoader.h"
-#include "spark/os/os.h"
-#include "spark/persistence/IDGenerator.h"
-#include "spark/utils/Dispatcher.h"
-#include "sparkd/md/platforms/IMDPlatform.h"
-#include "sparkd/md/platforms/mdp3_onixs/MDP3Main.h"
-
-#include <atomic>
-#include <memory>
-#include <thread>
-#include <vector>
-
-namespace bts::md {
-
-using bts::api::md::GlbxMarketDataConfig;
-using bts::domain::SecurityCriteria;
-using bts::md::IMDPlatform;
-using bts::persistence::IDGenerator;
-using std::shared_ptr;
-using std::string;
-using std::vector;
-
-struct OnixsGlbxSubscription::PImpl {
-   GlbxMarketDataConfig config;
-   shared_ptr<IDGenerator> sec_id_store;
-
-   PImpl(shared_ptr<IDGenerator> sec_id_store, GlbxMarketDataConfig config)
-       : config(std::move(config)), sec_id_store(std::move(sec_id_store)) {}
-
-   std::thread dispatcher_thread;
-   utils::Dispatcher *dispatcher{nullptr};
-   std::optional<utils::DispatcherTimer> heartbeat_timer;
-   bool is_running = false;
-   std::mutex mx;
-
-   ~PImpl() { stop(); }
-
-   /**
-    * Starts market data subscription on this thread.
-    * 
-    * @note Designed to be called from a thread callback inside `OnixsGlbxSubscription::start`.
-    * @note Blocks until subscription stops.
-    */
-   void run() {
-      {
-         BTS_LOCK_GUARD(mx);
-         if (!is_running) {
-            return;
-         }
-         const auto symbols = SymbolLoader::load(
-             SecurityCriteria::create({api::domain::VENUE_GLBX}));
-         if (symbols.empty()) {
-            LOG(md, Info, "OnixsGlbxSubscription aborting. No symbols loaded");
-            return;
-         }
-         LOG(md, Info,
-             "OnixsGlbxSubscription starting. t_id: {}; t_name: {}; config: "
-             "{}; symbols: {}",
-             os::get_thread_id(), os::get_thread_name(), config.DebugString(),
-             FMT_JOIN(symbols, ", "));
-         vector<shared_ptr<IMDPlatform>> platforms;
-         auto main = std::make_shared<MDP3Main>();
-         main->start({symbols.begin(), symbols.end()}, nullptr, nullptr);
-         platforms.emplace_back(std::move(main));
-         dispatcher = &utils::Dispatcher::current();
-         heartbeat_timer.emplace(*dispatcher, 250, [&]() {
-            for (auto &p : platforms) {
-               p->send_heartbeats();
-            }
-         });
-         heartbeat_timer->start();
-         is_running = true;
-      }
-      // This will block until dispatcher->terminate() is called.
-      // There is a slim, but non-zero chance that this is executed after
-      // 'stop' is called, since we 'run' outside of the lock.
-      dispatcher->run();
-   }
-
-   void stop() {
-      BTS_LOCK_GUARD(mx);
-      if (!is_running) {
-         return;
-      }
-      is_running = false;
-      if (dispatcher != nullptr) {
-         LOG(md, Diag, "OnixsGlbxSubscription stopping.");
-         dispatcher->io_service()->stop();
-      }
-      if (dispatcher_thread.joinable()) {
-         LOG(md, Diag, "OnixsGlbxSubscription joining dispatcher thread.");
-         dispatcher_thread.join();
-      }
-   }
-};
-
-OnixsGlbxSubscription::OnixsGlbxSubscription(
-    shared_ptr<IDGenerator> sec_id_store, GlbxMarketDataConfig config)
-    : me_(std::make_unique<PImpl>(std::move(sec_id_store), std::move(config))) {
-}
-
-OnixsGlbxSubscription::~OnixsGlbxSubscription() = default;
-
-void OnixsGlbxSubscription::start(
-    shared_ptr<IMarketDataListener> /*listener*/) {
-   std::atomic_thread_fence(std::memory_order_release);
-   bts_assert(!me_->dispatcher_thread.joinable());
-   me_->dispatcher_thread = std::thread{[&]() { me_->run(); }};
-}
-
-void OnixsGlbxSubscription::stop() { me_->stop(); }
-
-} // namespace bts::md
@@ -1,48 +0,0 @@
-#pragma once
-
-#include "spark/md/IMarketDataSource.h"
-
-#include <memory>
-
-namespace bts::api::md {
-class GlbxMarketDataConfig;
-}
-
-namespace bts::persistence {
-class IDGenerator;
-}
-
-namespace bts::md {
-
-/**
- * @brief Subscribes to market data at Glbx via Onixs
- *
- * @a https://www.onixs.biz/
- */
-class OnixsGlbxSubscription : public IMarketDataSource {
- public:
-   OnixsGlbxSubscription(
-       std::shared_ptr<persistence::IDGenerator> sec_id_store,
-       api::md::GlbxMarketDataConfig config);
-   OnixsGlbxSubscription(const OnixsGlbxSubscription &) noexcept = delete;
-   OnixsGlbxSubscription(OnixsGlbxSubscription &&) noexcept;
-   OnixsGlbxSubscription &operator=(const OnixsGlbxSubscription &) noexcept =
-       delete;
-   OnixsGlbxSubscription &operator=(OnixsGlbxSubscription &&) noexcept;
-   ~OnixsGlbxSubscription() override;
-
-   /**
-    * Starts market data subscription on a background thread.
-    *
-    * @note Does not block.
-    */
-   void start(std::shared_ptr<IMarketDataListener> listener) override;
-
-   void stop() override;
-
- private:
-   struct PImpl;
-   std::unique_ptr<PImpl> me_;
-};
-
-} // namespace bts::md
@@ -1,47 +0,0 @@
-#include "TaifexMarketDataProvider.h"
-#include "spark/api/domain/Venue.pb.h"
-#include "spark/api/md/MarketDataConfig.pb.h"
-#include "spark/persistence/IDGenerator.h"
-#include "sparkd/md/taifex/TaifexSubscription.h"
-
-#include <memory>
-#include <vector>
-
-namespace bts::md {
-
-using bts::api::domain::Venue;
-using bts::api::md::MarketDataConfig;
-using bts::persistence::IDGenerator;
-using std::pair;
-using std::shared_ptr;
-using std::string;
-using std::vector;
-
-struct TaifexMarketDataProvider::PImpl {
-   MarketDataConfig config;
-   shared_ptr<IDGenerator> sec_id_store;
-   vector<shared_ptr<TaifexSubscription>> sources;
-
-   PImpl(MarketDataConfig config, shared_ptr<IDGenerator> sec_id_store)
-       : config(std::move(config)), sec_id_store(std::move(sec_id_store)) {}
-};
-
-TaifexMarketDataProvider::TaifexMarketDataProvider(
-    MarketDataConfig config, shared_ptr<IDGenerator> sec_id_store)
-    : me_(std::make_unique<PImpl>(std::move(config), std::move(sec_id_store))) {
-}
-
-TaifexMarketDataProvider::~TaifexMarketDataProvider() = default;
-
-vector<pair<Venue, shared_ptr<IMarketDataSource>>>
-TaifexMarketDataProvider::get_sources() {
-   vector<pair<Venue, shared_ptr<IMarketDataSource>>> rv;
-   for (const auto &taifex : me_->config.taifex()) {
-      me_->sources.push_back(
-          std::make_shared<TaifexSubscription>(taifex, me_->sec_id_store));
-      rv.push_back({Venue::VENUE_TAIFEX, me_->sources.back()});
-   }
-   return rv;
-}
-
-} // namespace bts::md
@@ -1,33 +0,0 @@
-#pragma once
-
-#include "spark/md/IMarketDataProvider.h"
-
-#include <memory>
-
-namespace bts::persistence {
-class IDGenerator;
-}
-
-namespace bts::api::md {
-class MarketDataConfig;
-}
-
-namespace bts::md {
-
-class TaifexMarketDataProvider : public IMarketDataProvider {
- public:
-   explicit TaifexMarketDataProvider(
-       api::md::MarketDataConfig config,
-       std::shared_ptr<persistence::IDGenerator> sec_id_store);
-   ~TaifexMarketDataProvider();
-
-   std::vector<
-       std::pair<api::domain::Venue, std::shared_ptr<IMarketDataSource>>>
-   get_sources() override;
-
- private:
-   struct PImpl;
-   std::unique_ptr<PImpl> me_;
-};
-
-} // namespace bts::md
@@ -1,40 +0,0 @@
-#include "TaifexSubscription.h"
-#include "spark/api/md/TaifexMarketDataConfig.pb.h"
-#include "spark/utils/thread_manager.h"
-#include "sparkd/md/taifex/TaifexDataGateway.h"
-
-namespace bts::md {
-
-using bts::api::md::TaifexMarketDataConfig;
-using bts::md::taifex::TaifexDataGateway;
-using bts::persistence::IDGenerator;
-using std::shared_ptr;
-
-struct TaifexSubscription::PImpl {
-   TaifexMarketDataConfig config;
-   shared_ptr<IDGenerator> sec_id_store;
-
-   PImpl(TaifexMarketDataConfig config, shared_ptr<IDGenerator> sec_id_store)
-       : config(std::move(config)), sec_id_store(std::move(sec_id_store)) {}
-};
-
-TaifexSubscription::TaifexSubscription(
-    TaifexMarketDataConfig config, shared_ptr<IDGenerator> sec_id_store)
-    : me_(std::make_unique<PImpl>(std::move(config), std::move(sec_id_store))) {
-}
-
-TaifexSubscription::~TaifexSubscription() = default;
-
-void TaifexSubscription::start(shared_ptr<IMarketDataListener> listener) {
-   bts::threading::make_thread(
-       "taifex",
-       [config = me_->config, sec_id_store = me_->sec_id_store,
-        listener = std::move(listener)]() {
-          TaifexDataGateway::run(
-              config, std::move(listener), std::move(sec_id_store));
-       })
-       .detach();
-   // TODO: implement 'stop' (signal the thread above to terminate, then join it).
-}
-
-} // namespace bts::md
@@ -1,43 +0,0 @@
-#pragma once
-
-#include "spark/md/IMarketDataSource.h"
-
-#include <memory>
-
-namespace bts::api::md {
-class TaifexMarketDataConfig;
-}
-
-namespace bts::persistence {
-class IDGenerator;
-}
-
-namespace bts::md {
-
-/**
- * @brief Subscribes to market data at Taifex
- */
-class TaifexSubscription : public IMarketDataSource {
- public:
-   TaifexSubscription(
-       api::md::TaifexMarketDataConfig config,
-       std::shared_ptr<persistence::IDGenerator> sec_id_store);
-   TaifexSubscription(const TaifexSubscription &) noexcept = delete;
-   TaifexSubscription(TaifexSubscription &&) noexcept;
-   TaifexSubscription &operator=(const TaifexSubscription &) noexcept = delete;
-   TaifexSubscription &operator=(TaifexSubscription &&) noexcept;
-   ~TaifexSubscription() override;
-
-   /**
-    * Starts market data subscription on a background thread.
-    *
-    * @note Does not block.
-    */
-   void start(std::shared_ptr<IMarketDataListener> listener) override;
-
- private:
-   struct PImpl;
-   std::unique_ptr<PImpl> me_;
-};
-
-} // namespace bts::md
@@ -30,7 +30,7 @@
      <PrivateAssets>all</PrivateAssets>
    </PackageReference>
    <PackageReference Include="Google.Protobuf" Version="3.24.3" />
-    <PackageReference Include="IronPython" Version="3.4.1" />
+    <PackageReference Include="IronPython" Version="3.4.2" />
    <PackageReference Include="MathNet.Numerics" Version="5.0.0" />
    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.2.0" />
    <PackageReference Include="NAudio" Version="2.0.1" />
@@ -60,7 +60,7 @@
  <ItemGroup>
    <Resource Include="**/*.ico" />
    <Resource Include="**/*.png" />
-    <ConfigDefaults Include="../../lib_spark/spark/config/defaults/*.conf" />
+    <ConfigDefaults Include="../../lib/config/defaults/*.conf" />
  </ItemGroup>

  <ItemGroup>
@@ -128,8 +128,6 @@ namespace bts.api {
            Security.ExchangeDataOneofCase.Taifex => security.Taifex.SecurityDesc,
            Security.ExchangeDataOneofCase.Twse => security.Twse.SecurityDesc,
            Security.ExchangeDataOneofCase.None => security.Synthetic.Name,
-            Security.ExchangeDataOneofCase.Bybit => security.Bybit.SecurityDesc,
-            Security.ExchangeDataOneofCase.Gateio => security.Gateio.SecurityDesc,
            _ => throw new NotImplementedException("security_desc not implemented for exchange")
         };

@@ -181,8 +179,8 @@ namespace bts.api {
            domain.Venue.Cboe => Venue.Cboe,
            domain.Venue.Taifex => Venue.Taifex,
            domain.Venue.Twse => Venue.Twse,
-            domain.Venue.Bybit => Venue.Bybit,
-            domain.Venue.Gateio => Venue.Gateio,
+
+
            domain.Venue.Unspecified => Venue.Unset,
            _ => throw new NotImplementedException(),
         };
@@ -203,8 +201,7 @@ namespace bts.api {
            Venue.Cboe => domain.Venue.Cboe,
            Venue.Taifex => domain.Venue.Taifex,
            Venue.Twse => domain.Venue.Twse,
-            Venue.Bybit => domain.Venue.Bybit,
-            Venue.Gateio => domain.Venue.Gateio,
+
            Venue.Unset => domain.Venue.Unspecified,
            _ => throw new NotImplementedException(),
         };
				`@@ -1 +0,0 @@`
				`* Connects to Binance test nets for market data and execution`
				`@@ -1 +0,0 @@`
				`SPARKD__CONFIG_DIR=src/app_sparkd/configurations/bybit_dev`