CLEANUP: BYBIT Symbols.conf.

ADD: BYBIT MD Working, NaN's also Added.
ADD: BYBIT Securities showing up on frontend. Default Contract Spec
2025-05-26 11:38:19 +00:00 · 2025-05-26 11:01:20 +00:00 · 2025-05-22 16:17:19 +00:00 · 2025-05-22 16:17:19 +00:00 · 2025-05-22 16:17:19 +00:00 · 2025-05-22 16:17:19 +00:00
2096 changed files with 14830 additions and 9493 deletions
@@ -199,24 +199,27 @@
      "request": "launch",
      "program": "",
      "linux": {
-        "preLaunchTask": "Build Sparkd (Debug)",
-        "program": "${workspaceFolder}/build/sparkd-debug-clang/server/sparkd",
-        "args": [
-          "run"
-        ],
-        "stopAtEntry": false,
-        "cwd": "${workspaceFolder}/build/sparkd-debug-clang/server",
-        "environment": [],
-        "externalConsole": false,
-        "MIMode": "gdb",
-        "setupCommands": [
-          {
-            "description": "Enable pretty-printing for gdb",
-            "text": "-enable-pretty-printing",
-            "ignoreFailures": true
-          }
-        ]
-      }
+        "program": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd/sparkd",
+      },
+      "args": [
+        "run"
+      ],
+      "cwd": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd",
+      "stopAtEntry": false,
+      "externalConsole": false,
+      "MIMode": "gdb",
+      "setupCommands": [
+        {
+          "description": "Enable pretty-printing for gdb",
+          "text": "-enable-pretty-printing",
+          "ignoreFailures": true
+        },
+        {
+          "description": "Set Disassembly Flavor to Intel",
+          "text": "-gdb-set disassembly-flavor intel",
+          "ignoreFailures": true
+        }
+      ]
    },
    {
      "name": "Sparkd (Skip Build)",
@@ -224,74 +227,27 @@
      "request": "launch",
      "program": "",
      "linux": {
-        "program": "${workspaceFolder}/build/sparkd-debug-clang/server/sparkd",
-        "args": [
-          "run"
-        ],
-        "stopAtEntry": false,
-        "cwd": "${workspaceFolder}/build/sparkd-debug-clang/server",
-        "environment": [],
-        "externalConsole": false,
-        "MIMode": "gdb",
-        "setupCommands": [
-          {
-            "description": "Enable pretty-printing for gdb",
-            "text": "-enable-pretty-printing",
-            "ignoreFailures": true
-          }
-        ]
-      }
-    },
-    {
-      "name": "Sparkd Test",
-      "type": "cppdbg",
-      "request": "launch",
-      "program": "",
-      "linux": {
-        "preLaunchTask": "Build Sparkd (Debug)",
-        "program": "${workspaceFolder}/build/sparkd-debug-clang/server/sparkd",
-        "args": [
-          "test",
-          "${input:sparkdTest}",
-        ],
-        "stopAtEntry": false,
-        "cwd": "${workspaceFolder}/build/sparkd-debug-clang/server",
-        "environment": [],
-        "externalConsole": false,
-        "MIMode": "gdb",
-        "setupCommands": [
-          {
-            "description": "Enable pretty-printing for gdb",
-            "text": "-enable-pretty-printing",
-            "ignoreFailures": true
-          }
-        ]
-      }
-    },
-    {
-      "name": "Sparkd Test (Skip Build)",
-      "type": "cppdbg",
-      "request": "launch",
-      "program": "",
-      "linux": {
-        "program": "${workspaceFolder}/build/sparkd-debug-clang/server/sparkd",
-        "args": [
-          "test",
-          "${input:sparkdTest}",
-        ],
-        "stopAtEntry": false,
-        "cwd": "${workspaceFolder}/build/sparkd-debug-clang/server",
-        "environment": [],
-        "externalConsole": false,
-        "MIMode": "gdb",
-        "setupCommands": [
-          {
-            "description": "Enable pretty-printing for gdb",
-            "text": "-enable-pretty-printing",
-            "ignoreFailures": true
-          }
-        ]
-      }
+        "program": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd/sparkd",
+      },
+      "args": [
+        "run"
+      ],
+      "cwd": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd",
+      "stopAtEntry": false,
+      "externalConsole": false,
+      "MIMode": "gdb",
+      "setupCommands": [
+        {
+          "description": "Enable pretty-printing for gdb",
+          "text": "-enable-pretty-printing",
+          "ignoreFailures": true
+        },
+        {
+          "description": "Set Disassembly Flavor to Intel",
+          "text": "-gdb-set disassembly-flavor intel",
+          "ignoreFailures": true
+        }
+      ]
    },
  ],
  "inputs": [
@@ -22,9 +22,9 @@ set(
 # can run simultaneously without fighting over the same shm files (e.g., sparkd,
 # api server, etc.).
 set(SHARED_MEMORY_PREFIX "bts" CACHE STRING "Prefix to use when creating shared memory files")
-set(SPARK_BIN "${EXTERNALS_ROOT}/bin")
-set(SPARK_INCLUDE "${EXTERNALS_ROOT}/include")
-set(SPARK_LIB "${EXTERNALS_ROOT}/lib/$<CONFIG>")
+set(EXTERNALS_BIN "${EXTERNALS_ROOT}/bin")
+set(EXTERNALS_INCLUDE "${EXTERNALS_ROOT}/include")
+set(EXTERNALS_LIB "${EXTERNALS_ROOT}/lib/$<CONFIG>")
 set(BUILD_DOCS OFF CACHE BOOL "Build documentation")
 set(BUILD_SDK OFF CACHE BOOL "Build the sdk")
 set(EXPORT_MD_CONSUMER OFF CACHE BOOL "Export (extern) C functions for MD consumer interop")
@@ -36,8 +36,8 @@ else()
  set(BUILD_ADMIN_WEB_APP ON CACHE BOOL "Build the Admin WebApp")
 endif()

-include_directories(BEFORE SYSTEM ${SPARK_INCLUDE} ${EXTERNALS_ROOT}/include/zlib)
-link_directories(${SPARK_LIB})
+include_directories(BEFORE SYSTEM ${EXTERNALS_INCLUDE} ${EXTERNALS_ROOT}/include/zlib)
+link_directories(${EXTERNALS_LIB})

 add_compile_definitions(SHARED_MEMORY_PREFIX="${SHARED_MEMORY_PREFIX}")
 add_compile_definitions(APP_DATA_DIR="${APP_DATA_DIR}")
@@ -47,12 +47,8 @@ endif()
 add_compile_definitions(EXPORT_MD_CONSUMER="${EXPORT_MD_CONSUMER}")
 if(WIN32)
  # subdirs
-  add_subdirectory(src/lib)
+  add_subdirectory(src/lib_spark)
 else()
-  # make sparkd look for libspark.so in its directory
-  set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE)
-  set(CMAKE_INSTALL_RPATH ".")
-
  if(BUILD_SPARKD)
    # Note: All macros that begin w/ 'CCAPI' are ccapi's macros (defined and/or
    # read by their code). We override some below.
@@ -118,11 +114,14 @@ else()

  # subdirs
  if (BUILD_SPARKD)
+    add_subdirectory(src/lib_onixs_fix)
    add_subdirectory(src/lib_binance_execution)
-    add_subdirectory(src/server/sparkd)
+    add_subdirectory(src/lib_okx_execution)
+    add_subdirectory(src/lib_python_md)
+    add_subdirectory(src/app_sparkd)
    add_subdirectory(src/data_files)
  endif()
-  add_subdirectory(src/lib)
+  add_subdirectory(src/lib_spark)
  if (BUILD_ADMIN_WEB_APP)
    add_subdirectory(src/Spark.Admin.WebApp)
  endif()
@@ -132,37 +131,4 @@ else()
  if(BUILD_SDK)
    add_subdirectory(src/sdk)
  endif()
-
-  if(BUILD_SPARKD)
-    add_custom_command(
-      OUTPUT ${CMAKE_BINARY_DIR}/server
-      COMMAND mkdir -p ${CMAKE_BINARY_DIR}/server
-      COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/src/server/sparkd/sparkd ${CMAKE_BINARY_DIR}/server
-      COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/data_files ${CMAKE_BINARY_DIR}/server
-      COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/src/lib/libspark.so ${CMAKE_BINARY_DIR}/server
-      COMMAND ln ARGS -sf ${EXTERNALS_ROOT}/lib/${CMAKE_BUILD_TYPE}/libonixs-fix-engine.so.4 ${CMAKE_BINARY_DIR}/server
-      
-      COMMAND
-        ln ARGS -sf
-        ${EXTERNALS_ROOT}/lib/Release/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.16.0.0
-        ${CMAKE_BINARY_DIR}/server
-      COMMAND ln ARGS -sf ${EXTERNALS_ROOT}/lib/Release/libACE-5.7.5.so ${CMAKE_BINARY_DIR}/server
-      COMMAND mkdir ARGS -p ${CMAKE_BINARY_DIR}/server/plugins
-      COMMENT "Creating server runtime layout"
-      DEPENDS sparkd data_files_dir)
-
-    add_custom_target(server_layout ALL DEPENDS ${CMAKE_BINARY_DIR}/server)
-
-    install(
-      DIRECTORY "${CMAKE_BINARY_DIR}/server/plugins"
-      DESTINATION "${CMAKE_INSTALL_BINDIR}")
-    install(
-      FILES
-      "${SPARK_LIB}/libACE-5.7.5.so"
-      "${SPARK_LIB}/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.16.0.0"
-      "${SPARK_LIB}/libonixs-fix-engine.so.4.10.1"
-      "${SPARK_LIB}/libonixs-fix-engine.so.4"
-      "${SPARK_LIB}/libonixs-fix-engine.so"
-      DESTINATION "${CMAKE_INSTALL_BINDIR}")
-  endif()
 endif()
@@ -31,9 +31,8 @@
        "CMAKE_SHARED_LINKER_FLAGS_RELEASE": "/OPT:REF /OPT:ICF /LTCG",
        "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
        "EXTERNALS_ROOT": "${sourceDir}/externals/windows",
-        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0",
-        "DOTNET_WINDOWS_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0-windows",
+        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf",
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0;${sourceDir}/build/$<CONFIG>/bin/net8.0-windows",
        "EXPORT_MD_CONSUMER": {
          "type": "BOOL",
          "value": "ON"
@@ -58,11 +57,12 @@
        "APP_DATA_DIR_OVERRIDE": "${dollar}{SPARKD_DATA_DIR}",
        "CMAKE_BUILD_WITH_INSTALL_RPATH": {
          "type": "BOOL",
-          "value": "ON"
+          "value": "OFF"
        },
        "CMAKE_CXX_STANDARD": "20",
        "CMAKE_INSTALL_RPATH": ".",
        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
+        "CMAKE_PREFIX_PATH": "${sourceDir}/cmake/;${sourceDir}/.venv/lib/python3.10/site-packages/pybind11/share/cmake/pybind11",
        "CMAKE_EXE_LINKER_FLAGS": "-Wno-undef",
        "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
        "DATABENTO_USE_EXTERNAL_JSON": {
@@ -93,7 +93,7 @@
          "value": "ON"
        },
        "CCAPI_LOG_LEVEL": "Info",
-        "CONFIG_DEFAULTS_PATH": "config/profiles/md_dev/*.conf;config/profiles/default/*.conf"
+        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/md_dev/*.conf;spark/config/profiles/default/*.conf"
      }
    },

@@ -104,7 +104,7 @@
      "cacheVariables": {
        "CMAKE_BUILD_TYPE": "Release",
        "CCAPI_LOG_LEVEL": "Warn",
-        "CONFIG_DEFAULTS_PATH": "config/profiles/md_prod/*.conf;config/profiles/default/*.conf"
+        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/md_prod/*.conf;spark/config/profiles/default/*.conf"
      }
    },

@@ -186,7 +186,11 @@
        "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
-        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
+        "CMAKE_EXPORT_COMPILE_COMMANDS": {
+          "type": "BOOL",
+          "value": "ON"
+        },
+        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf",
        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0",
        "EXPORT_MD_CONSUMER": {
          "type": "BOOL",
@@ -227,7 +231,7 @@
      "cacheVariables": {
        "APP_DATA_DIR": "${dollar}{APPDATA}/bts/DataStream",
        "APP_DATA_DIR_OVERRIDE": "${dollar}{DATA_STREAM_DATA_DIR}",
-        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf",
+        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf;spark/config/profiles/data_stream/*.conf",
        "SHARED_MEMORY_PREFIX": "data_stream"
      }
    },
@@ -262,8 +266,7 @@
        "CMAKE_CXX_COMPILER": "clang++",
        "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
-        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
-        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf",
+        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf;spark/config/profiles/data_stream/*.conf",
        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
        "EXPORT_MD_CONSUMER": {
          "type": "BOOL",
@@ -0,0 +1,7 @@
+include_guard(GLOBAL)
+
+macro(export_to_parent)
+  foreach(var IN ITEMS ${ARGN})
+    set(${var} ${${var}} PARENT_SCOPE)
+  endforeach()
+endmacro()
@@ -0,0 +1,142 @@
+include_guard(DIRECTORY)
+include(${CMAKE_SOURCE_DIR}/cmake/ExportToParent.cmake)
+include(${CMAKE_SOURCE_DIR}/cmake/InstallBasic.cmake)
+include(${CMAKE_SOURCE_DIR}/cmake/ProtobufGenerateCpp.cmake)
+include(${CMAKE_SOURCE_DIR}/cmake/SetFirstDefined.cmake)
+include(${CMAKE_SOURCE_DIR}/cmake/safe_dep_glob.cmake)
+
+function(init_library)
+  set(_options SHARED STATIC)
+  set(_one_value_args
+    NAME
+    SOURCE_DIR
+    INCLUDE_DIR
+  )
+  set(_multi_value_args)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+  if (_ARGS_SHARED AND _ARGS_STATIC)
+    message(FATAL_ERROR "Invalid args passed to init_library: cannot specify STATIC and SHARED")
+  endif()
+  set_first_defined(_name _ARGS_NAME PROJECT_NAME)
+  set_first_defined(_source_dir _ARGS_SOURCE_DIR _name)
+  set_first_defined(_include_dir _ARGS_INCLUDE_DIR _source_dir)
+
+  # Cpp sources
+  file(
+    GLOB_RECURSE ${_name}_SOURCES
+    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
+    ${_source_dir}/*.cpp
+  )
+  # Note: cpp files inside 'tests' dir(s) are considered tests.
+  # Remove tests from main sources, and add them to test sources
+  foreach(item ${${_name}_SOURCES})
+    if(${item} MATCHES "tests/.*\.cpp$")
+      list(APPEND ${_name}_TEST_SOURCES ${item})
+    endif()
+  endforeach()
+  foreach(item ${${_name}_TEST_SOURCES})
+    list(REMOVE_ITEM ${_name}_SOURCES ${item})
+  endforeach()
+
+  # Proto sources 
+  file(
+    GLOB_RECURSE ${_name}_PROTO_SOURCES
+    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
+    ${_source_dir}/protobufs/*.proto
+  )
+  update_deps_file("${${_name}_PROTO_SOURCES};")
+  protobuf_generate_cpp(
+    CXX_SOURCES ${_name}_CXX_PROTO_SOURCES
+    CXX_HEADERS ${_name}_CXX_PROTO_HEADERS
+    CXX_OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}
+    PROTO_INCLUDE_DIRS
+      ${EXTERNALS_INCLUDE}
+      ${CMAKE_CURRENT_SOURCE_DIR}
+    PROTO_SOURCES
+      ${${_name}_PROTO_SOURCES}
+  )
+
+  export_to_parent(
+    ${_name}_SOURCES
+    ${_name}_TEST_SOURCES
+    ${_name}_PROTO_SOURCES
+    ${_name}_CXX_PROTO_SOURCES
+    ${_name}_CXX_PROTO_HEADERS
+  )
+
+  # Add main lib
+  if (_ARGS_STATIC)
+    add_library(${_name} STATIC ${${_name}_SOURCES} ${${_name}_CXX_PROTO_SOURCES})
+  elseif(_ARGS_SHARED)
+    add_library(${_name} SHARED ${${_name}_SOURCES} ${${_name}_CXX_PROTO_SOURCES})
+  else()
+    add_library(${_name} ${${_name}_SOURCES} ${${_name}_CXX_PROTO_SOURCES})
+  endif()
+  target_include_directories(
+    ${_name}
+    SYSTEM
+    BEFORE
+    PUBLIC
+      $<BUILD_INTERFACE:${EXTERNALS_ROOT}/include>
+      $<BUILD_INTERFACE:${EXTERNALS_ROOT}/include/zlib>
+  )
+  target_include_directories(
+    ${_name}
+    PUBLIC
+      $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
+      $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>
+  )
+  # Include protos (if any)
+  if (${_name}_PROTO_SOURCES)
+    target_include_directories(
+      ${_name}
+      PUBLIC
+        $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
+    )
+  endif()
+  target_link_directories(
+    ${_name}
+    PUBLIC
+      $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
+      $<BUILD_INTERFACE:${EXTERNALS_ROOT}/lib/$<CONFIG>>
+      $<INSTALL_INTERFACE:${CMAKE_INSTALL_LIBDIR}>
+  )
+  # Install main lib
+  foreach(_hdr ${${_name}_CXX_PROTO_HEADERS})
+    file(RELATIVE_PATH _relative_path ${CMAKE_CURRENT_BINARY_DIR} ${_hdr})
+    install(FILES ${_hdr} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/${_relative_path})
+  endforeach()
+  install_basic(
+    NAME ${_name}
+    INCLUDE_DIR ${_include_dir}
+  )
+
+  # Add tests executable (if there are any)
+  if (${_name}_TEST_SOURCES AND NOT WIN32)
+    add_executable(${_name}-tests ${${_name}_TEST_SOURCES})
+
+    get_target_property(_lib_type ${_name} TYPE)
+    if (_lib_type STREQUAL "SHARED_LIBRARY")
+      # Set RPATH so the tests exe knows where to find the shared lib.
+      set_target_properties(
+        ${_name}-tests
+        PROPERTIES
+          BUILD_RPATH "$ORIGIN"
+          INSTALL_RPATH "$ORIGIN/../${CMAKE_INSTALL_LIBDIR}"
+      )
+    endif()
+
+    # Link main lib
+    target_link_directories(
+      ${_name}-tests
+      PRIVATE
+        $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
+        $<INSTALL_INTERFACE:${CMAKE_INSTALL_LIBDIR}>
+    )
+    target_link_libraries(${_name}-tests PRIVATE ${_name})
+
+    # Catch2
+    find_package(Catch2 CONFIG REQUIRED COMPONENTS Catch2WithMain)
+    target_link_libraries(${_name}-tests PRIVATE Catch2::Catch2WithMain)
+  endif()
+endfunction()
@@ -0,0 +1,39 @@
+include_guard(DIRECTORY)
+include(GNUInstallDirs)
+include(${CMAKE_SOURCE_DIR}/cmake/PackBasic.cmake)
+include(${CMAKE_SOURCE_DIR}/cmake/SetFirstDefined.cmake)
+
+function(install_basic)
+  set(_options)
+  set(_one_value_args NAME INCLUDE_DIR)
+  set(_multi_value_args)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+  set_first_defined(_name _ARGS_NAME PROJECT_NAME)
+  set_first_defined(_include_dir _ARGS_INCLUDE_DIR _name)
+
+  file(
+    GLOB_RECURSE ${_name}_HEADERS
+    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
+    ${_include_dir}/*.h)
+
+  target_sources(${_name}
+    PUBLIC
+      FILE_SET HEADERS FILES ${${_name}_HEADERS})
+
+  install(
+    TARGETS ${_name}
+    EXPORT ${_name}Targets
+    RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
+    LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
+    ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
+    FILE_SET HEADERS DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}
+  )
+
+  install(
+    EXPORT ${_name}Targets
+    FILE ${_name}Targets.cmake
+    NAMESPACE ${_name}::
+    DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name})
+
+  pack_basic()
+endfunction()
@@ -0,0 +1,44 @@
+include_guard(DIRECTORY)
+include(CMakePackageConfigHelpers)
+include(${CMAKE_SOURCE_DIR}/cmake/SetFirstDefined.cmake)
+
+function(pack_basic)
+  set(_options)
+  set(_one_value_args NAME)
+  set(_multi_value_args)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+  set_first_defined(_name _ARGS_NAME PROJECT_NAME)
+
+  find_package(Git QUIET)
+
+  if(GIT_FOUND)
+    execute_process(
+      COMMAND ${GIT_EXECUTABLE} describe --tags --always --dirty
+      WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
+      OUTPUT_VARIABLE PROJECT_VERSION
+      OUTPUT_STRIP_TRAILING_WHITESPACE
+    )
+  endif()
+
+  if(NOT GIT_FOUND OR PROJECT_VERSION STREQUAL "")
+    set(PROJECT_VERSION "0.0.0")
+  endif()
+
+  write_basic_package_version_file(
+    "${CMAKE_CURRENT_BINARY_DIR}/${_name}ConfigVersion.cmake"
+    VERSION ${PROJECT_VERSION}
+    COMPATIBILITY AnyNewerVersion
+  )
+
+  configure_package_config_file(
+    "${CMAKE_CURRENT_SOURCE_DIR}/Config.cmake.in"
+    "${CMAKE_CURRENT_BINARY_DIR}/${_name}Config.cmake"
+    INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name}
+  )
+
+  install(
+    FILES
+      "${CMAKE_CURRENT_BINARY_DIR}/${_name}Config.cmake"
+      "${CMAKE_CURRENT_BINARY_DIR}/${_name}ConfigVersion.cmake"
+    DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name})
+endfunction()
@@ -1,12 +1,23 @@
-set(Protobuf_PROTOC_EXECUTABLE "${SPARK_BIN}/protoc")
-set(PROTOBUF_INCLUDE_DIRS "${SPARK_INCLUDE}")
+set(Protobuf_PROTOC_EXECUTABLE "${EXTERNALS_BIN}/protoc")
+set(PROTOBUF_INCLUDE_DIRS "${EXTERNALS_INCLUDE}")
 set(PROTOBUF_STATIC_LIB "libprotobuf")
+set(_protobuf_src_dir ${CMAKE_SOURCE_DIR}/src/lib_spark)
+set(_protobuf_bin_dir ${CMAKE_BINARY_DIR}/src/lib_spark)

-function(Protobuf_Generate_Cpp SRCS HDRS)
-  set(${SRCS})
-  set(${HDRS})
+function(Protobuf_Generate_Cpp)
+  set(_options)
+  set(_one_value_args CXX_SOURCES CXX_HEADERS CXX_OUT_DIR)
+  set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+  set(${_ARGS_CXX_SOURCES})
+  set(${_ARGS_CXX_HEADERS})

-  foreach(FIL ${ARGN})
+  set(_proto_include_cli_args)
+  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
+    list(APPEND _proto_include_cli_args -I ${_dir})
+  endforeach()
+
+  foreach(FIL ${_ARGS_PROTO_SOURCES})
    get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
    get_filename_component(FIL_WE ${FIL} NAME_WE)
    get_filename_component(FIL_DIR ${FIL} DIRECTORY)
@@ -14,48 +25,56 @@ function(Protobuf_Generate_Cpp SRCS HDRS)
      set(FIL_WE "${FIL_DIR}/${FIL_WE}")
    endif()

-    list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc")
-    list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h")
+    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc")
+    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h")

    add_custom_command(
-      OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc"
-             "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h"
+      OUTPUT "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
+             "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
      COMMAND
-        ${Protobuf_PROTOC_EXECUTABLE} ARGS --cpp_out=${CMAKE_BINARY_DIR}/src -I
-        ${PROTOBUF_INCLUDE_DIRS} -I ${CMAKE_SOURCE_DIR}/src ${ABS_FIL}
+        ${Protobuf_PROTOC_EXECUTABLE} ARGS --cpp_out=${_ARGS_CXX_OUT_DIR} ${_proto_include_cli_args} ${ABS_FIL}
      DEPENDS ${ABS_FIL}
      COMMENT "Running C++ protocol buffer compiler on ${FIL}"
      VERBATIM)
  endforeach()

-  set_source_files_properties(${${SRCS}} ${${HDRS}} PROPERTIES GENERATED TRUE)
+  set_source_files_properties(${${_ARGS_CXX_SOURCES}} ${${_ARGS_CXX_HEADERS}} PROPERTIES GENERATED TRUE)

  # remove some warnings from protobuf generated files
  if(WIN32)
-    set_source_files_properties(${${SRCS}} PROPERTIES COMPILE_FLAGS
+    set_source_files_properties(${${_ARGS_CXX_SOURCES}} PROPERTIES COMPILE_FLAGS
                                                      "/wd4309 /wd4838")
  else()
    set_source_files_properties(
-      ${${SRCS}}
+      ${${_ARGS_CXX_SOURCES}}
      PROPERTIES
        COMPILE_FLAGS
        "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
    )
  endif()

-  set(${SRCS}
-      ${${SRCS}}
+  set(${_ARGS_CXX_SOURCES}
+      ${${_ARGS_CXX_SOURCES}}
      PARENT_SCOPE)
-  set(${HDRS}
-      ${${HDRS}}
+  set(${_ARGS_CXX_HEADERS}
+      ${${_ARGS_CXX_HEADERS}}
      PARENT_SCOPE)
 endfunction()

-function(Service_Compiler_Generate_Cpp SRCS HDRS)
-  set(${SRCS})
-  set(${HDRS})
+function(Service_Compiler_Generate_Cpp)
+  set(_options)
+  set(_one_value_args CXX_SOURCES CXX_HEADERS CXX_OUT_DIR)
+  set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+  set(${_ARGS_CXX_SOURCES})
+  set(${_ARGS_CXX_HEADERS})

-  foreach(FIL ${ARGN})
+  set(_proto_include_cli_args)
+  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
+    list(APPEND _proto_include_cli_args -I ${_dir})
+  endforeach()
+
+  foreach(FIL ${_ARGS_PROTO_SOURCES})
    get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
    get_filename_component(FIL_WE ${FIL} NAME_WE)
    get_filename_component(FIL_DIR ${FIL} DIRECTORY)
@@ -63,58 +82,58 @@ function(Service_Compiler_Generate_Cpp SRCS HDRS)
      set(FIL_WE "${FIL_DIR}/${FIL_WE}")
    endif()

-    list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc")
-    list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h")
+    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc")
+    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h")

-    list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.cc")
-    list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.h")
+    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc")
+    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h")

-    list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.cc")
-    list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.h")
+    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc")
+    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h")

    set(TEMPLATES
-        ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/templates/cpp/client.pb.cc
-        ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/templates/cpp/client.pb.h
-        ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/templates/cpp/server.pb.cc
-        ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/templates/cpp/server.pb.h
+        ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/client.pb.cc
+        ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/client.pb.h
+        ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/server.pb.cc
+        ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/server.pb.h
    )

+    set(_plugin_py ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/plugin.py)
    if(WIN32)
+      set(_plugin_cmd ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/plugin.cmd)
      add_custom_command(
-        OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc"
-               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h"
-               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.cc"
-               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.h"
-               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.cc"
-               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.h"
+        OUTPUT "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h"
        COMMAND SET PYTHONPATH=${SPARK_PYTHON}
        COMMAND ${CMAKE_SOURCE_DIR}/.venv/Scripts/activate.bat
        COMMAND
-          ${Protobuf_PROTOC_EXECUTABLE} ARGS --cpp_out=${CMAKE_BINARY_DIR}/src
-          --plugin=protoc-gen-service=${CMAKE_SOURCE_DIR}/src/lib/api/service_compiler/plugin.cmd
-          --service_out=lang=cpp:${CMAKE_BINARY_DIR}/src -I
-          ${PROTOBUF_INCLUDE_DIRS} -I ${CMAKE_SOURCE_DIR}/src ${ABS_FIL}
-        DEPENDS ${ABS_FIL} ${TEMPLATES}
-                ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/plugin.py
+          ${Protobuf_PROTOC_EXECUTABLE} ARGS --cpp_out=${_ARGS_CXX_OUT_DIR}
+          --plugin=protoc-gen-service=${_plugin_cmd}
+          --service_out=lang=cpp:${_ARGS_CXX_OUT_DIR} -I
+          ${PROTOBUF_INCLUDE_DIRS} -I ${_protobuf_src_dir} ${ABS_FIL}
+        DEPENDS ${ABS_FIL} ${TEMPLATES} ${_plugin_cmd} ${_plugin_py}
        COMMENT "Running C++ service compiler on ${FIL}"
        VERBATIM)
    else()
      add_custom_command(
-        OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc"
-               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h"
-               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.cc"
-               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.h"
-               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.cc"
-               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.h"
+        OUTPUT "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h"
        COMMAND ${CMAKE_SOURCE_DIR}/tools/venv_exec.sh
        ARGS
          ${Protobuf_PROTOC_EXECUTABLE}
-          --cpp_out=${CMAKE_BINARY_DIR}/src
-          --plugin=protoc-gen-service=${CMAKE_SOURCE_DIR}/src/lib/api/service_compiler/plugin.py
-          --service_out=lang=cpp,make_file_name=${ABS_FIL}:${CMAKE_BINARY_DIR}/src
-          -I ${PROTOBUF_INCLUDE_DIRS} -I ${CMAKE_SOURCE_DIR}/src ${ABS_FIL}
-        DEPENDS ${ABS_FIL} ${TEMPLATES}
-                ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/plugin.py
+          --cpp_out=${_ARGS_CXX_OUT_DIR}
+          --plugin=protoc-gen-service=${_plugin_py}
+          --service_out=lang=cpp,make_file_name=${ABS_FIL}:${_ARGS_CXX_OUT_DIR}
+          ${_proto_include_cli_args} ${ABS_FIL}
+        DEPENDS ${ABS_FIL} ${TEMPLATES} ${_plugin_py}
        COMMENT "Running C++ service compiler on ${FIL}"
        VERBATIM)
    endif()
@@ -123,22 +142,22 @@ function(Service_Compiler_Generate_Cpp SRCS HDRS)

  # remove some warnings from protobuf generated files
  if(WIN32)
-    set_source_files_properties(${${SRCS}} PROPERTIES COMPILE_FLAGS
+    set_source_files_properties(${${_ARGS_CXX_SOURCES}} PROPERTIES COMPILE_FLAGS
                                                      "/wd4309 /wd4838")
  else()
    set_source_files_properties(
-      ${${SRCS}}
+      ${${_ARGS_CXX_SOURCES}}
      PROPERTIES
        COMPILE_FLAGS
        "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
    )
  endif()

-  set(${SRCS}
-      ${${SRCS}}
+  set(${_ARGS_CXX_SOURCES}
+      ${${_ARGS_CXX_SOURCES}}
      PARENT_SCOPE)
-  set(${HDRS}
-      ${${HDRS}}
+  set(${_ARGS_CXX_HEADERS}
+      ${${_ARGS_CXX_HEADERS}}
      PARENT_SCOPE)

 endfunction()
@@ -0,0 +1,83 @@
+include_guard(GLOBAL)
+
+function(protobuf_generate_cpp)
+  set(_options)
+  set(_one_value_args
+    CXX_SOURCES
+    CXX_HEADERS
+    CXX_OUT_DIR
+    PROTOC
+  )
+  set(_multi_value_args
+    PROTO_INCLUDE_DIRS
+    PROTO_SOURCES
+  )
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+
+  if (NOT DEFINED _ARGS_PROTOC)
+    if (WIN32)
+      set(_ARGS_PROTOC "${EXTERNALS_BIN}/protoc.exe")
+    else()
+      set(_ARGS_PROTOC "${EXTERNALS_BIN}/protoc")
+    endif()
+  endif()
+  set(${_ARGS_CXX_SOURCES})
+  set(${_ARGS_CXX_HEADERS})
+
+  set(_proto_include_cli_args)
+  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
+    list(APPEND _proto_include_cli_args -I ${_dir})
+  endforeach()
+
+  foreach(_fil ${_ARGS_PROTO_SOURCES})
+    get_filename_component(_abs_fil ${_fil} ABSOLUTE)
+    get_filename_component(_fil_we ${_fil} NAME_WE)
+    get_filename_component(_fil_dir ${_fil} DIRECTORY)
+    if(_fil_dir)
+      set(_fil_we "${_fil_dir}/${_fil_we}")
+    endif()
+    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.cc")
+    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.h")
+    message(STATUS "add_custom_command for ${_fil} -> ${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.h")
+    # TODO: Pass proto sources by relative path (so this fn can be called from
+    # arbitrary subdirs).
+    add_custom_command(
+      OUTPUT
+        "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.cc"
+        "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.h"
+      COMMAND
+        ${_ARGS_PROTOC}
+      ARGS
+        --cpp_out=${_ARGS_CXX_OUT_DIR} ${_proto_include_cli_args} ${_abs_fil}
+      DEPENDS ${_abs_fil}
+      COMMENT "Running C++ protocol buffer compiler on ${_fil}"
+      VERBATIM
+    )
+  endforeach()
+
+  set_source_files_properties(
+    ${${_ARGS_CXX_SOURCES}}
+    ${${_ARGS_CXX_HEADERS}}
+    PROPERTIES
+      GENERATED TRUE
+  )
+
+  # remove some warnings from protobuf generated files
+  if(WIN32)
+    set_source_files_properties(
+      ${${_ARGS_CXX_SOURCES}}
+      PROPERTIES
+        COMPILE_FLAGS "/wd4309 /wd4838"
+    )
+  else()
+    set_source_files_properties(
+      ${${_ARGS_CXX_SOURCES}}
+      PROPERTIES
+        COMPILE_FLAGS "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
+    )
+  endif()
+
+  set(${_ARGS_CXX_SOURCES} ${${_ARGS_CXX_SOURCES}} PARENT_SCOPE)
+  set(${_ARGS_CXX_HEADERS} ${${_ARGS_CXX_HEADERS}} PARENT_SCOPE)
+endfunction()
+
@@ -0,0 +1,61 @@
+include_guard(GLOBAL)
+
+function(protobuf_generate_cpp_config)
+  set(_options)
+  set(_one_value_args
+    PROTO_SOURCES_HEADER
+    PROTO_DEFAULTS_HEADER
+  )
+  set(_multi_value_args
+    PROTO_SOURCES
+    PROTO_DEFAULTS
+  )
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+
+  # Configure the 'config_proto_files' header. It provides an array of strings populated with the
+  # files in PROTO_SOURCES. At runtime, we use this array to generate a special 'config' proto
+  # message with fields for each message defined in said files. A global instance of this 'config'
+  # proto is then constructed, and populated with [compile-time] defaults, and [runtime] overrides
+  # (i.e., in spark.conf). Calls to `bts::config::get<T>()` retrieve the value from the field
+  # associated with type `T` in this global instance.
+  message(STATUS "PROTO_SOURCES: ${_ARGS_PROTO_SOURCES}")
+  message(STATUS "PROTO_SOURCES_HEADER: ${_ARGS_PROTO_SOURCES_HEADER}")
+  message(STATUS "PROTO_DEFAULTS_HEADER: ${_ARGS_PROTO_DEFAULTS_HEADER}")
+  if (_ARGS_PROTO_SOURCES_HEADER)
+    set(PROTO_SOURCES)
+    set(PROTO_SOURCES_CXX_ARRAY_ITEMS "")
+    foreach(_proto_path ${_ARGS_PROTO_SOURCES})
+      list(APPEND PROTO_SOURCES ${_proto_path})
+      string(APPEND PROTO_SOURCES_CXX_ARRAY_ITEMS "\"${_proto_path}\",\n")
+    endforeach()
+    list(LENGTH PROTO_SOURCES PROTO_SOURCES_LENGTH)
+    message(STATUS "CONFIGURING PROTO_SOURCES_HEADER): ${_ARGS_PROTO_SOURCES_HEADER}")
+    configure_file(${CMAKE_SOURCE_DIR}/cmake/config_proto_files.h.in ${_ARGS_PROTO_SOURCES_HEADER})
+  endif()
+
+  # Generate static defaults. Ideally, this would be a single string, but we use an array because
+  # the compiler complains if the string is longer than 65k.
+  set(_cfg_defaults_str "inline const char * config_static_defaults[] = { ")
+
+  foreach(_cfg_file_path ${_ARGS_PROTO_DEFAULTS})
+    file(READ ${_cfg_file_path} _cfg_file_content)
+    string(REPLACE "\"" "\\\"" _cfg_file_content_escaped ${_cfg_file_content})
+    string(REPLACE "\n" ";" _cfg_lines ${_cfg_file_content_escaped})
+
+    foreach(_line ${_cfg_lines})
+      string(APPEND _cfg_defaults_str "\n  \"${_line}\",")
+    endforeach(_line)
+  endforeach(_cfg_file_path)
+
+  string(APPEND _cfg_defaults_str "};")
+  set(_old_cfg_defaults_str "")
+
+  if(EXISTS ${_ARGS_PROTO_DEFAULTS_HEADER})
+    file(READ ${_ARGS_PROTO_DEFAULTS_HEADER} _old_cfg_defaults_str)
+  endif()
+
+  if(NOT _cfg_defaults_str STREQUAL _old_cfg_defaults_str)
+    file(WRITE "${_ARGS_PROTO_DEFAULTS_HEADER}" "${_cfg_defaults_str}")
+  endif()
+endfunction()
+
@@ -10,7 +10,7 @@ function(protobuf_generate_fs)
    set(_ARGS_TARGET_NAME "${PROJECT_NAME}_generate_fs_protos")
  endif()
  if (NOT DEFINED _ARGS_PROTOC)
-    set(_ARGS_PROTOC "${SPARK_BIN}/protoc")
+    set(_ARGS_PROTOC "${EXTERNALS_BIN}/protoc")
  endif()

  set(_proto_include_cli_args)
@@ -27,8 +27,8 @@ function(protobuf_generate_fs)
  endforeach()

  set(_venv_exec ${CMAKE_SOURCE_DIR}/tools/venv_exec.sh)
-  set(_plugin_py ${CMAKE_SOURCE_DIR}/src/lib/api/service_compiler/plugin.py)
-  set(_template_fs ${CMAKE_SOURCE_DIR}/src/lib/api/service_compiler/templates/fs/fable.fs)
+  set(_plugin_py ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/plugin.py)
+  set(_template_fs ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/fs/fable.fs)
  add_custom_command(
    OUTPUT ${_outputs}
    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
@@ -0,0 +1,10 @@
+include_guard(GLOBAL)
+
+macro(set_first_defined RESULT)
+  foreach(_arg ${ARGN})
+    if (DEFINED ${_arg})
+      set(${RESULT} ${${_arg}})
+      break()
+    endif()
+  endforeach()
+endmacro()
@@ -0,0 +1,11 @@
+#pragma once
+
+#include <array>
+
+namespace bts::@PROJECT_NAME@::protobufs {
+
+inline std::array<const char *, @PROTO_SOURCES_LENGTH@> config_proto_files = {
+@PROTO_SOURCES_CXX_ARRAY_ITEMS@
+};
+
+} // namespace bts::@PROJECT_NAME@::protobufs
@@ -1,24 +0,0 @@
-cmake_minimum_required(VERSION 3.5)
-
-# Explicitly set policy to convert relative paths to absolute (although this is a default policy,
-# setting it explicitly prevents a compile-warning)
-cmake_policy(SET CMP0076 NEW)
-
-# add protos to the library to facilitate source_group
-target_sources(libspark PUBLIC ${Config_Protos} ${NonConfig_Protos} ${Service_Protos})
-
-function(assign_source_group)
-  foreach(_source IN ITEMS ${ARGN})
-    if(IS_ABSOLUTE "${_source}")
-      file(RELATIVE_PATH _source_rel "${CMAKE_CURRENT_SOURCE_DIR}" "${_source}")
-    else()
-      set(_source_rel "${_source}")
-    endif()
-    get_filename_component(_source_path "${_source_rel}" PATH)
-    string(REPLACE "/" "\\" _source_path_msvc "${_source_path}")
-    source_group("${_source_path_msvc}" FILES "${_source}")
-    set(SOURCE_FILES "${_source}")
-  endforeach()
-endfunction(assign_source_group)
-
-assign_source_group(${SparkLib_Sources} ${Config_Protos} ${NonConfig_Protos} ${Service_Protos})
@@ -3,7 +3,7 @@ services:
    volumes:
      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/sparkdata/data
    build:
-      dockerfile: src/server/sparkd/Dockerfile
+      dockerfile: src/app_sparkd/Dockerfile
      args:
        - IMAGE=${SPARKD__IMAGE}
        - SOURCE_DIR=${SPARKD__SOURCE_DIR}
@@ -124,8 +124,8 @@ service OrderService {
 ```
 ### C++ Example
 ```cpp
-#include "lib/api/public/ServiceContext.h"
-#include "lib/api/services/OrderService.client.pb.h"
+#include "spark/api/public/ServiceContext.h"
+#include "spark/api/services/OrderService.client.pb.h"
 using namespace bts::api;
 ServiceContext ctx;
 auto orders = ctx.locate<orderman::OrderService>();
@@ -29,9 +29,9 @@ namespace sample {
 ```

 ```cpp
-#include "lib/api/public/ServiceContext.h"
-#include "spark/lib/domain_model/Security.h"
-#include "lib/api/services/MarketDataService.client.pb.h"
+#include "spark/api/public/ServiceContext.h"
+#include "spark/domain_model/Security.h"
+#include "spark/api/services/MarketDataService.client.pb.h"
 #include <thread>

 using namespace bts::api;
@@ -19,8 +19,8 @@
  </ItemGroup>

  <ItemGroup>
-    <Proto Include="../../lib/config/protos/data_stream_config.proto" CSName="DataStreamConfig">
-      <OutDir>$(ProtoOutputPath)/lib/config/protos</OutDir>
+    <Proto Include="../../lib_spark/spark/config/protos/data_stream_config.proto" CSName="DataStreamConfig">
+      <OutDir>$(ProtoOutputPath)/lib_spark/spark/config/protos</OutDir>
    </Proto>
  </ItemGroup>

@@ -43,7 +43,7 @@
      Condition="Exists($(ComputerSpecificConfig))" />

  <PropertyGroup Label="ProtoBuf">
-    <ProtocPluginBase>$(SparkRoot)\src\lib\api\service_compiler</ProtocPluginBase>
+    <ProtocPluginBase>$(SparkRoot)\src\lib_spark\spark\api\service_compiler</ProtocPluginBase>
  </PropertyGroup>
  <Choose>
    <When Condition="'$(OS)' == 'Windows_NT'">
@@ -24,7 +24,7 @@
    <PropertyGroup>
      <GenProtosCmd>$(protoc) %(Proto.FullPath)</GenProtosCmd>
      <GenProtosCmd>$(GenProtosCmd) --error_format=msvs</GenProtosCmd>
-      <GenProtosCmd>$(GenProtosCmd) --proto_path=$(SourceRoot)</GenProtosCmd>
+      <GenProtosCmd>$(GenProtosCmd) --proto_path=$(SourceRoot)/lib_spark</GenProtosCmd>
      <GenProtosCmd>$(GenProtosCmd) --plugin=&quot;protoc-gen-spark-csharp=$(ProtocPlugin)&quot;</GenProtosCmd>
      <GenProtosCmd>$(GenProtosCmd) --spark-csharp_out=lang=c#,templates=immutable.pb.cs,strip_out_path=1:%(Proto.OutDir)</GenProtosCmd>
      <GenProtosCmd>$(GenProtosCmd) --csharp_out=%(Proto.OutDir)</GenProtosCmd>
@@ -65,7 +65,7 @@
    <PropertyGroup>
      <GenProtosCmd>$(protoc) %(ProtoService.FullPath)</GenProtosCmd>
      <GenProtosCmd>$(GenProtosCmd) --error_format=msvs</GenProtosCmd>
-      <GenProtosCmd>$(GenProtosCmd) --proto_path=$(SourceRoot)</GenProtosCmd>
+      <GenProtosCmd>$(GenProtosCmd) --proto_path=$(SourceRoot)/lib_spark</GenProtosCmd>
      <GenProtosCmd>$(GenProtosCmd) --plugin=&quot;protoc-gen-service=$(ProtocPlugin)&quot;</GenProtosCmd>
      <GenProtosCmd>$(GenProtosCmd) --service_out=lang=c#,strip_out_path=1:%(ProtoService.OutDir)</GenProtosCmd>
      <GenProtosCmd>$(GenProtosCmd) --csharp_out=%(ProtoService.OutDir)</GenProtosCmd>
@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 3.10)
+cmake_minimum_required(VERSION 3.5)
 project(Spark.Admin.WebApp)

 include(GNUInstallDirs)
@@ -10,11 +10,11 @@ include(${CMAKE_SOURCE_DIR}/cmake/ProtobufGenerateFs.cmake)
 file(
  GLOB_RECURSE ${PROJECT_NAME}_PROTO_SOURCES
  RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
-  ../lib/api/*.proto
-  ../lib/api/services/*.proto
-  ../lib/domain_model/*.proto
-  ../lib/sparkproto/*.proto
-  ../lib/md/protos/*.proto
+  ../lib_spark/spark/api/*.proto
+  ../lib_spark/spark/api/services/*.proto
+  ../lib_spark/spark/domain_model/*.proto
+  ../lib_spark/spark/sparkproto/*.proto
+  ../lib_spark/spark/md/protos/*.proto
  ../../externals/linux/include/google/protobuf/descriptor.proto
  ../../externals/linux/include/google/protobuf/timestamp.proto
 )
@@ -44,7 +44,7 @@ set(_build_admin_web_app_deps
  ${_build_admin_web_app_py}
  ${CMAKE_CURRENT_SOURCE_DIR}/src/Spark.Admin.WebApp.fsproj
  ${CMAKE_CURRENT_SOURCE_DIR}/webpack.config.js
-  ${CMAKE_SOURCE_DIR}/src/lib/api/service_compiler/templates/fs/fable.fs
+  ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/fs/fable.fs
  ${CMAKE_CURRENT_SOURCE_DIR}/package-lock.json
  ${${PROJECT_NAME}_PROTO_SOURCES}
  ${_api_ts_SOURCES}
@@ -73,7 +73,7 @@ add_custom_command(
 protobuf_generate_js(
  TARGET_NAME generate_js_protos
  PROTO_INCLUDE_DIRS
-    ..
+    ../lib_spark
    ../../externals/linux/include
  PROTO_SOURCES ${${PROJECT_NAME}_PROTO_SOURCES}
  OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/proto.js
@@ -82,7 +82,7 @@ protobuf_generate_js(
 protobuf_generate_fs(
  TARGET_NAME generate_fs_protos
  PROTO_INCLUDE_DIRS
-    ..
+    ../lib_spark
    ../../externals/linux/include
  PROTO_SOURCES ${${PROJECT_NAME}_PROTO_SOURCES}
  OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/fable
@@ -184,4 +184,3 @@ install(
  DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/public"
  DESTINATION "${CMAKE_INSTALL_BINDIR}/${PROJECT_NAME}"
 )
-
@@ -0,0 +1,186 @@
+cmake_minimum_required(VERSION 3.14)
+project(sparkd)
+
+include(FetchContent)
+include(${CMAKE_SOURCE_DIR}/cmake/safe_dep_glob.cmake)
+include(${CMAKE_SOURCE_DIR}/cmake/ProtobufGenerateCpp.cmake)
+include(${CMAKE_SOURCE_DIR}/cmake/InstallBasic.cmake)
+include(GNUInstallDirs)
+
+if (NOT DEFINED PROJECT_VERSION)
+  set(PROJECT_VERSION 1.0.0)
+endif()
+
+# dependencies
+file(
+  GLOB_RECURSE ${PROJECT_NAME}_PROTO_SOURCES
+  RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
+  *.proto
+)
+file(
+  GLOB_RECURSE ${PROJECT_NAME}_SOURCES
+  RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
+  *.cpp
+)
+update_deps_file("${${PROJECT_NAME}_PROTO_SOURCES};${${PROJECT_NAME}_SOURCES}")
+
+protobuf_generate_cpp(
+  CXX_SOURCES ${PROJECT_NAME}_PROTO_CXX_SOURCES
+  CXX_HEADERS ${PROJECT_NAME}_PROTO_CXX_HEADERS
+  CXX_OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/include
+  PROTO_INCLUDE_DIRS
+    ${EXTERNALS_INCLUDE}
+    ${CMAKE_SOURCE_DIR}/src/lib_spark
+    ${CMAKE_CURRENT_SOURCE_DIR}
+  PROTO_SOURCES
+    ${${PROJECT_NAME}_PROTO_SOURCES}
+)
+add_executable(sparkd ${${PROJECT_NAME}_SOURCES} ${${PROJECT_NAME}_PROTO_CXX_SOURCES})
+
+# include paths
+target_include_directories(
+  sparkd
+  PUBLIC
+    $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
+    $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>/include
+    $<BUILD_INTERFACE:${emumba-deribit-connector-library_INCLUDE_DIRS}>
+)
+
+find_package(PkgConfig REQUIRED)
+pkg_check_modules(emumba-deribit-connector-library REQUIRED emumba-deribit-connector)
+
+# This dir contains rapidjsonConfig.cmake
+list(APPEND CMAKE_PREFIX_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake_modules)
+find_package(rapidjson REQUIRED)
+target_include_directories(
+  ${PROJECT_NAME} SYSTEM
+  PUBLIC
+    $<BUILD_INTERFACE:${rapidjson_INCLUDE_DIR}> 
+)
+add_dependencies(${PROJECT_NAME} rapidjson_project)
+
+# databento
+FetchContent_Declare(
+  databento
+  GIT_REPOSITORY https://github.com/databento/databento-cpp
+  GIT_TAG v0.31.0
+)
+FetchContent_MakeAvailable(databento)
+
+# deribit-demo log level
+if(NOT DEFINED DERIBIT_DEMO_LOG_LEVEL)
+  add_definitions(-DSPDLOG_ACTIVE_LEVEL=0)
+  message("-- Building of DERIBIT_DEMO_LOG_LEVEL=TRACE")
+elseif(DERIBIT_DEMO_LOG_LEVEL STREQUAL "TRACE")
+  add_definitions(-DSPDLOG_ACTIVE_LEVEL=0)
+  message("-- Building of DERIBIT_DEMO_LOG_LEVEL=TRACE")
+elseif(DERIBIT_DEMO_LOG_LEVEL STREQUAL "DEBUG")
+  add_definitions(-DSPDLOG_ACTIVE_LEVEL=1)
+  message("-- Building of DERIBIT_DEMO_LOG_LEVEL=DEBUG")
+elseif(DERIBIT_DEMO_LOG_LEVEL STREQUAL "INFO")
+  add_definitions(-DSPDLOG_ACTIVE_LEVEL=2)
+  message("-- Building of DERIBIT_DEMO_LOG_LEVEL=INFO")
+elseif(DERIBIT_DEMO_LOG_LEVEL STREQUAL "WARN")
+  add_definitions(-DSPDLOG_ACTIVE_LEVEL=3)
+  message("-- Building of DERIBIT_DEMO_LOG_LEVEL=WARN")
+elseif(DERIBIT_DEMO_LOG_LEVEL STREQUAL "ERROR")
+  add_definitions(-DSPDLOG_ACTIVE_LEVEL=4)
+  message("-- Building of DERIBIT_DEMO_LOG_LEVEL=ERROR")
+elseif(DERIBIT_DEMO_LOG_LEVEL STREQUAL "CRITICAL")
+  add_definitions(-DSPDLOG_ACTIVE_LEVEL=5)
+  message("-- Building of DERIBIT_DEMO_LOG_LEVEL=CRITICAL")
+elseif(DERIBIT_DEMO_LOG_LEVEL STREQUAL "OFF")
+  add_definitions(-DSPDLOG_ACTIVE_LEVEL=6)
+  message("-- Building of DERIBIT_DEMO_LOG_LEVEL=OFF")
+else()
+  add_definitions(-DSPDLOG_ACTIVE_LEVEL=0)
+  message("-- Building of DERIBIT_DEMO_LOG_LEVEL=TRACE")
+endif()
+
+target_link_directories(
+  ${PROJECT_NAME} PUBLIC
+  $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
+  $<INSTALL_INTERFACE:${CMAKE_INSTALL_LIBDIR}>
+)
+
+# Set RPATH so sparkd knows where to find our shared libs (e.g., libspark).
+# The dirs are relative to sparkd's path, represented by "$ORIGIN".
+set_target_properties(
+  ${PROJECT_NAME} PROPERTIES
+  BUILD_RPATH "$ORIGIN/../lib_spark;$ORIGIN/../lib_python_md"
+  INSTALL_RPATH "$ORIGIN/../${CMAKE_INSTALL_LIBDIR}"
+)
+
+# Python interop (for python md providers)
+add_dependencies(${PROJECT_NAME}
+  python_md
+  python_md_module
+  python_md_providers
+  copy_python_md_providers
+)
+include(${CMAKE_SOURCE_DIR}/cmake/SymLink.cmake)
+
+sym_link_local_lib(python_md)
+sym_link_file(
+  SOURCE ${CMAKE_BINARY_DIR}/src/lib_python_md/python_md_module.cpython-310-x86_64-linux-gnu.so
+  DEST ${CMAKE_CURRENT_BINARY_DIR}/python_md_module.cpython-310-x86_64-linux-gnu.so
+  INSTALL_DEST ${CMAKE_INSTALL_BINDIR}/
+)
+get_target_property(providers python_md_providers INTERFACE_PYTHON_SCRIPTS_DIR)
+sym_link_dir(
+  TARGET_NAME link_python_md_providers
+  SOURCE ${providers}
+  DEST ${CMAKE_CURRENT_BINARY_DIR}/python_md_providers
+)
+
+target_link_libraries(
+  sparkd
+  spark
+  binance_execution
+  okx_execution
+  python_md
+  python_md_module
+  databento::databento
+  ${emumba-deribit-connector-library_LIBRARIES}
+  tbb
+  quickfix
+  benchmark
+  Common++
+  Packet++
+  Pcap++
+  lttng-ust
+  -lpcap
+  -static-libgcc
+  -static-libstdc++
+  -pthread)
+
+# Onixs
+target_link_libraries(
+  ${PROJECT_NAME}
+  onixs-cmemdh
+  OnixS.IceImpactMulticastPriceFeedHandlerCpp
+  # gcc doesn't resolve onixs-fix-engine and ACE in runtime path unless we specify
+  # --no-as-needed. These are onixs deps that aren't called directly by our code.
+  # Note: this isn't an issue for clang
+  -Wl,--no-as-needed
+  onixs-fix-engine
+  ACE
+)
+
+# Catch2
+find_package(Catch2 CONFIG REQUIRED)
+target_link_libraries(${PROJECT_NAME} Catch2::Catch2)
+
+install_basic()
+install(
+  FILES
+    "${EXTERNALS_LIB}/libonixs-cmemdh.so"
+    "${EXTERNALS_LIB}/libonixs-cmemdh.so.4"
+    "${EXTERNALS_LIB}/libonixs-cmemdh.so.4.8.0"
+    "${EXTERNALS_LIB}/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.16.0.0"
+    "${EXTERNALS_LIB}/libonixs-fix-engine.so"
+    "${EXTERNALS_LIB}/libonixs-fix-engine.so.4"
+    "${EXTERNALS_LIB}/libonixs-fix-engine.so.4.10.1"
+    "${EXTERNALS_LIB}/libACE-5.7.5.so"
+  DESTINATION ${CMAKE_INSTALL_LIBDIR}
+)
@@ -0,0 +1,5 @@
+@PACKAGE_INIT@
+
+include("${CMAKE_CURRENT_LIST_DIR}/@PROJECT_NAME@Targets.cmake")
+
+check_required_components(@PROJECT_NAME@)
@@ -0,0 +1,19 @@
+FROM bluetradingsystems/linux_dotnet7
+
+ARG SOURCE_DIR="install/sparkd-release-gcc"
+ARG CONF_FILE="src/app_sparkd/spark.conf"
+
+USER "root"
+RUN apt-get update \
+  && apt-get install -y liblttng-ust-dev
+
+RUN apt-get update \
+  && apt-get install -y libpcap0.8-dev
+
+RUN apt-get update \
+  && apt-get install -y python3.10-dev
+
+USER "bts"
+COPY --chown=bts:bts "${SOURCE_DIR}" "."
+COPY --chown=bts:bts "${CONF_FILE}" ".local/share/bts/sparkd/conf/spark.conf"
+WORKDIR "/home/bts/bin"
@@ -0,0 +1,4 @@
+SPARKD__CONFIG_DIR=src/app_sparkd/configurations/alphaworx_prod
+SPARKD__SHM_SIZE=8gb
+SPARKD__MEMORY=8gb
+SPARKD__RUNTIME_CONF_DIR=/home/ubuntu/8000/sparkdata/conf/
@@ -1,4 +1,4 @@
-SPARKD__CONFIG_DIR=src/server/sparkd/configurations/alphaworx_zen_hk_live
+SPARKD__CONFIG_DIR=src/app_sparkd/configurations/alphaworx_zen_hk_live
 SPARKD__SHM_SIZE=8gb
 SPARKD__MEMORY=8gb
 SPARKD__IMAGE=sparkd_symbols
@@ -0,0 +1,2 @@
+SPARKD__CONFIG_DIR=src/app_sparkd/configurations/binance_dev
+SPARKD__SHM_SIZE=8gb
@@ -0,0 +1 @@
+SPARKD__CONFIG_DIR=src/app_sparkd/configurations/py_dummy_md
@@ -0,0 +1,38 @@
+PublisherOptions {
+  value {
+    network_interface: "0.0.0.0"
+    heartbeat_ms: 5000
+    heartbeat_boot_seconds: 60
+    skip_options: true
+    hot_initial: 1024
+    hot_growth: 1024
+    cold_initial: 8192
+    cold_growth: 2048
+    channel_prefix: ""
+    disable_heartbeat: true
+  }
+}
+
+Logging {
+  value {
+    initial_queue_size: 120001
+    max_queue_growth: 4
+    file {
+      default_level: "Info"
+      subsystem {
+        name: "md"
+        level: "Diag"
+      }
+    }
+  }
+}
+
+MarketDataConfig {
+  value {
+    python_md_providers {
+      venue: VENUE_BYBIT
+      module_path: "python_md_providers"
+      provider_type: "bybit_md_provider.BybitMdProvider"
+    }
+  }
+}
@@ -0,0 +1,22 @@
+SymbolLists {
+  value {
+    list {
+      key: "Bybit"
+      symbol: "BTC"
+      symbol: "ETH"    
+      symbol: "BNB"
+      symbol: "XRP"
+      symbol: "ADA"
+      symbol: "PRIME"
+    }
+  }
+}
+
+Symbols {
+  value {
+    venues {
+      name: "VENUE_BYBIT"
+      spot: @SymbolLists.list[Bybit].symbol
+    }
+  }
+}
@@ -0,0 +1,2 @@
+SPARKD__CONFIG_DIR=src/app_sparkd/configurations/dev
+SPARKD__SHM_SIZE=8gb
@@ -0,0 +1 @@
+SPARKD__CONFIG_DIR=src/app_sparkd/configurations/py_dummy_md
@@ -0,0 +1,38 @@
+PublisherOptions {
+  value {
+    network_interface: "0.0.0.0"
+    heartbeat_ms: 5000
+    heartbeat_boot_seconds: 60
+    skip_options: true
+    hot_initial: 1024
+    hot_growth: 1024
+    cold_initial: 8192
+    cold_growth: 2048
+    channel_prefix: ""
+    disable_heartbeat: true
+  }
+}
+
+Logging {
+  value {
+    initial_queue_size: 120001
+    max_queue_growth: 4
+    file {
+      default_level: "Info"
+      subsystem {
+        name: "md"
+        level: "Diag"
+      }
+    }
+  }
+}
+
+MarketDataConfig {
+  value {
+    python_md_providers {
+      venue: VENUE_GATEIO
+      module_path: "python_md_providers"
+      provider_type: "gateio_md_provider.GateIoMdProvider"
+    }
+  }
+}
@@ -0,0 +1,47 @@
+  SymbolLists {
+  value {
+    list {
+      key: "Gateio"
+      symbol: "BTC"
+      symbol: "CAKE"      
+      symbol: "ETH"
+      symbol: "MANA"
+      symbol: "OMG"
+      symbol: "GMX"
+      symbol: "ICP"
+      symbol: "LRC"
+      symbol: "MINA"
+      symbol: "ZIL"
+      symbol: "TURBO"
+      symbol: "OP"
+      symbol: "ZRX"
+      symbol: "VET"
+      symbol: "AR"
+      symbol: "IMX"
+      symbol: "HIGH"
+      symbol: "ARB"
+      symbol: "YFI"
+      symbol: "NEAR"
+      symbol: "OXY"
+      symbol: "WAVES"
+      symbol: "AMP"
+      symbol: "CELO"
+      symbol: "CRV"
+      symbol: "FIL"
+      symbol: "PEOPLE"
+      symbol: "IOTX"
+      symbol: "ATOM"
+      symbol: "NOT"      
+      symbol: "GMT"            
+    }
+  }
+}
+
+Symbols {
+  value {
+    venues {
+      name: "VENUE_GATEIO"
+      spot: @SymbolLists.list[Gateio].symbol
+    }
+  }
+}
@@ -0,0 +1 @@
+SPARKD__CONFIG_DIR=src/app_sparkd/configurations/py_dummy_md
@@ -0,0 +1,38 @@
+PublisherOptions {
+  value {
+    network_interface: "0.0.0.0"
+    heartbeat_ms: 5000
+    heartbeat_boot_seconds: 60
+    skip_options: true
+    hot_initial: 1024
+    hot_growth: 1024
+    cold_initial: 8192
+    cold_growth: 2048
+    channel_prefix: ""
+    disable_heartbeat: true
+  }
+}
+
+Logging {
+  value {
+    initial_queue_size: 120001
+    max_queue_growth: 4
+    file {
+      default_level: "Info"
+      subsystem {
+        name: "md"
+        level: "Diag"
+      }
+    }
+  }
+}
+
+MarketDataConfig {
+  value {
+    python_md_providers {
+      venue: VENUE_GLBX
+      module_path: "python_md_providers"
+      provider_type: "dummy_md_provider.DummyMdProvider"
+    }
+  }
+}
@@ -0,0 +1,17 @@
+SymbolLists {
+  value {
+    list {
+      key: "Glbx"
+      symbol: "ES"
+    }
+  }
+}
+
+Symbols {
+  value {
+    venues {
+      name: "VENUE_GLBX"
+      future: @SymbolLists.list[Glbx].symbol
+    }
+  }
+}
@@ -0,0 +1,2 @@
+SPARKD__CONFIG_DIR=src/app_sparkd/configurations/sim_dev
+SPARKD__SHM_SIZE=8gb
@@ -1,5 +1,5 @@
 #include "AlgoDefinitionServiceServer.h"
-#include "lib/utils/PBUtils.h"
+#include "spark/utils/PBUtils.h"

 namespace bts {
 namespace api {
@@ -1,9 +1,17 @@
 #pragma once

-#include "lib/api/public/ServiceContext.h"
-#include "lib/api/services/AlgoDefinitionService.server.pb.h"
-#include "lib/persistence/IStore.h"
-#include "lib/persistence/SqliteStore.h"
+#include "spark/api/public/ServiceContext.h"
+#include "spark/api/services/AlgoDefinitionService.server.pb.h"
+#include "spark/api/services/OrderService.client.pb.h"
+#include "spark/api/services/OrderService.pb.h"
+#include "spark/logging/logging.h"
+#include "spark/persistence/IStore.h"
+#include "spark/persistence/SqliteStore.h"
+#include "spark/utils/PBUtils.h"
+#include "spark/utils/TimeStamp.h"
+#include "spark/utils/bts_assert.h"
+#include "spark/utils/bts_no_copy.h"
+#include <memory>

 #include <boost/tokenizer.hpp>
 #include <fmt/format.h>
@@ -1,14 +1,14 @@
 #include "AnalyticsServiceServer.h"
-#include "lib/api/apicore.pb.h"
-#include "lib/api/services/AnalyticsService.pb.h"
-#include "lib/api/services/AnalyticsService.server.pb.h"
-#include "lib/logging/logging.h"
-#include "lib/persistence/IStore.h"
-#include "lib/persistence/SqliteStore.h"
-#include "lib/utils/PBUtils.h"
-#include "lib/utils/TimeStamp.h"
-#include "lib/utils/bts_assert.h"
-#include "server/sparkd/algo/AlgoDefinitionServiceServer.h"
+#include "spark/api/apicore.pb.h"
+#include "spark/api/services/AnalyticsService.pb.h"
+#include "spark/api/services/AnalyticsService.server.pb.h"
+#include "spark/logging/logging.h"
+#include "spark/persistence/IStore.h"
+#include "spark/persistence/SqliteStore.h"
+#include "spark/utils/PBUtils.h"
+#include "spark/utils/TimeStamp.h"
+#include "spark/utils/bts_assert.h"
+#include "sparkd/algo/AlgoDefinitionServiceServer.h"

 #include <boost/algorithm/string.hpp>
 #include <boost/tokenizer.hpp>
@@ -1,6 +1,7 @@
 #pragma once

-#include "lib/api/services/AnalyticsService.server.pb.h"
+#include "spark/api/services/AnalyticsService.server.pb.h"
+#include "spark/utils/bts_no_copy.h"

 namespace bts {
 namespace api {
@@ -1,13 +1,13 @@
 #include "AdminReport.h"

-#include "lib/api/public/ServiceContext.h"
-#include "lib/config/config.h"
-#include "lib/config/protos/logging.pb.h"
-#include "lib/logging/logging.h"
-#include "lib/quickbus/quickbus.h"
+#include "spark/api/public/ServiceContext.h"
+#include "spark/config/config.h"
+#include "spark/config/protos/logging.pb.h"
+#include "spark/logging/logging.h"
+#include "spark/quickbus/quickbus.h"

-#include "lib/api/service_helpers/AdminServiceUtils.h"
-#include "lib/api/services/AdminService.client.pb.h"
+#include "spark/api/service_helpers/AdminServiceUtils.h"
+#include "spark/api/services/AdminService.client.pb.h"

 #include <algorithm>
 #include <boost/algorithm/string.hpp>
@@ -1,12 +1,12 @@
 #include "AutoCert.h"
-#include "lib/utils/PBUtils.h"
+#include "spark/utils/PBUtils.h"

-#include "lib/jetfix/common/InMsgBuffer.h"
-#include "lib/utils/bts_chrono.h"
-#include "lib/utils/datetime.h"
+#include "spark/jetfix/common/InMsgBuffer.h"
+#include "spark/utils/bts_chrono.h"
+#include "spark/utils/datetime.h"

-#include "lib/jetfix/codec/i32_toa.h"
-#include "lib/utils/SecureLogon.h"
+#include "spark/jetfix/codec/i32_toa.h"
+#include "spark/utils/SecureLogon.h"
 #include <boost/array.hpp>
 #include <boost/asio.hpp>
 #include <boost/filesystem.hpp>
@@ -1,13 +1,13 @@
 #include "CMECert.h"
-#include "lib/logging/logging.h"
-#include "lib/orderman/ilink3/IL3Dispatcher.h"
-#include "lib/orderman/ilink3/IL3GW.h"
-#include "lib/orderman/ilink3/IL3Reactor.h"
-#include "lib/orderman/ilink3/ILink3Utils.h"
-#include "lib/os/os.h"
-#include "lib/utils/Dispatcher.h"
-#include "lib/utils/SecureLogon.h"
-#include "lib/utils/TimeStamp.h"
+#include "spark/logging/logging.h"
+#include "spark/orderman/ilink3/IL3Dispatcher.h"
+#include "spark/orderman/ilink3/IL3GW.h"
+#include "spark/orderman/ilink3/IL3Reactor.h"
+#include "spark/orderman/ilink3/ILink3Utils.h"
+#include "spark/os/os.h"
+#include "spark/utils/Dispatcher.h"
+#include "spark/utils/SecureLogon.h"
+#include "spark/utils/TimeStamp.h"
 #include <chrono>
 #include <fstream>
 #include <future>
@@ -1,6 +1,6 @@
 #include "ConfigCmd.h"
-#include "lib/config/ConfigQuerier.h"
-#include "lib/config/config.h"
+#include "spark/config/ConfigQuerier.h"
+#include "spark/config/config.h"

 namespace bts::cmds {

@@ -1,5 +1,5 @@
-#include "server/sparkd/cmds/FIXSim.h"
-#include "server/sparkd/exchange_sim/ExchangeSim.h"
+#include "sparkd/cmds/FIXSim.h"
+#include "sparkd/exchange_sim/ExchangeSim.h"

 namespace bts::cmds {

@@ -1,18 +1,18 @@
 #include "KeyCmd.h"

-#include "lib/config/config.h"
-#include "lib/config/protos/keycode.pb.h"
-#include "lib/logging/logging.h"
-#include "lib/os/os.h"
-#include "lib/utils/AuthToken.h"
-#include "lib/utils/FileUtils.h"
-#include "lib/utils/bts_assert.h"
-#include "lib/utils/crypto/CryptoInit.h"
-#include "lib/utils/crypto/KeyCodeHelper.h"
-#include "lib/utils/crypto/PrivateKey.h"
-#include "lib/utils/crypto/PublicKey.h"
-#include "lib/utils/datetime.h"
-#include "lib/utils/helper_macros.h"
+#include "spark/config/config.h"
+#include "spark/config/protos/keycode.pb.h"
+#include "spark/logging/logging.h"
+#include "spark/os/os.h"
+#include "spark/utils/AuthToken.h"
+#include "spark/utils/FileUtils.h"
+#include "spark/utils/bts_assert.h"
+#include "spark/utils/crypto/CryptoInit.h"
+#include "spark/utils/crypto/KeyCodeHelper.h"
+#include "spark/utils/crypto/PrivateKey.h"
+#include "spark/utils/crypto/PublicKey.h"
+#include "spark/utils/datetime.h"
+#include "spark/utils/helper_macros.h"

 #include <boost/algorithm/string/replace.hpp>
 #include <boost/date_time/gregorian/gregorian.hpp>
@@ -1,10 +1,10 @@
 #include "ParseHistoricalFIX.h"
-#include "lib/jetfix/codec/FIXMsg.h"
-#include "lib/jetfix/codec/FIXParser.h"
-#include "lib/jetfix/codec/Fields.h"
-#include "lib/jetfix/codec/Group.h"
-#include "lib/jetfix/codec/MessageTypes.h"
-#include "lib/utils/bts_chrono.h"
+#include "spark/jetfix/codec/FIXMsg.h"
+#include "spark/jetfix/codec/FIXParser.h"
+#include "spark/jetfix/codec/Fields.h"
+#include "spark/jetfix/codec/Group.h"
+#include "spark/jetfix/codec/MessageTypes.h"
+#include "spark/utils/bts_chrono.h"

 #include <fstream>
 #include <unordered_map>
@@ -1,7 +1,7 @@
-#include "server/sparkd/cmds/PrintHedgeOrders.h"
-#include "lib/api/services/OrderService.pb.h"
-#include "lib/persistence/SqliteStore.h"
-#include "lib/utils/PBUtils.h"
+#include "PrintHedgeOrders.h"
+#include "spark/api/services/OrderService.pb.h"
+#include "spark/persistence/SqliteStore.h"
+#include "spark/utils/PBUtils.h"

 namespace bts::cmds {

@@ -1,5 +1,5 @@
 #include "QuickTest.h"
-#include "lib/logging/logging.h"
+#include "spark/logging/logging.h"

 #include <thread>

@@ -0,0 +1,159 @@
+#include "Run.h"
+#include "spark/api/public/ServiceContext.h"
+#include "spark/config/config.h"
+#include "spark/config/protos/ports.pb.h"
+#include "spark/config/protos/quickbus_cfg.pb.h"
+#include "spark/io/http/curl_session.h"
+#include "spark/logging/logging.h"
+#include "spark/md/MBOProcessor.h"
+#include "spark/md/MBOServiceServer.h"
+#include "spark/md/MDClient.h"
+#include "spark/md/SyntheticSecurityService.h"
+#include "spark/md/relay/QueryServer.h"
+#include "spark/md/service/MarketDataServiceServer.h"
+#include "spark/metrics/metrics.h"
+#include "spark/metrics/system_metrics.h"
+#include "spark/orderman/OrderServiceServer.h"
+#include "spark/orderman/events/TradeSummarySignalDispatcher.h"
+#include "spark/orderman/order_factory/DynamicOrderRegistry.h"
+#include "spark/os/os.h"
+#include "spark/os/spark_data_directory.h"
+#include "spark/persistence/PersistenceServiceServer.h"
+#include "spark/plugin/SparkServices.h"
+#include "spark/plugin/plugin.h"
+#include "spark/quickbus/core/Quickbus.h"
+#include "spark/tracing/PerfLog.h"
+#include "spark/utils/Dispatcher.h"
+#include "spark/version.h"
+#include "sparkd/algo/AlgoDefinitionServiceServer.h"
+#include "sparkd/analytics/AnalyticsServiceServer.h"
+#include "sparkd/cmds/md/cmds/Run.h"
+#include "sparkd/execution/execution_gateway_provider.h"
+#include "sparkd/http/WebServer.h"
+#include "sparkd/md/MarketDataAuxiliaryServiceServer.h"
+#include "sparkd/md/recording/MDRecorder.h"
+#include "sparkd/position/MarginServiceServer.h"
+#include "sparkd/position/PositionServiceServer.h"
+#include "sparkd/position/import/BasePositionServiceServer.h"
+#include "sparkd/position/import/MoneyFileServiceServer.h"
+#include "sparkd/subprocess/SubProcessRunner.h"
+#include "sparkd/support/ErrorReportService.h"
+#include "sparkd/support/ShellLatencyServer.h"
+#include "sparkd/user/AdminServiceServer.h"
+#include "sparkd/user/AuthServiceServer.h"
+
+namespace bts::cmds {
+
+using bts::api::ServiceContext;
+using bts::md::MDClient;
+using bts::md::TradeSummarySignalDispatcher;
+using bts::metrics::MetricsContainer;
+using bts::orderman::DynamicOrderRegistry;
+using bts::orderman::ExecutionGatewayProvider;
+using bts::orderman::IExecutionGatewayProvider;
+using bts::orderman::OrderServiceServer;
+using bts::trace::PerfLog;
+using std::unique_ptr;
+using std::vector;
+
+static OrderServiceServer create_order_service_server(
+    ServiceContext &sc,
+    MDClient &mdclient,
+    DynamicOrderRegistry &dyn_order_registry,
+    MetricsContainer &stats,
+    PerfLog &perf_log,
+    TradeSummarySignalDispatcher &tss_dispatcher) {
+   vector<unique_ptr<IExecutionGatewayProvider>> gw_pros;
+   gw_pros.push_back(std::make_unique<ExecutionGatewayProvider>(
+       &perf_log, mdclient.oracle(), config::get<config::JetFIX>(),
+       [](const std::string &name) {
+          const spark::SparkDataDirectory data_dir;
+          return data_dir.load_config_json(name);
+       }));
+   return OrderServiceServer(
+       sc, mdclient, dyn_order_registry, std::move(gw_pros), stats, perf_log,
+       tss_dispatcher);
+}
+
+static void execute() {
+   using namespace bts;
+   LOG(Gen, Warn,
+       "------------------------------------ Sparkd " BTS_SPARK_VERSION_FULL
+       " launched "
+       "--------------------------------");
+   // Initializes curl for the application, and eventual cleanup in destructor
+   http::CurlSession global_curl_session;
+   auto qb_cfg = config::get<config::QuickbusCfg>().node();
+   auto &ports = config::get<config::Ports>();
+   qb_cfg.set_my_port(ports.qbus_port());
+   qbus::Quickbus quickbus(qb_cfg, qbus::QuickbusThreadOptions::DEFERRED_START);
+   api::ServiceContext sc(&quickbus);
+
+   trace::PerfLog perf_log;
+   metrics::MetricsContainer stats;
+   metrics::publish_system_metrics(stats);
+   http::SparkWebServer webserver(sc, stats, perf_log, ports.config_port());
+
+   persistence::PersistenceServiceServer settings_saver(sc, "settings");
+   user::AuthServiceServer login_server(sc);
+
+   bts::md::MBOProcessor mbo_processor;
+   bts::md::TradeSummarySignalDispatcher tss_dispatcher;
+   md::cmds::Run::run();
+
+   api::synthetic::SyntheticSecurityService synthetic_service(sc);
+   bts::md::MDClient mdclient(
+       &utils::Dispatcher::background(), &synthetic_service,
+       &mbo_processor.reader());
+
+   bts::md::QueryServerPtr md_query_server(
+       bts::md::run_query_server(sc, mdclient, stats));
+
+   bts::md::MBOServiceServer mboserver(
+       sc, utils::Dispatcher::background(), mdclient);
+   bts::md::MarketDataService mdserver(mdclient, sc, stats);
+
+   // initialize order plugins
+   orderman::DynamicOrderRegistry dyn_order_registry;
+   SparkServices plugin_services(dyn_order_registry);
+   plugin::init_plugins(plugin_services);
+   // finished initializing order plugins
+
+   auto order_service_server = create_order_service_server(
+       sc, mdclient, dyn_order_registry, stats, perf_log, tss_dispatcher);
+   support::ErrorReportServer error_report_service(sc);
+   admin::AdminServiceServer admin_service(sc);
+   bts::md::auxiliary::MarketDataAuxiliaryServiceServer md_aux_server(sc);
+   bts::api::orderman::algo::AlgoDefinitionServiceServer algo_definition_server(
+       sc);
+   bts::analytics::AnalyticsServiceServer analytics_server(sc);
+   position::BasePositionServiceServer base_pos_server(sc);
+   position::MoneyFileServiceServer money_file_server(sc);
+   position::PositionServiceServer position_service(sc, mdclient);
+   position::MarginServiceServer margin_service(sc, mdclient);
+   metrics::ShellLatencyServer shell_latency_service(
+       sc, stats, utils::Dispatcher::background());
+   bts::md::recording::MDRecorder::start_recording(mdclient);
+
+   auto bg_health =
+       std::make_shared<metrics::IntCounterMetric>("spark_background_thread");
+   utils::DispatcherTimer timer(utils::Dispatcher::background(), 5000, [=]() {
+      bg_health->increment();
+   });
+   timer.start();
+
+   utils::DispatcherTimer timer2(utils::Dispatcher::background(), 1000, [&]() {
+      perf_log.store_log_entries();
+   });
+   timer2.start();
+   stats.publish(bg_health);
+   subprocess::run_all();
+
+   quickbus.run();
+}
+
+Run::Run() : CLI::App("Run default sparkd services", "run") {
+   callback([]() { execute(); });
+}
+
+} // namespace bts::cmds
@@ -1,17 +1,17 @@
 #include "Span.h"

-#include "server/sparkd/position/MarginServiceCore.h"
-#include "server/sparkd/position/Position.h"
-#include "server/sparkd/position/PositionAggregator.h"
+#include "sparkd/position/MarginServiceCore.h"
+#include "sparkd/position/Position.h"
+#include "sparkd/position/PositionAggregator.h"

-#include "lib/orderman/orderman_utils/orderman_common.h"
+#include "spark/orderman/orderman_utils/orderman_common.h"

-#include "lib/api/public/ServiceContext.h"
-#include "lib/api/services/AuthService.client.pb.h"
-#include "lib/config/config.h"
-#include "lib/domain_model/Security.pb.h"
-#include "lib/utils/Dispatcher.h"
-#include "lib/utils/helper_macros.h"
+#include "spark/api/public/ServiceContext.h"
+#include "spark/api/services/AuthService.client.pb.h"
+#include "spark/config/config.h"
+#include "spark/domain_model/Security.pb.h"
+#include "spark/utils/Dispatcher.h"
+#include "spark/utils/helper_macros.h"

 #include <condition_variable>
 #include <future>
@@ -1,28 +1,27 @@
 #include "Web.h"
-#include "lib/api/public/ServiceContext.h"
-#include "lib/api/services/MarketDataService.pb.h"
-#include "lib/api/services/SyntheticSecurityService.pb.h"
-#include "lib/config/config.h"
-#include "lib/config/protos/market_data.pb.h"
-#include "lib/config/protos/ports.pb.h"
-#include "lib/config/protos/quickbus_cfg.pb.h"
-#include "lib/md/MBOProcessor.h"
-#include "lib/md/MDClient.h"
-#include "lib/md/SyntheticSecurityService.h"
-#include "lib/md/relay/QueryServer.h"
-#include "lib/metrics/metrics.h"
-#include "lib/metrics/system_metrics.h"
-#include "lib/quickbus/quickbus.h"
-#include "lib/tracing/PerfLog.h"
-#include "lib/utils/Dispatcher.h"
-#include "lib/utils/thread_manager.h"
+#include "spark/api/public/ServiceContext.h"
+#include "spark/api/services/MarketDataService.pb.h"
+#include "spark/api/services/SyntheticSecurityService.pb.h"
+#include "spark/config/config.h"
+#include "spark/config/protos/market_data.pb.h"
+#include "spark/config/protos/ports.pb.h"
+#include "spark/config/protos/quickbus_cfg.pb.h"
+#include "spark/md/MBOProcessor.h"
+#include "spark/md/MDClient.h"
+#include "spark/md/SyntheticSecurityService.h"
+#include "spark/md/relay/QueryServer.h"
+#include "spark/metrics/metrics.h"
+#include "spark/metrics/system_metrics.h"
+#include "spark/quickbus/quickbus.h"
+#include "spark/tracing/PerfLog.h"
+#include "spark/utils/Dispatcher.h"
+#include "spark/utils/thread_manager.h"
 #include "md/cmds/Cleanup.h"
 #include "md/cmds/Run.h"
-#include "server/sparkd/exchange_sim/ExchangeSim.h"
-#include "server/sparkd/http/WebServer.h"
-#include "server/sparkd/md/platforms/mdp3_onixs/TradeSummarySignalDispatcher.h"
-#include "server/sparkd/user/AdminServiceServer.h"
-#include "server/sparkd/user/AuthServiceServer.h"
+#include "sparkd/exchange_sim/ExchangeSim.h"
+#include "sparkd/http/WebServer.h"
+#include "sparkd/user/AdminServiceServer.h"
+#include "sparkd/user/AuthServiceServer.h"
 #include <optional>
 #include <string>
 #include <vector>
@@ -1,5 +1,5 @@
 #include "Validators.h"
-#include "lib/utils/FileUtils.h"
+#include "spark/utils/FileUtils.h"
 #include <fmt/format.h>

 namespace bts::cmds {
Author	SHA1	Message	Date
Adeel Khan	97b5ec49b4	CLEANUP: BYBIT Symbols.conf.	2025-05-26 11:38:19 +00:00
Adeel Khan	8aaadf1253	ADD: BYBIT MD Working, NaN's also Added.	2025-05-26 11:01:20 +00:00
Adeel Khan	13aa94863d	ADD: BYBIT Securities showing up on frontend. Default Contract Spec	2025-05-22 16:17:19 +00:00
Adeel Khan	29c951fbca	ADD: GATEIO md_provider with dynamic contract specs using Batch REST call	2025-05-22 16:17:19 +00:00
Adeel Khan	ae776cf75f	ADD: GATEIO md_provider with dynamic contract specs 1 REST call per Contract	2025-05-22 16:17:19 +00:00
Adeel Khan	f353a8e9fd	ADD: GATEIO md_provider with default contract specs	2025-05-22 16:17:19 +00:00
John Colvin	24f4c2bb27	Add gateio_dev config profile for Adeel	2025-05-21 04:15:30 +00:00
John Colvin	2960f19e60	cleanup: Remove unused spark.conf file Originally, this spark.conf was hardcoded into compose.yaml, and copied into the sparkd container on each deploy. It hasn't been used since config profiles were introduced.	2025-05-21 04:15:30 +00:00
John Colvin	653d96675b	Add py_dummy_md config profile to test py md plugin	2025-05-21 04:15:30 +00:00
John Colvin	ae2aacdb1c	deploy: Move configurations to new stutter dir "app_sparkd"	2025-05-21 04:15:30 +00:00
John Colvin	11a741eea5	Add protos for gateio	2025-05-21 04:15:30 +00:00
John Colvin	56d0217aad	Add protos for bybit	2025-05-21 04:15:30 +00:00
John Colvin	992b922d68	build: Move onix fix adapter to its own lib We should keep order execution dependencies out of lib_spark.	2025-05-21 04:14:12 +00:00
John Colvin	b1103d5edb	fix build: Move OnixsUtils to from lib_spark to app_sparkd They're only needed MDP3Main.cpp and ICEMain.cpp. We're one step closer to removing lib_spark's dependency on onixs.	2025-05-21 04:14:12 +00:00
John Colvin	1289992127	cleanup: Remove unnecessary onixs include from TrafixOrderStateHandler.h	2025-05-21 04:14:12 +00:00
John Colvin	239548774a	feat: Support python md plugins	2025-05-21 04:14:12 +00:00
John Colvin	d0b0ca42ab	Add "clear" fn to Book & BookSide Prelude to python market data bindings	2025-05-21 04:14:12 +00:00
John Colvin	51882570bd	Add "smart" parse fns for a couple proto enums Prelude to python market data bindings. Unlike the normal parse fns, the "smart" variants aren't case sensitive, and can match strings with or without TYPE_NAME_ prefixes.	2025-05-21 04:14:12 +00:00
John Colvin	3ad95cf0ae	build: Move okx execution to its own lib	2025-05-21 04:14:12 +00:00
John Colvin	02bcbb900a	cleanup: Remove unnecessary onixs include from IDGenerator.h	2025-05-21 04:14:12 +00:00
John Colvin	d8efc38eeb	cleanup: Remove unnecessary includes from MDP3Listener.h	2025-05-21 04:14:12 +00:00
John Colvin	a51143c94b	refactor: Organize libspark proj w/ stuttering include This new structure provides several benefits: * Allows us to install multiple CMake projects in the same destination dir (without include path conflicts). - This makes shipping the trading algo sdk a trivial excercise. Simply copy the lib and include dirs from the install dest. That's it! * Puts us in a position to break up the monolithic libspark into smaller modules, which are easier to manage. Windows specific notes: * Get rid of unnecessary script vs_filters.cmake * Define WebsocketConnectionGateway directly in cs (instead of proto) - The proto would no longer be in the include path passed to protoc. Moving it to c# because there wasn't a compelling reason for it to be a proto in the first place. * Set spark output prefix to "lib" on windows - So we get "libspark.dll" instead of "spark.dll", which conflicts with the csharp lib, "Spark.dll". * Copy libspark.dll to net8.0 & net8.0-windows in a single cmake build target fix build: Copy libspark to both dirs net8.0, net8.0-windows	2025-05-21 04:14:09 +00:00
John Colvin	5e401d49ac	cleanup: Move TradeSummarySignalDispatcher to libspark It is needed by TradeSummarySignal, which is in libspark. This prevents a circular dependency from libspark -> sparkd.	2025-05-20 20:20:07 +00:00
John Colvin	2761a52636	build: Clean up protobuf cmake scripts Make them more flexible, reusable. This is a precursor to porting over python md provider code from another branch.	2025-05-20 20:20:07 +00:00
John Colvin	b2f6421b1e	build: Rename external path vars w/ "EXTERNAL_" prefix in cmake The old "SPARK_" prefix felt like a misnomer.	2025-05-20 20:20:07 +00:00
John Colvin	236ab31fbc	Add a few arithmetic ops to TimeSpan	2025-05-20 20:20:07 +00:00
John Colvin	83b77c9fc0	diag: Log time it takes to start md sources So we can detect bottlenecks	2025-05-20 20:20:07 +00:00
John Colvin	9930b65ddc	refactor: Split MarketDataGateway up into providers/sources MarketDataGateway still exists, but is greatly simplified. It creates several `IMarketDataProvider`s, each of which generates a list of `IMarketDataSource`s. The market data sources provide market data to an `IMarketDataListener`. * EmumbaMarketDataProvider * CCApiMarketDataProvider * PyMarketDataProvider * OnixsMarketDataProvider * ReplayMarketDataProvider As a side effect, individual py market data sources are now explicitly limited to a single venue (it probably should have been this way all along, since the shared memory writers do not expect updates to involve multiple venues).	2025-05-20 20:20:07 +00:00
John Colvin	33093f3497	cleanup: Remove dependency on quickfix for string to_lower We shouldn't depend on external libs for trivial things like converting strings to lower case. Dependencies come at a cost. They add baggage in the form of internal dependencies and increased build time.	2025-05-20 20:20:07 +00:00
				`@@ -0,0 +1 @@`
				`SPARKD__CONFIG_DIR=src/app_sparkd/configurations/py_dummy_md`