build: Clean up client CMake

webapp check pt?
cleanup: Remove empty header FTXSecurity.h
2024-09-03 12:39:02 -04:00 · 2024-09-01 20:07:31 +00:00 · 2024-09-01 20:07:31 +00:00 · 2024-09-01 20:07:31 +00:00 · 2024-09-01 20:07:31 +00:00 · 2024-09-01 20:07:31 +00:00
530 changed files with 20611 additions and 3794 deletions
@@ -203,12 +203,12 @@
      "preLaunchTask": "Build Sparkd (Debug | Clang)",
      "program": "",
      "linux": {
-        "program": "${workspaceFolder}/build/sparkd-debug-clang/server/sparkd",
+        "program": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd/sparkd",
      },
      "args": [
        "run"
      ],
-      "cwd": "${workspaceFolder}/build/sparkd-debug-clang/server",
+      "cwd": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd",
      "stopAtEntry": false,
      "externalConsole": false,
      "MIMode": "gdb",
@@ -231,12 +231,12 @@
      "request": "launch",
      "program": "",
      "linux": {
-        "program": "${workspaceFolder}/build/sparkd-debug-clang/server/sparkd",
+        "program": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd/sparkd",
      },
      "args": [
-        "run"
+        "test-py-md-provider"
      ],
-      "cwd": "${workspaceFolder}/build/sparkd-debug-clang/server",
+      "cwd": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd",
      "stopAtEntry": false,
      "externalConsole": false,
      "MIMode": "gdb",
@@ -252,7 +252,7 @@
          "ignoreFailures": true
        }
      ]
-    }
+    },
  ],
  "inputs": [
    {
@@ -1,5 +1,5 @@
 cmake_minimum_required(VERSION 3.12)
-project(spark)
+project(spark_root)
 set(CMAKE_CXX_STANDARD 20)

 if(WIN32)
@@ -18,7 +18,7 @@ set(
 # SHARED_MEMORY_PREFIX serves as a kind of namespace so multiple applications
 # can run simultaneously without fighting over the same shm files (e.g., sparkd,
 # api server, etc.).
-set(SHARED_MEMORY_PREFIX "bts" CACHE STRING "Prefix to use when creating shared memory files")
+set(SHARED_MEMORY_PREFIX "\${BTS_SHARED_MEMORY_PREFIX}" CACHE STRING "Prefix to use when creating shared memory files")
 set(EXTERNALS_BIN "${EXTERNALS_ROOT}/bin")
 set(EXTERNALS_INCLUDE "${EXTERNALS_ROOT}/include")
 set(EXTERNALS_LIB "${EXTERNALS_ROOT}/lib/$<CONFIG>")
@@ -45,11 +45,6 @@ if(WIN32)
  # subdirs
  add_subdirectory(src/lib_spark)
 else()
-  # make sparkd look for libspark.so in its directory
-  # set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE)
-  # set(CMAKE_INSTALL_RPATH ".")
-  # set(CMAKE_INSTALL_PREFIX "${CMAKE_SOURCE_DIR}/install/$<CONFIG>")
-
  if(BUILD_SPARKD)
    # Note: All macros that begin w/ 'CCAPI' are ccapi's macros (defined and/or
    # read by their code). We override some below.
@@ -109,10 +104,11 @@ else()

  # subdirs
  if (BUILD_SPARKD)
+    add_subdirectory(src/lib_system)
+    add_subdirectory(src/lib_domain)
+    add_subdirectory(src/lib_python_md)
    add_subdirectory(src/app_sparkd)
    add_subdirectory(src/data_files)
-    add_subdirectory(src/lib_py_market_data)
-    # add_subdirectory(src/py_market_data_providers)
  endif()
  add_subdirectory(src/lib_spark)
  if (BUILD_ADMIN_WEB_APP)
@@ -294,6 +294,14 @@
      "inherits": "sparkd"
    },

+    {
+      "name": "sparkd-debug-clang-j6",
+      "configurePreset": "sparkd-debug-clang",
+      "displayName": "Sparkd (Debug | Clang | jobs=6)",
+      "inherits": "sparkd",
+      "jobs": 6
+    },
+
    {
      "name": "sparkd-debug-gcc",
      "configurePreset": "sparkd-debug-gcc",
@@ -0,0 +1,5 @@
+macro(export_to_parent)
+  foreach(var IN ITEMS ${ARGN})
+    set(${var} ${${var}} PARENT_SCOPE)
+  endforeach()
+endmacro()
@@ -1,53 +1,53 @@
-function(init_executable)
-  set(source_dir)
-  if (${ARGC} EQUAL 0)
-    set(source_dir ${PROJECT_NAME})
-  elseif(${ARGC} GREATER 1)
-    message(FATAL_ERROR "init_shared_library received invalid arguments. Usage: init_shared_library(<OPTIONAL:source_dir>)")
-  else()
-    set(source_dir ${ARGV})
-  endif()
+include_guard(DIRECTORY)
+include(${CMAKE_SOURCE_DIR}/cmake/ExportToParent.cmake)
+include(${CMAKE_SOURCE_DIR}/cmake/TernaryOperator.cmake)

-  set(_options)
-  set(_one_value_args CXX_SOURCES CXX_HEADERS CXX_OUT_DIR)
-  set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
+function(init_library)
+  set(_options SHARED STATIC)
+  set(_one_value_args SOURCE_DIR)
+  set(_multi_value_args)
  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-  if (${_ARGS_PROTO_SOURCES} EQUAL "GLOB_RECURSIVE")
-    file(
-      GLOB_RECURSE ${PROJECT_NAME}_PROTO_SOURCES
-      RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
-      *.proto
-    )
+
+  if (_ARGS_SHARED AND _ARGS_STATIC)
+    message(FATAL_ERROR "Invalid args passed to init_library: cannot specify STATIC and SHARED")
  endif()

-  set(${PROJECT_NAME}_SOURCES PARENT_SCOPE)
+  set_ternary(_source_dir _ARGS_SOURCE_DIR _ARGS_SOURCE_DIR ${PROJECT_NAME})
+  # Glob cpp sources
  file(
    GLOB_RECURSE ${PROJECT_NAME}_SOURCES
    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
-    ${source_dir}/*.cpp
+    ${_source_dir}/*.cpp
  )
-  set(${PROJECT_NAME}_SOURCES ${${PROJECT_NAME}_SOURCES} PARENT_SCOPE)
-  # set(${PROJECT_NAME}_Sources ${${PROJECT_NAME}_Sources} PARENT_SCOPE)
  # Note: cpp files inside 'tests' dir(s) are considered tests.
  # Remove tests from main sources, and add them to test sources
-  set(${PROJECT_NAME}_TEST_SOURCES)
  foreach(item ${${PROJECT_NAME}_SOURCES})
    if(${item} MATCHES "tests/*.cpp")
      list(REMOVE_ITEM ${PROJECT_NAME}_SOURCES ${item})
      list(APPEND ${PROJECT_NAME}_TEST_SOURCES ${item})
    endif()
-  set(${PROJECT_NAME}_TEST_SOURCES ${${PROJECT_NAME}_TEST_SOURCES} PARENT_SCOPE)
  endforeach()

-  # Add executable
-  add_executable(${PROJECT_NAME} ${${PROJECT_NAME}_SOURCES})
+  export_to_parent(
+    ${PROJECT_NAME}_SOURCES
+    ${PROJECT_NAME}_TEST_SOURCES
+  )
+
+  # Add main library
+  if (_ARGS_STATIC)
+    add_library(${PROJECT_NAME} STATIC ${${PROJECT_NAME}_SOURCES})
+  elseif(_ARGS_SHARED)
+    add_library(${PROJECT_NAME} SHARED ${${PROJECT_NAME}_SOURCES})
+  else()
+    add_library(${PROJECT_NAME} ${${PROJECT_NAME}_SOURCES})
+  endif()
+
  target_include_directories(
    ${PROJECT_NAME}
    PUBLIC
      $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
      $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>
  )
-
  target_link_directories(
    ${PROJECT_NAME}
    PUBLIC
@@ -57,7 +57,7 @@ function(init_executable)

  # Add tests executable (if there are any)
  if (${PROJECT_NAME}_TEST_SOURCES)
-    add_executable(${PROJECT_NAME}-tests ${${PROJECT_NAME}_TEST_SOURCEs})
+    add_executable(${PROJECT_NAME}-tests ${${PROJECT_NAME}_TEST_SOURCES})
    target_link_libraries(${PROJECT_NAME}-tests PRIVATE ${PROJECT_NAME})
    # Catch2
    find_package(Catch2 CONFIG REQUIRED COMPONENTS Catch2WithMain)
@@ -1,4 +1,5 @@
 function(init_shared_library)
+  include(${CMAKE_SOURCE_DIR}/cmake/ExportToParent.cmake)
  set(source_dir)
  if (${ARGC} EQUAL 0)
    set(source_dir ${PROJECT_NAME})
@@ -7,33 +8,35 @@ function(init_shared_library)
  else()
    set(source_dir ${ARGV})
  endif()
-  set(${PROJECT_NAME}_Sources PARENT_SCOPE)
+
+  # Glob cpp sources
  file(
-    GLOB_RECURSE ${PROJECT_NAME}_Sources
+    GLOB_RECURSE ${PROJECT_NAME}_SOURCES
    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
-    ${source_dir}/*.cpp)
-  set(${PROJECT_NAME}_Sources ${${PROJECT_NAME}_Sources} PARENT_SCOPE)
-  # set(${PROJECT_NAME}_Sources ${${PROJECT_NAME}_Sources} PARENT_SCOPE)
+    ${source_dir}/*.cpp
+  )
  # Note: cpp files inside 'tests' dir(s) are considered tests.
  # Remove tests from main sources, and add them to test sources
-  set(${PROJECT_NAME}_Test_Sources)
-  foreach(item ${${PROJECT_NAME}_Sources})
+  foreach(item ${${PROJECT_NAME}_SOURCES})
    if(${item} MATCHES "tests/*.cpp")
-      list(REMOVE_ITEM ${PROJECT_NAME}_Sources ${item})
-      list(APPEND ${PROJECT_NAME}_Test_Sources ${item})
+      list(REMOVE_ITEM ${PROJECT_NAME}_SOURCES ${item})
+      list(APPEND ${PROJECT_NAME}_TEST_SOURCES ${item})
    endif()
-  set(${PROJECT_NAME}_Test_Sources ${${PROJECT_NAME}_Test_Sources} PARENT_SCOPE)
  endforeach()

+  export_to_parent(
+    ${PROJECT_NAME}_SOURCES
+    ${PROJECT_NAME}_TEST_SOURCES
+  )
+
  # Add main library
-  add_library(${PROJECT_NAME} SHARED ${${PROJECT_NAME}_Sources})
+  add_library(${PROJECT_NAME} SHARED ${${PROJECT_NAME}_SOURCES})
  target_include_directories(
    ${PROJECT_NAME}
    PUBLIC
      $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
      $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>
  )
-
  target_link_directories(
    ${PROJECT_NAME}
    PUBLIC
@@ -42,8 +45,8 @@ function(init_shared_library)
  )

  # Add tests executable (if there are any)
-  if (${PROJECT_NAME}_Test_Sources)
-    add_executable(${PROJECT_NAME}-tests ${${PROJECT_NAME}_Test_Sources})
+  if (${PROJECT_NAME}_TEST_SOURCES)
+    add_executable(${PROJECT_NAME}-tests ${${PROJECT_NAME}_TEST_SOURCES})
    target_link_libraries(${PROJECT_NAME}-tests PRIVATE ${PROJECT_NAME})
    # Catch2
    find_package(Catch2 CONFIG REQUIRED COMPONENTS Catch2WithMain)
@@ -1,4 +1,5 @@
 include(${CMAKE_SOURCE_DIR}/cmake/PackBasic.cmake)
+include(GNUInstallDirs)

 function(install_basic_config)
  file(
@@ -1,6 +1,21 @@
 include(CMakePackageConfigHelpers)

 function(pack_basic)
+  find_package(Git QUIET)
+
+  if(GIT_FOUND)
+    execute_process(
+      COMMAND ${GIT_EXECUTABLE} describe --tags --always --dirty
+      WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
+      OUTPUT_VARIABLE PROJECT_VERSION
+      OUTPUT_STRIP_TRAILING_WHITESPACE
+    )
+  endif()
+
+  if(NOT GIT_FOUND OR PROJECT_VERSION STREQUAL "")
+    set(PROJECT_VERSION "0.0.0")
+  endif()
+
  write_basic_package_version_file(
    "${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}ConfigVersion.cmake"
    VERSION ${PROJECT_VERSION}
@@ -1,167 +0,0 @@
-set(Protobuf_PROTOC_EXECUTABLE "${EXTERNALS_BIN}/protoc")
-set(PROTOBUF_INCLUDE_DIRS "${EXTERNALS_INCLUDE}")
-set(PROTOBUF_STATIC_LIB "libprotobuf")
-set(_protobuf_src_dir ${CMAKE_SOURCE_DIR}/src/lib_spark)
-set(_protobuf_bin_dir ${CMAKE_BINARY_DIR}/src/lib_spark)
-
-function(Protobuf_Generate_Cpp)
-set(_options)
-  set(_one_value_args CXX_SOURCES CXX_HEADERS CXX_OUT_DIR)
-  set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-  set(${_ARGS_CXX_SOURCES})
-  set(${_ARGS_CXX_HEADERS})
-
-  set(_proto_include_cli_args)
-  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
-    list(APPEND _proto_include_cli_args -I ${_dir})
-  endforeach()
-
-  foreach(FIL ${_ARGS_PROTO_SOURCES})
-    get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
-    get_filename_component(FIL_WE ${FIL} NAME_WE)
-    get_filename_component(FIL_DIR ${FIL} DIRECTORY)
-    if(FIL_DIR)
-      set(FIL_WE "${FIL_DIR}/${FIL_WE}")
-    endif()
-
-    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc")
-    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h")
-
-    add_custom_command(
-      OUTPUT
-        "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
-        "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
-      COMMAND
-        ${Protobuf_PROTOC_EXECUTABLE}
-      ARGS
-        --cpp_out=${_ARGS_CXX_OUT_DIR} ${_proto_include_cli_args} ${ABS_FIL}
-      DEPENDS ${ABS_FIL}
-      COMMENT "Running C++ protocol buffer compiler on ${FIL}"
-      VERBATIM)
-  endforeach()
-
-  set_source_files_properties(${${_ARGS_CXX_SOURCES}} ${${_ARGS_CXX_HEADERS}} PROPERTIES GENERATED TRUE)
-
-  # remove some warnings from protobuf generated files
-  if(WIN32)
-    set_source_files_properties(${${_ARGS_CXX_SOURCES}} PROPERTIES COMPILE_FLAGS
-                                                      "/wd4309 /wd4838")
-  else()
-    set_source_files_properties(
-      ${${_ARGS_CXX_SOURCES}}
-      PROPERTIES
-        COMPILE_FLAGS
-        "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
-    )
-  endif()
-
-  set(${_ARGS_CXX_SOURCES}
-      ${${_ARGS_CXX_SOURCES}}
-      PARENT_SCOPE)
-  set(${_ARGS_CXX_HEADERS}
-      ${${_ARGS_CXX_HEADERS}}
-      PARENT_SCOPE)
-endfunction()
-
-function(Service_Compiler_Generate_Cpp)
-  set(_options)
-  set(_one_value_args CXX_SOURCES CXX_HEADERS CXX_OUT_DIR)
-  set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-  set(${_ARGS_CXX_SOURCES})
-  set(${_ARGS_CXX_HEADERS})
-
-  set(_proto_include_cli_args)
-  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
-    list(APPEND _proto_include_cli_args -I ${_dir})
-  endforeach()
-
-  foreach(FIL ${_ARGS_PROTO_SOURCES})
-    get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
-    get_filename_component(FIL_WE ${FIL} NAME_WE)
-    get_filename_component(FIL_DIR ${FIL} DIRECTORY)
-    if(FIL_DIR)
-      set(FIL_WE "${FIL_DIR}/${FIL_WE}")
-    endif()
-
-    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc")
-    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h")
-
-    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc")
-    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h")
-
-    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc")
-    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h")
-
-    set(TEMPLATES
-        ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/client.pb.cc
-        ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/client.pb.h
-        ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/server.pb.cc
-        ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/server.pb.h
-    )
-
-    set(_plugin_py ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/plugin.py)
-    if(WIN32)
-      set(_plugin_cmd ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/plugin.cmd)
-      add_custom_command(
-        OUTPUT "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
-               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
-               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc"
-               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h"
-               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc"
-               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h"
-        COMMAND SET PYTHONPATH=${SPARK_PYTHON}
-        COMMAND ${CMAKE_SOURCE_DIR}/.venv/Scripts/activate.bat
-        COMMAND
-          ${Protobuf_PROTOC_EXECUTABLE} ARGS --cpp_out=${_ARGS_CXX_OUT_DIR}
-          --plugin=protoc-gen-service=${_plugin_cmd}
-          --service_out=lang=cpp:${_ARGS_CXX_OUT_DIR} -I
-          ${PROTOBUF_INCLUDE_DIRS} -I ${_protobuf_src_dir} ${ABS_FIL}
-        DEPENDS ${ABS_FIL} ${TEMPLATES} ${_plugin_cmd} ${_plugin_py}
-        COMMENT "Running C++ service compiler on ${FIL}"
-        VERBATIM)
-    else()
-      add_custom_command(
-        OUTPUT "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
-               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
-               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc"
-               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h"
-               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc"
-               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h"
-        COMMAND ${CMAKE_SOURCE_DIR}/tools/venv_exec.sh
-        ARGS
-          ${Protobuf_PROTOC_EXECUTABLE}
-          --cpp_out=${_ARGS_CXX_OUT_DIR}
-          --plugin=protoc-gen-service=${_plugin_py}
-          --service_out=lang=cpp,make_file_name=${ABS_FIL}:${_ARGS_CXX_OUT_DIR}
-          ${_proto_include_cli_args} ${ABS_FIL}
-        DEPENDS ${ABS_FIL} ${TEMPLATES} ${_plugin_py}
-        COMMENT "Running C++ service compiler on ${FIL}"
-        VERBATIM)
-    endif()
-
-  endforeach()
-
-  # remove some warnings from protobuf generated files
-  if(WIN32)
-    set_source_files_properties(${${_ARGS_CXX_SOURCES}} PROPERTIES COMPILE_FLAGS
-                                                      "/wd4309 /wd4838")
-  else()
-    set_source_files_properties(
-      ${${_ARGS_CXX_SOURCES}}
-      PROPERTIES
-        COMPILE_FLAGS
-        "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
-    )
-  endif()
-
-  set(${_ARGS_CXX_SOURCES}
-      ${${_ARGS_CXX_SOURCES}}
-      PARENT_SCOPE)
-  set(${_ARGS_CXX_HEADERS}
-      ${${_ARGS_CXX_HEADERS}}
-      PARENT_SCOPE)
-
-endfunction()
-
@@ -0,0 +1,67 @@
+include_guard(GLOBAL)
+
+function(protobuf_generate_cpp)
+  set(_options)
+  set(_one_value_args CXX_SOURCES CXX_HEADERS CXX_OUT_DIR PROTOC)
+  set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+
+  if (NOT DEFINED _ARGS_PROTOC)
+    set(_ARGS_PROTOC "${EXTERNALS_BIN}/protoc")
+  endif()
+  set(${_ARGS_CXX_SOURCES})
+  set(${_ARGS_CXX_HEADERS})
+
+  set(_proto_include_cli_args)
+  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
+    list(APPEND _proto_include_cli_args -I ${_dir})
+  endforeach()
+
+  foreach(_fil ${_ARGS_PROTO_SOURCES})
+    get_filename_component(_abs_fil ${_fil} ABSOLUTE)
+    get_filename_component(_fil_we ${_fil} NAME_WE)
+    get_filename_component(_fil_dir ${_fil} DIRECTORY)
+    if(_fil_dir)
+      set(_fil_we "${_fil_dir}/${_fil_we}")
+    endif()
+    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.cc")
+    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.h")
+    add_custom_command(
+      OUTPUT
+        "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.cc"
+        "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.h"
+      COMMAND
+        ${_ARGS_PROTOC}
+      ARGS
+        --cpp_out=${_ARGS_CXX_OUT_DIR} ${_proto_include_cli_args} ${_abs_fil}
+      DEPENDS ${_abs_fil}
+      COMMENT "Running C++ protocol buffer compiler on ${_fil}"
+      VERBATIM
+    )
+  endforeach()
+
+  set_source_files_properties(
+    ${${_ARGS_CXX_SOURCES}}
+    ${${_ARGS_CXX_HEADERS}}
+    PROPERTIES
+      GENERATED TRUE
+  )
+
+  # remove some warnings from protobuf generated files
+  if(WIN32)
+    set_source_files_properties(
+      ${${_ARGS_CXX_SOURCES}}
+      PROPERTIES
+        COMPILE_FLAGS "/wd4309 /wd4838"
+    )
+  else()
+    set_source_files_properties(
+      ${${_ARGS_CXX_SOURCES}}
+      PROPERTIES
+        COMPILE_FLAGS "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
+    )
+  endif()
+
+  set(${_ARGS_CXX_SOURCES} ${${_ARGS_CXX_SOURCES}} PARENT_SCOPE)
+  set(${_ARGS_CXX_HEADERS} ${${_ARGS_CXX_HEADERS}} PARENT_SCOPE)
+endfunction()
@@ -0,0 +1,49 @@
+include_guard(DIRECTORY)
+
+function(protobuf_generate_fs)
+  set(_options)
+  set(_one_value_args TARGET_NAME OUT_DIR PROTOC)
+  set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+
+  if (NOT DEFINED _ARGS_TARGET_NAME)
+    set(_ARGS_TARGET_NAME "${PROJECT_NAME}_generate_fs_protos")
+  endif()
+  if (NOT DEFINED _ARGS_PROTOC)
+    set(_ARGS_PROTOC "${EXTERNALS_BIN}/protoc")
+  endif()
+
+  set(_proto_include_cli_args)
+  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
+    list(APPEND _proto_include_cli_args -I ${_dir})
+  endforeach()
+
+  set(_outputs)
+  foreach(_proto_source ${_ARGS_PROTO_SOURCES})
+    # get_filename_component(_abs_proto_source ${_proto_source} ABSOLUTE)
+    get_filename_component(_proto_source_we ${_proto_source} NAME_WE)
+    # get_filename_component(_proto_source_dir ${_proto_source} DIRECTORY)
+    list(APPEND _outputs ${_ARGS_OUT_DIR}/${_proto_source_we}.fable.fs)
+  endforeach()
+
+  set(_venv_exec ${CMAKE_SOURCE_DIR}/tools/venv_exec.sh)
+  set(_plugin_py ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/plugin.py)
+  set(_template_fs ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/fs/fable.fs)
+  add_custom_command(
+    OUTPUT ${_outputs}
+    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+    COMMAND ${_venv_exec}
+      ${_ARGS_PROTOC}
+      --plugin=protoc-gen-service=${_plugin_py}
+      --service_out=lang=fs,strip_out_path=1:${_ARGS_OUT_DIR}
+      ${_proto_include_cli_args}
+      ${_ARGS_PROTO_SOURCES}
+    DEPENDS
+      ${_venv_exec}
+      ${_plugin_py}
+      ${_template_fs}
+      ${CMAKE_CURRENT_SOURCE_DIR}/package-lock.json
+      ${_ARGS_PROTO_SOURCES}
+  )
+  add_custom_target(${_ARGS_TARGET_NAME} ALL DEPENDS "${_outputs}")
+endfunction()
@@ -0,0 +1,33 @@
+include_guard(DIRECTORY)
+
+function(protobuf_generate_js)
+  set(_options)
+  set(_one_value_args TARGET_NAME OUTPUT)
+  set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+
+  if (NOT DEFINED _ARGS_TARGET_NAME)
+    set(_ARGS_TARGET_NAME "${PROJECT_NAME}_generate_js_protos")
+  endif()
+
+  set(_proto_include_cli_args)
+  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
+    list(APPEND _proto_include_cli_args -p ${_dir})
+  endforeach()
+
+  add_custom_command(
+    OUTPUT ${_ARGS_OUTPUT}
+    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+    COMMAND npx
+      --no-install pbjs
+      -t json-module
+      -w commonjs
+      -o ${_ARGS_OUTPUT}
+      ${_proto_include_cli_args}
+      ${_ARGS_PROTO_SOURCES}
+    DEPENDS
+      ${CMAKE_CURRENT_SOURCE_DIR}/package-lock.json
+      ${_ARGS_PROTO_SOURCES}
+  )
+  add_custom_target(${_ARGS_TARGET_NAME} ALL DEPENDS ${_ARGS_OUTPUT})
+endfunction()
@@ -1,8 +0,0 @@
-macro(set_first_defined RESULT CANDIDATE_VARS)
-  foreach(_candidate_var ${CANDIDATE_VARS})
-    if (DEFINED _candidate_var)
-      set(${RESULT} ${_candidate_var})
-      break()
-    endif()
-  endforeach()
-endmacro()
@@ -1,11 +1,20 @@
-function(sym_link_dir TARGET_NAME SOURCE DEST)
+include_guard(DIRECTORY)
+include(${CMAKE_SOURCE_DIR}/cmake/TernaryOperator.cmake)
+
+function(sym_link_dir)
+  set(_options)
+  set(_one_value_args TARGET_NAME SOURCE DEST COMMENT)
+  set(_multi_value_args)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
  add_custom_command(
-    OUTPUT ${DEST}
-    COMMAND ln ARGS -sf ${SOURCE} ${DEST}
+    OUTPUT ${_ARGS_DEST}
+    COMMAND ln ARGS -sf ${_ARGS_SOURCE} ${_ARGS_DEST}
  )
+  set_ternary(_comment _ARGS_COMMENT "${_ARGS_COMMENT}" "linking dir ${_ARGS_SOURCE} -> ${_ARGS_DEST}")
  add_custom_target(
-    ${TARGET_NAME} ALL
-    DEPENDS ${DEST}
+    ${_ARGS_TARGET_NAME} ALL
+    DEPENDS ${_ARGS_DEST}
+    COMMENT "${_comment}"
  )
 endfunction()

@@ -1,3 +1,5 @@
+include_guard(GLOBAL)
+
 macro(set_ternary RESULT CONDITION TRUE_VALUE FALSE_VALUE)
  if(${CONDITION})
    set(${RESULT} ${TRUE_VALUE})
@@ -53,7 +53,7 @@ The status callback is called when the market for this security opens, closes, o
 // Subscribe to security trading status
 void IAlgoServices::subscribe_status(SecurityKey);
 // You implement
-virtual void on_status(SecurityKey, co2::TradingStatus) override;
+virtual void on_status(SecurityKey, domain::protobufs::TradingStatus) override;
 ```

 # Child Orders
@@ -1,55 +1,43 @@
 cmake_minimum_required(VERSION 3.5)
 project(Spark.Admin.WebApp)

+include(GNUInstallDirs)
 include(${CMAKE_SOURCE_DIR}/cmake/safe_dep_glob.cmake)
+include(${CMAKE_SOURCE_DIR}/cmake/SymLink.cmake)
+include(${CMAKE_SOURCE_DIR}/cmake/ProtobufGenerateJs.cmake)
+include(${CMAKE_SOURCE_DIR}/cmake/ProtobufGenerateFs.cmake)

-# dependencies
-set(PROTO_GLOBS
+file(
+  GLOB_RECURSE ${PROJECT_NAME}_PROTO_SOURCES
+  RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
+  ../lib_domain/domain/protobufs/*.proto
  ../lib_spark/spark/api/*.proto
  ../lib_spark/spark/api/services/*.proto
  ../lib_spark/spark/domain_model/*.proto
  ../lib_spark/spark/sparkproto/*.proto
  ../lib_spark/spark/md/protos/*.proto
  ../../externals/linux/include/google/protobuf/descriptor.proto
-  ../../externals/linux/include/google/protobuf/timestamp.proto)
+  ../../externals/linux/include/google/protobuf/timestamp.proto
+)

-file(
-  GLOB_RECURSE _proto_rel_SOURCES
-  RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
-  ${PROTO_GLOBS})
-
-file(
-  GLOB_RECURSE _proto_abs_SOURCES
-  ${PROTO_GLOBS})
-
-file(
-  GLOB_RECURSE _ts_SOURCES
-  RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
-  src/*.ts)
-
-file(
-  GLOB_RECURSE _fs_SOURCES
-  RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
-  src/*.fs)
+add_custom_command(
+  OUTPUT ${CMAKE_CURRENT_SOURCE_DIR}/package-lock.json
+  COMMAND npm install --include=dev
+  DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/package.json
+  WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+)

 set(_tools ${CMAKE_SOURCE_DIR}/tools)
 set(_quiet_exec ${_tools}/quiet_exec.sh)
 set(_venv_exec ${_tools}/venv_exec.sh)
 set(_tools_py ${CMAKE_SOURCE_DIR}/tools/py)

-# The tsconfig file is generated as a function of the cmake build type, and
-# binary dir. Depending on the build type, it extends either
-# 'tsconfig.debug.json' or 'tsconfig.release.json'.
 set(_tsconfig_base "${CMAKE_CURRENT_SOURCE_DIR}/tsconfig.$<IF:$<CONFIG:Debug>,debug,release>.json")
-set(WEBPACK_MODE $<IF:$<CONFIG:Debug>,"development","production">)
 set(_tsconfig "${CMAKE_CURRENT_BINARY_DIR}/tsconfig.json")

 set(_venv_python ${CMAKE_SOURCE_DIR}/.venv/bin/python)
 set(_gen_tsconfig_py ${_tools_py}/gen_tsconfig.py)
 set(_build_admin_web_app_py ${_tools_py}/build_admin_web_app.py)
-
-set(_gen_tsconfig_deps ${_venv_exec} ${_venv_python} ${_gen_tsconfig_py} ${_tsconfig_base})
-
 set(_build_admin_web_app_deps
  ${_quiet_exec}
  ${_venv_exec}
@@ -59,49 +47,133 @@ set(_build_admin_web_app_deps
  ${CMAKE_CURRENT_SOURCE_DIR}/webpack.config.js
  ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/fs/fable.fs
  ${CMAKE_CURRENT_SOURCE_DIR}/package-lock.json
-  ${_proto_rel_SOURCES}
+  ${${PROJECT_NAME}_PROTO_SOURCES}
  ${_api_ts_SOURCES}
  ${_ui_fs_SOURCES}
  ${_tsconfig})

 update_deps_file("${_build_admin_web_app_deps}")

-function(npm_install path)
-  add_custom_command(
-    OUTPUT ${path}/package-lock.json
-    COMMAND "npm" ARGS "install" "--include=dev"
-    DEPENDS ${path}/package.json
-    WORKING_DIRECTORY ${path}
-    COMMENT "Run 'npm install' in ${path}")
-endfunction()
-
-npm_install(${CMAKE_CURRENT_SOURCE_DIR})
-
 add_custom_command(
  OUTPUT ${_tsconfig}
  COMMAND ${_venv_exec}
  ARGS
-  ${_venv_python}
-  ${_gen_tsconfig_py}
-  --config-base ${_tsconfig_base}
-  --config-out ${_tsconfig}
-  --build-out ${CMAKE_CURRENT_BINARY_DIR}
-  DEPENDS ${_gen_tsconfig_deps})
+    ${_venv_python}
+    ${_gen_tsconfig_py}
+    --config-base ${_tsconfig_base}
+    --config-out ${_tsconfig}
+    --build-out ${CMAKE_CURRENT_BINARY_DIR}
+  DEPENDS
+    ${_venv_exec}
+    ${_venv_python}
+    ${_gen_tsconfig_py}
+    ${_tsconfig_base}
+    ${CMAKE_CURRENT_SOURCE_DIR}/package-lock.json
+)
+
+protobuf_generate_js(
+  TARGET_NAME generate_js_protos
+  PROTO_INCLUDE_DIRS
+    ../lib_domain
+    ../lib_spark
+    ../../externals/linux/include
+  PROTO_SOURCES ${${PROJECT_NAME}_PROTO_SOURCES}
+  OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/proto.js
+)
+
+protobuf_generate_fs(
+  TARGET_NAME generate_fs_protos
+  PROTO_INCLUDE_DIRS
+    ../lib_domain
+    ../lib_spark
+    ../../externals/linux/include
+  PROTO_SOURCES ${${PROJECT_NAME}_PROTO_SOURCES}
+  OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/fable
+)
+
+# Each input file maps to multiple output files with different extensions
+# (js, js.map, d.ts, fs). Transpile ts -> d.ts (and js), then d.ts -> fs.
+# Who knew front-end development could be so complicated? `\_(=_=)_/`
+file(
+  GLOB_RECURSE ${PROJECT_NAME}_TS_SOURCES
+  RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
+  src/*.ts
+)
+set(${PROJECT_NAME}_JS_PROTOS)
+set(${PROJECT_NAME}_TS_PROTOS)
+set(${PROJECT_NAME}_FS_PROTOS)
+foreach(_ts_source ${${PROJECT_NAME}_TS_SOURCES})
+  get_filename_component(_ts_source_we ${_ts_source} NAME_WE)
+  set(_prefix ${CMAKE_CURRENT_BINARY_DIR}/${_ts_source_we})
+  list(APPEND ${PROJECT_NAME}_JS_PROTOS ${_prefix}.js ${_prefix}.js.map)
+  list(APPEND ${PROJECT_NAME}_TS_PROTOS ${_prefix}.d.ts ${_prefix}.d.ts.map)
+  list(APPEND ${PROJECT_NAME}_FS_PROTOS ${_prefix}.fs)
+  add_custom_command(
+    OUTPUT ${_prefix}.fs
+    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+    COMMAND npx --no-install ts2fable ${_prefix}.d.ts ${_prefix}.fs
+    DEPENDS ${_prefix}.d.ts
+  )
+endforeach()

 add_custom_command(
-  OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/bundle.js
-  COMMAND ${_quiet_exec}
-  ARGS
-  ${_venv_exec}
-  ${_venv_python}
-  ${_build_admin_web_app_py}
-  --protos ${_proto_abs_SOURCES}
-  --tsconfig ${_tsconfig}
-  --mode ${WEBPACK_MODE}
-  --output-dir ${CMAKE_CURRENT_BINARY_DIR}
-  COMMENT "Building Spark.Admin.WebApp"
-  DEPENDS ${_build_admin_web_app_deps})
+  OUTPUT
+    ${${PROJECT_NAME}_JS_PROTOS}
+    ${${PROJECT_NAME}_TS_PROTOS}
+  WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+  COMMAND npx
+    --no-install tsc
+    -p ${_tsconfig}
+  DEPENDS
+    ${_tsconfig}
+    ${CMAKE_CURRENT_BINARY_DIR}/proto.js
+)
+add_custom_target(
+  transpile_type_script
+  ALL
+  DEPENDS
+    ${${PROJECT_NAME}_JS_PROTOS}
+    ${${PROJECT_NAME}_TS_PROTOS}
+)
+add_custom_target(generate_fsts_stubs ALL DEPENDS ${${PROJECT_NAME}_FS_PROTOS})

-add_custom_target(admin_web_app_bundle ALL DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/bundle.js)
+sym_link_dir(
+  TARGET_NAME link_webapp_public
+  SOURCE ${CMAKE_CURRENT_SOURCE_DIR}/public
+  DEST ${CMAKE_CURRENT_BINARY_DIR}/public
+)
+sym_link_dir(
+  TARGET_NAME link_webapp_root
+  SOURCE ${CMAKE_CURRENT_BINARY_DIR}
+  DEST ${CMAKE_SOURCE_DIR}/build/Spark.Admin.WebApp
+)

-install(DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}" DESTINATION "bin")
+file(
+  GLOB_RECURSE ${PROJECT_NAME}_FS_SOURCES
+  RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
+  src/*.fs
+)
+add_custom_command(
+  OUTPUT
+    ${CMAKE_CURRENT_BINARY_DIR}/bundle.js
+    $<$<CONFIG:DEBUG>:${CMAKE_CURRENT_BINARY_DIR}/bundle.js.map>
+  WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/src
+  COMMAND dotnet restore
+  COMMAND dotnet tool restore
+  COMMAND export SPARK_ADMIN_WEB_APP_OUTPUT_DIR=${CMAKE_CURRENT_BINARY_DIR}
+  COMMAND dotnet fable -o ${CMAKE_CURRENT_BINARY_DIR}
+  COMMAND npm run build -- --mode $<IF:$<CONFIG:Debug>,"development","production">
+  DEPENDS
+    ${${PROJECT_NAME}_FS_SOURCES}
+    ${${PROJECT_NAME}_FS_PROTOS}
+    ${CMAKE_CURRENT_SOURCE_DIR}/src/Spark.Admin.WebApp.fsproj
+)
+add_custom_target(
+  bundle_spark_admin_webapp ALL
+  DEPENDS
+    ${CMAKE_CURRENT_BINARY_DIR}/bundle.js
+    $<$<CONFIG:DEBUG>:${CMAKE_CURRENT_BINARY_DIR}/bundle.js.map>
+  COMMENT "Bundling Spark.Admin.WebApp"
+)
+
+install(DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}" DESTINATION "${CMAKE_INSTALL_BINDIR}")
@@ -42,7 +42,7 @@
    "ts-loader": "^9.4.2",
    "tsutils": "^3.21.0",
    "typescript": "^4.9.5",
-    "webpack-cli": "^5.0.1",
+    "webpack-cli": "^5.1.4",
    "webpack-dev-server": "^4.11.1",
    "ws": "^8.12.1"
  }
@@ -33,6 +33,7 @@
    <Compile Include="$(API)/fable/timestamp.fable.fs" />
    <Compile Include="$(API)/fable/apicore.fable.fs" />
    <Compile Include="$(API)/fable/apioptions.fable.fs" />
+    <Compile Include="$(API)/fable/aggressor_side.fable.fs" />
    <Compile Include="$(API)/fable/Security.fable.fs" />
    <Compile Include="$(API)/fable/SparkProtos.fable.fs" />
    <Compile Include="$(API)/fable/HelloService.fable.fs" />
@@ -2,7 +2,7 @@ cmake_minimum_required(VERSION 3.5)
 project(sparkd)

 include(${CMAKE_SOURCE_DIR}/cmake/safe_dep_glob.cmake)
-include(${CMAKE_SOURCE_DIR}/cmake/Protobuf.cmake)
+include(${CMAKE_SOURCE_DIR}/cmake/ProtobufGenerateCpp.cmake)
 include(GNUInstallDirs)

 if (NOT DEFINED PROJECT_VERSION)
@@ -31,15 +31,18 @@ protobuf_generate_cpp(
    ${CMAKE_SOURCE_DIR}/src/lib_spark
    ${CMAKE_CURRENT_SOURCE_DIR}
  PROTO_SOURCES
-    ${${PROJECT_NAME}_PROTO_SOURCES})
+    ${${PROJECT_NAME}_PROTO_SOURCES}
+)
 add_executable(sparkd ${${PROJECT_NAME}_SOURCES} ${${PROJECT_NAME}_PROTO_CXX_SOURCES})

 # include paths
 target_include_directories(
-  sparkd PUBLIC
-  $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
-  $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>/include
-  $<BUILD_INTERFACE:${emumba-deribit-connector-library_INCLUDE_DIRS}>)
+  sparkd
+  PUBLIC
+    $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
+    $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>/include
+    $<BUILD_INTERFACE:${emumba-deribit-connector-library_INCLUDE_DIRS}>
+)

 find_package(PkgConfig REQUIRED)
 pkg_check_modules(emumba-deribit-connector-library REQUIRED emumba-deribit-connector)
@@ -48,8 +51,9 @@ pkg_check_modules(emumba-deribit-connector-library REQUIRED emumba-deribit-conne
 list(APPEND CMAKE_PREFIX_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake_modules)
 find_package(rapidjson REQUIRED)
 target_include_directories(
-  ${PROJECT_NAME} SYSTEM PUBLIC
-  $<BUILD_INTERFACE:${rapidjson_INCLUDE_DIR}> 
+  ${PROJECT_NAME} SYSTEM
+  PUBLIC
+    $<BUILD_INTERFACE:${rapidjson_INCLUDE_DIR}> 
 )
 add_dependencies(${PROJECT_NAME} rapidjson_project)

@@ -83,53 +87,48 @@ else()
  message("-- Building of DERIBIT_DEMO_LOG_LEVEL=TRACE")
 endif()

-# add_dependencies(${PROJECT_NAME} py_market_data)
-list(APPEND CMAKE_PREFIX_PATH "${CMAKE_SOURCE_DIR}/install/$<CONFIG>/lib/cmake/py-market-data")
-
 target_link_directories(
  ${PROJECT_NAME} PUBLIC
  $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
  $<INSTALL_INTERFACE:${CMAKE_INSTALL_LIBDIR}>
 )

-# Set RPATH so sparkd knows where to find our shared libs (e.g., libspark, lib_py_market_data).
+# Set RPATH so sparkd knows where to find our shared libs (e.g., libspark).
 # The dirs are relative to sparkd's path, represented by "$ORIGIN".
 set_target_properties(
  ${PROJECT_NAME} PROPERTIES
-  BUILD_RPATH "$ORIGIN/../lib_spark;$ORIGIN/../lib_py_market_data"
+  BUILD_RPATH "$ORIGIN/../lib_spark;$ORIGIN/../lib_python_md"
  INSTALL_RPATH "$ORIGIN/../${CMAKE_INSTALL_LIBDIR}"
 )

+# Python interop (for python md providers)
+add_dependencies(${PROJECT_NAME}
+  python_md
+  python_md_module
+  python_md_providers
+  copy_python_md_providers
+)
 include(${CMAKE_SOURCE_DIR}/cmake/SymLink.cmake)

-sym_link_local_lib(py_market_data)
+sym_link_local_lib(python_md)
 sym_link_file(
-  TARGET_NAME link_py_market_data_module
-  SOURCE ${CMAKE_BINARY_DIR}/src/lib_py_market_data/py_market_data_module.cpython-310-x86_64-linux-gnu.so
-  DEST ${CMAKE_CURRENT_BINARY_DIR}/py_market_data_module.cpython-310-x86_64-linux-gnu.so
+  TARGET_NAME link_python_md_module
+  SOURCE ${CMAKE_BINARY_DIR}/src/lib_python_md/python_md_module.cpython-310-x86_64-linux-gnu.so
+  DEST ${CMAKE_CURRENT_BINARY_DIR}/python_md_module.cpython-310-x86_64-linux-gnu.so
  INSTALL_DEST ${CMAKE_INSTALL_BINDIR}/
 )
-# add_custom_command(
-#   OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/py_market_data_module.cpython-310-x86_64-linux-gnu.so
-#   COMMAND ${CMAKE_COMMAND} -E create_symlink
-#     ${CMAKE_BINARY_DIR}/src/lib_py_market_data/py_market_data_module.cpython-310-x86_64-linux-gnu.so
-#     ${CMAKE_CURRENT_BINARY_DIR}/py_market_data_module.cpython-310-x86_64-linux-gnu.so
-#   DEPENDS 
-#     ${CMAKE_BINARY_DIR}/src/lib_py_market_data/py_market_data_module.cpython-310-x86_64-linux-gnu.so
-#   COMMENT "Creating symlink to py_market_data_module.so"
-# )
-# add_custom_target(create_symlinks ALL
-#   DEPENDS ${CMAKE_BINARY_DIR}/src/lib_py_market_data/py_market_data_module.cpython-310-x86_64-linux-gnu.so)
-# install(
-#   FILES
-#   ${CMAKE_CURRENT_BINARY_DIR}/py_market_data_module.cpython-310-x86_64-linux-gnu.so
-#   DESTINATION ${CMAKE_INSTALL_BINDIR}/
-# )
+get_target_property(providers python_md_providers INTERFACE_PYTHON_SCRIPTS_DIR)
+sym_link_dir(
+  TARGET_NAME link_python_md_providers
+  SOURCE ${providers}
+  DEST ${CMAKE_CURRENT_BINARY_DIR}/python_md_providers
+)

 target_link_libraries(
  sparkd
-  py_market_data
-  py_market_data_module
+  system
+  python_md
+  python_md_module
  spark
  ${emumba-deribit-connector-library_LIBRARIES}
  ssl
@@ -0,0 +1,73 @@
+#include "TestPythonMdProviderCmd.h"
+#include "python_md/md_sink_impl.h"
+#include "python_md/py_md_executor.h"
+#include "spark/logging/logging.h"
+
+#include <pybind11/embed.h>
+#include <pybind11/pybind11.h>
+#include <string>
+#include <thread>
+
+using bts::md::py_interop::MdSinkImpl;
+using std::string;
+
+namespace bts::cmds {
+
+struct TestPythonMdProviderCmd::Args {
+   std::string py_md_provider;
+   std::vector<std::string> module_paths;
+};
+
+static void execute(
+    std::string py_md_provider, std::vector<std::string> module_paths) {
+   using namespace std::chrono_literals;
+   MdSinkImpl sink;
+   sink.set_securities_callback([](const auto &securities) {
+      LOG(md, Info, "Received securities: {}", securities.to_string());
+   });
+   sink.set_books_callback([](const auto &books) {
+      LOG(md, Info, "Received books: {}", books.to_string());
+   });
+   sink.set_trades_callback([](const auto &trades) {
+      LOG(md, Info, "Received trades: {}", trades.to_string());
+   });
+   try {
+      md::py_interop::PyMdExecutor py_md_exec{
+          std::move(py_md_provider), std::move(module_paths), &sink};
+      LOG(md, Info, "Starting py md executor.");
+      py_md_exec.start();
+      LOG(md, Info, "Stopping py md executor.");
+      py_md_exec.stop();
+      int i = 0;
+      while (true) {
+         std::cout << "Sleep loop count " << ++i << std::endl;
+         std::this_thread::sleep_for(3s);
+      }
+   } catch (const std::exception &ex) {
+      LOG(md, Error, "Py md executor threw: {}", string{ex.what()});
+   }
+}
+
+TestPythonMdProviderCmd::TestPythonMdProviderCmd()
+    : CLI::App("Test python md provider plugin", "test-py-md-provider"),
+      args_(std::make_unique<Args>()) {
+   add_option(
+       "--provider", args_->py_md_provider,
+       "Python md provider plugin (py file)");
+   add_option(
+       "--module-path", args_->module_paths, "Module path for python plugin");
+   callback([this]() {
+      if (args_->py_md_provider.empty()) {
+         args_->py_md_provider = "md_provider_impl.MdProviderImpl";
+      }
+      if (args_->module_paths.empty()) {
+         args_->module_paths.push_back(".");
+         args_->module_paths.push_back("./python_md_providers");
+      }
+      execute(std::move(args_->py_md_provider), std::move(args_->module_paths));
+   });
+}
+
+TestPythonMdProviderCmd::~TestPythonMdProviderCmd() = default;
+
+} // namespace bts::cmds
@@ -0,0 +1,22 @@
+#pragma once
+
+#include <CLI11/CLI11.hpp>
+#include <memory>
+
+namespace bts::cmds {
+
+/**
+ * This command is designed to run a quick sanity check on Python md provider C++ interop.
+ * It instantiates a python md provider (in C++) and calls <provider>.start() with a dummy
+ * sink that prints updates to console, but does not write md to shared memory.
+ */
+struct TestPythonMdProviderCmd : public CLI::App {
+   TestPythonMdProviderCmd();
+   ~TestPythonMdProviderCmd();
+
+ private:
+   struct Args;
+   std::unique_ptr<Args> args_;
+};
+
+} // namespace bts::cmds
@@ -1,6 +1,6 @@
 #pragma once

-#include "spark/core/FMT_IMPL.h"
+#include "system/FMT_IMPL.h"

 #include <emumba-deribit-connector/deribit_utils.h>
 #include <iostream>
@@ -1,7 +1,7 @@
 #include "Run.h"
 #include "Cleanup.h"
+#include "domain/security_criteria.h"
 #include "spark/config/protos/market_data.pb.h"
-#include "spark/domain/SecurityCriteria.h"
 #include "spark/logging/logging.h"
 #include "spark/md/SymbolLoader.h"
 #include "spark/md/UserSymbol.h"
@@ -60,7 +60,7 @@ static void run_onixs(
   std::vector<std::shared_ptr<bts::md::IMDPlatform>> platforms;
   // get all glbx symbols
   auto cme_symbols =
-       SymbolLoader::load(SecurityCriteria::create({api::domain::VENUE_GLBX}));
+       SymbolLoader::load(SecurityCriteria::create({domain::protobufs::VENUE_GLBX}));
   if (!cme_symbols.empty()) {
      auto *main = new bts::md::MDP3Main();
      main->start(
@@ -69,7 +69,7 @@ static void run_onixs(
   }
   // ice
   const auto ice_symbols =
-       SymbolLoader::load(SecurityCriteria::create({api::domain::VENUE_ICE}));
+       SymbolLoader::load(SecurityCriteria::create({domain::protobufs::VENUE_ICE}));
   if (!ice_symbols.empty()) {
      auto *imain = new bts::md::ice::ICEMain();
      imain->start({ice_symbols.begin(), ice_symbols.end()});
@@ -1,12 +1,9 @@
 #include "Tail.h"
-#include "spark/api/domain/SecurityType.pb.h"
-#include "spark/api/domain/Venue.pb.h"
-#include "spark/config/protos/market_data.pb.h"
-#include "spark/domain/SecurityCriteria.h"
+#include "domain/protobufs/security_type.pb.h"
+#include "domain/security_criteria.h"
 #include "spark/md/Diag/MDPeriodicDumper.h"
-#include "spark/md/Diag/ShmDumper.h"
 #include "spark/md/SymbolLoader.h"
-#include "spark/md/UserSymbol.h"
+
 #include <map>

 namespace bts::cmds::md::cmds {
@@ -44,7 +41,7 @@ Tail::Tail()

   callback([&] {
      const auto products = SymbolLoader::load(SecurityCriteria::create(
-          {}, {bts::api::domain::SECURITY_TYPE_FUTURE}));
+          {}, {domain::protobufs::SECURITY_TYPE_FUTURE}));
      bts::md::run_periodic_dumper(
          products, _args->mod, _args->format, _args->all, _args->qualifier);
   });
@@ -34,6 +34,8 @@ namespace ex_sim {
 GCC_DIAG_OFF("-Wdeprecated")
 GCC_DIAG_OFF("-Wignored-qualifiers")

+#define AGGRESSOR_SIDE(x) bts::domain::protobufs::AGGRESSOR_SIDE_##x
+
 api::Side convert(const FIX::Side &side) {
   switch (side) {
   case FIX::Side_BUY:
@@ -539,9 +541,9 @@ void ExchangeSimApplication::update_md(std::string const &sec_desc) {
             md::PriceSize_from_double<ex_sim::precision>(
                 t.price.value() * price_factor, t.quantity.value()),
             datetime::ll_now(),
-             t.bid_aggressor ? co2::AGGRESSOR_SIDE_BUY
-                             : (t.ask_aggressor ? co2::AGGRESSOR_SIDE_SELL
-                                                : co2::AGGRESSOR_SIDE_NONE));
+             t.bid_aggressor ? AGGRESSOR_SIDE(BUY)
+                             : (t.ask_aggressor ? AGGRESSOR_SIDE(SELL)
+                                                : AGGRESSOR_SIDE(NONE)));
      }

      // publish mbo
@@ -1,9 +1,9 @@
 #pragma once

-#include "spark/core/FMT_IMPL.h"
 #include "spark/domain_model/Security.pb.h"
 #include "spark/orderman/orderman_utils/orderman_common.h"
 #include "sparkd/exchange_sim/book/Order.h"
+#include "system/FMT_IMPL.h"

 #include <memory>
 #include <sstream>
@@ -1,25 +1,28 @@
 #include "spark/testing/catch_test_runner.h"
 #include "spark/utils/init_spark_lib.h"
-// #include "py-market-data/py_exec.h"
 #include "sparkd/cmds/AdminReport.h"
 #include "sparkd/cmds/ConfigCmd.h"
 #include "sparkd/cmds/KeyCmd.h"
 #include "sparkd/cmds/Md.h"
 #include "sparkd/cmds/QuickTest.h"
 #include "sparkd/cmds/Run.h"
+#include "sparkd/cmds/TestPythonMdProviderCmd.h"
 #include "sparkd/cmds/Web.h"
 #include "sparkd/cmds/crypto/CryptoCmd.h"
-#include "sparkd/md/MarketDataGateway.h"
+
 #include <CLI11/CLI11.hpp>
+#include <csignal>

 using namespace bts;

+void signal_handler(int signum) {
+   std::cerr << "Interrupt signal (" << signum << "). Exiting" << std::endl;
+   exit(signum);
+}
+
 int main(int argc, char **argv) {
-   if (argc < 40) {
-      // md::py_interop::PyExec::exec("/home/bts/spark/tools/py");
-      md::MarketDataGateway::test_py();
-      return 0;
-   }
+   std::signal(SIGINT, signal_handler);
+   std::signal(SIGTERM, signal_handler);
   utils::SparkLibInit init;

   // The catch testing framework has its own command line parser so
@@ -37,6 +40,7 @@ int main(int argc, char **argv) {
      app.add_subcommand(CLI::App_p(new cmds::Run()));
      app.add_subcommand(CLI::App_p(new cmds::KeyCmd()));
      app.add_subcommand(CLI::App_p(new cmds::Web()));
+      app.add_subcommand(CLI::App_p(new cmds::TestPythonMdProviderCmd()));
      app.add_subcommand(
          "test", "Run unit test(s)"); // so this shows up in help
      CLI11_PARSE(app, argc, argv);
@@ -9,10 +9,12 @@ namespace auxiliary {
 using namespace bts::co2;
 using namespace bts::api::md::auxiliary;

+#define AGGRESSOR_SIDE(x) bts::domain::protobufs::AGGRESSOR_SIDE_##x
+
 ChannelVolume::ChannelVolume(SecurityChannel *channel) : _channel(channel) {}

-const VolumeAtPriceLevel &
-ChannelVolume::add_trade(const TradeMessage &m, int &record_id) {
+const VolumeAtPriceLevel &ChannelVolume::add_trade(
+    const TradeMessage &m, int &record_id) {
   auto price_map_it = _sec_map.find(m.key());
   if (price_map_it == _sec_map.end()) {
      _sec_map[m.key()] = std::unordered_map<uint64_t, VolumeAtPriceLevel>();
@@ -31,12 +33,13 @@ ChannelVolume::add_trade(const TradeMessage &m, int &record_id) {
   }
   VolumeAtPriceLevel &vol = price_it->second;
   vol.set_volume(vol.volume() + m.size());
-   if (m.aggressor_side() == AggressorSide::AGGRESSOR_SIDE_BUY)
+   if (m.aggressor_side() == AGGRESSOR_SIDE(BUY)) {
      vol.set_buy_volume(vol.buy_volume() + m.size());
-   else if (m.aggressor_side() == AggressorSide::AGGRESSOR_SIDE_SELL)
+   } else if (m.aggressor_side() == AGGRESSOR_SIDE(SELL)) {
      vol.set_sell_volume(vol.sell_volume() + m.size());
-   else if (m.aggressor_side() == AggressorSide::AGGRESSOR_SIDE_NONE)
+   } else if (m.aggressor_side() == AGGRESSOR_SIDE(NONE)) {
      vol.set_passive_volume(vol.passive_volume() + m.size());
+   }
   return vol;
 }

@@ -52,4 +55,4 @@ bool ChannelVolume::match_update(const SecurityChannel &channel) {

 } // namespace auxiliary
 } // namespace md
-} // namespace bts
+} // namespace bts
@@ -0,0 +1,7 @@
+#include "IMarketDataProvider.h"
+
+namespace bts::md {
+
+IMarketDataProvider::~IMarketDataProvider() = default;
+
+} // namespace bts::md
@@ -0,0 +1,25 @@
+#pragma once
+
+#include "domain/protobufs/venue.pb.h"
+#include "sparkd/md/IMarketDataSource.h"
+
+#include <vector>
+
+namespace bts::md {
+
+/**
+ * @brief This is a single function interface, whose sole purpose is to provide a list of market
+ * data sources (per venue). A good example implementation is the `CCApiMarketDataProvider`, which
+ * reads `MarketDataConfig` and returns an `IMarketDataSource` for each venue configuration (e.g.,
+ * `bts::api::md::BinanceMarketDataConfig`, `bts::api::md::CoinbaseMarketDataConfig`, etc.).
+ */
+class IMarketDataProvider {
+ public:
+   virtual ~IMarketDataProvider();
+
+   virtual std::vector<
+       std::pair<domain::protobufs::Venue, std::shared_ptr<IMarketDataSource>>>
+   get_sources() = 0;
+};
+
+} // namespace bts::md
@@ -0,0 +1,9 @@
+#include "IMarketDataSource.h"
+
+namespace bts::md {
+
+IMarketDataSource::~IMarketDataSource() = default;
+
+void IMarketDataSource::stop() {}
+
+} // namespace bts::md
@@ -0,0 +1,18 @@
+#pragma once
+
+#include "spark/md/IMarketDataListener.h"
+
+namespace bts::md {
+
+/**
+ * @brief This interface represents a source of market data. Call `IMarketDataSource::start` to
+ * connect an `IMarketDataListener` to the source and begin receiving market data.
+ */
+class IMarketDataSource {
+ public:
+   virtual ~IMarketDataSource();
+   virtual void start(std::shared_ptr<IMarketDataListener> listener) = 0;
+   virtual void stop();
+};
+
+} // namespace bts::md
@@ -1,6 +1,6 @@
 #pragma once

-#include "spark/core/FMT_IMPL.h"
+#include "system/FMT_IMPL.h"

 #include <ostream>
 #include <sstream>
@@ -1,66 +1,40 @@
 #include "MarketDataGateway.h"
-#include "py_market_data/md_provider.h"
-#include "py_market_data/md_sink_impl.h"
-#include "spark/api/md/EmumbaDeribitMarketDataConfig.pb.h"
+#include "domain/security_id.h"
 #include "spark/api/md/MarketDataConfig.pb.h"
 #include "spark/api/md/MarketDataConfigUtils.h"
-#include "spark/api/md/MarketDataReplayConfig.pb.h"
-#include "spark/collections/Contains.h"
-#include "spark/config/protos/system_config.pb.h"
-#include "spark/core/FMT.h"
-#include "spark/domain/SecurityID.h"
-#include "spark/domain_model/Security.pb.h"
-#include "spark/io/ProtoPrinter.h"
+#include "spark/config/config.h"
+#include "spark/domain/DomainSecurityUtils.h"
 #include "spark/logging/logging.h"
 #include "spark/md/GeneralMarketDataListener.h"
 #include "spark/md/GeneralSecurityIndex.h"
+#include "spark/md/IMarketDataListener.h"
 #include "spark/md/MDProducer.h"
 #include "spark/os/os.h"
 #include "spark/persistence/IDGenerator.h"
-#include "spark/utils/bts_assert.h"
 #include "spark/utils/bts_fatal.h"
-#include "spark/utils/thread_manager.h"
-#include "sparkd/md/MarketDataListenerID.h"
-#include "sparkd/md/MarketDataRegistration.h"
-#include "sparkd/md/MarketDataRegistrationError.h"
-#include "sparkd/md/MarketDataRegistrationFactory.h"
-#include "sparkd/md/MarketDataSubscription.h"
-#include "sparkd/md/binance/BinanceSubscription.h"
-#include "sparkd/md/binance_us/BinanceUsSubscription.h"
-#include "sparkd/md/binance_usds_futures/BinanceUsdsFuturesSubscription.h"
-#include "sparkd/md/coinbase/CoinbaseSubscription.h"
-#include "sparkd/md/deribit/DeribitSubscription.h"
-#include "sparkd/md/emumba/EmumbaDeribitData.h"
-#include "sparkd/md/ftx/FTXSubscription.h"
-#include "sparkd/md/okx/OkxSubscription.h"
-#include "sparkd/md/platforms/mdp3_onixs/MDP3Main.h"
-#include "sparkd/md/recording/MDReplayer.h"
+#include "sparkd/md/IMarketDataProvider.h"
+#include "sparkd/md/OnixsMarketDataProvider.h"
+#include "sparkd/md/PyMarketDataProvider.h"
+#include "sparkd/md/PyMarketDataSubscription.h"
+#include "sparkd/md/ReplayMarketDataProvider.h"
+#include "sparkd/md/ccapi/CCApiMarketDataProvider.h"
+#include "sparkd/md/emumba/EmumbaMarketDataProvider.h"
+#include "system/FMT.h"

 #include <bits/stdint-uintn.h>
 #include <boost/filesystem.hpp>
 #include <boost/filesystem/path.hpp>
-#include <ccapi_cpp/ccapi_event_dispatcher.h>
 #include <memory>
-#include <optional>
-#include <pybind11/embed.h>
-#include <pybind11/pybind11.h>
-#include <unordered_map>
 #include <vector>

-namespace py = pybind11;
-
 namespace bts::md {

 using bts::api::md::MarketDataConfig;
-using bts::config::SystemConfig;
-//using bts::domain::Security;
 using bts::domain::SecurityID;
-using bts::io::ProtoPrinter;
 using bts::persistence::IDGenerator;
-using os::sparkd_wd;
+using std::shared_ptr;
 using std::string;
 using std::unique_ptr;
-using std::unordered_map;
 using std::vector;

 using Path = boost::filesystem::path;
@@ -74,51 +48,33 @@ static bool validate(const MarketDataConfig &cfg, vector<string> &errs) {
   return errs.size() == orig_n_errs;
 }

-struct MarketDataListenerData {
-   MarketDataListenerData() = default;
-   MarketDataListenerData(const MarketDataListenerData &) noexcept = delete;
-   MarketDataListenerData(MarketDataListenerData &&) noexcept = default;
-   MarketDataListenerData &operator=(const MarketDataListenerData &) noexcept =
-       delete;
-   MarketDataListenerData &operator=(MarketDataListenerData &&) noexcept =
-       default;
-   ~MarketDataListenerData() = default;
-
-   MarketDataListenerID id;
-   MarketDataSubscription subscription;
-};
-
-struct MDReplayData {
-   bts::api::md::MarketDataReplayConfig replay_conf;
-   bts::api::Venue venue;
-   bts::md::GeneralSecurityIndex sec_index;
-   bts::md::MDProducer md_producer;
-   bts::md::recording::MDReplayer md_replayer;
-
-   MDReplayData(const bts::api::md::MarketDataReplayConfig &replay_conf)
-       : replay_conf(replay_conf), venue(replay_conf.venue()), sec_index(),
-         md_replayer(replay_conf.venue(), sec_index, md_producer) {}
-};
-
-struct CCMDSourceData {
-   bts::md::GeneralSecurityIndex sec_index;
-   bts::md::MDProducer md_producer;
-   bts::md::GeneralMarketDataListener listener;
-
-   CCMDSourceData(bts::api::Venue venue)
-       : sec_index(), listener(sec_index, md_producer, venue) {}
-};
-
 struct MarketDataGateway::PImpl {
-   api::md::MarketDataConfig md_config =
-       config::get<api::md::MarketDataConfig>();
-   ccapi::EventDispatcher dispatcher;
-   MarketDataRegistrationFactory registration_factory;
+   struct MdSink {
+      GeneralSecurityIndex sec_index;
+      MDProducer md_producer;
+      shared_ptr<IMarketDataListener> listener;
+
+      MdSink(api::Venue venue, IDGenerator *sec_id_store)
+          : sec_index(), listener(std::make_shared<GeneralMarketDataListener>(
+                             sec_index, md_producer, venue, sec_id_store)) {}
+   };
+
+   MarketDataConfig config = config::get<MarketDataConfig>();
   IDGenerator &sec_id_store = IDGenerator::instance();
-   unordered_map<MarketDataListenerID, MarketDataListenerData> listeners;
-   std::vector<std::unique_ptr<MDReplayData>> replays = {};
-   // we create these and pass them to the cc md sources
-   std::vector<std::shared_ptr<CCMDSourceData>> cc_md_listeners = {};
+   vector<unique_ptr<IMarketDataProvider>> providers;
+   vector<unique_ptr<MdSink>> sinks;
+   vector<unique_ptr<IMarketDataSource>> sources;
+
+   PImpl() {
+      providers.emplace_back(std::make_unique<CCApiMarketDataProvider>(config));
+      providers.emplace_back(
+          std::make_unique<EmumbaMarketDataProvider>(config));
+      providers.emplace_back(std::make_unique<OnixsMarketDataProvider>(config));
+      providers.emplace_back(
+          std::make_unique<PyMarketDataProvider>(config, &sec_id_store));
+      providers.emplace_back(
+          std::make_unique<ReplayMarketDataProvider>(config, &sec_id_store));
+   }
 };

 MarketDataGateway::MarketDataGateway()
@@ -129,23 +85,20 @@ MarketDataGateway::~MarketDataGateway() = default;
 // starts the market data sources and then keeps the function going
 void MarketDataGateway::run() {
   vector<string> cfg_errs;
-   if (!validate(me_->md_config, cfg_errs)) {
+   if (!validate(me_->config, cfg_errs)) {
      BTS_FATAL_MSGF("Invalid MarketDataConfig:", FMT_JOIN(cfg_errs, "\n"));
      return;
   }
   LOG(md, Info, "MarketDataGateway::run starting t_id:{} t_name:{}",
       os::get_thread_id(), os::get_thread_name());
-   // _start_cc(me_->md_config);
-   if (!me_->md_config.emumba_deribit().empty()) {
-      bts_assert(me_->md_config.emumba_deribit().size() == 1);
-      _start_emumba_deribit(me_->md_config.emumba_deribit().at(0));
+   for (auto &provider : me_->providers) {
+      for (auto &[venue, source] : provider->get_sources()) {
+         const auto v = domain::DomainSecurityUtils::convert(venue);
+         auto &sink = me_->sinks.emplace_back(
+             std::make_unique<PImpl::MdSink>(v, &me_->sec_id_store));
+         source->start(sink->listener);
+      }
   }
-   if (!me_->md_config.glbx().empty()) {
-      bts_assert(me_->md_config.glbx().size() == 1);
-      _start_onix_glbx(me_->md_config.glbx().at(0));
-   }
-   _start_replays(me_->md_config);
-   start_py_();
   while (true) {
      std::this_thread::sleep_for(std::chrono::seconds(1));
   }
@@ -153,243 +106,4 @@ void MarketDataGateway::run() {
       os::get_thread_id(), os::get_thread_name());
 }

-void MarketDataGateway::test_py() {
-   start_py_();
-}
-
-void MarketDataGateway::start_py_() {
-   const Path wd{sparkd_wd()};
-   vector<string> module_paths{
-       (wd / "py_market_data_providers").string(),
-      //  (wd.parent_path() / "lib").string(),
-   };
-   start_py_(module_paths);
-}
-
-void MarketDataGateway::start_py_(
-    const std::vector<std::string> &module_paths) {
-   py::scoped_interpreter guard{}; // Start the Python interpreter
-
-   try {
-      // import sys
-      py::module sys = py::module::import("sys");
-      std::cout << "Appending module path" << std::endl;
-      for (const auto &module_path : module_paths) {
-         sys.attr("path").attr("append")(module_path.c_str());
-      }
-      std::cout << "Importing py_market_data module" << std::endl;
-      py::module spark_md = py::module::import("py_market_data_module");
-      std::cout << "Importing md_provider_impl module" << std::endl;
-      py::module my_implementation = py::module::import("md_provider_impl");
-      std::cout << "Instantiating MdProviderImpl" << std::endl;
-      py::object impl = my_implementation.attr("MdProviderImpl")();
-      std::cout << "Casting MdProviderImpl to MdProvider" << std::endl;
-      py_interop::MdProvider *provider = impl.cast<py_interop::MdProvider *>();
-      py_interop::MdSinkImpl sink;
-      std::cout << "Starting MdProvider(Impl)" << std::endl;
-      provider->start(&sink);
-      std::cout << "MdProvider(Impl) DONE" << std::endl;
-   } catch (const py::error_already_set &e) {
-      std::cerr << "Error: " << e.what() << std::endl;
-   }
-}
-
-// CCAPI //////////////////////////////////////////////////////////////////////
-///////////////////////////////////////////////////////////////////////////////
-MarketDataRegistration MarketDataGateway::_start_cc(
-    const MarketDataConfig &cfg) noexcept {
-   LOG(md, Info, "MarketDataGateway::_start_cc starting t_id:{} t_name:{}",
-       os::get_thread_id(), os::get_thread_name());
-   auto reg = me_->registration_factory.create();
-   bts_assert(reg.listener_id.has_value());
-   bts_assert(!CONTAINS_KEY(me_->listeners, *reg.listener_id));
-   LOG(md, Info, "MarketDataGateway subscribing to data:\n{}",
-       ProtoPrinter::to_string(cfg));
-
-   vector<string> cfg_errs;
-   if (!validate(cfg, cfg_errs)) {
-      return MarketDataRegistration{
-          {MarketDataRegistrationError::Type::CONFIG_ERROR,
-           fmt::to_string(fmt::join(cfg_errs, "\n"))}};
-   }
-
-   MarketDataListenerData &listener_data = me_->listeners[*reg.listener_id];
-   listener_data.id = *reg.listener_id;
-
-   // FTX
-   for (const auto &ftx : cfg.ftx()) {
-      auto p = me_->cc_md_listeners.emplace_back(
-          std::make_shared<CCMDSourceData>(bts::api::VENUE_FTX));
-      listener_data.subscription.add(ftx::FTXSubscription{
-          &me_->dispatcher, ftx, &me_->sec_id_store, &p->listener});
-   }
-
-   // Binance
-   for (const auto &binance : cfg.binance()) {
-      auto p = me_->cc_md_listeners.emplace_back(
-          std::make_shared<CCMDSourceData>(bts::api::VENUE_BINANCE));
-      listener_data.subscription.add(binance::BinanceSubscription{
-          &me_->dispatcher, binance, &me_->sec_id_store, &p->listener});
-   }
-
-   // Binance US
-   for (const auto &binance_us : cfg.binance_us()) {
-      auto p = me_->cc_md_listeners.emplace_back(
-          std::make_shared<CCMDSourceData>(bts::api::VENUE_BINANCE_US));
-      listener_data.subscription.add(binance_us::BinanceUsSubscription{
-          &me_->dispatcher, binance_us, &me_->sec_id_store, &p->listener});
-   }
-   // Coinbase
-   for (const auto &coinbase : cfg.coinbase()) {
-      auto p = me_->cc_md_listeners.emplace_back(
-          std::make_shared<CCMDSourceData>(bts::api::VENUE_COINBASE));
-      listener_data.subscription.add(coinbase::CoinbaseSubscription{
-          &me_->dispatcher, coinbase, &me_->sec_id_store, &p->listener});
-   }
-   // Binance USDS Futures
-   for (const auto &binance_ufs : cfg.binance_usds_futures()) {
-      auto p =
-          me_->cc_md_listeners.emplace_back(std::make_shared<CCMDSourceData>(
-              bts::api::VENUE_BINANCE_USDS_FUTURES));
-      listener_data.subscription.add(
-          binance_usds_futures::BinanceUsdsFuturesSubscription{
-              &me_->dispatcher, binance_ufs, &me_->sec_id_store, &p->listener});
-   }
-
-   // Deribit
-   for (const auto &deribit : cfg.deribit()) {
-      auto p = me_->cc_md_listeners.emplace_back(
-          std::make_shared<CCMDSourceData>(bts::api::VENUE_DERIBIT));
-      listener_data.subscription.add(deribit::DeribitSubscription{
-          &me_->dispatcher, deribit, &me_->sec_id_store, &p->listener});
-   }
-
-   // Okx
-   for (const auto &okx : cfg.okx()) {
-      auto p = me_->cc_md_listeners.emplace_back(
-          std::make_shared<CCMDSourceData>(bts::api::VENUE_OKX));
-      listener_data.subscription.add(okx::OkxSubscription{
-          &me_->dispatcher, okx, &me_->sec_id_store, &p->listener});
-   }
-
-   LOG(md, Info, "MarketDataGateway::_start_cc ending t_id:{} t_name:{}",
-       os::get_thread_id(), os::get_thread_name());
-   return reg;
-}
-
-void MarketDataGateway::unsubscribe(MarketDataListenerID id) {
-   LOG(md, Info, "Unsubscribing listener {}", id);
-   me_->listeners.erase(id);
-}
-/// glbx //////////////////////////////////////////////////////////////////////
-//////////////////////////////////////////////////////////////////////////////
-static void run_onixs() {
-   LOG(md, Info, "MarketDataGateway::run_onixs t_id:{} t_name",
-       os::get_thread_id(), os::get_thread_name());
-   bts::md::MBOWriter *mbo = nullptr;
-   bts::md::TradeSummarySignalDispatcher *tssd = nullptr;
-   std::vector<std::shared_ptr<bts::md::IMDPlatform>> platforms;
-   // get all glbx symbols
-   auto cme_symbols = SymbolLoader::load(
-       bts::domain::SecurityCriteria::create({api::domain::VENUE_GLBX}));
-   if (!cme_symbols.empty()) {
-      auto *main = new bts::md::MDP3Main();
-      main->start({cme_symbols.begin(), cme_symbols.end()}, mbo, tssd);
-      platforms.emplace_back(main);
-   }
-   // deribit
-   // auto *d_platform = new bts::md::EmMDListener()
-   // Timers
-   auto &disp = utils::Dispatcher::current();
-   utils::DispatcherTimer heartbeat_timer(disp, 250, [&]() {
-      for (auto &p : platforms)
-         p->send_heartbeats();
-   });
-   heartbeat_timer.start();
-
-   disp.run();
-}
-
-void MarketDataGateway::_start_onix_glbx(
-    bts::api::md::GlbxMarketDataConfig cfg) {
-   LOG(md, Info,
-       "MarketDataGateway::_start_onix_glbx starting t_id:{} t_name:{}. "
-       "cfg:\n{}",
-       os::get_thread_id(), os::get_thread_name(), cfg.DebugString());
-   bts::md::run_onixs();
-}
-
-// deribit ////////////////////////////////////////////////////////////////////
-///////////////////////////////////////////////////////////////////////////////
-
-void MarketDataGateway::_start_emumba_deribit(
-    bts::api::md::EmumbaDeribitMarketDataConfig cfg) {
-   LOG(md, Info,
-       "MarketDataGateway::_start_emumba_deribit starting t_id:{} t_name:{}. "
-       "cfg:\n{}",
-       os::get_thread_id(), os::get_thread_name(), cfg.DebugString());
-   auto start_deribit_thr = [=]() {
-      const auto config_path = Path{config::get<SystemConfig>().config_path()} /
-                               cfg.json_conf_file_name();
-      const auto conf =
-          bts::cmds::DeribitDemoConfiguration::read_file(config_path.string());
-      bts_assert_msg(conf.has_value(), "Failed to parse deribit config");
-      // log
-      for (const auto &s : conf.value().symbol_config_list) {
-         LOG(md, Info, "sub.symbol: {}", s.symbol_name);
-         LOG(md, Info, "sub.futs: {} sub.options: {}", s.interested_in_future,
-             s.interested_in_option);
-         for (const auto &i : s.interested_instruments) {
-            LOG(md, Info, "sub.instrument: {}", i);
-         }
-      }
-      EmumbaDeribitData ddata_obj;
-      EmumbaDeribitData *ddata = &ddata_obj;
-      // end log
-      bts_assert(conf.has_value());
-      LOG(md, Info, "Deribit conf: {}", conf.value());
-      ddata->real_handler =
-          std::make_shared<emumba::deribit_connector::deribit_handler>(
-              conf.value().symbol_config_list, conf.value().trading_config,
-              conf.value().credential_map, conf.value().logging_configuration,
-              conf.value().deribit_url, &ddata->listener);
-      // since the trading interface is housed inside the EmMDListener, we need to
-      // create and send emumba's trading_session_interface
-      for (const auto &p : conf.value().credential_map) {
-         const auto &sess = p.second;
-         auto ts_ptr = std::make_shared<
-             emumba::deribit_connector::trading_session_interface>(
-             ddata->real_handler->create_new_trading_session(sess));
-         ddata->trading_session_vec.push_back({sess, ts_ptr});
-      }
-      bts_assert(!ddata->trading_session_vec.empty());
-      ddata->listener.set_handler(
-          ddata->real_handler.get(), ddata->trading_session_vec);
-      ddata->real_handler->run();
-   };
-   threading::make_thread("Emumba", [&]() { start_deribit_thr(); }).detach();
-   LOG(md, Info, "MarketDataGateway::_start_deribit ending t_id:{} t_name:{}",
-       os::get_thread_id(), os::get_thread_name());
-}
-
-// replays ////////////////////////////////////////////////////////////////////
-///////////////////////////////////////////////////////////////////////////////
-void MarketDataGateway::_start_replays(
-    const bts::api::md::MarketDataConfig &cfg) {
-   LOG(md, Info, "MarketDataGateway::_start_replays starting t_id:{} t_name:{}",
-       os::get_thread_id(), os::get_thread_name());
-   for (const auto &r :
-        MarketDataConfigUtils::get_effective_replay_config(cfg)) {
-      me_->replays.emplace_back(std::make_unique<MDReplayData>(r));
-   }
-   for (const auto &replayer : me_->replays) {
-      std::string tid_name = "MDReplayer_" + Venue_Name(replayer->venue);
-      bts::threading::make_thread(tid_name, [&]() {
-         replayer->md_replayer.run();
-      }).detach();
-   }
-   LOG(md, Info, "MarketDataGateway::_start_replays ending t_id:{} t_name:{}",
-       os::get_thread_id(), os::get_thread_name());
-}
-
 } // namespace bts::md
@@ -1,43 +1,23 @@
 #pragma once

-#include "spark/api/md/GlbxMarketDataConfig.pb.h"
-#include "sparkd/md/MarketDataListenerID.h"
-
 #include <memory>

-namespace bts::api::md {
-class EmumbaDeribitMarketDataConfig;
-class MarketDataConfig;
-} // namespace bts::api::md
-
 namespace bts::md {

-class IMarketDataListener;
-class MarketDataRegistration;
-
+/**
+ * @brief The `MarketDataGateway` is responsible for starting all configured market data sources.
+ * It connects each source to a special `IMarketDataListener` that writes the data to shared memory.
+ */
 class MarketDataGateway {
 public:
   MarketDataGateway();
   ~MarketDataGateway();

-   /**
-    * @brief Connects a market data listener to a market data source
-    * @return MarketDataRegistration
-    */
   void run();
-   void unsubscribe(MarketDataListenerID id);
-   static void test_py();

 private:
   struct PImpl;
   std::unique_ptr<PImpl> me_;
-   MarketDataRegistration _start_cc(const api::md::MarketDataConfig &) noexcept;
-   static void start_py_();
-   static void start_py_(const std::vector<std::string> &module_paths);
-   static void _start_emumba_deribit(
-       bts::api::md::EmumbaDeribitMarketDataConfig cfg);
-   static void _start_onix_glbx(bts::api::md::GlbxMarketDataConfig cfg);
-   void _start_replays(const bts::api::md::MarketDataConfig &cfg);
 };

 } // namespace bts::md
@@ -1,11 +1,12 @@
 #pragma once

-#include "spark/md/IMarketDataListener.h"
-#include "spark/domain/Book.h"
-#include "spark/domain/Security.h"
+#include "domain/book.h"
+#include "domain/rfq.h"
+#include "domain/security.h"
+#include "domain/security_state.h"
+#include "domain/trade_record.h"
 #include "spark/domain/SecurityMarketDataStatistics.h"
-#include "spark/domain/SecurityState.h"
-#include "spark/domain/TradeRecord.h"
+#include "spark/md/IMarketDataListener.h"
 #include "sparkd/md/MarketDataError.h"

 namespace bts::md {
@@ -1,8 +1,8 @@
 #pragma once
 #ifndef _WIN32

-#include "spark/core/FMT_IMPL.h"
 #include "spark/core/ValueWrapper.h"
+#include "system/FMT_IMPL.h"

 namespace bts::md {

@@ -1,60 +1,60 @@
-#include "MarketDataSubscription.h"
-#include "sparkd/md/binance/BinanceSubscription.h"
-#include "sparkd/md/binance_us/BinanceUsSubscription.h"
-#include "sparkd/md/binance_usds_futures/BinanceUsdsFuturesSubscription.h"
-#include "sparkd/md/coinbase/CoinbaseSubscription.h"
-#include "sparkd/md/deribit/DeribitSubscription.h"
-#include "sparkd/md/ftx/FTXSubscription.h"
-#include "sparkd/md/okx/OkxSubscription.h"
+// #include "MarketDataSubscription.h"
+// #include "sparkd/md/binance/BinanceSubscription.h"
+// #include "sparkd/md/binance_us/BinanceUsSubscription.h"
+// #include "sparkd/md/binance_usds_futures/BinanceUsdsFuturesSubscription.h"
+// #include "sparkd/md/coinbase/CoinbaseSubscription.h"
+// #include "sparkd/md/deribit/DeribitSubscription.h"
+// #include "sparkd/md/ftx/FTXSubscription.h"
+// #include "sparkd/md/okx/OkxSubscription.h"

-#include <vector>
+// #include <vector>

-namespace bts::md {
+// namespace bts::md {

-using std::vector;
+// using std::vector;

-struct MarketDataSubscription::PImpl {
-   vector<ftx::FTXSubscription> ftx_subs;
-   vector<binance::BinanceSubscription> binance_subs;
-   vector<binance_us::BinanceUsSubscription> binance_us_subs;
-   vector<coinbase::CoinbaseSubscription> coinbase_subs;
-   vector<binance_usds_futures::BinanceUsdsFuturesSubscription>
-       binance_usds_futures_subs;
-   vector<deribit::DeribitSubscription> deribit_subs;
-   vector<okx::OkxSubscription> okx_subs;
-   // TODO: add more exchanges
-};
+// struct MarketDataSubscription::PImpl {
+//    vector<ftx::FTXSubscription> ftx_subs;
+//    vector<binance::BinanceSubscription> binance_subs;
+//    vector<binance_us::BinanceUsSubscription> binance_us_subs;
+//    vector<coinbase::CoinbaseSubscription> coinbase_subs;
+//    vector<binance_usds_futures::BinanceUsdsFuturesSubscription>
+//        binance_usds_futures_subs;
+//    vector<deribit::DeribitSubscription> deribit_subs;
+//    vector<okx::OkxSubscription> okx_subs;
+//    // TODO: add more exchanges
+// };

-MarketDataSubscription::MarketDataSubscription()
-    : me_(std::make_unique<PImpl>()) {}
+// MarketDataSubscription::MarketDataSubscription()
+//     : me_(std::make_unique<PImpl>()) {}

-MarketDataSubscription::~MarketDataSubscription() = default;
+// MarketDataSubscription::~MarketDataSubscription() = default;

-void MarketDataSubscription::add(binance::BinanceSubscription sub) {
-   me_->binance_subs.push_back(std::move(sub));
-}
+// void MarketDataSubscription::add(binance::BinanceSubscription sub) {
+//    me_->binance_subs.push_back(std::move(sub));
+// }

-void MarketDataSubscription::add(binance_us::BinanceUsSubscription sub) {
-   me_->binance_us_subs.push_back(std::move(sub));
-}
-void MarketDataSubscription::add(coinbase::CoinbaseSubscription sub) {
-   me_->coinbase_subs.push_back(std::move(sub));
-}
-void MarketDataSubscription::add(
-    binance_usds_futures::BinanceUsdsFuturesSubscription sub) {
-   me_->binance_usds_futures_subs.push_back(std::move(sub));
-}
+// void MarketDataSubscription::add(binance_us::BinanceUsSubscription sub) {
+//    me_->binance_us_subs.push_back(std::move(sub));
+// }
+// void MarketDataSubscription::add(coinbase::CoinbaseSubscription sub) {
+//    me_->coinbase_subs.push_back(std::move(sub));
+// }
+// void MarketDataSubscription::add(
+//     binance_usds_futures::BinanceUsdsFuturesSubscription sub) {
+//    me_->binance_usds_futures_subs.push_back(std::move(sub));
+// }

-void MarketDataSubscription::add(deribit::DeribitSubscription sub) {
-   me_->deribit_subs.push_back(std::move(sub));
-}
+// void MarketDataSubscription::add(deribit::DeribitSubscription sub) {
+//    me_->deribit_subs.push_back(std::move(sub));
+// }

-void MarketDataSubscription::add(ftx::FTXSubscription sub) {
-   me_->ftx_subs.push_back(std::move(sub));
-}
+// void MarketDataSubscription::add(ftx::FTXSubscription sub) {
+//    me_->ftx_subs.push_back(std::move(sub));
+// }

-void MarketDataSubscription::add(okx::OkxSubscription sub) {
-   me_->okx_subs.push_back(std::move(sub));
-}
+// void MarketDataSubscription::add(okx::OkxSubscription sub) {
+//    me_->okx_subs.push_back(std::move(sub));
+// }

-} // namespace bts::md
+// } // namespace bts::md
@@ -1,59 +1,59 @@
-#pragma once
+// #pragma once

-#include <memory>
+// #include <memory>

-namespace bts::md::binance {
-class BinanceSubscription;
-}
+// namespace bts::md::binance {
+// class BinanceSubscription;
+// }

-namespace bts::md::binance_us {
-class BinanceUsSubscription;
-}
-namespace bts::md::coinbase {
-class CoinbaseSubscription;
-}
-namespace bts::md::binance_usds_futures {
-class BinanceUsdsFuturesSubscription;
-}
+// namespace bts::md::binance_us {
+// class BinanceUsSubscription;
+// }
+// namespace bts::md::coinbase {
+// class CoinbaseSubscription;
+// }
+// namespace bts::md::binance_usds_futures {
+// class BinanceUsdsFuturesSubscription;
+// }

-namespace bts::md::deribit {
-class DeribitSubscription;
-}
+// namespace bts::md::deribit {
+// class DeribitSubscription;
+// }

-namespace bts::md::ftx {
-class FTXSubscription;
-}
+// namespace bts::md::ftx {
+// class FTXSubscription;
+// }

-namespace bts::md::okx {
-class OkxSubscription;
-}
+// namespace bts::md::okx {
+// class OkxSubscription;
+// }

-namespace bts::md {
+// namespace bts::md {

-class MarketDataSubscription {
- public:
-   MarketDataSubscription();
+// class MarketDataSubscription {
+//  public:
+//    MarketDataSubscription();

-   MarketDataSubscription(const MarketDataSubscription &) noexcept = delete;
-   MarketDataSubscription(MarketDataSubscription &&) noexcept = default;
-   MarketDataSubscription &operator=(const MarketDataSubscription &) noexcept =
-       delete;
-   MarketDataSubscription &operator=(MarketDataSubscription &&) noexcept =
-       default;
+//    MarketDataSubscription(const MarketDataSubscription &) noexcept = delete;
+//    MarketDataSubscription(MarketDataSubscription &&) noexcept = default;
+//    MarketDataSubscription &operator=(const MarketDataSubscription &) noexcept =
+//        delete;
+//    MarketDataSubscription &operator=(MarketDataSubscription &&) noexcept =
+//        default;

-   ~MarketDataSubscription();
+//    ~MarketDataSubscription();

-   void add(binance::BinanceSubscription);
-   void add(binance_us::BinanceUsSubscription);
-   void add(coinbase::CoinbaseSubscription);
-   void add(binance_usds_futures::BinanceUsdsFuturesSubscription);
-   void add(deribit::DeribitSubscription);
-   void add(ftx::FTXSubscription);
-   void add(okx::OkxSubscription);
+//    void add(binance::BinanceSubscription);
+//    void add(binance_us::BinanceUsSubscription);
+//    void add(coinbase::CoinbaseSubscription);
+//    void add(binance_usds_futures::BinanceUsdsFuturesSubscription);
+//    void add(deribit::DeribitSubscription);
+//    void add(ftx::FTXSubscription);
+//    void add(okx::OkxSubscription);

- private:
-   struct PImpl;
-   std::unique_ptr<PImpl> me_;
-};
+//  private:
+//    struct PImpl;
+//    std::unique_ptr<PImpl> me_;
+// };

-} // namespace bts::md
+// } // namespace bts::md
@@ -0,0 +1,47 @@
+#include "OnixsMarketDataProvider.h"
+#include "domain/protobufs/venue.pb.h"
+#include "spark/config/config.h"
+#include "spark/persistence/IDGenerator.h"
+#include "sparkd/md/glbx/OnixsGlbxSubscription.h"
+
+#include <boost/filesystem.hpp>
+#include <memory>
+#include <vector>
+
+namespace bts::md {
+
+using bts::api::md::MarketDataConfig;
+using bts::persistence::IDGenerator;
+using std::pair;
+using std::shared_ptr;
+using std::string;
+using std::vector;
+
+using Path = boost::filesystem::path;
+
+struct OnixsMarketDataProvider::PImpl {
+   MarketDataConfig config;
+   IDGenerator &sec_id_store = IDGenerator::instance();
+   shared_ptr<OnixsGlbxSubscription> source;
+};
+
+OnixsMarketDataProvider::OnixsMarketDataProvider()
+    : bts::md::OnixsMarketDataProvider(bts::config::get<MarketDataConfig>()) {}
+
+OnixsMarketDataProvider::OnixsMarketDataProvider(MarketDataConfig config)
+    : me_(std::make_unique<PImpl>()) {
+   me_->config = std::move(config);
+}
+
+OnixsMarketDataProvider::~OnixsMarketDataProvider() = default;
+
+vector<pair<domain::protobufs::Venue, shared_ptr<IMarketDataSource>>>
+OnixsMarketDataProvider::get_sources() {
+   vector<pair<domain::protobufs::Venue, shared_ptr<IMarketDataSource>>> rv;
+   me_->source =
+       std::make_shared<OnixsGlbxSubscription>(&me_->sec_id_store, me_->config);
+   rv.push_back({domain::protobufs::VENUE_GLBX, me_->source});
+   return rv;
+}
+
+} // namespace bts::md
@@ -0,0 +1,25 @@
+#pragma once
+
+#include "spark/api/md/MarketDataConfig.pb.h"
+#include "sparkd/md/IMarketDataProvider.h"
+#include "sparkd/md/IMarketDataSource.h"
+
+#include <memory>
+
+namespace bts::md {
+
+class OnixsMarketDataProvider : public IMarketDataProvider {
+ public:
+   OnixsMarketDataProvider();
+   explicit OnixsMarketDataProvider(api::md::MarketDataConfig config);
+   ~OnixsMarketDataProvider();
+
+   std::vector<std::pair<domain::protobufs::Venue, std::shared_ptr<IMarketDataSource>>>
+   get_sources() override;
+
+ private:
+   struct PImpl;
+   std::unique_ptr<PImpl> me_;
+};
+
+} // namespace bts::md
@@ -0,0 +1,58 @@
+#include "PyMarketDataProvider.h"
+#include "spark/api/md/MarketDataConfig.pb.h"
+#include "spark/os/os.h"
+#include "spark/persistence/IDGenerator.h"
+#include "sparkd/md/IMarketDataSource.h"
+#include "sparkd/md/MarketDataSubscription.h"
+#include "sparkd/md/PyMarketDataSubscription.h"
+
+namespace bts::md {
+
+using bts::os::sparkd_wd;
+using bts::persistence::IDGenerator;
+using std::pair;
+using std::shared_ptr;
+using std::string;
+using std::vector;
+
+using Path = boost::filesystem::path;
+
+struct PyMarketDataProvider::PImpl {
+   api::md::MarketDataConfig config;
+   IDGenerator *sec_id_store;
+   vector<shared_ptr<PyMarketDataSubscription>> py_md_subs = {};
+};
+
+PyMarketDataProvider::PyMarketDataProvider(
+    api::md::MarketDataConfig config, IDGenerator *sec_id_store)
+    : me_(std::make_unique<PImpl>()) {
+   me_->config = std::move(config);
+   me_->sec_id_store = sec_id_store;
+}
+
+PyMarketDataProvider::~PyMarketDataProvider() = default;
+
+vector<pair<domain::protobufs::Venue, shared_ptr<IMarketDataSource>>>
+PyMarketDataProvider::get_sources() {
+   vector<pair<domain::protobufs::Venue, shared_ptr<IMarketDataSource>>> rv;
+   const Path wd{sparkd_wd()};
+   vector<string> module_paths{
+       wd.string(),
+       (wd / "py_market_data_providers").string(),
+   };
+   for (const auto &py_plugin : me_->config.python_md_providers()) {
+      vector<string> module_path;
+      for (const auto &p : py_plugin.module_path()) {
+         module_path.emplace_back(p);
+      }
+      auto source = std::make_shared<PyMarketDataSubscription>(
+          py_plugin.provider_type(), module_path, me_->sec_id_store);
+      me_->py_md_subs.push_back(source);
+      rv.push_back(std::make_pair<
+                   domain::protobufs::Venue, shared_ptr<IMarketDataSource>>(
+          py_plugin.venue(), source));
+   }
+   return rv;
+}
+
+} // namespace bts::md
@@ -0,0 +1,27 @@
+#pragma once
+
+#include "spark/api/md/MarketDataConfig.pb.h"
+#include "spark/persistence/IDGenerator.h"
+#include "sparkd/md/IMarketDataProvider.h"
+
+#include <memory>
+
+namespace bts::md {
+
+class PyMarketDataProvider : public IMarketDataProvider {
+ public:
+   explicit PyMarketDataProvider(
+       api::md::MarketDataConfig config,
+       persistence::IDGenerator *sec_id_store);
+   ~PyMarketDataProvider();
+
+   std::vector<
+       std::pair<domain::protobufs::Venue, std::shared_ptr<IMarketDataSource>>>
+   get_sources() override;
+
+ private:
+   struct PImpl;
+   std::unique_ptr<PImpl> me_;
+};
+
+} // namespace bts::md
@@ -0,0 +1,249 @@
+#include "PyMarketDataSubscription.h"
+#include "domain/future_security.h"
+#include "domain/protobufs/aggressor_side.pb.h"
+#include "domain/protobufs/call_put.pb.h"
+#include "domain/protobufs/venue.h"
+#include "domain/security.h"
+#include "domain/security_id.h"
+#include "domain/spot_security.h"
+#include "domain/trade_record.h"
+#include "python_md/md_sink.h"
+#include "python_md/py_book.h"
+#include "python_md/py_md_executor.h"
+#include "python_md/py_security.h"
+#include "python_md/py_security_list.h"
+#include "python_md/py_trade.h"
+#include "spark/logging/logging.h"
+#include "spark/md/IMarketDataListener.h"
+#include "spark/os/os.h"
+#include "spark/persistence/IDGenerator.h"
+#include "spark/utils/bts_assert.h"
+#include "system/time_stamp.h"
+
+#include <memory>
+#include <optional>
+#include <pybind11/embed.h>
+#include <pybind11/pybind11.h>
+#include <vector>
+
+namespace bts::md {
+
+#define AGGRESSOR_SIDE(x) bts::domain::protobufs::AGGRESSOR_SIDE_##x
+
+using bts::domain::SecurityID;
+using bts::md::py_interop::PyBook;
+using bts::md::py_interop::PyBookList;
+using bts::md::py_interop::PyBookSide;
+using bts::md::py_interop::PyMdExecutor;
+using bts::md::py_interop::PySecurityList;
+using bts::md::py_interop::PyTrade;
+using bts::md::py_interop::PyTradeList;
+using bts::persistence::IDGenerator;
+using std::shared_ptr;
+using std::string;
+using std::vector;
+
+using Path = boost::filesystem::path;
+
+struct TradeTuple {
+   py_interop::PyTrade py_trade;
+   domain::TradeRecord trade;
+   domain::protobufs::Venue venue;
+};
+
+struct PyMarketDataSubscription::PImpl : private py_interop::MdSink {
+   PImpl(
+       string py_plugin, vector<string> module_paths, IDGenerator *sec_id_store)
+       : py_plugin_(std::move(py_plugin)),
+         module_paths_(std::move(module_paths)),
+         py_md_exec_(py_plugin_, module_paths_, this),
+         sec_id_store_(sec_id_store) {}
+
+   void start(shared_ptr<IMarketDataListener> listener) {
+      bts_assert(listener != nullptr);
+      this->listener_ = std::move(listener);
+      LOG(md, Info, "PyMarketDataSubscription::start t_id:{} t_name:{}",
+          os::get_thread_id(), os::get_thread_name());
+      py_md_exec_.start();
+   }
+
+   void stop() { py_md_exec_.stop(); }
+
+ private:
+   string py_plugin_;
+   vector<string> module_paths_;
+   PyMdExecutor py_md_exec_;
+   shared_ptr<IMarketDataListener> listener_;
+
+   IDGenerator *sec_id_store_;
+   // The main store for securities.
+   vector<domain::Security> securities_;
+   vector<const domain::Security *> security_ptrs_;
+   domain::Book book_;
+   // Since the python md provider isn't tethered to a single venue, we
+   // must know the venue for each individual trade.
+   vector<domain::TradeRecord> trades_;
+   vector<const domain::TradeRecord *> trade_ptrs_;
+
+   void on_securities(const PySecurityList &py_securities) override {
+      LOG(md, Info, "Received securities from python plugin. count: {}",
+          py_securities.size());
+      // `securities_` and `security_ptrs_` should already be empty, but no harm in clearing them
+      // again. We keep them as member variables to avoid (re)allocating memory in each callback.
+      securities_.clear();
+      security_ptrs_.clear();
+      for (const auto &py_sec : py_securities.items()) {
+         LOG(md, Info, "PyMarketDataSubscription adding security {}",
+             py_sec.name());
+         auto sec = to_bts_security_(py_sec);
+         if (!sec) {
+            LOG(md, Error,
+                "Failed to convert security {} to bts::domain::Security",
+                py_sec.name());
+            continue;
+         }
+         securities_.push_back(std::move(*sec));
+      }
+      // Populate vectors of security ptrs per each venue.
+      for (auto &sec : securities_) {
+         security_ptrs_.push_back(&sec);
+      }
+      // Pass securities to the sinks (per venue).
+      listener_->on_securities(security_ptrs_);
+      securities_.clear();
+      security_ptrs_.clear();
+   }
+
+   void on_books(const PyBookList &py_books) override {
+      for (auto &py_book : py_books.items()) {
+         bts_assert(book_.count_populated() == 0);
+         to_book_(py_book, book_);
+         listener_->on_book(book_);
+         book_.clear();
+      }
+   }
+
+   void on_trades(const PyTradeList &py_trades) override {
+      LOG(md, Info, "Received trades from python plugin. count: {}",
+          py_trades.size());
+      // `trades_` and `trade_ptrs_` should already be empty, but no harm in clearing them again.
+      // We keep them as member variables to avoid (re)allocating memory in each callback.
+      trades_.clear();
+      trade_ptrs_.clear();
+      for (const auto &py_trade : py_trades.items()) {
+         LOG(md, Info, "PyMarketDataSubscription received trade {}",
+             py_trade.to_string());
+         to_bts_trade_(py_trade, trades_.emplace_back());
+      }
+      for (const auto &t : trades_) {
+         trade_ptrs_.push_back(&t);
+      }
+      listener_->on_trades(trade_ptrs_);
+      trades_.clear();
+      trade_ptrs_.clear();
+   }
+
+   SecurityID get_sec_id_(const string &venue_name, const string &sec_name) {
+      domain::protobufs::Venue venue;
+      // Although an overload of 'get_id' accepts a string, we parse `venue_name` to the enum to
+      // normalize different spellings of the same venue. For example, "deribt", "DERIBIT", and
+      // "VENUE_deribit" all represent the VENUE_DERIBIT enum (see `venue_parse_smart` for details).
+      bts_assert(domain::protobufs::venue_parse_smart(venue_name, &venue));
+      auto sec_id = sec_id_store_->get_id(venue, sec_name, true);
+      bts_assert_msgf(
+          sec_id, "Failed to get security id for py_book. venue: {}; name: {}",
+          venue, sec_name);
+      return SecurityID{*sec_id};
+   }
+
+   std::optional<domain::Security> to_bts_security_(
+       const py_interop::PySecurity &py_sec) {
+      // We can't let python provider assign the id, because it is unaware of other
+      // md providers, so can't possibly guarentee a unique id.
+      bts_assert(py_sec.id() == 0);
+      domain::Security sec;
+      auto &sec_comm = sec.security_common;
+      if (!domain::protobufs::venue_parse_smart(
+              py_sec.venue(), &sec_comm.venue)) {
+         LOG(md, Error, "Failed to parse venue from '{}'", py_sec.venue());
+         return std::nullopt;
+      }
+      sec_comm.id = get_sec_id_(py_sec.venue(), py_sec.exchange_id());
+      sec_comm.exchange_sec_id = py_sec.exchange_id();
+      sec_comm.currency = py_sec.currency();
+      sec_comm.asset = py_sec.asset();
+      sec_comm.group = py_sec.group();
+      sec_comm.symbol = py_sec.asset();
+      if (sec_comm.symbol.empty()) {
+         sec_comm.symbol = py_sec.group();
+      }
+      sec_comm.desc = py_sec.name();
+      if (py_sec.is_spot()) {
+         sec.security = domain::SpotSecurity{};
+      } else if (py_sec.is_future()) {
+         domain::FutureSecurity bts_fut;
+         bts_fut.expiration =
+             core::TimeStamp::from_iso_string(py_sec.future().expiration());
+         sec.security = std::move(bts_fut);
+      } else if (py_sec.is_option()) {
+         domain::OptionSecurity opt;
+         const auto &py_opt = py_sec.option();
+         opt.expiration = core::TimeStamp::from_iso_string(py_opt.expiration());
+         opt.call_put = py_opt.is_call() ? domain::protobufs::CallPut::CALL
+                                         : domain::protobufs::CallPut::PUT;
+         opt.strike = domain::Price::from_double(py_opt.strike());
+         sec.security = std::move(opt);
+      } else {
+         LOG(md, Error,
+             "PySecurity has incomplete definition (not spot, future, or "
+             "option)");
+         return std::nullopt;
+      }
+      return sec;
+   }
+
+   void to_bts_trade_(const PyTrade &source, domain::TradeRecord &dest) {
+      dest.sec_id = get_sec_id_(source.venue(), source.exchange_security_id());
+      dest.time_stamp = core::TimeStamp::from_nanos(source.timestamp());
+      dest.agg_side = source.aggressor_side().starts_with('B') ||
+                              source.aggressor_side().starts_with('b')
+                          ? AGGRESSOR_SIDE(BUY)
+                          : AGGRESSOR_SIDE(SELL);
+      dest.trade_price = domain::Price::from_double(source.price());
+      *dest.trade_quantity = source.quantity();
+   }
+
+   void to_book_side_(const PyBookSide &source, domain::BookSide &dest) {
+      for (int i = 0; i < source.size(); i++) {
+         auto py_level = source.get_level(i);
+         const auto price = py_level.price();
+         const auto quantity = py_level.quantity();
+         bts_assert(!std::isnan(price));
+         bts_assert(quantity > 0);
+         *dest[i].total_quantity = quantity;
+         *dest[i].direct_quantity = quantity;
+         dest[i].price = domain::Price::from_double(price);
+      }
+   }
+
+   void to_book_(const PyBook &source, domain::Book &dest) {
+      to_book_side_(source.bids(), dest.bids());
+      to_book_side_(source.asks(), dest.asks());
+      dest.set_security_id(
+          get_sec_id_(source.venue(), source.exchange_security_id()));
+   }
+};
+
+PyMarketDataSubscription::PyMarketDataSubscription(
+    string py_plugin, vector<string> module_paths, IDGenerator *sec_id_store)
+    : me_(std::make_unique<PyMarketDataSubscription::PImpl>(
+          std::move(py_plugin), std::move(module_paths), sec_id_store)) {}
+
+PyMarketDataSubscription::~PyMarketDataSubscription() { me_->stop(); }
+
+void PyMarketDataSubscription::start(shared_ptr<IMarketDataListener> listener) {
+   me_->start(std::move(listener));
+}
+void PyMarketDataSubscription::stop() { me_->stop(); }
+
+} // namespace bts::md
@@ -0,0 +1,31 @@
+#pragma once
+
+#include "sparkd/md/IMarketDataSource.h"
+
+#include <memory>
+#include <string>
+#include <vector>
+
+namespace bts::persistence {
+class IDGenerator;
+}
+
+namespace bts::md {
+
+class PyMarketDataSubscription : public IMarketDataSource {
+ public:
+   explicit PyMarketDataSubscription(
+       std::string py_plugin,
+       std::vector<std::string> module_paths,
+       persistence::IDGenerator *sec_id_store);
+   ~PyMarketDataSubscription();
+
+   void start(std::shared_ptr<IMarketDataListener> listener) override;
+   void stop() override;
+
+ private:
+   struct PImpl;
+   std::unique_ptr<PImpl> me_;
+};
+
+} // namespace bts::md
@@ -0,0 +1,51 @@
+#include "ReplayMarketDataProvider.h"
+#include "spark/api/md/MarketDataConfig.pb.h"
+#include "spark/api/md/MarketDataConfigUtils.h"
+#include "spark/config/config.h"
+#include "spark/domain/DomainSecurityUtils.h"
+#include "spark/persistence/IDGenerator.h"
+#include "sparkd/md/IMarketDataSource.h"
+#include "sparkd/md/recording/MDReplayer.h"
+
+namespace bts::md {
+
+using bts::api::md::MarketDataConfig;
+using bts::md::recording::MDReplayer;
+using bts::persistence::IDGenerator;
+using std::pair;
+using std::shared_ptr;
+using std::string;
+using std::vector;
+
+struct ReplayMarketDataProvider::PImpl {
+   MarketDataConfig config;
+   IDGenerator *sec_id_store;
+   vector<pair<domain::protobufs::Venue, shared_ptr<IMarketDataSource>>>
+       sources;
+};
+
+ReplayMarketDataProvider::ReplayMarketDataProvider()
+    : ReplayMarketDataProvider(
+          bts::config::get<MarketDataConfig>(), &IDGenerator::instance()) {}
+
+ReplayMarketDataProvider::ReplayMarketDataProvider(
+    MarketDataConfig config, IDGenerator *sec_id_store)
+    : me_(std::make_unique<PImpl>()) {
+   me_->config = std::move(config);
+   me_->sec_id_store = sec_id_store;
+}
+
+ReplayMarketDataProvider::~ReplayMarketDataProvider() = default;
+
+vector<pair<domain::protobufs::Venue, shared_ptr<IMarketDataSource>>>
+ReplayMarketDataProvider::get_sources() {
+   const auto replay_cfgs =
+       MarketDataConfigUtils::get_effective_replay_config(me_->config);
+   for (const auto &r : replay_cfgs) {
+      const auto v = domain::DomainSecurityUtils::convert(r.venue());
+      me_->sources.push_back({v, std::make_shared<MDReplayer>(r.venue())});
+   }
+   return me_->sources;
+}
+
+} // namespace bts::md
@@ -0,0 +1,28 @@
+#pragma once
+
+#include "spark/api/md/MarketDataConfig.pb.h"
+#include "spark/persistence/IDGenerator.h"
+#include "sparkd/md/IMarketDataProvider.h"
+
+#include <memory>
+
+namespace bts::md {
+
+class ReplayMarketDataProvider : public IMarketDataProvider {
+ public:
+   ReplayMarketDataProvider();
+   explicit ReplayMarketDataProvider(
+       api::md::MarketDataConfig config,
+       persistence::IDGenerator *sec_id_store);
+   ~ReplayMarketDataProvider();
+
+   std::vector<
+       std::pair<domain::protobufs::Venue, std::shared_ptr<IMarketDataSource>>>
+   get_sources() override;
+
+ private:
+   struct PImpl;
+   std::unique_ptr<PImpl> me_;
+};
+
+} // namespace bts::md
@@ -0,0 +1,47 @@
+#include "ReplayMarketDataSubscription.h"
+#include "spark/api/md/MarketDataReplayConfig.pb.h"
+#include "spark/logging/logging.h"
+#include "spark/md/GeneralMarketDataListener.h"
+#include "spark/md/IMarketDataListener.h"
+#include "spark/md/MDProducer.h"
+#include "spark/utils/thread_manager.h"
+#include "sparkd/md/recording/MDReplayer.h"
+
+#include <ccapi_cpp/ccapi_event_dispatcher.h>
+#include <optional>
+
+namespace bts::md {
+
+using bts::api::md::MarketDataReplayConfig;
+using bts::md::recording::MDReplayer;
+using std::string;
+
+struct ReplayMarketDataSource::PImpl {
+   MarketDataReplayConfig config;
+
+   PImpl(const MarketDataReplayConfig &config) : config(config) {}
+};
+
+ReplayMarketDataSource::ReplayMarketDataSource(MarketDataReplayConfig config)
+    : me_(std::make_unique<PImpl>(config)) {
+   me_->config = std::move(config);
+}
+
+ReplayMarketDataSource::~ReplayMarketDataSource() = default;
+
+void ReplayMarketDataSource::start(
+    std::shared_ptr<IMarketDataListener> listener) {
+   LOG(md, Info, "MarketDataGateway::_start_replays starting t_id:{} t_name:{}",
+       os::get_thread_id(), os::get_thread_name());
+   auto md_replayer = std::make_shared<MDReplayer>(me_->config.venue());
+   std::string tid_name = "MDReplayer_" + Venue_Name(me_->config.venue());
+   bts::threading::make_thread(tid_name, [&, listener, md_replayer]() {
+      md_replayer->start(listener);
+   }).detach();
+   LOG(md, Info, "MarketDataGateway::_start_replays ending t_id:{} t_name:{}",
+       os::get_thread_id(), os::get_thread_name());
+}
+
+void ReplayMarketDataSource::stop() {}
+
+} // namespace bts::md
@@ -0,0 +1,25 @@
+#pragma once
+
+#include "spark/api/md/MarketDataReplayConfig.pb.h"
+#include "spark/md/IMarketDataListener.h"
+#include "sparkd/md/IMarketDataSource.h"
+
+#include <memory>
+
+namespace bts::md {
+
+class ReplayMarketDataSource : public IMarketDataSource {
+ public:
+   ReplayMarketDataSource();
+   explicit ReplayMarketDataSource(api::md::MarketDataReplayConfig config);
+   ~ReplayMarketDataSource();
+
+   void start(std::shared_ptr<IMarketDataListener> listener) override;
+   void stop() override;
+
+ private:
+   struct PImpl;
+   std::unique_ptr<PImpl> me_;
+};
+
+} // namespace bts::md
@@ -1,7 +1,7 @@
 #pragma once

-#include "spark/core/EnumMacros.h"
-#include "spark/core/FMT_IMPL.h"
+#include "system/FMT_IMPL.h"
+#include "system/enum_macros.h"

 #include <fmt/format.h>
 #include <nlohmann/json.hpp>
@@ -1,7 +1,5 @@
 #pragma once

-#include "spark/io/JsonDumper.h"
-#include "spark/io/JsonReader.h"
 #include "sparkd/md/binance/BinanceFilterType.h"
 #include "sparkd/md/binance/BinanceIcebergPartsFilter.h"
 #include "sparkd/md/binance/BinanceLotSizeFilter.h"
@@ -15,6 +13,8 @@
 #include "sparkd/md/binance/BinancePercentPriceFilter.h"
 #include "sparkd/md/binance/BinancePriceFilter.h"
 #include "sparkd/md/binance/BinanceTrailingDeltaFilter.h"
+#include "system/json/json_dumper.h"
+#include "system/json/json_reader.h"

 #include <nlohmann/json.hpp>
 #include <string>
@@ -1,7 +1,7 @@
 #pragma once

-#include "spark/core/EnumMacros.h"
-#include "spark/core/FMT_IMPL.h"
+#include "system/FMT_IMPL.h"
+#include "system/enum_macros.h"

 #include <fmt/format.h>
 #include <nlohmann/json.hpp>
@@ -1,8 +1,8 @@
 #pragma once

 #include "spark/io/JsonMacros.h"
-#include "spark/io/JsonReader.h"
 #include "sparkd/md/binance/BinanceFilterType.h"
+#include "system/json/json_reader.h"

 #include <iostream>
 #include <nlohmann/json.hpp>
@@ -1,7 +1,7 @@
 #pragma once

-#include "spark/core/EnumMacros.h"
-#include "spark/core/FMT_IMPL.h"
+#include "system/FMT_IMPL.h"
+#include "system/enum_macros.h"

 #include <fmt/format.h>
 #include <nlohmann/json.hpp>
@@ -1,7 +1,8 @@
 #pragma once

-#include "spark/io/JsonReader.h"
+#include "spark/logging/logging.h"
 #include "sparkd/md/binance/BinanceFilterType.h"
+#include "system/json/json_reader.h"

 #include <nlohmann/json.hpp>
 #include <string>
@@ -1,7 +1,7 @@
 #pragma once

-#include "spark/io/JsonReader.h"
 #include "sparkd/md/binance/BinanceFilterType.h"
+#include "system/json/json_reader.h"

 #include <nlohmann/json.hpp>

@@ -1,7 +1,7 @@
 #pragma once

-#include "spark/io/JsonReader.h"
 #include "sparkd/md/binance/BinanceFilterType.h"
+#include "system/json/json_reader.h"

 #include <nlohmann/json.hpp>
 #include <string>
@@ -1,7 +1,7 @@
 #pragma once

-#include "spark/io/JsonReader.h"
 #include "sparkd/md/binance/BinanceFilterType.h"
+#include "system/json/json_reader.h"

 #include <nlohmann/json.hpp>
 #include <string>
@@ -1,7 +1,7 @@
 #pragma once

-#include "spark/io/JsonReader.h"
 #include "sparkd/md/binance/BinanceFilterType.h"
+#include "system/json/json_reader.h"

 #include <nlohmann/json.hpp>

@@ -1,7 +1,7 @@
 #pragma once

-#include "spark/core/EnumMacros.h"
-#include "spark/core/FMT_IMPL.h"
+#include "system/FMT_IMPL.h"
+#include "system/enum_macros.h"

 #include <fmt/format.h>
 #include <nlohmann/json.hpp>
@@ -1,8 +1,8 @@
 #pragma once

 #include "spark/io/JsonMacros.h"
-#include "spark/io/JsonReader.h"
 #include "sparkd/md/binance/BinanceFilterType.h"
+#include "system/json/json_reader.h"

 #include <iostream>
 #include <nlohmann/json.hpp>
@@ -1,6 +1,6 @@
 #pragma once

-#include "spark/domain/SecurityID.h"
+#include "domain/security_id.h"
 #include "sparkd/md/ccapi/ICCApiMessageHandler.h"

 namespace bts::md {
@@ -1,6 +1,6 @@
 #pragma once
 #ifdef SPARK_ENABLE_CCAPI
-#include "spark/api/domain/SecurityType.pb.h"
+#include "domain/protobufs/security_type.pb.h"

 #include <nlohmann/json.hpp>
 #include <ostream>
@@ -1,12 +1,12 @@
 #ifdef SPARK_ENABLE_CCAPI
 #include "BinanceMiniTickerUpdate.h"
-#include "spark/core/TimeStamp.h"
-#include "spark/domain/Price.h"
-#include "spark/domain/Quantity.h"
+#include "domain/price.h"
+#include "domain/quantity.h"
 #include "spark/domain/SecurityMarketDataStatistics.h"
-#include "spark/io/JsonDumper.h"
-#include "spark/io/JsonReader.h"
 #include "sparkd/md/ccapi/CCApiReader.h"
+#include "system/json/json_dumper.h"
+#include "system/json/json_reader.h"
+#include "system/time_stamp.h"

 #include <ccapi_cpp/ccapi_message.h>
 #include <type_traits>
@@ -1,8 +1,8 @@
 #pragma once

-#include "spark/core/TimeStamp.h"
 #include "spark/domain/SecurityMarketDataStatistics.h"
 #include "sparkd/md/binance/BinanceMiniTickerEventType.h"
+#include "system/time_stamp.h"

 #include <nlohmann/json.hpp>
 #include <string>
@@ -1,8 +1,8 @@
 #pragma once

 #include "spark/io/JsonMacros.h"
-#include "spark/io/JsonReader.h"
 #include "sparkd/md/binance/BinanceFilterType.h"
+#include "system/json/json_reader.h"

 #include <nlohmann/json.hpp>
 #include <string>
@@ -1,7 +1,7 @@
 #pragma once

-#include "spark/core/EnumMacros.h"
-#include "spark/core/FMT_IMPL.h"
+#include "system/FMT_IMPL.h"
+#include "system/enum_macros.h"

 #include <fmt/format.h>
 #include <nlohmann/json.hpp>
@@ -1,8 +1,8 @@
 #pragma once

-#include "spark/io/JsonDumper.h"
-#include "spark/io/JsonReader.h"
 #include "sparkd/md/binance/BinanceFilterType.h"
+#include "system/json/json_dumper.h"
+#include "system/json/json_reader.h"

 #include <nlohmann/json.hpp>
 #include <string>
@@ -1,7 +1,7 @@
 #pragma once

-#include "spark/io/JsonReader.h"
 #include "sparkd/md/binance/BinanceFilterType.h"
+#include "system/json/json_reader.h"

 #include <nlohmann/json.hpp>
 #include <string>
@@ -1,7 +1,7 @@
 #pragma once

-#include "spark/core/EnumMacros.h"
-#include "spark/core/FMT_IMPL.h"
+#include "system/FMT_IMPL.h"
+#include "system/enum_macros.h"

 #include <fmt/format.h>
 #include <nlohmann/json.hpp>
@@ -1,8 +1,8 @@
 #pragma once

-#include "spark/io/JsonDumper.h"
-#include "spark/io/JsonReader.h"
 #include "sparkd/md/binance/BinanceFilterType.h"
+#include "system/json/json_dumper.h"
+#include "system/json/json_reader.h"

 #include <nlohmann/json.hpp>
 #include <string>
@@ -1,11 +1,11 @@
 #include "BinancePriceStatistics.h"
-#include "spark/core/TimeStamp.h"
-#include "spark/domain/Price.h"
-#include "spark/domain/Quantity.h"
+#include "domain/price.h"
+#include "domain/quantity.h"
 #include "spark/domain/SecurityMarketDataStatistics.h"
-#include "spark/io/JsonDumper.h"
-#include "spark/io/JsonReader.h"
 #include "sparkd/md/ccapi/CCApiReader.h"
+#include "system/json/json_dumper.h"
+#include "system/json/json_reader.h"
+#include "system/time_stamp.h"

 #include <ccapi_cpp/ccapi_macro.h>
 #include <ccapi_cpp/ccapi_message.h>
@@ -1,6 +1,6 @@
 #pragma once

-#include "spark/core/TimeStamp.h"
+#include "system/time_stamp.h"

 #include <cstdint>
 #include <nlohmann/json.hpp>
@@ -1,5 +1,5 @@
 #include "BinancePriceStatisticsAdaptor.h"
-#include "spark/domain/SecurityID.h"
+#include "domain/security_id.h"
 #include "spark/domain/SecurityMarketDataStatistics.h"
 #include "spark/md/IMarketDataListener.h"
 #include "spark/utils/bts_fatal.h"
@@ -1,6 +1,6 @@
 #pragma once

-#include "spark/domain/SecurityID.h"
+#include "domain/security_id.h"
 #include "sparkd/md/ccapi/ICCApiMessageHandler.h"

 #include <string>
@@ -1,8 +1,8 @@
 #pragma once

-#include "spark/api/domain/Currency.pb.h"
-#include "spark/core/EnumMacros.h"
-#include "spark/core/FMT_IMPL.h"
+#include "domain/protobufs/currency.pb.h"
+#include "system/FMT_IMPL.h"
+#include "system/enum_macros.h"

 #include <boost/algorithm/string/case_conv.hpp>
 #include <fmt/format.h>
@@ -60,7 +60,7 @@ class BinanceQuoteAsset {

   ENUM_BASIC_MEMBERS(BinanceQuoteAsset)

-   constexpr api::domain::Currency to_currency() const;
+   constexpr domain::protobufs::Currency to_currency() const;
 };

 ENUM_OUTPUT_OPS(BinanceQuoteAsset)
@@ -75,18 +75,18 @@ ENUM_EQUALITY_OPS(BinanceQuoteAsset)
 // Definition
 //////////////////////////////////////////////////

-constexpr api::domain::Currency BinanceQuoteAsset::to_currency() const {
+constexpr domain::protobufs::Currency BinanceQuoteAsset::to_currency() const {
   switch (value) {
   case BTC:
-      return api::domain::Currency::BTC;
+      return domain::protobufs::Currency::BTC;
   case ETH:
-      return api::domain::Currency::ETH;
+      return domain::protobufs::Currency::ETH;
   case USDC:
   case USDP:
   case USDS:
   case USDT:
   default:
-      return api::domain::Currency::USD;
+      return domain::protobufs::Currency::USD;
   }
 }

@@ -1,10 +1,8 @@
 #include "BinanceSecurityRequestor.h"
-#include "spark/api/domain/SecurityType.pb.h"
-#include "spark/api/domain/Venue.pb.h"
-#include "spark/collections/Contains.h"
-#include "spark/collections/Remove.h"
-#include "spark/domain/Security.h"
-#include "spark/domain/SecurityCriteria.h"
+#include "domain/protobufs/security_type.pb.h"
+#include "domain/protobufs/venue.pb.h"
+#include "domain/security.h"
+#include "domain/security_criteria.h"
 #include "spark/logging/logging.h"
 #include "spark/persistence/IDGenerator.h"
 #include "spark/utils/bts_fatal.h"
@@ -14,6 +12,8 @@
 #include "sparkd/md/ccapi/CCApiMessageHandler.h"
 #include "sparkd/md/ccapi/CCApiSession.h"
 #include "sparkd/md/ccapi/ICCApiMessageHandler.h"
+#include "system/collections/contains.h"
+#include "system/collections/remove.h"

 #include <algorithm>
 #include <ccapi_cpp/ccapi_event_dispatcher.h>
@@ -26,12 +26,12 @@

 namespace bts::md::binance {

-using api::domain::SecurityType;
-using api::domain::Venue;
-using api::domain::Venue_Name;
 using bts::domain::Security;
 using bts::domain::SecurityCriteria;
 using bts::domain::SecurityID;
+using bts::domain::protobufs::SecurityType;
+using bts::domain::protobufs::Venue;
+using bts::domain::protobufs::Venue_Name;
 using bts::persistence::IDGenerator;
 using std::string;
 using std::unordered_set;
@@ -53,7 +53,7 @@ using std::vector;
 #define CCAPI_VENUE_NAME CCAPI_EXCHANGE_NAME_BINANCE
 #define BINANCE_MSG_TYPE_NAME(x)                                               \
   std::string { BinanceMessageType{x}.name() }
-#define SEC_TYPE(x) bts::api::domain::SECURITY_TYPE_##x
+#define SEC_TYPE(x) bts::domain::protobufs::SECURITY_TYPE_##x

 static ccapi::Request exchange_info_req() {
   auto corr_id = CCApiCorrelationID::from_parts(
@@ -1,6 +1,6 @@
 #pragma once

-#include "spark/utils/Delegate.h"
+#include "system/delegate.h"

 #include <memory>
 #include <unordered_set>
@@ -1,9 +1,9 @@
 #include "BinanceSpot.h"
-#include "spark/api/domain/Venue.pb.h"
-#include "spark/domain/Quantity.h"
-#include "spark/domain/Security.h"
-#include "spark/io/JsonReader.h"
+#include "domain/protobufs/venue.pb.h"
+#include "domain/quantity.h"
+#include "domain/security.h"
 #include "spark/logging/logging.h"
+#include "system/json/json_reader.h"

 #include <ccapi_cpp/ccapi_message.h>
 #include <iostream>
@@ -12,15 +12,15 @@

 namespace bts::md::binance {

-using bts::api::domain::Venue;
 using bts::domain::Quantity;
 using bts::domain::Security;
 using bts::domain::SpotSecurity;
+using bts::domain::protobufs::Venue;
 using bts::io::JsonReader;
 using std::string;
 using std::vector;

-#define SEC_TYPE(x) bts::api::domain::SECURITY_TYPE_##x
+#define SEC_TYPE(x) bts::domain::protobufs::SECURITY_TYPE_##x

 static std::string deduce_sym_from_name(const std::string &name) {
   std::regex word_regex("(\\w+)");
@@ -1,13 +1,13 @@
 #pragma once

-#include "spark/api/domain/Venue.h"
-#include "spark/core/FMT_IMPL.h"
+#include "domain/protobufs/venue.h"
 #include "spark/io/JsonMacros.h"
 #include "sparkd/md/binance/BinanceFilterTuple.h"
 #include "sparkd/md/binance/BinanceInstrumentStatus.h"
 #include "sparkd/md/binance/BinanceOrderType.h"
 #include "sparkd/md/binance/BinancePermission.h"
 #include "sparkd/md/binance/BinanceQuoteAsset.h"
+#include "system/FMT_IMPL.h"

 #include <nlohmann/json.hpp>
 #include <ostream>
@@ -50,7 +50,7 @@ class BinanceSpot {
   static void parse(
       const ccapi::Message &msg,
       std::vector<domain::Security> &securities,
-       api::domain::Venue venue = api::domain::Venue::VENUE_BINANCE);
+       domain::protobufs::Venue venue = domain::protobufs::VENUE_BINANCE);
 };

 void from_json(const nlohmann::json &j, BinanceSpot &spot);
@@ -1,10 +1,8 @@
 #include "BinanceSubscription.h"
-#include "spark/api/domain/Venue.pb.h"
+#include "domain/protobufs/venue.pb.h"
+#include "domain/security.h"
+#include "domain/security_criteria.h"
 #include "spark/api/md/BinanceMarketDataConfig.pb.h"
-#include "spark/collections/UnorderedSetFactory.h"
-#include "spark/core/FMT.h"
-#include "spark/domain/Security.h"
-#include "spark/domain/SecurityCriteria.h"
 #include "spark/io/ProtoPrinter.h"
 #include "spark/io/QueryParams.h"
 #include "spark/logging/logging.h"
@@ -19,6 +17,8 @@
 #include "sparkd/md/ccapi/CCApiMessageLogger.h"
 #include "sparkd/md/ccapi/CCApiSession.h"
 #include "sparkd/md/ccapi/CCApiTradeAdaptor.h"
+#include "system/FMT.h"
+#include "system/collections/unordered_set_factory.h"

 #include <ccapi_cpp/ccapi_event_dispatcher.h>
 #include <ccapi_cpp/ccapi_macro.h>
@@ -34,12 +34,12 @@

 namespace bts::md::binance {

-using bts::api::domain::Venue;
-using bts::api::domain::Venue_Name;
 using bts::api::md::BinanceMarketDataConfig;
 using bts::domain::Security;
 using bts::domain::SecurityCriteria;
 using bts::domain::SecurityID;
+using bts::domain::protobufs::Venue;
+using bts::domain::protobufs::Venue_Name;
 using bts::io::ProtoPrinter;
 using bts::io::QueryParams;
 using bts::persistence::IDGenerator;
@@ -168,7 +168,7 @@ struct BinanceSubscription::PImpl {

   // The session automatically starts/stops in its ctr/dtr
   CCApiSession session;
-   IMarketDataListener *listener;
+   std::shared_ptr<IMarketDataListener> listener;
   BinanceSecurityRequestor sec_requestor;

   // These aliases are useful for quickly switching the handler types (e.g.,
@@ -191,11 +191,10 @@ struct BinanceSubscription::PImpl {
   explicit PImpl(
       ccapi::EventDispatcher *dispatcher,
       BinanceMarketDataConfig cfg,
-       IDGenerator *sec_id_store,
-       IMarketDataListener *listener)
+       IDGenerator *sec_id_store)
       : sec_id_store(sec_id_store), cfg(std::move(cfg)),
         criteria(SecurityCriteria::from_proto(this->cfg.security_criteria())),
-         session(dispatcher, mk_session_cfg(this->cfg), {}), listener(listener),
+         session(dispatcher, mk_session_cfg(this->cfg), {}),
         sec_requestor(
             &session,
             this->criteria,
@@ -206,13 +205,18 @@ struct BinanceSubscription::PImpl {
      LOG(md, Info, "Initializing BinanceSubscription:\n{}",
          ProtoPrinter::to_string(this->cfg));
      sec_requestor.callback().bind<&PImpl::on_securities>(*this);
+   }
+
+   void start(std::shared_ptr<IMarketDataListener> listener) {
+      bts_assert(listener != nullptr);
+      this->listener = std::move(listener);
      sec_requestor.request();
   }

   DepthHandler &mk_depth_handler(const string &sec_desc) {
      SecurityID sec_id{sec_id_store->get_or_add(VENUE_NAME, sec_desc)};
      auto [it, is_new] =
-          depth_handlers.try_emplace(sec_desc, sec_id, listener);
+          depth_handlers.try_emplace(sec_desc, sec_id, listener.get());
      bts_assert(is_new);
      return it->second;
   }
@@ -220,7 +224,7 @@ struct BinanceSubscription::PImpl {
   TickerHandler &mk_ticker_handler(const string &sec_desc) {
      SecurityID sec_id{sec_id_store->get_or_add(VENUE_NAME, sec_desc)};
      auto [it, is_new] =
-          ticker_handlers.try_emplace(sec_desc, sec_id, listener);
+          ticker_handlers.try_emplace(sec_desc, sec_id, listener.get());
      bts_assert_msgf(
          is_new, "Ticker handler already created for '{}'", sec_desc);
      return it->second;
@@ -236,7 +240,8 @@ struct BinanceSubscription::PImpl {
             SecurityID{sec_id_store->get_or_add(VENUE_NAME, sec_desc)});
      }
      auto [it, is_new] = stats_handlers.try_emplace(
-          hnd_id, std::make_unique<StatsHandler>(sec_desc_id_pairs, listener));
+          hnd_id,
+          std::make_unique<StatsHandler>(sec_desc_id_pairs, listener.get()));
      bts_assert_msgf(
          is_new, "StatsHandler already created with id '{}'", hnd_id);
      return *it->second;
@@ -325,10 +330,8 @@ struct BinanceSubscription::PImpl {
 BinanceSubscription::BinanceSubscription(
    ccapi::EventDispatcher *dispatcher,
    BinanceMarketDataConfig cfg,
-    IDGenerator *sec_id_store,
-    IMarketDataListener *listener)
-    : me_(std::make_unique<PImpl>(
-          dispatcher, std::move(cfg), sec_id_store, listener)) {}
+    IDGenerator *sec_id_store)
+    : me_(std::make_unique<PImpl>(dispatcher, std::move(cfg), sec_id_store)) {}

 BinanceSubscription::BinanceSubscription(BinanceSubscription &&) noexcept =
    default;
@@ -337,4 +340,8 @@ BinanceSubscription &BinanceSubscription::operator=(

 BinanceSubscription::~BinanceSubscription() = default;

+void BinanceSubscription::start(std::shared_ptr<IMarketDataListener> listener) {
+   me_->start(listener);
+}
+
 } // namespace bts::md::binance
@@ -1,5 +1,6 @@
 #pragma once

+#include "sparkd/md/IMarketDataSource.h"
 #include <memory>

 namespace ccapi {
@@ -25,13 +26,12 @@ namespace bts::md::binance {
 *
 * @a https://binance-docs.github.io/apidocs/spot/en/#exchange-information
 */
-class BinanceSubscription {
+class BinanceSubscription : public IMarketDataSource {
 public:
   BinanceSubscription(
       ccapi::EventDispatcher *dispatcher,
       api::md::BinanceMarketDataConfig cfg,
-       persistence::IDGenerator *sec_id_store,
-       IMarketDataListener *listener);
+       persistence::IDGenerator *sec_id_store);

   BinanceSubscription(const BinanceSubscription &) noexcept = delete;
   BinanceSubscription(BinanceSubscription &&) noexcept;
@@ -39,10 +39,9 @@ class BinanceSubscription {
       delete;
   BinanceSubscription &operator=(BinanceSubscription &&) noexcept;

-   ~BinanceSubscription();
+   ~BinanceSubscription() override;

-   void add_listener(IMarketDataListener *listener);
-   void remove_listener(IMarketDataListener *listener);
+   void start(std::shared_ptr<IMarketDataListener> listener) override;

 private:
   struct PImpl;
@@ -1,8 +1,8 @@
 #pragma once

-#include "spark/io/JsonDumper.h"
-#include "spark/io/JsonReader.h"
 #include "sparkd/md/binance/BinanceFilterType.h"
+#include "system/json/json_dumper.h"
+#include "system/json/json_reader.h"

 #include <nlohmann/json.hpp>

@@ -1,7 +1,7 @@
 #pragma once

-#include "spark/core/EnumMacros.h"
-#include "spark/core/FMT_IMPL.h"
+#include "system/FMT_IMPL.h"
+#include "system/enum_macros.h"

 #include <fmt/format.h>
 #include <nlohmann/json.hpp>
@@ -1,11 +1,11 @@
 #include "BinanceUsPriceStatistics.h"
-#include "spark/core/TimeStamp.h"
-#include "spark/domain/Price.h"
-#include "spark/domain/Quantity.h"
+#include "domain/price.h"
+#include "domain/quantity.h"
 #include "spark/domain/SecurityMarketDataStatistics.h"
-#include "spark/io/JsonDumper.h"
-#include "spark/io/JsonReader.h"
 #include "sparkd/md/ccapi/CCApiReader.h"
+#include "system/json/json_dumper.h"
+#include "system/json/json_reader.h"
+#include "system/time_stamp.h"

 #include <ccapi_cpp/ccapi_macro.h>
 #include <ccapi_cpp/ccapi_message.h>
@@ -1,6 +1,6 @@
 #pragma once

-#include "spark/core/TimeStamp.h"
+#include "system/time_stamp.h"

 #include <cstdint>
 #include <nlohmann/json.hpp>
@@ -1,5 +1,5 @@
 #include "BinanceUsPriceStatisticsAdaptor.h"
-#include "spark/domain/SecurityID.h"
+#include "domain/security_id.h"
 #include "spark/domain/SecurityMarketDataStatistics.h"
 #include "spark/md/IMarketDataListener.h"
 #include "spark/utils/bts_fatal.h"
@@ -1,6 +1,6 @@
 #pragma once

-#include "spark/domain/SecurityID.h"
+#include "domain/security_id.h"
 #include "sparkd/md/ccapi/ICCApiMessageHandler.h"

 #include <string>
@@ -1,10 +1,8 @@
 #include "BinanceUsSecurityRequestor.h"
-#include "spark/api/domain/SecurityType.pb.h"
-#include "spark/api/domain/Venue.pb.h"
-#include "spark/collections/Contains.h"
-#include "spark/collections/Remove.h"
-#include "spark/domain/Security.h"
-#include "spark/domain/SecurityCriteria.h"
+#include "domain/protobufs/security_type.pb.h"
+#include "domain/protobufs/venue.pb.h"
+#include "domain/security.h"
+#include "domain/security_criteria.h"
 #include "spark/logging/logging.h"
 #include "spark/persistence/IDGenerator.h"
 #include "sparkd/md/binance_us/BinanceUsMessageType.h"
@@ -13,6 +11,8 @@
 #include "sparkd/md/ccapi/CCApiMessageHandler.h"
 #include "sparkd/md/ccapi/CCApiSession.h"
 #include "sparkd/md/ccapi/ICCApiMessageHandler.h"
+#include "system/collections/contains.h"
+#include "system/collections/remove.h"

 #include <algorithm>
 #include <ccapi_cpp/ccapi_event_dispatcher.h>
@@ -25,12 +25,12 @@

 namespace bts::md::binance_us {

-using api::domain::SecurityType;
-using api::domain::Venue;
-using api::domain::Venue_Name;
 using bts::domain::Security;
 using bts::domain::SecurityCriteria;
 using bts::domain::SecurityID;
+using bts::domain::protobufs::SecurityType;
+using bts::domain::protobufs::Venue;
+using bts::domain::protobufs::Venue_Name;
 using bts::persistence::IDGenerator;
 using std::string;
 using std::unordered_set;
@@ -41,7 +41,7 @@ using std::vector;
 #define CCAPI_VENUE_NAME CCAPI_EXCHANGE_NAME_BINANCE_US
 #define BINANCE_US_MSG_TYPE_NAME(x)                                            \
   std::string { BinanceUsMessageType{x}.name() }
-#define SEC_TYPE(x) bts::api::domain::SECURITY_TYPE_##x
+#define SEC_TYPE(x) bts::domain::protobufs::SECURITY_TYPE_##x

 static ccapi::Request exchange_info_req() {
   auto corr_id = CCApiCorrelationID::from_parts(
@@ -1,6 +1,6 @@
 #pragma once

-#include "spark/utils/Delegate.h"
+#include "system/delegate.h"

 #include <memory>
 #include <unordered_set>
@@ -1,10 +1,8 @@
 #include "BinanceUsSubscription.h"
-#include "spark/api/domain/Venue.pb.h"
+#include "domain/protobufs/venue.pb.h"
+#include "domain/security.h"
+#include "domain/security_criteria.h"
 #include "spark/api/md/BinanceUsMarketDataConfig.pb.h"
-#include "spark/collections/UnorderedSetFactory.h"
-#include "spark/core/FMT.h"
-#include "spark/domain/Security.h"
-#include "spark/domain/SecurityCriteria.h"
 #include "spark/io/ProtoPrinter.h"
 #include "spark/io/QueryParams.h"
 #include "spark/logging/logging.h"
@@ -20,6 +18,8 @@
 #include "sparkd/md/ccapi/CCApiMessageLogger.h"
 #include "sparkd/md/ccapi/CCApiSession.h"
 #include "sparkd/md/ccapi/CCApiTradeAdaptor.h"
+#include "system/FMT.h"
+#include "system/collections/unordered_set_factory.h"

 #include <ccapi_cpp/ccapi_event_dispatcher.h>
 #include <ccapi_cpp/ccapi_event_handler.h>
@@ -36,16 +36,17 @@

 namespace bts::md::binance_us {

-using bts::api::domain::Venue;
-using bts::api::domain::Venue_Name;
 using bts::api::md::BinanceUsMarketDataConfig;
 using bts::domain::Security;
 using bts::domain::SecurityCriteria;
 using bts::domain::SecurityID;
+using bts::domain::protobufs::Venue;
+using bts::domain::protobufs::Venue_Name;
 using bts::io::ProtoPrinter;
 using bts::io::QueryParams;
 using bts::persistence::IDGenerator;
 using std::pair;
+using std::shared_ptr;
 using std::string;
 using std::uint32_t;
 using std::unique_ptr;
@@ -171,7 +172,7 @@ struct BinanceUsSubscription::PImpl {

   // The session automatically starts/stops in its ctr/dtr
   CCApiSession session;
-   IMarketDataListener *listener;
+   shared_ptr<IMarketDataListener> listener;
   BinanceUsSecurityRequestor sec_requestor;

   // These aliases are useful for quickly switching the handler types (e.g.,
@@ -188,17 +189,16 @@ struct BinanceUsSubscription::PImpl {
   // different map, 'stats_handlers' (unique ids are generated w/ a counter,
   // 'stats_handlers_id_counter').
   unordered_map<string, StatsHandler *> sec_stats_handler_map;
-   unordered_map<uint32_t, std::unique_ptr<StatsHandler>> stats_handlers;
+   unordered_map<uint32_t, unique_ptr<StatsHandler>> stats_handlers;
   uint32_t stats_handlers_id_counter = 0;

   explicit PImpl(
       ccapi::EventDispatcher *dispatcher,
       BinanceUsMarketDataConfig cfg,
-       IDGenerator *sec_id_store,
-       IMarketDataListener *listener)
+       IDGenerator *sec_id_store)
       : sec_id_store(sec_id_store), cfg(std::move(cfg)),
         criteria(SecurityCriteria::from_proto(this->cfg.security_criteria())),
-         session(dispatcher, mk_session_cfg(this->cfg), {}), listener(listener),
+         session(dispatcher, mk_session_cfg(this->cfg), {}),
         sec_requestor(
             &session,
             this->criteria,
@@ -209,13 +209,18 @@ struct BinanceUsSubscription::PImpl {
      LOG(md, Info, "Initializing BinanceUsSubscription:\n{}",
          ProtoPrinter::to_string(this->cfg));
      sec_requestor.callback().bind<&PImpl::on_securities>(*this);
+   }
+
+   void start(std::shared_ptr<IMarketDataListener> listener) {
+      bts_assert(listener != nullptr);
+      this->listener = std::move(listener);
      sec_requestor.request();
   }

   DepthHandler &mk_depth_handler(const string &sec_desc) {
      SecurityID sec_id{sec_id_store->get_or_add(VENUE_NAME, sec_desc)};
      auto [it, is_new] =
-          depth_handlers.try_emplace(sec_desc, sec_id, listener);
+          depth_handlers.try_emplace(sec_desc, sec_id, listener.get());
      bts_assert(is_new);
      return it->second;
   }
@@ -223,7 +228,7 @@ struct BinanceUsSubscription::PImpl {
   TickerHandler &mk_ticker_handler(const string &sec_desc) {
      SecurityID sec_id{sec_id_store->get_or_add(VENUE_NAME, sec_desc)};
      auto [it, is_new] =
-          ticker_handlers.try_emplace(sec_desc, sec_id, listener);
+          ticker_handlers.try_emplace(sec_desc, sec_id, listener.get());
      bts_assert_msgf(
          is_new, "Ticker handler already created for '{}'", sec_desc);
      return it->second;
@@ -239,7 +244,8 @@ struct BinanceUsSubscription::PImpl {
             SecurityID{sec_id_store->get_or_add(VENUE_NAME, sec_desc)});
      }
      auto [it, is_new] = stats_handlers.try_emplace(
-          hnd_id, std::make_unique<StatsHandler>(sec_desc_id_pairs, listener));
+          hnd_id,
+          std::make_unique<StatsHandler>(sec_desc_id_pairs, listener.get()));
      bts_assert_msgf(
          is_new, "StatsHandler already created with id '{}'", hnd_id);
      return *it->second;
@@ -328,10 +334,8 @@ struct BinanceUsSubscription::PImpl {
 BinanceUsSubscription::BinanceUsSubscription(
    ccapi::EventDispatcher *dispatcher,
    BinanceUsMarketDataConfig cfg,
-    IDGenerator *sec_id_store,
-    IMarketDataListener *listener)
-    : me_(std::make_unique<PImpl>(
-          dispatcher, std::move(cfg), sec_id_store, listener)) {}
+    IDGenerator *sec_id_store)
+    : me_(std::make_unique<PImpl>(dispatcher, std::move(cfg), sec_id_store)) {}

 BinanceUsSubscription::BinanceUsSubscription(
    BinanceUsSubscription &&) noexcept = default;
@@ -340,4 +344,8 @@ BinanceUsSubscription &BinanceUsSubscription::operator=(

 BinanceUsSubscription::~BinanceUsSubscription() = default;

+void BinanceUsSubscription::start(shared_ptr<IMarketDataListener> listener) {
+   me_->start(std::move(listener));
+}
+
 } // namespace bts::md::binance_us
@@ -1,5 +1,6 @@
 #pragma once

+#include "sparkd/md/IMarketDataSource.h"
 #include <memory>

 namespace ccapi {
@@ -25,13 +26,12 @@ namespace bts::md::binance_us {
 *
 * @a https://docs.binance.us/#get-exchange-information
 */
-class BinanceUsSubscription {
+class BinanceUsSubscription : public IMarketDataSource {
 public:
   BinanceUsSubscription(
       ccapi::EventDispatcher *dispatcher,
       api::md::BinanceUsMarketDataConfig cfg,
-       persistence::IDGenerator *sec_id_store,
-       IMarketDataListener *listener);
+       persistence::IDGenerator *sec_id_store);

   BinanceUsSubscription(const BinanceUsSubscription &) noexcept = delete;
   BinanceUsSubscription(BinanceUsSubscription &&) noexcept;
@@ -41,8 +41,7 @@ class BinanceUsSubscription {

   ~BinanceUsSubscription();

-   void add_listener(IMarketDataListener *listener);
-   void remove_listener(IMarketDataListener *listener);
+   void start(std::shared_ptr<IMarketDataListener> listener) override;

 private:
   struct PImpl;
@@ -1,10 +1,10 @@
 #include "BinanceUsdsFuture.h"
-#include "spark/domain/Price.h"
-#include "spark/domain/Quantity.h"
-#include "spark/domain/Security.h"
-#include "spark/io/JsonDumper.h"
-#include "spark/io/JsonReader.h"
+#include "domain/price.h"
+#include "domain/quantity.h"
+#include "domain/security.h"
 #include "spark/logging/logging.h"
+#include "system/json/json_dumper.h"
+#include "system/json/json_reader.h"

 #include <ccapi_cpp/ccapi_message.h>
 #include <iostream>
@@ -12,18 +12,18 @@

 namespace bts::md::binance_usds_futures {

-using bts::api::domain::SecurityType;
 using bts::domain::FutureSecurity;
 using bts::domain::Price;
 using bts::domain::Quantity;
 using bts::domain::Security;
+using bts::domain::protobufs::SecurityType;
 using bts::io::JsonDumper;
 using bts::io::JsonReader;
 using std::string;
 using std::vector;

-#define SEC_TYPE(x) bts::api::domain::SECURITY_TYPE_##x
-#define VENUE api::domain::Venue::VENUE_BINANCE_USDS_FUTURES;
+#define SEC_TYPE(x) bts::domain::protobufs::SECURITY_TYPE_##x
+#define VENUE bts::domain::protobufs::VENUE_BINANCE_USDS_FUTURES;

 static string deduce_sym_from_name(const string &name) {
   const std::regex word_regex("(\\w+)");
@@ -1,7 +1,7 @@
 #pragma once

-#include "spark/core/EnumMacros.h"
-#include "spark/core/FMT_IMPL.h"
+#include "system/FMT_IMPL.h"
+#include "system/enum_macros.h"

 #include <fmt/format.h>
 #include <nlohmann/json.hpp>
@@ -1,7 +1,5 @@
 #pragma once

-#include "spark/io/JsonDumper.h"
-#include "spark/io/JsonReader.h"
 #include "sparkd/md/binance/BinanceFilterType.h"
 #include "sparkd/md/binance/BinanceIcebergPartsFilter.h"
 #include "sparkd/md/binance/BinanceLotSizeFilter.h"
@@ -14,6 +12,8 @@
 #include "sparkd/md/binance_usds_futures/BinanceUsdsFuturesMaxNumAlgoOrdersFilter.h"
 #include "sparkd/md/binance_usds_futures/BinanceUsdsFuturesMaxNumOrdersFilter.h"
 #include "sparkd/md/binance_usds_futures/BinanceUsdsFuturesPercentPriceFilter.h"
+#include "system/json/json_dumper.h"
+#include "system/json/json_reader.h"

 #include <nlohmann/json.hpp>
 #include <string>
@@ -1,8 +1,8 @@
 #pragma once

-#include "spark/io/JsonDumper.h"
-#include "spark/io/JsonReader.h"
 #include "sparkd/md/binance/BinanceFilterType.h"
+#include "system/json/json_dumper.h"
+#include "system/json/json_reader.h"

 #include <nlohmann/json.hpp>
 #include <string>
@@ -1,8 +1,8 @@
 #pragma once

-#include "spark/io/JsonDumper.h"
-#include "spark/io/JsonReader.h"
 #include "sparkd/md/binance/BinanceFilterType.h"
+#include "system/json/json_dumper.h"
+#include "system/json/json_reader.h"

 #include <nlohmann/json.hpp>
 #include <string>
@@ -1,7 +1,7 @@
 #pragma once

-#include "spark/core/EnumMacros.h"
-#include "spark/core/FMT_IMPL.h"
+#include "system/FMT_IMPL.h"
+#include "system/enum_macros.h"

 #include <fmt/format.h>
 #include <nlohmann/json.hpp>
Author	SHA1	Message	Date
John Colvin	f3f4397ebb	build: Clean up client CMake	2024-09-03 12:39:02 -04:00
John Colvin	6c6d91506f	webapp check pt?	2024-09-01 20:07:31 +00:00
John Colvin	6cde245fa4	cleanup: Remove empty header FTXSecurity.h	2024-09-01 20:07:31 +00:00
John Colvin	f1ecf5c136	cleanup: Remove temporary redirect headers These were used to make code migration smoother (to the new lib_system and lib_domain) * spark/algorithm/TryCatch.h -> system/algorithm/try_catch.h * spark/core/BoolName.h -> system/bool_name.h * spark/core/EnumMacros.h -> system/enum_macros.h * spark/core/FMT.h -> system/FMT.h * spark/core/FMT_IMPL.h -> system/FMT_IMPL.h * spark/core/TimeSpan.h -> system/time_span.h * spark/core/TimeStamp.h -> system/time_stamp.h * spark/domain/Price.h -> domain/price.h * spark/io/JsonReader.h -> system/json/json_reader.h * spark/io/JsonDumper.h -> system/json/json_dumper.h	2024-09-01 20:07:31 +00:00
John Colvin	4ec034c3ac	refactor: Move RFQ to lib_domain	2024-09-01 20:07:31 +00:00
John Colvin	d93d5d4f76	refactor: Move AggressorSide proto to lib_domain Get rid of non-proto AggressorSide enum while we're at it (consolidate to 1 type)	2024-09-01 20:07:31 +00:00
John Colvin	34b69404ef	cleanup: Remove more unused code	2024-09-01 20:07:31 +00:00
John Colvin	88ebbceec5	refactor: Move a few Order types to lib_domain	2024-09-01 20:07:31 +00:00
John Colvin	abde05a5b6	refactor: Move TimeInForce to lib_domain	2024-09-01 20:07:31 +00:00
John Colvin	96660bcd44	cleanup: Remove unused code	2024-09-01 20:07:31 +00:00
John Colvin	a69b022e50	refactor: Move TradeRecord into lib_domain	2024-09-01 20:07:31 +00:00
John Colvin	d658e31849	refactor: Move TradingStatus enum to lib_domain	2024-09-01 20:07:31 +00:00
John Colvin	58f8187cab	build: Update webpack-cli in admin webapp	2024-09-01 20:06:01 +00:00
John Colvin	2198be05f8	build: Configure admin webapp build entirely in CMakeLists Get rid of build_admin_web_app.py	2024-09-01 20:06:01 +00:00
John Colvin	55c64c744a	refactor: Rename core_md -> domain	2024-09-01 20:06:01 +00:00
John Colvin	30166c4acc	refactor: Move core_md protos into protobufs dir Also rename files in snake_case (standard naming convention for protos) https://protobuf.dev/programming-guides/style/	2024-08-27 15:46:09 +00:00
John Colvin	fbc6622eb4	refactor: Move domain types into new lib core_md	2024-08-27 15:45:52 +00:00
John Colvin	299654e9d6	refactor: Move collections into lib_system	2024-08-27 15:40:05 +00:00
John Colvin	8ef2d42d3d	refactor: Move a handful of types/utils to lib_system	2024-08-27 15:40:05 +00:00
John Colvin	37fc4974a1	build: Add preset to build with 6 jobs (-j 6) Otherwise the devcontainer may hog too much cpu and cause the container to freeze	2024-08-27 15:40:05 +00:00
John Colvin	e7bc230c21	cleanup: Remove unused includes Mostly collection macro headers	2024-08-27 15:40:05 +00:00
John Colvin	55ba3be74d	cleanup (build): Move protobuf service generation cmake into lib_spark Because lib_spark owns all the services	2024-08-27 15:40:05 +00:00
John Colvin	b06c11d289	refactor: Rename protobuf out dir "include" -> "protobufs" This is a more accurate and precise name. The dir contains c++ code generated from .proto files - both sources (.cc) and headers (.pb.h).	2024-08-27 15:40:05 +00:00
John Colvin	18794bf3c8	refactor: Move ValueWrapper into lib_system So other projects can use it	2024-08-27 15:40:05 +00:00
John Colvin	7c70828452	Include bts_assert.h in bts_fatal.h We shouldn't have to include bts_assert.h when we're already including bts_fatal.h	2024-08-27 15:40:05 +00:00
John Colvin	bb4805763b	refactor: Split MarketDataGateway up into providers/sources MarketDataGateway still exists, but is greatly simplified. It creates several `IMarketDataProvider`s, each of which generates a list of `IMarketDataSource`s. The market data sources provide market data to an `IMarketDataListener`. * EmumbaMarketDataProvider * CCApiMarketDataProvider * PyMarketDataProvider * OnixsMarketDataProvider * ReplayMarketDataProvider As a side effect, individual py market data sources are now explicitly limited to a single venue (it probably should have been this way all along, since the shared memory writers do not expect updates to involve multiple venues).	2024-08-27 15:39:57 +00:00
John Colvin	753006be10	fix build: Remove leftover cmake setup From before recent build refactoring.	2024-08-27 15:35:58 +00:00
John Colvin	50e567edda	fix build: Remove old install target Leftover from before recent build refactoring.	2024-08-27 15:35:58 +00:00
John Colvin	49da74d60c	fix: Include GNUInstallDirs in InstallBasicConfig We reference variables defined therein (otherwise the burden of inclusion falls to the caller).	2024-08-27 15:35:58 +00:00
John Colvin	c66ac5b742	cleanup: Remove superfluous code	2024-08-27 15:35:58 +00:00
John Colvin	ecff63b090	improvement: Set SHARED_MEMORY_PREFIX to an environment variable So we can effectively define it at runtime. We don't want to have to produce different builds just to change the shared mem prefix.	2024-08-27 15:35:58 +00:00
John Colvin	8356d3e394	fix: Always prepend proc name to shared mem prefix Our shared memory model isn't really designed to be shared across processes. Maybe some day it will.	2024-08-27 15:35:58 +00:00
John Colvin	48ae662fca	cleanup: Remove redundant macro def	2024-08-27 15:35:58 +00:00
John Colvin	998d0e7133	Skip md recording thread loop if no recorders configured to run	2024-08-27 15:35:58 +00:00
John Colvin	45489850b7	refactor: Rename lib_py_market_data -> lib_python_md	2024-08-27 15:35:56 +00:00
John Colvin	5bb0c3654f	Handle signal interrupts explicitly in sparkd Otherwise the python interpretter, which is instantiated when we load python md plugins, prevents the application from exiting.	2024-08-27 15:35:04 +00:00
John Colvin	fa02d00bcc	check pt: Python cpp md interop refactor: Rename py-market-data -> py_market_data Because '-' is an illegal character in py module names build: Clean up py_market_data cmake proj fix build: Ensure py scripts use unix line endings I'm not sure why, but my sparkd build (linux) suddenly began failing to generate cpp headers for protos. The error message indicated it was due to an unexpected carriage return (\r) in plugin.py: /usr/bin/env: ‘python\r’: No such file or directory build: Add py_market_data_providers cmake proj For md providers implemented in python. They're loaded as plugins in sparkd Clean up MarketDataGateway python md plugins cleanup: Merge provider py scripts into lib_py_market_data proj doc: Add README and doxy docs for py md providers cleanup: Remove old py_market_data_providers project It has been merged into the py_market_data project. Clean up py_market_data install, config.cmake feat: Get python md plugins ready for prime time * Add cmd to run python md provider c++ interop sanity check	2024-08-23 16:37:08 +00:00
John Colvin	079015674d	Add lib_system proj for common utils refactor: Move Delegate.h to lib_system Rename delegate.h (lower case) refactor: Move bts_array_size.h to lib_system fix add lib system fix system fix move bts_array_size fix move delegate	2024-08-23 16:36:16 +00:00
John Colvin	4409a71fb0	vscode: Fix sparkd path in launch.json	2024-08-23 16:36:16 +00:00
John Colvin	06a2c814d0	build: Clean up libspark CMakeLists	2024-08-23 16:36:16 +00:00
John Colvin	ba90abf5ae	cleanup: Trivial whitespace cleanup in sparkd CMakeLists	2024-08-23 16:36:16 +00:00
John Colvin	e41979740a	build: Add cmake fns to symlink and install local libs The code needed to do this (inline) is ugly and verbose	2024-08-23 16:36:16 +00:00
John Colvin	6106bc21ca	cleanup: Use conventional UPPER_CASE var names in sparkd CMakeLists	2024-08-23 16:29:04 +00:00
John Colvin	6b028a2a42	cleanup: Trivial whitespace cleanup in Protobuf.cmake	2024-08-23 16:29:04 +00:00
John Colvin	7da3da74a9	build: Remove explicit targets from sparkd build preset So it builds the default targets (all). They were only listed explicitly in the first place so we could build specific targets in vs code using the dropdown	2024-08-23 16:29:04 +00:00
John Colvin	9d56826797	build: Set up necessary paths, symlinks for sparkd in sparkd cmake proj Rather than performing this step in the root CMakeLists, as we previously did in the "server_layout" target. This effectively replaces the "server_layout" target.	2024-08-23 16:29:04 +00:00
John Colvin	92159c01ba	build: Clean up data_files cmake proj (support --target clean)	2024-08-23 16:22:42 +00:00
John Colvin	8d260a979c	fix build: Generate proto hdrs in "include" dir (child of proj's bin dir) Reorganizing the sparkd proj with stuttering dirs broke the build because the generated protos were being dumped into a dir with the same name as the "sparkd" executable.	2024-08-23 16:22:42 +00:00
John Colvin	89305eb7cf	fix build: Update sparkd & risk webapp after moving libspark	2024-08-23 16:22:42 +00:00
John Colvin	bb1e113893	refactor: Parse input args in Protobuf_Generate_cpp So we can be more precise when generating protobufs. For example, now we can specify different include dirs when generating code for different group of proto messages - an obvious improvement over the previous system, which relied on a hardcoded superset of include dirs that would support every proto message in our code base.	2024-08-23 16:22:42 +00:00
John Colvin	28f27deb2e	refactor: Orgnaize libspark proj w/ stuttering include	2024-08-23 16:22:41 +00:00
John Colvin	9a1cc9664c	build: Add install targets for libspark	2024-08-23 14:40:50 +00:00
John Colvin	2e7189463a	vscode: Add debug cfg to skip sparkd build	2024-08-13 17:39:33 +00:00
John Colvin	8cde5bcd41	fix build: Add ShmSegmentName.cpp to windows client sources	2024-08-13 17:39:33 +00:00
John Colvin	43d0022878	Make a few timestamp fns const They should have been const in the first place	2024-08-13 17:39:32 +00:00
John Colvin	65133e8a5e	Add config profiles to libspark (check pt) This is part of a larger effort to unify the libspark build, and eliminate compile time differences between the builds used by our various services	2024-08-13 17:39:32 +00:00
John Colvin	781bc52ea1	Add api server to docker compose	2024-08-13 17:39:32 +00:00
John Colvin	8de20e51d7	Rename pingtest "sparkd-ping-test", make it depend on sparkd Add it to a "ping-test" profile, as we may add similar tests for other services	2024-08-13 17:39:32 +00:00
John Colvin	8e57209e52	cleanup: Remove unnecessary env vars from webapp docker compose	2024-08-13 17:39:32 +00:00
John Colvin	c5eba43ace	Change api server ports so they don't conflict w/ ds webapp	2024-08-13 17:39:32 +00:00
John Colvin	86810bb445	vscode: Fix sparkd debug config	2024-08-13 17:39:32 +00:00
John Colvin	ae2c116840	build: Resolve app data dir and shared mem prefix at runtime We really can't define them at compile time anymore as we build multiple exes that depend on libspark in the same dir (e.g., spark api server, pubsub, ds-publisher, webapp, etc.). That means one libspark.so (or dll) must serve all exes. We should compile a consistent libspark to mitigate the risk of running an exe with the wrong version (of libspark). For example, running pubsub with a libspark configured for spark api server, leftover from an earlier build.	2024-08-13 17:39:32 +00:00