check pt: Combine lib_py_market_data and provider py scripts into one project

I'm not sure why, but my sparkd build (linux) suddenly began failing to
generate cpp headers for protos. The error message indicated it was due
to an unexpected carriage return (\r) in plugin.py:

/usr/bin/env: ‘python\r’: No such file or directory

.vscode/launch.json

@@ -203,12 +203,12 @@
       "preLaunchTask": "Build Sparkd (Debug | Clang)",
       "program": "",
       "linux": {
-        "program": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd/sparkd",
+        "program": "${workspaceFolder}/build/sparkd-debug-clang/server/sparkd",
       },
       "args": [
         "run"
       ],
-      "cwd": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd",
+      "cwd": "${workspaceFolder}/build/sparkd-debug-clang/server",
       "stopAtEntry": false,
       "externalConsole": false,
       "MIMode": "gdb",
@@ -231,12 +231,12 @@
       "request": "launch",
       "program": "",
       "linux": {
-        "program": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd/sparkd",
+        "program": "${workspaceFolder}/build/sparkd-debug-clang/server/sparkd",
       },
       "args": [
-        "test-py-md-provider"
+        "run"
       ],
-      "cwd": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd",
+      "cwd": "${workspaceFolder}/build/sparkd-debug-clang/server",
       "stopAtEntry": false,
       "externalConsole": false,
       "MIMode": "gdb",
@@ -252,7 +252,7 @@
           "ignoreFailures": true
         }
       ]
-    },
+    }
   ],
   "inputs": [
     {

CMakeLists.txt

@@ -1,5 +1,5 @@
 cmake_minimum_required(VERSION 3.12)
-project(spark_root)
+project(spark)
 set(CMAKE_CXX_STANDARD 20)
 
 if(WIN32)
@@ -18,7 +18,7 @@ set(
 # SHARED_MEMORY_PREFIX serves as a kind of namespace so multiple applications
 # can run simultaneously without fighting over the same shm files (e.g., sparkd,
 # api server, etc.).
-set(SHARED_MEMORY_PREFIX "\${BTS_SHARED_MEMORY_PREFIX}" CACHE STRING "Prefix to use when creating shared memory files")
+set(SHARED_MEMORY_PREFIX "bts" CACHE STRING "Prefix to use when creating shared memory files")
 set(EXTERNALS_BIN "${EXTERNALS_ROOT}/bin")
 set(EXTERNALS_INCLUDE "${EXTERNALS_ROOT}/include")
 set(EXTERNALS_LIB "${EXTERNALS_ROOT}/lib/$<CONFIG>")
@@ -45,6 +45,11 @@ if(WIN32)
   # subdirs
   add_subdirectory(src/lib_spark)
 else()
+  # make sparkd look for libspark.so in its directory
+  # set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE)
+  # set(CMAKE_INSTALL_RPATH ".")
+  # set(CMAKE_INSTALL_PREFIX "${CMAKE_SOURCE_DIR}/install/$<CONFIG>")
+
   if(BUILD_SPARKD)
     # Note: All macros that begin w/ 'CCAPI' are ccapi's macros (defined and/or
     # read by their code). We override some below.
@@ -104,11 +109,10 @@ else()
 
   # subdirs
   if (BUILD_SPARKD)
-    add_subdirectory(src/lib_system)
-    add_subdirectory(src/lib_domain)
-    add_subdirectory(src/lib_python_md)
     add_subdirectory(src/app_sparkd)
     add_subdirectory(src/data_files)
+    add_subdirectory(src/lib_py_market_data)
+    # add_subdirectory(src/py_market_data_providers)
   endif()
   add_subdirectory(src/lib_spark)
   if (BUILD_ADMIN_WEB_APP)

CMakePresets.json

@@ -294,14 +294,6 @@
       "inherits": "sparkd"
     },
 
-    {
-      "name": "sparkd-debug-clang-j6",
-      "configurePreset": "sparkd-debug-clang",
-      "displayName": "Sparkd (Debug | Clang | jobs=6)",
-      "inherits": "sparkd",
-      "jobs": 6
-    },
-
     {
       "name": "sparkd-debug-gcc",
       "configurePreset": "sparkd-debug-gcc",

cmake/ExportToParent.cmake

@@ -1,5 +0,0 @@
-macro(export_to_parent)
-  foreach(var IN ITEMS ${ARGN})
-    set(${var} ${${var}} PARENT_SCOPE)
-  endforeach()
-endmacro()

cmake/InitExecutable.cmake

@@ -1,53 +1,53 @@
-include_guard(DIRECTORY)
-include(${CMAKE_SOURCE_DIR}/cmake/ExportToParent.cmake)
-include(${CMAKE_SOURCE_DIR}/cmake/TernaryOperator.cmake)
-
-function(init_library)
-  set(_options SHARED STATIC)
-  set(_one_value_args SOURCE_DIR)
-  set(_multi_value_args)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-
-  if (_ARGS_SHARED AND _ARGS_STATIC)
-    message(FATAL_ERROR "Invalid args passed to init_library: cannot specify STATIC and SHARED")
+function(init_executable)
+  set(source_dir)
+  if (${ARGC} EQUAL 0)
+    set(source_dir ${PROJECT_NAME})
+  elseif(${ARGC} GREATER 1)
+    message(FATAL_ERROR "init_shared_library received invalid arguments. Usage: init_shared_library(<OPTIONAL:source_dir>)")
+  else()
+    set(source_dir ${ARGV})
   endif()
 
-  set_ternary(_source_dir _ARGS_SOURCE_DIR _ARGS_SOURCE_DIR ${PROJECT_NAME})
-  # Glob cpp sources
+  set(_options)
+  set(_one_value_args CXX_SOURCES CXX_HEADERS CXX_OUT_DIR)
+  set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+  if (${_ARGS_PROTO_SOURCES} EQUAL "GLOB_RECURSIVE")
+    file(
+      GLOB_RECURSE ${PROJECT_NAME}_PROTO_SOURCES
+      RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
+      *.proto
+    )
+  endif()
+
+  set(${PROJECT_NAME}_SOURCES PARENT_SCOPE)
   file(
     GLOB_RECURSE ${PROJECT_NAME}_SOURCES
     RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
-    ${_source_dir}/*.cpp
+    ${source_dir}/*.cpp
   )
+  set(${PROJECT_NAME}_SOURCES ${${PROJECT_NAME}_SOURCES} PARENT_SCOPE)
+  # set(${PROJECT_NAME}_Sources ${${PROJECT_NAME}_Sources} PARENT_SCOPE)
   # Note: cpp files inside 'tests' dir(s) are considered tests.
   # Remove tests from main sources, and add them to test sources
+  set(${PROJECT_NAME}_TEST_SOURCES)
   foreach(item ${${PROJECT_NAME}_SOURCES})
     if(${item} MATCHES "tests/*.cpp")
       list(REMOVE_ITEM ${PROJECT_NAME}_SOURCES ${item})
       list(APPEND ${PROJECT_NAME}_TEST_SOURCES ${item})
     endif()
+  set(${PROJECT_NAME}_TEST_SOURCES ${${PROJECT_NAME}_TEST_SOURCES} PARENT_SCOPE)
   endforeach()
 
-  export_to_parent(
-    ${PROJECT_NAME}_SOURCES
-    ${PROJECT_NAME}_TEST_SOURCES
-  )
-
-  # Add main library
-  if (_ARGS_STATIC)
-    add_library(${PROJECT_NAME} STATIC ${${PROJECT_NAME}_SOURCES})
-  elseif(_ARGS_SHARED)
-    add_library(${PROJECT_NAME} SHARED ${${PROJECT_NAME}_SOURCES})
-  else()
-    add_library(${PROJECT_NAME} ${${PROJECT_NAME}_SOURCES})
-  endif()
-
+  # Add executable
+  add_executable(${PROJECT_NAME} ${${PROJECT_NAME}_SOURCES})
   target_include_directories(
     ${PROJECT_NAME}
     PUBLIC
       $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
       $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>
   )
+
   target_link_directories(
     ${PROJECT_NAME}
     PUBLIC
@@ -57,7 +57,7 @@ function(init_library)
 
   # Add tests executable (if there are any)
   if (${PROJECT_NAME}_TEST_SOURCES)
-    add_executable(${PROJECT_NAME}-tests ${${PROJECT_NAME}_TEST_SOURCES})
+    add_executable(${PROJECT_NAME}-tests ${${PROJECT_NAME}_TEST_SOURCEs})
     target_link_libraries(${PROJECT_NAME}-tests PRIVATE ${PROJECT_NAME})
     # Catch2
     find_package(Catch2 CONFIG REQUIRED COMPONENTS Catch2WithMain)

cmake/InitSharedLibrary.cmake

@@ -1,5 +1,4 @@
 function(init_shared_library)
-  include(${CMAKE_SOURCE_DIR}/cmake/ExportToParent.cmake)
   set(source_dir)
   if (${ARGC} EQUAL 0)
     set(source_dir ${PROJECT_NAME})
@@ -8,35 +7,33 @@ function(init_shared_library)
   else()
     set(source_dir ${ARGV})
   endif()
-
-  # Glob cpp sources
+  set(${PROJECT_NAME}_Sources PARENT_SCOPE)
   file(
-    GLOB_RECURSE ${PROJECT_NAME}_SOURCES
+    GLOB_RECURSE ${PROJECT_NAME}_Sources
     RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
-    ${source_dir}/*.cpp
-  )
+    ${source_dir}/*.cpp)
+  set(${PROJECT_NAME}_Sources ${${PROJECT_NAME}_Sources} PARENT_SCOPE)
+  # set(${PROJECT_NAME}_Sources ${${PROJECT_NAME}_Sources} PARENT_SCOPE)
   # Note: cpp files inside 'tests' dir(s) are considered tests.
   # Remove tests from main sources, and add them to test sources
-  foreach(item ${${PROJECT_NAME}_SOURCES})
+  set(${PROJECT_NAME}_Test_Sources)
+  foreach(item ${${PROJECT_NAME}_Sources})
     if(${item} MATCHES "tests/*.cpp")
-      list(REMOVE_ITEM ${PROJECT_NAME}_SOURCES ${item})
-      list(APPEND ${PROJECT_NAME}_TEST_SOURCES ${item})
+      list(REMOVE_ITEM ${PROJECT_NAME}_Sources ${item})
+      list(APPEND ${PROJECT_NAME}_Test_Sources ${item})
     endif()
+  set(${PROJECT_NAME}_Test_Sources ${${PROJECT_NAME}_Test_Sources} PARENT_SCOPE)
   endforeach()
 
-  export_to_parent(
-    ${PROJECT_NAME}_SOURCES
-    ${PROJECT_NAME}_TEST_SOURCES
-  )
-
   # Add main library
-  add_library(${PROJECT_NAME} SHARED ${${PROJECT_NAME}_SOURCES})
+  add_library(${PROJECT_NAME} SHARED ${${PROJECT_NAME}_Sources})
   target_include_directories(
     ${PROJECT_NAME}
     PUBLIC
       $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
       $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>
   )
+
   target_link_directories(
     ${PROJECT_NAME}
     PUBLIC
@@ -45,8 +42,8 @@ function(init_shared_library)
   )
 
   # Add tests executable (if there are any)
-  if (${PROJECT_NAME}_TEST_SOURCES)
-    add_executable(${PROJECT_NAME}-tests ${${PROJECT_NAME}_TEST_SOURCES})
+  if (${PROJECT_NAME}_Test_Sources)
+    add_executable(${PROJECT_NAME}-tests ${${PROJECT_NAME}_Test_Sources})
     target_link_libraries(${PROJECT_NAME}-tests PRIVATE ${PROJECT_NAME})
     # Catch2
     find_package(Catch2 CONFIG REQUIRED COMPONENTS Catch2WithMain)

cmake/InstallBasicConfig.cmake

@@ -1,5 +1,4 @@
 include(${CMAKE_SOURCE_DIR}/cmake/PackBasic.cmake)
-include(GNUInstallDirs)
 
 function(install_basic_config)
   file(

cmake/PackBasic.cmake

@@ -1,21 +1,6 @@
 include(CMakePackageConfigHelpers)
 
 function(pack_basic)
-  find_package(Git QUIET)
-
-  if(GIT_FOUND)
-    execute_process(
-      COMMAND ${GIT_EXECUTABLE} describe --tags --always --dirty
-      WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
-      OUTPUT_VARIABLE PROJECT_VERSION
-      OUTPUT_STRIP_TRAILING_WHITESPACE
-    )
-  endif()
-
-  if(NOT GIT_FOUND OR PROJECT_VERSION STREQUAL "")
-    set(PROJECT_VERSION "0.0.0")
-  endif()
-
   write_basic_package_version_file(
     "${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}ConfigVersion.cmake"
     VERSION ${PROJECT_VERSION}

cmake/Protobuf.cmake

@@ -0,0 +1,167 @@
+set(Protobuf_PROTOC_EXECUTABLE "${EXTERNALS_BIN}/protoc")
+set(PROTOBUF_INCLUDE_DIRS "${EXTERNALS_INCLUDE}")
+set(PROTOBUF_STATIC_LIB "libprotobuf")
+set(_protobuf_src_dir ${CMAKE_SOURCE_DIR}/src/lib_spark)
+set(_protobuf_bin_dir ${CMAKE_BINARY_DIR}/src/lib_spark)
+
+function(Protobuf_Generate_Cpp)
+set(_options)
+  set(_one_value_args CXX_SOURCES CXX_HEADERS CXX_OUT_DIR)
+  set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+  set(${_ARGS_CXX_SOURCES})
+  set(${_ARGS_CXX_HEADERS})
+
+  set(_proto_include_cli_args)
+  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
+    list(APPEND _proto_include_cli_args -I ${_dir})
+  endforeach()
+
+  foreach(FIL ${_ARGS_PROTO_SOURCES})
+    get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
+    get_filename_component(FIL_WE ${FIL} NAME_WE)
+    get_filename_component(FIL_DIR ${FIL} DIRECTORY)
+    if(FIL_DIR)
+      set(FIL_WE "${FIL_DIR}/${FIL_WE}")
+    endif()
+
+    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc")
+    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h")
+
+    add_custom_command(
+      OUTPUT
+        "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
+        "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
+      COMMAND
+        ${Protobuf_PROTOC_EXECUTABLE}
+      ARGS
+        --cpp_out=${_ARGS_CXX_OUT_DIR} ${_proto_include_cli_args} ${ABS_FIL}
+      DEPENDS ${ABS_FIL}
+      COMMENT "Running C++ protocol buffer compiler on ${FIL}"
+      VERBATIM)
+  endforeach()
+
+  set_source_files_properties(${${_ARGS_CXX_SOURCES}} ${${_ARGS_CXX_HEADERS}} PROPERTIES GENERATED TRUE)
+
+  # remove some warnings from protobuf generated files
+  if(WIN32)
+    set_source_files_properties(${${_ARGS_CXX_SOURCES}} PROPERTIES COMPILE_FLAGS
+                                                      "/wd4309 /wd4838")
+  else()
+    set_source_files_properties(
+      ${${_ARGS_CXX_SOURCES}}
+      PROPERTIES
+        COMPILE_FLAGS
+        "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
+    )
+  endif()
+
+  set(${_ARGS_CXX_SOURCES}
+      ${${_ARGS_CXX_SOURCES}}
+      PARENT_SCOPE)
+  set(${_ARGS_CXX_HEADERS}
+      ${${_ARGS_CXX_HEADERS}}
+      PARENT_SCOPE)
+endfunction()
+
+function(Service_Compiler_Generate_Cpp)
+  set(_options)
+  set(_one_value_args CXX_SOURCES CXX_HEADERS CXX_OUT_DIR)
+  set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+  set(${_ARGS_CXX_SOURCES})
+  set(${_ARGS_CXX_HEADERS})
+
+  set(_proto_include_cli_args)
+  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
+    list(APPEND _proto_include_cli_args -I ${_dir})
+  endforeach()
+
+  foreach(FIL ${_ARGS_PROTO_SOURCES})
+    get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
+    get_filename_component(FIL_WE ${FIL} NAME_WE)
+    get_filename_component(FIL_DIR ${FIL} DIRECTORY)
+    if(FIL_DIR)
+      set(FIL_WE "${FIL_DIR}/${FIL_WE}")
+    endif()
+
+    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc")
+    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h")
+
+    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc")
+    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h")
+
+    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc")
+    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h")
+
+    set(TEMPLATES
+        ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/client.pb.cc
+        ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/client.pb.h
+        ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/server.pb.cc
+        ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/server.pb.h
+    )
+
+    set(_plugin_py ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/plugin.py)
+    if(WIN32)
+      set(_plugin_cmd ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/plugin.cmd)
+      add_custom_command(
+        OUTPUT "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h"
+        COMMAND SET PYTHONPATH=${SPARK_PYTHON}
+        COMMAND ${CMAKE_SOURCE_DIR}/.venv/Scripts/activate.bat
+        COMMAND
+          ${Protobuf_PROTOC_EXECUTABLE} ARGS --cpp_out=${_ARGS_CXX_OUT_DIR}
+          --plugin=protoc-gen-service=${_plugin_cmd}
+          --service_out=lang=cpp:${_ARGS_CXX_OUT_DIR} -I
+          ${PROTOBUF_INCLUDE_DIRS} -I ${_protobuf_src_dir} ${ABS_FIL}
+        DEPENDS ${ABS_FIL} ${TEMPLATES} ${_plugin_cmd} ${_plugin_py}
+        COMMENT "Running C++ service compiler on ${FIL}"
+        VERBATIM)
+    else()
+      add_custom_command(
+        OUTPUT "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h"
+        COMMAND ${CMAKE_SOURCE_DIR}/tools/venv_exec.sh
+        ARGS
+          ${Protobuf_PROTOC_EXECUTABLE}
+          --cpp_out=${_ARGS_CXX_OUT_DIR}
+          --plugin=protoc-gen-service=${_plugin_py}
+          --service_out=lang=cpp,make_file_name=${ABS_FIL}:${_ARGS_CXX_OUT_DIR}
+          ${_proto_include_cli_args} ${ABS_FIL}
+        DEPENDS ${ABS_FIL} ${TEMPLATES} ${_plugin_py}
+        COMMENT "Running C++ service compiler on ${FIL}"
+        VERBATIM)
+    endif()
+
+  endforeach()
+
+  # remove some warnings from protobuf generated files
+  if(WIN32)
+    set_source_files_properties(${${_ARGS_CXX_SOURCES}} PROPERTIES COMPILE_FLAGS
+                                                      "/wd4309 /wd4838")
+  else()
+    set_source_files_properties(
+      ${${_ARGS_CXX_SOURCES}}
+      PROPERTIES
+        COMPILE_FLAGS
+        "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
+    )
+  endif()
+
+  set(${_ARGS_CXX_SOURCES}
+      ${${_ARGS_CXX_SOURCES}}
+      PARENT_SCOPE)
+  set(${_ARGS_CXX_HEADERS}
+      ${${_ARGS_CXX_HEADERS}}
+      PARENT_SCOPE)
+
+endfunction()
+

cmake/ProtobufGenerateCpp.cmake

@@ -1,67 +0,0 @@
-include_guard(GLOBAL)
-
-function(protobuf_generate_cpp)
-  set(_options)
-  set(_one_value_args CXX_SOURCES CXX_HEADERS CXX_OUT_DIR PROTOC)
-  set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-
-  if (NOT DEFINED _ARGS_PROTOC)
-    set(_ARGS_PROTOC "${EXTERNALS_BIN}/protoc")
-  endif()
-  set(${_ARGS_CXX_SOURCES})
-  set(${_ARGS_CXX_HEADERS})
-
-  set(_proto_include_cli_args)
-  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
-    list(APPEND _proto_include_cli_args -I ${_dir})
-  endforeach()
-
-  foreach(_fil ${_ARGS_PROTO_SOURCES})
-    get_filename_component(_abs_fil ${_fil} ABSOLUTE)
-    get_filename_component(_fil_we ${_fil} NAME_WE)
-    get_filename_component(_fil_dir ${_fil} DIRECTORY)
-    if(_fil_dir)
-      set(_fil_we "${_fil_dir}/${_fil_we}")
-    endif()
-    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.cc")
-    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.h")
-    add_custom_command(
-      OUTPUT
-        "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.cc"
-        "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.h"
-      COMMAND
-        ${_ARGS_PROTOC}
-      ARGS
-        --cpp_out=${_ARGS_CXX_OUT_DIR} ${_proto_include_cli_args} ${_abs_fil}
-      DEPENDS ${_abs_fil}
-      COMMENT "Running C++ protocol buffer compiler on ${_fil}"
-      VERBATIM
-    )
-  endforeach()
-
-  set_source_files_properties(
-    ${${_ARGS_CXX_SOURCES}}
-    ${${_ARGS_CXX_HEADERS}}
-    PROPERTIES
-      GENERATED TRUE
-  )
-
-  # remove some warnings from protobuf generated files
-  if(WIN32)
-    set_source_files_properties(
-      ${${_ARGS_CXX_SOURCES}}
-      PROPERTIES
-        COMPILE_FLAGS "/wd4309 /wd4838"
-    )
-  else()
-    set_source_files_properties(
-      ${${_ARGS_CXX_SOURCES}}
-      PROPERTIES
-        COMPILE_FLAGS "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
-    )
-  endif()
-
-  set(${_ARGS_CXX_SOURCES} ${${_ARGS_CXX_SOURCES}} PARENT_SCOPE)
-  set(${_ARGS_CXX_HEADERS} ${${_ARGS_CXX_HEADERS}} PARENT_SCOPE)
-endfunction()

cmake/ProtobufGenerateFs.cmake

@@ -1,49 +0,0 @@
-include_guard(DIRECTORY)
-
-function(protobuf_generate_fs)
-  set(_options)
-  set(_one_value_args TARGET_NAME OUT_DIR PROTOC)
-  set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-
-  if (NOT DEFINED _ARGS_TARGET_NAME)
-    set(_ARGS_TARGET_NAME "${PROJECT_NAME}_generate_fs_protos")
-  endif()
-  if (NOT DEFINED _ARGS_PROTOC)
-    set(_ARGS_PROTOC "${EXTERNALS_BIN}/protoc")
-  endif()
-
-  set(_proto_include_cli_args)
-  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
-    list(APPEND _proto_include_cli_args -I ${_dir})
-  endforeach()
-
-  set(_outputs)
-  foreach(_proto_source ${_ARGS_PROTO_SOURCES})
-    # get_filename_component(_abs_proto_source ${_proto_source} ABSOLUTE)
-    get_filename_component(_proto_source_we ${_proto_source} NAME_WE)
-    # get_filename_component(_proto_source_dir ${_proto_source} DIRECTORY)
-    list(APPEND _outputs ${_ARGS_OUT_DIR}/${_proto_source_we}.fable.fs)
-  endforeach()
-
-  set(_venv_exec ${CMAKE_SOURCE_DIR}/tools/venv_exec.sh)
-  set(_plugin_py ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/plugin.py)
-  set(_template_fs ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/fs/fable.fs)
-  add_custom_command(
-    OUTPUT ${_outputs}
-    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
-    COMMAND ${_venv_exec}
-      ${_ARGS_PROTOC}
-      --plugin=protoc-gen-service=${_plugin_py}
-      --service_out=lang=fs,strip_out_path=1:${_ARGS_OUT_DIR}
-      ${_proto_include_cli_args}
-      ${_ARGS_PROTO_SOURCES}
-    DEPENDS
-      ${_venv_exec}
-      ${_plugin_py}
-      ${_template_fs}
-      ${CMAKE_CURRENT_SOURCE_DIR}/package-lock.json
-      ${_ARGS_PROTO_SOURCES}
-  )
-  add_custom_target(${_ARGS_TARGET_NAME} ALL DEPENDS "${_outputs}")
-endfunction()

cmake/ProtobufGenerateJs.cmake

@@ -1,33 +0,0 @@
-include_guard(DIRECTORY)
-
-function(protobuf_generate_js)
-  set(_options)
-  set(_one_value_args TARGET_NAME OUTPUT)
-  set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-
-  if (NOT DEFINED _ARGS_TARGET_NAME)
-    set(_ARGS_TARGET_NAME "${PROJECT_NAME}_generate_js_protos")
-  endif()
-
-  set(_proto_include_cli_args)
-  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
-    list(APPEND _proto_include_cli_args -p ${_dir})
-  endforeach()
-
-  add_custom_command(
-    OUTPUT ${_ARGS_OUTPUT}
-    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
-    COMMAND npx
-      --no-install pbjs
-      -t json-module
-      -w commonjs
-      -o ${_ARGS_OUTPUT}
-      ${_proto_include_cli_args}
-      ${_ARGS_PROTO_SOURCES}
-    DEPENDS
-      ${CMAKE_CURRENT_SOURCE_DIR}/package-lock.json
-      ${_ARGS_PROTO_SOURCES}
-  )
-  add_custom_target(${_ARGS_TARGET_NAME} ALL DEPENDS ${_ARGS_OUTPUT})
-endfunction()

cmake/SetFirstDefined.cmake

@@ -0,0 +1,8 @@
+macro(set_first_defined RESULT CANDIDATE_VARS)
+  foreach(_candidate_var ${CANDIDATE_VARS})
+    if (DEFINED _candidate_var)
+      set(${RESULT} ${_candidate_var})
+      break()
+    endif()
+  endforeach()
+endmacro()

cmake/SymLink.cmake

@@ -1,20 +1,11 @@
-include_guard(DIRECTORY)
-include(${CMAKE_SOURCE_DIR}/cmake/TernaryOperator.cmake)
-
-function(sym_link_dir)
-  set(_options)
-  set(_one_value_args TARGET_NAME SOURCE DEST COMMENT)
-  set(_multi_value_args)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+function(sym_link_dir TARGET_NAME SOURCE DEST)
   add_custom_command(
-    OUTPUT ${_ARGS_DEST}
-    COMMAND ln ARGS -sf ${_ARGS_SOURCE} ${_ARGS_DEST}
+    OUTPUT ${DEST}
+    COMMAND ln ARGS -sf ${SOURCE} ${DEST}
   )
-  set_ternary(_comment _ARGS_COMMENT "${_ARGS_COMMENT}" "linking dir ${_ARGS_SOURCE} -> ${_ARGS_DEST}")
   add_custom_target(
-    ${_ARGS_TARGET_NAME} ALL
-    DEPENDS ${_ARGS_DEST}
-    COMMENT "${_comment}"
+    ${TARGET_NAME} ALL
+    DEPENDS ${DEST}
   )
 endfunction()
 

cmake/TernaryOperator.cmake

@@ -1,5 +1,3 @@
-include_guard(GLOBAL)
-
 macro(set_ternary RESULT CONDITION TRUE_VALUE FALSE_VALUE)
   if(${CONDITION})
     set(${RESULT} ${TRUE_VALUE})

doc/api/content/server/ref/subscriptions.md

@@ -53,7 +53,7 @@ The status callback is called when the market for this security opens, closes, o
 // Subscribe to security trading status
 void IAlgoServices::subscribe_status(SecurityKey);
 // You implement
-virtual void on_status(SecurityKey, domain::protobufs::TradingStatus) override;
+virtual void on_status(SecurityKey, co2::TradingStatus) override;
 ```
 
 # Child Orders

src/Spark.Admin.WebApp/CMakeLists.txt

@@ -1,43 +1,55 @@
 cmake_minimum_required(VERSION 3.5)
 project(Spark.Admin.WebApp)
 
-include(GNUInstallDirs)
 include(${CMAKE_SOURCE_DIR}/cmake/safe_dep_glob.cmake)
-include(${CMAKE_SOURCE_DIR}/cmake/SymLink.cmake)
-include(${CMAKE_SOURCE_DIR}/cmake/ProtobufGenerateJs.cmake)
-include(${CMAKE_SOURCE_DIR}/cmake/ProtobufGenerateFs.cmake)
 
-file(
-  GLOB_RECURSE ${PROJECT_NAME}_PROTO_SOURCES
-  RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
-  ../lib_domain/domain/protobufs/*.proto
+# dependencies
+set(PROTO_GLOBS
   ../lib_spark/spark/api/*.proto
   ../lib_spark/spark/api/services/*.proto
   ../lib_spark/spark/domain_model/*.proto
   ../lib_spark/spark/sparkproto/*.proto
   ../lib_spark/spark/md/protos/*.proto
   ../../externals/linux/include/google/protobuf/descriptor.proto
-  ../../externals/linux/include/google/protobuf/timestamp.proto
-)
+  ../../externals/linux/include/google/protobuf/timestamp.proto)
 
-add_custom_command(
-  OUTPUT ${CMAKE_CURRENT_SOURCE_DIR}/package-lock.json
-  COMMAND npm install --include=dev
-  DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/package.json
-  WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
-)
+file(
+  GLOB_RECURSE _proto_rel_SOURCES
+  RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
+  ${PROTO_GLOBS})
+
+file(
+  GLOB_RECURSE _proto_abs_SOURCES
+  ${PROTO_GLOBS})
+
+file(
+  GLOB_RECURSE _ts_SOURCES
+  RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
+  src/*.ts)
+
+file(
+  GLOB_RECURSE _fs_SOURCES
+  RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
+  src/*.fs)
 
 set(_tools ${CMAKE_SOURCE_DIR}/tools)
 set(_quiet_exec ${_tools}/quiet_exec.sh)
 set(_venv_exec ${_tools}/venv_exec.sh)
 set(_tools_py ${CMAKE_SOURCE_DIR}/tools/py)
 
+# The tsconfig file is generated as a function of the cmake build type, and
+# binary dir. Depending on the build type, it extends either
+# 'tsconfig.debug.json' or 'tsconfig.release.json'.
 set(_tsconfig_base "${CMAKE_CURRENT_SOURCE_DIR}/tsconfig.$<IF:$<CONFIG:Debug>,debug,release>.json")
+set(WEBPACK_MODE $<IF:$<CONFIG:Debug>,"development","production">)
 set(_tsconfig "${CMAKE_CURRENT_BINARY_DIR}/tsconfig.json")
 
 set(_venv_python ${CMAKE_SOURCE_DIR}/.venv/bin/python)
 set(_gen_tsconfig_py ${_tools_py}/gen_tsconfig.py)
 set(_build_admin_web_app_py ${_tools_py}/build_admin_web_app.py)
+
+set(_gen_tsconfig_deps ${_venv_exec} ${_venv_python} ${_gen_tsconfig_py} ${_tsconfig_base})
+
 set(_build_admin_web_app_deps
   ${_quiet_exec}
   ${_venv_exec}
@@ -47,133 +59,49 @@ set(_build_admin_web_app_deps
   ${CMAKE_CURRENT_SOURCE_DIR}/webpack.config.js
   ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/fs/fable.fs
   ${CMAKE_CURRENT_SOURCE_DIR}/package-lock.json
-  ${${PROJECT_NAME}_PROTO_SOURCES}
+  ${_proto_rel_SOURCES}
   ${_api_ts_SOURCES}
   ${_ui_fs_SOURCES}
   ${_tsconfig})
 
 update_deps_file("${_build_admin_web_app_deps}")
 
+function(npm_install path)
+  add_custom_command(
+    OUTPUT ${path}/package-lock.json
+    COMMAND "npm" ARGS "install" "--include=dev"
+    DEPENDS ${path}/package.json
+    WORKING_DIRECTORY ${path}
+    COMMENT "Run 'npm install' in ${path}")
+endfunction()
+
+npm_install(${CMAKE_CURRENT_SOURCE_DIR})
+
 add_custom_command(
   OUTPUT ${_tsconfig}
   COMMAND ${_venv_exec}
   ARGS
-    ${_venv_python}
-    ${_gen_tsconfig_py}
-    --config-base ${_tsconfig_base}
-    --config-out ${_tsconfig}
-    --build-out ${CMAKE_CURRENT_BINARY_DIR}
-  DEPENDS
-    ${_venv_exec}
-    ${_venv_python}
-    ${_gen_tsconfig_py}
-    ${_tsconfig_base}
-    ${CMAKE_CURRENT_SOURCE_DIR}/package-lock.json
-)
-
-protobuf_generate_js(
-  TARGET_NAME generate_js_protos
-  PROTO_INCLUDE_DIRS
-    ../lib_domain
-    ../lib_spark
-    ../../externals/linux/include
-  PROTO_SOURCES ${${PROJECT_NAME}_PROTO_SOURCES}
-  OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/proto.js
-)
-
-protobuf_generate_fs(
-  TARGET_NAME generate_fs_protos
-  PROTO_INCLUDE_DIRS
-    ../lib_domain
-    ../lib_spark
-    ../../externals/linux/include
-  PROTO_SOURCES ${${PROJECT_NAME}_PROTO_SOURCES}
-  OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/fable
-)
-
-# Each input file maps to multiple output files with different extensions
-# (js, js.map, d.ts, fs). Transpile ts -> d.ts (and js), then d.ts -> fs.
-# Who knew front-end development could be so complicated? `\_(=_=)_/`
-file(
-  GLOB_RECURSE ${PROJECT_NAME}_TS_SOURCES
-  RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
-  src/*.ts
-)
-set(${PROJECT_NAME}_JS_PROTOS)
-set(${PROJECT_NAME}_TS_PROTOS)
-set(${PROJECT_NAME}_FS_PROTOS)
-foreach(_ts_source ${${PROJECT_NAME}_TS_SOURCES})
-  get_filename_component(_ts_source_we ${_ts_source} NAME_WE)
-  set(_prefix ${CMAKE_CURRENT_BINARY_DIR}/${_ts_source_we})
-  list(APPEND ${PROJECT_NAME}_JS_PROTOS ${_prefix}.js ${_prefix}.js.map)
-  list(APPEND ${PROJECT_NAME}_TS_PROTOS ${_prefix}.d.ts ${_prefix}.d.ts.map)
-  list(APPEND ${PROJECT_NAME}_FS_PROTOS ${_prefix}.fs)
-  add_custom_command(
-    OUTPUT ${_prefix}.fs
-    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
-    COMMAND npx --no-install ts2fable ${_prefix}.d.ts ${_prefix}.fs
-    DEPENDS ${_prefix}.d.ts
-  )
-endforeach()
+  ${_venv_python}
+  ${_gen_tsconfig_py}
+  --config-base ${_tsconfig_base}
+  --config-out ${_tsconfig}
+  --build-out ${CMAKE_CURRENT_BINARY_DIR}
+  DEPENDS ${_gen_tsconfig_deps})
 
 add_custom_command(
-  OUTPUT
-    ${${PROJECT_NAME}_JS_PROTOS}
-    ${${PROJECT_NAME}_TS_PROTOS}
-  WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
-  COMMAND npx
-    --no-install tsc
-    -p ${_tsconfig}
-  DEPENDS
-    ${_tsconfig}
-    ${CMAKE_CURRENT_BINARY_DIR}/proto.js
-)
-add_custom_target(
-  transpile_type_script
-  ALL
-  DEPENDS
-    ${${PROJECT_NAME}_JS_PROTOS}
-    ${${PROJECT_NAME}_TS_PROTOS}
-)
-add_custom_target(generate_fsts_stubs ALL DEPENDS ${${PROJECT_NAME}_FS_PROTOS})
+  OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/bundle.js
+  COMMAND ${_quiet_exec}
+  ARGS
+  ${_venv_exec}
+  ${_venv_python}
+  ${_build_admin_web_app_py}
+  --protos ${_proto_abs_SOURCES}
+  --tsconfig ${_tsconfig}
+  --mode ${WEBPACK_MODE}
+  --output-dir ${CMAKE_CURRENT_BINARY_DIR}
+  COMMENT "Building Spark.Admin.WebApp"
+  DEPENDS ${_build_admin_web_app_deps})
 
-sym_link_dir(
-  TARGET_NAME link_webapp_public
-  SOURCE ${CMAKE_CURRENT_SOURCE_DIR}/public
-  DEST ${CMAKE_CURRENT_BINARY_DIR}/public
-)
-sym_link_dir(
-  TARGET_NAME link_webapp_root
-  SOURCE ${CMAKE_CURRENT_BINARY_DIR}
-  DEST ${CMAKE_SOURCE_DIR}/build/Spark.Admin.WebApp
-)
+add_custom_target(admin_web_app_bundle ALL DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/bundle.js)
 
-file(
-  GLOB_RECURSE ${PROJECT_NAME}_FS_SOURCES
-  RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
-  src/*.fs
-)
-add_custom_command(
-  OUTPUT
-    ${CMAKE_CURRENT_BINARY_DIR}/bundle.js
-    $<$<CONFIG:DEBUG>:${CMAKE_CURRENT_BINARY_DIR}/bundle.js.map>
-  WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/src
-  COMMAND dotnet restore
-  COMMAND dotnet tool restore
-  COMMAND export SPARK_ADMIN_WEB_APP_OUTPUT_DIR=${CMAKE_CURRENT_BINARY_DIR}
-  COMMAND dotnet fable -o ${CMAKE_CURRENT_BINARY_DIR}
-  COMMAND npm run build -- --mode $<IF:$<CONFIG:Debug>,"development","production">
-  DEPENDS
-    ${${PROJECT_NAME}_FS_SOURCES}
-    ${${PROJECT_NAME}_FS_PROTOS}
-    ${CMAKE_CURRENT_SOURCE_DIR}/src/Spark.Admin.WebApp.fsproj
-)
-add_custom_target(
-  bundle_spark_admin_webapp ALL
-  DEPENDS
-    ${CMAKE_CURRENT_BINARY_DIR}/bundle.js
-    $<$<CONFIG:DEBUG>:${CMAKE_CURRENT_BINARY_DIR}/bundle.js.map>
-  COMMENT "Bundling Spark.Admin.WebApp"
-)
-
-install(DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}" DESTINATION "${CMAKE_INSTALL_BINDIR}")
+install(DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}" DESTINATION "bin")

src/Spark.Admin.WebApp/package.json

@@ -42,7 +42,7 @@
     "ts-loader": "^9.4.2",
     "tsutils": "^3.21.0",
     "typescript": "^4.9.5",
-    "webpack-cli": "^5.1.4",
+    "webpack-cli": "^5.0.1",
     "webpack-dev-server": "^4.11.1",
     "ws": "^8.12.1"
   }

src/Spark.Admin.WebApp/src/Spark.Admin.WebApp.fsproj

@@ -33,7 +33,6 @@
     <Compile Include="$(API)/fable/timestamp.fable.fs" />
     <Compile Include="$(API)/fable/apicore.fable.fs" />
     <Compile Include="$(API)/fable/apioptions.fable.fs" />
-    <Compile Include="$(API)/fable/aggressor_side.fable.fs" />
     <Compile Include="$(API)/fable/Security.fable.fs" />
     <Compile Include="$(API)/fable/SparkProtos.fable.fs" />
     <Compile Include="$(API)/fable/HelloService.fable.fs" />

src/app_sparkd/CMakeLists.txt

@@ -2,7 +2,7 @@ cmake_minimum_required(VERSION 3.5)
 project(sparkd)
 
 include(${CMAKE_SOURCE_DIR}/cmake/safe_dep_glob.cmake)
-include(${CMAKE_SOURCE_DIR}/cmake/ProtobufGenerateCpp.cmake)
+include(${CMAKE_SOURCE_DIR}/cmake/Protobuf.cmake)
 include(GNUInstallDirs)
 
 if (NOT DEFINED PROJECT_VERSION)
@@ -31,18 +31,15 @@ protobuf_generate_cpp(
     ${CMAKE_SOURCE_DIR}/src/lib_spark
     ${CMAKE_CURRENT_SOURCE_DIR}
   PROTO_SOURCES
-    ${${PROJECT_NAME}_PROTO_SOURCES}
-)
+    ${${PROJECT_NAME}_PROTO_SOURCES})
 add_executable(sparkd ${${PROJECT_NAME}_SOURCES} ${${PROJECT_NAME}_PROTO_CXX_SOURCES})
 
 # include paths
 target_include_directories(
-  sparkd
-  PUBLIC
-    $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
-    $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>/include
-    $<BUILD_INTERFACE:${emumba-deribit-connector-library_INCLUDE_DIRS}>
-)
+  sparkd PUBLIC
+  $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
+  $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>/include
+  $<BUILD_INTERFACE:${emumba-deribit-connector-library_INCLUDE_DIRS}>)
 
 find_package(PkgConfig REQUIRED)
 pkg_check_modules(emumba-deribit-connector-library REQUIRED emumba-deribit-connector)
@@ -51,9 +48,8 @@ pkg_check_modules(emumba-deribit-connector-library REQUIRED emumba-deribit-conne
 list(APPEND CMAKE_PREFIX_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake_modules)
 find_package(rapidjson REQUIRED)
 target_include_directories(
-  ${PROJECT_NAME} SYSTEM
-  PUBLIC
-    $<BUILD_INTERFACE:${rapidjson_INCLUDE_DIR}> 
+  ${PROJECT_NAME} SYSTEM PUBLIC
+  $<BUILD_INTERFACE:${rapidjson_INCLUDE_DIR}> 
 )
 add_dependencies(${PROJECT_NAME} rapidjson_project)
 
@@ -87,48 +83,53 @@ else()
   message("-- Building of DERIBIT_DEMO_LOG_LEVEL=TRACE")
 endif()
 
+# add_dependencies(${PROJECT_NAME} py_market_data)
+list(APPEND CMAKE_PREFIX_PATH "${CMAKE_SOURCE_DIR}/install/$<CONFIG>/lib/cmake/py-market-data")
+
 target_link_directories(
   ${PROJECT_NAME} PUBLIC
   $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
   $<INSTALL_INTERFACE:${CMAKE_INSTALL_LIBDIR}>
 )
 
-# Set RPATH so sparkd knows where to find our shared libs (e.g., libspark).
+# Set RPATH so sparkd knows where to find our shared libs (e.g., libspark, lib_py_market_data).
 # The dirs are relative to sparkd's path, represented by "$ORIGIN".
 set_target_properties(
   ${PROJECT_NAME} PROPERTIES
-  BUILD_RPATH "$ORIGIN/../lib_spark;$ORIGIN/../lib_python_md"
+  BUILD_RPATH "$ORIGIN/../lib_spark;$ORIGIN/../lib_py_market_data"
   INSTALL_RPATH "$ORIGIN/../${CMAKE_INSTALL_LIBDIR}"
 )
 
-# Python interop (for python md providers)
-add_dependencies(${PROJECT_NAME}
-  python_md
-  python_md_module
-  python_md_providers
-  copy_python_md_providers
-)
 include(${CMAKE_SOURCE_DIR}/cmake/SymLink.cmake)
 
-sym_link_local_lib(python_md)
+sym_link_local_lib(py_market_data)
 sym_link_file(
-  TARGET_NAME link_python_md_module
-  SOURCE ${CMAKE_BINARY_DIR}/src/lib_python_md/python_md_module.cpython-310-x86_64-linux-gnu.so
-  DEST ${CMAKE_CURRENT_BINARY_DIR}/python_md_module.cpython-310-x86_64-linux-gnu.so
+  TARGET_NAME link_py_market_data_module
+  SOURCE ${CMAKE_BINARY_DIR}/src/lib_py_market_data/py_market_data_module.cpython-310-x86_64-linux-gnu.so
+  DEST ${CMAKE_CURRENT_BINARY_DIR}/py_market_data_module.cpython-310-x86_64-linux-gnu.so
   INSTALL_DEST ${CMAKE_INSTALL_BINDIR}/
 )
-get_target_property(providers python_md_providers INTERFACE_PYTHON_SCRIPTS_DIR)
-sym_link_dir(
-  TARGET_NAME link_python_md_providers
-  SOURCE ${providers}
-  DEST ${CMAKE_CURRENT_BINARY_DIR}/python_md_providers
-)
+# add_custom_command(
+#   OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/py_market_data_module.cpython-310-x86_64-linux-gnu.so
+#   COMMAND ${CMAKE_COMMAND} -E create_symlink
+#     ${CMAKE_BINARY_DIR}/src/lib_py_market_data/py_market_data_module.cpython-310-x86_64-linux-gnu.so
+#     ${CMAKE_CURRENT_BINARY_DIR}/py_market_data_module.cpython-310-x86_64-linux-gnu.so
+#   DEPENDS 
+#     ${CMAKE_BINARY_DIR}/src/lib_py_market_data/py_market_data_module.cpython-310-x86_64-linux-gnu.so
+#   COMMENT "Creating symlink to py_market_data_module.so"
+# )
+# add_custom_target(create_symlinks ALL
+#   DEPENDS ${CMAKE_BINARY_DIR}/src/lib_py_market_data/py_market_data_module.cpython-310-x86_64-linux-gnu.so)
+# install(
+#   FILES
+#   ${CMAKE_CURRENT_BINARY_DIR}/py_market_data_module.cpython-310-x86_64-linux-gnu.so
+#   DESTINATION ${CMAKE_INSTALL_BINDIR}/
+# )
 
 target_link_libraries(
   sparkd
-  system
-  python_md
-  python_md_module
+  py_market_data
+  py_market_data_module
   spark
   ${emumba-deribit-connector-library_LIBRARIES}
   ssl

src/app_sparkd/sparkd/cmds/TestPythonMdProviderCmd.cpp

@@ -1,73 +0,0 @@
-#include "TestPythonMdProviderCmd.h"
-#include "python_md/md_sink_impl.h"
-#include "python_md/py_md_executor.h"
-#include "spark/logging/logging.h"
-
-#include <pybind11/embed.h>
-#include <pybind11/pybind11.h>
-#include <string>
-#include <thread>
-
-using bts::md::py_interop::MdSinkImpl;
-using std::string;
-
-namespace bts::cmds {
-
-struct TestPythonMdProviderCmd::Args {
-   std::string py_md_provider;
-   std::vector<std::string> module_paths;
-};
-
-static void execute(
-    std::string py_md_provider, std::vector<std::string> module_paths) {
-   using namespace std::chrono_literals;
-   MdSinkImpl sink;
-   sink.set_securities_callback([](const auto &securities) {
-      LOG(md, Info, "Received securities: {}", securities.to_string());
-   });
-   sink.set_books_callback([](const auto &books) {
-      LOG(md, Info, "Received books: {}", books.to_string());
-   });
-   sink.set_trades_callback([](const auto &trades) {
-      LOG(md, Info, "Received trades: {}", trades.to_string());
-   });
-   try {
-      md::py_interop::PyMdExecutor py_md_exec{
-          std::move(py_md_provider), std::move(module_paths), &sink};
-      LOG(md, Info, "Starting py md executor.");
-      py_md_exec.start();
-      LOG(md, Info, "Stopping py md executor.");
-      py_md_exec.stop();
-      int i = 0;
-      while (true) {
-         std::cout << "Sleep loop count " << ++i << std::endl;
-         std::this_thread::sleep_for(3s);
-      }
-   } catch (const std::exception &ex) {
-      LOG(md, Error, "Py md executor threw: {}", string{ex.what()});
-   }
-}
-
-TestPythonMdProviderCmd::TestPythonMdProviderCmd()
-    : CLI::App("Test python md provider plugin", "test-py-md-provider"),
-      args_(std::make_unique<Args>()) {
-   add_option(
-       "--provider", args_->py_md_provider,
-       "Python md provider plugin (py file)");
-   add_option(
-       "--module-path", args_->module_paths, "Module path for python plugin");
-   callback([this]() {
-      if (args_->py_md_provider.empty()) {
-         args_->py_md_provider = "md_provider_impl.MdProviderImpl";
-      }
-      if (args_->module_paths.empty()) {
-         args_->module_paths.push_back(".");
-         args_->module_paths.push_back("./python_md_providers");
-      }
-      execute(std::move(args_->py_md_provider), std::move(args_->module_paths));
-   });
-}
-
-TestPythonMdProviderCmd::~TestPythonMdProviderCmd() = default;
-
-} // namespace bts::cmds

src/app_sparkd/sparkd/cmds/TestPythonMdProviderCmd.h

@@ -1,22 +0,0 @@
-#pragma once
-
-#include <CLI11/CLI11.hpp>
-#include <memory>
-
-namespace bts::cmds {
-
-/**
- * This command is designed to run a quick sanity check on Python md provider C++ interop.
- * It instantiates a python md provider (in C++) and calls <provider>.start() with a dummy
- * sink that prints updates to console, but does not write md to shared memory.
- */
-struct TestPythonMdProviderCmd : public CLI::App {
-   TestPythonMdProviderCmd();
-   ~TestPythonMdProviderCmd();
-
- private:
-   struct Args;
-   std::unique_ptr<Args> args_;
-};
-
-} // namespace bts::cmds

src/app_sparkd/sparkd/cmds/crypto/DeribitDemoConfiguration.h

@@ -1,6 +1,6 @@
 #pragma once
 
-#include "system/FMT_IMPL.h"
+#include "spark/core/FMT_IMPL.h"
 
 #include <emumba-deribit-connector/deribit_utils.h>
 #include <iostream>

src/app_sparkd/sparkd/cmds/md/cmds/Run.cpp

@@ -1,7 +1,7 @@
 #include "Run.h"
 #include "Cleanup.h"
-#include "domain/security_criteria.h"
 #include "spark/config/protos/market_data.pb.h"
+#include "spark/domain/SecurityCriteria.h"
 #include "spark/logging/logging.h"
 #include "spark/md/SymbolLoader.h"
 #include "spark/md/UserSymbol.h"
@@ -60,7 +60,7 @@ static void run_onixs(
    std::vector<std::shared_ptr<bts::md::IMDPlatform>> platforms;
    // get all glbx symbols
    auto cme_symbols =
-       SymbolLoader::load(SecurityCriteria::create({domain::protobufs::VENUE_GLBX}));
+       SymbolLoader::load(SecurityCriteria::create({api::domain::VENUE_GLBX}));
    if (!cme_symbols.empty()) {
       auto *main = new bts::md::MDP3Main();
       main->start(
@@ -69,7 +69,7 @@ static void run_onixs(
    }
    // ice
    const auto ice_symbols =
-       SymbolLoader::load(SecurityCriteria::create({domain::protobufs::VENUE_ICE}));
+       SymbolLoader::load(SecurityCriteria::create({api::domain::VENUE_ICE}));
    if (!ice_symbols.empty()) {
       auto *imain = new bts::md::ice::ICEMain();
       imain->start({ice_symbols.begin(), ice_symbols.end()});

src/app_sparkd/sparkd/cmds/md/cmds/Tail.cpp

@@ -1,9 +1,12 @@
 #include "Tail.h"
-#include "domain/protobufs/security_type.pb.h"
-#include "domain/security_criteria.h"
+#include "spark/api/domain/SecurityType.pb.h"
+#include "spark/api/domain/Venue.pb.h"
+#include "spark/config/protos/market_data.pb.h"
+#include "spark/domain/SecurityCriteria.h"
 #include "spark/md/Diag/MDPeriodicDumper.h"
+#include "spark/md/Diag/ShmDumper.h"
 #include "spark/md/SymbolLoader.h"
-
+#include "spark/md/UserSymbol.h"
 #include <map>
 
 namespace bts::cmds::md::cmds {
@@ -41,7 +44,7 @@ Tail::Tail()
 
    callback([&] {
       const auto products = SymbolLoader::load(SecurityCriteria::create(
-          {}, {domain::protobufs::SECURITY_TYPE_FUTURE}));
+          {}, {bts::api::domain::SECURITY_TYPE_FUTURE}));
       bts::md::run_periodic_dumper(
           products, _args->mod, _args->format, _args->all, _args->qualifier);
    });

src/app_sparkd/sparkd/exchange_sim/ExchangeSimApplication.cpp

@@ -34,8 +34,6 @@ namespace ex_sim {
 GCC_DIAG_OFF("-Wdeprecated")
 GCC_DIAG_OFF("-Wignored-qualifiers")
 
-#define AGGRESSOR_SIDE(x) bts::domain::protobufs::AGGRESSOR_SIDE_##x
-
 api::Side convert(const FIX::Side &side) {
    switch (side) {
    case FIX::Side_BUY:
@@ -541,9 +539,9 @@ void ExchangeSimApplication::update_md(std::string const &sec_desc) {
              md::PriceSize_from_double<ex_sim::precision>(
                  t.price.value() * price_factor, t.quantity.value()),
              datetime::ll_now(),
-             t.bid_aggressor ? AGGRESSOR_SIDE(BUY)
-                             : (t.ask_aggressor ? AGGRESSOR_SIDE(SELL)
-                                                : AGGRESSOR_SIDE(NONE)));
+             t.bid_aggressor ? co2::AGGRESSOR_SIDE_BUY
+                             : (t.ask_aggressor ? co2::AGGRESSOR_SIDE_SELL
+                                                : co2::AGGRESSOR_SIDE_NONE));
       }
 
       // publish mbo

src/app_sparkd/sparkd/exchange_sim/book/Book.h

@@ -1,9 +1,9 @@
 #pragma once
 
+#include "spark/core/FMT_IMPL.h"
 #include "spark/domain_model/Security.pb.h"
 #include "spark/orderman/orderman_utils/orderman_common.h"
 #include "sparkd/exchange_sim/book/Order.h"
-#include "system/FMT_IMPL.h"
 
 #include <memory>
 #include <sstream>

src/app_sparkd/sparkd/main.cpp

@@ -1,28 +1,25 @@
 #include "spark/testing/catch_test_runner.h"
 #include "spark/utils/init_spark_lib.h"
+// #include "py-market-data/py_exec.h"
 #include "sparkd/cmds/AdminReport.h"
 #include "sparkd/cmds/ConfigCmd.h"
 #include "sparkd/cmds/KeyCmd.h"
 #include "sparkd/cmds/Md.h"
 #include "sparkd/cmds/QuickTest.h"
 #include "sparkd/cmds/Run.h"
-#include "sparkd/cmds/TestPythonMdProviderCmd.h"
 #include "sparkd/cmds/Web.h"
 #include "sparkd/cmds/crypto/CryptoCmd.h"
-
+#include "sparkd/md/MarketDataGateway.h"
 #include <CLI11/CLI11.hpp>
-#include <csignal>
 
 using namespace bts;
 
-void signal_handler(int signum) {
-   std::cerr << "Interrupt signal (" << signum << "). Exiting" << std::endl;
-   exit(signum);
-}
-
 int main(int argc, char **argv) {
-   std::signal(SIGINT, signal_handler);
-   std::signal(SIGTERM, signal_handler);
+   if (argc < 40) {
+      // md::py_interop::PyExec::exec("/home/bts/spark/tools/py");
+      md::MarketDataGateway::test_py();
+      return 0;
+   }
    utils::SparkLibInit init;
 
    // The catch testing framework has its own command line parser so
@@ -40,7 +37,6 @@ int main(int argc, char **argv) {
       app.add_subcommand(CLI::App_p(new cmds::Run()));
       app.add_subcommand(CLI::App_p(new cmds::KeyCmd()));
       app.add_subcommand(CLI::App_p(new cmds::Web()));
-      app.add_subcommand(CLI::App_p(new cmds::TestPythonMdProviderCmd()));
       app.add_subcommand(
           "test", "Run unit test(s)"); // so this shows up in help
       CLI11_PARSE(app, argc, argv);

src/app_sparkd/sparkd/md/ChannelVolume.cpp

@@ -9,12 +9,10 @@ namespace auxiliary {
 using namespace bts::co2;
 using namespace bts::api::md::auxiliary;
 
-#define AGGRESSOR_SIDE(x) bts::domain::protobufs::AGGRESSOR_SIDE_##x
-
 ChannelVolume::ChannelVolume(SecurityChannel *channel) : _channel(channel) {}
 
-const VolumeAtPriceLevel &ChannelVolume::add_trade(
-    const TradeMessage &m, int &record_id) {
+const VolumeAtPriceLevel &
+ChannelVolume::add_trade(const TradeMessage &m, int &record_id) {
    auto price_map_it = _sec_map.find(m.key());
    if (price_map_it == _sec_map.end()) {
       _sec_map[m.key()] = std::unordered_map<uint64_t, VolumeAtPriceLevel>();
@@ -33,13 +31,12 @@ const VolumeAtPriceLevel &ChannelVolume::add_trade(
    }
    VolumeAtPriceLevel &vol = price_it->second;
    vol.set_volume(vol.volume() + m.size());
-   if (m.aggressor_side() == AGGRESSOR_SIDE(BUY)) {
+   if (m.aggressor_side() == AggressorSide::AGGRESSOR_SIDE_BUY)
       vol.set_buy_volume(vol.buy_volume() + m.size());
-   } else if (m.aggressor_side() == AGGRESSOR_SIDE(SELL)) {
+   else if (m.aggressor_side() == AggressorSide::AGGRESSOR_SIDE_SELL)
       vol.set_sell_volume(vol.sell_volume() + m.size());
-   } else if (m.aggressor_side() == AGGRESSOR_SIDE(NONE)) {
+   else if (m.aggressor_side() == AggressorSide::AGGRESSOR_SIDE_NONE)
       vol.set_passive_volume(vol.passive_volume() + m.size());
-   }
    return vol;
 }
 
@@ -55,4 +52,4 @@ bool ChannelVolume::match_update(const SecurityChannel &channel) {
 
 } // namespace auxiliary
 } // namespace md
-} // namespace bts
+} // namespace bts

src/app_sparkd/sparkd/md/IMarketDataProvider.cpp

@@ -1,7 +0,0 @@
-#include "IMarketDataProvider.h"
-
-namespace bts::md {
-
-IMarketDataProvider::~IMarketDataProvider() = default;
-
-} // namespace bts::md

src/app_sparkd/sparkd/md/IMarketDataProvider.h

@@ -1,25 +0,0 @@
-#pragma once
-
-#include "domain/protobufs/venue.pb.h"
-#include "sparkd/md/IMarketDataSource.h"
-
-#include <vector>
-
-namespace bts::md {
-
-/**
- * @brief This is a single function interface, whose sole purpose is to provide a list of market
- * data sources (per venue). A good example implementation is the `CCApiMarketDataProvider`, which
- * reads `MarketDataConfig` and returns an `IMarketDataSource` for each venue configuration (e.g.,
- * `bts::api::md::BinanceMarketDataConfig`, `bts::api::md::CoinbaseMarketDataConfig`, etc.).
- */
-class IMarketDataProvider {
- public:
-   virtual ~IMarketDataProvider();
-
-   virtual std::vector<
-       std::pair<domain::protobufs::Venue, std::shared_ptr<IMarketDataSource>>>
-   get_sources() = 0;
-};
-
-} // namespace bts::md

src/app_sparkd/sparkd/md/IMarketDataSource.cpp

@@ -1,9 +0,0 @@
-#include "IMarketDataSource.h"
-
-namespace bts::md {
-
-IMarketDataSource::~IMarketDataSource() = default;
-
-void IMarketDataSource::stop() {}
-
-} // namespace bts::md

src/app_sparkd/sparkd/md/IMarketDataSource.h

@@ -1,18 +0,0 @@
-#pragma once
-
-#include "spark/md/IMarketDataListener.h"
-
-namespace bts::md {
-
-/**
- * @brief This interface represents a source of market data. Call `IMarketDataSource::start` to
- * connect an `IMarketDataListener` to the source and begin receiving market data.
- */
-class IMarketDataSource {
- public:
-   virtual ~IMarketDataSource();
-   virtual void start(std::shared_ptr<IMarketDataListener> listener) = 0;
-   virtual void stop();
-};
-
-} // namespace bts::md

src/app_sparkd/sparkd/md/MarketDataError.h

@@ -1,6 +1,6 @@
 #pragma once
 
-#include "system/FMT_IMPL.h"
+#include "spark/core/FMT_IMPL.h"
 
 #include <ostream>
 #include <sstream>

src/app_sparkd/sparkd/md/MarketDataGateway.cpp

@@ -1,40 +1,66 @@
 #include "MarketDataGateway.h"
-#include "domain/security_id.h"
+#include "py_market_data/md_provider.h"
+#include "py_market_data/md_sink_impl.h"
+#include "spark/api/md/EmumbaDeribitMarketDataConfig.pb.h"
 #include "spark/api/md/MarketDataConfig.pb.h"
 #include "spark/api/md/MarketDataConfigUtils.h"
-#include "spark/config/config.h"
-#include "spark/domain/DomainSecurityUtils.h"
+#include "spark/api/md/MarketDataReplayConfig.pb.h"
+#include "spark/collections/Contains.h"
+#include "spark/config/protos/system_config.pb.h"
+#include "spark/core/FMT.h"
+#include "spark/domain/SecurityID.h"
+#include "spark/domain_model/Security.pb.h"
+#include "spark/io/ProtoPrinter.h"
 #include "spark/logging/logging.h"
 #include "spark/md/GeneralMarketDataListener.h"
 #include "spark/md/GeneralSecurityIndex.h"
-#include "spark/md/IMarketDataListener.h"
 #include "spark/md/MDProducer.h"
 #include "spark/os/os.h"
 #include "spark/persistence/IDGenerator.h"
+#include "spark/utils/bts_assert.h"
 #include "spark/utils/bts_fatal.h"
-#include "sparkd/md/IMarketDataProvider.h"
-#include "sparkd/md/OnixsMarketDataProvider.h"
-#include "sparkd/md/PyMarketDataProvider.h"
-#include "sparkd/md/PyMarketDataSubscription.h"
-#include "sparkd/md/ReplayMarketDataProvider.h"
-#include "sparkd/md/ccapi/CCApiMarketDataProvider.h"
-#include "sparkd/md/emumba/EmumbaMarketDataProvider.h"
-#include "system/FMT.h"
+#include "spark/utils/thread_manager.h"
+#include "sparkd/md/MarketDataListenerID.h"
+#include "sparkd/md/MarketDataRegistration.h"
+#include "sparkd/md/MarketDataRegistrationError.h"
+#include "sparkd/md/MarketDataRegistrationFactory.h"
+#include "sparkd/md/MarketDataSubscription.h"
+#include "sparkd/md/binance/BinanceSubscription.h"
+#include "sparkd/md/binance_us/BinanceUsSubscription.h"
+#include "sparkd/md/binance_usds_futures/BinanceUsdsFuturesSubscription.h"
+#include "sparkd/md/coinbase/CoinbaseSubscription.h"
+#include "sparkd/md/deribit/DeribitSubscription.h"
+#include "sparkd/md/emumba/EmumbaDeribitData.h"
+#include "sparkd/md/ftx/FTXSubscription.h"
+#include "sparkd/md/okx/OkxSubscription.h"
+#include "sparkd/md/platforms/mdp3_onixs/MDP3Main.h"
+#include "sparkd/md/recording/MDReplayer.h"
 
 #include <bits/stdint-uintn.h>
 #include <boost/filesystem.hpp>
 #include <boost/filesystem/path.hpp>
+#include <ccapi_cpp/ccapi_event_dispatcher.h>
 #include <memory>
+#include <optional>
+#include <pybind11/embed.h>
+#include <pybind11/pybind11.h>
+#include <unordered_map>
 #include <vector>
 
+namespace py = pybind11;
+
 namespace bts::md {
 
 using bts::api::md::MarketDataConfig;
+using bts::config::SystemConfig;
+//using bts::domain::Security;
 using bts::domain::SecurityID;
+using bts::io::ProtoPrinter;
 using bts::persistence::IDGenerator;
-using std::shared_ptr;
+using os::sparkd_wd;
 using std::string;
 using std::unique_ptr;
+using std::unordered_map;
 using std::vector;
 
 using Path = boost::filesystem::path;
@@ -48,33 +74,51 @@ static bool validate(const MarketDataConfig &cfg, vector<string> &errs) {
    return errs.size() == orig_n_errs;
 }
 
+struct MarketDataListenerData {
+   MarketDataListenerData() = default;
+   MarketDataListenerData(const MarketDataListenerData &) noexcept = delete;
+   MarketDataListenerData(MarketDataListenerData &&) noexcept = default;
+   MarketDataListenerData &operator=(const MarketDataListenerData &) noexcept =
+       delete;
+   MarketDataListenerData &operator=(MarketDataListenerData &&) noexcept =
+       default;
+   ~MarketDataListenerData() = default;
+
+   MarketDataListenerID id;
+   MarketDataSubscription subscription;
+};
+
+struct MDReplayData {
+   bts::api::md::MarketDataReplayConfig replay_conf;
+   bts::api::Venue venue;
+   bts::md::GeneralSecurityIndex sec_index;
+   bts::md::MDProducer md_producer;
+   bts::md::recording::MDReplayer md_replayer;
+
+   MDReplayData(const bts::api::md::MarketDataReplayConfig &replay_conf)
+       : replay_conf(replay_conf), venue(replay_conf.venue()), sec_index(),
+         md_replayer(replay_conf.venue(), sec_index, md_producer) {}
+};
+
+struct CCMDSourceData {
+   bts::md::GeneralSecurityIndex sec_index;
+   bts::md::MDProducer md_producer;
+   bts::md::GeneralMarketDataListener listener;
+
+   CCMDSourceData(bts::api::Venue venue)
+       : sec_index(), listener(sec_index, md_producer, venue) {}
+};
+
 struct MarketDataGateway::PImpl {
-   struct MdSink {
-      GeneralSecurityIndex sec_index;
-      MDProducer md_producer;
-      shared_ptr<IMarketDataListener> listener;
-
-      MdSink(api::Venue venue, IDGenerator *sec_id_store)
-          : sec_index(), listener(std::make_shared<GeneralMarketDataListener>(
-                             sec_index, md_producer, venue, sec_id_store)) {}
-   };
-
-   MarketDataConfig config = config::get<MarketDataConfig>();
+   api::md::MarketDataConfig md_config =
+       config::get<api::md::MarketDataConfig>();
+   ccapi::EventDispatcher dispatcher;
+   MarketDataRegistrationFactory registration_factory;
    IDGenerator &sec_id_store = IDGenerator::instance();
-   vector<unique_ptr<IMarketDataProvider>> providers;
-   vector<unique_ptr<MdSink>> sinks;
-   vector<unique_ptr<IMarketDataSource>> sources;
-
-   PImpl() {
-      providers.emplace_back(std::make_unique<CCApiMarketDataProvider>(config));
-      providers.emplace_back(
-          std::make_unique<EmumbaMarketDataProvider>(config));
-      providers.emplace_back(std::make_unique<OnixsMarketDataProvider>(config));
-      providers.emplace_back(
-          std::make_unique<PyMarketDataProvider>(config, &sec_id_store));
-      providers.emplace_back(
-          std::make_unique<ReplayMarketDataProvider>(config, &sec_id_store));
-   }
+   unordered_map<MarketDataListenerID, MarketDataListenerData> listeners;
+   std::vector<std::unique_ptr<MDReplayData>> replays = {};
+   // we create these and pass them to the cc md sources
+   std::vector<std::shared_ptr<CCMDSourceData>> cc_md_listeners = {};
 };
 
 MarketDataGateway::MarketDataGateway()
@@ -85,20 +129,23 @@ MarketDataGateway::~MarketDataGateway() = default;
 // starts the market data sources and then keeps the function going
 void MarketDataGateway::run() {
    vector<string> cfg_errs;
-   if (!validate(me_->config, cfg_errs)) {
+   if (!validate(me_->md_config, cfg_errs)) {
       BTS_FATAL_MSGF("Invalid MarketDataConfig:", FMT_JOIN(cfg_errs, "\n"));
       return;
    }
    LOG(md, Info, "MarketDataGateway::run starting t_id:{} t_name:{}",
        os::get_thread_id(), os::get_thread_name());
-   for (auto &provider : me_->providers) {
-      for (auto &[venue, source] : provider->get_sources()) {
-         const auto v = domain::DomainSecurityUtils::convert(venue);
-         auto &sink = me_->sinks.emplace_back(
-             std::make_unique<PImpl::MdSink>(v, &me_->sec_id_store));
-         source->start(sink->listener);
-      }
+   // _start_cc(me_->md_config);
+   if (!me_->md_config.emumba_deribit().empty()) {
+      bts_assert(me_->md_config.emumba_deribit().size() == 1);
+      _start_emumba_deribit(me_->md_config.emumba_deribit().at(0));
    }
+   if (!me_->md_config.glbx().empty()) {
+      bts_assert(me_->md_config.glbx().size() == 1);
+      _start_onix_glbx(me_->md_config.glbx().at(0));
+   }
+   _start_replays(me_->md_config);
+   start_py_();
    while (true) {
       std::this_thread::sleep_for(std::chrono::seconds(1));
    }
@@ -106,4 +153,243 @@ void MarketDataGateway::run() {
        os::get_thread_id(), os::get_thread_name());
 }
 
+void MarketDataGateway::test_py() {
+   start_py_();
+}
+
+void MarketDataGateway::start_py_() {
+   const Path wd{sparkd_wd()};
+   vector<string> module_paths{
+       (wd / "py_market_data_providers").string(),
+      //  (wd.parent_path() / "lib").string(),
+   };
+   start_py_(module_paths);
+}
+
+void MarketDataGateway::start_py_(
+    const std::vector<std::string> &module_paths) {
+   py::scoped_interpreter guard{}; // Start the Python interpreter
+
+   try {
+      // import sys
+      py::module sys = py::module::import("sys");
+      std::cout << "Appending module path" << std::endl;
+      for (const auto &module_path : module_paths) {
+         sys.attr("path").attr("append")(module_path.c_str());
+      }
+      std::cout << "Importing py_market_data module" << std::endl;
+      py::module spark_md = py::module::import("py_market_data_module");
+      std::cout << "Importing md_provider_impl module" << std::endl;
+      py::module my_implementation = py::module::import("md_provider_impl");
+      std::cout << "Instantiating MdProviderImpl" << std::endl;
+      py::object impl = my_implementation.attr("MdProviderImpl")();
+      std::cout << "Casting MdProviderImpl to MdProvider" << std::endl;
+      py_interop::MdProvider *provider = impl.cast<py_interop::MdProvider *>();
+      py_interop::MdSinkImpl sink;
+      std::cout << "Starting MdProvider(Impl)" << std::endl;
+      provider->start(&sink);
+      std::cout << "MdProvider(Impl) DONE" << std::endl;
+   } catch (const py::error_already_set &e) {
+      std::cerr << "Error: " << e.what() << std::endl;
+   }
+}
+
+// CCAPI //////////////////////////////////////////////////////////////////////
+///////////////////////////////////////////////////////////////////////////////
+MarketDataRegistration MarketDataGateway::_start_cc(
+    const MarketDataConfig &cfg) noexcept {
+   LOG(md, Info, "MarketDataGateway::_start_cc starting t_id:{} t_name:{}",
+       os::get_thread_id(), os::get_thread_name());
+   auto reg = me_->registration_factory.create();
+   bts_assert(reg.listener_id.has_value());
+   bts_assert(!CONTAINS_KEY(me_->listeners, *reg.listener_id));
+   LOG(md, Info, "MarketDataGateway subscribing to data:\n{}",
+       ProtoPrinter::to_string(cfg));
+
+   vector<string> cfg_errs;
+   if (!validate(cfg, cfg_errs)) {
+      return MarketDataRegistration{
+          {MarketDataRegistrationError::Type::CONFIG_ERROR,
+           fmt::to_string(fmt::join(cfg_errs, "\n"))}};
+   }
+
+   MarketDataListenerData &listener_data = me_->listeners[*reg.listener_id];
+   listener_data.id = *reg.listener_id;
+
+   // FTX
+   for (const auto &ftx : cfg.ftx()) {
+      auto p = me_->cc_md_listeners.emplace_back(
+          std::make_shared<CCMDSourceData>(bts::api::VENUE_FTX));
+      listener_data.subscription.add(ftx::FTXSubscription{
+          &me_->dispatcher, ftx, &me_->sec_id_store, &p->listener});
+   }
+
+   // Binance
+   for (const auto &binance : cfg.binance()) {
+      auto p = me_->cc_md_listeners.emplace_back(
+          std::make_shared<CCMDSourceData>(bts::api::VENUE_BINANCE));
+      listener_data.subscription.add(binance::BinanceSubscription{
+          &me_->dispatcher, binance, &me_->sec_id_store, &p->listener});
+   }
+
+   // Binance US
+   for (const auto &binance_us : cfg.binance_us()) {
+      auto p = me_->cc_md_listeners.emplace_back(
+          std::make_shared<CCMDSourceData>(bts::api::VENUE_BINANCE_US));
+      listener_data.subscription.add(binance_us::BinanceUsSubscription{
+          &me_->dispatcher, binance_us, &me_->sec_id_store, &p->listener});
+   }
+   // Coinbase
+   for (const auto &coinbase : cfg.coinbase()) {
+      auto p = me_->cc_md_listeners.emplace_back(
+          std::make_shared<CCMDSourceData>(bts::api::VENUE_COINBASE));
+      listener_data.subscription.add(coinbase::CoinbaseSubscription{
+          &me_->dispatcher, coinbase, &me_->sec_id_store, &p->listener});
+   }
+   // Binance USDS Futures
+   for (const auto &binance_ufs : cfg.binance_usds_futures()) {
+      auto p =
+          me_->cc_md_listeners.emplace_back(std::make_shared<CCMDSourceData>(
+              bts::api::VENUE_BINANCE_USDS_FUTURES));
+      listener_data.subscription.add(
+          binance_usds_futures::BinanceUsdsFuturesSubscription{
+              &me_->dispatcher, binance_ufs, &me_->sec_id_store, &p->listener});
+   }
+
+   // Deribit
+   for (const auto &deribit : cfg.deribit()) {
+      auto p = me_->cc_md_listeners.emplace_back(
+          std::make_shared<CCMDSourceData>(bts::api::VENUE_DERIBIT));
+      listener_data.subscription.add(deribit::DeribitSubscription{
+          &me_->dispatcher, deribit, &me_->sec_id_store, &p->listener});
+   }
+
+   // Okx
+   for (const auto &okx : cfg.okx()) {
+      auto p = me_->cc_md_listeners.emplace_back(
+          std::make_shared<CCMDSourceData>(bts::api::VENUE_OKX));
+      listener_data.subscription.add(okx::OkxSubscription{
+          &me_->dispatcher, okx, &me_->sec_id_store, &p->listener});
+   }
+
+   LOG(md, Info, "MarketDataGateway::_start_cc ending t_id:{} t_name:{}",
+       os::get_thread_id(), os::get_thread_name());
+   return reg;
+}
+
+void MarketDataGateway::unsubscribe(MarketDataListenerID id) {
+   LOG(md, Info, "Unsubscribing listener {}", id);
+   me_->listeners.erase(id);
+}
+/// glbx //////////////////////////////////////////////////////////////////////
+//////////////////////////////////////////////////////////////////////////////
+static void run_onixs() {
+   LOG(md, Info, "MarketDataGateway::run_onixs t_id:{} t_name",
+       os::get_thread_id(), os::get_thread_name());
+   bts::md::MBOWriter *mbo = nullptr;
+   bts::md::TradeSummarySignalDispatcher *tssd = nullptr;
+   std::vector<std::shared_ptr<bts::md::IMDPlatform>> platforms;
+   // get all glbx symbols
+   auto cme_symbols = SymbolLoader::load(
+       bts::domain::SecurityCriteria::create({api::domain::VENUE_GLBX}));
+   if (!cme_symbols.empty()) {
+      auto *main = new bts::md::MDP3Main();
+      main->start({cme_symbols.begin(), cme_symbols.end()}, mbo, tssd);
+      platforms.emplace_back(main);
+   }
+   // deribit
+   // auto *d_platform = new bts::md::EmMDListener()
+   // Timers
+   auto &disp = utils::Dispatcher::current();
+   utils::DispatcherTimer heartbeat_timer(disp, 250, [&]() {
+      for (auto &p : platforms)
+         p->send_heartbeats();
+   });
+   heartbeat_timer.start();
+
+   disp.run();
+}
+
+void MarketDataGateway::_start_onix_glbx(
+    bts::api::md::GlbxMarketDataConfig cfg) {
+   LOG(md, Info,
+       "MarketDataGateway::_start_onix_glbx starting t_id:{} t_name:{}. "
+       "cfg:\n{}",
+       os::get_thread_id(), os::get_thread_name(), cfg.DebugString());
+   bts::md::run_onixs();
+}
+
+// deribit ////////////////////////////////////////////////////////////////////
+///////////////////////////////////////////////////////////////////////////////
+
+void MarketDataGateway::_start_emumba_deribit(
+    bts::api::md::EmumbaDeribitMarketDataConfig cfg) {
+   LOG(md, Info,
+       "MarketDataGateway::_start_emumba_deribit starting t_id:{} t_name:{}. "
+       "cfg:\n{}",
+       os::get_thread_id(), os::get_thread_name(), cfg.DebugString());
+   auto start_deribit_thr = [=]() {
+      const auto config_path = Path{config::get<SystemConfig>().config_path()} /
+                               cfg.json_conf_file_name();
+      const auto conf =
+          bts::cmds::DeribitDemoConfiguration::read_file(config_path.string());
+      bts_assert_msg(conf.has_value(), "Failed to parse deribit config");
+      // log
+      for (const auto &s : conf.value().symbol_config_list) {
+         LOG(md, Info, "sub.symbol: {}", s.symbol_name);
+         LOG(md, Info, "sub.futs: {} sub.options: {}", s.interested_in_future,
+             s.interested_in_option);
+         for (const auto &i : s.interested_instruments) {
+            LOG(md, Info, "sub.instrument: {}", i);
+         }
+      }
+      EmumbaDeribitData ddata_obj;
+      EmumbaDeribitData *ddata = &ddata_obj;
+      // end log
+      bts_assert(conf.has_value());
+      LOG(md, Info, "Deribit conf: {}", conf.value());
+      ddata->real_handler =
+          std::make_shared<emumba::deribit_connector::deribit_handler>(
+              conf.value().symbol_config_list, conf.value().trading_config,
+              conf.value().credential_map, conf.value().logging_configuration,
+              conf.value().deribit_url, &ddata->listener);
+      // since the trading interface is housed inside the EmMDListener, we need to
+      // create and send emumba's trading_session_interface
+      for (const auto &p : conf.value().credential_map) {
+         const auto &sess = p.second;
+         auto ts_ptr = std::make_shared<
+             emumba::deribit_connector::trading_session_interface>(
+             ddata->real_handler->create_new_trading_session(sess));
+         ddata->trading_session_vec.push_back({sess, ts_ptr});
+      }
+      bts_assert(!ddata->trading_session_vec.empty());
+      ddata->listener.set_handler(
+          ddata->real_handler.get(), ddata->trading_session_vec);
+      ddata->real_handler->run();
+   };
+   threading::make_thread("Emumba", [&]() { start_deribit_thr(); }).detach();
+   LOG(md, Info, "MarketDataGateway::_start_deribit ending t_id:{} t_name:{}",
+       os::get_thread_id(), os::get_thread_name());
+}
+
+// replays ////////////////////////////////////////////////////////////////////
+///////////////////////////////////////////////////////////////////////////////
+void MarketDataGateway::_start_replays(
+    const bts::api::md::MarketDataConfig &cfg) {
+   LOG(md, Info, "MarketDataGateway::_start_replays starting t_id:{} t_name:{}",
+       os::get_thread_id(), os::get_thread_name());
+   for (const auto &r :
+        MarketDataConfigUtils::get_effective_replay_config(cfg)) {
+      me_->replays.emplace_back(std::make_unique<MDReplayData>(r));
+   }
+   for (const auto &replayer : me_->replays) {
+      std::string tid_name = "MDReplayer_" + Venue_Name(replayer->venue);
+      bts::threading::make_thread(tid_name, [&]() {
+         replayer->md_replayer.run();
+      }).detach();
+   }
+   LOG(md, Info, "MarketDataGateway::_start_replays ending t_id:{} t_name:{}",
+       os::get_thread_id(), os::get_thread_name());
+}
+
 } // namespace bts::md

src/app_sparkd/sparkd/md/MarketDataGateway.h

@@ -1,23 +1,43 @@
 #pragma once
 
+#include "spark/api/md/GlbxMarketDataConfig.pb.h"
+#include "sparkd/md/MarketDataListenerID.h"
+
 #include <memory>
 
+namespace bts::api::md {
+class EmumbaDeribitMarketDataConfig;
+class MarketDataConfig;
+} // namespace bts::api::md
+
 namespace bts::md {
 
-/**
- * @brief The `MarketDataGateway` is responsible for starting all configured market data sources.
- * It connects each source to a special `IMarketDataListener` that writes the data to shared memory.
- */
+class IMarketDataListener;
+class MarketDataRegistration;
+
 class MarketDataGateway {
  public:
    MarketDataGateway();
    ~MarketDataGateway();
 
+   /**
+    * @brief Connects a market data listener to a market data source
+    * @return MarketDataRegistration
+    */
    void run();
+   void unsubscribe(MarketDataListenerID id);
+   static void test_py();
 
  private:
    struct PImpl;
    std::unique_ptr<PImpl> me_;
+   MarketDataRegistration _start_cc(const api::md::MarketDataConfig &) noexcept;
+   static void start_py_();
+   static void start_py_(const std::vector<std::string> &module_paths);
+   static void _start_emumba_deribit(
+       bts::api::md::EmumbaDeribitMarketDataConfig cfg);
+   static void _start_onix_glbx(bts::api::md::GlbxMarketDataConfig cfg);
+   void _start_replays(const bts::api::md::MarketDataConfig &cfg);
 };
 
 } // namespace bts::md

src/app_sparkd/sparkd/md/MarketDataListener.h

@@ -1,12 +1,11 @@
 #pragma once
 
-#include "domain/book.h"
-#include "domain/rfq.h"
-#include "domain/security.h"
-#include "domain/security_state.h"
-#include "domain/trade_record.h"
-#include "spark/domain/SecurityMarketDataStatistics.h"
 #include "spark/md/IMarketDataListener.h"
+#include "spark/domain/Book.h"
+#include "spark/domain/Security.h"
+#include "spark/domain/SecurityMarketDataStatistics.h"
+#include "spark/domain/SecurityState.h"
+#include "spark/domain/TradeRecord.h"
 #include "sparkd/md/MarketDataError.h"
 
 namespace bts::md {

src/app_sparkd/sparkd/md/MarketDataListenerID.h

@@ -1,8 +1,8 @@
 #pragma once
 #ifndef _WIN32
 
+#include "spark/core/FMT_IMPL.h"
 #include "spark/core/ValueWrapper.h"
-#include "system/FMT_IMPL.h"
 
 namespace bts::md {
 

src/app_sparkd/sparkd/md/MarketDataSubscription.cpp

@@ -1,60 +1,60 @@
-// #include "MarketDataSubscription.h"
-// #include "sparkd/md/binance/BinanceSubscription.h"
-// #include "sparkd/md/binance_us/BinanceUsSubscription.h"
-// #include "sparkd/md/binance_usds_futures/BinanceUsdsFuturesSubscription.h"
-// #include "sparkd/md/coinbase/CoinbaseSubscription.h"
-// #include "sparkd/md/deribit/DeribitSubscription.h"
-// #include "sparkd/md/ftx/FTXSubscription.h"
-// #include "sparkd/md/okx/OkxSubscription.h"
+#include "MarketDataSubscription.h"
+#include "sparkd/md/binance/BinanceSubscription.h"
+#include "sparkd/md/binance_us/BinanceUsSubscription.h"
+#include "sparkd/md/binance_usds_futures/BinanceUsdsFuturesSubscription.h"
+#include "sparkd/md/coinbase/CoinbaseSubscription.h"
+#include "sparkd/md/deribit/DeribitSubscription.h"
+#include "sparkd/md/ftx/FTXSubscription.h"
+#include "sparkd/md/okx/OkxSubscription.h"
 
-// #include <vector>
+#include <vector>
 
-// namespace bts::md {
+namespace bts::md {
 
-// using std::vector;
+using std::vector;
 
-// struct MarketDataSubscription::PImpl {
-//    vector<ftx::FTXSubscription> ftx_subs;
-//    vector<binance::BinanceSubscription> binance_subs;
-//    vector<binance_us::BinanceUsSubscription> binance_us_subs;
-//    vector<coinbase::CoinbaseSubscription> coinbase_subs;
-//    vector<binance_usds_futures::BinanceUsdsFuturesSubscription>
-//        binance_usds_futures_subs;
-//    vector<deribit::DeribitSubscription> deribit_subs;
-//    vector<okx::OkxSubscription> okx_subs;
-//    // TODO: add more exchanges
-// };
+struct MarketDataSubscription::PImpl {
+   vector<ftx::FTXSubscription> ftx_subs;
+   vector<binance::BinanceSubscription> binance_subs;
+   vector<binance_us::BinanceUsSubscription> binance_us_subs;
+   vector<coinbase::CoinbaseSubscription> coinbase_subs;
+   vector<binance_usds_futures::BinanceUsdsFuturesSubscription>
+       binance_usds_futures_subs;
+   vector<deribit::DeribitSubscription> deribit_subs;
+   vector<okx::OkxSubscription> okx_subs;
+   // TODO: add more exchanges
+};
 
-// MarketDataSubscription::MarketDataSubscription()
-//     : me_(std::make_unique<PImpl>()) {}
+MarketDataSubscription::MarketDataSubscription()
+    : me_(std::make_unique<PImpl>()) {}
 
-// MarketDataSubscription::~MarketDataSubscription() = default;
+MarketDataSubscription::~MarketDataSubscription() = default;
 
-// void MarketDataSubscription::add(binance::BinanceSubscription sub) {
-//    me_->binance_subs.push_back(std::move(sub));
-// }
+void MarketDataSubscription::add(binance::BinanceSubscription sub) {
+   me_->binance_subs.push_back(std::move(sub));
+}
 
-// void MarketDataSubscription::add(binance_us::BinanceUsSubscription sub) {
-//    me_->binance_us_subs.push_back(std::move(sub));
-// }
-// void MarketDataSubscription::add(coinbase::CoinbaseSubscription sub) {
-//    me_->coinbase_subs.push_back(std::move(sub));
-// }
-// void MarketDataSubscription::add(
-//     binance_usds_futures::BinanceUsdsFuturesSubscription sub) {
-//    me_->binance_usds_futures_subs.push_back(std::move(sub));
-// }
+void MarketDataSubscription::add(binance_us::BinanceUsSubscription sub) {
+   me_->binance_us_subs.push_back(std::move(sub));
+}
+void MarketDataSubscription::add(coinbase::CoinbaseSubscription sub) {
+   me_->coinbase_subs.push_back(std::move(sub));
+}
+void MarketDataSubscription::add(
+    binance_usds_futures::BinanceUsdsFuturesSubscription sub) {
+   me_->binance_usds_futures_subs.push_back(std::move(sub));
+}
 
-// void MarketDataSubscription::add(deribit::DeribitSubscription sub) {
-//    me_->deribit_subs.push_back(std::move(sub));
-// }
+void MarketDataSubscription::add(deribit::DeribitSubscription sub) {
+   me_->deribit_subs.push_back(std::move(sub));
+}
 
-// void MarketDataSubscription::add(ftx::FTXSubscription sub) {
-//    me_->ftx_subs.push_back(std::move(sub));
-// }
+void MarketDataSubscription::add(ftx::FTXSubscription sub) {
+   me_->ftx_subs.push_back(std::move(sub));
+}
 
-// void MarketDataSubscription::add(okx::OkxSubscription sub) {
-//    me_->okx_subs.push_back(std::move(sub));
-// }
+void MarketDataSubscription::add(okx::OkxSubscription sub) {
+   me_->okx_subs.push_back(std::move(sub));
+}
 
-// } // namespace bts::md
+} // namespace bts::md

src/app_sparkd/sparkd/md/MarketDataSubscription.h

@@ -1,59 +1,59 @@
-// #pragma once
+#pragma once
 
-// #include <memory>
+#include <memory>
 
-// namespace bts::md::binance {
-// class BinanceSubscription;
-// }
+namespace bts::md::binance {
+class BinanceSubscription;
+}
 
-// namespace bts::md::binance_us {
-// class BinanceUsSubscription;
-// }
-// namespace bts::md::coinbase {
-// class CoinbaseSubscription;
-// }
-// namespace bts::md::binance_usds_futures {
-// class BinanceUsdsFuturesSubscription;
-// }
+namespace bts::md::binance_us {
+class BinanceUsSubscription;
+}
+namespace bts::md::coinbase {
+class CoinbaseSubscription;
+}
+namespace bts::md::binance_usds_futures {
+class BinanceUsdsFuturesSubscription;
+}
 
-// namespace bts::md::deribit {
-// class DeribitSubscription;
-// }
+namespace bts::md::deribit {
+class DeribitSubscription;
+}
 
-// namespace bts::md::ftx {
-// class FTXSubscription;
-// }
+namespace bts::md::ftx {
+class FTXSubscription;
+}
 
-// namespace bts::md::okx {
-// class OkxSubscription;
-// }
+namespace bts::md::okx {
+class OkxSubscription;
+}
 
-// namespace bts::md {
+namespace bts::md {
 
-// class MarketDataSubscription {
-//  public:
-//    MarketDataSubscription();
+class MarketDataSubscription {
+ public:
+   MarketDataSubscription();
 
-//    MarketDataSubscription(const MarketDataSubscription &) noexcept = delete;
-//    MarketDataSubscription(MarketDataSubscription &&) noexcept = default;
-//    MarketDataSubscription &operator=(const MarketDataSubscription &) noexcept =
-//        delete;
-//    MarketDataSubscription &operator=(MarketDataSubscription &&) noexcept =
-//        default;
+   MarketDataSubscription(const MarketDataSubscription &) noexcept = delete;
+   MarketDataSubscription(MarketDataSubscription &&) noexcept = default;
+   MarketDataSubscription &operator=(const MarketDataSubscription &) noexcept =
+       delete;
+   MarketDataSubscription &operator=(MarketDataSubscription &&) noexcept =
+       default;
 
-//    ~MarketDataSubscription();
+   ~MarketDataSubscription();
 
-//    void add(binance::BinanceSubscription);
-//    void add(binance_us::BinanceUsSubscription);
-//    void add(coinbase::CoinbaseSubscription);
-//    void add(binance_usds_futures::BinanceUsdsFuturesSubscription);
-//    void add(deribit::DeribitSubscription);
-//    void add(ftx::FTXSubscription);
-//    void add(okx::OkxSubscription);
+   void add(binance::BinanceSubscription);
+   void add(binance_us::BinanceUsSubscription);
+   void add(coinbase::CoinbaseSubscription);
+   void add(binance_usds_futures::BinanceUsdsFuturesSubscription);
+   void add(deribit::DeribitSubscription);
+   void add(ftx::FTXSubscription);
+   void add(okx::OkxSubscription);
 
-//  private:
-//    struct PImpl;
-//    std::unique_ptr<PImpl> me_;
-// };
+ private:
+   struct PImpl;
+   std::unique_ptr<PImpl> me_;
+};
 
-// } // namespace bts::md
+} // namespace bts::md

src/app_sparkd/sparkd/md/OnixsMarketDataProvider.cpp

@@ -1,47 +0,0 @@
-#include "OnixsMarketDataProvider.h"
-#include "domain/protobufs/venue.pb.h"
-#include "spark/config/config.h"
-#include "spark/persistence/IDGenerator.h"
-#include "sparkd/md/glbx/OnixsGlbxSubscription.h"
-
-#include <boost/filesystem.hpp>
-#include <memory>
-#include <vector>
-
-namespace bts::md {
-
-using bts::api::md::MarketDataConfig;
-using bts::persistence::IDGenerator;
-using std::pair;
-using std::shared_ptr;
-using std::string;
-using std::vector;
-
-using Path = boost::filesystem::path;
-
-struct OnixsMarketDataProvider::PImpl {
-   MarketDataConfig config;
-   IDGenerator &sec_id_store = IDGenerator::instance();
-   shared_ptr<OnixsGlbxSubscription> source;
-};
-
-OnixsMarketDataProvider::OnixsMarketDataProvider()
-    : bts::md::OnixsMarketDataProvider(bts::config::get<MarketDataConfig>()) {}
-
-OnixsMarketDataProvider::OnixsMarketDataProvider(MarketDataConfig config)
-    : me_(std::make_unique<PImpl>()) {
-   me_->config = std::move(config);
-}
-
-OnixsMarketDataProvider::~OnixsMarketDataProvider() = default;
-
-vector<pair<domain::protobufs::Venue, shared_ptr<IMarketDataSource>>>
-OnixsMarketDataProvider::get_sources() {
-   vector<pair<domain::protobufs::Venue, shared_ptr<IMarketDataSource>>> rv;
-   me_->source =
-       std::make_shared<OnixsGlbxSubscription>(&me_->sec_id_store, me_->config);
-   rv.push_back({domain::protobufs::VENUE_GLBX, me_->source});
-   return rv;
-}
-
-} // namespace bts::md

src/app_sparkd/sparkd/md/OnixsMarketDataProvider.h

@@ -1,25 +0,0 @@
-#pragma once
-
-#include "spark/api/md/MarketDataConfig.pb.h"
-#include "sparkd/md/IMarketDataProvider.h"
-#include "sparkd/md/IMarketDataSource.h"
-
-#include <memory>
-
-namespace bts::md {
-
-class OnixsMarketDataProvider : public IMarketDataProvider {
- public:
-   OnixsMarketDataProvider();
-   explicit OnixsMarketDataProvider(api::md::MarketDataConfig config);
-   ~OnixsMarketDataProvider();
-
-   std::vector<std::pair<domain::protobufs::Venue, std::shared_ptr<IMarketDataSource>>>
-   get_sources() override;
-
- private:
-   struct PImpl;
-   std::unique_ptr<PImpl> me_;
-};
-
-} // namespace bts::md

src/app_sparkd/sparkd/md/PyMarketDataProvider.cpp

@@ -1,58 +0,0 @@
-#include "PyMarketDataProvider.h"
-#include "spark/api/md/MarketDataConfig.pb.h"
-#include "spark/os/os.h"
-#include "spark/persistence/IDGenerator.h"
-#include "sparkd/md/IMarketDataSource.h"
-#include "sparkd/md/MarketDataSubscription.h"
-#include "sparkd/md/PyMarketDataSubscription.h"
-
-namespace bts::md {
-
-using bts::os::sparkd_wd;
-using bts::persistence::IDGenerator;
-using std::pair;
-using std::shared_ptr;
-using std::string;
-using std::vector;
-
-using Path = boost::filesystem::path;
-
-struct PyMarketDataProvider::PImpl {
-   api::md::MarketDataConfig config;
-   IDGenerator *sec_id_store;
-   vector<shared_ptr<PyMarketDataSubscription>> py_md_subs = {};
-};
-
-PyMarketDataProvider::PyMarketDataProvider(
-    api::md::MarketDataConfig config, IDGenerator *sec_id_store)
-    : me_(std::make_unique<PImpl>()) {
-   me_->config = std::move(config);
-   me_->sec_id_store = sec_id_store;
-}
-
-PyMarketDataProvider::~PyMarketDataProvider() = default;
-
-vector<pair<domain::protobufs::Venue, shared_ptr<IMarketDataSource>>>
-PyMarketDataProvider::get_sources() {
-   vector<pair<domain::protobufs::Venue, shared_ptr<IMarketDataSource>>> rv;
-   const Path wd{sparkd_wd()};
-   vector<string> module_paths{
-       wd.string(),
-       (wd / "py_market_data_providers").string(),
-   };
-   for (const auto &py_plugin : me_->config.python_md_providers()) {
-      vector<string> module_path;
-      for (const auto &p : py_plugin.module_path()) {
-         module_path.emplace_back(p);
-      }
-      auto source = std::make_shared<PyMarketDataSubscription>(
-          py_plugin.provider_type(), module_path, me_->sec_id_store);
-      me_->py_md_subs.push_back(source);
-      rv.push_back(std::make_pair<
-                   domain::protobufs::Venue, shared_ptr<IMarketDataSource>>(
-          py_plugin.venue(), source));
-   }
-   return rv;
-}
-
-} // namespace bts::md

src/app_sparkd/sparkd/md/PyMarketDataProvider.h

@@ -1,27 +0,0 @@
-#pragma once
-
-#include "spark/api/md/MarketDataConfig.pb.h"
-#include "spark/persistence/IDGenerator.h"
-#include "sparkd/md/IMarketDataProvider.h"
-
-#include <memory>
-
-namespace bts::md {
-
-class PyMarketDataProvider : public IMarketDataProvider {
- public:
-   explicit PyMarketDataProvider(
-       api::md::MarketDataConfig config,
-       persistence::IDGenerator *sec_id_store);
-   ~PyMarketDataProvider();
-
-   std::vector<
-       std::pair<domain::protobufs::Venue, std::shared_ptr<IMarketDataSource>>>
-   get_sources() override;
-
- private:
-   struct PImpl;
-   std::unique_ptr<PImpl> me_;
-};
-
-} // namespace bts::md

src/app_sparkd/sparkd/md/PyMarketDataSubscription.cpp

@@ -1,249 +0,0 @@
-#include "PyMarketDataSubscription.h"
-#include "domain/future_security.h"
-#include "domain/protobufs/aggressor_side.pb.h"
-#include "domain/protobufs/call_put.pb.h"
-#include "domain/protobufs/venue.h"
-#include "domain/security.h"
-#include "domain/security_id.h"
-#include "domain/spot_security.h"
-#include "domain/trade_record.h"
-#include "python_md/md_sink.h"
-#include "python_md/py_book.h"
-#include "python_md/py_md_executor.h"
-#include "python_md/py_security.h"
-#include "python_md/py_security_list.h"
-#include "python_md/py_trade.h"
-#include "spark/logging/logging.h"
-#include "spark/md/IMarketDataListener.h"
-#include "spark/os/os.h"
-#include "spark/persistence/IDGenerator.h"
-#include "spark/utils/bts_assert.h"
-#include "system/time_stamp.h"
-
-#include <memory>
-#include <optional>
-#include <pybind11/embed.h>
-#include <pybind11/pybind11.h>
-#include <vector>
-
-namespace bts::md {
-
-#define AGGRESSOR_SIDE(x) bts::domain::protobufs::AGGRESSOR_SIDE_##x
-
-using bts::domain::SecurityID;
-using bts::md::py_interop::PyBook;
-using bts::md::py_interop::PyBookList;
-using bts::md::py_interop::PyBookSide;
-using bts::md::py_interop::PyMdExecutor;
-using bts::md::py_interop::PySecurityList;
-using bts::md::py_interop::PyTrade;
-using bts::md::py_interop::PyTradeList;
-using bts::persistence::IDGenerator;
-using std::shared_ptr;
-using std::string;
-using std::vector;
-
-using Path = boost::filesystem::path;
-
-struct TradeTuple {
-   py_interop::PyTrade py_trade;
-   domain::TradeRecord trade;
-   domain::protobufs::Venue venue;
-};
-
-struct PyMarketDataSubscription::PImpl : private py_interop::MdSink {
-   PImpl(
-       string py_plugin, vector<string> module_paths, IDGenerator *sec_id_store)
-       : py_plugin_(std::move(py_plugin)),
-         module_paths_(std::move(module_paths)),
-         py_md_exec_(py_plugin_, module_paths_, this),
-         sec_id_store_(sec_id_store) {}
-
-   void start(shared_ptr<IMarketDataListener> listener) {
-      bts_assert(listener != nullptr);
-      this->listener_ = std::move(listener);
-      LOG(md, Info, "PyMarketDataSubscription::start t_id:{} t_name:{}",
-          os::get_thread_id(), os::get_thread_name());
-      py_md_exec_.start();
-   }
-
-   void stop() { py_md_exec_.stop(); }
-
- private:
-   string py_plugin_;
-   vector<string> module_paths_;
-   PyMdExecutor py_md_exec_;
-   shared_ptr<IMarketDataListener> listener_;
-
-   IDGenerator *sec_id_store_;
-   // The main store for securities.
-   vector<domain::Security> securities_;
-   vector<const domain::Security *> security_ptrs_;
-   domain::Book book_;
-   // Since the python md provider isn't tethered to a single venue, we
-   // must know the venue for each individual trade.
-   vector<domain::TradeRecord> trades_;
-   vector<const domain::TradeRecord *> trade_ptrs_;
-
-   void on_securities(const PySecurityList &py_securities) override {
-      LOG(md, Info, "Received securities from python plugin. count: {}",
-          py_securities.size());
-      // `securities_` and `security_ptrs_` should already be empty, but no harm in clearing them
-      // again. We keep them as member variables to avoid (re)allocating memory in each callback.
-      securities_.clear();
-      security_ptrs_.clear();
-      for (const auto &py_sec : py_securities.items()) {
-         LOG(md, Info, "PyMarketDataSubscription adding security {}",
-             py_sec.name());
-         auto sec = to_bts_security_(py_sec);
-         if (!sec) {
-            LOG(md, Error,
-                "Failed to convert security {} to bts::domain::Security",
-                py_sec.name());
-            continue;
-         }
-         securities_.push_back(std::move(*sec));
-      }
-      // Populate vectors of security ptrs per each venue.
-      for (auto &sec : securities_) {
-         security_ptrs_.push_back(&sec);
-      }
-      // Pass securities to the sinks (per venue).
-      listener_->on_securities(security_ptrs_);
-      securities_.clear();
-      security_ptrs_.clear();
-   }
-
-   void on_books(const PyBookList &py_books) override {
-      for (auto &py_book : py_books.items()) {
-         bts_assert(book_.count_populated() == 0);
-         to_book_(py_book, book_);
-         listener_->on_book(book_);
-         book_.clear();
-      }
-   }
-
-   void on_trades(const PyTradeList &py_trades) override {
-      LOG(md, Info, "Received trades from python plugin. count: {}",
-          py_trades.size());
-      // `trades_` and `trade_ptrs_` should already be empty, but no harm in clearing them again.
-      // We keep them as member variables to avoid (re)allocating memory in each callback.
-      trades_.clear();
-      trade_ptrs_.clear();
-      for (const auto &py_trade : py_trades.items()) {
-         LOG(md, Info, "PyMarketDataSubscription received trade {}",
-             py_trade.to_string());
-         to_bts_trade_(py_trade, trades_.emplace_back());
-      }
-      for (const auto &t : trades_) {
-         trade_ptrs_.push_back(&t);
-      }
-      listener_->on_trades(trade_ptrs_);
-      trades_.clear();
-      trade_ptrs_.clear();
-   }
-
-   SecurityID get_sec_id_(const string &venue_name, const string &sec_name) {
-      domain::protobufs::Venue venue;
-      // Although an overload of 'get_id' accepts a string, we parse `venue_name` to the enum to
-      // normalize different spellings of the same venue. For example, "deribt", "DERIBIT", and
-      // "VENUE_deribit" all represent the VENUE_DERIBIT enum (see `venue_parse_smart` for details).
-      bts_assert(domain::protobufs::venue_parse_smart(venue_name, &venue));
-      auto sec_id = sec_id_store_->get_id(venue, sec_name, true);
-      bts_assert_msgf(
-          sec_id, "Failed to get security id for py_book. venue: {}; name: {}",
-          venue, sec_name);
-      return SecurityID{*sec_id};
-   }
-
-   std::optional<domain::Security> to_bts_security_(
-       const py_interop::PySecurity &py_sec) {
-      // We can't let python provider assign the id, because it is unaware of other
-      // md providers, so can't possibly guarentee a unique id.
-      bts_assert(py_sec.id() == 0);
-      domain::Security sec;
-      auto &sec_comm = sec.security_common;
-      if (!domain::protobufs::venue_parse_smart(
-              py_sec.venue(), &sec_comm.venue)) {
-         LOG(md, Error, "Failed to parse venue from '{}'", py_sec.venue());
-         return std::nullopt;
-      }
-      sec_comm.id = get_sec_id_(py_sec.venue(), py_sec.exchange_id());
-      sec_comm.exchange_sec_id = py_sec.exchange_id();
-      sec_comm.currency = py_sec.currency();
-      sec_comm.asset = py_sec.asset();
-      sec_comm.group = py_sec.group();
-      sec_comm.symbol = py_sec.asset();
-      if (sec_comm.symbol.empty()) {
-         sec_comm.symbol = py_sec.group();
-      }
-      sec_comm.desc = py_sec.name();
-      if (py_sec.is_spot()) {
-         sec.security = domain::SpotSecurity{};
-      } else if (py_sec.is_future()) {
-         domain::FutureSecurity bts_fut;
-         bts_fut.expiration =
-             core::TimeStamp::from_iso_string(py_sec.future().expiration());
-         sec.security = std::move(bts_fut);
-      } else if (py_sec.is_option()) {
-         domain::OptionSecurity opt;
-         const auto &py_opt = py_sec.option();
-         opt.expiration = core::TimeStamp::from_iso_string(py_opt.expiration());
-         opt.call_put = py_opt.is_call() ? domain::protobufs::CallPut::CALL
-                                         : domain::protobufs::CallPut::PUT;
-         opt.strike = domain::Price::from_double(py_opt.strike());
-         sec.security = std::move(opt);
-      } else {
-         LOG(md, Error,
-             "PySecurity has incomplete definition (not spot, future, or "
-             "option)");
-         return std::nullopt;
-      }
-      return sec;
-   }
-
-   void to_bts_trade_(const PyTrade &source, domain::TradeRecord &dest) {
-      dest.sec_id = get_sec_id_(source.venue(), source.exchange_security_id());
-      dest.time_stamp = core::TimeStamp::from_nanos(source.timestamp());
-      dest.agg_side = source.aggressor_side().starts_with('B') ||
-                              source.aggressor_side().starts_with('b')
-                          ? AGGRESSOR_SIDE(BUY)
-                          : AGGRESSOR_SIDE(SELL);
-      dest.trade_price = domain::Price::from_double(source.price());
-      *dest.trade_quantity = source.quantity();
-   }
-
-   void to_book_side_(const PyBookSide &source, domain::BookSide &dest) {
-      for (int i = 0; i < source.size(); i++) {
-         auto py_level = source.get_level(i);
-         const auto price = py_level.price();
-         const auto quantity = py_level.quantity();
-         bts_assert(!std::isnan(price));
-         bts_assert(quantity > 0);
-         *dest[i].total_quantity = quantity;
-         *dest[i].direct_quantity = quantity;
-         dest[i].price = domain::Price::from_double(price);
-      }
-   }
-
-   void to_book_(const PyBook &source, domain::Book &dest) {
-      to_book_side_(source.bids(), dest.bids());
-      to_book_side_(source.asks(), dest.asks());
-      dest.set_security_id(
-          get_sec_id_(source.venue(), source.exchange_security_id()));
-   }
-};
-
-PyMarketDataSubscription::PyMarketDataSubscription(
-    string py_plugin, vector<string> module_paths, IDGenerator *sec_id_store)
-    : me_(std::make_unique<PyMarketDataSubscription::PImpl>(
-          std::move(py_plugin), std::move(module_paths), sec_id_store)) {}
-
-PyMarketDataSubscription::~PyMarketDataSubscription() { me_->stop(); }
-
-void PyMarketDataSubscription::start(shared_ptr<IMarketDataListener> listener) {
-   me_->start(std::move(listener));
-}
-void PyMarketDataSubscription::stop() { me_->stop(); }
-
-} // namespace bts::md

src/app_sparkd/sparkd/md/PyMarketDataSubscription.h

@@ -1,31 +0,0 @@
-#pragma once
-
-#include "sparkd/md/IMarketDataSource.h"
-
-#include <memory>
-#include <string>
-#include <vector>
-
-namespace bts::persistence {
-class IDGenerator;
-}
-
-namespace bts::md {
-
-class PyMarketDataSubscription : public IMarketDataSource {
- public:
-   explicit PyMarketDataSubscription(
-       std::string py_plugin,
-       std::vector<std::string> module_paths,
-       persistence::IDGenerator *sec_id_store);
-   ~PyMarketDataSubscription();
-
-   void start(std::shared_ptr<IMarketDataListener> listener) override;
-   void stop() override;
-
- private:
-   struct PImpl;
-   std::unique_ptr<PImpl> me_;
-};
-
-} // namespace bts::md

src/app_sparkd/sparkd/md/ReplayMarketDataProvider.cpp

@@ -1,51 +0,0 @@
-#include "ReplayMarketDataProvider.h"
-#include "spark/api/md/MarketDataConfig.pb.h"
-#include "spark/api/md/MarketDataConfigUtils.h"
-#include "spark/config/config.h"
-#include "spark/domain/DomainSecurityUtils.h"
-#include "spark/persistence/IDGenerator.h"
-#include "sparkd/md/IMarketDataSource.h"
-#include "sparkd/md/recording/MDReplayer.h"
-
-namespace bts::md {
-
-using bts::api::md::MarketDataConfig;
-using bts::md::recording::MDReplayer;
-using bts::persistence::IDGenerator;
-using std::pair;
-using std::shared_ptr;
-using std::string;
-using std::vector;
-
-struct ReplayMarketDataProvider::PImpl {
-   MarketDataConfig config;
-   IDGenerator *sec_id_store;
-   vector<pair<domain::protobufs::Venue, shared_ptr<IMarketDataSource>>>
-       sources;
-};
-
-ReplayMarketDataProvider::ReplayMarketDataProvider()
-    : ReplayMarketDataProvider(
-          bts::config::get<MarketDataConfig>(), &IDGenerator::instance()) {}
-
-ReplayMarketDataProvider::ReplayMarketDataProvider(
-    MarketDataConfig config, IDGenerator *sec_id_store)
-    : me_(std::make_unique<PImpl>()) {
-   me_->config = std::move(config);
-   me_->sec_id_store = sec_id_store;
-}
-
-ReplayMarketDataProvider::~ReplayMarketDataProvider() = default;
-
-vector<pair<domain::protobufs::Venue, shared_ptr<IMarketDataSource>>>
-ReplayMarketDataProvider::get_sources() {
-   const auto replay_cfgs =
-       MarketDataConfigUtils::get_effective_replay_config(me_->config);
-   for (const auto &r : replay_cfgs) {
-      const auto v = domain::DomainSecurityUtils::convert(r.venue());
-      me_->sources.push_back({v, std::make_shared<MDReplayer>(r.venue())});
-   }
-   return me_->sources;
-}
-
-} // namespace bts::md

src/app_sparkd/sparkd/md/ReplayMarketDataProvider.h

@@ -1,28 +0,0 @@
-#pragma once
-
-#include "spark/api/md/MarketDataConfig.pb.h"
-#include "spark/persistence/IDGenerator.h"
-#include "sparkd/md/IMarketDataProvider.h"
-
-#include <memory>
-
-namespace bts::md {
-
-class ReplayMarketDataProvider : public IMarketDataProvider {
- public:
-   ReplayMarketDataProvider();
-   explicit ReplayMarketDataProvider(
-       api::md::MarketDataConfig config,
-       persistence::IDGenerator *sec_id_store);
-   ~ReplayMarketDataProvider();
-
-   std::vector<
-       std::pair<domain::protobufs::Venue, std::shared_ptr<IMarketDataSource>>>
-   get_sources() override;
-
- private:
-   struct PImpl;
-   std::unique_ptr<PImpl> me_;
-};
-
-} // namespace bts::md

src/app_sparkd/sparkd/md/ReplayMarketDataSubscription.cpp

@@ -1,47 +0,0 @@
-#include "ReplayMarketDataSubscription.h"
-#include "spark/api/md/MarketDataReplayConfig.pb.h"
-#include "spark/logging/logging.h"
-#include "spark/md/GeneralMarketDataListener.h"
-#include "spark/md/IMarketDataListener.h"
-#include "spark/md/MDProducer.h"
-#include "spark/utils/thread_manager.h"
-#include "sparkd/md/recording/MDReplayer.h"
-
-#include <ccapi_cpp/ccapi_event_dispatcher.h>
-#include <optional>
-
-namespace bts::md {
-
-using bts::api::md::MarketDataReplayConfig;
-using bts::md::recording::MDReplayer;
-using std::string;
-
-struct ReplayMarketDataSource::PImpl {
-   MarketDataReplayConfig config;
-
-   PImpl(const MarketDataReplayConfig &config) : config(config) {}
-};
-
-ReplayMarketDataSource::ReplayMarketDataSource(MarketDataReplayConfig config)
-    : me_(std::make_unique<PImpl>(config)) {
-   me_->config = std::move(config);
-}
-
-ReplayMarketDataSource::~ReplayMarketDataSource() = default;
-
-void ReplayMarketDataSource::start(
-    std::shared_ptr<IMarketDataListener> listener) {
-   LOG(md, Info, "MarketDataGateway::_start_replays starting t_id:{} t_name:{}",
-       os::get_thread_id(), os::get_thread_name());
-   auto md_replayer = std::make_shared<MDReplayer>(me_->config.venue());
-   std::string tid_name = "MDReplayer_" + Venue_Name(me_->config.venue());
-   bts::threading::make_thread(tid_name, [&, listener, md_replayer]() {
-      md_replayer->start(listener);
-   }).detach();
-   LOG(md, Info, "MarketDataGateway::_start_replays ending t_id:{} t_name:{}",
-       os::get_thread_id(), os::get_thread_name());
-}
-
-void ReplayMarketDataSource::stop() {}
-
-} // namespace bts::md

src/app_sparkd/sparkd/md/ReplayMarketDataSubscription.h

@@ -1,25 +0,0 @@
-#pragma once
-
-#include "spark/api/md/MarketDataReplayConfig.pb.h"
-#include "spark/md/IMarketDataListener.h"
-#include "sparkd/md/IMarketDataSource.h"
-
-#include <memory>
-
-namespace bts::md {
-
-class ReplayMarketDataSource : public IMarketDataSource {
- public:
-   ReplayMarketDataSource();
-   explicit ReplayMarketDataSource(api::md::MarketDataReplayConfig config);
-   ~ReplayMarketDataSource();
-
-   void start(std::shared_ptr<IMarketDataListener> listener) override;
-   void stop() override;
-
- private:
-   struct PImpl;
-   std::unique_ptr<PImpl> me_;
-};
-
-} // namespace bts::md

src/app_sparkd/sparkd/md/binance/BinanceContractType.h

@@ -1,7 +1,7 @@
 #pragma once
 
-#include "system/FMT_IMPL.h"
-#include "system/enum_macros.h"
+#include "spark/core/EnumMacros.h"
+#include "spark/core/FMT_IMPL.h"
 
 #include <fmt/format.h>
 #include <nlohmann/json.hpp>

src/app_sparkd/sparkd/md/binance/BinanceFilterTuple.h

@@ -1,5 +1,7 @@
 #pragma once
 
+#include "spark/io/JsonDumper.h"
+#include "spark/io/JsonReader.h"
 #include "sparkd/md/binance/BinanceFilterType.h"
 #include "sparkd/md/binance/BinanceIcebergPartsFilter.h"
 #include "sparkd/md/binance/BinanceLotSizeFilter.h"
@@ -13,8 +15,6 @@
 #include "sparkd/md/binance/BinancePercentPriceFilter.h"
 #include "sparkd/md/binance/BinancePriceFilter.h"
 #include "sparkd/md/binance/BinanceTrailingDeltaFilter.h"
-#include "system/json/json_dumper.h"
-#include "system/json/json_reader.h"
 
 #include <nlohmann/json.hpp>
 #include <string>

src/app_sparkd/sparkd/md/binance/BinanceFilterType.h

@@ -1,7 +1,7 @@
 #pragma once
 
-#include "system/FMT_IMPL.h"
-#include "system/enum_macros.h"
+#include "spark/core/EnumMacros.h"
+#include "spark/core/FMT_IMPL.h"
 
 #include <fmt/format.h>
 #include <nlohmann/json.hpp>

src/app_sparkd/sparkd/md/binance/BinanceIcebergPartsFilter.h

@@ -1,8 +1,8 @@
 #pragma once
 
 #include "spark/io/JsonMacros.h"
+#include "spark/io/JsonReader.h"
 #include "sparkd/md/binance/BinanceFilterType.h"
-#include "system/json/json_reader.h"
 
 #include <iostream>
 #include <nlohmann/json.hpp>

src/app_sparkd/sparkd/md/binance/BinanceInstrumentStatus.h

@@ -1,7 +1,7 @@
 #pragma once
 
-#include "system/FMT_IMPL.h"
-#include "system/enum_macros.h"
+#include "spark/core/EnumMacros.h"
+#include "spark/core/FMT_IMPL.h"
 
 #include <fmt/format.h>
 #include <nlohmann/json.hpp>

src/app_sparkd/sparkd/md/binance/BinanceLotSizeFilter.h

@@ -1,8 +1,7 @@
 #pragma once
 
-#include "spark/logging/logging.h"
+#include "spark/io/JsonReader.h"
 #include "sparkd/md/binance/BinanceFilterType.h"
-#include "system/json/json_reader.h"
 
 #include <nlohmann/json.hpp>
 #include <string>

src/app_sparkd/sparkd/md/binance/BinanceMarketLotSizeFilter.h

@@ -1,7 +1,7 @@
 #pragma once
 
+#include "spark/io/JsonReader.h"
 #include "sparkd/md/binance/BinanceFilterType.h"
-#include "system/json/json_reader.h"
 
 #include <nlohmann/json.hpp>
 

src/app_sparkd/sparkd/md/binance/BinanceMaxNumAlgoOrdersFilter.h

@@ -1,7 +1,7 @@
 #pragma once
 
+#include "spark/io/JsonReader.h"
 #include "sparkd/md/binance/BinanceFilterType.h"
-#include "system/json/json_reader.h"
 
 #include <nlohmann/json.hpp>
 #include <string>

src/app_sparkd/sparkd/md/binance/BinanceMaxNumOrdersFilter.h

@@ -1,7 +1,7 @@
 #pragma once
 
+#include "spark/io/JsonReader.h"
 #include "sparkd/md/binance/BinanceFilterType.h"
-#include "system/json/json_reader.h"
 
 #include <nlohmann/json.hpp>
 #include <string>

src/app_sparkd/sparkd/md/binance/BinanceMaxPositionFilter.h

@@ -1,7 +1,7 @@
 #pragma once
 
+#include "spark/io/JsonReader.h"
 #include "sparkd/md/binance/BinanceFilterType.h"
-#include "system/json/json_reader.h"
 
 #include <nlohmann/json.hpp>
 

src/app_sparkd/sparkd/md/binance/BinanceMessageType.h

@@ -1,7 +1,7 @@
 #pragma once
 
-#include "system/FMT_IMPL.h"
-#include "system/enum_macros.h"
+#include "spark/core/EnumMacros.h"
+#include "spark/core/FMT_IMPL.h"
 
 #include <fmt/format.h>
 #include <nlohmann/json.hpp>

src/app_sparkd/sparkd/md/binance/BinanceMinNotionalFilter.h

@@ -1,8 +1,8 @@
 #pragma once
 
 #include "spark/io/JsonMacros.h"
+#include "spark/io/JsonReader.h"
 #include "sparkd/md/binance/BinanceFilterType.h"
-#include "system/json/json_reader.h"
 
 #include <iostream>
 #include <nlohmann/json.hpp>

src/app_sparkd/sparkd/md/binance/BinanceMiniTickerAdaptor.h

@@ -1,6 +1,6 @@
 #pragma once
 
-#include "domain/security_id.h"
+#include "spark/domain/SecurityID.h"
 #include "sparkd/md/ccapi/ICCApiMessageHandler.h"
 
 namespace bts::md {

src/app_sparkd/sparkd/md/binance/BinanceMiniTickerEventType.h

@@ -1,6 +1,6 @@
 #pragma once
 #ifdef SPARK_ENABLE_CCAPI
-#include "domain/protobufs/security_type.pb.h"
+#include "spark/api/domain/SecurityType.pb.h"
 
 #include <nlohmann/json.hpp>
 #include <ostream>

src/app_sparkd/sparkd/md/binance/BinanceMiniTickerUpdate.cpp

@@ -1,12 +1,12 @@
 #ifdef SPARK_ENABLE_CCAPI
 #include "BinanceMiniTickerUpdate.h"
-#include "domain/price.h"
-#include "domain/quantity.h"
+#include "spark/core/TimeStamp.h"
+#include "spark/domain/Price.h"
+#include "spark/domain/Quantity.h"
 #include "spark/domain/SecurityMarketDataStatistics.h"
+#include "spark/io/JsonDumper.h"
+#include "spark/io/JsonReader.h"
 #include "sparkd/md/ccapi/CCApiReader.h"
-#include "system/json/json_dumper.h"
-#include "system/json/json_reader.h"
-#include "system/time_stamp.h"
 
 #include <ccapi_cpp/ccapi_message.h>
 #include <type_traits>

src/app_sparkd/sparkd/md/binance/BinanceMiniTickerUpdate.h

@@ -1,8 +1,8 @@
 #pragma once
 
+#include "spark/core/TimeStamp.h"
 #include "spark/domain/SecurityMarketDataStatistics.h"
 #include "sparkd/md/binance/BinanceMiniTickerEventType.h"
-#include "system/time_stamp.h"
 
 #include <nlohmann/json.hpp>
 #include <string>

src/app_sparkd/sparkd/md/binance/BinanceNotionalFilter.h

@@ -1,8 +1,8 @@
 #pragma once
 
 #include "spark/io/JsonMacros.h"
+#include "spark/io/JsonReader.h"
 #include "sparkd/md/binance/BinanceFilterType.h"
-#include "system/json/json_reader.h"
 
 #include <nlohmann/json.hpp>
 #include <string>

src/app_sparkd/sparkd/md/binance/BinanceOrderType.h

@@ -1,7 +1,7 @@
 #pragma once
 
-#include "system/FMT_IMPL.h"
-#include "system/enum_macros.h"
+#include "spark/core/EnumMacros.h"
+#include "spark/core/FMT_IMPL.h"
 
 #include <fmt/format.h>
 #include <nlohmann/json.hpp>

src/app_sparkd/sparkd/md/binance/BinancePercentPriceBySideFilter.h

@@ -1,8 +1,8 @@
 #pragma once
 
+#include "spark/io/JsonDumper.h"
+#include "spark/io/JsonReader.h"
 #include "sparkd/md/binance/BinanceFilterType.h"
-#include "system/json/json_dumper.h"
-#include "system/json/json_reader.h"
 
 #include <nlohmann/json.hpp>
 #include <string>

src/app_sparkd/sparkd/md/binance/BinancePercentPriceFilter.h

@@ -1,7 +1,7 @@
 #pragma once
 
+#include "spark/io/JsonReader.h"
 #include "sparkd/md/binance/BinanceFilterType.h"
-#include "system/json/json_reader.h"
 
 #include <nlohmann/json.hpp>
 #include <string>

src/app_sparkd/sparkd/md/binance/BinancePermission.h

@@ -1,7 +1,7 @@
 #pragma once
 
-#include "system/FMT_IMPL.h"
-#include "system/enum_macros.h"
+#include "spark/core/EnumMacros.h"
+#include "spark/core/FMT_IMPL.h"
 
 #include <fmt/format.h>
 #include <nlohmann/json.hpp>

src/app_sparkd/sparkd/md/binance/BinancePriceFilter.h

@@ -1,8 +1,8 @@
 #pragma once
 
+#include "spark/io/JsonDumper.h"
+#include "spark/io/JsonReader.h"
 #include "sparkd/md/binance/BinanceFilterType.h"
-#include "system/json/json_dumper.h"
-#include "system/json/json_reader.h"
 
 #include <nlohmann/json.hpp>
 #include <string>

src/app_sparkd/sparkd/md/binance/BinancePriceStatistics.cpp

@@ -1,11 +1,11 @@
 #include "BinancePriceStatistics.h"
-#include "domain/price.h"
-#include "domain/quantity.h"
+#include "spark/core/TimeStamp.h"
+#include "spark/domain/Price.h"
+#include "spark/domain/Quantity.h"
 #include "spark/domain/SecurityMarketDataStatistics.h"
+#include "spark/io/JsonDumper.h"
+#include "spark/io/JsonReader.h"
 #include "sparkd/md/ccapi/CCApiReader.h"
-#include "system/json/json_dumper.h"
-#include "system/json/json_reader.h"
-#include "system/time_stamp.h"
 
 #include <ccapi_cpp/ccapi_macro.h>
 #include <ccapi_cpp/ccapi_message.h>

src/app_sparkd/sparkd/md/binance/BinancePriceStatistics.h

@@ -1,6 +1,6 @@
 #pragma once
 
-#include "system/time_stamp.h"
+#include "spark/core/TimeStamp.h"
 
 #include <cstdint>
 #include <nlohmann/json.hpp>

src/app_sparkd/sparkd/md/binance/BinancePriceStatisticsAdaptor.cpp

@@ -1,5 +1,5 @@
 #include "BinancePriceStatisticsAdaptor.h"
-#include "domain/security_id.h"
+#include "spark/domain/SecurityID.h"
 #include "spark/domain/SecurityMarketDataStatistics.h"
 #include "spark/md/IMarketDataListener.h"
 #include "spark/utils/bts_fatal.h"

src/app_sparkd/sparkd/md/binance/BinancePriceStatisticsAdaptor.h

@@ -1,6 +1,6 @@
 #pragma once
 
-#include "domain/security_id.h"
+#include "spark/domain/SecurityID.h"
 #include "sparkd/md/ccapi/ICCApiMessageHandler.h"
 
 #include <string>

src/app_sparkd/sparkd/md/binance/BinanceQuoteAsset.h

@@ -1,8 +1,8 @@
 #pragma once
 
-#include "domain/protobufs/currency.pb.h"
-#include "system/FMT_IMPL.h"
-#include "system/enum_macros.h"
+#include "spark/api/domain/Currency.pb.h"
+#include "spark/core/EnumMacros.h"
+#include "spark/core/FMT_IMPL.h"
 
 #include <boost/algorithm/string/case_conv.hpp>
 #include <fmt/format.h>
@@ -60,7 +60,7 @@ class BinanceQuoteAsset {
 
    ENUM_BASIC_MEMBERS(BinanceQuoteAsset)
 
-   constexpr domain::protobufs::Currency to_currency() const;
+   constexpr api::domain::Currency to_currency() const;
 };
 
 ENUM_OUTPUT_OPS(BinanceQuoteAsset)
@@ -75,18 +75,18 @@ ENUM_EQUALITY_OPS(BinanceQuoteAsset)
 // Definition
 //////////////////////////////////////////////////
 
-constexpr domain::protobufs::Currency BinanceQuoteAsset::to_currency() const {
+constexpr api::domain::Currency BinanceQuoteAsset::to_currency() const {
    switch (value) {
    case BTC:
-      return domain::protobufs::Currency::BTC;
+      return api::domain::Currency::BTC;
    case ETH:
-      return domain::protobufs::Currency::ETH;
+      return api::domain::Currency::ETH;
    case USDC:
    case USDP:
    case USDS:
    case USDT:
    default:
-      return domain::protobufs::Currency::USD;
+      return api::domain::Currency::USD;
    }
 }
 

src/app_sparkd/sparkd/md/binance/BinanceSecurityRequestor.cpp

@@ -1,8 +1,10 @@
 #include "BinanceSecurityRequestor.h"
-#include "domain/protobufs/security_type.pb.h"
-#include "domain/protobufs/venue.pb.h"
-#include "domain/security.h"
-#include "domain/security_criteria.h"
+#include "spark/api/domain/SecurityType.pb.h"
+#include "spark/api/domain/Venue.pb.h"
+#include "spark/collections/Contains.h"
+#include "spark/collections/Remove.h"
+#include "spark/domain/Security.h"
+#include "spark/domain/SecurityCriteria.h"
 #include "spark/logging/logging.h"
 #include "spark/persistence/IDGenerator.h"
 #include "spark/utils/bts_fatal.h"
@@ -12,8 +14,6 @@
 #include "sparkd/md/ccapi/CCApiMessageHandler.h"
 #include "sparkd/md/ccapi/CCApiSession.h"
 #include "sparkd/md/ccapi/ICCApiMessageHandler.h"
-#include "system/collections/contains.h"
-#include "system/collections/remove.h"
 
 #include <algorithm>
 #include <ccapi_cpp/ccapi_event_dispatcher.h>
@@ -26,12 +26,12 @@
 
 namespace bts::md::binance {
 
+using api::domain::SecurityType;
+using api::domain::Venue;
+using api::domain::Venue_Name;
 using bts::domain::Security;
 using bts::domain::SecurityCriteria;
 using bts::domain::SecurityID;
-using bts::domain::protobufs::SecurityType;
-using bts::domain::protobufs::Venue;
-using bts::domain::protobufs::Venue_Name;
 using bts::persistence::IDGenerator;
 using std::string;
 using std::unordered_set;
@@ -53,7 +53,7 @@ using std::vector;
 #define CCAPI_VENUE_NAME CCAPI_EXCHANGE_NAME_BINANCE
 #define BINANCE_MSG_TYPE_NAME(x)                                               \
    std::string { BinanceMessageType{x}.name() }
-#define SEC_TYPE(x) bts::domain::protobufs::SECURITY_TYPE_##x
+#define SEC_TYPE(x) bts::api::domain::SECURITY_TYPE_##x
 
 static ccapi::Request exchange_info_req() {
    auto corr_id = CCApiCorrelationID::from_parts(

src/app_sparkd/sparkd/md/binance/BinanceSecurityRequestor.h

@@ -1,6 +1,6 @@
 #pragma once
 
-#include "system/delegate.h"
+#include "spark/utils/Delegate.h"
 
 #include <memory>
 #include <unordered_set>

src/app_sparkd/sparkd/md/binance/BinanceSpot.cpp

@@ -1,9 +1,9 @@
 #include "BinanceSpot.h"
-#include "domain/protobufs/venue.pb.h"
-#include "domain/quantity.h"
-#include "domain/security.h"
+#include "spark/api/domain/Venue.pb.h"
+#include "spark/domain/Quantity.h"
+#include "spark/domain/Security.h"
+#include "spark/io/JsonReader.h"
 #include "spark/logging/logging.h"
-#include "system/json/json_reader.h"
 
 #include <ccapi_cpp/ccapi_message.h>
 #include <iostream>
@@ -12,15 +12,15 @@
 
 namespace bts::md::binance {
 
+using bts::api::domain::Venue;
 using bts::domain::Quantity;
 using bts::domain::Security;
 using bts::domain::SpotSecurity;
-using bts::domain::protobufs::Venue;
 using bts::io::JsonReader;
 using std::string;
 using std::vector;
 
-#define SEC_TYPE(x) bts::domain::protobufs::SECURITY_TYPE_##x
+#define SEC_TYPE(x) bts::api::domain::SECURITY_TYPE_##x
 
 static std::string deduce_sym_from_name(const std::string &name) {
    std::regex word_regex("(\\w+)");

src/app_sparkd/sparkd/md/binance/BinanceSpot.h

@@ -1,13 +1,13 @@
 #pragma once
 
-#include "domain/protobufs/venue.h"
+#include "spark/api/domain/Venue.h"
+#include "spark/core/FMT_IMPL.h"
 #include "spark/io/JsonMacros.h"
 #include "sparkd/md/binance/BinanceFilterTuple.h"
 #include "sparkd/md/binance/BinanceInstrumentStatus.h"
 #include "sparkd/md/binance/BinanceOrderType.h"
 #include "sparkd/md/binance/BinancePermission.h"
 #include "sparkd/md/binance/BinanceQuoteAsset.h"
-#include "system/FMT_IMPL.h"
 
 #include <nlohmann/json.hpp>
 #include <ostream>
@@ -50,7 +50,7 @@ class BinanceSpot {
    static void parse(
        const ccapi::Message &msg,
        std::vector<domain::Security> &securities,
-       domain::protobufs::Venue venue = domain::protobufs::VENUE_BINANCE);
+       api::domain::Venue venue = api::domain::Venue::VENUE_BINANCE);
 };
 
 void from_json(const nlohmann::json &j, BinanceSpot &spot);

src/app_sparkd/sparkd/md/binance/BinanceSubscription.cpp

@@ -1,8 +1,10 @@
 #include "BinanceSubscription.h"
-#include "domain/protobufs/venue.pb.h"
-#include "domain/security.h"
-#include "domain/security_criteria.h"
+#include "spark/api/domain/Venue.pb.h"
 #include "spark/api/md/BinanceMarketDataConfig.pb.h"
+#include "spark/collections/UnorderedSetFactory.h"
+#include "spark/core/FMT.h"
+#include "spark/domain/Security.h"
+#include "spark/domain/SecurityCriteria.h"
 #include "spark/io/ProtoPrinter.h"
 #include "spark/io/QueryParams.h"
 #include "spark/logging/logging.h"
@@ -17,8 +19,6 @@
 #include "sparkd/md/ccapi/CCApiMessageLogger.h"
 #include "sparkd/md/ccapi/CCApiSession.h"
 #include "sparkd/md/ccapi/CCApiTradeAdaptor.h"
-#include "system/FMT.h"
-#include "system/collections/unordered_set_factory.h"
 
 #include <ccapi_cpp/ccapi_event_dispatcher.h>
 #include <ccapi_cpp/ccapi_macro.h>
@@ -34,12 +34,12 @@
 
 namespace bts::md::binance {
 
+using bts::api::domain::Venue;
+using bts::api::domain::Venue_Name;
 using bts::api::md::BinanceMarketDataConfig;
 using bts::domain::Security;
 using bts::domain::SecurityCriteria;
 using bts::domain::SecurityID;
-using bts::domain::protobufs::Venue;
-using bts::domain::protobufs::Venue_Name;
 using bts::io::ProtoPrinter;
 using bts::io::QueryParams;
 using bts::persistence::IDGenerator;
@@ -168,7 +168,7 @@ struct BinanceSubscription::PImpl {
 
    // The session automatically starts/stops in its ctr/dtr
    CCApiSession session;
-   std::shared_ptr<IMarketDataListener> listener;
+   IMarketDataListener *listener;
    BinanceSecurityRequestor sec_requestor;
 
    // These aliases are useful for quickly switching the handler types (e.g.,
@@ -191,10 +191,11 @@ struct BinanceSubscription::PImpl {
    explicit PImpl(
        ccapi::EventDispatcher *dispatcher,
        BinanceMarketDataConfig cfg,
-       IDGenerator *sec_id_store)
+       IDGenerator *sec_id_store,
+       IMarketDataListener *listener)
        : sec_id_store(sec_id_store), cfg(std::move(cfg)),
          criteria(SecurityCriteria::from_proto(this->cfg.security_criteria())),
-         session(dispatcher, mk_session_cfg(this->cfg), {}),
+         session(dispatcher, mk_session_cfg(this->cfg), {}), listener(listener),
          sec_requestor(
              &session,
              this->criteria,
@@ -205,18 +206,13 @@ struct BinanceSubscription::PImpl {
       LOG(md, Info, "Initializing BinanceSubscription:\n{}",
           ProtoPrinter::to_string(this->cfg));
       sec_requestor.callback().bind<&PImpl::on_securities>(*this);
-   }
-
-   void start(std::shared_ptr<IMarketDataListener> listener) {
-      bts_assert(listener != nullptr);
-      this->listener = std::move(listener);
       sec_requestor.request();
    }
 
    DepthHandler &mk_depth_handler(const string &sec_desc) {
       SecurityID sec_id{sec_id_store->get_or_add(VENUE_NAME, sec_desc)};
       auto [it, is_new] =
-          depth_handlers.try_emplace(sec_desc, sec_id, listener.get());
+          depth_handlers.try_emplace(sec_desc, sec_id, listener);
       bts_assert(is_new);
       return it->second;
    }
@@ -224,7 +220,7 @@ struct BinanceSubscription::PImpl {
    TickerHandler &mk_ticker_handler(const string &sec_desc) {
       SecurityID sec_id{sec_id_store->get_or_add(VENUE_NAME, sec_desc)};
       auto [it, is_new] =
-          ticker_handlers.try_emplace(sec_desc, sec_id, listener.get());
+          ticker_handlers.try_emplace(sec_desc, sec_id, listener);
       bts_assert_msgf(
           is_new, "Ticker handler already created for '{}'", sec_desc);
       return it->second;
@@ -240,8 +236,7 @@ struct BinanceSubscription::PImpl {
              SecurityID{sec_id_store->get_or_add(VENUE_NAME, sec_desc)});
       }
       auto [it, is_new] = stats_handlers.try_emplace(
-          hnd_id,
-          std::make_unique<StatsHandler>(sec_desc_id_pairs, listener.get()));
+          hnd_id, std::make_unique<StatsHandler>(sec_desc_id_pairs, listener));
       bts_assert_msgf(
           is_new, "StatsHandler already created with id '{}'", hnd_id);
       return *it->second;
@@ -330,8 +325,10 @@ struct BinanceSubscription::PImpl {
 BinanceSubscription::BinanceSubscription(
     ccapi::EventDispatcher *dispatcher,
     BinanceMarketDataConfig cfg,
-    IDGenerator *sec_id_store)
-    : me_(std::make_unique<PImpl>(dispatcher, std::move(cfg), sec_id_store)) {}
+    IDGenerator *sec_id_store,
+    IMarketDataListener *listener)
+    : me_(std::make_unique<PImpl>(
+          dispatcher, std::move(cfg), sec_id_store, listener)) {}
 
 BinanceSubscription::BinanceSubscription(BinanceSubscription &&) noexcept =
     default;
@@ -340,8 +337,4 @@ BinanceSubscription &BinanceSubscription::operator=(
 
 BinanceSubscription::~BinanceSubscription() = default;
 
-void BinanceSubscription::start(std::shared_ptr<IMarketDataListener> listener) {
-   me_->start(listener);
-}
-
 } // namespace bts::md::binance

src/app_sparkd/sparkd/md/binance/BinanceSubscription.h

@@ -1,6 +1,5 @@
 #pragma once
 
-#include "sparkd/md/IMarketDataSource.h"
 #include <memory>
 
 namespace ccapi {
@@ -26,12 +25,13 @@ namespace bts::md::binance {
  *
  * @a https://binance-docs.github.io/apidocs/spot/en/#exchange-information
  */
-class BinanceSubscription : public IMarketDataSource {
+class BinanceSubscription {
  public:
    BinanceSubscription(
        ccapi::EventDispatcher *dispatcher,
        api::md::BinanceMarketDataConfig cfg,
-       persistence::IDGenerator *sec_id_store);
+       persistence::IDGenerator *sec_id_store,
+       IMarketDataListener *listener);
 
    BinanceSubscription(const BinanceSubscription &) noexcept = delete;
    BinanceSubscription(BinanceSubscription &&) noexcept;
@@ -39,9 +39,10 @@ class BinanceSubscription : public IMarketDataSource {
        delete;
    BinanceSubscription &operator=(BinanceSubscription &&) noexcept;
 
-   ~BinanceSubscription() override;
+   ~BinanceSubscription();
 
-   void start(std::shared_ptr<IMarketDataListener> listener) override;
+   void add_listener(IMarketDataListener *listener);
+   void remove_listener(IMarketDataListener *listener);
 
  private:
    struct PImpl;

src/app_sparkd/sparkd/md/binance/BinanceTrailingDeltaFilter.h

@@ -1,8 +1,8 @@
 #pragma once
 
+#include "spark/io/JsonDumper.h"
+#include "spark/io/JsonReader.h"
 #include "sparkd/md/binance/BinanceFilterType.h"
-#include "system/json/json_dumper.h"
-#include "system/json/json_reader.h"
 
 #include <nlohmann/json.hpp>
 

src/app_sparkd/sparkd/md/binance_us/BinanceUsMessageType.h

@@ -1,7 +1,7 @@
 #pragma once
 
-#include "system/FMT_IMPL.h"
-#include "system/enum_macros.h"
+#include "spark/core/EnumMacros.h"
+#include "spark/core/FMT_IMPL.h"
 
 #include <fmt/format.h>
 #include <nlohmann/json.hpp>

src/app_sparkd/sparkd/md/binance_us/BinanceUsPriceStatistics.cpp

@@ -1,11 +1,11 @@
 #include "BinanceUsPriceStatistics.h"
-#include "domain/price.h"
-#include "domain/quantity.h"
+#include "spark/core/TimeStamp.h"
+#include "spark/domain/Price.h"
+#include "spark/domain/Quantity.h"
 #include "spark/domain/SecurityMarketDataStatistics.h"
+#include "spark/io/JsonDumper.h"
+#include "spark/io/JsonReader.h"
 #include "sparkd/md/ccapi/CCApiReader.h"
-#include "system/json/json_dumper.h"
-#include "system/json/json_reader.h"
-#include "system/time_stamp.h"
 
 #include <ccapi_cpp/ccapi_macro.h>
 #include <ccapi_cpp/ccapi_message.h>

src/app_sparkd/sparkd/md/binance_us/BinanceUsPriceStatistics.h

@@ -1,6 +1,6 @@
 #pragma once
 
-#include "system/time_stamp.h"
+#include "spark/core/TimeStamp.h"
 
 #include <cstdint>
 #include <nlohmann/json.hpp>

src/app_sparkd/sparkd/md/binance_us/BinanceUsPriceStatisticsAdaptor.cpp

@@ -1,5 +1,5 @@
 #include "BinanceUsPriceStatisticsAdaptor.h"
-#include "domain/security_id.h"
+#include "spark/domain/SecurityID.h"
 #include "spark/domain/SecurityMarketDataStatistics.h"
 #include "spark/md/IMarketDataListener.h"
 #include "spark/utils/bts_fatal.h"

src/app_sparkd/sparkd/md/binance_us/BinanceUsPriceStatisticsAdaptor.h

@@ -1,6 +1,6 @@
 #pragma once
 
-#include "domain/security_id.h"
+#include "spark/domain/SecurityID.h"
 #include "sparkd/md/ccapi/ICCApiMessageHandler.h"
 
 #include <string>

src/app_sparkd/sparkd/md/binance_us/BinanceUsSecurityRequestor.cpp

@@ -1,8 +1,10 @@
 #include "BinanceUsSecurityRequestor.h"
-#include "domain/protobufs/security_type.pb.h"
-#include "domain/protobufs/venue.pb.h"
-#include "domain/security.h"
-#include "domain/security_criteria.h"
+#include "spark/api/domain/SecurityType.pb.h"
+#include "spark/api/domain/Venue.pb.h"
+#include "spark/collections/Contains.h"
+#include "spark/collections/Remove.h"
+#include "spark/domain/Security.h"
+#include "spark/domain/SecurityCriteria.h"
 #include "spark/logging/logging.h"
 #include "spark/persistence/IDGenerator.h"
 #include "sparkd/md/binance_us/BinanceUsMessageType.h"
@@ -11,8 +13,6 @@
 #include "sparkd/md/ccapi/CCApiMessageHandler.h"
 #include "sparkd/md/ccapi/CCApiSession.h"
 #include "sparkd/md/ccapi/ICCApiMessageHandler.h"
-#include "system/collections/contains.h"
-#include "system/collections/remove.h"
 
 #include <algorithm>
 #include <ccapi_cpp/ccapi_event_dispatcher.h>
@@ -25,12 +25,12 @@
 
 namespace bts::md::binance_us {
 
+using api::domain::SecurityType;
+using api::domain::Venue;
+using api::domain::Venue_Name;
 using bts::domain::Security;
 using bts::domain::SecurityCriteria;
 using bts::domain::SecurityID;
-using bts::domain::protobufs::SecurityType;
-using bts::domain::protobufs::Venue;
-using bts::domain::protobufs::Venue_Name;
 using bts::persistence::IDGenerator;
 using std::string;
 using std::unordered_set;
@@ -41,7 +41,7 @@ using std::vector;
 #define CCAPI_VENUE_NAME CCAPI_EXCHANGE_NAME_BINANCE_US
 #define BINANCE_US_MSG_TYPE_NAME(x)                                            \
    std::string { BinanceUsMessageType{x}.name() }
-#define SEC_TYPE(x) bts::domain::protobufs::SECURITY_TYPE_##x
+#define SEC_TYPE(x) bts::api::domain::SECURITY_TYPE_##x
 
 static ccapi::Request exchange_info_req() {
    auto corr_id = CCApiCorrelationID::from_parts(

src/app_sparkd/sparkd/md/binance_us/BinanceUsSecurityRequestor.h

@@ -1,6 +1,6 @@
 #pragma once
 
-#include "system/delegate.h"
+#include "spark/utils/Delegate.h"
 
 #include <memory>
 #include <unordered_set>

src/app_sparkd/sparkd/md/binance_us/BinanceUsSubscription.cpp

@@ -1,8 +1,10 @@
 #include "BinanceUsSubscription.h"
-#include "domain/protobufs/venue.pb.h"
-#include "domain/security.h"
-#include "domain/security_criteria.h"
+#include "spark/api/domain/Venue.pb.h"
 #include "spark/api/md/BinanceUsMarketDataConfig.pb.h"
+#include "spark/collections/UnorderedSetFactory.h"
+#include "spark/core/FMT.h"
+#include "spark/domain/Security.h"
+#include "spark/domain/SecurityCriteria.h"
 #include "spark/io/ProtoPrinter.h"
 #include "spark/io/QueryParams.h"
 #include "spark/logging/logging.h"
@@ -18,8 +20,6 @@
 #include "sparkd/md/ccapi/CCApiMessageLogger.h"
 #include "sparkd/md/ccapi/CCApiSession.h"
 #include "sparkd/md/ccapi/CCApiTradeAdaptor.h"
-#include "system/FMT.h"
-#include "system/collections/unordered_set_factory.h"
 
 #include <ccapi_cpp/ccapi_event_dispatcher.h>
 #include <ccapi_cpp/ccapi_event_handler.h>
@@ -36,17 +36,16 @@
 
 namespace bts::md::binance_us {
 
+using bts::api::domain::Venue;
+using bts::api::domain::Venue_Name;
 using bts::api::md::BinanceUsMarketDataConfig;
 using bts::domain::Security;
 using bts::domain::SecurityCriteria;
 using bts::domain::SecurityID;
-using bts::domain::protobufs::Venue;
-using bts::domain::protobufs::Venue_Name;
 using bts::io::ProtoPrinter;
 using bts::io::QueryParams;
 using bts::persistence::IDGenerator;
 using std::pair;
-using std::shared_ptr;
 using std::string;
 using std::uint32_t;
 using std::unique_ptr;
@@ -172,7 +171,7 @@ struct BinanceUsSubscription::PImpl {
 
    // The session automatically starts/stops in its ctr/dtr
    CCApiSession session;
-   shared_ptr<IMarketDataListener> listener;
+   IMarketDataListener *listener;
    BinanceUsSecurityRequestor sec_requestor;
 
    // These aliases are useful for quickly switching the handler types (e.g.,
@@ -189,16 +188,17 @@ struct BinanceUsSubscription::PImpl {
    // different map, 'stats_handlers' (unique ids are generated w/ a counter,
    // 'stats_handlers_id_counter').
    unordered_map<string, StatsHandler *> sec_stats_handler_map;
-   unordered_map<uint32_t, unique_ptr<StatsHandler>> stats_handlers;
+   unordered_map<uint32_t, std::unique_ptr<StatsHandler>> stats_handlers;
    uint32_t stats_handlers_id_counter = 0;
 
    explicit PImpl(
        ccapi::EventDispatcher *dispatcher,
        BinanceUsMarketDataConfig cfg,
-       IDGenerator *sec_id_store)
+       IDGenerator *sec_id_store,
+       IMarketDataListener *listener)
        : sec_id_store(sec_id_store), cfg(std::move(cfg)),
          criteria(SecurityCriteria::from_proto(this->cfg.security_criteria())),
-         session(dispatcher, mk_session_cfg(this->cfg), {}),
+         session(dispatcher, mk_session_cfg(this->cfg), {}), listener(listener),
          sec_requestor(
              &session,
              this->criteria,
@@ -209,18 +209,13 @@ struct BinanceUsSubscription::PImpl {
       LOG(md, Info, "Initializing BinanceUsSubscription:\n{}",
           ProtoPrinter::to_string(this->cfg));
       sec_requestor.callback().bind<&PImpl::on_securities>(*this);
-   }
-
-   void start(std::shared_ptr<IMarketDataListener> listener) {
-      bts_assert(listener != nullptr);
-      this->listener = std::move(listener);
       sec_requestor.request();
    }
 
    DepthHandler &mk_depth_handler(const string &sec_desc) {
       SecurityID sec_id{sec_id_store->get_or_add(VENUE_NAME, sec_desc)};
       auto [it, is_new] =
-          depth_handlers.try_emplace(sec_desc, sec_id, listener.get());
+          depth_handlers.try_emplace(sec_desc, sec_id, listener);
       bts_assert(is_new);
       return it->second;
    }
@@ -228,7 +223,7 @@ struct BinanceUsSubscription::PImpl {
    TickerHandler &mk_ticker_handler(const string &sec_desc) {
       SecurityID sec_id{sec_id_store->get_or_add(VENUE_NAME, sec_desc)};
       auto [it, is_new] =
-          ticker_handlers.try_emplace(sec_desc, sec_id, listener.get());
+          ticker_handlers.try_emplace(sec_desc, sec_id, listener);
       bts_assert_msgf(
           is_new, "Ticker handler already created for '{}'", sec_desc);
       return it->second;
@@ -244,8 +239,7 @@ struct BinanceUsSubscription::PImpl {
              SecurityID{sec_id_store->get_or_add(VENUE_NAME, sec_desc)});
       }
       auto [it, is_new] = stats_handlers.try_emplace(
-          hnd_id,
-          std::make_unique<StatsHandler>(sec_desc_id_pairs, listener.get()));
+          hnd_id, std::make_unique<StatsHandler>(sec_desc_id_pairs, listener));
       bts_assert_msgf(
           is_new, "StatsHandler already created with id '{}'", hnd_id);
       return *it->second;
@@ -334,8 +328,10 @@ struct BinanceUsSubscription::PImpl {
 BinanceUsSubscription::BinanceUsSubscription(
     ccapi::EventDispatcher *dispatcher,
     BinanceUsMarketDataConfig cfg,
-    IDGenerator *sec_id_store)
-    : me_(std::make_unique<PImpl>(dispatcher, std::move(cfg), sec_id_store)) {}
+    IDGenerator *sec_id_store,
+    IMarketDataListener *listener)
+    : me_(std::make_unique<PImpl>(
+          dispatcher, std::move(cfg), sec_id_store, listener)) {}
 
 BinanceUsSubscription::BinanceUsSubscription(
     BinanceUsSubscription &&) noexcept = default;
@@ -344,8 +340,4 @@ BinanceUsSubscription &BinanceUsSubscription::operator=(
 
 BinanceUsSubscription::~BinanceUsSubscription() = default;
 
-void BinanceUsSubscription::start(shared_ptr<IMarketDataListener> listener) {
-   me_->start(std::move(listener));
-}
-
 } // namespace bts::md::binance_us

src/app_sparkd/sparkd/md/binance_us/BinanceUsSubscription.h

@@ -1,6 +1,5 @@
 #pragma once
 
-#include "sparkd/md/IMarketDataSource.h"
 #include <memory>
 
 namespace ccapi {
@@ -26,12 +25,13 @@ namespace bts::md::binance_us {
  *
  * @a https://docs.binance.us/#get-exchange-information
  */
-class BinanceUsSubscription : public IMarketDataSource {
+class BinanceUsSubscription {
  public:
    BinanceUsSubscription(
        ccapi::EventDispatcher *dispatcher,
        api::md::BinanceUsMarketDataConfig cfg,
-       persistence::IDGenerator *sec_id_store);
+       persistence::IDGenerator *sec_id_store,
+       IMarketDataListener *listener);
 
    BinanceUsSubscription(const BinanceUsSubscription &) noexcept = delete;
    BinanceUsSubscription(BinanceUsSubscription &&) noexcept;
@@ -41,7 +41,8 @@ class BinanceUsSubscription : public IMarketDataSource {
 
    ~BinanceUsSubscription();
 
-   void start(std::shared_ptr<IMarketDataListener> listener) override;
+   void add_listener(IMarketDataListener *listener);
+   void remove_listener(IMarketDataListener *listener);
 
  private:
    struct PImpl;

src/app_sparkd/sparkd/md/binance_usds_futures/BinanceUsdsFuture.cpp

@@ -1,10 +1,10 @@
 #include "BinanceUsdsFuture.h"
-#include "domain/price.h"
-#include "domain/quantity.h"
-#include "domain/security.h"
+#include "spark/domain/Price.h"
+#include "spark/domain/Quantity.h"
+#include "spark/domain/Security.h"
+#include "spark/io/JsonDumper.h"
+#include "spark/io/JsonReader.h"
 #include "spark/logging/logging.h"
-#include "system/json/json_dumper.h"
-#include "system/json/json_reader.h"
 
 #include <ccapi_cpp/ccapi_message.h>
 #include <iostream>
@@ -12,18 +12,18 @@
 
 namespace bts::md::binance_usds_futures {
 
+using bts::api::domain::SecurityType;
 using bts::domain::FutureSecurity;
 using bts::domain::Price;
 using bts::domain::Quantity;
 using bts::domain::Security;
-using bts::domain::protobufs::SecurityType;
 using bts::io::JsonDumper;
 using bts::io::JsonReader;
 using std::string;
 using std::vector;
 
-#define SEC_TYPE(x) bts::domain::protobufs::SECURITY_TYPE_##x
-#define VENUE bts::domain::protobufs::VENUE_BINANCE_USDS_FUTURES;
+#define SEC_TYPE(x) bts::api::domain::SECURITY_TYPE_##x
+#define VENUE api::domain::Venue::VENUE_BINANCE_USDS_FUTURES;
 
 static string deduce_sym_from_name(const string &name) {
    const std::regex word_regex("(\\w+)");

src/app_sparkd/sparkd/md/binance_usds_futures/BinanceUsdsFuturesContractType.h

@@ -1,7 +1,7 @@
 #pragma once
 
-#include "system/FMT_IMPL.h"
-#include "system/enum_macros.h"
+#include "spark/core/EnumMacros.h"
+#include "spark/core/FMT_IMPL.h"
 
 #include <fmt/format.h>
 #include <nlohmann/json.hpp>

src/app_sparkd/sparkd/md/binance_usds_futures/BinanceUsdsFuturesFilterTuple.h

@@ -1,5 +1,7 @@
 #pragma once
 
+#include "spark/io/JsonDumper.h"
+#include "spark/io/JsonReader.h"
 #include "sparkd/md/binance/BinanceFilterType.h"
 #include "sparkd/md/binance/BinanceIcebergPartsFilter.h"
 #include "sparkd/md/binance/BinanceLotSizeFilter.h"
@@ -12,8 +14,6 @@
 #include "sparkd/md/binance_usds_futures/BinanceUsdsFuturesMaxNumAlgoOrdersFilter.h"
 #include "sparkd/md/binance_usds_futures/BinanceUsdsFuturesMaxNumOrdersFilter.h"
 #include "sparkd/md/binance_usds_futures/BinanceUsdsFuturesPercentPriceFilter.h"
-#include "system/json/json_dumper.h"
-#include "system/json/json_reader.h"
 
 #include <nlohmann/json.hpp>
 #include <string>

src/app_sparkd/sparkd/md/binance_usds_futures/BinanceUsdsFuturesMaxNumAlgoOrdersFilter.h

@@ -1,8 +1,8 @@
 #pragma once
 
+#include "spark/io/JsonDumper.h"
+#include "spark/io/JsonReader.h"
 #include "sparkd/md/binance/BinanceFilterType.h"
-#include "system/json/json_dumper.h"
-#include "system/json/json_reader.h"
 
 #include <nlohmann/json.hpp>
 #include <string>

src/app_sparkd/sparkd/md/binance_usds_futures/BinanceUsdsFuturesMaxNumOrdersFilter.h

@@ -1,8 +1,8 @@
 #pragma once
 
+#include "spark/io/JsonDumper.h"
+#include "spark/io/JsonReader.h"
 #include "sparkd/md/binance/BinanceFilterType.h"
-#include "system/json/json_dumper.h"
-#include "system/json/json_reader.h"
 
 #include <nlohmann/json.hpp>
 #include <string>

src/app_sparkd/sparkd/md/binance_usds_futures/BinanceUsdsFuturesMessageType.h

@@ -1,7 +1,7 @@
 #pragma once
 
-#include "system/FMT_IMPL.h"
-#include "system/enum_macros.h"
+#include "spark/core/EnumMacros.h"
+#include "spark/core/FMT_IMPL.h"
 
 #include <fmt/format.h>
 #include <nlohmann/json.hpp>