activ md check-pt

Started to resolve circular deps between libspark <-> sparkd

CMakeLists.txt

@@ -115,6 +115,9 @@ else()
     add_subdirectory(src/server/sparkd)
     add_subdirectory(src/data_files)
   endif()
+  if (BUILD_ACTIV_MD)
+    add_subdirectory(src/app_activ-tester)
+  endif()
   add_subdirectory(src/lib)
   if (BUILD_ADMIN_WEB_APP)
     add_subdirectory(src/Spark.Admin.WebApp)
@@ -127,6 +130,36 @@ else()
   endif()
 
   if(BUILD_SPARKD)
+    function(copy_shared_lib lib_file_name)
+      add_custom_command(OUTPUT ${CMAKE_BINARY_DIR}/src/server/sparkd/${lib_file_name}
+        COMMAND ${CMAKE_COMMAND} -E copy 
+        ${EXTERNALS_ROOT}/lib/Debug/${lib_file_name}
+        ${CMAKE_BINARY_DIR}/src/server/sparkd/${lib_file_name}
+        DEPENDS ${EXTERNALS_ROOT}/lib/Debug/${lib_file_name})
+    endfunction()
+
+    copy_shared_lib(libActivAgentApplication_x86-64_ubuntu22-04_gcc123_mdid.so)
+    copy_shared_lib(libActivBase_x86-64_ubuntu22-04_gcc123_mdid.so)
+    copy_shared_lib(libActivFramework_x86-64_ubuntu22-04_gcc123_mdid.so)
+    copy_shared_lib(libActivOneApi_x86-64_ubuntu22-04_gcc123_mdid.so)
+    copy_shared_lib(libActivDatabase_x86-64_ubuntu22-04_gcc123_mdid.so)
+    copy_shared_lib(libActivServices_x86-64_ubuntu22-04_gcc123_mdid.so)
+    copy_shared_lib(libBoostAtomic_x86-64_ubuntu22-04_gcc123_mdid.so)
+    copy_shared_lib(libBoostChrono_x86-64_ubuntu22-04_gcc123_mdid.so)
+    copy_shared_lib(libBoostFilesystem_x86-64_ubuntu22-04_gcc123_mdid.so)
+    copy_shared_lib(libBoostProgramOptions_x86-64_ubuntu22-04_gcc123_mdid.so)
+    add_custom_target(activ_shared_libs ALL DEPENDS
+      ${CMAKE_BINARY_DIR}/src/server/sparkd/libActivAgentApplication_x86-64_ubuntu22-04_gcc123_mdid.so
+      ${CMAKE_BINARY_DIR}/src/server/sparkd/libActivBase_x86-64_ubuntu22-04_gcc123_mdid.so
+      ${CMAKE_BINARY_DIR}/src/server/sparkd/libActivFramework_x86-64_ubuntu22-04_gcc123_mdid.so
+      ${CMAKE_BINARY_DIR}/src/server/sparkd/libActivOneApi_x86-64_ubuntu22-04_gcc123_mdid.so
+      ${CMAKE_BINARY_DIR}/src/server/sparkd/libActivDatabase_x86-64_ubuntu22-04_gcc123_mdid.so
+      ${CMAKE_BINARY_DIR}/src/server/sparkd/libActivServices_x86-64_ubuntu22-04_gcc123_mdid.so
+      ${CMAKE_BINARY_DIR}/src/server/sparkd/libBoostAtomic_x86-64_ubuntu22-04_gcc123_mdid.so
+      ${CMAKE_BINARY_DIR}/src/server/sparkd/libBoostChrono_x86-64_ubuntu22-04_gcc123_mdid.so
+      ${CMAKE_BINARY_DIR}/src/server/sparkd/libBoostFilesystem_x86-64_ubuntu22-04_gcc123_mdid.so
+      ${CMAKE_BINARY_DIR}/src/server/sparkd/libBoostProgramOptions_x86-64_ubuntu22-04_gcc123_mdid.so)
+
     add_custom_command(
       OUTPUT ${CMAKE_BINARY_DIR}/server
       COMMAND mkdir -p ${CMAKE_BINARY_DIR}/server
@@ -151,6 +184,11 @@ else()
       FILES
       "${SPARK_LIB}/libACE-5.7.5.so"
       "${SPARK_LIB}/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.16.0.0"
+      "${SPARK_LIB}/libActivFramework_x86-64_ubuntu22-04_gcc123_mdid.so"
+      "${SPARK_LIB}/libActivOneApi_x86-64_ubuntu22-04_gcc123_mdid.so"
+      "${SPARK_LIB}/libActivServices_x86-64_ubuntu22-04_gcc123_mdid.so"
+      "${SPARK_LIB}/libBoostAtomic_x86-64_ubuntu22-04_gcc123_mdid.so"
+      "${SPARK_LIB}/libBoostFilesystem_x86-64_ubuntu22-04_gcc123_mdid.so"
       DESTINATION "${SPARK_INSTALL_PREFIX}/bin")
   endif()
 endif()

CMakePresets.json

@@ -113,13 +113,29 @@
       "description": "Debug build compiled w/ gcc",
       "binaryDir": "${sourceDir}/build/gcc/debug",
       "cacheVariables": {
-        "CMAKE_C_COMPILER": "gcc",
-        "CMAKE_CXX_COMPILER": "g++",
+        "CMAKE_C_COMPILER": "gcc-12",
+        "CMAKE_CXX_COMPILER": "g++-12",
         "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -Wno-stringop-overread -DDEBUG",
         "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/gcc/debug/sparkd"
       }
     },
 
+    {
+      "name": "activ-tester",
+      "inherits": "sparkd-debug-gcc",
+      "displayName": "Activ Tester Debug (gcc)",
+      "description": "Activ Tester Debug build compiled w/ gcc",
+      "binaryDir": "${sourceDir}/build/${presetName}",
+      "cacheVariables": {
+        "BUILD_ACTIV_MD": {
+          "type": "BOOL",
+          "value": "ON"
+        },
+        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}/sparkd",
+        "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake"
+      }
+    },
+
     {
       "name": "sparkd-release-clang",
       "inherits": "sparkd-release",
@@ -291,6 +307,13 @@
       "inherits": "sparkd"
     },
 
+    {
+      "name": "activ-tester",
+      "configurePreset": "activ-tester",
+      "displayName": "Activ Tester Debug (gcc)",
+      "inherits": "sparkd"
+    },
+
     {
       "name": "sparkd-release-clang",
       "configurePreset": "sparkd-release-clang",

arham_spark.conf

@@ -0,0 +1,124 @@
+# Symbols {
+#   value {
+#     venues {
+#       name: "VENUE_BINANCE_US"
+#       spot: "BTC"
+#       spot: "ETH"
+#     }
+#   }
+# }
+# 
+# MarketDataConfig {
+#   value {
+#     binance_us {
+#       num_depth_levels: 5
+#       security_criteria {
+#         clauses {
+#           security_types: SECURITY_TYPE_SPOT
+#           symbols: "BTC"
+#         }
+#       }
+#     }
+#   }
+# }
+
+
+MarketDataConfig {
+  value {
+    # binance {
+    #   num_depth_levels: 5
+    #   security_criteria {
+    #     clauses {
+    #       security_types: SECURITY_TYPE_SPOT
+    #       symbols: "BTC"
+    #       symbols: "ETH"
+    #     }
+    #   }
+    #   url_rest_base: "https://testnet.binance.vision"
+    #   url_ws_base: "wss://testnet.binance.vision"
+    # }
+    # binance_usds_futures {
+    #   num_depth_levels: 5
+    #   security_criteria {
+    #     clauses {
+    #       security_types: SECURITY_TYPE_FUTURE
+    #       symbols: "BTC"
+    #       symbols: "ETH"
+    #     }
+    #   }
+    #   security_desc_formattings {
+    #     expression: "^(.+)(USD)(?=_)(.*)"
+    #     format: "$1USDT$3"
+    #   }
+    #   url_rest_base: "https://testnet.binancefuture.com"
+    #   url_ws_base: "wss://fstream.binance.com"
+    # }
+    binance_us {
+      num_depth_levels: 5
+      security_criteria {
+        clauses {
+          security_types: SECURITY_TYPE_SPOT
+          symbols: "BTC"
+          symbols: "ETH"
+        }
+      }
+      url_rest_base: "https://api.binance.us"
+      url_ws_base: "wss://stream.binance.us:9443"
+    }
+    # deribit {
+    #   num_depth_levels: 10
+    #   security_criteria {
+    #     clauses {
+    #       security_types: SECURITY_TYPE_SPOT
+    #       security_types: SECURITY_TYPE_FUTURE
+    #       security_types: SECURITY_TYPE_OPTION
+    #       security_types: SECURITY_TYPE_SPREAD
+    #       symbols: "BTC"
+    #       symbols: "ETH"
+    #     }
+    #   }
+    #   currencies: "BTC"
+    #   currencies: "USDC"
+    #   url_rest_base: "https://test.deribit.com"
+    #   url_ws_base: "wss://test.deribit.com"
+    # }
+    # okx {
+    #   num_depth_levels: 5
+    #   security_criteria {
+    #     clauses {
+    #       security_types: SECURITY_TYPE_SPOT
+    #       security_types: SECURITY_TYPE_FUTURE
+    #       security_types: SECURITY_TYPE_OPTION
+    #       symbols: "BTC"
+    #       symbols: "ETH"
+    #     }
+    #   }
+    #   url_rest_base: "https://aws.okx.com"
+    #   url_ws_base: "wss://wsaws.okx.com:8443"
+    #   is_protocol_ping_pong_enabled: true
+    #   protocol_ping_interval_ms: 60000
+    #   protocol_pong_timeout_ms: 30000
+    #   is_application_ping_pong_enabled: true
+    #   application_ping_interval_ms: 20000
+    #   application_pong_timeout_ms: 10000
+    # }
+    # recording {
+    #   num_of_seconds_to_run: 300
+    #   venue: VENUE_DERIBIT
+    # }
+    # replay {
+    #   venue: VENUE_DERIBIT
+    # }
+  }
+}
+
+Symbols {
+  value {
+    venues {
+      name: "VENUE_BINANCE_US"
+      spot: "BTC"
+      spot: "ETH"
+    }
+  }
+}
+

beeda

@@ -0,0 +1,210 @@
+cmake_minimum_required(VERSION 3.12)
+project(spark)
+set(CMAKE_CXX_STANDARD 20)
+
+if(WIN32)
+  set(platform "windows")
+else()
+  set(platform "linux")
+endif()
+
+set(
+  EXTERNALS_ROOT
+  "${CMAKE_SOURCE_DIR}/externals/${platform}"
+  CACHE
+  STRING
+  "External dependencies root folder (should contain folders: lib, bin, include")
+set(SPARK_BIN "${EXTERNALS_ROOT}/bin")
+set(SPARK_INCLUDE "${EXTERNALS_ROOT}/include")
+set(SPARK_LIB "${EXTERNALS_ROOT}/lib/$<CONFIG>")
+set(BUILD_API_SERVER OFF CACHE BOOL "Build the api server")
+set(BUILD_DOCS OFF CACHE BOOL "Build documentation")
+set(BUILD_SDK OFF CACHE BOOL "Build the sdk")
+if (WIN32)
+  set(BUILD_SPARKD OFF CACHE BOOL "Build sparkd")
+  set(BUILD_ADMIN_WEB_APP OFF CACHE BOOL "Build the Admin WebApp")
+else()
+  set(BUILD_SPARKD ON CACHE BOOL "Build sparkd")
+  set(BUILD_ADMIN_WEB_APP ON CACHE BOOL "Build the Admin WebApp")
+endif()
+
+include_directories(BEFORE SYSTEM ${SPARK_INCLUDE} ${EXTERNALS_ROOT}/include/zlib)
+link_directories(${SPARK_LIB})
+
+if(BUILD_API_SERVER)
+  add_compile_definitions(SPARK_ENABLE_API_SERVER)
+endif()
+
+if(WIN32)
+  add_compile_definitions(SHARED_MEMORY_PREFIX="spark")
+  add_compile_definitions(APP_DATA_DIR="\$\{APPDATA\}/bts/")
+  # subdirs
+  add_subdirectory(src/lib)
+else()
+  # make sparkd look for libspark.so in its directory
+  set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE)
+  set(CMAKE_INSTALL_RPATH ".")
+  set(CMAKE_INSTALL_PREFIX "${CMAKE_SOURCE_DIR}/install")
+  set(SPARK_INSTALL_PREFIX "${CMAKE_INSTALL_PREFIX}/$<CONFIG>")
+
+  if(BUILD_API_SERVER)
+    add_compile_definitions(APP_DATA_DIR="\$\{HOME\}/spark_api/")
+    add_compile_definitions(SHARED_MEMORY_PREFIX="spark_api")
+  elseif(BUILD_SPARKD)
+    add_compile_definitions(SHARED_MEMORY_PREFIX="sparkd")
+    add_compile_definitions(APP_DATA_DIR="\$\{HOME\}/sparkdata/")
+
+    # Note: All macros that begin w/ 'CCAPI' are ccapi's macros (defined and/or
+    # read by their code). We override some below.
+    add_compile_definitions(SPARK_ENABLE_CCAPI)
+
+    # This allows us to override hardcoded exchange-urls with ones read from
+    # config (i.e., resolved at runtime rather than compile-time)
+    add_compile_definitions(CCAPI_EXPOSE_INTERNAL)
+
+    # Enable ccapi services
+    add_compile_definitions(CCAPI_ENABLE_SERVICE_MARKET_DATA)
+    add_compile_definitions(CCAPI_ENABLE_SERVICE_EXECUTION_MANAGEMENT)
+    # Enable ccapi exchanges
+    add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE)
+    add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE_COIN_FUTURES)
+    add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE_US)
+    add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE_USDS_FUTURES)
+    add_compile_definitions(CCAPI_ENABLE_EXCHANGE_COINBASE)
+    add_compile_definitions(CCAPI_ENABLE_EXCHANGE_DERIBIT)
+    add_compile_definitions(CCAPI_ENABLE_EXCHANGE_OKX)
+
+    set(CCAPI_LOG_LEVEL
+      "Warn"
+      CACHE STRING "<Fatal|Error|Warn|Info|Debug|Trace> Log level for ccapi")
+    string(TOUPPER "${CCAPI_LOG_LEVEL}" _ccapi_log_level_upper)
+    add_compile_definitions("CCAPI_ENABLE_LOG_${_ccapi_log_level_upper}")
+
+    # ** Binance **
+    add_compile_definitions(CCAPI_BINANCE_URL_REST_BASE="https://api.binance.com")
+    add_compile_definitions(BINANCE_INSTRUMENTS_PATH="/api/v3/exchangeInfo")
+    add_compile_definitions(BINANCE_PRICE_STATS_PATH="/api/v3/ticker/24hr")
+
+    # ** Binance US **
+    add_compile_definitions(BINANCE_US_INSTRUMENTS_PATH="/api/v3/exchangeInfo")
+    add_compile_definitions(BINANCE_US_PRICE_STATS_PATH="/api/v3/ticker/24hr")
+
+    # ** Binance USDS Futures **
+    add_compile_definitions(CCAPI_BINANCE_USDS_FUTURES_URL_REST_BASE="https://dapi.binance.com")
+    add_compile_definitions(BINANCE_USDS_FUTURES_INSTRUMENTS_PATH="/dapi/v1/exchangeInfo")
+    add_compile_definitions(BINANCE_USDS_FUTURES_PRICE_STATS_PATH="/dapi/v1/ticker/24hr")
+
+    # ** Deribit **
+    add_compile_definitions(DERIBIT_INSTRUMENT_PATH="/api/v2/public/get_instrument")
+    add_compile_definitions(DERIBIT_INSTRUMENTS_PATH="/api/v2/public/get_instruments")
+    add_compile_definitions(DERIBIT_COMBOS_PATH="/api/v2/public/get_combos")
+    add_compile_definitions(DERIBIT_TICKER_PATH="/api/v2/public/ticker")
+    add_compile_definitions(DERIBIT_CURRENCIES_METHOD="public/get_currencies")
+    add_compile_definitions(DERIBIT_INSTRUMENT_METHOD="public/get_instrument")
+    add_compile_definitions(DERIBIT_INSTRUMENTS_METHOD="public/get_instruments")
+    add_compile_definitions(DERIBIT_COMBO_DETAILS_METHOD="public/get_combo_details")
+    add_compile_definitions(DERIBIT_COMBOS_METHOD="public/get_combos")
+
+    # ** Okx **
+    add_compile_definitions(OKX_INSTRUMENTS_PATH="/api/v5/public/instruments")
+    add_compile_definitions(OKX_UNDERLYING_PATH="/api/v5/public/underlying")
+  endif()
+
+  # subdirs
+  if (BUILD_SPARKD)
+    add_subdirectory(src/server/sparkd)
+    add_subdirectory(src/data_files)
+  endif()
+  add_subdirectory(src/lib)
+  if (BUILD_ADMIN_WEB_APP)
+    add_subdirectory(src/Spark.Admin.WebApp)
+  endif()
+  if(BUILD_DOCS)
+    add_subdirectory(doc/user)
+  endif()
+  if(BUILD_SDK)
+    add_subdirectory(src/sdk)
+  endif()
+
+  if(BUILD_SPARKD)
+    function(copy_shared_lib lib_file_name)
+      add_custom_command(OUTPUT ${CMAKE_BINARY_DIR}/src/server/sparkd/${lib_file_name}
+        COMMAND ${CMAKE_COMMAND} -E copy 
+        ${EXTERNALS_ROOT}/lib/Debug/${lib_file_name}
+        ${CMAKE_BINARY_DIR}/src/server/sparkd/${lib_file_name}
+        DEPENDS ${EXTERNALS_ROOT}/lib/Debug/${lib_file_name})
+      add_custom_target(copy_shared_lib_${lib_file_name} ALL DEPENDS ${CMAKE_BINARY_DIR}/src/server/sparkd/${lib_file_name})
+    endfunction()
+
+    add_custom_command(
+      OUTPUT ${CMAKE_BINARY_DIR}/server
+      COMMAND mkdir -p ${CMAKE_BINARY_DIR}/server
+      COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/src/server/sparkd/sparkd ${CMAKE_BINARY_DIR}/server
+      COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/data_files ${CMAKE_BINARY_DIR}/server
+      COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/src/lib/libspark.so ${CMAKE_BINARY_DIR}/server
+      COMMAND
+        ln ARGS -sf
+        ${EXTERNALS_ROOT}/lib/Release/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.16.0.0
+        ${CMAKE_BINARY_DIR}/server
+      COMMAND
+        ${CMAKE_COMMAND} -E copy
+        ${EXTERNALS_ROOT}/lib/Debug/libActivAgentApplication_x86-64_ubuntu22-04_gcc114_mdid.so
+        ${CMAKE_BINARY_DIR}/server
+      COMMAND
+        ${CMAKE_COMMAND} -E copy
+        ${EXTERNALS_ROOT}/lib/Debug/libActivBase_x86-64_ubuntu22-04_gcc114_mdid.so
+        ${CMAKE_BINARY_DIR}/server
+      COMMAND
+        ${CMAKE_COMMAND} -E copy
+        ${EXTERNALS_ROOT}/lib/Debug/libActivFramework_x86-64_ubuntu22-04_gcc114_mdid.so
+        ${CMAKE_BINARY_DIR}/server
+      COMMAND
+        ${CMAKE_COMMAND} -E copy
+        ${EXTERNALS_ROOT}/lib/Debug/libActivOneApi_x86-64_ubuntu22-04_gcc114_mdid.so
+        ${CMAKE_BINARY_DIR}/server
+      COMMAND
+        ${CMAKE_COMMAND} -E copy
+        ${EXTERNALS_ROOT}/lib/Debug/libActivDatabase_x86-64_ubuntu22-04_gcc114_mdid.so
+        ${CMAKE_BINARY_DIR}/server
+      COMMAND
+        ${CMAKE_COMMAND} -E copy
+        ${EXTERNALS_ROOT}/lib/Debug/libActivServices_x86-64_ubuntu22-04_gcc114_mdid.so
+        ${CMAKE_BINARY_DIR}/server
+      COMMAND
+        ${CMAKE_COMMAND} -E copy
+        ${EXTERNALS_ROOT}/lib/Debug/libBoostAtomic_x86-64_ubuntu22-04_gcc114_mdid.so
+        ${CMAKE_BINARY_DIR}/server
+      COMMAND
+        ${CMAKE_COMMAND} -E copy
+        ${EXTERNALS_ROOT}/lib/Debug/libBoostChrono_x86-64_ubuntu22-04_gcc114_mdid.so
+        ${CMAKE_BINARY_DIR}/server
+      COMMAND
+        ${CMAKE_COMMAND} -E copy
+        ${EXTERNALS_ROOT}/lib/Debug/libBoostFilesystem_x86-64_ubuntu22-04_gcc114_mdid.so
+        ${CMAKE_BINARY_DIR}/server
+      COMMAND
+        ${CMAKE_COMMAND} -E copy
+        ${EXTERNALS_ROOT}/lib/Debug/libBoostProgramOptions_x86-64_ubuntu22-04_gcc114_mdid.so
+        ${CMAKE_BINARY_DIR}/server
+      COMMAND ln ARGS -sf ${EXTERNALS_ROOT}/lib/Release/libACE-5.7.5.so ${CMAKE_BINARY_DIR}/server
+      COMMAND mkdir ARGS -p ${CMAKE_BINARY_DIR}/server/plugins
+      COMMENT "Creating server runtime layout"
+      DEPENDS sparkd data_files_dir)
+
+    add_custom_target(server_layout ALL DEPENDS ${CMAKE_BINARY_DIR}/server)
+
+    install(
+      DIRECTORY "${CMAKE_BINARY_DIR}/server/plugins"
+      DESTINATION "${SPARK_INSTALL_PREFIX}/bin")
+    install(
+      FILES
+      "${SPARK_LIB}/libACE-5.7.5.so"
+      "${SPARK_LIB}/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.16.0.0"
+      "${SPARK_LIB}/libActivFramework_x86-64_ubuntu22-04_gcc114_mdid.so"
+      "${SPARK_LIB}/libActivOneApi_x86-64_ubuntu22-04_gcc114_mdid.so"
+      "${SPARK_LIB}/libActivServices_x86-64_ubuntu22-04_gcc114_mdid.so"
+      "${SPARK_LIB}/libBoostAtomic_x86-64_ubuntu22-04_gcc114_mdid.so"
+      "${SPARK_LIB}/libBoostFilesystem_x86-64_ubuntu22-04_gcc114_mdid.so"
+      DESTINATION "${SPARK_INSTALL_PREFIX}/bin")
+  endif()
+endif()

cfgl

@@ -0,0 +1,33 @@
+diff.astextplain.textconv=astextplain
+filter.lfs.clean=git-lfs clean -- %f
+filter.lfs.smudge=git-lfs smudge -- %f
+filter.lfs.process=git-lfs filter-process
+filter.lfs.required=true
+http.sslbackend=openssl
+http.sslcainfo=C:/Program Files/Git/mingw64/etc/ssl/certs/ca-bundle.crt
+core.autocrlf=true
+core.fscache=true
+core.symlinks=false
+pull.rebase=false
+credential.helper=manager
+credential.https://dev.azure.com.usehttppath=true
+init.defaultbranch=master
+status.short=true
+core.repositoryformatversion=0
+core.filemode=false
+core.bare=false
+core.logallrefupdates=true
+core.symlinks=false
+core.ignorecase=true
+submodule.active=.
+remote.origin.url=git@bitbucket.org:bluetradesys/spark.git
+remote.origin.fetch=+refs/heads/*:refs/remotes/origin/*
+branch.spark_crypto.dev.remote=origin
+branch.spark_crypto.dev.merge=refs/heads/spark_crypto.dev
+submodule.docker/linux.url=git@bitbucket.org:bluetradesys/linux_docker.git
+submodule.docker/windows.url=git@bitbucket.org:bluetradesys/windows_docker.git
+submodule.externals.url=git@bitbucket.org:bluetradesys/externals.git
+branch.development.remote=origin
+branch.development.merge=refs/heads/development
+user.name=John Colvin
+user.email=john.colvin@bluetradesys.com

cfgl.wsl

@@ -0,0 +1,12 @@
+core.repositoryformatversion=0
+core.filemode=true
+core.bare=false
+core.logallrefupdates=true
+remote.origin.url=git@bitbucket.org:bluetradesys/spark.git
+remote.origin.fetch=+refs/heads/*:refs/remotes/origin/*
+branch.spark_crypto.dev.remote=origin
+branch.spark_crypto.dev.merge=refs/heads/spark_crypto.dev
+branch.development.remote=origin
+branch.development.merge=refs/heads/development
+submodule.externals.active=true
+submodule.externals.url=git@bitbucket.org:bluetradesys/externals.git

cmake/InitExecutable.cmake

@@ -0,0 +1,8 @@
+function(init_executable source_dir)
+  file(
+    GLOB_RECURSE ${PROJECT_NAME}_Sources
+    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
+    ${source_dir}/*.cpp)
+  add_executable(${PROJECT_NAME} ${${PROJECT_NAME}_Sources})
+  target_include_directories(${PROJECT_NAME} PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
+endfunction(init_executable)

ops_notes.txt

@@ -24,3 +24,6 @@
   those re-entered explicitly
 - With this release we add per user location and smp id tags. At least
   the location will have to be set through the admin UI for every user
+
+libBoostFilesystem_x86-64_ubuntu22-04_gcc123_mdid.so:    ELF 64-bit LSB shared object, x86-64, version 1 (GNU/Linux), dynamically linked, BuildID[sha1]=75427020663511112b3aadeef5023e4f368aafd4, stripped
+/usr/lib/x86_64-linux-gnu/libboost_filesystem.so.1.74.0: ELF 64-bit LSB shared object, x86-64, version 1 (SYSV), dynamically linked, BuildID[sha1]=f0cb6f85ac07ed93816bae06c584d4a1fabb0e70, stripped

src/app_activ-tester/CMakeLists.txt

@@ -0,0 +1,51 @@
+cmake_minimum_required(VERSION 3.23)
+project(activ-tester)
+include(${CMAKE_SOURCE_DIR}/cmake/InitExecutable.cmake)
+init_executable(activ-tester)
+target_link_libraries(${PROJECT_NAME} PUBLIC
+    protobuf
+    double-conversion
+    boost_system
+    boost_thread
+    boost_filesystem
+    boost_regex
+    boost_chrono
+    boost_date_time
+    boost_atomic
+    SQLiteCpp
+    sqlite3
+    curl
+    uuid
+    ssl
+    crypto
+    debug fmtd
+    optimized fmt
+    tbb
+    cryptopp
+    pugixml
+    libspark
+    lttng-ust
+    Pcap++
+    Packet++
+    Common++
+    ssh2
+    BoostAtomic_x86-64_ubuntu22-04_gcc123_mdid
+    BoostChrono_x86-64_ubuntu22-04_gcc123_mdid
+    BoostFilesystem_x86-64_ubuntu22-04_gcc123_mdid
+    BoostProgramOptions_x86-64_ubuntu22-04_gcc123_mdid
+    ActivAgentApplication_x86-64_ubuntu22-04_gcc123_mdid
+    ActivBase_x86-64_ubuntu22-04_gcc123_mdid
+    ActivDatabase_x86-64_ubuntu22-04_gcc123_mdid
+    ActivFramework_x86-64_ubuntu22-04_gcc123_mdid
+    ActivOneApi_x86-64_ubuntu22-04_gcc123_mdid
+    ActivServices_x86-64_ubuntu22-04_gcc123_mdid
+    -lrt
+    -ldl
+    -lpcap
+    -static-libgcc
+    -static-libstdc++
+    -pthread)
+
+# cli11
+find_package(CLI11 CONFIG REQUIRED)
+target_link_libraries(${PROJECT_NAME} PUBLIC CLI11::CLI11)

src/app_activ-tester/activ-tester/main.cpp

@@ -0,0 +1,75 @@
+// #include "lib/md/cme/CMEActivSession.h"
+// #include "lib/testing/catch_test_runner.h"
+// #include "lib/utils/init_spark_lib.h"
+// #include "server/sparkd/cmds/AdminReport.h"
+// #include "server/sparkd/cmds/ConfigCmd.h"
+// #include "server/sparkd/cmds/KeyCmd.h"
+// #include "server/sparkd/cmds/Md.h"
+// #include "server/sparkd/cmds/QuickTest.h"
+// #include "server/sparkd/cmds/Run.h"
+// #include "server/sparkd/cmds/Web.h"
+// #include "server/sparkd/cmds/crypto/CryptoCmd.h"
+
+#include <CLI/CLI.hpp>
+#include <functional>
+#include <iostream>
+#include <string>
+// using namespace bts;
+
+void print_args(int argc, char **argv) {
+   if (argc < 2) {
+      std::cout << "No args" << std::endl;
+      return;
+   }
+   for (int i = 1; i < argc; i++) {
+      std::cout << "arg " << i << ": " << argv[i] << std::endl;
+   }
+}
+
+int main(int argc, char **argv) {
+   print_args(argc, argv);
+   CLI::App app("Futures trading system", "sparkd");
+   app.require_subcommand();
+   // app.add_subcommand("")
+   return 0;
+}
+
+// int main(int argc, char **argv) {
+//    utils::SparkLibInit init;
+
+//    // The catch testing framework has its own command line parser so
+//    // if we are testing we just forward the entire command to catch
+//    if (argc > 1 && std::string(argv[1]) == "test")
+//       return run_tests(argc, argv);
+//    else {
+//       CLI::App app("Futures trading system", "sparkd");
+//       app.require_subcommand();
+//       app.add_subcommand(CLI::App_p(new cmds::AdminReport()));
+//       app.add_subcommand(CLI::App_p(new cmds::CryptoCmd()));
+//       app.add_subcommand(CLI::App_p(new cmds::ConfigCmd()));
+//       app.add_subcommand(CLI::App_p(new cmds::Md()));
+//       app.add_subcommand(CLI::App_p(new cmds::QuickTest()));
+//       app.add_subcommand(CLI::App_p(new cmds::Run()));
+//       app.add_subcommand(CLI::App_p(new cmds::KeyCmd()));
+//       app.add_subcommand(CLI::App_p(new cmds::Web()));
+
+//       // Activ md example
+//       struct {
+//          std::string host{"aop-ny4-replay-1w.activfinancial.com"};
+//          std::string user{"bltdsy-replay"};
+//          std::string password{"redhawk0281"};
+//          CLI::App *cmd;
+//       } activ;
+//       activ.cmd = app.add_subcommand("activ-md", "Run Activ md demo");
+//       activ.cmd->add_option("--host", activ.host, "Activ host (server)");
+//       activ.cmd->add_option("--user", activ.user, "Activ login user");
+//       activ.cmd->add_option("--pass", activ.password, "Activ login password");
+//       activ.cmd->callback([&activ]() {
+//          md::cme::CMEActivSession::run(activ.host, activ.user, activ.password);
+//       });
+
+//       app.add_subcommand(
+//           "test", "Run unit test(s)"); // so this shows up in help
+//       CLI11_PARSE(app, argc, argv);
+//    }
+// }

src/lib/CMakeLists.txt

@@ -55,7 +55,7 @@ target_include_directories(
 
 get_link_libraries(_link_libs)
 print_list(_link_libs VERBOSE)
-target_link_libraries(${PROJECT_NAME} ${_link_libs})
+target_link_libraries(${PROJECT_NAME} PUBLIC ${_link_libs})
 
 # Copy binaries to Dotnet build dirs (for csprojs that depend on libspark)
 if (DEFINED DOTNET_BINARY_DIR)

src/lib/cmake/GetLinuxServerLinkLibraries.cmake

@@ -25,6 +25,16 @@ function(get_link_libraries dest)
     Packet++
     Common++
     ssh2
+    BoostAtomic_x86-64_ubuntu22-04_gcc123_mdid
+    BoostChrono_x86-64_ubuntu22-04_gcc123_mdid
+    BoostFilesystem_x86-64_ubuntu22-04_gcc123_mdid
+    BoostProgramOptions_x86-64_ubuntu22-04_gcc123_mdid
+    ActivAgentApplication_x86-64_ubuntu22-04_gcc123_mdid
+    ActivBase_x86-64_ubuntu22-04_gcc123_mdid
+    ActivDatabase_x86-64_ubuntu22-04_gcc123_mdid
+    ActivFramework_x86-64_ubuntu22-04_gcc123_mdid
+    ActivOneApi_x86-64_ubuntu22-04_gcc123_mdid
+    ActivServices_x86-64_ubuntu22-04_gcc123_mdid
     -lrt
     -ldl
     -lpcap

src/lib/datastreams/Tables/tests/stream_tests.cpp

@@ -1,11 +1,11 @@
 
 #include "catch/catch.hpp"
-#include "lib/api/service_helpers/AdminServiceUtils.h"
-#include "lib/api/services/AdminService.server.pb.h"
-#include "lib/config/config.h"
+// #include "lib/api/service_helpers/AdminServiceUtils.h"
+// #include "lib/api/services/AdminService.server.pb.h"
+// #include "lib/config/config.h"
 #include "lib/datastreams/Tables/StreamData.h"
-#include "lib/utils/bts_chrono.h"
-#include "lib/utils/helper_macros.h"
+// #include "lib/utils/bts_chrono.h"
+// #include "lib/utils/helper_macros.h"
 #include <memory>
 
 using namespace bts::streams;

src/lib/io/JsonReader.h

@@ -37,6 +37,8 @@ class JsonReader {
 
    template <class T>
    bool read(const std::string &name, T &result) const;
+   template <class T>
+   bool try_read(const std::string &name, T &result) const;
 
    template <class T>
    bool read_one_or_more(
@@ -84,6 +86,14 @@ class JsonReader {
    template <class T>
    bool read(T &result) const;
 
+   /**
+    * Reads a json field into \p result
+    *
+    * @note Doesn't log error if read op fails (unlike `read`)
+    */
+   template <class T>
+   bool try_read(T &result) const;
+
    template <class T>
    T read() const;
 
@@ -135,6 +145,23 @@ inline bool JsonReader::read(const std::string &name, T &result) const {
    return true;
 }
 
+template <class T>
+inline bool JsonReader::try_read(const std::string &name, T &result) const {
+   if (!json_ || !json_->is_object()) {
+      return false;
+   }
+   auto it = json_->find(name);
+   if (it == json_->end()) {
+      return false;
+   }
+   try {
+      it->get_to(result);
+   } catch (...) {
+      return false;
+   }
+   return true;
+}
+
 template <class T>
 inline bool JsonReader::read_one_or_more(
     const std::string &name, std::vector<T> &result, bool append) const {
@@ -219,6 +246,18 @@ inline bool JsonReader::read(T &result) const {
    }
 }
 
+template <class T>
+inline bool JsonReader::try_read(T &result) const {
+   try {
+      json_->get_to(result);
+      return true;
+   } catch (...) {
+      const auto j_str = json_->dump(2);
+      LOG(md, Error, "Exception parsing json:\n{}", j_str);
+      return false;
+   }
+}
+
 template <class T>
 inline T JsonReader::read() const {
    T result;
@@ -262,7 +301,7 @@ inline T JsonReader::read(const std::string &name) const {
 template <class T>
 inline bool JsonReader::read_or(
     const std::string &name, T &result, T default_value) const {
-   if (read(name, result)) {
+   if (try_read(name, result)) {
       return true;
    }
    result = default_value;

src/lib/md/cme/CMEActivSession.cpp

@@ -0,0 +1,52 @@
+#include "CMEActivSession.h"
+#include "lib/logging/logging.h"
+#include "lib/md/cme/CMEActivSessionHandler.h"
+
+#include <ActivMiddleware/ActivBase/Exception.h>
+#include <ActivMiddleware/ActivBase/FieldTypes/ActivFieldTypes.h>
+#include <ActivMiddleware/ActivBase/FieldTypes/FieldMap.h>
+#include <ActivOneApi/Constants/FieldIds.h>
+#include <ActivOneApi/Session.h>
+#include <exception>
+
+namespace bts::md::cme {
+
+using Activ::FieldMap;
+using Activ::OneApi::Session;
+using std::string;
+
+void CMEActivSession::run(const string &host, const string &user, const string &password) {
+   try {
+      // Session callback handler.
+      CMEActivSessionHandler sessionHandler;
+
+      FieldMap fields;
+      // Construct the Session with a default configuration passing in our sessionHandler.
+      Session session(fields, sessionHandler);
+
+      FieldMap connectParameters;
+      
+      connectParameters.Set(Activ::OneApi::FID_HOST, host.c_str());
+      connectParameters.Set(Activ::OneApi::FID_USER_ID, user.c_str());
+      connectParameters.Set(Activ::OneApi::FID_PASSWORD, password.c_str());
+
+      // Connect synchronously with a 10 second timeout (will throw on failure).
+      session.Connect(connectParameters, 10000);
+
+      // Alternatively an asynchronous connection can be used.
+      // Connection completion is notified via ISessionHandler::OnSessionConnect().
+      // session.AsyncConnect(connectParameters);
+
+      // Run the internal session message loop.
+      session.Run();
+   } catch (const Activ::Exception &ex) {
+      std::cerr << "Error running CMEActivSession (activ): " << ex.GetMessage()
+                << std::endl;
+   } catch (const std::exception &ex) {
+      std::cerr << "Error running CMEActivSession: " << ex.what() << std::endl;
+      // LOG(md, Error, "Error running CMEActivSession: {}",
+      //     std::string{ex.what()});
+   }
+}
+
+} // namespace bts::md::cme

src/lib/md/cme/CMEActivSession.h

@@ -0,0 +1,15 @@
+#pragma once
+
+#include <string>
+
+namespace bts::md::cme {
+
+class CMEActivSession {
+ public:
+   static void run(
+       const std::string &host,
+       const std::string &user,
+       const std::string &password);
+};
+
+} // namespace bts::md::cme

src/lib/md/cme/CMEActivSessionHandler.cpp

@@ -0,0 +1,32 @@
+#include "CMEActivSessionHandler.h"
+#include <ActivMiddleware/ActivBase/ActivBase.h>
+#include <ActivOneApi/Session.h>
+
+namespace bts::md::cme {
+
+using Activ::LogType;
+using Activ::StatusCode;
+using Activ::String;
+using Activ::OneApi::Session;
+
+void CMEActivSessionHandler::OnSessionLogMessage(
+    Session & /*session*/, const LogType log_type, const String &message) {
+   std::cout << Activ::LogTypeToString(log_type) << ": " << message
+             << std::endl;
+}
+
+void CMEActivSessionHandler::OnSessionConnect(Session & /*session*/) {
+   std::cout << "connected" << std::endl;
+}
+
+void CMEActivSessionHandler::OnSessionDisconnect(Session & /*session*/) {
+   std::cout << "disconnected" << std::endl;
+}
+
+void CMEActivSessionHandler::OnSessionError(
+    Session & /*session*/, const StatusCode statusCode) {
+   std::string err{Activ::StatusCodeToString(statusCode)};
+   std::cerr << "CMEActivSessionHandler::OnSessionError. " << err << std::endl;
+}
+
+} // namespace bts::md::cme

src/lib/md/cme/CMEActivSessionHandler.h

@@ -0,0 +1,24 @@
+#pragma once
+
+#include <ActivMiddleware/ActivBase/ActivBase.h>
+#include <ActivOneApi/ISessionHandler.h>
+
+namespace bts::md::cme {
+
+class CMEActivSessionHandler : public Activ::OneApi::ISessionHandler {
+ public:
+   static void run();
+
+   void OnSessionLogMessage(
+       Activ::OneApi::Session &session,
+       const Activ::LogType log_type,
+       const Activ::String &message) override;
+
+   void OnSessionConnect(Activ::OneApi::Session &session) override;
+   void OnSessionDisconnect(Activ::OneApi::Session &session) override;
+   void OnSessionError(
+       Activ::OneApi::Session &session,
+       const Activ::StatusCode statusCode) override;
+};
+
+} // namespace bts::md::cme

src/lib/md/cme/MDP3Utils.cpp

@@ -12,6 +12,7 @@
 #include "lib/utils/datetime.h"
 #include "lib/utils/download.h"
 
+#include <ActivOneApi/OneApi.h>
 #include <boost/filesystem.hpp>
 #include <iostream>
 #include <mutex>

src/lib/md/platforms/IMDPlatform.h

@@ -0,0 +1,13 @@
+#pragma once
+
+namespace bts {
+namespace md {
+
+class IMDPlatform {
+ public:
+   virtual void send_heartbeats();
+   virtual ~IMDPlatform() {}
+};
+
+} // namespace md
+} // namespace bts

src/lib/md/platforms/emumba/DeribitDemoConfiguration.cpp

@@ -0,0 +1,204 @@
+#include "DeribitDemoConfiguration.h"
+#include "lib/io/JsonReader.h"
+#include "lib/logging/logging.h"
+#include "lib/utils/FileUtils.h"
+#include "rapidjson/istreamwrapper.h"
+#include <emumba-deribit-connector/deribit_utils.h>
+#include <exception>
+#include <fstream>
+#include <list>
+#include <nlohmann/detail/macro_scope.hpp>
+#include <nlohmann/json.hpp>
+#include <optional>
+
+namespace emumba::deribit_connector::configuration {
+
+NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(
+    emumba::deribit_connector::configuration::subscription_config,
+    public_book,
+    public_index,
+    public_markprice,
+    public_quote,
+    public_ticker,
+    public_trade,
+    private_portfolio,
+    use_raw_interval,
+    book_depth,
+    requests_limit);
+// NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(
+//     emumba::deribit_connector::configuration::symbol_config,
+//     symbol_name,
+//     currency_type,
+//     cred_index,
+//     interested_instruments,
+//     interested_subscriptions,
+//     interested_in_future,
+//     interested_in_future_combo,
+//     interested_in_option,
+//     interested_in_option_combo);
+
+inline void to_json(
+    nlohmann::json &j,
+    const emumba ::deribit_connector::configuration::symbol_config &s) {
+   j["symbol_name"] = s.symbol_name;
+   j["currency_type"] = s.currency_type;
+   j["cred_index"] = s.cred_index;
+   j["interested_instruments"] = s.interested_instruments;
+   j["interested_subscriptions"] = s.interested_subscriptions;
+   j["interested_in_future"] = s.interested_in_future;
+   j["interested_in_future_combo"] = s.interested_in_future_combo;
+   j["interested_in_option"] = s.interested_in_option;
+   j["interested_in_option_combo"] = s.interested_in_option_combo;
+}
+
+inline void from_json(
+    const nlohmann::json &j,
+    emumba::deribit_connector::configuration::symbol_config &s) {
+   j.at("symbol_name").get_to(s.symbol_name);
+   j.at("currency_type").get_to(s.currency_type);
+   j.at("cred_index").get_to(s.cred_index);
+   j.at("interested_instruments").get_to(s.interested_instruments);
+   j.at("interested_subscriptions").get_to(s.interested_subscriptions);
+   j.at("interested_in_future").get_to(s.interested_in_future);
+   j.at("interested_in_future_combo").get_to(s.interested_in_future_combo);
+   j.at("interested_in_option").get_to(s.interested_in_option);
+   j.at("interested_in_option_combo").get_to(s.interested_in_option_combo);
+}
+
+inline void to_json(nlohmann::json &j, const logging_config &l) {
+   j["severity"] = l.severity;
+   j["console"] = l.is_console_enabled;
+   j["file"] = l.is_file_enabled;
+   j["queuesize"] = l.queue_size;
+}
+
+inline void from_json(const nlohmann::json &j, logging_config &l) {
+   bts::io::JsonReader jr{&j};
+   jr.read("severity", l.severity);
+   jr.read("console", l.is_console_enabled);
+   jr.read("file", l.is_file_enabled);
+   jr.read("queuesize", l.queue_size);
+}
+
+NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(
+    reconnection_config, retry_count, retry_interval, reception_timeout);
+NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(order_tracker, uuid, start_id);
+
+inline void to_json(nlohmann::json &j, const trading_session_config &t) {
+   j["orders_tracker"] = t.orders_tracker;
+   j["core"] = t.core;
+   j["cancel_on_disconnect"] = t.cancel_on_disconnect;
+   j["session"] = nlohmann::json{{"heartbeat_interval", t.heartbeat_interval}};
+}
+
+inline void from_json(const nlohmann::json &j, trading_session_config &ts) {
+   j.at("orders_tracker").get_to(ts.orders_tracker);
+   j.at("core").get_to(ts.core);
+   j.at("cancel_on_disconnect").get_to(ts.cancel_on_disconnect);
+   j.at("session").at("heartbeat_interval").get_to(ts.heartbeat_interval);
+}
+
+} // namespace emumba::deribit_connector::configuration
+
+namespace bts::cmds {
+
+static constexpr const char *test_site() {
+   return "wss://test.deribit.com/ws/api/v2";
+}
+
+static std::string read_url(nlohmann::json &doc) {
+   io::JsonReader jr{&doc};
+   std::string url;
+   jr.try_read("uri", url);
+   if (url.empty()) {
+      return test_site();
+   }
+   return url;
+}
+
+template <class Obj>
+static void read_instruments(
+    Obj &obj, std::unordered_set<std::string> &instruments) {
+   const char *instruments_field = "instruments";
+   bts_assert(obj.HasMember(instruments_field));
+   for (auto &instrument : obj[instruments_field].GetArray()) {
+      instruments.insert(instrument.GetString());
+   }
+}
+
+static std::list<emumba::deribit_connector::configuration::symbol_config>
+read_symbol_config(rapidjson::Document &doc) {
+   using RJR = RapidJsonReader;
+
+   std::list<emumba::deribit_connector::configuration::symbol_config> syms_out;
+   for (auto &sym_in : doc["symbol"].GetArray()) {
+      auto &sym = syms_out.emplace_back();
+      sym.symbol_name = RJR::get_string(sym_in, "id");
+      sym.interested_in_future = RJR::get_bool(sym_in, "interested_in_future");
+      sym.interested_in_option = RJR::get_bool(sym_in, "interested_in_option");
+      sym.currency_type = RJR::get_string(sym_in, "currency_type", "usd");
+      sym.cred_index = 1;
+
+      read_instruments(sym_in, sym.interested_instruments);
+      auto chan_in = sym_in["channel"].GetObject();
+
+      auto &subs = sym.interested_subscriptions;
+      subs.requests_limit = RJR::get_uint(sym_in, "requests_limit", 20u);
+      subs.book_depth = RJR::get_uint(sym_in, "book_depth", 5u);
+      subs.public_book = RJR::get_bool(chan_in, "PUBLIC_BOOK");
+      subs.public_quote = RJR::get_bool(chan_in, "PUBLIC_QUOTE");
+      subs.public_trade = RJR::get_bool(chan_in, "PUBLIC_TRADE");
+      subs.public_ticker = RJR::get_bool(chan_in, "PUBLIC_TICKER");
+      subs.public_markprice = RJR::get_bool(chan_in, "PUBLIC_MARKPRICE");
+      subs.public_index = RJR::get_bool(chan_in, "PUBLIC_INDEX");
+      // subs.private_orders = RJR::get_bool(chan_in, "PRIVATE_ORDERS");
+      // subs.private_trades = RJR::get_bool(chan_in, "PRIVATE_TRADES");
+      subs.private_portfolio = RJR::get_bool(chan_in, "PRIVATE_PORTFOLIO");
+      subs.use_raw_interval = RJR::get_bool(sym_in, "raw_enable", true);
+   }
+   return syms_out;
+}
+
+static std::
+    map<uint, emumba::deribit_connector::configuration::credential_config>
+    read_credential_config(rapidjson::Document &doc) {
+   using RJR = RapidJsonReader;
+   using namespace emumba::deribit_connector;
+
+   std::map<uint, configuration::credential_config> rv;
+   for (auto &cred_in : doc["credentials"].GetArray()) {
+      configuration::credential_config rv_i;
+      rv_i.client_name = RJR::get_string(cred_in, "client_name");
+      rv_i.client_id = RJR::get_string(cred_in, "id");
+      rv_i.index = RJR::get_uint(cred_in, "subaccount_index");
+      rv_i.secret_key = RJR::get_string(cred_in, "secret_key");
+      rv.insert(
+          std::pair<uint, configuration::credential_config>(rv_i.index, rv_i));
+   }
+   return rv;
+}
+
+std::optional<DeribitDemoConfiguration> DeribitDemoConfiguration::read_file(
+    const std::string &config_path) {
+   nlohmann::json conf_j;
+   try {
+   conf_j =
+       nlohmann::json::parse(bts::spartan::FileUtils::read_file(config_path));
+   } catch (const std::exception &ex) {
+      LOG(md, Error, "Error in parsing config from {}: {}", config_path,
+          static_cast<int>(conf_j.GetParseError()));
+      return std::nullopt;
+   }
+
+   DeribitDemoConfiguration conf;
+   io::JsonReader jr{&conf_j};
+   jr.try_read("app_logging", conf.logging_configuration);
+   conf.logging_configuration = read_app_logging(doc);
+   conf.deribit_url = read_url(conf_j);
+   conf.trading_config = read_trading_session_config(doc);
+   conf.symbol_config_list = read_symbol_config(doc);
+   conf.credential_map = read_credential_config(doc);
+   return conf;
+}
+
+} // namespace bts::cmds

src/lib/md/platforms/emumba/DeribitDemoConfiguration.h

@@ -0,0 +1,54 @@
+#pragma once
+
+#include "lib/core/FMT_IMPL.h"
+
+#include <emumba-deribit-connector/deribit_utils.h>
+#include <iostream>
+#include <list>
+#include <map>
+#include <optional>
+#include <sstream>
+#include <string>
+
+namespace bts::cmds {
+
+struct DeribitDemoConfiguration {
+   std::string deribit_url;
+   std::list<emumba::deribit_connector::configuration::symbol_config>
+       symbol_config_list;
+   emumba::deribit_connector::configuration::logging_config
+       logging_configuration;
+   emumba::deribit_connector::configuration::trading_session_config
+       trading_config;
+   std::map<uint, emumba::deribit_connector::configuration::credential_config>
+       credential_map;
+
+   static std::optional<DeribitDemoConfiguration> read_file(
+       const std::string &path);
+};
+
+inline std::ostream &operator<<(
+    std::ostream &out,
+    emumba::deribit_connector::configuration::credential_config const &c) {
+   out << "index: " << c.index << " client_name: " << c.client_name
+       << " client_id: " << c.client_id << " secret_key: " << c.secret_key;
+   return out;
+}
+
+inline std::ostream &operator<<(
+    std::ostream &out, DeribitDemoConfiguration const &c) {
+   out << "deribit_url: " << c.deribit_url
+       << " symbols: " << c.symbol_config_list.size()
+       << " core: " << c.trading_config.core
+       << " heartbeat_interval: " << c.trading_config.heartbeat_interval
+       << " cancel_on_disconnect: " << c.trading_config.cancel_on_disconnect
+       << " credential_map: " << c.credential_map.size();
+   for (const auto &p : c.credential_map) {
+      out << " " << p.second;
+   }
+   return out;
+}
+
+} // namespace bts::cmds
+
+FMT_IMPL_OSTREAM(bts::cmds::DeribitDemoConfiguration);

src/lib/md/platforms/emumba/EMTradingSession.h

@@ -0,0 +1,16 @@
+#pragma once
+
+#include "lib/md/platforms/emumba/DeribitDemoConfiguration.h"
+#include <emumba-deribit-connector/deribit_handler.h>
+#include <emumba-deribit-connector/deribit_listener.h>
+#include <emumba-deribit-connector/trading_session_interface.h>
+
+namespace bts::orderman {
+
+struct EMTradingSession {
+   emumba::deribit_connector::configuration::credential_config credential;
+   std::shared_ptr<emumba::deribit_connector::trading_session_interface>
+       trading_interface;
+};
+
+} // namespace bts::orderman

src/lib/md/platforms/emumba/EmumbaMain.h

@@ -0,0 +1,21 @@
+#pragma once
+
+#include "lib/md/platforms/IMDPlatform.h"
+#include <memory>
+
+namespace bts::md {
+
+class EmumbaMain : public IMDPlatform {
+ public:
+   EmumbaMain();
+   virtual ~EmumbaMain();
+   // IMDPlatform implementation
+   void send_heartbeats() override;
+   void run();
+
+ private:
+   struct PImpl;
+   std::unique_ptr<PImpl> me;
+};
+
+} // namespace bts::md

src/server/sparkd/cmds/crypto/DeribitDemoConfiguration.cpp

@@ -1,160 +0,0 @@
-#include "DeribitDemoConfiguration.h"
-#include "RapidJsonReader.h"
-#include "lib/logging/logging.h"
-#include "rapidjson/istreamwrapper.h"
-#include <emumba-deribit-connector/deribit_utils.h>
-#include <fstream>
-#include <list>
-#include <optional>
-#include <rapidjson/document.h>
-
-namespace bts::cmds {
-
-static constexpr const char *test_site() {
-   return "wss://test.deribit.com/ws/api/v2";
-}
-
-static emumba::deribit_connector::configuration::logging_config
-read_app_logging(rapidjson::Document &doc) {
-   emumba::deribit_connector::configuration::logging_config lc;
-   if (!doc.HasMember("app_logging")) {
-      return lc;
-   }
-   auto &logging_in = doc["app_logging"];
-   lc.severity = logging_in["severity"].GetString();
-   lc.is_console_enabled = logging_in["console"].GetBool();
-   lc.is_file_enabled = logging_in["file"].GetBool();
-   if (logging_in.HasMember("queuesize")) {
-      lc.queue_size = logging_in["queuesize"].GetInt();
-   }
-   return lc;
-}
-
-static std::string read_url(rapidjson::Document &doc) {
-   if (!doc.HasMember("uri")) {
-      return test_site();
-   }
-   std::string deribit_url = doc["uri"].GetString();
-   if (deribit_url == "") {
-      return test_site();
-   }
-   return deribit_url;
-}
-
-static emumba::deribit_connector::configuration::trading_session_config
-read_trading_session_config(typename rapidjson::Document &doc) {
-   using namespace emumba::deribit_connector;
-   // order_tracker
-   configuration::order_tracker order_tracker;
-   order_tracker.uuid = doc["uuid"].GetString();
-   order_tracker.start_id = doc["start_id"].GetUint();
-   // reconnection_config
-   configuration::reconnection_config rec_config;
-   rec_config.retry_count =
-       doc["reconnection"].GetObject()["retry_count"].GetInt();
-   rec_config.retry_interval =
-       doc["reconnection"].GetObject()["retry_interval"].GetInt();
-   rec_config.reception_timeout =
-       doc["reconnection"].GetObject()["reception_timeout"].GetInt();
-
-   emumba::deribit_connector::configuration::trading_session_config tsc;
-   tsc.orders_tracker = order_tracker;
-   tsc.core = doc["core"].GetUint();
-   // TODO
-   // auto &creds = doc["credentials"];
-   // TODO
-   // tsc.client_name = creds["client_name"].GetString();
-   // tsc.client_id = creds["id"].GetString();
-   // tsc.secret_key = creds["secret_key"].GetString();
-   tsc.heartbeat_interval =
-       doc["session"].GetObject()["heartbeat_interval"].GetInt();
-   tsc.cancel_on_disconnect = doc["cancel_on_disconnect"].GetBool();
-   tsc.reconnect_config = rec_config;
-   return tsc;
-}
-
-template <class Obj>
-static void read_instruments(
-    Obj &obj, std::unordered_set<std::string> &instruments) {
-   const char *instruments_field = "instruments";
-   bts_assert(obj.HasMember(instruments_field));
-   for (auto &instrument : obj[instruments_field].GetArray()) {
-      instruments.insert(instrument.GetString());
-   }
-}
-
-static std::list<emumba::deribit_connector::configuration::symbol_config>
-read_symbol_config(rapidjson::Document &doc) {
-   using RJR = RapidJsonReader;
-
-   std::list<emumba::deribit_connector::configuration::symbol_config> syms_out;
-   for (auto &sym_in : doc["symbol"].GetArray()) {
-      auto &sym = syms_out.emplace_back();
-      sym.symbol_name = RJR::get_string(sym_in, "id");
-      sym.interested_in_future = RJR::get_bool(sym_in, "interested_in_future");
-      sym.interested_in_option = RJR::get_bool(sym_in, "interested_in_option");
-      sym.currency_type = RJR::get_string(sym_in, "currency_type", "usd");
-      sym.cred_index = 1;
-
-      read_instruments(sym_in, sym.interested_instruments);
-      auto chan_in = sym_in["channel"].GetObject();
-
-      auto &subs = sym.interested_subscriptions;
-      subs.requests_limit = RJR::get_uint(sym_in, "requests_limit", 20u);
-      subs.book_depth = RJR::get_uint(sym_in, "book_depth", 5u);
-      subs.public_book = RJR::get_bool(chan_in, "PUBLIC_BOOK");
-      subs.public_quote = RJR::get_bool(chan_in, "PUBLIC_QUOTE");
-      subs.public_trade = RJR::get_bool(chan_in, "PUBLIC_TRADE");
-      subs.public_ticker = RJR::get_bool(chan_in, "PUBLIC_TICKER");
-      subs.public_markprice = RJR::get_bool(chan_in, "PUBLIC_MARKPRICE");
-      subs.public_index = RJR::get_bool(chan_in, "PUBLIC_INDEX");
-      // subs.private_orders = RJR::get_bool(chan_in, "PRIVATE_ORDERS");
-      // subs.private_trades = RJR::get_bool(chan_in, "PRIVATE_TRADES");
-      subs.private_portfolio = RJR::get_bool(chan_in, "PRIVATE_PORTFOLIO");
-      subs.use_raw_interval = RJR::get_bool(sym_in, "raw_enable", true);
-   }
-   return syms_out;
-}
-
-static std::
-    map<uint, emumba::deribit_connector::configuration::credential_config>
-    read_credential_config(rapidjson::Document &doc) {
-   using RJR = RapidJsonReader;
-   using namespace emumba::deribit_connector;
-
-   std::map<uint, configuration::credential_config> rv;
-   for (auto &cred_in : doc["credentials"].GetArray()) {
-      configuration::credential_config rv_i;
-      rv_i.client_name = RJR::get_string(cred_in, "client_name");
-      rv_i.client_id = RJR::get_string(cred_in, "id");
-      rv_i.index = RJR::get_uint(cred_in, "subaccount_index");
-      rv_i.secret_key = RJR::get_string(cred_in, "secret_key");
-      rv.insert(
-          std::pair<uint, configuration::credential_config>(rv_i.index, rv_i));
-   }
-   return rv;
-}
-
-std::optional<DeribitDemoConfiguration> DeribitDemoConfiguration::read_file(
-    const std::string &config_path) {
-   std::ifstream configFile(config_path);
-   rapidjson::IStreamWrapper isw{configFile};
-
-   rapidjson::Document doc{};
-   auto &res = doc.ParseStream(isw);
-   if (res.HasParseError()) {
-      LOG(md, Error, "Error in parsing config from {}: {}", config_path,
-          static_cast<int>(res.GetParseError()));
-      return std::nullopt;
-   }
-
-   DeribitDemoConfiguration conf;
-   conf.logging_configuration = read_app_logging(doc);
-   conf.deribit_url = read_url(doc);
-   conf.trading_config = read_trading_session_config(doc);
-   conf.symbol_config_list = read_symbol_config(doc);
-   conf.credential_map = read_credential_config(doc);
-   return conf;
-}
-
-} // namespace bts::cmds

src/server/sparkd/cmds/crypto/DeribitDemoConfiguration.h

@@ -1,54 +1,3 @@
 #pragma once
 
-#include "lib/core/FMT_IMPL.h"
-
-#include <emumba-deribit-connector/deribit_utils.h>
-#include <iostream>
-#include <list>
-#include <map>
-#include <optional>
-#include <sstream>
-#include <string>
-
-namespace bts::cmds {
-
-struct DeribitDemoConfiguration {
-   std::string deribit_url;
-   std::list<emumba::deribit_connector::configuration::symbol_config>
-       symbol_config_list;
-   emumba::deribit_connector::configuration::logging_config
-       logging_configuration;
-   emumba::deribit_connector::configuration::trading_session_config
-       trading_config;
-   std::map<uint, emumba::deribit_connector::configuration::credential_config>
-       credential_map;
-
-   static std::optional<DeribitDemoConfiguration> read_file(
-       const std::string &path);
-};
-
-inline std::ostream &operator<<(
-    std::ostream &out,
-    emumba::deribit_connector::configuration::credential_config const &c) {
-   out << "index: " << c.index << " client_name: " << c.client_name
-       << " client_id: " << c.client_id << " secret_key: " << c.secret_key;
-   return out;
-}
-
-inline std::ostream &operator<<(
-    std::ostream &out, DeribitDemoConfiguration const &c) {
-   out << "deribit_url: " << c.deribit_url
-       << " symbols: " << c.symbol_config_list.size()
-       << " core: " << c.trading_config.core
-       << " heartbeat_interval: " << c.trading_config.heartbeat_interval
-       << " cancel_on_disconnect: " << c.trading_config.cancel_on_disconnect
-       << " credential_map: " << c.credential_map.size();
-   for (const auto &p : c.credential_map) {
-      out << " " << p.second;
-   }
-   return out;
-}
-
-} // namespace bts::cmds
-
-FMT_IMPL_OSTREAM(bts::cmds::DeribitDemoConfiguration);
+#include "lib/md/platforms/emumba/DeribitDemoConfiguration.h"

src/server/sparkd/main.cpp

@@ -1,37 +1,71 @@
-#include "lib/testing/catch_test_runner.h"
-#include "lib/utils/init_spark_lib.h"
-#include "server/sparkd/cmds/AdminReport.h"
-#include "server/sparkd/cmds/ConfigCmd.h"
-#include "server/sparkd/cmds/KeyCmd.h"
-#include "server/sparkd/cmds/Md.h"
-#include "server/sparkd/cmds/QuickTest.h"
-#include "server/sparkd/cmds/Run.h"
-#include "server/sparkd/cmds/Web.h"
-#include "server/sparkd/cmds/crypto/CryptoCmd.h"
-#include <CLI11/CLI11.hpp>
+// #include "lib/md/cme/CMEActivSession.h"
+// #include "lib/testing/catch_test_runner.h"
+// #include "lib/utils/init_spark_lib.h"
+// #include "server/sparkd/cmds/AdminReport.h"
+// #include "server/sparkd/cmds/ConfigCmd.h"
+// #include "server/sparkd/cmds/KeyCmd.h"
+// #include "server/sparkd/cmds/Md.h"
+// #include "server/sparkd/cmds/QuickTest.h"
+// #include "server/sparkd/cmds/Run.h"
+// #include "server/sparkd/cmds/Web.h"
+// #include "server/sparkd/cmds/crypto/CryptoCmd.h"
+// #include <CLI11/CLI11.hpp>
 
-using namespace bts;
+#include <iostream>
+#include <string>
+// using namespace bts;
 
-int main(int argc, char **argv) {
-   utils::SparkLibInit init;
-
-   // The catch testing framework has its own command line parser so
-   // if we are testing we just forward the entire command to catch
-   if (argc > 1 && std::string(argv[1]) == "test")
-      return run_tests(argc, argv);
-   else {
-      CLI::App app("Futures trading system", "sparkd");
-      app.require_subcommand();
-      app.add_subcommand(CLI::App_p(new cmds::AdminReport()));
-      app.add_subcommand(CLI::App_p(new cmds::CryptoCmd()));
-      app.add_subcommand(CLI::App_p(new cmds::ConfigCmd()));
-      app.add_subcommand(CLI::App_p(new cmds::Md()));
-      app.add_subcommand(CLI::App_p(new cmds::QuickTest()));
-      app.add_subcommand(CLI::App_p(new cmds::Run()));
-      app.add_subcommand(CLI::App_p(new cmds::KeyCmd()));
-      app.add_subcommand(CLI::App_p(new cmds::Web()));
-      app.add_subcommand(
-          "test", "Run unit test(s)"); // so this shows up in help
-      CLI11_PARSE(app, argc, argv);
+void print_args(int argc, char **argv) {
+   if (argc < 2) {
+      std::cout << "No args" << std::endl;
+      return;
+   }
+   for (int i = 1; i < argc; i++) {
+      std::cout << "arg " << i << ": " << argv[i] << std::endl;
    }
 }
+
+int main(int argc, char **argv) {
+   print_args(argc, argv);
+   return 0;
+}
+
+// int main(int argc, char **argv) {
+//    utils::SparkLibInit init;
+
+//    // The catch testing framework has its own command line parser so
+//    // if we are testing we just forward the entire command to catch
+//    if (argc > 1 && std::string(argv[1]) == "test")
+//       return run_tests(argc, argv);
+//    else {
+//       CLI::App app("Futures trading system", "sparkd");
+//       app.require_subcommand();
+//       app.add_subcommand(CLI::App_p(new cmds::AdminReport()));
+//       app.add_subcommand(CLI::App_p(new cmds::CryptoCmd()));
+//       app.add_subcommand(CLI::App_p(new cmds::ConfigCmd()));
+//       app.add_subcommand(CLI::App_p(new cmds::Md()));
+//       app.add_subcommand(CLI::App_p(new cmds::QuickTest()));
+//       app.add_subcommand(CLI::App_p(new cmds::Run()));
+//       app.add_subcommand(CLI::App_p(new cmds::KeyCmd()));
+//       app.add_subcommand(CLI::App_p(new cmds::Web()));
+
+//       // Activ md example
+//       struct {
+//          std::string host{"aop-ny4-replay-1w.activfinancial.com"};
+//          std::string user{"bltdsy-replay"};
+//          std::string password{"redhawk0281"};
+//          CLI::App *cmd;
+//       } activ;
+//       activ.cmd = app.add_subcommand("activ-md", "Run Activ md demo");
+//       activ.cmd->add_option("--host", activ.host, "Activ host (server)");
+//       activ.cmd->add_option("--user", activ.user, "Activ login user");
+//       activ.cmd->add_option("--pass", activ.password, "Activ login password");
+//       activ.cmd->callback([&activ]() {
+//          md::cme::CMEActivSession::run(activ.host, activ.user, activ.password);
+//       });
+
+//       app.add_subcommand(
+//           "test", "Run unit test(s)"); // so this shows up in help
+//       CLI11_PARSE(app, argc, argv);
+//    }
+// }

src/server/sparkd/md/platforms/IMDPlatform.h

@@ -1,13 +1,3 @@
 #pragma once
 
-namespace bts {
-namespace md {
-
-class IMDPlatform {
- public:
-   virtual void send_heartbeats();
-   virtual ~IMDPlatform() {}
-};
-
-} // namespace md
-} // namespace bts
+#include "lib/md/platforms/IMDPlatform.h"

src/server/sparkd/md/platforms/emumba/EMTradingSession.h

@@ -1,16 +1,3 @@
 #pragma once
 
-#include "server/sparkd/cmds/crypto/DeribitDemoConfiguration.h"
-#include <emumba-deribit-connector/deribit_handler.h>
-#include <emumba-deribit-connector/deribit_listener.h>
-#include <emumba-deribit-connector/trading_session_interface.h>
-
-namespace bts::orderman {
-
-struct EMTradingSession {
-   emumba::deribit_connector::configuration::credential_config credential;
-   std::shared_ptr<emumba::deribit_connector::trading_session_interface>
-       trading_interface;
-};
-
-} // namespace bts::orderman
+#include "lib/md/platforms/emumba/EMTradingSession.h"

src/server/sparkd/md/platforms/emumba/EmumbaMain.h

@@ -1,6 +1,6 @@
 #pragma once
 
-#include "server/sparkd/md/platforms/IMDPlatform.h"
+#include "lib/md/platforms/IMDPlatform.h"
 #include <memory>
 
 namespace bts::md {

tools/pack_sparkd.sh

@@ -0,0 +1,85 @@
+#!/usr/bin/env bash
+
+function _exit() {
+  if [[ "$#" -gt 1 ]]; then
+    # When there are multiple args, the last arg is the exit code.
+    echo "${@:1:($# - 1)}"
+    exit "${@: -1}"
+  fi
+  echo "$1"
+  exit 0
+}
+
+help_text="\
+Package a self-contained sparkd executable in a tar.gz file
+
+Usage: pack_sparkd.sh [OPTIONS]
+
+Positionals:
+  OUT                The output path for the resultant sparkd tar.gz
+
+Options:
+  --out OUT          The output path for the resultant sparkd tar.gz
+  --force            Overwrite existing sparkd tar.gz
+  -h,--help          Print this help message and exit
+
+"
+
+dest=""
+pkg_name=""
+declare -i overwrite_dest=0
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in 
+    --out)
+      if [[ -z "$2" ]]; then
+        _exit "OUT (--out) must be non-empty" 1
+      fi
+      dest="$2"
+      shift 2
+      ;;
+    --force)
+      overwrite_dest=1
+      shift 1
+      ;;
+    -h)
+      ;&
+    --help)
+      _exit "$help_text" 0
+      ;;
+    *)
+  esac
+done
+
+tools_dir="$(dirname "$(readlink -f ${BASH_SOURCE[0]})")"
+repo_dir="$tools_dir/.."
+build_dir="$repo_dir/build/gcc/release"
+install_dir="$repo_dir/install/Release/"
+revision=$(git rev-parse --short HEAD)
+timestamp=$(date +'%Y-%m%d-%H%M')
+
+if [[ -z "$dest" ]]; then
+  pkg_name="sparkd_${revision}_${timestamp}.tar.gz"
+  dest="$repo_dir/$pkg_name"
+else
+  pkg_name="$(basename "$dest")"
+fi
+
+if [ -e "$dest" ]; then
+  echo "The output file '$dest' already exists (use --force to overwrite)"
+  exit 1
+fi
+eval "\
+  $tools_dir/build_sparkd.sh --compiler gcc --build-type release && \
+  cmake --build $build_dir --target install"
+if [[ $? -ne 0 ]]; then
+  exit $?
+fi
+cd "$install_dir"
+echo "Creating '$pkg_name'"
+tar -czf "$pkg_name" "sparkd"
+if [[ $? -ne 0 ]]; then
+  exit $?
+fi
+
+mv "$pkg_name" "$dest"

vcpkg.json

@@ -1,5 +1,6 @@
 {
   "dependencies": [
-    "fmt"
+    "fmt",
+    "cli11"
   ]
 }