build: Add CPM pkgs

These replace include files/libs previously tracked in externals

* absl
* benchmark
* concurrentqueue
* date
* double-conversion
* fmt
* json
* range-v3
* readerwriterqueue
* rxcpp
* websocketpp

.gitignore

@@ -51,3 +51,4 @@ trace_data/
 /.venv*/*
 .cache/
 src/web/api/tsconfig.*.g.json
+/.cpm_cache/**

CMakeLists.txt

@@ -23,63 +23,61 @@ set(BUILD_WEB ON CACHE BOOL "Build the web server")
 include_directories(BEFORE SYSTEM ${SPARK_INCLUDE} ${EXTERNALS_ROOT}/include/zlib)
 link_directories(${SPARK_LIB})
 
-if(WIN32)
-  # flags
-  set(
-    CMAKE_CXX_FLAGS
-    "${CMAKE_CXX_FLAGS} \
-    /MP \
-    /WX /wd4251 /wd4250 /wd4251 /wd4244 /wd4267 /wd4355 /wd4275 /wd4345 \
-    /bigobj \
-    /DCURL_STATICLIB \
-    /D_WINSOCK_DEPRECATED_NO_WARNINGS \
-    /D__TBB_NO_IMPLICIT_LINKAGE \
-    /DBOOST_ALL_NO_LIB \
-    /std:c++17 \
-    /D_SCL_SECURE_NO_WARNINGS /D_CRT_SECURE_NO_WARNINGS /D_WIN32_WINNT=0x0600")
-  set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /MTd")
-  set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_DEBUG} /MT")
-  set(
-    CMAKE_SHARED_LINKER_FLAGS_RELEASE
-    "${CMAKE_SHARED_LINKER_FLAGS_RELEASE} \
-    /DEBUG \
-    /OPT:REF \
-    /OPT:ICF \
-    /LTCG")
+include(${CMAKE_SOURCE_DIR}/cmake/CPM.cmake)
 
+# CPM packages
+CPMAddPackage("gh:nlohmann/json@3.11.2")
+CPMAddPackage("gh:fmtlib/fmt#8.1.1")
+CPMAddPackage("gh:ericniebler/range-v3#0.12.0")
+CPMAddPackage("gh:ReactiveX/RxCpp@4.1.1")
+CPMAddPackage("gh:google/double-conversion@3.2.1")
+CPMAddPackage("gh:cameron314/concurrentqueue@1.0.3")
+CPMAddPackage("gh:cameron314/readerwriterqueue@1.0.6")
+CPMAddPackage("gh:HowardHinnant/date@3.0.1")
+
+# websocketpp
+CPMAddPackage("gh:zaphoyd/websocketpp#0.8.2")
+add_library(websocketpp INTERFACE)
+set_target_properties(websocketpp PROPERTIES CMAKE_CXX_FLAGS "-Wno-null-pointer-subtraction")
+
+# abseil
+CPMAddPackage(
+  NAME absl
+  GITHUB_REPOSITORY abseil/abseil-cpp
+  GIT_TAG 20220623.1
+  DOWNLOAD_ONLY True
+)
+add_library(absl INTERFACE)
+
+string(TOUPPER ${CMAKE_BUILD_TYPE} _build_type_u)
+# benchmark
+CPMAddPackage(
+  NAME benchmark
+  GITHUB_REPOSITORY google/benchmark
+  VERSION 1.5.0
+  OPTIONS
+  "BENCHMARK_ENABLE_TESTING OFF"
+  "CMAKE_CXX_FLAGS_${_build_type_u} -Wno-unused-but-set-variable -Wno-deprecated-declarations"
+  "BENCHMARK_DOWNLOAD_DEPENDENCIES ON"
+  "CMAKE_CXX_STANDARD 17"
+)
+
+if(WIN32)
   # subdirs
   add_subdirectory(src/lib)
 else()
-  # make sparkd look for libspark.so in its directory
-  set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE)
-  set(CMAKE_INSTALL_RPATH ".")
-  set(CMAKE_INSTALL_PREFIX "${CMAKE_SOURCE_DIR}/install/${CMAKE_BUILD_TYPE}/sparkd")
   add_compile_definitions(SPARK_ENABLE_CCAPI)
   add_compile_definitions(CCAPI_ENABLE_SERVICE_MARKET_DATA)
   add_compile_definitions(CCAPI_ENABLE_EXCHANGE_FTX)
   add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE)
   add_compile_definitions(CCAPI_ENABLE_LOG_WARN)
 
-  # flags
-  set(
-    CMAKE_CXX_FLAGS_DEBUG
-    "${CMAKE_CXX_FLAGS_DEBUG} \
-    -fPIC \
-    -Wall \
-    -pedantic \
-    -Wextra \
-    -Werror \
-    -Wno-missing-braces \
-    -Wno-ignored-qualifiers \
-    --std=c++17 -fdiagnostics-color")
-  set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_DEBUG} -fno-omit-frame-pointer -DNDEBUG -O3")
-  set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -DDEBUG")
-  set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wno-undef")
-
   # subdirs
   add_subdirectory(src/server/sparkd)
+  add_dependencies(sparkd benchmark)
   add_subdirectory(src/lib)
-  if (BUILD_WEB)
+  add_dependencies(libspark benchmark)
+  if(BUILD_WEB)
     add_subdirectory(src/web)
   endif()
   add_subdirectory(doc/user)
@@ -96,9 +94,9 @@ else()
     COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/data_files ${CMAKE_BINARY_DIR}/server
     COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/src/lib/libspark.so ${CMAKE_BINARY_DIR}/server
     COMMAND
-      ln ARGS -sf
-      ${EXTERNALS_ROOT}/lib/Release/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.16.0.0
-      ${CMAKE_BINARY_DIR}/server
+    ln ARGS -sf
+    ${EXTERNALS_ROOT}/lib/Release/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.16.0.0
+    ${CMAKE_BINARY_DIR}/server
     COMMAND ln ARGS -sf ${EXTERNALS_ROOT}/lib/Release/libACE-5.7.5.so ${CMAKE_BINARY_DIR}/server
     COMMAND mkdir ARGS -p ${CMAKE_BINARY_DIR}/server/plugins
     COMMENT "Creating server runtime layout"

CMakePresets.json

@@ -0,0 +1,218 @@
+{
+  "version": 4,
+  "cmakeMinimumRequired": {
+    "major": 3,
+    "minor": 23,
+    "patch": 0
+  },
+
+  "configurePresets": [
+    {
+      "name": "spark",
+      "condition": {
+        "type": "equals",
+        "lhs": "${hostSystemName}",
+        "rhs": "Windows"
+      },
+      "generator": "Visual Studio 17 2022",
+      "binaryDir": "${sourceDir}/build",
+      "cacheVariables": {
+        "CMAKE_CXX_FLAGS": "/MP /WX /wd4250 /wd4251 /wd4244 /wd4267 /wd4355 /wd4275 /wd4345 /bigobj /EHsc /DCURL_STATICLIB /D_WINSOCK_DEPRECATED_NO_WARNINGS /D__TBB_NO_IMPLICIT_LINKAGE /DBOOST_ALL_NO_LIB /std:c++17 /D_SCL_SECURE_NO_WARNINGS /D_CRT_SECURE_NO_WARNINGS /D_WIN32_WINNT=0x0600",
+        "CMAKE_CXX_FLAGS_DEBUG": "/MTd",
+        "CMAKE_CXX_FLAGS_RELEASE": "/MT",
+        "CMAKE_SHARED_LINKER_FLAGS_RELEASE": "/DEBUG /OPT:REF /OPT:ICF /LTCG",
+        "EXTERNALS_ROOT": "${sourceDir}/externals/windows"
+      }
+    },
+
+    {
+      "name": "sparkd",
+      "condition": {
+        "type": "equals",
+        "lhs": "${hostSystemName}",
+        "rhs": "Linux"
+      },
+      "hidden": true,
+      "generator": "Ninja",
+      "binaryDir": "${sourceDir}/build/${presetName}",
+      "cacheVariables": {
+        "platform": "",
+        "CMAKE_BUILD_WITH_INSTALL_RPATH": {
+          "type": "BOOL",
+          "value": "ON"
+        },
+        "CMAKE_INSTALL_RPATH": ".",
+        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}/sparkd",
+        "CMAKE_EXE_LINKER_FLAGS": "-Wno-undef",
+        "CPM_SOURCE_CACHE": "${sourceDir}/.cpm_cache/",
+        "EXTERNALS_ROOT": "${sourceDir}/externals/linux",
+        "CMAKE_CXX_STANDARD": "17",
+        "WEB_TSCONFIG_NAME": "${presetName}",
+        "BUILD_SDK": {
+          "type": "BOOL",
+          "value": "OFF"
+        },
+        "BUILD_WEB": {
+          "type": "BOOL",
+          "value": "ON"
+        }
+      }
+    },
+
+    {
+      "name": "sparkd-dev-clang",
+      "inherits": "sparkd",
+      "displayName": "Development (clang)",
+      "description": "Debug build compiled w/ clang",
+      "cacheVariables": {
+        "CMAKE_BUILD_TYPE": "Debug",
+        "CMAKE_EXPORT_COMPILE_COMMANDS": {
+          "type": "BOOL",
+          "value": "ON"
+        },
+        "CMAKE_C_COMPILER": "clang",
+        "CMAKE_CXX_COMPILER": "clang++",
+        "CMAKE_CXX_FLAGS_DEBUG": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-deprecated-builtins -Wno-array-parameter -Wno-ignored-qualifiers --std=c++17 -fdiagnostics-color"
+      }
+    },
+
+    {
+      "name": "sparkd-dev-gcc",
+      "inherits": "sparkd",
+      "displayName": "Development (gcc)",
+      "description": "Debug build compiled w/ gcc",
+      "cacheVariables": {
+        "CMAKE_BUILD_TYPE": "Debug",
+        "CMAKE_EXPORT_COMPILE_COMMANDS": {
+          "type": "BOOL",
+          "value": "ON"
+        },
+        "CMAKE_C_COMPILER": "gcc",
+        "CMAKE_CXX_COMPILER": "g++",
+        "CMAKE_CXX_FLAGS_DEBUG": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++17 -fdiagnostics-color -Wno-stringop-overread -DDEBUG"
+      }
+    },
+
+    {
+      "name": "sparkd-prod-clang",
+      "inherits": "sparkd",
+      "displayName": "Production (clang)",
+      "description": "Release build compiled w/ clang",
+      "cacheVariables": {
+        "CMAKE_BUILD_TYPE": "Release",
+        "CMAKE_C_COMPILER": "clang",
+        "CMAKE_CXX_COMPILER": "clang++",
+        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++17 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3"
+      }
+    },
+
+    {
+      "name": "sparkd-prod-gcc",
+      "inherits": "sparkd",
+      "displayName": "Production (gcc)",
+      "description": "Release build compiled w/ gcc",
+      "cacheVariables": {
+        "CMAKE_BUILD_TYPE": "Release",
+        "CMAKE_C_COMPILER": "gcc",
+        "CMAKE_CXX_COMPILER": "g++",
+        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++17 -fdiagnostics-color -Wno-stringop-overread -fno-omit-frame-pointer -DNDEBUG -O3"
+      }
+    },
+
+    {
+      "name": "sparkd-publish",
+      "inherits": "sparkd",
+      "displayName": "Publish",
+      "description": "Publish build compiled w/ gcc and ready to ship",
+      "cacheVariables": {
+        "CMAKE_CXX_COMPILER": "g++"
+      }
+    }
+  ],
+
+  "buildPresets": [
+    {
+      "name": "spark",
+      "condition": {
+        "type": "equals",
+        "lhs": "${hostSystemName}",
+        "rhs": "Windows"
+      },
+      "hidden": true,
+      "jobs": 4
+    },
+
+    {
+      "name": "spark-dev",
+      "displayName": "Development",
+      "inherits": "spark",
+      "configurePreset": "spark",
+      "configuration": "Debug"
+    },
+
+    {
+      "name": "spark-prod",
+      "displayName": "Production",
+      "inherits": "spark",
+      "configurePreset": "spark",
+      "configuration": "Release"
+    },
+
+    {
+      "name": "sparkd",
+      "condition": {
+        "type": "equals",
+        "lhs": "${hostSystemName}",
+        "rhs": "Linux"
+      },
+      "hidden": true,
+      "jobs": 4,
+      "targets": [
+        "data_files_dir",
+        "doc_manual_bundle",
+        "install",
+        "libspark",
+        "rapidjson_project",
+        "server_layout",
+        "sparkd",
+        "web_ui_bundle"
+      ]
+    },
+
+    {
+      "name": "sparkd-dev-clang",
+      "configurePreset": "sparkd-dev-clang",
+      "displayName": "Development (clang)",
+      "inherits": "sparkd"
+    },
+
+    {
+      "name": "sparkd-dev-gcc",
+      "configurePreset": "sparkd-dev-gcc",
+      "displayName": "Development (gcc)",
+      "inherits": "sparkd"
+    },
+
+    {
+      "name": "sparkd-prod-clang",
+      "configurePreset": "sparkd-prod-clang",
+      "displayName": "Production (clang)",
+      "inherits": "sparkd"
+    },
+
+    {
+      "name": "sparkd-prod-gcc",
+      "configurePreset": "sparkd-prod-gcc",
+      "displayName": "Production (gcc)",
+      "inherits": "sparkd"
+    },
+
+    {
+      "name": "sparkd-publish",
+      "configurePreset": "sparkd-publish",
+      "displayName": "Publish",
+      "inherits": "sparkd",
+      "targets": "install"
+    }
+  ]
+}

cmake/CPM.cmake

@@ -0,0 +1,23 @@
+# CPM Code (https://github.com/cpm-cmake/CPM.cmake/releases/latest/download/get_cpm.cmake)
+set(CPM_DOWNLOAD_VERSION 0.36.0)
+
+if(CPM_SOURCE_CACHE)
+  set(CPM_DOWNLOAD_LOCATION "${CPM_SOURCE_CACHE}/cpm/CPM_${CPM_DOWNLOAD_VERSION}.cmake")
+elseif(DEFINED ENV{CPM_SOURCE_CACHE})
+  set(CPM_DOWNLOAD_LOCATION "$ENV{CPM_SOURCE_CACHE}/cpm/CPM_${CPM_DOWNLOAD_VERSION}.cmake")
+else()
+  set(CPM_DOWNLOAD_LOCATION "${CMAKE_BINARY_DIR}/cmake/CPM_${CPM_DOWNLOAD_VERSION}.cmake")
+endif()
+
+# Expand relative path. This is important if the provided path contains a tilde (~)
+get_filename_component(CPM_DOWNLOAD_LOCATION ${CPM_DOWNLOAD_LOCATION} ABSOLUTE)
+
+if(NOT(EXISTS ${CPM_DOWNLOAD_LOCATION}))
+  message(STATUS "Downloading CPM.cmake to ${CPM_DOWNLOAD_LOCATION}")
+  file(DOWNLOAD
+    https://github.com/cpm-cmake/CPM.cmake/releases/download/v${CPM_DOWNLOAD_VERSION}/CPM.cmake
+    ${CPM_DOWNLOAD_LOCATION}
+  )
+endif()
+
+include(${CPM_DOWNLOAD_LOCATION})

src/lib/CMakeLists.txt

@@ -110,57 +110,64 @@ target_include_directories(
 if(WIN32)
   target_link_libraries(
     libspark
-    libcurl
+    crypt32
     debug
-    fmtd
-    optimized
     fmt
-    libtbb
-    libboost_system
-    libboost_thread
-    libboost_filesystem
-    libboost_regex
+    fmtd
+    libboost_atomic
     libboost_chrono
     libboost_date_time
-    libboost_atomic
+    libboost_filesystem
+    libboost_regex
+    libboost_system
+    libboost_thread
+    libcrypto
+    libcurl
     libprotobuf
     libprotoc
     libssl
-    libcrypto
-    crypt32)
+    libtbb
+    optimized)
 else()
   target_link_libraries(
     libspark
-    protobuf
-    double-conversion
-    boost_system
-    boost_thread
-    boost_filesystem
-    boost_regex
-    boost_chrono
-    boost_date_time
-    boost_atomic
-    SQLiteCpp
-    sqlite3
-    curl
-    uuid
-    ssl
-    crypto
-    fmt
-    tbb
-    cryptopp
-    pugixml
-    lttng-ust
-    Pcap++
-    Packet++
-    Common++
-    ssh2
-    -lrt
     -ldl
     -lpcap
+    -lrt
+    -pthread
     -static-libgcc
     -static-libstdc++
-    -pthread)
+    Common++
+    Packet++
+    Pcap++
+    SQLiteCpp
+    boost_atomic
+    boost_chrono
+    boost_date_time
+    boost_filesystem
+    boost_regex
+    boost_system
+    boost_thread
+    concurrentqueue
+    crypto
+    cryptopp
+    curl
+    date
+    double-conversion
+    fmt
+    lttng-ust
+    nlohmann_json
+    protobuf
+    pugixml
+    range-v3
+    readerwriterqueue
+    rxcpp
+    sqlite3
+    ssh2
+    ssl
+    tbb
+    uuid
+    websocketpp)
 
   install(TARGETS libspark DESTINATION bin)
 endif()

src/lib/api/core/ApiClientInterop.cpp

@@ -186,7 +186,7 @@ CAPI void api_client_send(
     void *extra) {
    LOG(Api, Diag, "send rpc {}", ((int64_t)buffer));
    std::string strbuf(buffer, buffer_length);
-   qbus::RPCRequest request(move(strbuf));
+   qbus::RPCRequest request(std::move(strbuf));
    api_client_send_internal(
        pclient, std::move(request), cb, rpc_connect_timeout_ms(), extra);
 }

src/lib/api/core/ApiServerInterop.cpp

@@ -60,7 +60,7 @@ CAPI void api_server_call_callback(
        reinterpret_cast<qbus::ReplyHandler **>(ppvhandler);
    std::string reply_buffer((const char *)buffer, (size_t)length);
    qbus::RPCReplyPtr replyptr =
-       std::make_shared<qbus::RPCReply>(move(reply_buffer));
+       std::make_shared<qbus::RPCReply>(std::move(reply_buffer));
    (**pphandler)(replyptr);
    delete (*pphandler);
    *pphandler = nullptr;

src/lib/api/public/ClientChannel.h

@@ -72,7 +72,7 @@ class ClientChannel {
                        response_buffer + header_size,
                        response_size - header_size);
                    if (response->ParseFromZeroCopyStream(&data_stream)) {
-                      on_response(Result<Response>(move(response)));
+                      on_response(Result<Response>(std::move(response)));
                       return;
                    }
                 } else {

src/lib/api/public/QuickbusClientChannel.cpp

@@ -28,7 +28,7 @@ void QuickbusClientChannel::send_core(
       (*on_response)(buffer, length);
       delete on_response;
    };
-   OnReply *heap_on_response = new OnReply(move(on_response));
+   OnReply *heap_on_response = new OnReply(std::move(on_response));
    api_client_send(
        _pclient, request_envalope.c_str(), request_envalope.size(), trampoline,
        heap_on_response);

src/lib/api/service_compiler/templates/cpp/client.pb.cc

@@ -11,8 +11,8 @@ using namespace rxcpp::operators;
 {{#each service}}
 
 // Constructor
-{{name}}::{{name}}(std::unique_ptr<bts::api::ClientChannel> pchannel) : _pchannel(move(pchannel)) {}
-{{name}}::{{name}}({{name}}&& other) : _pchannel(move(other._pchannel))  { }
+{{name}}::{{name}}(std::unique_ptr<bts::api::ClientChannel> pchannel) : _pchannel(std::move(pchannel)) {}
+{{name}}::{{name}}({{name}}&& other) : _pchannel(std::move(other._pchannel))  { }
 
 {{#each method}}
    {{#if server_streaming}}

src/lib/md/MBOProcessor.cpp

@@ -12,7 +12,7 @@
 #include <vector>
 
 #ifndef _WIN32
-#include <btree/set.h>
+#include <btree/btree_set.h>
 #endif
 #include <tbb/concurrent_hash_map.h>
 #include <tbb/concurrent_queue.h>
@@ -119,7 +119,7 @@ struct MBOProcessorImpl {
       // after draining publish the set in small batches
       // re-check input queue regularly
 
-      btree::set<MBOWorkingState *> dirty;
+      btree::btree_set<MBOWorkingState *> dirty;
       std::vector<MBOWorkingState *> clean;
 
       while (running) {

src/lib/md/MBOProcessor.h

@@ -6,7 +6,7 @@
 #include "lib/domain_model/SecurityKey.h"
 #include "lib/utils/TimeStamp.h"
 
-#include <btree/map.h>
+#include <btree/btree_map.h>
 
 #include <map>
 #include <memory>
@@ -102,8 +102,8 @@ struct MBOWorkingState {
    // Used for 'change' messages which update price
    std::unordered_map<ExchangeOrderKey, key_t> orderid_index;
 
-   btree::map<key_t, Data, LessBid> bid;
-   btree::map<key_t, Data> ask;
+   btree::btree_map<key_t, Data, LessBid> bid;
+   btree::btree_map<key_t, Data> ask;
 };
 
 void copy_to_snapshot(MBOWorkingState &state, MBOSnapshot *destination);

src/lib/md/MDConsumer.cpp

@@ -15,7 +15,7 @@
 #include "lib/utils/sane_algorithm.h"
 
 #include <algorithm>
-#include <btree/map.h>
+#include <btree/btree_map.h>
 #include <map>
 #include <set>
 #include <string>
@@ -181,9 +181,9 @@ class MDConsumerInst {
       return false;
    }
 
-   btree::map<security_key, tuple<const ColdEntry *, const void *>>
+   btree::btree_map<security_key, tuple<const ColdEntry *, const void *>>
    search_handles(const std::set<security_key> &keys) {
-      btree::map<security_key, tuple<const ColdEntry *, const void *>> result;
+      btree::btree_map<security_key, tuple<const ColdEntry *, const void *>> result;
       // Walk hot and cold lists and fill in any found handles
       if (!_cold.is_initialized())
          return result;
@@ -783,7 +783,7 @@ class MDConsumerImpl {
       // recurse to find details
       map<security_key, SecurityPtr> leg_map;
       for (auto &leg : fill_leg_details(legs, depth + 1)) {
-         leg_map.insert(make_pair(leg->id(), move(leg)));
+         leg_map.insert(make_pair(leg->id(), std::move(leg)));
       }
 
       //

src/lib/md/MDConsumerHandle.h

@@ -218,6 +218,7 @@ class BTS_EXPORT MDConsumerHandle {
       switch (_type) {
       case HANDLETYPE_VOID:
          bts_assert_msg(false, "Handle not initialized");
+         break;
       case HANDLETYPE_DEPTH:
          return convert_security(book.cdata->security);
       case HANDLETYPE_BBO:

src/lib/md/MDProducer.cpp

@@ -173,7 +173,7 @@ MDProducerImpl &MDProducer::get_producer(
    } else {
       newp.reset(new MDProducerImpl(symbol, type, exchange, _platform, _flags));
    }
-   _producers.push_back(move(newp));
+   _producers.push_back(std::move(newp));
    return *_producers.back();
 }
 

src/lib/md/SecurityCache.cpp

@@ -5,7 +5,7 @@
 #include "lib/utils/Dispatcher.h"
 #include "lib/utils/LockBox.h"
 
-#include <btree/map.h>
+#include <btree/btree_map.h>
 #include <mutex>
 #include <tbb/concurrent_queue.h>
 #include <unordered_map>
@@ -14,8 +14,8 @@ namespace bts {
 namespace md {
 
 struct SecurityCacheImpl {
-   utils::LockBox<btree::map<SecurityKey, SecurityPtr>> cache;
-   utils::LockBox<btree::map<ExchangeSecurityKey, SecurityPtr>> ex_cache;
+   utils::LockBox<btree::btree_map<SecurityKey, SecurityPtr>> cache;
+   utils::LockBox<btree::btree_map<ExchangeSecurityKey, SecurityPtr>> ex_cache;
    tbb::concurrent_queue<SecurityPtr> update_queue;
    void enqueue_update_db(const SecurityPtr &);
    void flush();
@@ -52,7 +52,7 @@ SecurityCache::SecurityCache() : _me(std::make_shared<SecurityCacheImpl>()) {}
 SecurityCache::~SecurityCache() {}
 
 const Security *SecurityCache::try_get_security(SecurityKey key) const {
-   return _me->cache.lock([key](btree::map<SecurityKey, SecurityPtr> &t) {
+   return _me->cache.lock([key](btree::btree_map<SecurityKey, SecurityPtr> &t) {
       auto iter = t.find(key);
       if (iter != t.end()) {
          return iter->second.get();
@@ -62,7 +62,7 @@ const Security *SecurityCache::try_get_security(SecurityKey key) const {
 }
 const Security *SecurityCache::try_get_security(ExchangeSecurityKey key) const {
    return _me->ex_cache.lock(
-       [key](btree::map<ExchangeSecurityKey, SecurityPtr> &t) {
+       [key](btree::btree_map<ExchangeSecurityKey, SecurityPtr> &t) {
           auto iter = t.find(key);
           if (iter != t.end()) {
              return iter->second.get();

src/lib/md/SecurityIDRecorder.cpp

@@ -210,7 +210,6 @@ void SecurityIDRecorderState::work_handles() {
    Security security;
    MDProducerHandle handle;
    vector<MDProducerHandle> retry;
-   int count = 0;
    while (pending_handle.try_pop(handle)) {
       security.Clear();
       // if a spread does not have all of its legs we need to wait
@@ -220,7 +219,6 @@ void SecurityIDRecorderState::work_handles() {
          retry.push_back(handle);
       } else {
          convert_security(handle.security(), &security);
-         ++count;
          assign_key(security, [handle](const api::Security &security) mutable {
             handle.set_key(security.id());
          });
@@ -254,7 +252,7 @@ void SecurityIDRecorderState::work_db() {
 
       // If we're unable to create a descriptor try again later
       if (descriptor == "") {
-         defer.push_back(move(temp));
+         defer.push_back(std::move(temp));
          continue;
       }
 
@@ -262,10 +260,10 @@ void SecurityIDRecorderState::work_db() {
       if (key != invalid_security_key) {
          temp->security.set_id(key);
          temp->callback(temp->security);
-         existing_securities.push_back(move(temp));
+         existing_securities.push_back(std::move(temp));
       } else {
          // always push in parallel
-         new_entries.push_back(move(temp));
+         new_entries.push_back(std::move(temp));
          new_descriptors.push_back(descriptor);
          ++count;
          // Limit the amount of time we spend writing to sqlite during startup
@@ -279,7 +277,7 @@ void SecurityIDRecorderState::work_db() {
    for (auto &d : defer) {
       LOG(md, Info, "SecurityIDRecorderState::work_db work.push d: {}",
           bts::securities::descriptor(d->security));
-      work.push(move(d));
+      work.push(std::move(d));
    }
 
    if (!new_entries.empty()) {

src/lib/md/SecurityIndex.h

@@ -133,10 +133,10 @@ class ConcurrentSecurityIndex {
          // support std::move
          accessor->second = entry.release();
          return accessor->second;
-      } else {
-         LOG(md, Error, "Found duplicate entry for platform id '{}' - ", key);
-         bts_assert(false);
       }
+      LOG(md, Error, "Found duplicate entry for platform id '{}' - ", key);
+      bts_assert(false);
+      return nullptr;
    }
 
    SecurityData *add_security(ExchangeKey key, MDProducerHandle &handle) {

src/lib/md/SecurityMapper.cpp

@@ -8,12 +8,12 @@
 #include "lib/domain_model/Security.h"
 #include "lib/utils/parse_enum.h"
 #ifndef _WIN32
-#include <btree/map.h>
+#include <btree/btree_map.h>
 
 namespace bts {
 namespace md {
 struct DisplayFactorMapper : public ISecurityMapper {
-   btree::map<UserSymbol, int> _mapped;
+   btree::btree_map<UserSymbol, int> _mapped;
    DisplayFactorMapper() : _mapped() {
       for (const auto &ov : config::get<DisplayFactorOverride>().override()) {
          if (ov.has_price_multiplier()) {
@@ -51,7 +51,7 @@ struct DisplayFactorMapper : public ISecurityMapper {
 };
 
 struct CppOverride : public ISecurityMapper {
-   btree::map<std::string, double> _map;
+   btree::btree_map<std::string, double> _map;
    CppOverride() : _map() {
       for (const auto &p : config::get<config::CppConfig>().cpp_entry()) {
          _map.insert(p);

src/lib/md/SecurityStore.cpp

@@ -4,7 +4,7 @@
 #include "lib/utils/Dispatcher.h"
 #include "lib/utils/LockBox.h"
 
-#include <btree/map.h>
+#include <btree/btree_map.h>
 #include <mutex>
 #include <tbb/concurrent_queue.h>
 #include <unordered_map>
@@ -19,7 +19,7 @@ static domain::SecurityID create_sec_id(
 }
 
 struct SecurityStore::PImpl {
-   utils::LockBox<btree::map<domain::SecurityID, SecurityPtr>> cache;
+   utils::LockBox<btree::btree_map<domain::SecurityID, SecurityPtr>> cache;
    tbb::concurrent_queue<SecurityStore::SecurityPtr> update_queue;
 
    void enqueue_update_db(const SecurityPtr &);
@@ -47,7 +47,7 @@ SecurityStore::~SecurityStore() {}
 const domain::Security *SecurityStore::try_get_security(
     domain::SecurityID key) const {
    return me_->cache.lock(
-       [key](btree::map<domain::SecurityID, SecurityPtr> &t)
+       [key](btree::btree_map<domain::SecurityID, SecurityPtr> &t)
            -> const domain::Security * {
           auto iter = t.find(key);
           if (iter != t.end()) {

src/lib/md/test/md2_tests.cpp

@@ -1238,7 +1238,7 @@ struct Co2IndexTests {
          temp->security.mutable_glbx()->set_exchange_id(id);
          temp->exchange_id = id;
          temp->handle.key(count);
-         index.add_security(id, move(temp));
+         index.add_security(id, std::move(temp));
       }
    }
 

src/lib/orderman/ilink3/IL3Parser.h

@@ -5,7 +5,7 @@
 #include "lib/md/SecurityOracle.h"
 #include "lib/orderman/orderman_utils/ExecutionReport.h"
 #include <boost/pool/object_pool.hpp>
-#include <btree/map.h>
+#include <btree/btree_map.h>
 #include <optional>
 
 namespace bts {

src/lib/orderman/orderman_utils/BookUtils.cpp

@@ -31,13 +31,12 @@ void BookUtils::change_unit_qty(
    const BookSide& book_in, std::vector<md::PriceSize>& book_out, int unit_qty) {
    //
    bts_assert(unit_qty > 0);
-   int mlev = 0, mlev_qty_used = 0;
+   int mlev_qty_used = 0;
    std::vector<md::PriceSize> unit_lev_span;
    for (uint32_t i = 0; i < book_in.size();) {
       int mlev_qty_rem = book_in[i].size() - mlev_qty_used;
       if (mlev_qty_rem == 0) {
          mlev_qty_used = 0;
-         mlev++;
          i++;
          continue;
       }
@@ -49,7 +48,6 @@ void BookUtils::change_unit_qty(
          } else {
             unit_lev_span.emplace_back(book_in[i].int_price(), mlev_qty_rem);
             mlev_qty_used = 0;
-            mlev++;
             i++;
          }
       } else {

src/lib/utils/SecureLogon.cpp

@@ -93,6 +93,7 @@ std::tuple<string, string> load_session_keys(const string &session) {
 
    string error = "Failed to find secure logon key for session: " + session;
    bts_assert_msg(false, error);
+   return {"", ""};
 }
 
 LogonData calculate_hmac(
@@ -127,6 +128,7 @@ LogonData calculate_hmac(
       bts_assert_msg(false, e.what());
    }
    bts_assert(false);
+   return {};
 }
 
 // TEST_CASE("LoadSessionKeys", "") {

src/lib/utils/SharedMemory.cpp

@@ -74,14 +74,14 @@ struct BoostLowLevelSharedMemoryImpl {
 
    void add_region(shared_memory_object block) {
       mapped_region region(block, block.get_mode());
-      _mappings.push_back(make_pair(move(block), move(region)));
+      _mappings.push_back(make_pair(std::move(block), std::move(region)));
    }
 
    Segment create(const string &name, size_t size) {
       try {
          shared_memory_object block(create_only, name.c_str(), read_write);
          block.truncate(size);
-         add_region(move(block));
+         add_region(std::move(block));
          return make_segment(size, _mappings.back().second.get_address());
       } catch (...) {
          return make_segment(0, nullptr);

src/lib/utils/sane_algorithm.h

@@ -1,5 +1,6 @@
 #pragma once
 #include <algorithm>
+#include <limits>
 
 namespace bts {
 namespace sane {
@@ -57,4 +58,4 @@ void map(const TSrc &source, TDest &dest, const TFn &fn) {
 }
 
 } // namespace sane
-} // namespace bts
+} // namespace bts

src/server/sparkd/CMakeLists.txt

@@ -67,24 +67,24 @@ endif()
 # linking
 target_link_libraries(
   sparkd
-  libspark
   ${emumba-deribit-connector-library_LIBRARIES}
-  ssl
-  cryptopp
-  tbb
-  onixs-cmemdh-all-4.8.0
-  OnixS.IceImpactMulticastPriceFeedHandlerCpp
-  ACE
-  quickfix
-  benchmark
-  Common++
-  Packet++
-  Pcap++
-  lttng-ust
   -lpcap
+  -pthread
   -static-libgcc
   -static-libstdc++
-  -pthread)
+  ACE
+  Common++
+  OnixS.IceImpactMulticastPriceFeedHandlerCpp
+  Packet++
+  Pcap++
+  benchmark
+  cryptopp
+  libspark
+  lttng-ust
+  onixs-cmemdh-all-4.8.0
+  quickfix
+  ssl
+  tbb)
 
 add_custom_command(
   TARGET sparkd

src/server/sparkd/cmds/Benchmark.cpp

@@ -31,12 +31,12 @@
 #include <boost/pool/singleton_pool.hpp>
 #include <cstdlib>
 #include <iostream>
-#include <moodycamel/concurrentqueue.h>
-#include <moodycamel/readerwriterqueue.h>
+#include <concurrentqueue.h>
+#include <readerwriterqueue.h>
 #include <mutex>
 #include <sys/time.h>
 
-#include <btree/set.h>
+#include <btree/btree_set.h>
 #include <set>
 #include <unordered_set>
 

src/server/sparkd/cmds/crypto/DeribitDemoConfiguration.cpp

@@ -141,8 +141,9 @@ std::optional<DeribitDemoConfiguration> DeribitDemoConfiguration::read_file(
    rapidjson::IStreamWrapper isw{configFile};
 
    rapidjson::Document doc{};
-   if (doc.ParseStream(isw).HasParseError()) {
-      LOG(md, Error, "Error in parsing config from {}", config_path);
+   auto &res = doc.ParseStream(isw);
+   if (res.HasParseError()) {
+      LOG(md, Error, "Error in parsing config from {}: {}", config_path, res.GetParseError());
       return std::nullopt;
    }
 

src/server/sparkd/cmds/spartan/domain/book/tests/synthetic_book_tests.cpp

@@ -395,9 +395,7 @@ void test_synth_book(LegMarkets synth) {
    // test every permutation of the legs.  All permutations of a particular
    // set of legs should produce the same synthetic book.
    auto perms = bts::spartan::Permutation::all(synth);
-   int counter = 0;
    for (const auto &p : perms) {
-      counter++;
       SmallBook book_out;
       SmallBook book_out_historic;
       bts::spartan::mdp3::book_utils::compute_synthetic_book(book_out_historic, p, 10);

src/server/sparkd/http/ApiGateway.cpp

@@ -114,7 +114,7 @@ struct ApiGatewayImpl {
       api_header.AppendToString(&buffer);
       buffer += body;
       // call send
-      Responder *presponder = new Responder(move(reply));
+      Responder *presponder = new Responder(std::move(reply));
       api_client_send(
           pclient, buffer.c_str(), buffer.size(),
           [](const char *buffer, size_t length, void *extra) {
@@ -144,10 +144,10 @@ struct ApiGatewayImpl {
       // call api_client_subscribe() store the TableSubscription and the id and
       // the reply channel When a tableupate fires attach the id to the reply
       // and send it over the channel Responder* preply = new
-      // Responder(move(reply));
+      // Responder(std::move(reply));
       auto id = request.request_id();
       SubscriptionContext *pcontext =
-          new SubscriptionContext{id, move(reply), nullptr};
+          new SubscriptionContext{id, std::move(reply), nullptr};
 
       auto table = api_client_subscribe(
           pclient, request.method_name().c_str(),

src/server/sparkd/md/platforms/emumba/EmMDListener.cpp

@@ -412,7 +412,8 @@ bts::orderman::OpResult EmMDListener::launch_new_order(
    me->order_acc_map.insert({ord_key.value(), trading_session});
    auto bts_sec_key = sec_ptr->handle.security().key;
    me->order_state_map.insert(
-       {ord_key.value(), {bts_sec_key, order.order_def.order_side}});
+       {static_cast<std::uint64_t>(ord_key.value()),
+        {bts_sec_key, order.order_def.order_side}});
    LOG(OrderMan, Info,
        "IEMOrderMuxer::launch_new_order em_ord_key: {} sec_key: {} tid: {} "
        "sec: {} p: {} "
@@ -533,14 +534,15 @@ void EmMDListener::on_order_update(
            order_state.exchange_order_state);
    // these are all logs
    LOG(OrderMan, Info,
-       "IEMOrderMuxer::on_order_update em_ord_key: {} tid: {} sub_acct_idx: "
-       "{} "
+       "IEMOrderMuxer::on_order_update em_ord_key: {} tid: {} sub_acct_idx: {} "
        "ec.value: {} ec.message: {} ec.details: {} update_type: {} "
-       "exchange_key: {} status: "
-       "{} pr: {} qty: {} filled_pr: {} fills: {} n_trades: {}",
+       "exchange_key: {} status: {} pr: {} qty: {} filled_pr: {} fills: {} "
+       "n_trades: {}",
        order_key.value(), os::get_thread_id(), order_state.subaccount_index,
-       ec.value(), ec.message(), ec.details(), order_state.order_update_type,
-       exchange_order_state.exchange_order_id, order_state.order_status,
+       ec.value(), ec.message(), ec.details(),
+       static_cast<std::uint64_t>(order_state.order_update_type),
+       exchange_order_state.exchange_order_id,
+       static_cast<std::uint64_t>(order_state.order_status),
        order_state.px.to_double(), extract_qty(order_state.qty),
        order_state.avg_price.to_double(), extract_qty(order_state.total_fills),
        order_state.trades.size());
@@ -1210,4 +1212,4 @@ double EmMDListener::_estimate_leg_fill(security_key key, api::Side side) {
 
 /* #endregion */
 
-} // namespace bts::md
+} // namespace bts::md

src/server/sparkd/md/platforms/emumba/EmumbaMain.cpp

@@ -21,6 +21,7 @@
 
 #include <boost/filesystem.hpp>
 #include <memory>
+#include <optional>
 #include <set>
 
 using namespace std;
@@ -47,8 +48,8 @@ struct EmumbaMain::PImpl {
          listener(sec_index, producer) {
       auto config_path =
           os::sparkdata_path("conf") + "deribit_demo_config.json";
-      std::optional<bts::cmds::DeribitDemoConfiguration> conf =
-          bts::cmds::DeribitDemoConfiguration::read_file(config_path);
+      auto conf = bts::cmds::DeribitDemoConfiguration::read_file(config_path);
+      bts_assert_msg(conf.has_value(), "Failed to parse deribit config");
       // log
       for (const auto &s : conf.value().symbol_config_list) {
          LOG(md, Info, "sub.symbol: {}", s.symbol_name);

src/server/sparkd/md/platforms/ice_onixs/ICEMain.cpp

@@ -104,7 +104,7 @@ void ICEMain::start(std::vector<UserSymbol> &symbols) {
       unique_ptr<onixs_ice::Handler> phandler =
           prepare_handler(channel, license.path.string());
       _listeners.push_back(prepare_listener(*phandler, channel));
-      _handlers.push_back(std::make_pair(channel, move(phandler)));
+      _handlers.push_back(std::make_pair(channel, std::move(phandler)));
 
       std::stringstream symstr;
       for (const auto &sym : channel.symbols)

src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp

@@ -122,7 +122,7 @@ create_feed_engines(const vector<CMEChannel> &channels) {
    // Loop through settings and instantiate feed engines
    for (const auto &s : settings) {
       unique_ptr<onixs::FeedEngine> temp(new onixs::FeedEngine(s.second));
-      result.insert(make_pair(s.first, move(temp)));
+      result.insert(make_pair(s.first, std::move(temp)));
    }
 
    return result;
@@ -168,7 +168,7 @@ void MDP3Main::start(
    for (const auto &channel : channels) {
       unique_ptr<onixs::Handler> phandler = prepare_handler(channel.id, mbo);
       _listeners.push_back(prepare_listener(*phandler, channel, mbo, tssd));
-      _handlers.insert(std::make_pair(channel.id, move(phandler)));
+      _handlers.insert(std::make_pair(channel.id, std::move(phandler)));
       std::stringstream symstr;
       for (const auto &sym : channel.symbols) {
          symstr << sym.symbol() << "-" << SecurityType_Name(sym.type()) << " ";

tools/build_sparkd.sh

@@ -1,4 +1,5 @@
 #!/usr/bin/env bash
 
 tools_path=$(dirname $(readlink -f $0))
-eval "$tools_path/venv_exec.sh python $tools_path/py/build_sparkd.py $@"
+eval "$tools_path/ensure_venv.sh" 1> /dev/stdout 2> /dev/stderr
+eval "$tools_path/venv_exec.sh python $tools_path/py/build_sparkd.py $@" 1> /dev/stdout 2> /dev/stderr

tools/ensure_venv.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+declare tools_dir="$(readlink -f $(dirname ${BASH_SOURCE[0]}))"
+declare repo_dir="$tools_dir/.."
+declare venv_dir="$repo_dir/.venv"
+declare venv_activate_sh="$repo_dir/.venv/bin/activate"
+declare requirements_txt="$repo_dir/requirements.txt"
+
+if [[ ! -f "$venv_activate_sh" ]]; then
+  echo "Creating python virtual environment '.venv'"
+  python3 -m pip install --upgrade pip
+  python3 -m venv $venv_dir
+  source $venv_activate_sh
+  echo "Upgrading pip"
+  python -m pip install --upgrade pip
+else
+  source $venv_activate_sh
+fi
+
+pip install -q -r $requirements_txt

tools/logging/pretty.sh

@@ -1,2 +0,0 @@
-#!/bin/bash
- tac ~/sparkdata/data/logging/log  | grep "Sparkd launched" -m 1 -B 9999 | tac | 

tools/logging/show_last_run.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+declare num_lines=9999
+if [[ "$#" -gt 0 ]]; then
+  set num_lines=$1
+fi
+
+tac ~/sparkdata/data/logging/log | grep "Sparkd launched" -m 1 -B $num_lines | tac | less

tools/py/build_sparkd.py

@@ -60,7 +60,7 @@ class CMakeConfigureCommand:
 def _parse_compiler(arg: str or None):
     '''Parses compiler input (from cli) and returns a tuple of c, cxx compilers respectively'''
     known_compilers = {
-        'gcc': CMakeCompilerPair('gcc', 'gcc-11', 'g++-11'),
+        'gcc': CMakeCompilerPair('gcc', 'gcc', 'g++'),
         'clang': CMakeCompilerPair('clang', 'clang', 'clang++')
     }
     if arg in known_compilers: