Update externals

Removed fmt and range-3, since we get them from vcpkg now
Set up devcontainer for sparkd
2024-08-28 23:39:34 -04:00 · 2024-08-28 23:26:28 -04:00 · 2024-08-28 23:25:04 -04:00 · 2024-08-28 00:05:46 -04:00 · 2024-08-28 00:05:46 -04:00 · 2024-08-28 00:05:46 -04:00
336 changed files with 132817 additions and 114064 deletions
@@ -0,0 +1,129 @@
+FROM ubuntu:22.04
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+# Note installations that take a while (more than 10s)
+
+RUN apt-get update
+RUN apt-get install -y ca-certificates
+RUN apt-get install -y ccache
+RUN apt-get install -y curl
+RUN apt-get install -y fish
+RUN apt-get install -y gdb
+# git-all: ~250s
+RUN apt-get install -y git-all
+RUN apt-get install -y gnupg
+# liblttng-ust-dev: ~30s
+RUN apt-get install -y liblttng-ust-dev
+RUN apt-get install -y libssh2-1-dev
+RUN apt-get install -y locales
+RUN apt-get install -y ninja-build
+# openssh-server: ~40s
+RUN apt-get install -y openssh-server
+RUN apt-get install -y pkg-config
+RUN apt-get install -y software-properties-common
+RUN apt-get install -y sudo
+RUN apt-get install -y tzdata
+RUN apt-get install -y vim
+RUN apt-get install -y zip
+
+# Python: ~30s
+RUN apt-get install -y \
+   python3-pip \
+   python3-venv \
+   && python3 -m pip install --upgrade pip
+
+# GCC
+ARG gcc_major=13
+RUN \
+   add-apt-repository ppa:ubuntu-toolchain-r/test \
+   && apt-get update \
+   && apt-get install -y g++-$gcc_major
+
+# Clang
+ARG clang_major=17
+RUN \
+   curl -fsSL https://apt.llvm.org/llvm-snapshot.gpg.key 2> /dev/null \
+   | gpg --dearmor - \
+   | tee /usr/share/keyrings/llvm-snapshot.gpg > /dev/null \
+   && echo "deb [signed-by=/usr/share/keyrings/llvm-snapshot.gpg] http://apt.llvm.org/$(lsb_release -cs)/ llvm-toolchain-$(lsb_release -cs)-$clang_major main" \
+   | tee -a "/etc/apt/sources.list.d/llvm.list" \
+   && apt-get update \
+   && apt-get install -y \
+   clang-$clang_major \
+   clang-format-$clang_major \
+   clangd-$clang_major
+
+# CMake
+RUN \
+   curl -fsSL https://apt.kitware.com/keys/kitware-archive-latest.asc 2> /dev/null \
+   | gpg --dearmor - \
+   | tee /usr/share/keyrings/kitware-archive-keyring.gpg > /dev/null \
+   && echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" \
+   | tee /etc/apt/sources.list.d/kitware.list > /dev/null \
+   && apt-get update \
+   && apt-get install -y cmake
+
+# Dotnet
+RUN apt-get update && apt-get install -y dotnet7
+
+# Node
+ARG node_major=18
+RUN mkdir -p /etc/apt/keyrings \
+   && curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key \
+   | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg \
+   && echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$node_major.x nodistro main" \
+   | tee /etc/apt/sources.list.d/nodesource.list \
+   && apt-get update \
+   && apt-get install -y nodejs
+
+# Emumba (deps)
+RUN apt-get update && apt-get -y install rapidjson-dev
+# Emumba
+COPY emumba /emumba
+RUN dpkg -i /emumba/emumba-deribit-connector-3.3.1-Linux.LICENSE_DISABLED.deb \
+    && rm -r /emumba
+
+# Pcap
+RUN apt-get update && apt-get install -y libpcap0.8-dev
+
+RUN apt-get update && apt-get install -y \
+    docker.io \
+    docker-compose \
+    docker-compose-v2
+
+# Set up default executables (update-alternatives)
+RUN \
+   update-alternatives --install /usr/bin/clang clang /usr/bin/clang-$clang_major 100; \
+   update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-$clang_major 100; \
+   update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-$clang_major 100; \
+   update-alternatives --install /usr/bin/clangd clangd /usr/bin/clangd-$clang_major 100; \
+   update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-$gcc_major 100; \
+   update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-$gcc_major 100
+
+# Add compiler symlinks for ccache 
+RUN \
+   ln -s /usr/bin/ccache /usr/local/bin/clang; \
+   ln -s /usr/bin/ccache /usr/local/bin/clang++; \
+   ln -s /usr/bin/ccache /usr/local/bin/clang-$clang_major; \
+   ln -s /usr/bin/ccache /usr/local/bin/clang++-$clang_major; \
+   ln -s /usr/bin/ccache /usr/local/bin/gcc; \
+   ln -s /usr/bin/ccache /usr/local/bin/g++
+
+RUN locale-gen en_US.UTF-8
+
+ARG docker_user="bts"
+ARG docker_uid="1000"
+ARG docker_pass="bts"
+
+RUN \
+   groupadd -g "$docker_uid" "$docker_user" \
+   && useradd \
+   -m \
+   -u "$docker_uid" \
+   -g "$docker_uid" \
+   -G sudo \
+   --password "$(openssl passwd -1 "$docker_pass")" \
+   "$docker_user"
+
+RUN usermod -aG docker "$docker_user"
@@ -0,0 +1,43 @@
+{
+  "dockerFile": "Dockerfile",
+  // Creates local dir to mount to HOME in docker 
+  "initializeCommand": "mkdir -p ${localEnv:HOME}/.bts/devcontainer-mounts/${devcontainerId}",
+  "hostRequirements": {
+    "memory": "4gb",
+    "storage": "6gb"
+  },
+  "postCreateCommand": "/home/bts/spark/docker/sparkd/dev/devcontainer_post_create.sh",
+  "containerUser": "bts",
+  "workspaceMount": "source=${localWorkspaceFolder},target=/home/bts/spark,type=bind",
+  "workspaceFolder": "/home/bts/spark",
+  "updateRemoteUserUID": true,
+  "shutdownAction": "none",
+  // Note: If running multiple containers on the same host, each must use a
+  // unique set of ports. Usually we just add 1000 to each port as needed. For
+  // example, if 8000-8003 is taken, go with 9000-9003, then 10000-10003, etc.
+  "runArgs": [
+    "--shm-size",
+    "2gb",
+    "--network",
+    "host"
+  ],
+  "mounts": [
+    "source=${localEnv:HOME}/.bts/devcontainer-mounts/${devcontainerId},target=/home/bts,type=bind",
+    "source=/var/run/docker.sock,target=/var/run/docker.sock,type=bind"
+  ],
+  "customizations": {
+    "vscode": {
+      "settings": {
+        "clangd.arguments": [
+          "--compile-commands-dir=${workspaceFolder}/build/clang/debug"
+        ]
+      },
+      "extensions": [
+        "eamodio.gitlens",
+        "llvm-vs-code-extensions.vscode-clangd",
+        "ms-vscode.cmake-tools",
+        "ms-vscode.cpptools"
+      ]
+    }
+  }
+}
@@ -42,7 +42,7 @@ trace_data/
 /.ignore
 *.rej
 .clangd/
-*CMakeDeps.cmake
+*CMakeDeps*.cmake
 /sniper.ipynb
 /src/.clang-format
 /src/server/sparkd/cmds/spartan/analysis/.ipynb_checkpoints
@@ -1,9 +1,6 @@
 [submodule "externals"]
 	path = externals
 	url = git@bitbucket.org:bluetradesys/externals.git
-[submodule "docker/linux"]
-	path = docker/linux
-	url = git@bitbucket.org:bluetradesys/linux_docker.git
-[submodule "docker/windows"]
-	path = docker/windows
-	url = git@bitbucket.org:bluetradesys/windows_docker.git
+[submodule "vcpkg"]
+	path = vcpkg
+	url = https://github.com/microsoft/vcpkg.git
@@ -1,5 +1,6 @@
 cmake_minimum_required(VERSION 3.12)
 project(spark)
+set(CMAKE_CXX_STANDARD 20)

 if(WIN32)
  set(platform "windows")
@@ -13,101 +14,49 @@ set(
  CACHE
  STRING
  "External dependencies root folder (should contain folders: lib, bin, include")
-set(SPARK_PYTHON "${EXTERNALS_ROOT}/python")
-set(SPARK_BIN "${EXTERNALS_ROOT}/bin")
-set(SPARK_INCLUDE "${EXTERNALS_ROOT}/include")
-set(SPARK_LIB "${EXTERNALS_ROOT}/lib/${CMAKE_BUILD_TYPE}")
+set(EXTERNALS_BIN "${EXTERNALS_ROOT}/bin")
+set(EXTERNALS_INCLUDE "${EXTERNALS_ROOT}/include")
+set(EXTERNALS_LIB "${EXTERNALS_ROOT}/lib/$<CONFIG>")
+set(BUILD_DOCS OFF CACHE BOOL "Build documentation")
 set(BUILD_SDK OFF CACHE BOOL "Build the sdk")
-set(BUILD_WEB ON CACHE BOOL "Build the web server")
+set(EXPORT_MD_CONSUMER OFF CACHE BOOL "Export (extern) C functions for MD consumer interop")
+if (WIN32)
+  set(BUILD_SPARKD OFF CACHE BOOL "Build sparkd")
+  set(BUILD_WEB OFF CACHE BOOL "Build the web server")
+else()
+  set(BUILD_SPARKD ON CACHE BOOL "Build sparkd")
+  set(BUILD_WEB ON CACHE BOOL "Build the web server")
+endif()

-include_directories(BEFORE SYSTEM ${SPARK_INCLUDE} ${EXTERNALS_ROOT}/include/zlib)
-link_directories(${SPARK_LIB})
+include_directories(BEFORE SYSTEM ${EXTERNALS_INCLUDE} ${EXTERNALS_ROOT}/include/zlib)
+link_directories(${EXTERNALS_LIB})

+add_compile_definitions(EXPORT_MD_CONSUMER="${EXPORT_MD_CONSUMER}")
 if(WIN32)
-  # flags
-  set(
-    CMAKE_CXX_FLAGS
-    "${CMAKE_CXX_FLAGS} \
-    /MP \
-    /WX /wd4251 /wd4250 /wd4251 /wd4244 /wd4267 /wd4355 /wd4275 /wd4345 \
-    /bigobj \
-    /DCURL_STATICLIB \
-    /D_WINSOCK_DEPRECATED_NO_WARNINGS \
-    /D__TBB_NO_IMPLICIT_LINKAGE \
-    /DBOOST_ALL_NO_LIB \
-    /std:c++17 \
-    /D_SCL_SECURE_NO_WARNINGS /D_CRT_SECURE_NO_WARNINGS /D_WIN32_WINNT=0x0600")
-  set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /MTd")
-  set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_DEBUG} /MT")
-  set(
-    CMAKE_SHARED_LINKER_FLAGS_RELEASE
-    "${CMAKE_SHARED_LINKER_FLAGS_RELEASE} \
-    /DEBUG \
-    /OPT:REF \
-    /OPT:ICF \
-    /LTCG")
-
  # subdirs
  add_subdirectory(src/lib)
 else()
-  # make sparkd look for libspark.so in its directory
-  set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE)
-  set(CMAKE_INSTALL_RPATH ".")
-  set(CMAKE_INSTALL_PREFIX "${CMAKE_SOURCE_DIR}/install/${CMAKE_BUILD_TYPE}/sparkd")
  add_compile_definitions(SPARK_ENABLE_CCAPI)
  add_compile_definitions(CCAPI_ENABLE_SERVICE_MARKET_DATA)
+  add_compile_definitions(CCAPI_ENABLE_SERVICE_EXECUTION_MANAGEMENT)
  add_compile_definitions(CCAPI_ENABLE_EXCHANGE_FTX)
  add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE)
+  add_compile_definitions(CCAPI_ENABLE_EXCHANGE_DERIBIT)
  add_compile_definitions(CCAPI_ENABLE_LOG_WARN)

-  # flags
-  set(
-    CMAKE_CXX_FLAGS_DEBUG
-    "${CMAKE_CXX_FLAGS_DEBUG} \
-    -fPIC \
-    -Wall \
-    -pedantic \
-    -Wextra \
-    -Werror \
-    -Wno-missing-braces \
-    -Wno-ignored-qualifiers \
-    --std=c++17 -fdiagnostics-color")
-  set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_DEBUG} -fno-omit-frame-pointer -DNDEBUG -O3")
-  set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -DDEBUG")
-  set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wno-undef")
-
  # subdirs
-  add_subdirectory(src/server/sparkd)
+  if (BUILD_SPARKD)
+    add_subdirectory(src/server/sparkd)
+    add_subdirectory(src/data_files)
+  endif()
  add_subdirectory(src/lib)
  if (BUILD_WEB)
    add_subdirectory(src/web)
  endif()
-  add_subdirectory(doc/user)
+  if(BUILD_DOCS)
+    add_subdirectory(doc/user)
+  endif()
  if(BUILD_SDK)
    add_subdirectory(src/sdk)
  endif()
-  add_subdirectory(src/data_files)
-
-  add_custom_command(
-    OUTPUT ${CMAKE_BINARY_DIR}/server
-    COMMAND mkdir -p ${CMAKE_BINARY_DIR}/server
-    COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/src/server/sparkd/sparkd ${CMAKE_BINARY_DIR}/server
-    COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/web ${CMAKE_BINARY_DIR}/server
-    COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/data_files ${CMAKE_BINARY_DIR}/server
-    COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/src/lib/libspark.so ${CMAKE_BINARY_DIR}/server
-    COMMAND
-      ln ARGS -sf
-      ${EXTERNALS_ROOT}/lib/Release/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.16.0.0
-      ${CMAKE_BINARY_DIR}/server
-    COMMAND ln ARGS -sf ${EXTERNALS_ROOT}/lib/Release/libACE-5.7.5.so ${CMAKE_BINARY_DIR}/server
-    COMMAND mkdir ARGS -p ${CMAKE_BINARY_DIR}/server/plugins
-    COMMENT "Creating server runtime layout"
-    DEPENDS sparkd data_files_dir)
-
-  add_custom_target(server_layout ALL DEPENDS ${CMAKE_BINARY_DIR}/server)
-
-  install(DIRECTORY "${CMAKE_BINARY_DIR}/server/plugins" DESTINATION bin)
-  install(FILES "${CMAKE_SOURCE_DIR}/externals/linux/lib/${CMAKE_BUILD_TYPE}/libACE-5.7.5.so" DESTINATION bin)
-  install(FILES "${CMAKE_SOURCE_DIR}/externals/linux/lib/${CMAKE_BUILD_TYPE}/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.16.0.0" DESTINATION bin)
-  install(FILES "${CMAKE_SOURCE_DIR}/externals/src/emumba/sparkd.lic" DESTINATION bin)
 endif()
@@ -0,0 +1,215 @@
+{
+  "version": 4,
+  "cmakeMinimumRequired": {
+    "major": 3,
+    "minor": 23,
+    "patch": 0
+  },
+
+  "configurePresets": [
+    {
+      "name": "spark",
+      "condition": {
+        "type": "equals",
+        "lhs": "${hostSystemName}",
+        "rhs": "Windows"
+      },
+      "generator": "Visual Studio 17 2022",
+      "binaryDir": "${sourceDir}/build/${presetName}",
+      "environment": {
+        "SPARK_CMAKE_CXX_FLAGS": "/MP /WX /wd4250 /wd4251 /wd4244 /wd4267 /wd4355 /wd4275 /wd4345 /bigobj /EHsc /DCURL_STATICLIB /D_WINSOCK_DEPRECATED_NO_WARNINGS /D__TBB_NO_IMPLICIT_LINKAGE /DBOOST_ALL_NO_LIB /std:c++20 /D_SCL_SECURE_NO_WARNINGS /D_CRT_SECURE_NO_WARNINGS /D_WIN32_WINNT=0x0600"
+      },
+      "cacheVariables": {
+        "CMAKE_CXX_FLAGS": "$env{SPARK_CMAKE_CXX_FLAGS}",
+        "CMAKE_CXX_FLAGS_DEBUG": "$env{SPARK_CMAKE_CXX_FLAGS} /MTd",
+        "CMAKE_CXX_FLAGS_RELEASE": "$env{SPARK_CMAKE_CXX_FLAGS} /MT",
+        "CMAKE_CXX_STANDARD": "20",
+        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
+        "CMAKE_SHARED_LINKER_FLAGS_DEBUG": "/DEBUG",
+        "CMAKE_SHARED_LINKER_FLAGS_RELEASE": "/OPT:REF /OPT:ICF /LTCG",
+        "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
+        "EXTERNALS_ROOT": "${sourceDir}/externals/windows",
+        "CONFIG_DEFAULTS_PATH": "config/defaults/*.conf",
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0;${sourceDir}/build/$<CONFIG>/bin/net7.0-windows",
+        "EXPORT_MD_CONSUMER": {
+          "type": "BOOL",
+          "value": "ON"
+        }
+      }
+    },
+
+    {
+      "name": "sparkd",
+      "condition": {
+        "type": "equals",
+        "lhs": "${hostSystemName}",
+        "rhs": "Linux"
+      },
+      "hidden": true,
+      "generator": "Ninja",
+      "binaryDir": "${sourceDir}/build/${presetName}",
+      "cacheVariables": {
+        "platform": "",
+        "CMAKE_CXX_STANDARD": "20",
+        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
+        "CMAKE_EXE_LINKER_FLAGS": "-Wno-undef",
+        "CONFIG_DEFAULTS_PATH": "config/defaults/*.conf",
+        "EXTERNALS_ROOT": "${sourceDir}/externals/linux",
+        "BUILD_SDK": {
+          "type": "BOOL",
+          "value": "OFF"
+        },
+        "BUILD_ADMIN_WEB_APP": {
+          "type": "BOOL",
+          "value": "ON"
+        }
+      }
+    },
+
+    {
+      "name": "sparkd-debug",
+      "inherits": "sparkd",
+      "hidden": true,
+      "cacheVariables": {
+        "CMAKE_BUILD_TYPE": "Debug",
+        "CMAKE_EXPORT_COMPILE_COMMANDS": {
+          "type": "BOOL",
+          "value": "ON"
+        },
+        "CCAPI_LOG_LEVEL": "Info"
+      }
+    },
+
+    {
+      "name": "sparkd-release",
+      "inherits": "sparkd",
+      "hidden": true,
+      "cacheVariables": {
+        "CMAKE_BUILD_TYPE": "Release",
+        "CCAPI_LOG_LEVEL": "Warn"
+      }
+    },
+
+    {
+      "name": "sparkd-debug-clang",
+      "inherits": "sparkd-debug",
+      "displayName": "Sparkd (Debug | Clang)",
+      "description": "Debug build compiled w/ clang",
+      "cacheVariables": {
+        "CMAKE_C_COMPILER": "clang",
+        "CMAKE_CXX_COMPILER": "clang++",
+        "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG"
+      }
+    },
+
+    {
+      "name": "sparkd-debug-gcc",
+      "inherits": "sparkd-debug",
+      "displayName": "Sparkd (Debug | Gcc)",
+      "description": "Debug build compiled w/ gcc",
+      "cacheVariables": {
+        "CMAKE_C_COMPILER": "gcc",
+        "CMAKE_CXX_COMPILER": "g++",
+        "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -Wno-stringop-overread -DDEBUG"
+      }
+    },
+
+    {
+      "name": "sparkd-release-clang",
+      "inherits": "sparkd-release",
+      "displayName": "Sparkd (Release | Clang)",
+      "description": "Release build compiled w/ clang",
+      "cacheVariables": {
+        "CMAKE_C_COMPILER": "clang",
+        "CMAKE_CXX_COMPILER": "clang++",
+        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3"
+      }
+    },
+
+    {
+      "name": "sparkd-release-gcc",
+      "inherits": "sparkd-release",
+      "displayName": "Sparkd (Release | Gcc)",
+      "description": "Release build compiled w/ gcc",
+      "cacheVariables": {
+        "CMAKE_C_COMPILER": "gcc",
+        "CMAKE_CXX_COMPILER": "g++",
+        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers -Wno-stringop-overflow --std=c++20 -fdiagnostics-color -Wno-stringop-overread -Wno-maybe-uninitialized -fno-omit-frame-pointer -DNDEBUG -O3"
+      }
+    }
+  ],
+
+  "buildPresets": [
+    {
+      "name": "spark",
+      "condition": {
+        "type": "equals",
+        "lhs": "${hostSystemName}",
+        "rhs": "Windows"
+      },
+      "hidden": true
+    },
+
+    {
+      "name": "spark-debug",
+      "displayName": "Spark (Debug)",
+      "inherits": "spark",
+      "configurePreset": "spark",
+      "configuration": "Debug"
+    },
+
+    {
+      "name": "spark-release",
+      "displayName": "Spark (Release)",
+      "inherits": "spark",
+      "configurePreset": "spark",
+      "configuration": "Release"
+    },
+
+    {
+      "name": "sparkd",
+      "condition": {
+        "type": "equals",
+        "lhs": "${hostSystemName}",
+        "rhs": "Linux"
+      },
+      "hidden": true
+    },
+
+    {
+      "name": "sparkd-debug-clang",
+      "configurePreset": "sparkd-debug-clang",
+      "displayName": "Sparkd (Debug | Clang)",
+      "inherits": "sparkd"
+    },
+
+    {
+      "name": "sparkd-debug-clang-j6",
+      "configurePreset": "sparkd-debug-clang",
+      "displayName": "Sparkd (Debug | Clang | jobs=6)",
+      "inherits": "sparkd",
+      "jobs": 6
+    },
+
+    {
+      "name": "sparkd-debug-gcc",
+      "configurePreset": "sparkd-debug-gcc",
+      "displayName": "Sparkd (Debug | Gcc)",
+      "inherits": "sparkd"
+    },
+
+    {
+      "name": "sparkd-release-clang",
+      "configurePreset": "sparkd-release-clang",
+      "displayName": "Sparkd (Release | Clang)",
+      "inherits": "sparkd"
+    },
+
+    {
+      "name": "sparkd-release-gcc",
+      "configurePreset": "sparkd-release-gcc",
+      "displayName": "Sparkd (Release | Gcc)",
+      "inherits": "sparkd"
+    }
+  ]
+}
@@ -0,0 +1,39 @@
+include(${CMAKE_SOURCE_DIR}/cmake/PackBasic.cmake)
+include(GNUInstallDirs)
+
+function(install_basic)
+  set(_options)
+  set(_one_value_args)
+  set(_multi_value_args HEADERS)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+  if (_ARGS_HEADERS)
+    set(${PROJECT_NAME}_HEADERS ${_ARGS_HEADERS})
+  else()
+    file(
+      GLOB_RECURSE ${PROJECT_NAME}_HEADERS
+      RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
+      *.h
+    )
+  endif()
+
+  target_sources(${PROJECT_NAME}
+    PUBLIC
+      FILE_SET HEADERS FILES ${${PROJECT_NAME}_HEADERS})
+
+  install(
+    TARGETS ${PROJECT_NAME}
+    EXPORT ${PROJECT_NAME}Targets
+    RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
+    LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
+    ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
+    FILE_SET HEADERS DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}
+  )
+
+  install(
+    EXPORT ${PROJECT_NAME}Targets
+    FILE ${PROJECT_NAME}Targets.cmake
+    NAMESPACE ${PROJECT_NAME}::
+    DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME})
+
+  pack_basic()
+endfunction()
@@ -0,0 +1,36 @@
+include(CMakePackageConfigHelpers)
+
+function(pack_basic)
+  find_package(Git QUIET)
+
+  if(GIT_FOUND)
+    execute_process(
+      COMMAND ${GIT_EXECUTABLE} describe --tags --always --dirty
+      WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
+      OUTPUT_VARIABLE PROJECT_VERSION
+      OUTPUT_STRIP_TRAILING_WHITESPACE
+    )
+  endif()
+
+  if(NOT GIT_FOUND OR PROJECT_VERSION STREQUAL "")
+    set(PROJECT_VERSION "0.0.0")
+  endif()
+
+  write_basic_package_version_file(
+    "${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}ConfigVersion.cmake"
+    VERSION ${PROJECT_VERSION}
+    COMPATIBILITY AnyNewerVersion
+  )
+
+  configure_package_config_file(
+    "${CMAKE_CURRENT_SOURCE_DIR}/Config.cmake.in"
+    "${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}Config.cmake"
+    INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}
+  )
+
+  install(
+    FILES
+      "${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}Config.cmake"
+      "${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}ConfigVersion.cmake"
+    DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME})
+endfunction()
@@ -0,0 +1,12 @@
+function(print_list var)
+  set(CMAKE_MESSAGE_INDENT "")
+  set(_var_str "")
+  foreach(item IN LISTS ${var})
+    set(_var_str "${_var_str}\n  ${item}")
+  endforeach()
+  if ("${ARGN}" STREQUAL "VERBOSE")
+    message(VERBOSE "***${var}:${_var_str}")
+  else()
+    message("***${var}:${_var_str}")
+  endif()
+endfunction()
@@ -1,5 +1,5 @@
-set(Protobuf_PROTOC_EXECUTABLE "${SPARK_BIN}/protoc")
-set(PROTOBUF_INCLUDE_DIRS "${SPARK_INCLUDE}")
+set(Protobuf_PROTOC_EXECUTABLE "${EXTERNALS_BIN}/protoc")
+set(PROTOBUF_INCLUDE_DIRS "${EXTERNALS_INCLUDE}")
 set(PROTOBUF_STATIC_LIB "libprotobuf")

 function(Protobuf_Generate_Cpp SRCS HDRS)
@@ -4,8 +4,6 @@ VisualStudioVersion = 17.3.32922.545
 MinimumVisualStudioVersion = 15.0.26124.0
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_api", "src\client\spark_api\spark_api.csproj", "{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}"
 EndProject
-Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "libspark", "build\src\lib\libspark.vcxproj", "{FDC3A021-386E-3A2C-A7EB-CBE94A95BE25}"
-EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Spark", "src\client\spark\Spark.csproj", "{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}"
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "sparkcmd", "src\client\sparkcmd\sparkcmd.csproj", "{387566C6-C6F6-412C-A30C-9855BBCB4816}"
@@ -13,9 +11,6 @@ EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_core", "src\client\spark_core\spark_core.csproj", "{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}"
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_sdk", "src\client\spark_sdk\spark_sdk.csproj", "{6963C314-83C4-4667-B0D0-C2F7AE208C29}"
-	ProjectSection(ProjectDependencies) = postProject
-		{FDC3A021-386E-3A2C-A7EB-CBE94A95BE25} = {FDC3A021-386E-3A2C-A7EB-CBE94A95BE25}
-	EndProjectSection
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_test", "src\client\spark_test\spark_test.csproj", "{ECD78CC5-138F-4809-A764-2358C2AC2E82}"
 EndProject
@@ -3,15 +3,16 @@
 using mbts.co2;

 namespace mbts.analytics {
+   using Size = System.Single;
   public class VolSurfaceMDUnit : IEquatable<VolSurfaceMDUnit> {
      public readonly double bid_price, ask_price;
-      public readonly uint bid_size, ask_size;
+      public readonly Size bid_size, ask_size;
      public readonly double mark_price = double.NaN;
      private static VolSurfaceMDUnit _empty =
         new VolSurfaceMDUnit(Double.NaN, 0, Double.NaN, 0, Double.NaN);

-      public VolSurfaceMDUnit(double this_bid_price, uint this_bid_size,
-                               double this_ask_price, uint this_ask_size,
+      public VolSurfaceMDUnit(double this_bid_price, Size this_bid_size,
+                               double this_ask_price, Size this_ask_size,
                               double this_mark_price) {
         bid_price = this_bid_price;
         bid_size = this_bid_size;
@@ -28,7 +29,7 @@ namespace mbts.analytics {

      public VolSurfaceMDUnit(NativePriceSize bid, NativePriceSize ask,
                              double mark_price)
-      : this(bid.price, bid.size, ask.price, ask.size, mark_price) {
+      : this(bid.price, bid.size_f, ask.price, ask.size_f, mark_price) {
      }

      public bool is_two_sided() { return bid_size != 0 && ask_size != 0; }
@@ -41,7 +42,7 @@ namespace mbts.analytics {
      }

      public double get_weighted_mid_price(bool get_one_sided_mid = true) {
-         uint sum_q = bid_size + ask_size;
+         Size sum_q = bid_size + ask_size;
         if (sum_q == 0) {
            return Double.NaN;
         }
@@ -52,10 +52,16 @@ namespace bts {
         string rv = "MergedMDDaemon Options:\n";
         foreach (var kv in sec_map) {
            var sec = kv.Value.security;
+            var md = kv.Value.md;
+            ulong time = 0;
+            NativePriceSize bid, ask;
+            var has_md = md.get_bbo(ref time, out bid, out ask);
            if (sec.HasOption) {
               rv += $"id: {kv.Key} desc: {sec.security_desc()} sym: {sec.Symbol} " +
                     $"exp: {sec.security_expiration()} " +
-                     $"k: {sec.Option.Strike} cp: {sec.Option.CallPut}" + "\n";
+                     $"k: {sec.Option.Strike} cp: {sec.Option.CallPut} " +
+                     $"bid: p: {bid.price} q: {bid.size} qf: {bid.size_f} ask: p: {ask.price} q: {ask.size} qf: {ask.size_f}" +
+                     "\n";
            }
         }
         Log.Gen.info(rv);
@@ -218,10 +218,14 @@ namespace mbts.spark {
         background = buy_sell_background_fill_color;
         hoverbackground = _mk_hover_bg_brush(dark, _darkbg, 0);

-         (_num_root_orders, _filled, _quantity) = _root_nodes
-            .Select(n => (1u, n.filled_quantity(), n.quantity()))
-            .Sum();
-
+         _num_root_orders = 0;
+         _filled = 0;
+         _quantity = 0;
+         foreach (var n in _root_nodes) {
+            _num_root_orders++;
+            _filled += n.filled_quantity();
+            _quantity += n.quantity();
+         }
         if (_num_root_orders == 1) {
            var order = _root_nodes.First();
            border = _owner.model.order_gateway.factory.get_metadata(order).Color.rgb_brush();
@@ -1,12 +1,10 @@
 using mbts.analytics;

 using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
+

 namespace BCG {
+   using Size = System.Single;
   public enum BidAskAlertZone { buy, sell, none }
   public class BBOPacket {
      public enum MarketSide {
@@ -15,8 +13,8 @@ namespace BCG {

      private double _bid_price => md.bid_price;
      private double _ask_price => md.ask_price;
-      private UInt32 _bid_size => md.bid_size;
-      private UInt32 _ask_size => md.ask_size;
+      private Size _bid_size => md.bid_size;
+      private Size _ask_size => md.ask_size;
      private static BBOPacket _null_pack = new BBOPacket(VolSurfaceMDUnit.empty);
      public readonly VolSurfaceMDUnit md;

@@ -24,21 +22,6 @@ namespace BCG {
         this.md = md;
      }

-      public static bool bid_through_theo(BBOPacket md, mbts.spark.TheoPoint tp, bool is_call) {
-         return false;
-         // if (tp != null && md != null && md._md_row != null)
-         //    return get_bid_price(md) > (is_call ? tp.call : tp.put);
-         // else
-         //    return false;
-      }
-
-      public static bool ask_through_theo(BBOPacket md, mbts.spark.TheoPoint tp, bool is_call) {
-         // if (tp != null && md != null && md._md_row != null)
-         //    return get_ask_price(md) < (is_call ? tp.call : tp.put);
-         // else
-         return false;
-      }
-
      public static double get_mid_price(BBOPacket md) {
         return (get_bid_price(md) + get_ask_price(md)) / 2.0;
      }
@@ -47,16 +30,16 @@ namespace BCG {
         return md != null ? md._bid_price : Double.NaN;
      }

-      public static UInt32 get_bid_size(BBOPacket md) {
-         return md != null ? md._bid_size : (UInt32)0;
+      public static Size get_bid_size(BBOPacket md) {
+         return md != null ? md._bid_size : (Size)0;
      }

      public static double get_ask_price(BBOPacket md) {
         return md != null ? md._ask_price : Double.NaN;
      }

-      public static UInt32 get_ask_size(BBOPacket md) {
-         return md != null ? md._ask_size : (UInt32)0;
+      public static Size get_ask_size(BBOPacket md) {
+         return md != null ? md._ask_size : (Size)0;
      }
   }
 }
@@ -190,13 +190,24 @@ namespace mbts.analytics {
         }
      }

-      void _compute_swipe_sum() =>
-         (swipe_sum_puts, swipe_sum_calls, swipe_sum_net) = _visible_selected_cells
-            .SelectMany(tup =>
-               tup.exps.Select(exp => {
-                  var (p, c, n) = tup.row.get_cell_values(exp);
-                  return (p.if_nan(0), c.if_nan(0), n.if_nan(0));
-               }))
-            .Sum();
+      void _compute_swipe_sum() {
+         swipe_sum_puts = 0;
+         swipe_sum_calls = 0;
+         swipe_sum_net = 0;
+         foreach (var (row, exps) in _visible_selected_cells) {
+            foreach (var exp in exps) {
+               var (p, c, n) = row.get_cell_values(exp);
+               if (!p.is_nan()) {
+                  swipe_sum_puts += p;
+               }
+               if (!c.is_nan()) {
+                  swipe_sum_calls += c;
+               }
+               if (!n.is_nan()) {
+                  swipe_sum_net += n;
+               }
+            }
+         }
+      }
   }
 }
@@ -1,4 +1,5 @@
 <Project Sdk="Microsoft.NET.Sdk">
+
  <PropertyGroup>
    <TargetFramework>net7.0-windows</TargetFramework>
    <UseWPF>true</UseWPF>
@@ -21,33 +22,39 @@
    <RootNamespace>bts</RootNamespace>
    <TreatWarningsAsErrors>true</TreatWarningsAsErrors>
    <WarningsAsErrors />
-    <Version>$(major).$(minor).$(patch)</Version>
-    <Version Condition="'$(version)'=='..'">1.0.0</Version>
  </PropertyGroup>
+
  <ItemGroup>
+    <PackageReference Include="CommandLineParser" Version="2.9.1" />
+    <PackageReference Include="GitVersion.MsBuild" Version="5.12.0">
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+      <PrivateAssets>all</PrivateAssets>
+    </PackageReference>
+    <PackageReference Include="Google.Protobuf" Version="3.24.3" />
+    <PackageReference Include="IronPython" Version="3.4.1" />
    <PackageReference Include="MathNet.Numerics" Version="5.0.0" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.2.0" />
+    <PackageReference Include="NAudio" Version="2.0.1" />
+    <PackageReference Include="NUnit" Version="3.13.3" />
+    <PackageReference Include="NUnit3TestAdapter" Version="4.2.1" />
+    <PackageReference Include="Newtonsoft.Json" Version="13.0.1" />
+    <PackageReference Include="OxyPlot.Core" Version="2.1.2" />
+    <PackageReference Include="OxyPlot.Wpf" Version="2.1.2" />
+    <PackageReference Include="protobuf-net" Version="3.2.26" />
    <PackageReference Include="System.Composition" Version="7.0.0-preview.2.22152.2" />
    <PackageReference Include="System.Composition.AttributedModel" Version="7.0.0-preview.2.22152.2" />
    <PackageReference Include="System.Composition.Hosting" Version="7.0.0-preview.2.22152.2" />
    <PackageReference Include="System.Composition.TypedParts" Version="7.0.0-preview.2.22152.2" />
-    <PackageReference Include="System.Reactive.Core" Version="5.0.0" />
-    <PackageReference Include="System.Reactive.Interfaces" Version="5.0.0" />
-    <PackageReference Include="System.Reactive.Linq" Version="5.0.0" />
-    <PackageReference Include="System.Reactive.PlatformServices" Version="5.0.0" />
-    <PackageReference Include="CommandLineParser" Version="2.9.1" />
-    <ProjectReference Include="../spark_api/spark_api.csproj" />
-    <ProjectReference Include="../spark_core/spark_core.csproj" />
-    <PackageReference Include="Google.Protobuf" Version="3.21.4" />
-    <PackageReference Include="IronPython" Version="2.7.12" />
-    <PackageReference Include="NAudio" Version="2.0.1" />
-    <PackageReference Include="Newtonsoft.Json" Version="13.0.1" />
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.2.0" />
-    <PackageReference Include="NUnit" Version="3.13.3" />
-    <PackageReference Include="NUnit3TestAdapter" Version="4.2.1" />
-    <PackageReference Include="System.Reactive" Version="5.0.0" />
-    <PackageReference Include="protobuf-net" Version="3.1.17" />
-    <PackageReference Include="OxyPlot.Core" Version="2.1.2" />
-    <PackageReference Include="OxyPlot.Wpf" Version="2.1.2" />
+    <PackageReference Include="System.Reactive" Version="6.0.0" />
+    <PackageReference Include="System.Reactive.Core" Version="6.0.0" />
+    <PackageReference Include="System.Reactive.Interfaces" Version="6.0.0" />
+    <PackageReference Include="System.Reactive.Linq" Version="6.0.0" />
+    <PackageReference Include="System.Reactive.PlatformServices" Version="6.0.0" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\spark_api\spark_api.csproj" />
+    <ProjectReference Include="..\spark_core\spark_core.csproj" />
  </ItemGroup>

  <ItemGroup>
@@ -55,15 +62,20 @@
    <Resource Include="**/*.png" />
    <ConfigDefaults Include="../../lib/config/defaults/*.conf" />
  </ItemGroup>
+
  <ItemGroup>
    <Resource Remove="logo.ico" />
  </ItemGroup>
+
  <ItemGroup>
    <Content Include="logo.ico" />
  </ItemGroup>
+
  <ItemGroup>
    <EmbeddedResource Include="Sounds/game_sound_correct.wav" />
    <EmbeddedResource Include="Sounds/whistle.wav" />
  </ItemGroup>
+
  <Target Name="diagnostic" AfterTargets="PreBuildEvent" />
+
 </Project>
@@ -162,7 +162,7 @@

               // Compare bids
               if (direct.int_price == implied.int_price) {
-                  bid[x] = new NativePriceSize(direct.int_price, direct.size, implied.size);
+                  bid[x] = new NativePriceSize(direct.int_price, direct.size_f, implied.size_f);
                  dbid_idx++;
                  ibid_idx++;
               }
@@ -185,7 +185,7 @@

               // Compare asks
               if (direct.int_price == implied.int_price) {
-                  ask[x] = new NativePriceSize(direct.int_price, direct.size, implied.size);
+                  ask[x] = new NativePriceSize(direct.int_price, direct.size_f, implied.size_f);
                  dask_idx++;
                  iask_idx++;
               }
@@ -1,14 +1,15 @@
 namespace mbts.co2 {
   using System.Runtime.InteropServices;

+   using Size = System.Single;
   [StructLayout(LayoutKind.Sequential)]
   public readonly struct NativePriceLevel {
      readonly IntPrice _bid;
      readonly IntPrice _ask;
-      readonly uint _bid_size;
-      readonly uint _ask_size;
+      readonly Size _bid_size;
+      readonly Size _ask_size;

-      public NativePriceLevel(IntPrice bid, uint bid_size, IntPrice ask, uint ask_size) {
+      public NativePriceLevel(IntPrice bid, Size bid_size, IntPrice ask, Size ask_size) {
         _bid = bid;
         _bid_size = bid_size;
         _ask = ask;
@@ -5,27 +5,29 @@ namespace mbts.co2 {
   using System;
   using System.Runtime.InteropServices;

+   using Size = System.Single;
+
   public readonly struct NativePriceSize : IEquatable<NativePriceSize> {
      readonly IntPrice _price;
-      readonly uint _size;
-      readonly uint _direct_size;
+      readonly Size _size;
+      readonly Size _direct_size;

      static NativePriceSize() =>
         Log.assert(Marshal.SizeOf<NativePriceSize>() == 16);

-      public NativePriceSize(IntPrice price, uint size) {
+      public NativePriceSize(IntPrice price, Size size) {
         _price = price;
         _size = size;
         _direct_size = size;
      }

-      public NativePriceSize(long mantissa, sbyte exponent, uint size) {
+      public NativePriceSize(long mantissa, sbyte exponent, Size size) {
         _price = new IntPrice(mantissa, exponent);
         _size = size;
         _direct_size = size;
      }

-      public NativePriceSize(IntPrice price, uint direct_size, uint implied_size) {
+      public NativePriceSize(IntPrice price, Size direct_size, Size implied_size) {
         _price = price;
         _size = direct_size + implied_size;
         _direct_size = direct_size;
@@ -36,15 +38,18 @@ namespace mbts.co2 {
      public IntPrice int_price => _price;
      public long mantissa => _price.mantissa;
      public sbyte exponent => _price.exponent;
-      public uint size => _size;
-      public uint direct_size => _direct_size;
-      public uint implied_size => _size - _direct_size;
+      public uint size => (uint)_size;
+      public uint direct_size => (uint)_direct_size;
+      public uint implied_size => (uint)(_size - _direct_size);
+      public Size size_f => _size;
+      public Size direct_size_f => _direct_size;
+      public Size implied_size_f => _size - _direct_size;
      public double price => _price.price();
      public bool is_nan => _price.is_nan;


      public override string ToString() =>
-         $"{price} {size} ({implied_size})";
+         $"{price} {size_f} ({implied_size})";

      public override bool Equals(object obj) =>
         obj is NativePriceSize p ? Equals(p) : base.Equals(obj);
@@ -3,13 +3,14 @@ using mbts.co2;

 namespace bts.md
 {
+   using Size = System.Single;
   public class PriceSize 
   {      
-      public uint size { get; }
-      public uint direct_size { get; }
-      public uint implied_size { get; }
+      public Size size { get; }
+      public Size direct_size { get; }
+      public Size implied_size { get; }
      public double price { get; }
-      public PriceSize(uint sz, uint direct_sz, uint implied_sz, double prx)
+      public PriceSize(Size sz, Size direct_sz, Size implied_sz, double prx)
      {
         size = sz;
         direct_size = direct_sz;
@@ -1,6 +1,7 @@
 namespace bts.api {
   using System;
   using System.Linq;
+   using System.Reflection.Metadata.Ecma335;

   public static class SecurityExtensions {
      public static Security.ContractOneofCase contract_case(this SecurityType type) =>
@@ -96,6 +97,8 @@ namespace bts.api {
            return security.Ice.SecurityDesc;
         } else if (security.HasDeribit) {
            return security.Deribit.SecurityDesc;
+         } else if (security.HasFtx) {
+            return security.Ftx.SecurityDesc;
         }
         Log.assert(false, "security_desc not implemented for exchange");
         return null;
@@ -136,6 +139,7 @@ namespace bts.api {
            domain.Venue.Bts => Venue.Bts,
            domain.Venue.Sim => Venue.Sim,
            domain.Venue.Deribit => Venue.Deribit,
+            domain.Venue.Ftx => Venue.Ftx,
            domain.Venue.Glbx => Venue.Glbx,
            domain.Venue.None => Venue.Unset,
            _ => throw new NotImplementedException(),
@@ -146,6 +150,7 @@ namespace bts.api {
            Venue.Bts => domain.Venue.Bts,
            Venue.Sim => domain.Venue.Sim,
            Venue.Deribit => domain.Venue.Deribit,
+            Venue.Ftx => domain.Venue.Ftx,
            Venue.Glbx => domain.Venue.Glbx,
            Venue.Unset => domain.Venue.None,
            _ => throw new NotImplementedException(),
@@ -45,7 +45,7 @@
         string.IsNullOrEmpty(exch_desc) ? format_option(symbol, option) : exch_desc;

      static string _format(SecurityImmutable? security) =>
-         security.HasValue ? _format(security.Value) : "Unknown-security";
+         security.HasValue ? _format(security.Value) : "Unknown-security1";

      static string _format(SecurityImmutable security, bool exchange_specific_formatting = true) {
         switch (security.ContractCase) {
@@ -71,8 +71,10 @@
               return string.Join("", _format_legs(new ContractLegEnumerable(security.Spread.Legs, 1)));
            case Security.ContractOneofCase.Synthetic:
               return security.Synthetic.Name;
+            case Security.ContractOneofCase.Spot:
+               return security.security_desc();
            default:
-               return "Unknown-security";
+               return "Unknown-security2";
         }
      }

@@ -79,6 +79,40 @@ Misc:
 To search for a publisher for an event search for make_stream<Event_Name>, for 
 example make_stream<OrderStateUpdateEvent>()

+--------------------------------------------------------------------------
+market data
+--------------------------------------------------------------------------
+adding exchanges:
+the following places need to accommodate exchanges
+managed:
+- SecurityExtensions: public static string security_desc(this SecurityImmutable security) ..
+- SecurityExtensions: public static Venue to_venue(this domain.Venue api_venue) ..
+- SecurityExtensions: public static domain.Venue to_api_venue(this Venue venue) ..
+C++
+- AdminServiceUtils.cpp, 6 switch statements should compilation error
+- Symbols.proto in Symbols
+- Symbols.conf exchange section in SymbolLists and Symbols
+- Security.proto, 3 places 
+- PlatformLoader.cpp, in get_system_md_symbols(), could be concverted into switch
+- FixedSecurity.cpp, exchange_id(..) and convert_security(..) both switch
+- QueryServer set_mic, switch
+- VenueOption name(..) switch
+
+---------------------------------------------------------------------------
+adding security type
+---------------------------------------------------------------------------
+ modified:   src/lib/api/service_helpers/AdminServiceUtils.cpp
+        modified:   src/lib/domain_model/Security.cpp
+        modified:   src/lib/domain_model/Security.proto
+        modified:   src/lib/md/MDConsumer.cpp
+        modified:   src/lib/md/PlatformLoader.cpp
+        modified:   src/lib/md/SecurityMapper.cpp
+        modified:   src/lib/md/Shm/FixedSecurity.cpp
+        modified:   src/lib/md/Shm/FixedSecurity.h
+        modified:   src/lib/md/ccApi/CCMain.cpp
+        modified:   src/lib/md/ccApi/CCMarketDataListener.cpp
+        modified:   src/lib/md/relay/RelayClient.cpp
+        modified:   src/lib/orderman/limiter/Limiter.cpp

 ALGO:
 ----
@@ -1 +0,0 @@
-set(sources api/analytics/ExpirationDriver.proto;api/analytics/TimeWeightedInterpolatedDriver.proto;api/apicore.proto;api/apigateway.proto;api/apioptions.proto;api/core/ApiClientInterop.cpp;api/core/ApiServerInterop.cpp;api/domain/Currency.proto;api/domain/SecurityCriteria.proto;api/domain/SecurityCriteriaClause.proto;api/domain/SecurityExpiration.proto;api/domain/SecurityMetadata.proto;api/domain/SecurityMetadataCollection.proto;api/domain/SecurityMetadataItem.proto;api/domain/SecurityType.proto;api/domain/TickDisplaySettings.proto;api/domain/TickFactors.proto;api/domain/Venue.proto;api/io/URL.proto;api/md/MarketDataSubscriptionParams.proto;api/md/binance/BinanceDepthStreamParams.proto;api/md/binance/BinanceMarketDataSubscriptionParams.proto;api/md/deribit/DeribitMarketDataSubscriptionParams.proto;api/md/ftx/FTXDepthStreamParams.proto;api/md/ftx/FTXMarketDataSubscriptionParams.proto;api/md/glbx/GlbxMarketDataSubscriptionParams.proto;api/md/huobi/HuobiMarketDataSubscriptionParams.proto;api/public/ApiServer.cpp;api/public/ClientChannel.cpp;api/public/QuickbusClientChannel.cpp;api/public/ServerChannel.cpp;api/public/ServiceContext.cpp;api/service_helpers/AdminServiceUtils.cpp;api/service_helpers/AlgoServiceUtils.cpp;api/service_helpers/PositionUtilities.cpp;api/services/AdminService.proto;api/services/AlgoDefinitionService.proto;api/services/AlgoService.proto;api/services/AnalyticsService.proto;api/services/ApiUnitTestService.proto;api/services/AuthService.proto;api/services/BasePositionService.proto;api/services/ErrorReportService.proto;api/services/HelloService.proto;api/services/MBOService.proto;api/services/MarginService.proto;api/services/MarketDataAuxiliaryService.proto;api/services/MarketDataRelay.proto;api/services/MarketDataService.proto;api/services/MoneyFileService.proto;api/services/OrderService.proto;api/services/OutOfProcessURLListener.proto;api/services/PersistenceService.proto;api/services/PositionService.proto;api/services/ShellDiagnosticService.proto;api/services/ShellLatencyService.proto;api/services/SyntheticSecurityService.proto;api/test/ApiTests.cpp;api/test/HelloServiceServer.cpp;communincation/websockets/tests/WebsocketSyncClientTests.cpp;config/ConfigCheck.cpp;config/ConfigDownloader.cpp;config/ConfigErrorCollector.cpp;config/ConfigLog.cpp;config/ConfigPropertyEvaluator.cpp;config/ConfigValueQueue.cpp;config/ConfigurationException.cpp;config/ConfigurationState.cpp;config/PreprocessInputStream.cpp;config/config.cpp;config/defaults/MarketData.conf;config/defaults/Ports.conf;config/defaults/SecurityMetadataCollection.conf;config/defaults/Symbols.conf;config/defaults/cme_reference_data.conf;config/defaults/edge.conf;config/defaults/jetfix.conf;config/defaults/keycode.conf;config/defaults/logging.conf;config/defaults/orderman_cfg.conf;config/defaults/orders_fills_cleanup.conf;config/defaults/position_cfg.conf;config/defaults/span.conf;config/defaults/spreader.conf;config/defaults/testkeys.conf;config/defaults/threads.conf;config/defaults/toggles.conf;config/extensions/extensions.proto;config/invalid/invalid.proto;config/protos/algo_types.proto;config/protos/cme_reference_data.proto;config/protos/exchange_sim_cfg.proto;config/protos/jetfix.proto;config/protos/keycode.proto;config/protos/logging.proto;config/protos/market_data.proto;config/protos/mercury_cfg.proto;config/protos/orderman_cfg.proto;config/protos/orders_fills_cleanup.proto;config/protos/perflog_cfg.proto;config/protos/ports.proto;config/protos/position_cfg.proto;config/protos/quickbus_cfg.proto;config/protos/span.proto;config/protos/spartan.proto;config/protos/spreader.proto;config/protos/subprocess.proto;config/protos/symbols.proto;config/protos/system.proto;config/protos/testkeys.proto;config/protos/threads.proto;config/protos/toggles.proto;config/protos/ui.proto;config/tests/HTTPServer.cpp;config/tests/config_ref_tests.cpp;config/tests/config_tests.cpp;config/text_format/text_format.cpp;domain/AggressorSide.cpp;domain/NewOrder.cpp;domain/OrderDefinition.cpp;domain/OrderStatus.cpp;domain/OrderUpdate.cpp;domain/Side.cpp;domain/TimeInForce.cpp;domain_model/Security.cpp;domain_model/Security.proto;domain_model/SecurityTests.cpp;domain_model/sod.cpp;domain_model/tests/domain_tests.cpp;expression/BooleanGrammar.cpp;expression/ExpressionEvaluator.cpp;expression/ParserExceptions.cpp;expression/Value.cpp;expression/tests/expression_parser_tests.cpp;io/QueryParams.cpp;io/URL.cpp;io/http/CurlHandle.cpp;io/http/HTTPClient.cpp;io/http/HTTPHeaders.cpp;io/http/HTTPRequest.cpp;io/http/HTTPResponse.cpp;io/websocket/WebsocketClient.cpp;io/websocket/WebsocketClientConfiguration.cpp;io/websocket/WebsocketError.cpp;io/websocket/WebsocketMessage.cpp;io/websocket/WebsocketSSLClientInternal.cpp;io/websocket/tests/WebsocketClientTests.cpp;jetfix/codec/FIXArray.cpp;jetfix/common/OutFIX.cpp;jetfix/connection/AsyncReader.cpp;jetfix/connection/IConnection.cpp;jetfix/connection/SocketConnection.cpp;jetfix/gateway/CoreGateway.cpp;jetfix/gateway/JetFIXGatLine truncated
@@ -1 +0,0 @@
-set(sources api/analytics/ExpirationDriver.proto;api/analytics/TimeWeightedInterpolatedDriver.proto;api/apicore.proto;api/apigateway.proto;api/apioptions.proto;api/core/ApiClientInterop.cpp;api/core/ApiServerInterop.cpp;api/domain/Currency.proto;api/domain/SecurityCriteria.proto;api/domain/SecurityCriteriaClause.proto;api/domain/SecurityExpiration.proto;api/domain/SecurityMetadata.proto;api/domain/SecurityMetadataCollection.proto;api/domain/SecurityMetadataItem.proto;api/domain/SecurityType.proto;api/domain/TickDisplaySettings.proto;api/domain/TickFactors.proto;api/domain/Venue.proto;api/io/URL.proto;api/md/MarketDataSubscriptionParams.proto;api/md/binance/BinanceDepthStreamParams.proto;api/md/binance/BinanceMarketDataSubscriptionParams.proto;api/md/deribit/DeribitMarketDataSubscriptionParams.proto;api/md/ftx/FTXDepthStreamParams.proto;api/md/ftx/FTXMarketDataSubscriptionParams.proto;api/md/glbx/GlbxMarketDataSubscriptionParams.proto;api/md/huobi/HuobiMarketDataSubscriptionParams.proto;api/public/ApiServer.cpp;api/public/ClientChannel.cpp;api/public/QuickbusClientChannel.cpp;api/public/ServerChannel.cpp;api/public/ServiceContext.cpp;api/service_helpers/AdminServiceUtils.cpp;api/service_helpers/AlgoServiceUtils.cpp;api/service_helpers/PositionUtilities.cpp;api/services/AdminService.proto;api/services/AlgoDefinitionService.proto;api/services/AlgoService.proto;api/services/AnalyticsService.proto;api/services/ApiUnitTestService.proto;api/services/AuthService.proto;api/services/BasePositionService.proto;api/services/ErrorReportService.proto;api/services/HelloService.proto;api/services/MBOService.proto;api/services/MarginService.proto;api/services/MarketDataAuxiliaryService.proto;api/services/MarketDataRelay.proto;api/services/MarketDataService.proto;api/services/MoneyFileService.proto;api/services/OrderService.proto;api/services/OutOfProcessURLListener.proto;api/services/PersistenceService.proto;api/services/PositionService.proto;api/services/ShellDiagnosticService.proto;api/services/ShellLatencyService.proto;api/services/SyntheticSecurityService.proto;api/test/ApiTests.cpp;api/test/HelloServiceServer.cpp;config/ConfigCheck.cpp;config/ConfigDownloader.cpp;config/ConfigErrorCollector.cpp;config/ConfigLog.cpp;config/ConfigPropertyEvaluator.cpp;config/ConfigValueQueue.cpp;config/ConfigurationException.cpp;config/ConfigurationState.cpp;config/PreprocessInputStream.cpp;config/config.cpp;config/defaults/MarketData.conf;config/defaults/Ports.conf;config/defaults/SecurityMetadataCollection.conf;config/defaults/Symbols.conf;config/defaults/cme_reference_data.conf;config/defaults/edge.conf;config/defaults/jetfix.conf;config/defaults/keycode.conf;config/defaults/logging.conf;config/defaults/orderman_cfg.conf;config/defaults/orders_fills_cleanup.conf;config/defaults/position_cfg.conf;config/defaults/span.conf;config/defaults/spreader.conf;config/defaults/testkeys.conf;config/defaults/threads.conf;config/defaults/toggles.conf;config/extensions/extensions.proto;config/invalid/invalid.proto;config/protos/algo_types.proto;config/protos/cme_reference_data.proto;config/protos/exchange_sim_cfg.proto;config/protos/jetfix.proto;config/protos/keycode.proto;config/protos/logging.proto;config/protos/market_data.proto;config/protos/mercury_cfg.proto;config/protos/orderman_cfg.proto;config/protos/orders_fills_cleanup.proto;config/protos/perflog_cfg.proto;config/protos/ports.proto;config/protos/position_cfg.proto;config/protos/quickbus_cfg.proto;config/protos/span.proto;config/protos/spartan.proto;config/protos/spreader.proto;config/protos/subprocess.proto;config/protos/symbols.proto;config/protos/system.proto;config/protos/testkeys.proto;config/protos/threads.proto;config/protos/toggles.proto;config/protos/ui.proto;config/tests/HTTPServer.cpp;config/tests/config_ref_tests.cpp;config/tests/config_tests.cpp;config/text_format/text_format.cpp;domain_model/Security.cpp;domain_model/Security.proto;domain_model/SecurityTests.cpp;domain_model/sod.cpp;domain_model/tests/domain_tests.cpp;expression/BooleanGrammar.cpp;expression/ExpressionEvaluator.cpp;expression/ParserExceptions.cpp;expression/Value.cpp;expression/tests/expression_parser_tests.cpp;io/QueryParams.cpp;io/URL.cpp;io/http/CurlHandle.cpp;io/http/HTTPClient.cpp;io/http/HTTPHeaders.cpp;io/http/HTTPRequest.cpp;io/http/HTTPResponse.cpp;io/websocket/WebsocketClient.cpp;io/websocket/WebsocketClientConfiguration.cpp;io/websocket/WebsocketError.cpp;io/websocket/WebsocketMessage.cpp;io/websocket/WebsocketSSLClientInternal.cpp;io/websocket/tests/WebsocketClientTests.cpp;logging/LoggerHub.cpp;logging/Sink.cpp;logging/logging.cpp;logging/tests/logging_tests.cpp;md/CCApiDepthWalker.cpp;md/CCApiListenerAdaptor.cpp;md/CCApiRequestor.cpp;md/Diag/MDPeriodicDumper.cpp;md/Diag/ShmDumper.cpp;md/MBOProcessor.cpp;md/MBOServiceServer.cpp;md/MDClient.cpp;md/MDConsumer.cpp;md/MDConsumerHandle.cpp;md/MDConsumerInterop.cpp;md/MDProducer.cpp;md/MDProducerHandle.cpp;md/MarketDataError.cpp;md/MarketDataGateway.cpLine truncated
@@ -1,170 +1,84 @@
 cmake_minimum_required(VERSION 3.5)
 project(libspark)
 set(CMAKE_SHARED_LIBRARY_PREFIX "")
+include(GNUInstallDirs)
 include(${CMAKE_SOURCE_DIR}/cmake/safe_dep_glob.cmake)
 include(${CMAKE_SOURCE_DIR}/cmake/Protobuf.cmake)
+include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/GenerateDefaultConfig.cmake)
+include(${CMAKE_SOURCE_DIR}/cmake/PrintList.cmake)

-# DEPENDENCIES
-file(
-  GLOB_RECURSE Config_Protos
-  RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
-  config/protos/*.proto
-  api/domain/SecurityMetadataCollection.proto)
-file(
-  GLOB_RECURSE Service_Protos
-  RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
-  api/services/*.proto)
-file(
-  GLOB_RECURSE NonConfig_Protos
-  RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
-  *.proto)
-list(REMOVE_ITEM NonConfig_Protos ${Service_Protos} ${Config_Protos})
-file(
-  GLOB_RECURSE Conf_Defaults
-  RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
-  config/defaults/*.conf)
+if (DEFINED DOTNET_BINARY_DIR OR DEFINED DOTNET_WINDOWS_BINARY_DIR)
+  include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/AddCopyBinariesTarget.cmake)
+endif()

 if(WIN32)
-  file(
-    GLOB_RECURSE SparkLib_Sources
-    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
-    api/*.cpp
-    config/*.cpp
-    domain_model/*.cpp
-    expression/*.cpp
-    logging/*.cpp
-    io/*.cpp
-    md/*.cpp
-    os/*.cpp
-    quickbus/*.cpp
-    sparkproto/*.cpp
-    testing/*.cpp
-    utils/*.cpp)
+  if(BUILD_SPARKD)
+    message(FATAL_ERROR "Cannot build sparkd on Windows (Please set BUILD_SPARKD to False)")
+  endif()
+
+  include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/GetWindowsClientSources.cmake)
+  include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/GetWindowsClientLinkLibraries.cmake)
+elseif(${BUILD_SPARKD})
+  include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/GetLinuxServerSources.cmake)
+  include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/GetLinuxServerLinkLibraries.cmake)
 else()
-  file(
-    GLOB_RECURSE SparkLib_Sources
-    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
-    *.cpp)
+  include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/GetLinuxClientSources.cmake)
+  include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/GetLinuxClientLinkLibraries.cmake)
 endif()

-update_deps_file("${SparkLib_Sources};${Config_Protos};${Service_Protos};${NonConfig_Protos};${Conf_Defaults}")
-# END DEPENDENCIES
+# Cpp sources
+set(${PROJECT_NAME}_Sources)
+get_cpp_sources(${PROJECT_NAME}_Sources)
+print_list(${PROJECT_NAME}_Sources VERBOSE)

-protobuf_generate_cpp(PROTO_SRCS PROTO_HDRS ${NonConfig_Protos} ${Config_Protos})
-service_compiler_generate_cpp(SERVICE_COMPILER_SRCS SERVICE_COMPILER_HDRS ${Service_Protos})
+# Proto sources 
+get_proto_sources(_cfg_protos _non_cfg_protos _service_protos _cfg_defaults)
+print_list(_cfg_protos VERBOSE)
+print_list(_non_cfg_protos VERBOSE)
+print_list(_service_protos VERBOSE)
+print_list(_cfg_defaults VERBOSE)

-# generate list of config protos
-set(config_proto_file ${CMAKE_CURRENT_BINARY_DIR}/config_proto_files.h)
+update_deps_file("${${PROJECT_NAME}_Sources};${_cfg_protos};${_service_protos};${_non_cfg_protos};${_cfg_defaults}")

-# generate a string with the current config files
-set(config_proto_str "const char * config_proto_files[] = {  // generated file\n")
-foreach(protopath ${Config_Protos})
-  get_filename_component(protoname ${protopath} NAME)
-  string(APPEND config_proto_str "\"${protoname}\",\n")
-endforeach(protopath)
-string(APPEND config_proto_str "} ;\n")
+# Generate proto cpp
+message(STATUS "Generating proto cpp")
+protobuf_generate_cpp(_proto_srcs _proto_hdrs ${_non_cfg_protos} ${_cfg_protos})
+service_compiler_generate_cpp(SERVICE_COMPILER_SRCS SERVICE_COMPILER_HDRS ${_service_protos})
+message(STATUS "Generating default config")
+generate_default_config(_cfg_protos _cfg_defaults)
+add_library(${PROJECT_NAME} SHARED ${${PROJECT_NAME}_Sources} ${_proto_srcs} ${SERVICE_COMPILER_SRCS})

-# compare with file and if there is any difference update the file
-set(old_config_proto_str "")
-if(EXISTS "${config_proto_file}")
-  file(READ "${config_proto_file}" old_config_proto_str)
-endif()
-if(NOT old_config_proto_str STREQUAL config_proto_str)
-  file(WRITE "${config_proto_file}" "${config_proto_str}")
-endif()
-
-# generate static defaults
-set(config_defaults_file ${CMAKE_CURRENT_BINARY_DIR}/config_defaults.h)
-# Defaults cannot be a single string because the compiler complains about a string longer than 65k
-set(config_defaults_str "const char * config_static_defaults[] = { ")
-foreach(cfg_file_path ${Conf_Defaults})
-  file(READ ${cfg_file_path} cfg_file_content)
-  string(REPLACE "\"" "\\\"" cfg_file_content_escaped ${cfg_file_content})
-  string(REPLACE "\n" ";" cfg_lines ${cfg_file_content_escaped})
-  foreach(line ${cfg_lines})
-    string(APPEND config_defaults_str "\n  \"${line}\",")
-  endforeach(line)
-endforeach(cfg_file_path)
-string(APPEND config_defaults_str "};")
-set(old_defaults_str "")
-if(EXISTS ${config_defaults_file})
-  file(READ ${config_defaults_file} old_defaults_str)
-endif()
-if(NOT config_defaults_str STREQUAL old_defaults_str)
-  file(WRITE "${config_defaults_file}" "${config_defaults_str}")
-endif()
-
-# lib
-if(WIN32)
-  add_library(libspark SHARED ${SparkLib_Sources} ${PROTO_SRCS} ${SERVICE_COMPILER_SRCS})
-else()
-  add_library(libspark SHARED ${SparkLib_Sources} ${PROTO_SRCS} ${SERVICE_COMPILER_SRCS})
-endif()
-
-# include paths
 target_include_directories(
-   libspark PUBLIC
-   ${CMAKE_SOURCE_DIR}/src
-   ${CMAKE_BINARY_DIR}/src)
+  ${PROJECT_NAME} PUBLIC
+  $<BUILD_INTERFACE:${CMAKE_SOURCE_DIR}/src>
+  $<BUILD_INTERFACE:${CMAKE_BINARY_DIR}/src>
+  $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>
+)

-if(WIN32)
-  target_link_libraries(
-    libspark
-    libcurl
-    debug
-    fmtd
-    optimized
-    fmt
-    libtbb
-    libboost_system
-    libboost_thread
-    libboost_filesystem
-    libboost_regex
-    libboost_chrono
-    libboost_date_time
-    libboost_atomic
-    libprotobuf
-    libprotoc
-    libssl
-    libcrypto
-    crypt32)
-else()
-  target_link_libraries(
-    libspark
-    protobuf
-    double-conversion
-    boost_system
-    boost_thread
-    boost_filesystem
-    boost_regex
-    boost_chrono
-    boost_date_time
-    boost_atomic
-    SQLiteCpp
-    sqlite3
-    curl
-    uuid
-    ssl
-    crypto
-    fmt
-    tbb
-    cryptopp
-    pugixml
-    lttng-ust
-    Pcap++
-    Packet++
-    Common++
-    ssh2
-    -lrt
-    -ldl
-    -lpcap
-    -static-libgcc
-    -static-libstdc++
-    -pthread)
+get_link_libraries(_link_libs)
+print_list(_link_libs VERBOSE)
+target_link_libraries(${PROJECT_NAME} ${_link_libs})

-  install(TARGETS libspark DESTINATION bin)
+# Copy binaries to Dotnet build dirs (for csprojs that depend on libspark)
+if (DEFINED DOTNET_BINARY_DIR)
+  add_copy_binaries_target(
+    copy_to_dotnet_binary_dir
+    "${DOTNET_BINARY_DIR}"
+    "Copy ${PROJECT_NAME} bins to Dotnet build dir")
 endif()

 if(WIN32)
-  include(${CMAKE_SOURCE_DIR}/tools/vs_filters.cmake)
+  include(${CMAKE_SOURCE_DIR}/cmake/vs_filters.cmake)
 endif()
+
+foreach(_hdr ${_proto_hdrs})
+  file(RELATIVE_PATH _relative_path ${CMAKE_CURRENT_BINARY_DIR}/protobufs ${_hdr})
+  install(FILES ${_hdr} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/${_relative_path})
+endforeach()
+include(${CMAKE_SOURCE_DIR}/cmake/InstallBasic.cmake)
+file(
+  GLOB_RECURSE _hdrs
+  RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
+  *.h
+)
+install_basic(HEADERS ${_hdrs})
@@ -0,0 +1,5 @@
+@PACKAGE_INIT@
+
+include("${CMAKE_CURRENT_LIST_DIR}/@PROJECT_NAME@Targets.cmake")
+
+check_required_components(@PROJECT_NAME@)
@@ -13,7 +13,7 @@
 #include "lib/config/protos/quickbus_cfg.pb.h"
 #include "lib/config/protos/system.pb.h"

-#ifdef WIN32
+#ifdef _WIN32
 #include <excpt.h>
 #endif

@@ -59,7 +59,7 @@ CAPI Quickbus *api_create_quickbus(const char *qbconfig, long length) {
 }

 CAPI void api_destroy_quickbus(Quickbus *qbus) {
-#ifndef WIN32
+#ifndef _WIN32
   if (qbus != nullptr) {
      delete qbus;
   }
@@ -186,7 +186,7 @@ CAPI void api_client_send(
    void *extra) {
   LOG(Api, Diag, "send rpc {}", ((int64_t)buffer));
   std::string strbuf(buffer, buffer_length);
-   qbus::RPCRequest request(move(strbuf));
+   qbus::RPCRequest request(std::move(strbuf));
   api_client_send_internal(
       pclient, std::move(request), cb, rpc_connect_timeout_ms(), extra);
 }
@@ -60,7 +60,7 @@ CAPI void api_server_call_callback(
       reinterpret_cast<qbus::ReplyHandler **>(ppvhandler);
   std::string reply_buffer((const char *)buffer, (size_t)length);
   qbus::RPCReplyPtr replyptr =
-       std::make_shared<qbus::RPCReply>(move(reply_buffer));
+       std::make_shared<qbus::RPCReply>(std::move(reply_buffer));
   (**pphandler)(replyptr);
   delete (*pphandler);
   *pphandler = nullptr;
@@ -0,0 +1,11 @@
+syntax = "proto3";
+
+import "lib/api/domain/Venue.proto";
+
+package bts.api.md;
+
+message DepthStreamConfig {
+  bts.api.domain.Venue venue = 1;
+  repeated string symbols = 2;
+  uint32 num_levels = 3;
+}
@@ -0,0 +1,10 @@
+syntax = "proto3";
+
+import "lib/api/md/DepthStreamConfig.proto";
+
+package bts.api.md;
+
+message MarketDataConfig {
+  repeated bts.api.md.DepthStreamConfig depth_streams = 1;
+  uint32 num_dispatcher_threads = 2;
+}
@@ -1,19 +0,0 @@
-syntax = "proto3";
-
-import "lib/api/md/binance/BinanceMarketDataSubscriptionParams.proto";
-import "lib/api/md/deribit/DeribitMarketDataSubscriptionParams.proto";
-import "lib/api/md/ftx/FTXMarketDataSubscriptionParams.proto";
-import "lib/api/md/glbx/GlbxMarketDataSubscriptionParams.proto";
-import "lib/api/md/huobi/HuobiMarketDataSubscriptionParams.proto";
-
-package bts.api.md;
-
-message MarketDataSubscriptionParams {
-   oneof VenueParams {
-      bts.api.md.glbx.GlbxMarketDataSubscriptionParams glbx = 1;
-      bts.api.md.deribit.DeribitMarketDataSubscriptionParams deribit = 2;
-      bts.api.md.binance.BinanceMarketDataSubscriptionParams binance = 3;
-      bts.api.md.huobi.HuobiMarketDataSubscriptionParams huobi = 4;
-      bts.api.md.ftx.FTXMarketDataSubscriptionParams ftx = 5;
-   }
-}
@@ -1,7 +0,0 @@
-syntax = "proto3";
-
-package bts.api.md.deribit;
-
-message DeribitMarketDataSubscriptionParams {
-   // TODO
-}
@@ -1,9 +0,0 @@
-syntax = "proto3";
-
-package bts.api.md.ftx;
-
-message FTXDepthStreamParams {
-  repeated string symbols = 1;
-  uint32 num_levels = 2;
-  uint32 num_dispatcher_threads = 3;
-}
@@ -1,9 +0,0 @@
-syntax = "proto3";
-
-import "lib/api/md/ftx/FTXDepthStreamParams.proto";
-
-package bts.api.md.ftx;
-
-message FTXMarketDataSubscriptionParams {
-  repeated FTXDepthStreamParams depth_streams = 2;
-}
@@ -1,7 +0,0 @@
-syntax = "proto3";
-
-package bts.api.md.glbx;
-
-message GlbxMarketDataSubscriptionParams {
-   // TODO
-}
@@ -1,7 +0,0 @@
-syntax = "proto3";
-
-package bts.api.md.huobi;
-
-message HuobiMarketDataSubscriptionParams {
-   // TODO
-}
@@ -72,7 +72,7 @@ class ClientChannel {
                       response_buffer + header_size,
                       response_size - header_size);
                   if (response->ParseFromZeroCopyStream(&data_stream)) {
-                      on_response(Result<Response>(move(response)));
+                      on_response(Result<Response>(std::move(response)));
                      return;
                   }
                } else {
@@ -28,7 +28,7 @@ void QuickbusClientChannel::send_core(
      (*on_response)(buffer, length);
      delete on_response;
   };
-   OnReply *heap_on_response = new OnReply(move(on_response));
+   OnReply *heap_on_response = new OnReply(std::move(on_response));
   api_client_send(
       _pclient, request_envalope.c_str(), request_envalope.size(), trampoline,
       heap_on_response);
@@ -11,8 +11,8 @@ using namespace rxcpp::operators;
 {{#each service}}

 // Constructor
-{{name}}::{{name}}(std::unique_ptr<bts::api::ClientChannel> pchannel) : _pchannel(move(pchannel)) {}
-{{name}}::{{name}}({{name}}&& other) : _pchannel(move(other._pchannel))  { }
+{{name}}::{{name}}(std::unique_ptr<bts::api::ClientChannel> pchannel) : _pchannel(std::move(pchannel)) {}
+{{name}}::{{name}}({{name}}&& other) : _pchannel(std::move(other._pchannel))  { }

 {{#each method}}
   {{#if server_streaming}}
@@ -52,6 +52,7 @@ bool AdminServiceUtils::has_exchange(
      rv = conn.has_ice_connection();
      break;
   case api::Venue::VENUE_DERIBIT: // TODO
+   case api::Venue::VENUE_FTX:
   case api::Venue::VENUE_BTS:
   case api::Venue::VENUE_CBOE:
   case api::Venue::VENUE_CBOECFE:
@@ -161,6 +162,7 @@ AdminServiceUtils::resolve_connection(
      rv = resolve_sim_connection(data, account, sec);
      break;
   case api::Venue::VENUE_DERIBIT:
+   case api::Venue::VENUE_FTX:
      rv = resolve_sim_connection(data, account, sec);
      break;
   case api::Venue::VENUE_CBOE:
@@ -410,6 +412,7 @@ std::string AdminServiceUtils::find_account_exchange_account(
   }
   case api::Venue::VENUE_UNSET:
   case api::Venue::VENUE_DERIBIT: // TODO
+   case api::Venue::VENUE_FTX:     // TODO
   case api::Venue::VENUE_BTS:
   case api::Venue::VENUE_CBOE:
   case api::Venue::VENUE_CBOECFE:
@@ -686,6 +689,7 @@ bts::api::orderman::RoutingRecord AdminServiceUtils::get_routing_record(
      rv.set_customer_or_firm(sim_account_data->origin());
      return rv;
   } break;
+   case api::VENUE_FTX: // TODO
   case api::Venue::VENUE_UNSET:
   case api::Venue::VENUE_BTS:
   case api::Venue::VENUE_CBOE:
@@ -1075,6 +1079,7 @@ std::string AdminServiceUtils::find_user_exchange_id(
      break;
   }
   case api::Venue::VENUE_DERIBIT: // TODO
+   case api::Venue::VENUE_FTX:     // TODO
   case api::Venue::VENUE_UNSET:
   case api::Venue::VENUE_BTS:
   case api::Venue::VENUE_CBOE:
@@ -1103,6 +1108,7 @@ std::string AdminServiceUtils::find_user_company_id(
      break;
   }
   case api::Venue::VENUE_DERIBIT:
+   case api::Venue::VENUE_FTX: // TODO
   case api::Venue::VENUE_SIM:
   case api::Venue::VENUE_SIMICE:
   case api::Venue::VENUE_GLBX:
@@ -1640,6 +1646,8 @@ api::SecurityType AdminServiceUtils::security_type(
      return api::SecurityType::SECURITYTYPE_FUTURE;
   case api::Security::ContractCase::kOption:
      return api::SecurityType::SECURITYTYPE_OPTION;
+   case api::Security::ContractCase::kSpot:
+      return api::SecurityType::SECURITYTYPE_STOCK;
   case api::Security::ContractCase::kSpread:
      return api::SecurityType::SECURITYTYPE_SPREAD;
   case api::Security::ContractCase::kSynthetic:
@@ -0,0 +1,15 @@
+function(add_copy_binaries_target tgt_name dest_dir)
+  set(_dest_files)
+  foreach (_dest_dir ${dest_dir})
+    set(_dest_file ${_dest_dir}/${PROJECT_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
+    list(APPEND _dest_files ${_dest_file})
+    add_custom_command(
+      OUTPUT ${_dest_file}
+      COMMAND ${CMAKE_COMMAND}
+      ARGS -E copy
+        ${CMAKE_CURRENT_BINARY_DIR}/$<CONFIG>/${PROJECT_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}
+        ${_dest_file}
+      DEPENDS ${PROJECT_NAME})
+  endforeach()
+  add_custom_target(${tgt_name} ALL DEPENDS ${_dest_files} COMMENT ${ARGN})
+endfunction()
@@ -0,0 +1,53 @@
+function(generate_default_config cfg_protos cfg_defaults)
+  # generate list of config protos
+  set(_cfg_proto_file ${CMAKE_CURRENT_BINARY_DIR}/config_proto_files.h)
+
+  # generate a string with the current config files
+  set(_cfg_proto_str "const char * config_proto_files[] = {  // generated file\n")
+
+  foreach(_proto_path ${${cfg_protos}})
+    get_filename_component(_proto_name ${_proto_path} NAME)
+    string(APPEND _cfg_proto_str "\"${_proto_name}\",\n")
+  endforeach(_proto_path)
+
+  string(APPEND _cfg_proto_str "} ;\n")
+
+  # compare with file and if there is any difference update the file
+  set(_old_cfg_proto_str "")
+
+  if(EXISTS "${_cfg_proto_file}")
+    file(READ "${_cfg_proto_file}" _old_cfg_proto_str)
+  endif()
+
+  if(NOT _old_cfg_proto_str STREQUAL _cfg_proto_str)
+    file(WRITE "${_cfg_proto_file}" "${_cfg_proto_str}")
+  endif()
+
+  # generate static defaults
+  set(_cfg_defaults_file ${CMAKE_CURRENT_BINARY_DIR}/config_defaults.h)
+
+  # Defaults cannot be a single string because the compiler complains about a
+  # string longer than 65k
+  set(_cfg_defaults_str "const char * config_static_defaults[] = { ")
+
+  foreach(_cfg_file_path ${${cfg_defaults}})
+    file(READ ${_cfg_file_path} _cfg_file_content)
+    string(REPLACE "\"" "\\\"" _cfg_file_content_escaped ${_cfg_file_content})
+    string(REPLACE "\n" ";" _cfg_lines ${_cfg_file_content_escaped})
+
+    foreach(_line ${_cfg_lines})
+      string(APPEND _cfg_defaults_str "\n  \"${_line}\",")
+    endforeach(_line)
+  endforeach(_cfg_file_path)
+
+  string(APPEND _cfg_defaults_str "};")
+  set(_old_cfg_defaults_str "")
+
+  if(EXISTS ${_cfg_defaults_file})
+    file(READ ${_cfg_defaults_file} _old_cfg_defaults_str)
+  endif()
+
+  if(NOT _cfg_defaults_str STREQUAL _old_cfg_defaults_str)
+    file(WRITE "${_cfg_defaults_file}" "${_cfg_defaults_str}")
+  endif()
+endfunction()
@@ -0,0 +1,23 @@
+function(get_link_libraries dest)
+  set(${dest}
+    curl
+    tbb
+    boost_system
+    boost_thread
+    boost_filesystem
+    boost_regex
+    boost_chrono
+    boost_date_time
+    boost_atomic
+    protobuf
+    protoc
+    ssl
+    crypto
+    debug fmtd
+    optimized fmt
+    cryptopp
+    SQLiteCpp
+    sqlite3
+    uuid
+    PARENT_SCOPE)
+endfunction()
@@ -0,0 +1,172 @@
+function(get_proto_sources cfg_srcs non_cfg_srcs service_srcs cfg_defaults)
+  file(
+    GLOB_RECURSE _cfg_srcs
+    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
+    config/protos/*.proto
+    api/domain/SecurityMetadataCollection.proto)
+  file(
+    GLOB_RECURSE _service_srcs
+    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
+    api/services/*.proto)
+  file(
+    GLOB_RECURSE _non_cfg_srcs
+    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
+    *.proto)
+  list(REMOVE_ITEM _non_cfg_srcs ${_service_srcs} ${_cfg_srcs})
+
+  file(
+    GLOB_RECURSE _cfg_defaults
+    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
+    ${CONFIG_DEFAULTS_PATH})
+
+  set(_cfg_defaults)
+
+  foreach(cfg_path ${CONFIG_DEFAULTS_PATH})
+    file(
+      GLOB_RECURSE cfg_files
+      RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
+      ${cfg_path})
+    list(APPEND _cfg_defaults ${cfg_files})
+  endforeach()
+
+  set(${cfg_srcs} ${_cfg_srcs} PARENT_SCOPE)
+  set(${non_cfg_srcs} ${_non_cfg_srcs} PARENT_SCOPE)
+  set(${service_srcs} ${_service_srcs} PARENT_SCOPE)
+  set(${cfg_defaults} ${_cfg_defaults} PARENT_SCOPE)
+endfunction()
+
+function(get_cpp_sources dest)
+  set(${dest}
+    api/core/ApiClientInterop.cpp
+    api/core/ApiServerInterop.cpp
+    api/md/MarketDataConfigUtils.cpp
+    api/public/ApiServer.cpp
+    api/public/ClientChannel.cpp
+    api/public/QuickbusClientChannel.cpp
+    api/public/ServerChannel.cpp
+    api/public/ServiceContext.cpp
+    api/test/ApiTests.cpp
+    api/test/HelloServiceServer.cpp
+    config/ConfigDownloader.cpp
+    config/ConfigErrorCollector.cpp
+    config/ConfigPropertyEvaluator.cpp
+    config/ConfigQuerier.cpp
+    config/ConfigValueQueue.cpp
+    config/ConfigurationException.cpp
+    config/ConfigurationState.cpp
+    config/PreprocessInputStream.cpp
+    config/config.cpp
+    config/tests/HTTPServer.cpp
+    config/tests/config_ref_tests.cpp
+    config/tests/config_tests.cpp
+    config/text_format/text_format.cpp
+    core/TimeSpan.cpp
+    core/TimeStamp.cpp
+    domain/DomainSecurityUtils.cpp
+    domain/SecurityCriteria.cpp
+    domain/SecurityCriteriaClause.cpp
+    domain_model/Security.cpp
+    domain_model/SecurityTests.cpp
+    domain_model/sod.cpp
+    domain_model/tests/domain_tests.cpp
+    expression/BooleanGrammar.cpp
+    expression/ExpressionEvaluator.cpp
+    expression/ParserExceptions.cpp
+    expression/Value.cpp
+    expression/tests/expression_parser_tests.cpp
+    io/http/CurlHandle.cpp
+    io/http/HTTPClient.cpp
+    io/http/HTTPHeaders.cpp
+    io/http/HTTPRequest.cpp
+    io/http/HTTPResponse.cpp
+    logging/LoggerHub.cpp
+    logging/Sink.cpp
+    logging/logging.cpp
+    logging/tests/logging_tests.cpp
+    md/Diag/MDPeriodicDumper.cpp
+    md/Diag/ShmDumper.cpp
+    md/MBOProcessor.cpp
+    md/MBOServiceServer.cpp
+    md/MDClient.cpp
+    md/MDConsumer.cpp
+    md/MDConsumerHandle.cpp
+    md/MDConsumerInterop.cpp
+    md/MDProducer.cpp
+    md/MDProducerHandle.cpp
+    md/MemorySegmentUtils.cpp
+    md/PriceSize.cpp
+    md/Product.cpp
+    md/SecurityCache.cpp
+    md/SecurityDB.cpp
+    md/SecurityIDRecorder.cpp
+    md/SecurityMapper.cpp
+    md/SecurityMarketState.cpp
+    md/SecurityOracle.cpp
+    md/SecurityStore.cpp
+    md/SettlementRecorder.cpp
+    md/Shm/FixedSecurity.cpp
+    md/SyntheticSecurityService.cpp
+    md/cme/MDP3Utils.cpp
+    md/common.cpp
+    md/ice/ICEUtils.cpp
+    md/relay/QueryServer.cpp
+    md/relay/RelayClient.cpp
+    md/relay/RelayRecorder.cpp
+    md/service/MarketDataServiceServer.cpp
+    md/test/Co2ManagedTestHelper.cpp
+    md/test/Co2TestInstrument.cpp
+    md/test/mbo_tests.cpp
+    md/test/md2_tests.cpp
+    md/test/md_test_utils.cpp
+    md/test/synthetic_book_merge_tests.cpp
+    os/os.cpp
+    os/tests/os_tests.cpp
+    quickbus/core/LocalNode.cpp
+    quickbus/core/Quickbus.cpp
+    quickbus/core/SubscriptionID.cpp
+    quickbus/core/TableID.cpp
+    quickbus/message_model/Update.cpp
+    quickbus/message_model/merge_updates.cpp
+    quickbus/network/QBConnection.cpp
+    quickbus/network/QBConnectionManager.cpp
+    quickbus/network/endpoint_description.cpp
+    quickbus/pubsub/TablePublisher.cpp
+    quickbus/pubsub/TableSubscriber.cpp
+    quickbus/pubsub/tests/qb_pubsub_tests.cpp
+    quickbus/routing/INode.cpp
+    quickbus/routing/NodeID.cpp
+    quickbus/routing/Router.cpp
+    quickbus/rpc/RPCClient.cpp
+    quickbus/rpc/RPCServer.cpp
+    quickbus/rpc/RPCSessionID.cpp
+    quickbus/rpc/rpc_utils.cpp
+    quickbus/rpc/tests/qb_rpc_tests.cpp
+    quickbus/tests/quickbus_tests.cpp
+    testing/catch_test_runner.cpp
+    utils/AuthSessionManager.cpp
+    utils/AuthToken.cpp
+    utils/Dispatcher.cpp
+    utils/FileUtils.cpp
+    utils/GrowableSPSCQueue.cpp
+    utils/HTTPStream.cpp
+    utils/MathUtils.cpp
+    utils/PBUtils.cpp
+    utils/Puid.cpp
+    utils/SecureLogon.cpp
+    utils/SharedMemory.cpp
+    utils/Sqlite_utils.cpp
+    utils/Uuid.cpp
+    utils/datetime.cpp
+    utils/download.cpp
+    utils/file_utils.cpp
+    utils/init_spark_lib.cpp
+    utils/is_unittest_environment.cpp
+    utils/sftp.cpp
+    utils/stats.cpp
+    utils/tests/AuthTokenTests.cpp
+    utils/tests/IDGeneratorTests.cpp
+    utils/tests/MathUtilsTests.cpp
+    utils/tests/SingleWriterConcurrentVectorTests.cpp
+    utils/thread_manager.cpp
+    PARENT_SCOPE)
+endfunction()
@@ -0,0 +1,35 @@
+function(get_link_libraries dest)
+  set(${dest}
+    protobuf
+    double-conversion
+    boost_system
+    boost_thread
+    boost_filesystem
+    boost_regex
+    boost_chrono
+    boost_date_time
+    boost_atomic
+    SQLiteCpp
+    sqlite3
+    curl
+    uuid
+    ssl
+    crypto
+    debug fmtd
+    optimized fmt
+    tbb
+    cryptopp
+    pugixml
+    lttng-ust
+    Pcap++
+    Packet++
+    Common++
+    ssh2
+    -lrt
+    -ldl
+    -lpcap
+    -static-libgcc
+    -static-libstdc++
+    -pthread
+    PARENT_SCOPE)
+endfunction()
@@ -0,0 +1,45 @@
+function(get_proto_sources cfg_srcs non_cfg_srcs service_srcs cfg_defaults)
+  file(
+    GLOB_RECURSE _cfg_srcs
+    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
+    config/protos/*.proto
+    api/domain/SecurityMetadataCollection.proto
+    api/md/MarketDataConfig.proto)
+  file(
+    GLOB_RECURSE _service_srcs
+    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
+    api/services/*.proto)
+  file(
+    GLOB_RECURSE _non_cfg_srcs
+    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
+    *.proto)
+  list(REMOVE_ITEM _non_cfg_srcs ${_service_srcs} ${_cfg_srcs})
+
+  file(
+    GLOB_RECURSE _cfg_defaults
+    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
+    ${CONFIG_DEFAULTS_PATH})
+
+  set(_cfg_defaults)
+
+  foreach(cfg_path ${CONFIG_DEFAULTS_PATH})
+    file(
+      GLOB_RECURSE cfg_files
+      RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
+      ${cfg_path})
+    list(APPEND _cfg_defaults ${cfg_files})
+  endforeach()
+  
+  set(${cfg_srcs} ${_cfg_srcs} PARENT_SCOPE)
+  set(${non_cfg_srcs} ${_non_cfg_srcs} PARENT_SCOPE)
+  set(${service_srcs} ${_service_srcs} PARENT_SCOPE)
+  set(${cfg_defaults} ${_cfg_defaults} PARENT_SCOPE)
+endfunction()
+
+function(get_cpp_sources dest)
+  file(
+    GLOB_RECURSE _dest
+    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
+    *.cpp)
+  set(${dest} ${_dest} PARENT_SCOPE)
+endfunction()
@@ -0,0 +1,20 @@
+function(get_link_libraries dest)
+  find_package(fmt CONFIG REQUIRED)
+  set(${dest}
+    crypt32
+    fmt::fmt-header-only
+    libcrypto
+    libcurl
+    libboost_system
+    libboost_thread
+    libboost_filesystem
+    libboost_regex
+    libboost_chrono
+    libboost_date_time
+    libboost_atomic
+    libprotobuf
+    libprotoc
+    libssl
+    libtbb
+    PARENT_SCOPE)
+endfunction()
@@ -0,0 +1,162 @@
+function(get_proto_sources cfg_srcs non_cfg_srcs service_srcs cfg_defaults)
+  file(
+    GLOB_RECURSE _cfg_srcs
+    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
+    config/protos/*.proto
+    api/domain/SecurityMetadataCollection.proto)
+  file(
+    GLOB_RECURSE _service_srcs
+    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
+    api/services/*.proto)
+  file(
+    GLOB_RECURSE _non_cfg_srcs
+    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
+    *.proto)
+  list(REMOVE_ITEM _non_cfg_srcs ${_service_srcs} ${_cfg_srcs})
+
+  file(
+    GLOB_RECURSE _cfg_defaults
+    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
+    ${CONFIG_DEFAULTS_PATH})
+
+  set(_cfg_defaults)
+
+  foreach(cfg_path ${CONFIG_DEFAULTS_PATH})
+    file(
+      GLOB_RECURSE cfg_files
+      RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
+      ${cfg_path})
+    list(APPEND _cfg_defaults ${cfg_files})
+  endforeach()
+
+  set(${cfg_srcs} ${_cfg_srcs} PARENT_SCOPE)
+  set(${non_cfg_srcs} ${_non_cfg_srcs} PARENT_SCOPE)
+  set(${service_srcs} ${_service_srcs} PARENT_SCOPE)
+  set(${cfg_defaults} ${_cfg_defaults} PARENT_SCOPE)
+endfunction()
+
+function(get_cpp_sources dest)
+  set(${dest}
+    api/core/ApiClientInterop.cpp
+    api/core/ApiServerInterop.cpp
+    api/public/ApiServer.cpp
+    api/public/ClientChannel.cpp
+    api/public/QuickbusClientChannel.cpp
+    api/public/ServerChannel.cpp
+    api/public/ServiceContext.cpp
+    api/test/ApiTests.cpp
+    api/test/HelloServiceServer.cpp
+    config/ConfigCheck.cpp
+    config/ConfigDownloader.cpp
+    config/ConfigErrorCollector.cpp
+    config/ConfigPropertyEvaluator.cpp
+    config/ConfigValueQueue.cpp
+    config/ConfigurationException.cpp
+    config/ConfigurationState.cpp
+    config/PreprocessInputStream.cpp
+    config/ConfigLog.cpp
+    config/config.cpp
+    config/tests/HTTPServer.cpp
+    config/tests/config_ref_tests.cpp
+    config/tests/config_tests.cpp
+    config/text_format/text_format.cpp
+    domain_model/Security.cpp
+    domain_model/SecurityTests.cpp
+    domain_model/sod.cpp
+    domain_model/tests/domain_tests.cpp
+    expression/BooleanGrammar.cpp
+    expression/ExpressionEvaluator.cpp
+    expression/ParserExceptions.cpp
+    expression/Value.cpp
+    expression/tests/expression_parser_tests.cpp
+    logging/LoggerHub.cpp
+    logging/Sink.cpp
+    logging/logging.cpp
+    logging/tests/logging_tests.cpp
+    md/Diag/MDPeriodicDumper.cpp
+    md/Diag/ShmDumper.cpp
+    md/MBOProcessor.cpp
+    md/MBOServiceServer.cpp
+    md/MDClient.cpp
+    md/MDConsumer.cpp
+    md/MDConsumerHandle.cpp
+    md/MDConsumerInterop.cpp
+    md/MDProducer.cpp
+    md/MDProducerHandle.cpp
+    md/MarketDataError.cpp
+    md/MarketDataGateway.cpp
+    md/MarketDataRegistrationFactory.cpp
+    md/MemorySegmentUtils.cpp
+    md/PlatformLoader.cpp
+    md/PriceSize.cpp
+    md/Product.cpp
+    md/SecurityCache.cpp
+    md/SecurityDB.cpp
+    md/SecurityMapper.cpp
+    md/SecurityMarketState.cpp
+    md/SecurityOracle.cpp
+    md/SecurityStore.cpp
+    md/Shm/FixedSecurity.cpp
+    md/SyntheticSecurityService.cpp
+    md/cme/MDP3Utils.cpp
+    md/common.cpp
+    md/ice/ICEUtils.cpp
+    md/relay/QueryServer.cpp
+    md/relay/RelayClient.cpp
+    md/relay/RelayRecorder.cpp
+    md/service/MarketDataServiceServer.cpp
+    md/test/Co2ManagedTestHelper.cpp
+    md/test/Co2TestInstrument.cpp
+    md/test/mbo_tests.cpp
+    md/test/md2_tests.cpp
+    md/test/md_test_utils.cpp
+    md/test/synthetic_book_merge_tests.cpp
+    os/os.cpp
+    os/tests/os_tests.cpp
+    quickbus/core/LocalNode.cpp
+    quickbus/core/Quickbus.cpp
+    quickbus/core/SubscriptionID.cpp
+    quickbus/core/TableID.cpp
+    quickbus/message_model/Update.cpp
+    quickbus/message_model/merge_updates.cpp
+    quickbus/network/QBConnection.cpp
+    quickbus/network/QBConnectionManager.cpp
+    quickbus/network/endpoint_description.cpp
+    quickbus/pubsub/TablePublisher.cpp
+    quickbus/pubsub/TableSubscriber.cpp
+    quickbus/pubsub/tests/qb_pubsub_tests.cpp
+    quickbus/routing/INode.cpp
+    quickbus/routing/NodeID.cpp
+    quickbus/routing/Router.cpp
+    quickbus/rpc/RPCClient.cpp
+    quickbus/rpc/RPCServer.cpp
+    quickbus/rpc/RPCSessionID.cpp
+    quickbus/rpc/rpc_utils.cpp
+    quickbus/rpc/tests/qb_rpc_tests.cpp
+    quickbus/tests/quickbus_tests.cpp
+    testing/catch_test_runner.cpp
+    utils/AuthSessionManager.cpp
+    utils/AuthToken.cpp
+    utils/Dispatcher.cpp
+    utils/FileUtils.cpp
+    utils/GrowableSPSCQueue.cpp
+    utils/HTTPStream.cpp
+    utils/MathUtils.cpp
+    utils/PBUtils.cpp
+    utils/Puid.cpp
+    utils/SecureLogon.cpp
+    utils/SharedMemory.cpp
+    utils/Uuid.cpp
+    utils/datetime.cpp
+    utils/download.cpp
+    utils/file_utils.cpp
+    utils/init_spark_lib.cpp
+    utils/is_unittest_environment.cpp
+    utils/sftp.cpp
+    utils/stats.cpp
+    utils/tests/AuthTokenTests.cpp
+    utils/tests/MathUtilsTests.cpp
+    utils/tests/SingleWriterConcurrentVectorTests.cpp
+    utils/thread_manager.cpp
+    PARENT_SCOPE)
+endfunction()
@@ -29,7 +29,7 @@
 #include "lib/config_defaults.h"
 #include "lib/config_proto_files.h"

-#ifdef WIN32
+#ifdef _WIN32
 #include <Winsock2.h>
 #include <d2d1.h>
 #else
@@ -566,7 +566,7 @@ void ConfigurationState::generate_all_config_proto(
                &bts_config_dp, config_message, message_desc, ++field_number);
         }
      } else {
-#ifdef WIN32
+#ifdef _WIN32
         // on linux the linker might throw away config protos that
         // are not used in sparkd. On windows we build a dll so that
         // should not happen
@@ -19,7 +19,7 @@ SymbolList {
        # symbol: "ES"
        # symbol: "FYT"
        # symbol: "FOB"
-        # symbol: "GC"
+        symbol: "GC"
        # symbol: "GE"
        # symbol: "HE"
        # symbol: "HG"
@@ -37,7 +37,7 @@ SymbolList {
        # symbol: "NUB"
        # symbol: "RB"
        # symbol: "RBT"
-        # symbol: "SI"
+        symbol: "SI"
        # symbol: "SOM"
        # symbol: "SR1"
        # symbol: "SR3"
@@ -84,6 +84,11 @@ SymbolLists
         key: "DERIBIT"
         symbol: @SymbolList.symbol
      }
+      list
+      {
+         key: "FTX"
+         symbol: @SymbolList.symbol
+      }
   }
 }

@@ -141,5 +146,10 @@ Symbols {
            option: @SymbolLists.list[DERIBIT].symbol
            spread: @SymbolLists.list[DERIBIT].symbol
        }
+        ftx {
+            future: @SymbolLists.list[FTX].symbol
+            option: @SymbolLists.list[FTX].symbol
+            spread: @SymbolLists.list[FTX].symbol
+        }
    }
 }
@@ -89,7 +89,7 @@ message JetFIX {
    optional uint32 window_duration_msecs = 345
        [ default = 1000 ]; // leave it alone

-    optional uint32 max_connection_attempts = 346 [ default = 10 ];
+    optional uint32 max_connection_attempts = 346 [ default = 3 ];
    optional bool auto_fix_seq_num = 347 [ default = false ];
    optional bool use_ilink3 = 348 [ default = false ];
  }
@@ -23,4 +23,5 @@ message Symbols {
  optional SecurityExchangeList ice = 30;
  optional SecurityExchangeList simice = 40;
  optional SecurityExchangeList deribit = 50;
+  optional SecurityExchangeList ftx = 60;
 }
@@ -33,17 +33,32 @@ std::string descriptor(api::Security const &sec) {
   return "No descriptor for non glbx / ice exchange";
 }

-api::SecurityType security_type(api::Security const &sec) {
-   if (sec.has_future())
+api::SecurityType security_type(api::Security const &security) {
+   switch (security.Contract_case()) {
+   case api::Security::ContractCase::kFuture:
      return api::SecurityType::SECURITYTYPE_FUTURE;
-   else if (sec.has_option())
+   case api::Security::ContractCase::kOption:
      return api::SecurityType::SECURITYTYPE_OPTION;
-   else if (sec.has_spread())
+   case api::Security::ContractCase::kSpot:
+      return api::SecurityType::SECURITYTYPE_STOCK;
+   case api::Security::ContractCase::kSpread:
      return api::SecurityType::SECURITYTYPE_SPREAD;
-   else if (sec.has_synthetic())
+   case api::Security::ContractCase::kSynthetic:
      return api::SecurityType::SECURITYTYPE_SYNTHETIC;
-   else
+   case api::Security::ContractCase::CONTRACT_NOT_SET:
      return api::SecurityType::SECURITYTYPE_UNSET;
+   }
+   return api::SecurityType::SECURITYTYPE_UNSET;
+   // if (sec.has_future())
+   //    return api::SecurityType::SECURITYTYPE_FUTURE;
+   // else if (sec.has_option())
+   //    return api::SecurityType::SECURITYTYPE_OPTION;
+   // else if (sec.has_spread())
+   //    return api::SecurityType::SECURITYTYPE_SPREAD;
+   // else if (sec.has_synthetic())
+   //    return api::SecurityType::SECURITYTYPE_SYNTHETIC;
+   // else
+   //    return api::SecurityType::SECURITYTYPE_UNSET;
 }

 boost::gregorian::date smallest_leg(
@@ -14,6 +14,7 @@ enum Venue {
     VENUE_ICE = 306;
     VENUE_SIMICE = 3;
     VENUE_DERIBIT = 400;
+     VENUE_FTX = 410;
 }

 enum SecurityType {
@@ -102,6 +103,10 @@ message DeribitExchangeData {
    int64 exchange_id = 10;
    string security_desc = 20;
 }
+message FTXExchangeData {
+    int64 exchange_id = 10;
+    string security_desc = 20;
+}
 message CBOECFEExchangeData {
    int64 exchange_id = 10;
    string security_desc = 20;
@@ -137,6 +142,9 @@ message OptionContract {
    PutOrCall call_put = 5;
 }

+message SpotContract{
+}
+
 message ContractLeg {
    Security security = 10;
    double quantity = 20;
@@ -189,6 +197,7 @@ message Security {
   oneof Contract {
       FutureContract future = 100;
       OptionContract option = 101;
+       SpotContract spot = 102;
       SpreadContract spread = 110;
       SyntheticContract synthetic = 120;
   }
@@ -198,6 +207,7 @@ message Security {
       CBOECFEExchangeData cfe = 220;
       ICEExchangeData ice = 230;
       DeribitExchangeData deribit = 240;
+       FTXExchangeData ftx = 250;
   }
 }
 message SecurityList {
@@ -1,6 +1,6 @@
 #pragma once

-#ifndef WIN32
+#ifndef _WIN32
 #include <bits/stdint-uintn.h>
 #endif
 #include <cstdint>
@@ -1,4 +1,4 @@
-#ifndef WIN32
+#ifndef _WIN32
 #include "OutFIX.h"

 namespace bts {
@@ -1,4 +1,4 @@
-#ifndef WIN32
+#ifndef _WIN32
 #include "AsyncReader.h"

 #include "IConnection.h"
@@ -1,3 +1,3 @@
-#ifndef WIN32
+#ifndef _WIN32
 #include "IConnection.h"
 #endif
@@ -1,4 +1,4 @@
-#ifndef WIN32
+#ifndef _WIN32
 #include "JetFIXGateway.h"

 namespace bts {
@@ -34,7 +34,7 @@ struct LogRec {
 };
 GCC_DIAG_RESTORE()

-#ifndef WIN32
+#ifndef _WIN32
 // static_assert(sizeof(LogRec) == 256, "LogRec should be 128 bytes");
 #endif

@@ -4,6 +4,7 @@
 #include <fmt/ostream.h>
 #include <iostream>
 #include <type_traits>
+#include <vector>

 namespace bts {
 namespace log {
@@ -12,7 +13,12 @@ template <class Tuple, std::size_t... Is>
 std::string format_tuple(
    const char *msg_template, Tuple const &t, std::index_sequence<Is...>) {
   try {
-      return fmt::format(msg_template, std::get<Is>(t)...);
+      using ctx = fmt::format_context;
+      std::vector<fmt::basic_format_arg<ctx>> fmt_args;
+      (fmt_args.push_back(fmt::detail::make_arg<ctx>(std::get<Is>(t))), ...);
+      return fmt::vformat(
+          msg_template,
+          fmt::basic_format_args<ctx>(fmt_args.data(), fmt_args.size()));
   } catch (fmt::format_error const &e) {
      return fmt::format(
          "Bad log message template: '{}' Error: '{}'", msg_template, e.what());
@@ -13,7 +13,7 @@
 #include <fstream>
 #include <iostream>

-#ifndef WIN32
+#ifndef _WIN32
 #include <syslog.h>
 #endif

@@ -22,7 +22,7 @@ namespace log {

 struct InitSysLog {
   InitSysLog() {
-#ifndef WIN32
+#ifndef _WIN32
      openlog("spark", 0, LOG_USER);
 #endif
   }
@@ -72,7 +72,7 @@ Sink console_sink(LoggingConfig const &) {
 }

 Sink syslog_sink(LoggingConfig const &cfg) {
-#ifdef WIN32
+#ifdef _WIN32
   return null_sink();
 #else
   auto f = formatter(cfg);
@@ -4,6 +4,8 @@
 #include "lib/utils/bts_chrono.h"
 #include "lib/logging/logging.h"

+#include <fmt/chrono.h>
+
 namespace bts {
 int logging_tests;

@@ -0,0 +1,57 @@
+#pragma once
+#ifdef SPARK_ENABLE_CCAPI
+#include "lib/api/domain/Venue.pb.h"
+#include "lib/md/MarketDataListenerID.h"
+
+#include <boost/algorithm/string/split.hpp>
+#include <fmt/format.h>
+#include <optional>
+#include <string>
+#include <tuple>
+
+namespace bts::md {
+
+class CCApiCorrelationID {
+ public:
+   static std::string fmt_market_depth(
+       api::domain::Venue venue,
+       std::string desc,
+       MarketDataListenerID listener_id);
+
+   static std::optional<
+       std::tuple<api::domain::Venue, std::string, MarketDataListenerID>>
+   parse_market_depth(const std::string &corr_id_str);
+};
+
+//////////////////////////////////////////////////
+// Definition
+//////////////////////////////////////////////////
+
+inline std::string CCApiCorrelationID::fmt_market_depth(
+    api::domain::Venue venue,
+    std::string desc,
+    MarketDataListenerID listener_id) {
+   return fmt::format(
+       "{}.{}.{}", api::domain::Venue_Name(venue), desc, listener_id);
+}
+
+inline std::optional<
+    std::tuple<api::domain::Venue, std::string, MarketDataListenerID>>
+CCApiCorrelationID::parse_market_depth(const std::string &corr_id_str) {
+   std::vector<std::string> parts;
+   boost::algorithm::split(parts, corr_id_str, [](auto c) { return c == '.'; });
+   if (parts.size() != 3) {
+      return std::nullopt;
+   }
+   api::domain::Venue venue;
+   if (!api::domain::Venue_Parse(parts.at(0), &venue)) {
+      return std::nullopt;
+   }
+   auto sec_desc = parts.at(1);
+   auto listener_id = MarketDataListenerID{
+       static_cast<std::uint32_t>(std::stoul(parts.at(2)))};
+   return std::make_tuple(venue, sec_desc, listener_id);
+}
+
+} // namespace bts::md
+#endif
@@ -1,243 +1,184 @@
 #ifdef SPARK_ENABLE_CCAPI
 #include "CCApiListenerAdaptor.h"
+#include "lib/api/md/MarketDataConfig.pb.h"
 #include "lib/domain/BUY_SELL.h"
+#include "lib/domain/SecurityID.h"
 #include "lib/logging/logging.h"
+#include "lib/md/CCApiCorrelationID.h"
 #include "lib/md/CCApiDepthWalker.h"
 #include "lib/md/CCApiSecurityParser.h"
+#include "lib/md/IMarketDataListener.h"
+#include "lib/md/MarketDataListenerData.h"
+#include "lib/persistence/IDGenerator.h"
+#include "lib/utils/ContainsKey.h"
+
+#include <memory>
 #include <string>
+#include <unordered_map>
+#include <vector>

 namespace bts::md {

-static bool parse_depth_msg(domain::Book &book, const ccapi::Message &msg) {
-   if (msg.getType() != ccapi::Message::Type::MARKET_DATA_EVENTS_MARKET_DEPTH) {
-      return false;
-   }
-
-   // The market data arrives in the following order:
-   //    * bid levels (by price descending)
-   //    * ask levels (by price ascending)
-   CCApiDepthWalker bid_walker{
-       "BID_PRICE", "BID_SIZE", &book[bts::domain::Side::Buy]};
-   CCApiDepthWalker ask_walker{
-       "ASK_PRICE", "ASK_SIZE", &book[bts::domain::Side::Sell]};
-   const auto &levels = msg.getElementList();
-   for (const auto &level : levels) {
-      // Each level has 2 entries, either:
-      //    ["BID_PRICE", "BID_SIZE"], or
-      //    ["ASK_PRICE", "ASK_SIZE"]
-      const auto &props = level.getNameValueMap();
-      for (const auto &[name, value] : props) {
-         bid_walker.parse_field(name, value) ||
-             ask_walker.parse_field(name, value);
-      }
-   }
-
-   // validate the book
-   bts_assert(bid_walker.price_idx == bid_walker.qty_idx);
-   bts_assert(ask_walker.price_idx == ask_walker.qty_idx);
-   bts_assert(book.is_sorted());
-   bts_assert(book[BUY].count_populated() == bid_walker.price_idx);
-   bts_assert(book[SELL].count_populated() == ask_walker.price_idx);
-   return true;
-}
-
-static domain::Book parse_depth_msg(const ccapi::Message &msg) {
-   domain::Book book;
-   parse_depth_msg(book, msg);
-   return book;
-}
-
-static void parse_securities_msg(
-    const ccapi::Message &msg,
-    std::unordered_map<std::string, std::unique_ptr<domain::Security>>
-        &securities,
-    std::vector<const domain::Security *> &added_secs,
-    std::vector<std::pair<const domain::Security *, const domain::Security *>>
-        &changed_secs,
-    std::vector<const domain::Security *> &all_secs) {
-   // std::vector<domain::Security
-   auto new_secs = CCApiSecurityParser::parse(msg.getElementList());
-   for (const auto &sec : new_secs) {
-      const auto &sym = sec.security_common.symbol;
-      bts_assert(!sym.empty());
-      auto it = securities.find(sym);
-      if (it == securities.end()) {
-         auto sec_ptr = std::make_unique<domain::Security>(sec);
-         added_secs.push_back(sec_ptr.get());
-         all_secs.push_back(sec_ptr.get());
-         securities[sym] = std::move(sec_ptr);
-         bts_assert(added_secs.back() != nullptr);
-         bts_assert(added_secs.back()->security_common.symbol == sym);
-         continue;
-      }
-      changed_secs.push_back({it->second.get(), &sec});
-   }
-   // auto added_msg = fmt::join(added_secs)
-   std::vector<std::string> added_sec_syms;
-   for (const auto *s : added_secs) {
-      added_sec_syms.push_back(s->security_common.symbol);
-   }
-   std::vector<std::string> changed_sec_syms;
-   for (const auto &[old_s, new_s] : changed_secs) {
-      added_sec_syms.push_back(fmt::format(
-          "{} -> {}", old_s->security_common.symbol,
-          new_s->security_common.symbol));
-   }
-   LOG(md, Info, "CCApi securities update. added: {}; changed: {}",
-       fmt::to_string(fmt::join(added_sec_syms, ", ")),
-       fmt::to_string(fmt::join(changed_sec_syms, ", ")));
-}
+using namespace bts::domain;
+using std::string;
+using std::unique_ptr;
+using std::unordered_map;
+using std::vector;

 struct CCApiListenerAdaptor::PImpl {
-   std::unordered_map<std::string, std::unique_ptr<domain::Security>>
-       securities;
-   std::unordered_map<std::uint32_t, IMarketDataListener *const> listeners;
-   MarketDataListenerID sec_id_counter{900'000};
+   PImpl(
+       IMarketDataListener *const listener,
+       persistence::IDGenerator *const sec_id_store)
+       : listener(listener), sec_id_store(sec_id_store) {}

-   void on_instruments(
-       const ccapi::Message &msg, IMarketDataListener &listener) {
-      std::vector<const domain::Security *> added_secs;
-      std::vector<std::pair<const domain::Security *, const domain::Security *>>
-          changed_secs;
-      std::vector<const domain::Security *> all_secs;
-      parse_securities_msg(msg, securities, added_secs, changed_secs, all_secs);
-      listener.on_securities(all_secs);
+   IMarketDataListener *const listener;
+
+   persistence::IDGenerator *const sec_id_store;
+   unordered_map<SecurityID, unique_ptr<Security>> securities;
+
+   void on_instruments(const ccapi::Message &msg, api::domain::Venue venue) {
+      vector<const Security *> all_secs;
+      auto new_secs = CCApiSecurityParser::parse(msg.getElementList());
+      for (const auto &sec : new_secs) {
+         const auto &desc = sec.security_common.desc;
+         bts_assert(!desc.empty());
+         auto sec_id = SecurityID{
+             sec_id_store->get_or_add(api::domain::Venue_Name(venue), desc)};
+         if (CONTAINS_KEY(securities, sec_id)) {
+            // don't replace existing secs (in case code downstream relies on
+            // ptr equality)
+            continue;
+         }
+         auto sec_ptr = std::make_unique<Security>(sec);
+         sec_ptr->security_common.id.value() = sec_id_store->get_or_add(
+             api::domain::Venue_Name(venue), sec_ptr->security_common.desc);
+         all_secs.push_back(sec_ptr.get());
+         securities[sec_id] = std::move(sec_ptr);
+      }
+      listener->on_securities(all_secs);
   }

-   void on_book(const ccapi::Message &msg, IMarketDataListener &listener) {
-      auto book = parse_depth_msg(msg);
-      listener.on_book(book);
+   void on_instruments(const ccapi::Message &msg, const string &corr_id) {
+      // Correlation ID is the Venue
+      api::domain::Venue venue;
+      if (!api::domain::Venue_Parse(corr_id, &venue)) {
+         LOG(md, Error, "Failed to parse venue from ccapi correlation id '{}'",
+             corr_id);
+         return;
+      }
+      on_instruments(msg, venue);
+   }
+
+   void on_book(const ccapi::Message &msg, SecurityID sec_id) {
+      if (msg.getType() !=
+          ccapi::Message::Type::MARKET_DATA_EVENTS_MARKET_DEPTH) {
+         return;
+      }
+
+      Book book;
+      book.set_security_id(sec_id);
+      // The market data arrives in the following order:
+      //    * bid levels (by price descending)
+      //    * ask levels (by price ascending)
+      CCApiDepthWalker bid_walker{"BID_PRICE", "BID_SIZE", &book[BUY]};
+      CCApiDepthWalker ask_walker{"ASK_PRICE", "ASK_SIZE", &book[SELL]};
+      const auto &levels = msg.getElementList();
+      for (const auto &level : levels) {
+         // Each level has 2 entries, either:
+         //    ["BID_PRICE", "BID_SIZE"], or
+         //    ["ASK_PRICE", "ASK_SIZE"]
+         const auto &props = level.getNameValueMap();
+         for (const auto &[name, value] : props) {
+            bid_walker.parse_field(name, value) ||
+                ask_walker.parse_field(name, value);
+         }
+      }
+
+      // validate the book
+      bts_assert(bid_walker.price_idx == bid_walker.qty_idx);
+      bts_assert(ask_walker.price_idx == ask_walker.qty_idx);
+      bts_assert(book.is_sorted());
+      bts_assert(book[BUY].count_populated() == bid_walker.price_idx);
+      bts_assert(book[SELL].count_populated() == ask_walker.price_idx);
+      listener->on_book(book);
+   }
+
+   void on_book(const ccapi::Message &msg, const string &corr_id) {
+      // Correlation ID is a formatted string:
+      // "<venue>.<sec_desc>.<listener_id>"
+      const auto corr_id_res = CCApiCorrelationID::parse_market_depth(corr_id);
+      bts_assert(corr_id_res.has_value());
+      const auto [venue, sec_desc, listener_id] = *corr_id_res;
+      LOG(md, Diag, "listener_id: {}", listener_id);
+      const auto sec_id =
+          sec_id_store->get_or_add(api::domain::Venue_Name(venue), sec_desc);
+      on_book(msg, SecurityID{sec_id});
+   }
+
+   SecurityID get_sec_id(
+       const api::domain::Venue venue, const string &sec_desc) {
+      const auto sec_id =
+          sec_id_store->get_or_add(api::domain::Venue_Name(venue), sec_desc);
+      return SecurityID{sec_id};
   }
 };

-CCApiListenerAdaptor::CCApiListenerAdaptor() : me_(std::make_unique<PImpl>()) {}
+CCApiListenerAdaptor::CCApiListenerAdaptor(
+    IMarketDataListener *const listener,
+    persistence::IDGenerator *const sec_id_store)
+    : me_(std::make_unique<PImpl>(listener, sec_id_store)) {}

 CCApiListenerAdaptor::~CCApiListenerAdaptor() = default;

-bool CCApiListenerAdaptor::empty() const { return me_->listeners.empty(); }
-
-void CCApiListenerAdaptor::register_listener(
-    IMarketDataListener *const listener, MarketDataListenerID id) {
-   bts_assert_msgf(
-       me_->listeners.find(id.value()) == me_->listeners.end(),
-       "MarketDataListener w/ id '{}' already registered", id);
-   me_->listeners.insert({id.value(), listener});
-}
-
-void CCApiListenerAdaptor::unregister_listener(MarketDataListenerID id) {
-   me_->listeners.erase(id.value());
-}
-
-static const char *Message_Type_Name(ccapi::Message::Type t) {
-   switch (t) {
-   case ccapi::Message::Type::UNKNOWN:
-      return "UNKNOWN";
-   case ccapi::Message::Type::AUTHORIZATION_SUCCESS:
-      return "AUTHORIZATION_SUCCESS";
-   case ccapi::Message::Type::AUTHORIZATION_FAILURE:
-      return "AUTHORIZATION_FAILURE";
-   case ccapi::Message::Type::MARKET_DATA_EVENTS_MARKET_DEPTH:
-      return "MARKET_DATA_EVENTS_MARKET_DEPTH";
-   case ccapi::Message::Type::MARKET_DATA_EVENTS_TRADE:
-      return "MARKET_DATA_EVENTS_TRADE";
-   case ccapi::Message::Type::MARKET_DATA_EVENTS_AGG_TRADE:
-      return "MARKET_DATA_EVENTS_AGG_TRADE";
-   case ccapi::Message::Type::EXECUTION_MANAGEMENT_EVENTS_ORDER_UPDATE:
-      return "EXECUTION_MANAGEMENT_EVENTS_ORDER_UPDATE";
-   case ccapi::Message::Type::EXECUTION_MANAGEMENT_EVENTS_PRIVATE_TRADE:
-      return "EXECUTION_MANAGEMENT_EVENTS_PRIVATE_TRADE";
-   case ccapi::Message::Type::SUBSCRIPTION_STARTED:
-      return "SUBSCRIPTION_STARTED";
-   case ccapi::Message::Type::SUBSCRIPTION_FAILURE:
-      return "SUBSCRIPTION_FAILURE";
-   case ccapi::Message::Type::SESSION_CONNECTION_UP:
-      return "SESSION_CONNECTION_UP";
-   case ccapi::Message::Type::SESSION_CONNECTION_DOWN:
-      return "SESSION_CONNECTION_DOWN";
-   case ccapi::Message::Type::INCORRECT_STATE_FOUND:
-      return "INCORRECT_STATE_FOUND";
-   case ccapi::Message::Type::CREATE_ORDER:
-      return "CREATE_ORDER";
-   case ccapi::Message::Type::CANCEL_ORDER:
-      return "CANCEL_ORDER";
-   case ccapi::Message::Type::GET_ORDER:
-      return "GET_ORDER";
-   case ccapi::Message::Type::GET_OPEN_ORDERS:
-      return "GET_OPEN_ORDERS";
-   case ccapi::Message::Type::CANCEL_OPEN_ORDERS:
-      return "CANCEL_OPEN_ORDERS";
-   case ccapi::Message::Type::GET_ACCOUNTS:
-      return "GET_ACCOUNTS";
-   case ccapi::Message::Type::GET_ACCOUNT_BALANCES:
-      return "GET_ACCOUNT_BALANCES";
-   case ccapi::Message::Type::GET_ACCOUNT_POSITIONS:
-      return "GET_ACCOUNT_POSITIONS";
-   case ccapi::Message::Type::GET_RECENT_TRADES:
-      return "GET_RECENT_TRADES";
-   case ccapi::Message::Type::GET_RECENT_AGG_TRADES:
-      return "GET_RECENT_AGG_TRADES";
-   case ccapi::Message::Type::GET_INSTRUMENT:
-      return "GET_INSTRUMENT";
-   case ccapi::Message::Type::GET_INSTRUMENTS:
-      return "GET_INSTRUMENTS";
-   case ccapi::Message::Type::RESPONSE_ERROR:
-      return "RESPONSE_ERROR";
-   case ccapi::Message::Type::REQUEST_FAILURE:
-      return "REQUEST_FAILURE";
-   case ccapi::Message::Type::GENERIC_ERROR:
-      return "GENERIC_ERROR";
-   case ccapi::Message::Type::CUSTOM:
-      return "CUSTOM";
-   case ccapi::Message::Type::FIX:
-      return "FIX";
-   case ccapi::Message::Type::FIX_FAILURE:
-      return "FIX_FAILURE";
-   case ccapi::Message::Type::GENERIC_PUBLIC_REQUEST:
-      return "GENERIC_PUBLIC_REQUEST";
-   case ccapi::Message::Type::GENERIC_PUBLIC_SUBSCRIPTION:
-      return "GENERIC_PUBLIC_SUBSCRIPTION";
-   case ccapi::Message::Type::GENERIC_PRIVATE_REQUEST:
-      return "GENERIC_PRIVATE_REQUEST";
-   default:
-      return "UNKNOWN";
-   }
-}
-
 bool CCApiListenerAdaptor::processEvent(
    const ccapi::Event &event, ccapi::Session *) {
-   LOG(md, Error, "Received ccapi event: {}", event.toString());
+   LOG(md, Diag, "Received ccapi event: {}", event.toString());
+   switch (event.getType()) {
+   case ccapi::Event::Type::SESSION_STATUS:
+   case ccapi::Event::Type::SUBSCRIPTION_STATUS:
+   case ccapi::Event::Type::REQUEST_STATUS:
+      // No processing needed
+      return true;
+   // one-time callback representing the reply to a ccapi 'REQUEST'
+   // Note: the GET_INSTRUMENTS msg arrives in an event of this type
+   case ccapi::Event::Type::RESPONSE:
+   // stream of callbacks representing the reply to a ccapi 'SUBSCRIBE'
+   // Note: the MARKET_DATA_EVENTS_MARKET_DEPTH msg arrives in an event of this
+   // type
+   case ccapi::Event::Type::SUBSCRIPTION_DATA:
+      break;
+   case ccapi::Event::Type::UNKNOWN:
+   case ccapi::Event::Type::AUTHORIZATION_STATUS:
+   case ccapi::Event::Type::FIX:
+   case ccapi::Event::Type::FIX_STATUS:
+      LOG(md, Error, "Unexpected ccapi event type '{}'",
+          ccapi::Event::typeToString(event.getType()));
+      break;
+   }
+
   const auto &msgs = event.getMessageList();
   for (const auto &msg : msgs) {
      const auto &corr_ids = msg.getCorrelationIdList();
      for (const auto &corr_id : corr_ids) {
-         MarketDataListenerID listener_id{
-             static_cast<std::uint32_t>(std::stoul(corr_id))};
-         auto listener_it = me_->listeners.find(listener_id.value());
-         if (listener_it == me_->listeners.end()) {
-            LOG(md, Warn,
-                "Received event associated w/ unknown correlationId '{}'",
-                corr_id);
-            continue;
-         }
-
-         auto &listener = *listener_it->second;
         switch (msg.getType()) {
-         case ccapi::Message::Type::GET_INSTRUMENTS:
-            me_->on_instruments(msg, listener);
+         case ccapi::Message::Type::GET_INSTRUMENTS: {
+            // This message should only arrive with a RESPONSE event
+            bts_assert(event.getType() == ccapi::Event::Type::RESPONSE);
+            me_->on_instruments(msg, corr_id);
            break;
+         }
         case ccapi::Message::Type::MARKET_DATA_EVENTS_MARKET_DEPTH: {
-            me_->on_book(msg, listener);
+            // This message should only arrive with a SUBSCRIPTION_DATA event
+            bts_assert(
+                event.getType() == ccapi::Event::Type::SUBSCRIPTION_DATA);
+            me_->on_book(msg, corr_id);
            break;
         }
         default:
-            std::string msg_type_name = Message_Type_Name(msg.getType());
-            LOG(md, Warn, "Unexpected CCApi event type '{}'", msg_type_name);
+            LOG(md, Warn, "Unexpected ccapi message type '{}'",
+                ccapi::Message::typeToString(msg.getType()));
            break;
         }
-         // switch (event.getType()) {
-         //    case ccapi::Event::Type::UNKNOWN:
-         //    break;
-         // }
      }
   }
   return true;
@@ -1,28 +1,33 @@
 #pragma once
 #ifdef SPARK_ENABLE_CCAPI
-#include "lib/md/IMarketDataListener.h"
-#include "lib/md/MarketDataListenerID.h"
+#include "lib/api/domain/Venue.pb.h"
+
 #include <ccapi_cpp/ccapi_event_handler.h>
+#include <memory>
+
+namespace bts::api::md {
+class MarketDataConfig;
+}
+
+namespace bts::persistence {
+class IDGenerator;
+}

 namespace bts::md {

+class IMarketDataListener;
+
 class CCApiListenerAdaptor : public ccapi::EventHandler {
 public:
-   CCApiListenerAdaptor();
+   CCApiListenerAdaptor(
+       IMarketDataListener *const listener,
+       persistence::IDGenerator *const sec_id_store);
+
   ~CCApiListenerAdaptor();

-   MarketDataListenerID register_listener(IMarketDataListener *const listener);
-
-   void register_listener(
-       IMarketDataListener *const listener, MarketDataListenerID id);
-
-   void unregister_listener(MarketDataListenerID id);
-
   bool processEvent(
       const ccapi::Event &event, ccapi::Session *session) override;

-   bool empty() const;
-
 private:
   struct PImpl;
   std::unique_ptr<PImpl> me_;
@@ -1,38 +0,0 @@
-#ifdef SPARK_ENABLE_CCAPI
-#include "CCApiRequestor.h"
-#include "lib/logging/logging.h"
-#include <optional>
-
-namespace bts::md {
-
-struct CCApiRequestor::PImpl {
-   ccapi::Request req;
-   ccapi::EventHandler *handler;
-   ccapi::SessionOptions opts;
-   ccapi::SessionConfigs cfgs;
-   std::optional<ccapi::Session> session;
-};
-
-CCApiRequestor::CCApiRequestor(
-    ccapi::EventHandler *const handler,
-    ccapi::SessionOptions opts,
-    ccapi::SessionConfigs cfgs)
-    : me_(std::make_unique<PImpl>()) {
-   // Store 'opts' and 'cfgs' in PImpl for diagnostic purposes
-   me_->opts = std::move(opts);
-   me_->cfgs = std::move(cfgs);
-   me_->handler = handler;
-}
-
-CCApiRequestor::~CCApiRequestor() { me_->session->stop(); }
-
-void CCApiRequestor::request(ccapi::Request &req) {
-   LOG(md, Info, "Making ccapi request: {}", req.toString());
-   if (!me_->session) {
-      me_->session.emplace(me_->opts, me_->cfgs, me_->handler);
-   }
-   me_->session->sendRequest(req);
-}
-
-} // namespace bts::md
-#endif
@@ -1,27 +0,0 @@
-#pragma once
-#ifdef SPARK_ENABLE_CCAPI
-#include <ccapi_cpp/ccapi_event_handler.h>
-#include <ccapi_cpp/ccapi_session.h>
-#include <ccapi_cpp/ccapi_session_options.h>
-#include <memory>
-
-namespace bts::md {
-
-class CCApiRequestor {
- public:
-   explicit CCApiRequestor(
-       ccapi::EventHandler *const handler,
-       ccapi::SessionOptions opts,
-       ccapi::SessionConfigs cfgs);
-
-   ~CCApiRequestor();
-
-   void request(ccapi::Request &req);
-
- private:
-   struct PImpl;
-   std::unique_ptr<PImpl> me_;
-};
-
-} // namespace bts::md::ftx
-#endif
@@ -0,0 +1,91 @@
+#ifdef SPARK_ENABLE_CCAPI
+#include "CCApiSession.h"
+#include "lib/api/md/DepthStreamConfig.pb.h"
+#include "lib/md/CCApiCorrelationID.h"
+#include "lib/md/CCApiListenerAdaptor.h"
+#include "lib/md/MarketDataListenerData.h"
+#include "lib/md/MarketDataRegistrationFactory.h"
+#include "lib/persistence/IDGenerator.h"
+
+#include <boost/algorithm/string/case_conv.hpp>
+#include <ccapi_cpp/ccapi_event_dispatcher.h>
+#include <ccapi_cpp/ccapi_request.h>
+#include <ccapi_cpp/ccapi_session.h>
+#include <ccapi_cpp/ccapi_session_configs.h>
+#include <ccapi_cpp/ccapi_session_options.h>
+#include <ccapi_cpp/ccapi_subscription.h>
+
+namespace bts::md {
+
+struct CCApiSession::PImpl {
+   PImpl(
+       const MarketDataListenerData listener_data,
+       bts::persistence::IDGenerator *const sec_id_store)
+       : listener_data(listener_data), sec_id_store(sec_id_store),
+         ccapi_dispatcher(
+             static_cast<int>(listener_data.cfg.num_dispatcher_threads())),
+         ccapi_listener(listener_data.listener, sec_id_store),
+         ccapi_session(
+             ccapi_opts, ccapi_cfgs, &ccapi_listener, &ccapi_dispatcher) {}
+
+   MarketDataListenerData listener_data;
+   bts::persistence::IDGenerator *sec_id_store;
+
+   ccapi::SessionOptions ccapi_opts;
+   ccapi::SessionConfigs ccapi_cfgs;
+
+   // Dispatches ccapi callbacks
+   ccapi::EventDispatcher ccapi_dispatcher;
+
+   // Handles ccapi sec/md callbacks
+   CCApiListenerAdaptor ccapi_listener;
+
+   // Manages ccapi connection to exchanges
+   ccapi::Session ccapi_session;
+
+   void request_securities(api::domain::Venue venue) {
+      auto venue_name = api::domain::Venue_Name(venue);
+      auto venue_name_lower = boost::algorithm::to_lower_copy(venue_name);
+      // The 'venue' in the request must be lower-case (ccapi expects this). The
+      // correlation id, however, should remain unchanged (in the callback, we
+      // parse the venue with 'api::domain::Venue_Parse')
+      ccapi::Request req(
+          ccapi::Request::Operation::GET_INSTRUMENTS, venue_name_lower);
+      req.setCorrelationId(venue_name);
+      ccapi_session.sendRequest(req);
+   }
+
+   void subscribe_md(const api::md::DepthStreamConfig &ds) {
+      const auto opts =
+          fmt::format("{}={}", CCAPI_MARKET_DEPTH_MAX, ds.num_levels());
+      auto venue_name = bts::api::domain::Venue_Name(ds.venue());
+      boost::algorithm::to_lower(venue_name);
+      for (const auto &symbol : ds.symbols()) {
+         const auto corr_id = CCApiCorrelationID::fmt_market_depth(
+             ds.venue(), symbol, listener_data.id);
+         ccapi::Subscription sub(
+             venue_name, symbol, CCAPI_MARKET_DEPTH, opts, corr_id);
+         ccapi_session.subscribe(sub);
+      }
+   }
+};
+
+CCApiSession::CCApiSession(
+    MarketDataListenerData listener_data,
+    bts::persistence::IDGenerator *sec_id_store)
+    : me_(std::make_unique<PImpl>(std::move(listener_data), sec_id_store)) {
+
+   std::unordered_set<api::domain::Venue> venues;
+   for (auto ds : me_->listener_data.cfg.depth_streams()) {
+      // Limit one sec request per Venue
+      if (venues.insert(ds.venue()).second) {
+         me_->request_securities(ds.venue());
+      }
+      me_->subscribe_md(ds);
+   }
+}
+
+CCApiSession::~CCApiSession() { me_->ccapi_session.stop(); }
+
+} // namespace bts::md
+#endif
@@ -0,0 +1,37 @@
+#pragma once
+#ifdef SPARK_ENABLE_CCAPI
+#include "lib/api/domain/Venue.pb.h"
+#include <memory>
+
+namespace ccapi {
+class Session;
+}
+
+namespace bts::persistence {
+class IDGenerator;
+}
+
+namespace bts::md {
+
+struct MarketDataListenerData;
+
+class CCApiSession {
+ public:
+   explicit CCApiSession(
+       MarketDataListenerData listener_data,
+       bts::persistence::IDGenerator *sec_id_store);
+
+   CCApiSession(const CCApiSession &) = delete;
+   CCApiSession(CCApiSession &&) = default;
+   CCApiSession &operator=(const CCApiSession &) = delete;
+   CCApiSession &operator=(CCApiSession &&) = default;
+
+   ~CCApiSession();
+
+ private:
+   struct PImpl;
+   std::unique_ptr<PImpl> me_;
+};
+
+} // namespace bts::md
+#endif
@@ -60,7 +60,7 @@ std::string print_book(
         auto a = ask[x - 1];
         if (!os::is_nan(a.price())) {
            snprintf(
-                buffer, 512, "%5s %14s | %14.7f %5d\n", "", "", a.price(),
+                buffer, 512, "%5s %14s | %14.7f %5.5f\n", "", "", a.price(),
                a.size());
            ss << buffer;
         }
@@ -69,7 +69,7 @@ std::string print_book(
      for (uint32_t x = 0; x < COCONUT_MAX_DEPTH; ++x) {
         auto b = bid[x];
         if (!os::is_nan(b.price())) {
-            snprintf(buffer, 512, "%5d %14.7f | \n", b.size(), b.price());
+            snprintf(buffer, 512, "%5.5f %14.7f | \n", b.size(), b.price());
            ss << buffer;
         }
      }
@@ -4,9 +4,11 @@
 #include "MDProducerHandle.h"
 #include "lib/domain_model/Security.h"
 #include "lib/domain_model/SecurityKey.h"
+#include "lib/utils/OSTREAM_FMT_IMPL.h"
 #include "lib/utils/TimeStamp.h"

 #include <btree/map.h>
+#include <fmt/format.h>

 #include <map>
 #include <memory>
@@ -169,3 +171,5 @@ class MBOProcessor {
 namespace std {
 std::ostream &operator<<(std::ostream &os, const bts::md::inc::Update &);
 }
+
+OSTREAM_FMT_IMPL(bts::md::inc::Update);
@@ -613,7 +613,8 @@ class MDConsumerImpl {

      for (auto &s : symbols) {
         if (s.type() == SECURITYTYPE_FUTURE ||
-             s.type() == SECURITYTYPE_OPTION) {
+             s.type() == SECURITYTYPE_OPTION ||
+             s.type() == api::SECURITYTYPE_STOCK) {
            _instances.push_back(make_shared<MDConsumerInst>(
                s.symbol(), s.type(), s.exchange(), shared, platform));
         }
@@ -783,7 +784,7 @@ class MDConsumerImpl {
      // recurse to find details
      map<security_key, SecurityPtr> leg_map;
      for (auto &leg : fill_leg_details(legs, depth + 1)) {
-         leg_map.insert(make_pair(leg->id(), move(leg)));
+         leg_map.insert(make_pair(leg->id(), std::move(leg)));
      }

      //
@@ -218,6 +218,7 @@ class BTS_EXPORT MDConsumerHandle {
      switch (_type) {
      case HANDLETYPE_VOID:
         bts_assert_msg(false, "Handle not initialized");
+         break;
      case HANDLETYPE_DEPTH:
         return convert_security(book.cdata->security);
      case HANDLETYPE_BBO:
@@ -1,4 +1,4 @@
-#ifdef _WIN32
+#if defined(EXPORT_MD_CONSUMER)
 #include "MDConsumer.h"
 #include "SecurityCache.h"
 #include "UserSymbol.h"
@@ -134,7 +134,7 @@ MDProducer::MDProducer(Platform platform, MDProducerFlags flags)
    : _mutex(new std::mutex()), _producers(), _id_recorder(),
      _platform(platform), _flags(flags),
      _smapper(flags & MDProducerFlags::SecurityMapper) {}
-#ifndef WIN32
+#ifndef _WIN32
 MDProducer::MDProducer(
    Platform platform, SecurityIDRecorder recorder, MDProducerFlags flags)
    : _mutex(new std::mutex()), _producers(),
@@ -173,7 +173,7 @@ MDProducerImpl &MDProducer::get_producer(
   } else {
      newp.reset(new MDProducerImpl(symbol, type, exchange, _platform, _flags));
   }
-   _producers.push_back(move(newp));
+   _producers.push_back(std::move(newp));
   return *_producers.back();
 }

@@ -1,145 +1,65 @@
 #ifdef SPARK_ENABLE_CCAPI
-#include "lib/md/CCApiListenerAdaptor.h"
-#include <ccapi_cpp/ccapi_macro.h>
 #include "MarketDataGateway.h"
-#include "MarketDataRegistration.h"
-#include "lib/api/md/MarketDataSubscriptionParams.pb.h"
-#include "lib/api/md/binance/BinanceMarketDataSubscriptionParams.pb.h"
+#include "lib/api/domain/Venue.pb.h"
+#include "lib/api/md/DepthStreamConfig.pb.h"
+#include "lib/api/md/MarketDataConfig.pb.h"
 #include "lib/core/ProtoPrinter.h"
 #include "lib/domain/Venue.h"
 #include "lib/logging/logging.h"
-#include "lib/md/CompositeMarketDataListener.h"
+#include "lib/md/CCApiSession.h"
 #include "lib/md/IMarketDataListener.h"
 #include "lib/md/MarketDataListenerData.h"
 #include "lib/md/MarketDataListenerID.h"
-#include "lib/md/binance/BinanceDepthStreamSiteKey.h"
-#include "lib/md/binance/BinanceDiffDepthSubscription.h"
-#include "lib/md/binance/BinanceDiffDepthUpdate.h"
+#include "lib/md/MarketDataRegistration.h"
+#include "lib/md/MarketDataRegistrationError.h"
+#include "lib/md/MarketDataRegistrationFactory.h"
+#include "lib/persistence/IDGenerator.h"
 #include "lib/utils/ContainsKey.h"
 #include "lib/utils/bts_assert.h"
-#include <ccapi_cpp/ccapi_session.h>
+
 #include <optional>
 #include <unordered_map>
+#include <vector>

 namespace bts::md {

+using std::string;
+using std::vector;
+
+static void _validate(
+    const api::md::DepthStreamConfig &ds,
+    std::uint32_t index,
+    vector<string> &errors) {
+   if (!api::domain::Venue_IsValid(ds.venue())) {
+      errors.push_back(fmt::format(
+          "DepthStreamConfig at index {} 'venue' is invalid", index));
+   }
+   if (ds.num_levels() == 0) {
+      errors.push_back(fmt::format(
+          "DepthStreamConfig at index {} 'num_levels' is 0", index));
+   }
+   if (ds.symbols().empty()) {
+      errors.push_back(fmt::format(
+          "DepthStreamConfig at index {} 'symbols' is empty", index));
+   }
+}
+
+static void _validate(
+    const api::md::MarketDataConfig &cfg, vector<string> errs) {
+   if (cfg.depth_streams().empty()) {
+      errs.push_back("MarketDataConfig 'depth_streams' is empty");
+   }
+   std::uint32_t ds_idx = 0;
+   for (const auto &ds : cfg.depth_streams()) {
+      _validate(ds, ds_idx++, errs);
+   }
+}
+
 struct MarketDataGateway::PImpl {
   MarketDataRegistrationFactory registration_factory;
-   std::unordered_map<
-       binance::BinanceDepthStreamSiteKey,
-       binance::BinanceDiffDepthSubscription>
-       binance_subs;
-   std::unordered_map<MarketDataListenerID, MarketDataListenerData>
-       registered_listeners;
-
- private:
-   ccapi::SessionOptions ccapi_opts_;
-   ccapi::SessionConfigs ccapi_cfgs_;
-   std::optional<ccapi::EventDispatcher> ccapi_dispatcher_;
-   std::optional<ccapi::Session> ccapi_session_;
-   std::optional<CCApiListenerAdaptor> ccapi_listener_;
-
- public:
-   CCApiListenerAdaptor &ccapi_listener() {
-      if (!ccapi_listener_) {
-         return ccapi_listener_.emplace();
-      }
-      return *ccapi_listener_;
-   }
-
-   ccapi::Session &ccapi_session() {
-      if (!ccapi_session_) {
-         if (!ccapi_dispatcher_) {
-            ccapi_dispatcher_.emplace(2);
-         }
-         ccapi_session_.emplace(
-             ccapi_opts_, ccapi_cfgs_, &ccapi_listener(), &*ccapi_dispatcher_);
-      }
-      return *ccapi_session_;
-   }
-   //////////////////////////////////////////////////
-   // Binance
-   //////////////////////////////////////////////////
-
-   void subscribe_binance(
-       const api::md::binance::BinanceMarketDataSubscriptionParams &sub,
-       IMarketDataListener *listener) {
-      std::vector<binance::BinanceDiffDepthSubscription *> new_subs;
-      for (const auto &depth_stream : sub.depth_streams()) {
-         md::binance::BinanceDepthStreamSiteKey site_key(depth_stream);
-         auto [sub_it, is_new_sub] = binance_subs.try_emplace(
-             site_key, md::binance::BinanceDepthStreamParams{depth_stream});
-         if (is_new_sub) {
-            new_subs.push_back(&sub_it->second);
-            continue;
-         }
-         auto &sub = sub_it->second;
-         // add symbols to piggy back on existing subscription rather than
-         // create a new one
-         for (const auto &symbol : depth_stream.symbols()) {
-            sub.add_listener(symbol, listener);
-         }
-      }
-      for (auto *sub : new_subs) {
-         sub->connect();
-      }
-   }
-
-   void unsubscribe_binance(
-       const api::md::binance::BinanceMarketDataSubscriptionParams &sub,
-       IMarketDataListener *listener) {
-      for (const auto &depth_stream : sub.depth_streams()) {
-         md::binance::BinanceDepthStreamSiteKey site_key(depth_stream);
-         auto sub_it = binance_subs.find(site_key);
-         bts_assert(sub_it != binance_subs.end());
-         auto &sub = sub_it->second;
-         for (const auto &symbol : depth_stream.symbols()) {
-            bts_assert(sub.remove_listener(symbol, listener));
-         }
-      }
-   }
-
-   //////////////////////////////////////////////////
-   // FTX
-   //////////////////////////////////////////////////
-
-   void subscribe_ftx(
-       const api::md::ftx::FTXMarketDataSubscriptionParams &params,
-       IMarketDataListener *listener,
-       MarketDataListenerID listener_id) {
-      if (params.depth_streams().empty()) {
-         return;
-      }
-
-      bts_assert(params.depth_streams().size() == 1);
-      auto &depth_stream = params.depth_streams().at(0);
-      std::vector<std::string> symbols;
-
-      for (const auto &symbol : depth_stream.symbols()) {
-         symbols.push_back(symbol);
-      }
-
-      ccapi_listener().register_listener(listener, listener_id);
-      auto opts = fmt::format(
-          "{}={}", CCAPI_MARKET_DEPTH_MAX, depth_stream.num_levels());
-      for (const auto &symbol : symbols) {
-         std::string corr_id = fmt::format("{}", listener_id);
-         ccapi::Subscription sub(
-             "binance", symbol, CCAPI_MARKET_DEPTH, opts, corr_id);
-         ccapi_session().subscribe(sub);
-      }
-   }
-
-   void unsubscribe_ftx(MarketDataListenerID listener_id) {
-      bts_assert_msg(
-          ccapi_listener_.has_value(),
-          "Ignoring request to unsubscribe from FTX market data. "
-          "CCApiListenerAdaptor not created.");
-      ccapi_listener_->unregister_listener(listener_id);
-      if (ccapi_listener_->empty()) {
-         ccapi_session_.reset();
-      }
-   }
+   bts::persistence::IDGenerator &sec_id_store =
+       bts::persistence::IDGenerator::insance();
+   std::unordered_map<MarketDataListenerID, CCApiSession> sessions;
 };

 MarketDataGateway::MarketDataGateway()
@@ -148,72 +68,35 @@ MarketDataGateway::MarketDataGateway()
 MarketDataGateway::~MarketDataGateway() = default;

 MarketDataRegistration MarketDataGateway::subscribe(
-    const MarketDataSubscription &sub, IMarketDataListener *listener) noexcept {
+    const api::md::MarketDataConfig &cfg,
+    IMarketDataListener *listener) noexcept {
   auto reg = me_->registration_factory.create();
   bts_assert(reg.listener_id.has_value());
-   bts_assert(
-       me_->registered_listeners.find(*reg.listener_id) ==
-       me_->registered_listeners.end());
+   bts_assert(me_->sessions.find(*reg.listener_id) == me_->sessions.end());
   LOG(md, bts::Info, "MarketDataGateway subscribing to data.\n{}\n",
-       core::ProtoPrinter<api::md::MarketDataSubscriptionParams>(
-           &sub.params()));
+       core::ProtoPrinterFactory::create(&cfg));

-   auto &listener_data = me_->registered_listeners[*reg.listener_id];
-   listener_data.id = *reg.listener_id;
-   listener_data.listener = listener;
-   listener_data.subscription = sub;
-
-   switch (sub.venue()) {
-   case api::domain::Venue::BINANCE:
-      bts_assert(
-          sub.params().VenueParams_case() ==
-          api::md::MarketDataSubscriptionParams::kBinance);
-      me_->subscribe_binance(sub.params().binance(), listener);
-      break;
-   case api::domain::Venue::FTX:
-      bts_assert(
-          sub.params().VenueParams_case() ==
-          api::md::MarketDataSubscriptionParams::kFtx);
-      LOG(md, bts::Info, "MarketDataGateway subscribing to data.\n{}\n",
-          core::ProtoPrinter<api::md::MarketDataSubscriptionParams>(
-              &sub.params()));
-      me_->subscribe_ftx(sub.params().ftx(), listener, listener_data.id);
-      break;
-   default:
+   vector<string> cfg_errs;
+   _validate(cfg, cfg_errs);
+   if (!cfg_errs.empty()) {
      return MarketDataRegistration{
-          MarketDataListenerID{0},
-          MarketDataRegstrationError::Type::NOT_IMPLEMENTED};
+          {MarketDataRegistrationError::Type::CONFIG_ERROR,
+           fmt::to_string(fmt::join(cfg_errs, "\n"))}};
   }

+   MarketDataListenerData listener_data;
+   listener_data.id = *reg.listener_id;
+   listener_data.listener = listener;
+   listener_data.cfg = cfg;
+   me_->sessions.try_emplace(
+       listener_data.id, listener_data, &me_->sec_id_store);
   return reg;
 }

 void MarketDataGateway::unsubscribe(MarketDataListenerID id) {
   LOG(md, bts::Info, "Unsubscribing listener {}", id);
-   bts_assert(CONTAINS_KEY(me_->registered_listeners, id));
-   auto &ld = me_->registered_listeners[id];
-   auto *listener = ld.listener;
-   bts_assert(listener);
-   auto &sub = ld.subscription;
-
-   switch (sub.venue()) {
-   case api::domain::Venue::BINANCE:
-      bts_assert(
-          sub.params().VenueParams_case() ==
-          api::md::MarketDataSubscriptionParams::kBinance);
-      me_->unsubscribe_binance(sub.params().binance(), listener);
-      break;
-   case api::domain::Venue::FTX:
-      bts_assert(
-          sub.params().VenueParams_case() ==
-          api::md::MarketDataSubscriptionParams::kFtx);
-      me_->unsubscribe_ftx(id);
-      break;
-   default:
-      break;
-   }
-
-   me_->registered_listeners.erase(id);
+   bts_assert(CONTAINS_KEY(me_->sessions, id));
+   me_->sessions.erase(id);
 }

 } // namespace bts::md
@@ -1,14 +1,10 @@
 #pragma once
 #include "MarketDataListenerID.h"
-#include "MarketDataRegistrationFactory.h"
-#include "MarketDataSubscription.h"
-#include "lib/domain/SecurityID.h"
-#include <cstdint>
 #include <memory>
-#include <optional>
-#include <string>
-#include <unordered_map>
-#include <vector>
+
+namespace bts::api::md {
+class MarketDataConfig;
+}

 namespace bts::md {

@@ -25,7 +21,7 @@ class MarketDataGateway {
    * @return MarketDataRegistration
    */
   MarketDataRegistration subscribe(
-       const MarketDataSubscription &, IMarketDataListener *) noexcept;
+       const api::md::MarketDataConfig &, IMarketDataListener *) noexcept;

   void unsubscribe(MarketDataListenerID id);

@@ -1,16 +1,14 @@
 #pragma once
-#include "lib/domain/SecurityID.h"
+#include "lib/api/md/MarketDataConfig.pb.h"
 #include "lib/md/IMarketDataListener.h"
 #include "lib/md/MarketDataListenerID.h"
-#include "lib/md/MarketDataSubscription.h"
-#include <unordered_set>

 namespace bts::md {

 struct MarketDataListenerData {
   MarketDataListenerID id;
   IMarketDataListener *listener;
-   MarketDataSubscription subscription;
+   api::md::MarketDataConfig cfg;
 };

 } // namespace bts::md
@@ -8,10 +8,25 @@ namespace bts::md {

 class MarketDataRegistration {
 public:
+   MarketDataRegistration() = default;
+   explicit MarketDataRegistration(MarketDataListenerID listener_id);
+   explicit MarketDataRegistration(MarketDataRegistrationError error);
   // will have a value when there is no error
   std::optional<MarketDataListenerID> listener_id;
   // domain::Venue venue = domain::Venue::Glbx;
-   MarketDataRegstrationError error = {};
+   MarketDataRegistrationError error = {};
 };

+//////////////////////////////////////////////////
+// Definition
+//////////////////////////////////////////////////
+
+inline MarketDataRegistration::MarketDataRegistration(
+    MarketDataListenerID listener_id)
+    : listener_id(listener_id), error() {}
+
+inline MarketDataRegistration::MarketDataRegistration(
+    MarketDataRegistrationError error)
+    : listener_id(std::nullopt), error(error) {}
+
 } // namespace bts::md
@@ -1,9 +1,10 @@
 #pragma once
 #include <optional>
+#include <string>

 namespace bts::md {

-struct MarketDataRegstrationError {
+struct MarketDataRegistrationError {
   enum class Type {
      NO_ERROR = 0,
      NETWORK_ERROR = 1,
@@ -14,7 +15,8 @@ struct MarketDataRegstrationError {
      // Any error that occurred on our end.  As this error type is vague, the
      // accompanying error message should provide adequate information and
      // context
-      INTERNAL_ERROR = 3
+      INTERNAL_ERROR = 3,
+      CONFIG_ERROR = 4
   };

   Type error_type{Type::NO_ERROR};
@@ -1,59 +0,0 @@
-#pragma once
-#include "lib/api/domain/Venue.pb.h"
-#include "lib/api/md/MarketDataSubscriptionParams.pb.h"
-#include "lib/api/md/deribit/DeribitMarketDataSubscriptionParams.pb.h"
-#include "lib/api/md/glbx/GlbxMarketDataSubscriptionParams.pb.h"
-#include <variant>
-
-namespace bts::md {
-
-class MarketDataSubscription {
- public:
-   api::domain::Venue venue() const;
-
-   MarketDataSubscription() = default;
-   explicit MarketDataSubscription(
-       api::md::MarketDataSubscriptionParams params);
-
-   MarketDataSubscription(const MarketDataSubscription &) = default;
-   MarketDataSubscription(MarketDataSubscription &&) = default;
-   MarketDataSubscription &operator=(const MarketDataSubscription &) = default;
-   MarketDataSubscription &operator=(MarketDataSubscription &&) = default;
-
-   const api::md::MarketDataSubscriptionParams &params() const;
-
- private:
-   api::md::MarketDataSubscriptionParams params_{};
-};
-
-//////////////////////////////////////////////////
-// Definition
-//////////////////////////////////////////////////
-
-inline MarketDataSubscription::MarketDataSubscription(
-    api::md::MarketDataSubscriptionParams params)
-    : params_(std::move(params)) {}
-
-inline const api::md::MarketDataSubscriptionParams &
-MarketDataSubscription::params() const {
-   return params_;
-}
-
-inline api::domain::Venue MarketDataSubscription::venue() const {
-   switch (params_.VenueParams_case()) {
-   case api::md::MarketDataSubscriptionParams::VenueParamsCase::kBinance:
-      return api::domain::Venue::BINANCE;
-   case api::md::MarketDataSubscriptionParams::VenueParamsCase::kDeribit:
-      return api::domain::Venue::DERIBIT;
-   case api::md::MarketDataSubscriptionParams::VenueParamsCase::kFtx:
-      return api::domain::Venue::FTX;
-   case api::md::MarketDataSubscriptionParams::VenueParamsCase::kGlbx:
-      return api::domain::Venue::GLBX;
-   case api::md::MarketDataSubscriptionParams::VenueParamsCase::kHuobi:
-      return api::domain::Venue::HUOBI;
-   default:
-      return api::domain::Venue::NONE;
-   }
-}
-
-} // namespace bts::md
@@ -235,6 +235,30 @@ size_t get_system_md_symbols(
                &result);
      }
   }
+   {
+      if (exchange == api::VENUE_FTX || exchange == api::VENUE_UNSET) {
+         auto const &e = config::get<config::system_symbols::Symbols>().ftx();
+         if (type == api::SECURITYTYPE_FUTURE ||
+             type == api::SECURITYTYPE_UNSET) {
+            add_symbols_to_result(
+                e.future(), api::VENUE_FTX, api::SECURITYTYPE_FUTURE, &result);
+         }
+         if (type == api::SECURITYTYPE_OPTION ||
+             type == api::SECURITYTYPE_UNSET) {
+            add_symbols_to_result(
+                e.option(), api::VENUE_FTX, api::SECURITYTYPE_OPTION, &result);
+         }
+         if (type == api::SECURITYTYPE_STOCK ||
+             type == api::SECURITYTYPE_UNSET) {
+            add_symbols_to_result(
+                e.option(), api::VENUE_FTX, api::SECURITYTYPE_STOCK, &result);
+         }
+         if (type == api::SECURITYTYPE_SPREAD ||
+             type == api::SECURITYTYPE_UNSET)
+            add_symbols_to_result(
+                e.spread(), api::VENUE_FTX, api::SECURITYTYPE_SPREAD, &result);
+      }
+   }
   for (const auto &u : result) {
      LOG(md, Info, "get_system_md_symbols loaded sym: {} exch: {} type: {}",
          u.symbol(), Venue_Name(u.exchange()), SecurityType_Name(u.type()));
@@ -19,15 +19,15 @@ namespace bts {
 namespace md {

 typedef double Price;
-typedef uint32_t Size;
+typedef float Size;

 int to_exponent(double value);

 class PriceSize {
   IntPrice _price;
   // merged size
-   uint32_t _size;
-   uint32_t _direct_size;
+   Size _size;
+   Size _direct_size;

 public:
   PriceSize(IntPrice price, Size size)
@@ -92,13 +92,13 @@ struct PriceLevel {

   IntPrice _bid;
   IntPrice _ask;
-   uint32_t _bid_size;
-   uint32_t _ask_size;
+   Size _bid_size;
+   Size _ask_size;

   const IntPrice &bid_price() const { return _bid; }
   const IntPrice &ask_price() const { return _ask; }
-   uint32_t bid_size() const { return _bid_size; }
-   uint32_t ask_size() const { return _ask_size; }
+   Size bid_size() const { return _bid_size; }
+   Size ask_size() const { return _ask_size; }

   bool operator==(const PriceLevel &other) const {
      return _ask_size == other._ask_size && _bid_size == other._bid_size &&
@@ -1,6 +1,6 @@
 #pragma once
 #include "lib/domain_model/Security.pb.h"
-#include "lib/utils/optional.h"
+#include <optional>
 #include <string>

 namespace bts {
@@ -10,4 +10,4 @@ std::string product_to_string(const api::Product &);
 std::optional<api::Product> string_to_product(const std::string &);

 } // namespace api
-} // namespace bts
+} // namespace bts
@@ -1,6 +1,6 @@
 #include "SecurityDB.h"
 #include "lib/logging/logging.h"
-#ifndef WIN32
+#ifndef _WIN32

 #include <SQLiteCpp/SQLiteCpp.h>
 #include <sqlite/sqlite3.h>
@@ -1,5 +1,5 @@
 #include "lib/domain_model/Security.h"
-#ifndef WIN32
+#ifndef _WIN32
 #include "SecurityIDRecorder.h"

 #include "lib/config/config.h"
@@ -210,7 +210,6 @@ void SecurityIDRecorderState::work_handles() {
   Security security;
   MDProducerHandle handle;
   vector<MDProducerHandle> retry;
-   int count = 0;
   while (pending_handle.try_pop(handle)) {
      security.Clear();
      // if a spread does not have all of its legs we need to wait
@@ -220,7 +219,6 @@ void SecurityIDRecorderState::work_handles() {
         retry.push_back(handle);
      } else {
         convert_security(handle.security(), &security);
-         ++count;
         assign_key(security, [handle](const api::Security &security) mutable {
            handle.set_key(security.id());
         });
@@ -254,7 +252,7 @@ void SecurityIDRecorderState::work_db() {

      // If we're unable to create a descriptor try again later
      if (descriptor == "") {
-         defer.push_back(move(temp));
+         defer.push_back(std::move(temp));
         continue;
      }

@@ -262,10 +260,10 @@ void SecurityIDRecorderState::work_db() {
      if (key != invalid_security_key) {
         temp->security.set_id(key);
         temp->callback(temp->security);
-         existing_securities.push_back(move(temp));
+         existing_securities.push_back(std::move(temp));
      } else {
         // always push in parallel
-         new_entries.push_back(move(temp));
+         new_entries.push_back(std::move(temp));
         new_descriptors.push_back(descriptor);
         ++count;
         // Limit the amount of time we spend writing to sqlite during startup
@@ -279,7 +277,7 @@ void SecurityIDRecorderState::work_db() {
   for (auto &d : defer) {
      LOG(md, Info, "SecurityIDRecorderState::work_db work.push d: {}",
          bts::securities::descriptor(d->security));
-      work.push(move(d));
+      work.push(std::move(d));
   }

   if (!new_entries.empty()) {
@@ -1,5 +1,5 @@
 #pragma once
-#ifndef WIN32
+#ifndef _WIN32
 #include "MDProducerHandle.h"
 #include "common.h"

@@ -133,10 +133,10 @@ class ConcurrentSecurityIndex {
         // support std::move
         accessor->second = entry.release();
         return accessor->second;
-      } else {
-         LOG(md, Error, "Found duplicate entry for platform id '{}' - ", key);
-         bts_assert(false);
      }
+      LOG(md, Error, "Found duplicate entry for platform id '{}' - ", key);
+      bts_assert(false);
+      return nullptr;
   }

   SecurityData *add_security(ExchangeKey key, MDProducerHandle &handle) {
@@ -3,10 +3,11 @@
 #include "PriceSize.h"
 #include "UserSymbol.h"
 #include "lib/config/config.h"
-#include "lib/config/protos/market_data.pb.h"
 #include "lib/config/protos/cme_reference_data.pb.h"
+#include "lib/config/protos/market_data.pb.h"
 #include "lib/domain_model/Security.h"
 #include "lib/utils/parse_enum.h"
+
 #ifndef _WIN32
 #include <btree/map.h>

@@ -29,10 +30,12 @@ struct DisplayFactorMapper : public ISecurityMapper {
                   UserSymbol{uppercase_copy(ov.symbol()), *venue, *type},
                   -to_exponent(ov.price_multiplier())));
            } else {
-               for (auto t :
-                    {SecurityType::SECURITYTYPE_FUTURE,
-                     SecurityType::SECURITYTYPE_SPREAD,
-                     SecurityType::SECURITYTYPE_OPTION}) {
+               for (auto t : {
+                        SecurityType::SECURITYTYPE_FUTURE,
+                        SecurityType::SECURITYTYPE_SPREAD,
+                        SecurityType::SECURITYTYPE_OPTION,
+                        SecurityType::SECURITYTYPE_STOCK,
+                    }) {
                  _mapped.insert(std::make_pair(
                      UserSymbol{uppercase_copy(ov.symbol()), *venue, t},
                      -to_exponent(ov.price_multiplier())));
@@ -1,4 +1,4 @@
-#ifndef WIN32
+#ifndef _WIN32
 #include "SettlementRecorder.h"
 #include "lib/config/config.h"
 #include "lib/config/protos/system.pb.h"
@@ -1,3 +1,4 @@
+#include "lib/domain_model/Security.pb.h"
 #include "lib/utils/bts_assert.h"
 #ifndef NO_CPP0X

@@ -99,6 +100,9 @@ std::string security_descriptor(const api::Security &security) {
          result);
      break;
   }
+   case Security::ContractCase::kSpot: {
+      break;
+   }
   case Security::ContractCase::kSpread: {
      // If the security key of a leg is invalid return an error
      const auto &s = security.spread();
@@ -190,6 +194,10 @@ void convert_security(const FixedSecurity &source, Security *presult) {
      dest.mutable_deribit()->set_security_desc(source.security_desc());
      dest.mutable_deribit()->set_exchange_id(source.numeric_id);
      break;
+   case Venue::VENUE_FTX:
+      dest.mutable_ftx()->set_security_desc(source.security_desc());
+      dest.mutable_ftx()->set_exchange_id(source.numeric_id);
+      break;
   case Venue::VENUE_BTS:
      break;
   default:
@@ -239,7 +247,11 @@ void convert_security(const FixedSecurity &source, Security *presult) {
      bts_assert(dest.symbol() == source.symbol());
      break;
   }
-   case SECURITYTYPE_STOCK:
+   case SECURITYTYPE_STOCK: {
+      bts_assert(source.size == sizeof(FixedStockSecurity));
+      dest.mutable_spot();
+      break;
+   }
   case SECURITYTYPE_UNSET:
   case SecurityType_INT_MAX_SENTINEL_DO_NOT_USE_:
   case SecurityType_INT_MIN_SENTINEL_DO_NOT_USE_:
@@ -255,6 +267,8 @@ SecurityType securitytype_of_contractcase(Security::ContractCase cc) {
      return SECURITYTYPE_FUTURE;
   case Security::ContractCase::kOption:
      return SECURITYTYPE_OPTION;
+   case Security::ContractCase::kSpot:
+      return api::SECURITYTYPE_STOCK;
   case Security::ContractCase::kSpread:
      return SECURITYTYPE_SPREAD;
   case Security::ContractCase::kSynthetic:
@@ -288,6 +302,8 @@ uint64_t exchange_id(const Security &security) {
      return security.ice().exchange_id();
   case Security::ExchangeDataCase::kDeribit:
      return security.deribit().exchange_id();
+   case Security::ExchangeDataCase::kFtx:
+      return security.ftx().exchange_id();
   // Debatable whether this should be an error or not.
   // If is possible that we might create a spread with no exchange in the
   // future and if we did this would be a runtime error.
@@ -338,6 +354,10 @@ void convert_security(
      dest.numeric_id = src.deribit().exchange_id();
      dest.security_desc(src.deribit().security_desc());
      break;
+   case Security::ExchangeDataCase::kFtx:
+      dest.numeric_id = src.ftx().exchange_id();
+      dest.security_desc(src.ftx().security_desc());
+      break;
   case Security::ExchangeDataCase::EXCHANGEDATA_NOT_SET:
      break;
   }
@@ -219,6 +219,7 @@ void with_security(
   switch (type) {
   case api::SECURITYTYPE_STOCK: {
      FixedStockSecurity security;
+      bts_assert(security.type == type);
      cb(security);
      return;
   }
@@ -1,4 +1,4 @@
-#ifndef WIN32
+#ifndef _WIN32
 #include "SyntheticSecurityService.h"

 #include "lib/api/public/ServiceContext.h"
Author	SHA1	Message	Date
John Colvin	caad455784	Update externals Removed fmt and range-3, since we get them from vcpkg now	2024-08-28 23:39:34 -04:00
John Colvin	ac35c789ce	Set up devcontainer for sparkd With config from development branch	2024-08-28 23:26:28 -04:00
John Colvin	1361d7cf5d	Port build configuration over from development branch It's cleaner here, and makes transitioning between branches easier. The new build_spark.py uses cmake --preset and dotnet build.	2024-08-28 23:25:04 -04:00
John Colvin	7b30dbf6b4	build: Update nuget packages in Spark.csproj	2024-08-28 00:05:46 -04:00
John Colvin	7c96b54101	fix build (spark client): Remove deps on preview features Namely, the 'Sum' extension function for an IEnumerable of tuples.	2024-08-28 00:05:46 -04:00
John Colvin	c6e4f090f5	build (windows): Update to c++ 20	2024-08-28 00:05:46 -04:00
John Colvin	dee847a30c	fix build (windows): Update libfmt to 10.2.1	2024-08-28 00:05:10 -04:00
John Colvin	b0c69f7a02	fix build: Remove unnecessary cpp files from win build	2024-08-27 23:28:16 -04:00
John Colvin	ae2e892311	build: Add vcpkg submodule	2024-08-27 13:14:14 -04:00
John Colvin	99e9dbb886	cleanup: Remove unused docker submodules	2024-08-27 13:13:16 -04:00
Taha Afzal	a606a4749f	Merging from spark_crypto.cme	2023-03-20 20:14:25 -04:00
Taha Afzal	fa22cbd386	Some name changes and cleanup	2023-03-20 10:23:21 -04:00
Taha Afzal	6612a6d7e1	Created a moat, SpreaderFractionalQtyHelper, around SpreaderOrder that normalizes the data coming in and de-normalizes the data going out	2023-03-09 16:59:27 -05:00
Taha Afzal	e6b2d6f0b8	Using EMExecutionGateway instead of CCAPIExecutionGateway	2023-03-08 10:25:56 -05:00
Taha Afzal	b5828fbadc	Merge branch 'spark_crypto.prod' into spark_crypto.ccapi.fractional_qty_md	2023-03-07 22:10:11 -05:00
Taha Afzal	3844d9f3ee	Merging from prod	2023-03-07 21:55:17 -05:00
Taha Afzal	309b0e64b2	Adding better logs	2023-01-16 21:46:27 -05:00
Taha Afzal	cfdca08cfe	CME Template 8	2023-01-06 11:52:30 -05:00
Taha Afzal	51c89fb8cc	Merge branch 'spark_crypto.prod' into spark_crypto.cme	2022-12-21 20:38:42 -05:00
Taha Afzal	ded674d87c	Enabling CME market data, and adding some new logs to track initialization of resources	2022-12-19 14:15:04 -05:00
Taha Afzal	648e6206ab	Deactivating FTX market data	2022-11-21 15:41:47 -05:00
Taha Afzal	ef07463a7d	Deribit execution via CCAPI works now (account to key resolution needs to be done)	2022-11-11 23:34:24 -05:00
Taha Afzal	124817d60d	Adding test that used CCAPI to send orders	2022-10-31 22:22:03 -04:00
Taha Afzal	828afb6173	TC2 can now show show fractional qtys	2022-10-24 22:02:21 -04:00
Taha Afzal	852782232c	Merge branch 'spark_crypto.ccapi.fractional_qty_md' of bitbucket.org:bluetradesys/spark into spark_crypto.ccapi.fractional_qty_md	2022-10-24 21:57:53 -04:00
Taha Afzal	add0c3412c	PriceSizeMessage proto's size changed to float from uint32	2022-10-24 21:57:45 -04:00
Taha Afzal	e20b7338c4	Merge branch 'spark_crypto.ccapi.fractional_qty_md' of bitbucket.org:bluetradesys/spark into spark_crypto.ccapi.fractional_qty_md	2022-10-24 13:45:19 -04:00
Taha Afzal	b5da4b3129	Changing managed PriceSize structure's size into floats	2022-10-24 13:45:14 -04:00
Taha Afzal	9f7bafa423	Deribit market data is now using floating point qtys	2022-10-24 13:40:09 -04:00
Taha Afzal	95e99f4855	Changing the Size in PriceSize to float	2022-10-22 13:06:50 -04:00
Taha Afzal	4c4bc3d451	Reducing the logs and couts that were put while interfacing with CC	2022-10-21 13:36:21 -04:00
Taha Afzal	649be1e74e	Enabling spot market dtaa in the UI	2022-10-21 09:13:49 -04:00
Taha Afzal	776c69d96c	Enabling spot market data	2022-10-21 09:12:55 -04:00
Taha Afzal	d371e09e72	Merging	2022-10-20 17:05:47 -04:00
Taha Afzal	c035befe7d	Removing temporary security creation from CCMarketDataListener	2022-10-20 14:06:06 -04:00
Taha Afzal	be3bf93154	UI changes in SecurityExtensions to accomodate FTX	2022-10-20 14:01:01 -04:00
Taha Afzal	9ec5a2d394	Removing #include of a deleted generated header	2022-10-20 11:14:44 -04:00
Taha Afzal	0dbdc6cd62	Merging CCListener work from a different branch	2022-10-20 10:39:06 -04:00
Taha Afzal	0eeb37427f	Fixing the build, unused variable	2022-10-19 22:46:15 -04:00
Taha Afzal	25cd86d44f	QueryServer while handling mic's use switch for exchange instead of ifs	2022-10-19 17:36:02 -04:00
Taha Afzal	bf6a232b5b	IDGenerator is a thread-safe singleton now	2022-10-19 15:29:40 -04:00
John Colvin	47fdae4a71	chore: Remove unnecessary includes	2022-10-19 15:09:24 -04:00
Taha Afzal	32d1714cf6	CCMarketDataListener now listens to CC and writes data to shared memory	2022-10-19 13:55:52 -04:00
John Colvin	df0018a8c0	chore: Remove unused files CCApiRequestor.h/cpp	2022-10-19 13:55:44 -04:00
John Colvin	200919d714	refactor: Return error if md cfg invalid rather than assert/crash	2022-10-19 13:55:44 -04:00
John Colvin	1a2a6b3fb2	fix: Fix typo in 'MarketDataRegistrationError' type name The 'Registration' part was misspelled 'Regstration'	2022-10-19 13:55:44 -04:00
John Colvin	ab5ec98d78	refactor: Hook up ccapi 'on_securities' callback Clean up ccapi md code while we're at it	2022-10-19 13:55:44 -04:00
John Colvin	fd47c3aca9	chore: Remove Binance-specific md impl from MarketDataGateway It was creating needless clutter, as it was written before we knew about ccapi. The new plan is to use a simple adaptor to interface with ccapi for every exchange md. Note: There are a handful of files related to the historic Binance impl in the folder /lib/md/binance. I'm keeping them around for now, since they are generally independent/isolated, and aren't causing clutter or compromsing other code. Some of the utils there may come in handy down the road.	2022-10-19 13:55:44 -04:00
John Colvin	24fb5d0041	refactor: Replace FTX ccapi impl w/ exchange agnostic one	2022-10-19 13:55:44 -04:00
John Colvin	74a4fa133c	chore: Include <string> in MarketDataRegistrationError Apparently <string> was included in all source files that referenced MarketDataRegistrationError by coicidence (else the build would have been broken)	2022-10-19 13:55:44 -04:00
John Colvin	febba9b3f7	build: Add -g to CMAKE_CXX_FLAGS_DEBUG This flag tells compilers (both clang and gcc) to produce debug info	2022-10-19 13:53:29 -04:00
John Colvin	ad2bc86688	chore: Sort includes	2022-10-12 22:11:47 -04:00
John Colvin	dd84ec6720	fix build (g++-12 rel): Add '-Wnomaybe-uninitialized' flag For some reason, this warning isn't suppressed in ccapi_date.h, which is included as a system header (like all headers in externals). According to GCC's documentation, compiler warnings shouldn't be emitted from system headers. Disabling the warning for now so we can focus on bigger fish.	2022-10-12 20:58:24 -04:00
John Colvin	9d79490a20	fix build (client): Change 'WIN32' macros to '_WIN32' Not sure exactly how this broke, but the WIN32 macro is no longer defined.	2022-10-12 20:13:18 -04:00
John Colvin	24059df312	refactor: Remove redundant optional.h It was from a time when Windows std didn't provide the 'optional' header	2022-10-12 20:13:18 -04:00
John Colvin	41862474e9	build: Fix build_spark.py (broken since CMakePresets introduced) Pass additional cmake opts through cli. Specifically, those that were moved from CMakeLists.txt to CMakePrests.json	2022-10-12 20:13:18 -04:00
John Colvin	ccbb1dd5c9	build (refactor): Add cmake_utils.py in py/core So it can be referenced by both client/server build scripts	2022-10-12 20:13:18 -04:00
John Colvin	e24c767c41	Update docker/linux for ubuntu 22.10 image	2022-10-12 20:13:18 -04:00
John Colvin	2b23c3d881	build (web): Update npm packages	2022-10-12 20:13:18 -04:00
John Colvin	0306da2a58	build (refactor): Remove unused cmake var SPARK_PYTHON	2022-10-12 19:29:51 -04:00
John Colvin	511856f989	build (refactor): Change some cmake cache vars to options	2022-10-12 19:29:51 -04:00
John Colvin	09ae3c23a5	build: Fix build_sparkd.py (broken since CMakePresets introduced) Pass additional cmake opts through cli. Specifically, those that were moved from CMakeLists.txt to CMakePrests.json	2022-10-12 19:29:51 -04:00
John Colvin	9dddecbc06	fix build: Update externals Disables a few warnings-as-errors in btree source files	2022-10-12 19:29:51 -04:00
John Colvin	02e31ecae6	fix build (gcc-12, clang>=11): Update externals Disables a few warnings-as-errors in catch2 source files	2022-10-12 19:29:51 -04:00
John Colvin	8722d80f2d	build: Replace setup_builds w/ CMakePresets.json Clean up CMakeLists a bit in the process: * Define more cmake vars in CMakePresets * Split linux/windows presets into separate files	2022-10-12 19:29:51 -04:00
John Colvin	8360c3f63f	build (sparkd): Automatically create py venv in build script	2022-10-12 19:29:51 -04:00
John Colvin	c310544783	build (sparkd): Pipe 'eval' cmd stdout/err back to shell So we can actually see the build output	2022-10-12 19:29:51 -04:00
John Colvin	e9ca7748a4	fix build: Compile with gcc/g++ (version agnostic) We don't really need a strict dependency on a compiler version, though best practice is to use the latest gcc (12).	2022-10-12 19:29:51 -04:00
John Colvin	83ee97419c	tools (log): Fix 'pretty' script & rename 'show_last_run'	2022-10-12 19:29:51 -04:00
John Colvin	b6a1dc61db	fix build (gcc): Disable warning stringop-overread This warning (as error) was introduced with g++ 12	2022-10-12 19:29:51 -04:00
John Colvin	4fb04aafd7	diag: Log emumba cfg parse error code	2022-10-12 19:29:51 -04:00
John Colvin	51cb7d91ba	refactor: Include <limits> in sane_algorithm.h Although the build was fine, many methods in this header file depend on 'limits' so it is appropriate to include it. Presumably all references to such methods are made from source files that include <limits> by coincidence.	2022-10-12 19:29:50 -04:00
John Colvin	14dd9e0204	fix build (gcc): Don't let switch case fall through Even if the case is simply 'bts_assert(false)'	2022-10-12 19:29:50 -04:00
John Colvin	ee330fdf6d	build: Benign changes to remove errors in gcc 12	2022-10-12 19:29:50 -04:00
John Colvin	78b82db3e7	build: Benign changes to remove errors in clang 15	2022-10-12 19:29:50 -04:00
				`@@ -1 +0,0 @@`
				set(sources api/analytics/ExpirationDriver.proto;api/analytics/TimeWeightedInterpolatedDriver.proto;api/apicore.proto;api/apigateway.proto;api/apioptions.proto;api/core/ApiClientInterop.cpp;api/core/ApiServerInterop.cpp;api/domain/Currency.proto;api/domain/SecurityCriteria.proto;api/domain/SecurityCriteriaClause.proto;api/domain/SecurityExpiration.proto;api/domain/SecurityMetadata.proto;api/domain/SecurityMetadataCollection.proto;api/domain/SecurityMetadataItem.proto;api/domain/SecurityType.proto;api/domain/TickDisplaySettings.proto;api/domain/TickFactors.proto;api/domain/Venue.proto;api/io/URL.proto;api/md/MarketDataSubscriptionParams.proto;api/md/binance/BinanceDepthStreamParams.proto;api/md/binance/BinanceMarketDataSubscriptionParams.proto;api/md/deribit/DeribitMarketDataSubscriptionParams.proto;api/md/ftx/FTXDepthStreamParams.proto;api/md/ftx/FTXMarketDataSubscriptionParams.proto;api/md/glbx/GlbxMarketDataSubscriptionParams.proto;api/md/huobi/HuobiMarketDataSubscriptionParams.proto;api/public/ApiServer.cpp;api/public/ClientChannel.cpp;api/public/QuickbusClientChannel.cpp;api/public/ServerChannel.cpp;api/public/ServiceContext.cpp;api/service_helpers/AdminServiceUtils.cpp;api/service_helpers/AlgoServiceUtils.cpp;api/service_helpers/PositionUtilities.cpp;api/services/AdminService.proto;api/services/AlgoDefinitionService.proto;api/services/AlgoService.proto;api/services/AnalyticsService.proto;api/services/ApiUnitTestService.proto;api/services/AuthService.proto;api/services/BasePositionService.proto;api/services/ErrorReportService.proto;api/services/HelloService.proto;api/services/MBOService.proto;api/services/MarginService.proto;api/services/MarketDataAuxiliaryService.proto;api/services/MarketDataRelay.proto;api/services/MarketDataService.proto;api/services/MoneyFileService.proto;api/services/OrderService.proto;api/services/OutOfProcessURLListener.proto;api/services/PersistenceService.proto;api/services/PositionService.proto;api/services/ShellDiagnosticService.proto;api/services/ShellLatencyService.proto;api/services/SyntheticSecurityService.proto;api/test/ApiTests.cpp;api/test/HelloServiceServer.cpp;communincation/websockets/tests/WebsocketSyncClientTests.cpp;config/ConfigCheck.cpp;config/ConfigDownloader.cpp;config/ConfigErrorCollector.cpp;config/ConfigLog.cpp;config/ConfigPropertyEvaluator.cpp;config/ConfigValueQueue.cpp;config/ConfigurationException.cpp;config/ConfigurationState.cpp;config/PreprocessInputStream.cpp;config/config.cpp;config/defaults/MarketData.conf;config/defaults/Ports.conf;config/defaults/SecurityMetadataCollection.conf;config/defaults/Symbols.conf;config/defaults/cme_reference_data.conf;config/defaults/edge.conf;config/defaults/jetfix.conf;config/defaults/keycode.conf;config/defaults/logging.conf;config/defaults/orderman_cfg.conf;config/defaults/orders_fills_cleanup.conf;config/defaults/position_cfg.conf;config/defaults/span.conf;config/defaults/spreader.conf;config/defaults/testkeys.conf;config/defaults/threads.conf;config/defaults/toggles.conf;config/extensions/extensions.proto;config/invalid/invalid.proto;config/protos/algo_types.proto;config/protos/cme_reference_data.proto;config/protos/exchange_sim_cfg.proto;config/protos/jetfix.proto;config/protos/keycode.proto;config/protos/logging.proto;config/protos/market_data.proto;config/protos/mercury_cfg.proto;config/protos/orderman_cfg.proto;config/protos/orders_fills_cleanup.proto;config/protos/perflog_cfg.proto;config/protos/ports.proto;config/protos/position_cfg.proto;config/protos/quickbus_cfg.proto;config/protos/span.proto;config/protos/spartan.proto;config/protos/spreader.proto;config/protos/subprocess.proto;config/protos/symbols.proto;config/protos/system.proto;config/protos/testkeys.proto;config/protos/threads.proto;config/protos/toggles.proto;config/protos/ui.proto;config/tests/HTTPServer.cpp;config/tests/config_ref_tests.cpp;config/tests/config_tests.cpp;config/text_format/text_format.cpp;domain/AggressorSide.cpp;domain/NewOrder.cpp;domain/OrderDefinition.cpp;domain/OrderStatus.cpp;domain/OrderUpdate.cpp;domain/Side.cpp;domain/TimeInForce.cpp;domain_model/Security.cpp;domain_model/Security.proto;domain_model/SecurityTests.cpp;domain_model/sod.cpp;domain_model/tests/domain_tests.cpp;expression/BooleanGrammar.cpp;expression/ExpressionEvaluator.cpp;expression/ParserExceptions.cpp;expression/Value.cpp;expression/tests/expression_parser_tests.cpp;io/QueryParams.cpp;io/URL.cpp;io/http/CurlHandle.cpp;io/http/HTTPClient.cpp;io/http/HTTPHeaders.cpp;io/http/HTTPRequest.cpp;io/http/HTTPResponse.cpp;io/websocket/WebsocketClient.cpp;io/websocket/WebsocketClientConfiguration.cpp;io/websocket/WebsocketError.cpp;io/websocket/WebsocketMessage.cpp;io/websocket/WebsocketSSLClientInternal.cpp;io/websocket/tests/WebsocketClientTests.cpp;jetfix/codec/FIXArray.cpp;jetfix/common/OutFIX.cpp;jetfix/connection/AsyncReader.cpp;jetfix/connection/IConnection.cpp;jetfix/connection/SocketConnection.cpp;jetfix/gateway/CoreGateway.cpp;jetfix/gateway/JetFIXGatLine truncated