build: Upgrade (Windows) c++ to stdlib 20

Features which are now standard in .Net

.gitignore

@@ -51,3 +51,5 @@ trace_data/
 /.venv*/*
 .cache/
 src/web/api/tsconfig.*.g.json
+**/CMakeDeps.cmake
+**/CMakeDeps_win.cmake

.gitmodules

@@ -7,3 +7,6 @@
 [submodule "docker/windows"]
 	path = docker/windows
 	url = git@bitbucket.org:bluetradesys/windows_docker.git
+[submodule "vcpkg"]
+	path = vcpkg
+	url = https://github.com/microsoft/vcpkg.git

CMakeLists.txt

@@ -1,6 +1,9 @@
 cmake_minimum_required(VERSION 3.12)
 project(spark)
 
+include("${CMAKE_SOURCE_DIR}/vcpkg/scripts/buildsystems/vcpkg.cmake")
+set(CMAKE_CXX_STANDARD 20)
+
 if(WIN32)
   set(platform "windows")
 else()
@@ -35,7 +38,7 @@ if(WIN32)
     /D_WINSOCK_DEPRECATED_NO_WARNINGS \
     /D__TBB_NO_IMPLICIT_LINKAGE \
     /DBOOST_ALL_NO_LIB \
-    /std:c++17 \
+    /std:c++20 \
     /D_SCL_SECURE_NO_WARNINGS /D_CRT_SECURE_NO_WARNINGS /D_WIN32_WINNT=0x0600")
   set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /MTd")
   set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_DEBUG} /MT")

doc/user/CMakeDeps.cmake

@@ -1 +0,0 @@
-set(sources /home/taha/spark/doc/user/build.sh;content/_index.md;content/bts_spark/Glossary/_index.md;content/bts_spark/Help/_index.md;content/bts_spark/Help/about_spark.png;content/bts_spark/Help/image39.png;content/bts_spark/Help/image58.png;content/bts_spark/Launcher/_index.md;content/bts_spark/Launcher/image38.png;content/bts_spark/Launcher/image68.png;content/bts_spark/Logging-In-to-the-System/_index.md;content/bts_spark/Logging-In-to-the-System/image3.png;content/bts_spark/Logging-In-to-the-System/image64.png;content/bts_spark/Market/_index.md;content/bts_spark/Market/change_securities.png;content/bts_spark/Market/image11.png;content/bts_spark/Market/image14.png;content/bts_spark/Market/image20.png;content/bts_spark/Market/image21.png;content/bts_spark/Market/image23.png;content/bts_spark/Market/image24.png;content/bts_spark/Market/image34.png;content/bts_spark/Market/image49.png;content/bts_spark/Market/image53.png;content/bts_spark/Market/image66.png;content/bts_spark/Market/image69.png;content/bts_spark/Market/market_view.png;content/bts_spark/Market/settings_icon.png;content/bts_spark/Market/time_and_sales.png;content/bts_spark/My-Windows/_index.md;content/bts_spark/My-Windows/image15.png;content/bts_spark/My-Windows/image16.png;content/bts_spark/My-Windows/image37.png;content/bts_spark/Settings/_index.md;content/bts_spark/Settings/image51.png;content/bts_spark/Settings/image9.png;content/bts_spark/Spreader/_index.md;content/bts_spark/Spreader/image10.png;content/bts_spark/Spreader/image13.png;content/bts_spark/Spreader/image18.png;content/bts_spark/Spreader/image19.png;content/bts_spark/Spreader/image28.png;content/bts_spark/Spreader/image29.png;content/bts_spark/Spreader/image31.png;content/bts_spark/Spreader/image35.png;content/bts_spark/Spreader/image40.png;content/bts_spark/Spreader/image43.png;content/bts_spark/Spreader/image54.png;content/bts_spark/Spreader/image55.png;content/bts_spark/Spreader/image56.png;content/bts_spark/Spreader/image6.png;content/bts_spark/Spreader/image7.png;content/bts_spark/Spreader/image70.png;content/bts_spark/Spreader/image72.png;content/bts_spark/Spreader/image8.png;content/bts_spark/Trading/OrderTypes/index.md;content/bts_spark/Trading/_index.md;content/bts_spark/Trading/algo_studio.png;content/bts_spark/Trading/collapse_all_icon.png;content/bts_spark/Trading/cover_algo_control.png;content/bts_spark/Trading/expand_all_icon.png;content/bts_spark/Trading/image1.png;content/bts_spark/Trading/image17.png;content/bts_spark/Trading/image2.png;content/bts_spark/Trading/image25.png;content/bts_spark/Trading/image27.png;content/bts_spark/Trading/image33.png;content/bts_spark/Trading/image42.png;content/bts_spark/Trading/image44.png;content/bts_spark/Trading/image47.png;content/bts_spark/Trading/image48.png;content/bts_spark/Trading/image5.png;content/bts_spark/Trading/image50.png;content/bts_spark/Trading/image52.png;content/bts_spark/Trading/image57.png;content/bts_spark/Trading/image59.png;content/bts_spark/Trading/image60.png;content/bts_spark/Trading/image61.png;content/bts_spark/Trading/image62.png;content/bts_spark/Trading/image73.png;content/bts_spark/Trading/image74.png;content/bts_spark/Trading/join_algo_control.png;content/bts_spark/Trading/ladder.png;content/bts_spark/Trading/ladder_badges.png;content/bts_spark/Trading/ladder_main_part.png;content/bts_spark/Trading/lean_algo_control.png;content/bts_spark/Trading/mini_orderbook.png;content/bts_spark/Trading/oco_algo_control.png;content/bts_spark/Trading/offset_algo_control.png;content/bts_spark/Trading/pnl_calculation.png;content/bts_spark/Trading/pnl_window.png;content/bts_spark/Trading/recreate_icon.png;content/bts_spark/Trading/settings_icon.png;content/bts_spark/Trading/spark_menu_order_book.png;content/bts_spark/Trading/top_algo_control.png;content/bts_spark/Trading/trigger_algo_control.png;content/bts_spark/_index.md;content/release_notes/_index.md;content/system_requirements/_index.md;static/book.js;static/favicon.ico;static/image.js;templates/index.html;templates/page.html;templates/section.html;templates/shortcodes/back_button.html)

docker/sparkd/dev/Dockerfile

@@ -0,0 +1,135 @@
+FROM ubuntu:latest
+
+ARG DEBIAN_FRONTEND=noninteractive
+ENV TZ=America/New_York
+
+# light deps (should be quick to download/install)
+RUN apt-get update && apt-get install -y \
+    ca-certificates \
+    curl \
+    gnupg \
+    liblttng-ust-dev \
+    libpcap0.8-dev \
+    locales \
+    ninja-build \
+    software-properties-common \
+    sqlite3 \
+    tzdata
+
+# clang
+RUN apt-get update && apt-get install -y \
+    clang-13 \
+    clang-format-13 \
+    clangd-13 \
+    clang-15 \
+    clang-format-15 \
+    clangd-15
+
+# g++
+RUN apt-get update && apt-get install -y \
+    g++-11 \
+    g++-12
+
+# python
+RUN apt-get update && apt-get install -y \
+    python3-pip \
+    python3-venv \
+    && python3 -m pip install --upgrade pip
+
+# dotnet
+RUN apt-get update && apt-get install -y \
+    dotnet7
+
+# ssh
+RUN apt-get update && apt-get install -y \
+    libssh2-1-dev \
+    openssh-server
+
+# cmake
+RUN \
+    curl -fsSL https://apt.kitware.com/keys/kitware-archive-latest.asc 2> /dev/null \
+    | gpg --dearmor - | tee /etc/apt/trusted.gpg.d/kitware.gpg > /dev/null \
+    && apt-add-repository "deb https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" \
+    && apt-get update \
+    && apt-get install -y cmake
+
+# node
+ARG node_major=18
+RUN mkdir -p /etc/apt/keyrings \
+    && curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key \
+    | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg \
+    && echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$node_major.x nodistro main" \
+    | tee /etc/apt/sources.list.d/nodesource.list \
+    && apt-get update \
+    && apt-get install -y nodejs
+
+# ccache is done here so that ccache sees clang++ installed, not clang-10
+RUN apt-get update && apt-get install -y \
+    ccache 
+
+# emumba (light deps)
+RUN apt-get update && apt-get install -y \
+    build-essential \
+    doxygen \
+    python3-sphinx \
+    graphviz \
+    libgtest-dev \
+    libssl-dev \
+    rapidjson-dev \
+    zlib1g-dev \
+    pkg-config
+
+# emumba (boost)
+RUN apt-get update && apt-get install -y \
+    libboost-filesystem-dev \
+    libboost-program-options-dev \
+    libboost-test-dev \
+    libboost-date-time-dev
+
+# dev utils
+RUN apt-get update && apt-get install -y \ 
+    fish \
+    git-all \
+    sudo \
+    vim \
+    wget
+
+# set up default executables (update-alternatives)
+RUN \
+    update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-13 100; \
+    update-alternatives --install /usr/bin/clang clang /usr/bin/clang-13 100; \
+    update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-13 100; \
+    update-alternatives --install /usr/bin/clangd clangd /usr/bin/clangd-13 100; \
+    update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-11 100; \
+    update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 100
+
+# Add compiler symlinks for ccache 
+RUN \
+    ln -s /usr/bin/ccache /usr/local/bin/clang; \
+    ln -s /usr/bin/ccache /usr/local/bin/clang++; \
+    ln -s /usr/bin/ccache /usr/local/bin/clang-13; \
+    ln -s /usr/bin/ccache /usr/local/bin/clang++-13; \
+    ln -s /usr/bin/ccache /usr/local/bin/clang-15; \
+    ln -s /usr/bin/ccache /usr/local/bin/clang++-15; \
+    ln -s /usr/bin/ccache /usr/local/bin/gcc; \
+    ln -s /usr/bin/ccache /usr/local/bin/g++; \
+    ln -s /usr/bin/ccache /usr/local/bin/gcc-11; \
+    ln -s /usr/bin/ccache /usr/local/bin/g++-11; \
+    ln -s /usr/bin/ccache /usr/local/bin/gcc-12; \
+    ln -s /usr/bin/ccache /usr/local/bin/g++-12
+
+RUN locale-gen en_US.UTF-8
+
+ARG docker_user="bts"
+ARG docker_uid="1000"
+ARG docker_pass="bts"
+
+RUN \
+    groupadd -g "$docker_uid" "$docker_user" \
+    && useradd \
+    -m \
+    -u "$docker_uid" \
+    -g "$docker_uid" \
+    -G sudo \
+    --password "$(openssl passwd -1 "$docker_pass")" \
+    "$docker_user"

docker/sparkd/dev/README.md

@@ -0,0 +1,64 @@
+[dev containers]: https://code.visualstudio.com/docs/devcontainers/containers
+[dev containers ext]: https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-containers
+[docker repo sparkd-dev]: https://hub.docker.com/repository/docker/bluetradingsystems/sparkd-dev/general
+# Sparkd Dev Container
+## How to create a VS Code [Dev Container][dev containers]
+
+1. Make sure the **[Dev Containers][dev containers ext]** extension is installed
+1. Make sure your Docker image is up to date
+   ```
+   docker pull bluetradingsystems/sparkd-dev:latest
+   ```
+1. Copy **devcontainer.json** from this folder to **$workspaceFolder/.devcontainer/**.
+
+1. In the VS Code Command Palette, run **Dev Containers: Reopen in Container**
+
+---
+## How to create a Sparkd Container in the terminal (without VS Code)
+
+1. Make sure your Docker image is up to date
+   ```
+   docker pull bluetradingsystems/sparkd-dev:latest
+   ```
+2. Start the container
+   ```
+   # Replace <cont_name> with a name for the container. The container's
+   # "$HOME" dir will be mounted to "$HOME/.sparkd-dev/mounts/<cont_name> on
+   # the host
+   
+   ./start_docker.sh --container <cont_name> --mount-existing
+   ```
+   * *Replace <cont_name> with a name for the container*
+   * *The container's `$HOME` dir will be mounted to
+   `$HOME/.sparkd-dev/mounts<cont_name>` on the host*<br>
+   * *Run: `./start_docker.sh --help` to see additional options*
+3. *(Optional) Attach VS Code to the running container.*<br>
+Run **Dev Containers: Attach to Running Container** from the command palette,
+then select your container in the dropdown
+
+---
+## How to update the Docker image
+
+1. Change the image by modifying the Dockerfile (don't tag arbitrary
+  images on your machine and push them to our repo on Docker Hub)
+
+1. After modifying the Dockerfile, (re)build the image
+   ```
+   # Run in this dir
+   docker build -t bluetradingsystems/sparkd-dev:latest .
+   ```
+   ***Note:** Your docker login must have write permissions to push to the
+   [sparkd-dev][docker repo sparkd-dev] repository*
+
+2. Commit the changes to Dockerfile in git
+
+3. Push the new image to Docker Hub
+
+   ```
+   docker push bluetradingsystems/sparkd-dev:latest
+   ```
+   ***Note:** The image doesn't have to be on Docker Hub, we can simply build
+   it from scratch on our local machines (with the Dockerfile). However, this
+   takes time. It is significantly quicker to download a prebuilt image, so we
+   store it on a remote repository to save time when setting up a new dev
+   machine.*

docker/sparkd/dev/bashrc

@@ -0,0 +1,123 @@
+# ~/.bashrc: executed by bash(1) for non-login shells.
+# see /usr/share/doc/bash/examples/startup-files (in the package bash-doc)
+# for examples
+
+# If not running interactively, don't do anything
+case $- in
+    *i*) ;;
+      *) return;;
+esac
+
+# don't put duplicate lines or lines starting with space in the history.
+# See bash(1) for more options
+HISTCONTROL=ignoreboth
+
+# append to the history file, don't overwrite it
+shopt -s histappend
+
+# for setting history length see HISTSIZE and HISTFILESIZE in bash(1)
+HISTSIZE=1000
+HISTFILESIZE=2000
+
+# check the window size after each command and, if necessary,
+# update the values of LINES and COLUMNS.
+shopt -s checkwinsize
+
+# If set, the pattern "**" used in a pathname expansion context will
+# match all files and zero or more directories and subdirectories.
+#shopt -s globstar
+
+# make less more friendly for non-text input files, see lesspipe(1)
+[ -x /usr/bin/lesspipe ] && eval "$(SHELL=/bin/sh lesspipe)"
+
+# set variable identifying the chroot you work in (used in the prompt below)
+if [ -z "${debian_chroot:-}" ] && [ -r /etc/debian_chroot ]; then
+    debian_chroot=$(cat /etc/debian_chroot)
+fi
+
+# set a fancy prompt (non-color, unless we know we "want" color)
+case "$TERM" in
+    xterm-color|*-256color) color_prompt=yes;;
+esac
+
+# uncomment for a colored prompt, if the terminal has the capability; turned
+# off by default to not distract the user: the focus in a terminal window
+# should be on the output of commands, not on the prompt
+#force_color_prompt=yes
+
+if [ -n "$force_color_prompt" ]; then
+    if [ -x /usr/bin/tput ] && tput setaf 1 >&/dev/null; then
+	# We have color support; assume it's compliant with Ecma-48
+	# (ISO/IEC-6429). (Lack of such support is extremely rare, and such
+	# a case would tend to support setf rather than setaf.)
+	color_prompt=yes
+    else
+	color_prompt=
+    fi
+fi
+
+if [ "$color_prompt" = yes ]; then
+    PS1='${debian_chroot:+($debian_chroot)}\[\033[01;32m\]\u@\h\[\033[00m\]:\[\033[01;34m\]\w\[\033[00m\]\$ '
+else
+    PS1='${debian_chroot:+($debian_chroot)}\u@\h:\w\$ '
+fi
+unset color_prompt force_color_prompt
+
+# If this is an xterm set the title to user@host:dir
+case "$TERM" in
+xterm*|rxvt*)
+    PS1="\[\e]0;${debian_chroot:+($debian_chroot)}\u@\h: \w\a\]$PS1"
+    ;;
+*)
+    ;;
+esac
+
+# enable color support of ls and also add handy aliases
+if [ -x /usr/bin/dircolors ]; then
+    test -r ~/.dircolors && eval "$(dircolors -b ~/.dircolors)" || eval "$(dircolors -b)"
+    alias ls='ls --color=auto'
+    #alias dir='dir --color=auto'
+    #alias vdir='vdir --color=auto'
+
+    alias grep='grep --color=auto'
+    alias fgrep='fgrep --color=auto'
+    alias egrep='egrep --color=auto'
+fi
+
+# colored GCC warnings and errors
+#export GCC_COLORS='error=01;31:warning=01;35:note=01;36:caret=01;32:locus=01:quote=01'
+
+# some more ls aliases
+alias ll='ls -alF'
+alias la='ls -A'
+alias l='ls -CF'
+
+# Add an "alert" alias for long running commands.  Use like so:
+#   sleep 10; alert
+alias alert='notify-send --urgency=low -i "$([ $? = 0 ] && echo terminal || echo error)" "$(history|tail -n1|sed -e '\''s/^\s*[0-9]\+\s*//;s/[;&|]\s*alert$//'\'')"'
+
+# Alias definitions.
+# You may want to put all your additions into a separate file like
+# ~/.bash_aliases, instead of adding them here directly.
+# See /usr/share/doc/bash-doc/examples in the bash-doc package.
+
+if [ -f ~/.bash_aliases ]; then
+    . ~/.bash_aliases
+fi
+
+# enable programmable completion features (you don't need to enable
+# this, if it's already enabled in /etc/bash.bashrc and /etc/profile
+# sources /etc/bash.bashrc).
+if ! shopt -oq posix; then
+  if [ -f /usr/share/bash-completion/bash_completion ]; then
+    . /usr/share/bash-completion/bash_completion
+  elif [ -f /etc/bash_completion ]; then
+    . /etc/bash_completion
+  fi
+fi
+
+if [[ ! -v XDG_CONFIG_HOME ]]; then
+  export XDG_CONFIG_HOME="$HOME/.config"
+fi
+
+export PATH="$HOME/.local/bin:$PATH"

docker/sparkd/dev/config.fish

@@ -0,0 +1,25 @@
+if ! set -q XDG_CONFIG_HOME
+    set -g XDG_CONFIG_HOME "$HOME/.config"
+end
+set fish_config_home $XDG_CONFIG_HOME/fish
+if test -f "$fish_config_home/aliases.fish"
+    source "$fish_config_home/aliases.fish"
+end
+# enable general colors and case insensitive search in 'less'
+set -x LESS "$LESS -R -Q -I"
+set -x LC_ALL en_US.UTF-8
+# enable syntax highlighting in 'less'
+# set highlight color scheme in 'less' (fg)(bg)
+set -g LESS_TERMCAP_md (tput bold)(tput setaf 36)
+set -g LESS_TERMCAP_me (tput sgr0)
+set -g LESS_TERMCAP_so (tput setaf 232)(tput setab 49)
+
+set -q TERM
+if test $status != 0; or test $TERM = 'xterm'
+    set -g TERM 'xterm-256color'
+end
+
+# Fix impossible-to-see navy blue command text color
+set fish_color_command F8F8F2
+
+set -g PATH "$HOME/.local/bin" "$PATH"

docker/sparkd/dev/devcontainer.json

@@ -0,0 +1,32 @@
+{
+  "image": "bluetradingsystems/sparkd-dev:latest",
+  "initializeCommand": "${localWorkspaceFolder}/docker/sparkd/dev/devcontainer_initialize.sh",
+  "hostRequirements": {
+    "memory": "4gb",
+    "storage": "6gb"
+  },
+  "postCreateCommand": "/home/bts/spark/docker/sparkd/dev/devcontainer_post_create.sh",
+  "containerUser": "bts",
+  "workspaceMount": "source=${localWorkspaceFolder},target=/home/bts/spark,type=bind",
+  "workspaceFolder": "/home/bts/spark",
+  "updateRemoteUserUID": true,
+  "shutdownAction": "none",
+  // Note: If running multiple containers on the same host, each must use a
+  // unique set of ports. Usually we just add 1000 to each port as needed. For
+  // example, if 8000-8003 is taken, go with 9000-9003, then 10000-10003, etc.
+  "runArgs": [
+    "--shm-size",
+    "2gb",
+    "-p",
+    "8000:8000",
+    "-p",
+    "8001:8001",
+    "-p",
+    "8002:8002",
+    "-p",
+    "8003:8003"
+  ],
+  "mounts": [
+    "source=${localEnv:HOME}/.sparkd-dev/mounts/sparkd-dev,target=/home/bts,type=bind"
+  ]
+}

docker/sparkd/dev/devcontainer_initialize.sh

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+mkdir -p "$HOME/.sparkd-dev/mounts/sparkd-dev"

docker/sparkd/dev/devcontainer_post_create.sh

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+
+this_dir="$(realpath "$(dirname "${BASH_SOURCE[0]}")")"
+repo_dir="$(cd "$this_dir" && git rev-parse --show-toplevel)"
+tools_dir="$repo_dir/tools"
+gen_tag="GENERATED BY $(basename "${BASH_SOURCE[0]}")"
+
+function copy_if_not_exist() {
+  src="$1"
+  dest="$2"
+  if [[ ! -e "$src" ]]; then
+    echo "Unable to copy $src to $dest. Source file doesn't exist"
+    exit 1
+  fi
+  if [[ ! -e "$dest" ]]; then
+    cp "$src" "$dest"
+  fi
+}
+
+# Init bashrc
+echo "Initializing bashrc"
+copy_if_not_exist "$this_dir/bashrc" "$HOME/.bashrc"
+
+# Init fish config
+echo "Initializing fish config"
+fish_cfg_dir="$HOME/.config/fish/"
+mkdir -p "$fish_cfg_dir"
+exec "$tools_dir/write_tag_block.sh" \
+  --tag "$gen_tag" \
+  --replace \
+  --in-place \
+  --file "$fish_cfg_dir/config.fish" \
+  --text "$(cat "$this_dir/config.fish")"

docker/sparkd/dev/set_user.sh

@@ -0,0 +1,250 @@
+#!/usr/bin/env bash
+
+function _exit() {
+  if [[ "$#" -gt 1 ]]; then
+    # When there are multiple args, the last arg is the exit code.
+    echo "${@:1:($# - 1)}"
+    exit "${@: -1}"
+  fi
+  echo "$1"
+  exit 0
+}
+
+help_text="\
+Add or update a user's uid
+
+This script is intended to be run in a Docker container as a means to obtain
+file permissions in mounted directories owned by the specified UID.
+
+Usage: set_user.sh [OPTIONS] USER UID
+
+Positionals:
+  USER               The name of the user
+
+Options:
+  --user USER        The name of the user (defaults to 'bts')
+  --uid UID          The new uid to give USER (defaults to '1000')
+  --group GROUP      The primary group to give USER (defaults to USER)
+  --gid GID          The primary gid to give USER (defaults to UID)
+  --pass PASSWORD    The password to give USER (defaults to USER, only applies
+                     if USER hasn't been created yet) 
+  --modify-existing  Update UID/GID even if USER already exists
+  -h,--help          Print this help message and exit
+
+"
+
+function _add_group() {
+  if [[ "$#" -ne 2 ]]; then
+    _exit "'_add_group' requires exactly 2 args (group, gid). Received $#." 1
+  fi
+  local group="$1"
+  local gid="$2"
+  # Check if gid already exists
+  local existing_group=""
+  existing_group="$(getent group "$gid" | cut -d : -f 1)"
+  if [[ -n "$existing_group" ]]; then
+    if [[ "$existing_group" != "$group" ]]; then
+      echo "\
+The gid '$gid' is already in use by group '$existing_group' (will not rename \
+the group)"
+    fi
+    return
+  fi
+
+  # Check if group with same name already exists
+  local existing_gid=""
+  existing_gid="$(getent group "$group" | cut -d : -f 3)"
+  if [[ -n "$existing_gid" ]]; then
+    # The group already exists. Only modify if --modify-existing was specified.
+    if [[ "$modify_existing" -eq 0 ]]; then
+      _exit "\
+The group '$group' already exists with gid '$existing_gid'.
+Specify --modify-existing to change the gid anyways." 1
+    fi
+    echo "Changing gid for group '$group' from '$existing_gid' to '$gid'"
+    groupmod -g "$gid" "$group"
+  else
+    # The group doesn't exist. Add it
+    echo "Adding group '$group:$gid'"
+    groupadd -g "$gid" "$group"
+  fi
+}
+
+function _add_user() {
+  local user="$1"
+  local uid="$2"
+  local group="$3"
+  local gid="$4"
+  local pass="$5"
+  if [[ "$#" -ne 5 ]]; then
+    _exit "'_add_user' requires exactly 5 args (user, uid, group, gid, pass). \
+Received $#." 1
+  fi
+
+  # Check if uid is being used by another user
+  local existing_user=""
+  existing_user="$(getent passwd "$uid" | cut -d : -f 1)"
+  if [[ -n "$existing_user" ]]; then
+    if [[ "$existing_user" != "$user" ]]; then
+      _exit "The uid '$uid' is already in use by user '$existing_user'." 1
+    fi
+
+    echo "The user '$user' already has the specified uid '$uid'."
+    # Modify gid if necessary
+    local existing_gid=""
+    existing_gid="$(id -g "$user" 2> /dev/null)"
+    if [[ "$existing_gid" = "$gid" ]]; then
+      echo "The user '$user' already has the specified gid '$gid'."
+      return
+    fi
+
+    if [[ "$modify_existing" -eq 0 ]]; then
+      _exit "\
+The user '$user' is associated with the primary group '$existing_gid'.
+Specify --modify-existing to change the gid anyways." 1
+    fi
+
+    echo "Changing primary group for '$user' from '$existing_gid' to '$gid'."
+    usermod -g "$gid" "$user"
+    return
+  fi
+
+  # Check if user with same name already exists
+  local existing_uid=""
+  existing_uid="$(id -u "$user" 2> /dev/null)"
+  if [[ -n "$existing_uid" ]]; then
+    if [[ "$existing_uid" = "$uid" ]]; then
+      _exit "The user '$user' already has the specified uid '$uid'" 0
+    fi
+
+    if [[ "$modify_existing" -eq 0 ]]; then
+      _exit "\
+User '$user' already exists with uid '$existing_uid'.
+Specify --modify-existing to change the uid anyways." 1
+    fi
+
+    echo "Changing uid for user '$user' from '$existing_uid' to '$uid'"
+    if ! usermod -u "$uid" -g "$gid" "$user"; then
+      _exit "usermod exited with code $?" 1
+    fi
+    return
+  fi
+
+  # The user doesn't exist. Add it
+  echo "Adding user '$user:$uid' (group='$group:$gid', password='$pass')"
+  useradd -m -u "$uid" -g "$gid" -G sudo --password "$(openssl passwd -1 "$pass")" "$user"
+}
+
+function _update_fish_dir_perms() {
+  local user="$1"
+  local uid="$2"
+  local gid="$3"
+  # When a fish shell session starts, a temp dir named '/tmp/fish.<user>' is
+  # created with the user's current uid/gid. If such a dir exists for the
+  # specified USER, make sure the uid/gid match the new values.
+  fish_dir="/tmp/fish.$user"
+  if [[ -d "$fish_dir" ]]; then
+    perms="$uid:$gid"
+    existing_perms="$(stat -c %u:%g "$fish_dir")"
+    if [[ "$existing_perms" != "$perms" ]]; then
+      echo "Changing uid:gid for dir '$fish_dir' from '$existing_perms' to '$perms'"
+      chown -R "$perms" "$fish_dir"
+    fi
+  fi
+}
+
+user=""
+group=""
+pass=""
+declare -i uid=0
+declare -i gid=0
+declare -i modify_existing=0
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --user)
+      if [[ -z "$2" ]]; then
+        _exit "USER (--user) must be non-empty" 1
+      elif [[ "$2" = "root" ]]; then
+        _exit "USER (--user) cannot be 'root'" 1
+      fi
+      user="$2"
+      shift 2
+      ;;
+    --group)
+      if [[ -z "$2" ]]; then
+        _exit "GROUP (--group) must be non-empty" 1
+      elif [[ "$2" = "root" ]]; then
+        _exit "GROUP (--group) cannot be 'root'" 1
+      fi
+      group="$2"
+      shift 2
+      ;;
+    --uid)
+      if [[ -z "$2" ]]; then
+        _exit "UID (--uid) must be non-empty" 1
+      fi
+      uid="$2"
+      shift 2
+      ;;
+    --gid)
+      if [[ -z "$2" ]]; then
+        _exit "GID (--gid) must be non-empty" 1
+      fi
+      gid="$2"
+      shift 2
+      ;;
+    --pass)
+      if [[ -z "$2" ]]; then
+        _exit "PASS (--pass) must be non-empty" 1
+      fi
+      pass="$2"
+      shift 2
+      ;;
+    --modify-existing)
+      modify_existing=1
+      shift 1
+      ;;
+    -h)
+      ;&
+    --help)
+      _exit "$help_text" 0
+      ;;
+    *)
+      if [[ -z "$1" ]]; then
+        _exit "Invalid positional argument '$1'" 1
+      elif [[ -z "$user" ]]; then
+        user="$1"
+      elif [[ -z "$uid" ]]; then
+        uid="$1"
+      else
+        _exit "Unexpected argument '$1'" 1
+      fi
+      shift 1
+      ;;
+  esac
+done
+
+if [[ -z "$user" ]]; then
+  user="bts"
+fi
+
+if [[ $uid -eq 0 ]]; then
+  uid=1000
+fi
+
+# In general, a user's primary group/gid matches the user's name/uid.
+# If group/gid args weren't specified, set them to name/uid.
+if [[ -z "$group" ]]; then
+  group="$user"
+fi
+if [[ $gid -eq 0 ]]; then
+  gid=$uid
+fi
+if [[ -z "$pass" ]]; then
+  pass="$user"
+fi
+
+_add_group "$group" "$gid"
+_add_user "$user" "$uid" "$group" "$gid" "$pass"
+_update_fish_dir_perms "$user" "$uid" "$gid"

docker/sparkd/dev/start_docker.sh

@@ -0,0 +1,230 @@
+#!/usr/bin/env bash
+
+function _exit() {
+  if [[ "$#" -gt 1 ]]; then
+    # When there are multiple args, the last arg is the exit code.
+    echo "${@:1:($# - 1)}"
+    exit "${@: -1}"
+  fi
+  echo "$1"
+  exit 0
+}
+
+help_text="\
+Start a docker dev container
+Usage: start_docker.sh [OPTIONS] image container
+
+Positionals:
+  image             The image to instantiate
+  container         The name for the container
+
+Options:
+  -u,--user USER      The user to add in the container (defaults to 'bts')
+  -p,--pass PASS      The password for USER in the container (defaults to USER)
+  -i,--image IMAGE    The image to instantiate (defaults to 'bluetradingsystems/sparkd-dev:latest')
+  -c,--container CONT The name for the container
+  -m,--mount MOUNT    The name for the folder (outside the container) to create/mount to the home
+                      directory in the container. Mount folders are added here: $mount_prefix
+  --mount-existing    Mount the specified dir even if it already exists
+  --port <cont_port>:<host_port> OR [<cont_port>:<host_port>,...] Port mapping(s). May be
+                      specified multiple times.
+  -h,--help           Print this help message and exit
+
+"
+
+mount_prefix="$HOME/.sparkd-dev/mounts"
+declare docker_user=""
+declare docker_pass=""
+declare image_name=""
+declare cont_name=""
+declare mount_name=""
+declare -i mount_existing=0
+declare -a port_mappings=()
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    -u)
+      ;&
+    --user)
+      if [[ -z "$2" ]]; then
+        _exit "User (-u,--user) must be non-empty" 1
+      fi
+      docker_user="$2"
+      shift 2
+      ;;
+    -p)
+      ;&
+    --pass)
+      if [[ -z "$2" ]]; then
+        _exit "Password (-p,--pass) must be non-empty" 1
+      fi
+      docker_pass="$2"
+      shift 2
+      ;;
+    -i)
+      ;&
+    --image)
+      if [[ -n "$image_name" ]]; then
+        _exit "Unexpected argument '$1'" 1
+      elif [[ -z "$2" ]]; then
+        _exit "Image (-i,--image) must be non-empty" 1
+      fi
+      image_name="$2"
+      shift 2
+      ;;
+    -c)
+      ;&
+    --container)
+      if [[ -n "$cont_name" ]]; then
+        _exit "Unexpected argument '$1'" 1
+      elif [[ -z "$2" ]]; then
+        _exit "Container (-c,--container) must be non-empty" 1
+      fi
+      cont_name="$2"
+      shift 2
+      ;;
+    --port)
+      # Parse port(s) arg, and append the result to 'port_mappings'
+      # Note: port(s) may be specified in bracketed notation:
+      #   [a:b,c:d,...,y:z]
+      next_ports="$(echo "$2" | sed -e 's/\[//g' -e 's/\]//g' -e 's/,/ /g')"
+      if [[ "${#port_mappings[@]}" -gt 0 ]]; then
+        next_ports=" $next_ports"
+      fi
+      port_mappings+=("$next_ports")
+      shift 2
+      ;;
+    -m)
+      ;&
+    --mount)
+      if [[ -z "$2" ]]; then
+        echo "Mount (-m,--mount) must be non-empty"
+      fi
+      mount_name="$2"
+      shift 2
+      ;;
+    --mount-existing)
+      mount_existing=1
+      shift 1
+      ;;
+    -h)
+      ;&
+    --help)
+      _exit "$help_text" 0
+      ;;
+    *)
+      if [[ -z "$1" ]]; then
+        _exit "Invalid positional argument '$1'" "$help_text" 1
+      elif [[ -z "$image_name" ]]; then
+        image_name="$1"
+      elif [[ -z "$cont_name" ]]; then
+        cont_name="$1"
+      else
+        _exit "Unexpected argument '$1'" "$help_text" 1
+      fi
+      shift 1
+      ;;
+  esac
+done
+
+
+if [[ -z "$image_name" ]]; then
+  image_name="bluetradingsystems/sparkd-dev:latest"
+fi
+
+host_name="$(hostname)"
+if [[ -n "$cont_name" ]]; then
+  host_name="$host_name.$cont_name"
+fi
+
+if [[ -z "$mount_name" ]]; then
+  mount_name="$cont_name"
+fi
+
+if [[ -z "$mount_name" && -z "$cont_name" ]]; then
+  _exit "Please specify mount (-m,--mount) and/or container (-c,--container)" 1
+fi
+
+if [[ -z "$docker_user" ]]; then
+  docker_user="bts"
+fi
+
+if [[ "$docker_user" = "root" ]]; then
+  _exit "USER (--user) cannot be 'root'" 1
+fi
+
+cont_home="/home/$docker_user"
+home_src="$mount_prefix/$mount_name"
+home_tgt="$cont_home"
+this_dir="$(dirname "$(realpath "${BASH_SOURCE[0]}")")"
+repo_src="$(cd "$this_dir" && git rev-parse --show-toplevel)"
+repo_tgt="$cont_home/spark"
+working_dir="$cont_home"
+
+if [[ $mount_existing -eq 0 && -e "$home_src" ]]; then
+  # Don't overwrite existing dir (might be mounted in another container)
+  _exit "Mount dir '$home_src' already exists (use --mount-existing to mount \
+an existing directory)" 1
+fi
+
+mkdir -p "$home_src"
+
+if [[ "${#port_mappings[@]}" -eq 0 ]]; then
+  port_mappings=("8000:8000" "8001:8001" "8002:8002" "8003:8003")
+fi
+
+# Adapt port_mappings into args for `docker run` by prepending "-p " to each
+# item
+declare -a docker_port_args=()
+for port_mapping in "${port_mappings[@]}"; do
+  docker_port_args+=("-p $port_mapping")
+done
+
+echo "Starting container '$cont_name'
+image_name  : $image_name
+cont_name   : $cont_name
+home_src    : $home_src
+home_tgt    : $home_tgt
+repo_src    : $repo_src
+repo_tgt    : $repo_tgt
+working dir : $working_dir
+ports       : ${port_mappings[*]}
+"
+
+# DON'T wrap $docker_port_args in quotes. We want bash to split args by
+# whitespace
+docker run \
+  --net bridge \
+  -h "$host_name" \
+  -e "TZ=America/New_York" \
+  --cap-add=SYS_PTRACE \
+  --security-opt seccomp=unconfined \
+  --name "$cont_name" \
+  -v "$home_src:$home_tgt" \
+  -v "$repo_src:$repo_tgt" \
+  -w "$working_dir" \
+  ${docker_port_args[*]} \
+  --shm-size=4024m \
+  -t \
+  -i \
+  -d \
+  -e DISPLAY="$DISPLAY" \
+  "$image_name"
+
+# Add user w/ same uid and gid as the current user outside of docker, so the
+# user inside the docker has file permissions in the mounted dir.
+docker_group="$docker_user"
+docker_uid="$(id -u "$USER")"
+docker_gid="$(id -g "$USER")"
+if [[ -z "$docker_pass" ]]; then
+  docker_pass="$docker_user"
+fi
+docker exec -u root -it "$cont_name" "$repo_tgt/docker/sparkd/dev/set_user.sh" \
+--user "$docker_user" \
+--uid "$docker_uid" \
+--group "$docker_group" \
+--gid "$docker_gid" \
+--pass "$docker_pass"
+
+# Run same script VS Code runs to initialize the container
+docker exec -u "$docker_user" -it "$cont_name" "$repo_tgt/docker/sparkd/dev/devcontainer_post_create.sh"

docker/sparkd/docker_shell.sh

@@ -0,0 +1,64 @@
+#!/usr/bin/env bash
+
+function print_help_and_exit() {
+  cat <<EOF
+Open a shell inside the docker container
+Usage: docker_shell.sh [OPTIONS] [CONTAINER]
+
+Options:
+  -u,--user         The user to login with in the container (defaults to the current user)
+  -c,--container    The container name (may also be specified as positional arg)
+  -h,--help         Print this help message and exit
+
+EOF
+  exit "$1"
+}
+
+
+declare docker_user=""
+declare cont_name=""
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    -u)
+      ;&
+    --user)
+      if [[ -z "$2" ]]; then
+        echo "User (-u,--user) must be non-empty"
+        print_help_and_exit 1
+      fi
+      docker_user="$2"
+      shift 2
+      ;;
+    -h)
+      ;&
+    --help)
+      print_help_and_exit 0
+      ;;
+    -c)
+      ;&
+    --container)
+      shift 1
+      ;&
+    *)
+      if [[ ! -z "$cont_name" ]]; then
+        echo "Unexpected argument '$1'"
+        print_help_and_exit 1
+      fi
+      cont_name="$1"
+      shift 1
+      ;;
+  esac
+done
+
+
+if [[ -z "$cont_name" ]]; then
+  echo "Please specify the container ([-c,--container])"
+  print_help_and_exit 1
+fi
+
+if [[ -z "$docker_user" ]]; then
+  docker_user="$USER"
+fi
+
+docker exec -u "$docker_user" -it "$cont_name" bash

docker/sparkd/start_docker.sh

@@ -0,0 +1,233 @@
+#!/usr/bin/env bash
+
+mount_prefix="$HOME/.spark-dev/mounts"
+
+function print_help_and_exit() {
+  cat <<EOF
+Start a docker dev container
+Usage: start_docker.sh [OPTIONS] image container
+
+Positionals:
+  image             The image to instantiate
+  container         The name for the container
+
+Options:
+  -u,--user USER      The user to add in the container (defaults to the current user)
+  -p,--pass PASS      The password for USER in the container (defaults to USER)
+  -i,--image IMAGE    The image to instantiate
+  -c,--container CONT The name for the container
+  -m,--mount MOUNT    The name for the folder (outside the container) to create/mount to the home
+                      directory in the container. Mount folders are added here: $mount_prefix
+  --mount-existing    Mount the specified dir even if it already exists
+  --port <cont_port>:<host_port> OR [<cont_port>:<host_port>,...] Port mapping(s). May be
+                      specified multiple times.
+  -h,--help           Print this help message and exit
+
+EOF
+  exit "$1"
+}
+
+
+declare docker_user=""
+declare docker_pass=""
+declare image_name=""
+declare cont_name=""
+declare mount_name=""
+declare -i mount_existing=0
+declare -a port_mappings=()
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    -u)
+      ;&
+    --user)
+      if [[ -z "$2" ]]; then
+        echo "User (-u,--user) must be non-empty"
+        print_help_and_exit 1
+      fi
+      docker_user="$2"
+      shift 2
+      ;;
+    -p)
+      ;&
+    --pass)
+      if [[ -z "$2" ]]; then
+        echo "Password (-p,--pass) must be non-empty"
+        print_help_and_exit 1
+      fi
+      docker_pass="$2"
+      shift 2
+      ;;
+    -i)
+      ;&
+    --image)
+      if [[ -n "$image_name" ]]; then
+        echo "Unexpected argument '$1'"
+        print_help_and_exit 1
+      elif [[ -z "$2" ]]; then
+        echo "Image (-i,--image) must be non-empty"
+        print_help_and_exit 1
+      fi
+      image_name="$2"
+      shift 2
+      ;;
+    -c)
+      ;&
+    --container)
+      if [[ -n "$cont_name" ]]; then
+        echo "Unexpected argument '$1'"
+        print_help_and_exit 1
+      elif [[ -z "$2" ]]; then
+        echo "Container (-c,--container) must be non-empty"
+        print_help_and_exit 1
+      fi
+      cont_name="$2"
+      shift 2
+      ;;
+    --port)
+      # Parse port(s) arg, and append the result to 'port_mappings'
+      # Note: port(s) may be specified in bracketed notation:
+      #   [a:b,c:d,...,y:z]
+      next_ports="$(echo "$2" | sed -e 's/\[//g' -e 's/\]//g' -e 's/,/ /g')"
+      if [[ "${#port_mappings[@]}" -gt 0 ]]; then
+        next_ports=" $next_ports"
+      fi
+      port_mappings+=("$next_ports")
+      shift 2
+      ;;
+    -m)
+      ;&
+    --mount)
+      if [[ -z "$2" ]]; then
+        echo "Mount (-m,--mount) must be non-empty"
+      fi
+      mount_name="$2"
+      shift 2
+      ;;
+    --mount-existing)
+      mount_existing=1
+      shift 1
+      ;;
+    -h)
+      ;&
+    --help)
+      print_help_and_exit 0
+      ;;
+    *)
+      if [[ -z "$1" ]]; then
+        echo "Invalid positional argument '$1'"
+        print_help_and_exit 1
+      elif [[ -z "$image_name" ]]; then
+        image_name="$1"
+      elif [[ -z "$cont_name" ]]; then
+        cont_name="$1"
+      else
+        echo "Unexpected argument '$1'"
+        print_help_and_exit 1
+      fi
+      shift 1
+      ;;
+  esac
+done
+
+
+if [[ -z "$image_name" ]]; then
+  image_name="bluetradingsystems/spark-crypto-dev"
+fi
+
+host_name="$(hostname)"
+if [[ -n "$cont_name" ]]; then
+  host_name="$host_name.$cont_name"
+fi
+
+if [[ -z "$mount_name" ]]; then
+  mount_name="$cont_name"
+fi
+
+if [[ -z "$mount_name" && -z "$cont_name" ]]; then
+  echo "Please specify mount (-m,--mount) and/or container (-c,--container)"
+  print_help_and_exit 1
+fi
+
+if [[ -z "$docker_user" ]]; then
+  docker_user="$USER"
+fi
+
+home_src=$mount_prefix/$mount_name
+home_tgt=$HOME
+repo_src=$(git rev-parse --show-superproject-working-tree)
+repo_tgt=$HOME/$(basename "$repo_src")
+working_dir=$HOME
+
+if [[ $mount_existing -eq 0 && -e "$home_src" ]]; then
+  # Don't overwrite existing dir (might be mounted in another container)
+  echo "Mount dir '$home_src' already exists (use --mount-existing to mount an existing directory)"
+  print_help_and_exit 1
+fi
+
+mkdir -p "$home_src"
+
+if [[ "${#port_mappings[@]}" -eq 0 ]]; then
+  port_mappings=("8000:8000" "8001:8001" "8002:8002" "8003:8003")
+fi
+
+# Adapt port_mappings into args for `docker run` by prepending "-p " to each
+# item
+declare -a docker_port_args=()
+for port_mapping in "${port_mappings[@]}"; do
+  docker_port_args+=("-p $port_mapping")
+done
+
+echo "Starting container '$cont_name'
+image_name  : $image_name
+cont_name   : $cont_name
+home_src    : $home_src
+home_tgt    : $home_tgt
+repo_src    : $repo_src
+repo_tgt    : $repo_tgt
+working dir : $working_dir
+ports       : ${port_mappings[*]}
+"
+
+# DON'T wrap $docker_port_args in quotes. We want bash to split args by
+# whitespace
+docker run \
+  --net bridge \
+  -h "$host_name" \
+  -e "TZ=America/New_York" \
+  --cap-add=SYS_PTRACE \
+  --security-opt seccomp=unconfined \
+  --name "$cont_name" \
+  -v "$home_src:$home_tgt" \
+  -v "$repo_src:$repo_tgt" \
+  -w "$working_dir" \
+  ${docker_port_args[*]} \
+  --shm-size=4024m \
+  -t \
+  -i \
+  -d \
+  -e DISPLAY="$DISPLAY" \
+  "$image_name" \
+  bash
+
+# Add user w/ same uid and gid as the current user outside of docker, so the
+# user inside the docker has file permissions in the mounted dir.
+docker_group="$docker_user"
+docker_user_uid="$(id -u "$USER")"
+docker_user_gid="$(id -g "$USER")"
+
+echo "Adding user in container '$cont_name'
+name : $docker_user
+uid  : $docker_user_uid
+gid  : $docker_user_gid"
+
+if [[ -z "$docker_pass" ]]; then
+  docker_pass="$docker_user"
+  echo "Giving user '$docker_user' a secure password: $docker_pass"
+fi
+
+docker exec -u root -it "$cont_name" groupadd -g "$docker_user_gid" "$docker_group"
+docker exec -u root -it "$cont_name" useradd -u "$docker_user_uid" -g "$docker_user_gid" "$docker_user"
+docker exec -u root -it "$cont_name" usermod --password "$(openssl passwd -1 "$docker_pass")" "$docker_user"
+docker exec -u root -it "$cont_name" usermod -aG sudo "$docker_user"
+docker exec -u root -it "$cont_name" "$repo_tgt/externals/src/emumba/install_emumba-deribit-connector.sh"

src/client/spark/AW2/ViewModel/SettingsEditors/VolSurfaceAdvancedSettingsEditorVM.cs

@@ -14,7 +14,7 @@ namespace mbts.spark {
    class VolSurfaceAdvancedSettingsEditorVM : ViewModelBase {
       readonly VolSurfaceAdvancedSettings _settings;
 
-      public VolSurfaceAdvancedSettingsEditorVM(VolSurfaceAdvancedSettings settings, string vs_name, Action<VolSurfaceAdvancedSettings> commit) {
+      public VolSurfaceAdvancedSettingsEditorVM(VolSurfaceAdvancedSettings settings, string vs_name) {
          _settings = settings;
          _settings.theo_handle_measure = VolSurfaceEnums.GeneralChangeMeasure.Absolute;
          _settings.slide_handle_measure = VolSurfaceEnums.GeneralChangeMeasure.Percentage;
@@ -23,7 +23,7 @@ namespace mbts.spark {
          if (_settings.precomputed_theo_handles == null)
             _settings.precomputed_theo_handles = new List<double>();
          vol_surface_name = vs_name;
-         this.commit = new DelegateCommand(() => request_commit?.Invoke(ProtobufSerializationHelper.clone(_settings)));
+         commit = new DelegateCommand(() => request_commit?.Invoke(ProtobufSerializationHelper.clone(_settings)));
       }
 
       public event Action<VolSurfaceAdvancedSettings> request_commit;

src/client/spark/AW2/ViewModel/VolSurfaceVM.cs

@@ -121,21 +121,24 @@ namespace mbts.spark {
             if (settings == null) {
                return null;
             }
-            // return new VolSurfaceAdvancedSettingsEditor {
-            //    DataContext = new VolSurfaceAdvancedSettingsEditorVM(
-            //       VolSurfaceUtilityFunctions.clone_proto(settings.advanced_settings),
-            //       settings.core_settings.vol_surface_name,
-            //       advanced_settings => {
-            //          var current_settings = _get_volsurface_settings();
-            //          if (current_settings == null) BTSMessageBox.Show("Error changing settings", "VS Settings", System.Windows.MessageBoxButton.OK, System.Windows.MessageBoxImage.Error, true, true, true);
-            //          else {
-            //             current_settings.advanced_settings = advanced_settings;
-            //             _model.set_candidate_settings(current_settings);
-            //          }
-            //       }
-            //    )
-            // };
-            return null;
+            var vm = new VolSurfaceAdvancedSettingsEditorVM(
+               VolSurfaceUtilityFunctions.clone_proto(settings.advanced_settings),
+               settings.core_settings.vol_surface_name);
+            vm.request_commit += commit_advanced_settings;
+            return new VolSurfaceAdvancedSettingsEditor {
+               DataContext = vm
+            };
+
+            async void commit_advanced_settings(VolSurfaceAdvancedSettings value) {
+               var current_settings = await _get_volsurface_settings();
+               if (current_settings == null) {
+                  BTSMessageBox.Show("Error changing settings", "VS Settings");
+               }
+               else {
+                  current_settings.advanced_settings = value;
+                  _model.set_candidate_settings(current_settings);
+               }
+            }
          }
 
          Window? mk_grid_settings_win() {

src/client/spark/Analytics/Core/Settings/VolSurfaceAdvancedSettings.cs

@@ -9,26 +9,34 @@ namespace mbts.analytics
    [ProtoBuf.ProtoContract(Name = "VolSurfaceAdvancedSettings")]
    public class VolSurfaceAdvancedSettings
    {
+      public static VolSurfaceAdvancedSettings Default =>
+         new() {
+            number_of_cashed_saved_days = double.NaN,
+            precomputed_theo_handles = new(0),
+            theo_handle_measure = VolSurfaceEnums.GeneralChangeMeasure.Absolute,
+            precomputed_slide_handles = new(0),
+            slide_handle_measure = VolSurfaceEnums.GeneralChangeMeasure.Percentage,
+            publish_from_client = false,
+            publish_from_client_user = string.Empty,
+            strike_normalization_axis = VolSurfaceEnums.VolCurveStrikeAxis.RawStrike,
+         };
+
       [ProtoBuf.ProtoMember(1)]
-      [System.ComponentModel.DefaultValue(Double.NaN)]
       public double number_of_cashed_saved_days;
       
       [ProtoBuf.ProtoMember(10)]      
       public List<double> precomputed_theo_handles;
       
       [ProtoBuf.ProtoMember(15)]
-      [System.ComponentModel.DefaultValue(VolSurfaceEnums.GeneralChangeMeasure.Absolute)]
       public VolSurfaceEnums.GeneralChangeMeasure theo_handle_measure;
       
       [ProtoBuf.ProtoMember(20)]      
       public List<double> precomputed_slide_handles;
       
       [ProtoBuf.ProtoMember(25)]
-      [System.ComponentModel.DefaultValue(VolSurfaceEnums.GeneralChangeMeasure.Percentage)]
       public VolSurfaceEnums.GeneralChangeMeasure slide_handle_measure;
 
       [ProtoBuf.ProtoMember(30)]
-      //[System.ComponentModel.DefaultValue(false)]
       public bool publish_from_client;
 
       [ProtoBuf.ProtoMember(40)]
@@ -38,5 +46,11 @@ namespace mbts.analytics
       [ProtoBuf.ProtoMember(50)]
       [System.ComponentModel.DefaultValue(VolSurfaceEnums.VolCurveStrikeAxis.RawStrike)]
       public VolSurfaceEnums.VolCurveStrikeAxis strike_normalization_axis;
+      
+      public void ensure_no_nulls() {
+         precomputed_theo_handles ??= new(0);
+         precomputed_slide_handles ??= new(0);
+         publish_from_client_user ??= string.Empty;
+      }
    }
 }

src/client/spark/Analytics/Core/Settings/VolSurfaceSettings.cs

@@ -25,7 +25,8 @@ namespace mbts.analytics {
             volpath_settings = new(),
             safety_limits = new(),
          },
-         polynomial_model_settings = new()
+         polynomial_model_settings = new(),
+         advanced_settings = VolSurfaceAdvancedSettings.Default
       };
 
       [ProtoMember(10)]
@@ -79,6 +80,8 @@ namespace mbts.analytics {
          if (core_settings.time_settings == null) {
             core_settings.time_settings = new VolSurfaceTimeSettings();
          }
+         advanced_settings ??= VolSurfaceAdvancedSettings.Default;
+         advanced_settings.ensure_no_nulls();
          return this;
       }
    }

src/client/spark/Analytics/Pricing/VolSurfaceInterpGreeksProducer.cs

@@ -40,10 +40,8 @@ namespace mbts.analytics {
       /// last computed min/max driver for each exp
       private IDictionary<VolSurfaceExpiration, Tuple<double, double>>
       _computed_driver_range_map = new Dictionary<VolSurfaceExpiration, Tuple<double, double>>();
-      private readonly IEnumerable<double> _default_abs_moves =
-         new double[] { 300, -200, -125, -100, -75, -50, -25, 0, 25, 50, 75, 100, 125, 200, 300 };
+      private readonly IEnumerable<double> _default_abs_moves =  config.Configuration.get<bts.config.VolSurface>().default_abs_moves;
       private readonly double _cushion_percentage = 25.0;
-
       public event
       Action<IDictionary<VolSurfaceExpiration, IEnumerable<Theos>>, double, double> on_theos =
          (o, vt, tt) => { };
@@ -52,7 +50,7 @@ namespace mbts.analytics {
          (o, vt, tt) => { };
 
       public VolSurfaceInterpGreeksProducer(int vs_id) {
-         Log.Gen.info($"VolSurfaceInterpGreeksProducer.cons vs_id: {vs_id}");
+        Log.Gen.info($"VolSurfaceInterpGreeksProducer.cons vs_id: {vs_id}, _default_abs_moves: {string.Join(", ", _default_abs_moves)}");
          _vs_id = vs_id;
       }
 
@@ -72,7 +70,7 @@ namespace mbts.analytics {
       }
 
       public void set_vol_data(VolSurfaceInput vs_input) {
-         Log.Gen.info($"VolSurfaceInterpGreeksProducer.set_vol_data vs_id: {_vs_id}");
+         Log.Gen.info($"VolSurfaceInterpGreeksProducer.set_vol_data vs_id: {_vs_id} precomputed_theo_handles?: {string.Join(", ", (vs_input?.surface_settings?.advanced_settings?.precomputed_theo_handles).empty_if_null())}");
          _vs_input = vs_input != null ? VolSurfaceUtilityFunctions.clone_proto(vs_input) : null;
          _compute_and_fire(InputType.Vol);
       }

src/client/spark/Modules/Analytics/VolSurface/View/VolSurfaceAdvancedSettingsEditor.xaml

@@ -0,0 +1,58 @@
+<Window
+  x:Class="mbts.ui.shell.VolSurfaceAdvancedSettingsEditor"
+  x:ClassModifier="internal"
+  xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation"
+  xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
+  xmlns:c="clr-namespace:mbts.ui.controls"
+  xmlns:cv="clr-namespace:mbts.ui.converters"
+  Title="{Binding vol_surface_name, StringFormat='{}Advanced Settings \&apos;{0}\&apos;'}"
+  Style="{StaticResource BTSDialog}"
+  SizeToContent="WidthAndHeight"
+  TextElement.FontSize="10">
+  <StackPanel>
+    <Grid Margin="5" Grid.IsSharedSizeScope="True">
+      <Grid.RowDefinitions>
+        <RowDefinition Height="Auto"/>
+        <RowDefinition Height="Auto"/>
+        <RowDefinition Height="Auto"/>
+        <RowDefinition Height="Auto"/>
+        <RowDefinition Height="Auto"/>
+      </Grid.RowDefinitions>
+      <Grid.ColumnDefinitions>
+        <ColumnDefinition Width="Auto" SharedSizeGroup="label"/>
+        <ColumnDefinition Width="*"/>
+      </Grid.ColumnDefinitions>
+      <GroupBox Grid.ColumnSpan="2" Padding="3">
+        <GroupBox.Header>
+          <TextBlock Text="Theos" FontWeight="Bold"/>
+        </GroupBox.Header>
+        <Grid>
+          <Grid.RowDefinitions>
+            <RowDefinition Height="Auto"/>
+            <RowDefinition Height="Auto"/>
+          </Grid.RowDefinitions>
+          <Grid.ColumnDefinitions>
+            <ColumnDefinition Width="Auto" SharedSizeGroup="label"/>
+            <ColumnDefinition Width="Auto"/>
+          </Grid.ColumnDefinitions>
+          <c:RadioButtonsControl Grid.ColumnSpan="2" Margin="0,0,0,5" ItemsSource="{Binding general_change_measures}" SelectedItem="{Binding theo_handle_measure}" IsEnabled="False"/>
+          <TextBlock Grid.Row="1" Text="Precomputed Handles" VerticalAlignment="Center"/>
+          <TextBox
+            x:Name="PART_Handles"
+            Validation.ErrorTemplate="{x:Null}"
+            Grid.Row="1"
+            Grid.Column="1"
+            Width="160"
+            Margin="5,0,0,0"
+            Text="{Binding precomputed_theo_handles, Converter={x:Static cv:EnumerableDoubleStringConverter.Instance}}" />
+        </Grid>
+      </GroupBox>
+      <TextBlock Visibility="Collapsed" Grid.Row="1" VerticalAlignment="Center" Margin="5,0,0,0" Text="Num Cash Saved Days" />
+      <c:TickUpDownDouble Visibility="Collapsed" Grid.Row="1" Grid.Column="1" HorizontalAlignment="Stretch" Margin="14,5,5,5" Value="{Binding number_of_cash_saved_days}"/>
+      <TextBlock Grid.Row="2" VerticalAlignment="Center" Margin="5,0,0,0" Text="Strike Normalization Axis"/>
+      <ComboBox Grid.Row="2" Grid.Column="1" HorizontalAlignment="Stretch" Margin="14,5,5,5" ItemsSource="{Binding strike_normalization_axes}" SelectedValue="{Binding selected_strike_normalization_axis}"/>
+      <CheckBox Grid.Row="3" Margin="7,0,0,0" Content="Publish From Client" IsChecked="{Binding publish_from_client}"/>
+    </Grid>
+    <c:DialogSkirtOKCancel OKCommand="{Binding commit}" />
+  </StackPanel>
+</Window>

src/client/spark/Modules/Analytics/VolSurface/View/VolSurfaceAdvancedSettingsEditor.xaml.cs

@@ -0,0 +1,11 @@
+namespace mbts.ui.shell {
+   using System.Windows;
+
+   /// <summary>
+   /// Interaction logic for VolSurfaceAdvancedSettingsEditor.xaml
+   /// </summary>
+   partial class VolSurfaceAdvancedSettingsEditor : Window {
+      public VolSurfaceAdvancedSettingsEditor() =>
+         InitializeComponent();
+   }
+}

src/client/spark/Modules/Trading/Ladder/View/LadderGridHost.cs

@@ -87,7 +87,7 @@ namespace mbts.spark {
          length = _initial_row_count,
       };
       LadderPriceList _pending_price_list = null;
-      IComparer<double> _price_comparer;
+      IComparer<double> _price_comparer = MathUtils.epsilon_comparer.reverse();
       double _last_origin_price;
       readonly MarketBreakLines _mkt_break_lines;
       NativePriceSize _last_trade;

src/client/spark/view/Spark/Utilities/BBOVM.cs

@@ -14,6 +14,9 @@ namespace mbts.spark {
       uint _bid_size, _ask_size;
       ulong _time;
 
+      public BBOVM() =>
+         _dispose = Disposable.Create(() => Disposable.dispose(ref _md_poll));
+
       public SecurityImmutable? security => _security_md?.security;
       public SecurityMD? security_md {
          get => _security_md;

src/client/spark_api/data/ticks/TickExtensions.cs

@@ -63,7 +63,11 @@ namespace mbts {
          if (ticks_per_base.is_int()) {
             return true;
          }
-         return tick.tick_unit > tick.tick_base && tick.tick_base == 1;
+         // The Log10 check accounts for [non-perp] futures at Deribit, which use TickFactors 25/10
+         // (e.g., 10.0, 12.5, 15.0, etc.). It was added 11/02/2023, when it replaced the
+         // commented out line below.
+         // return tick.tick_unit > tick.tick_base && tick.tick_base == 1;
+         return tick.tick_unit > tick.tick_base && Math.Log10(tick.tick_base).is_int();
       }
 
       public static bool is_valid(this RegionalTickFactors regional_tick) =>
@@ -667,7 +671,8 @@ namespace mbts {
          if (negate) {
             value = -value;
          }
-         var text = core_impl();
+         var text = tick.tick_unit == 25 && tick.tick_base == 10 ?
+            deribit_future_impl() : core_impl();
          if (!settings.show_zero_integer) {
             // Hide the integer if its zero.  This could be better optimized,
             // but speed shouldn't be an issue, so it doesn't seem worth the time/effort.
@@ -721,6 +726,37 @@ namespace mbts {
                return "Overflow";
             }
          }
+
+         string deribit_future_impl() {
+            try {
+               // TODO: Replace this hack with a suitable long term solution
+               //
+               // The hack: [non-perp] futures at Deribit tick in units of '2.5' (e.g., 10.0, 12.5,
+               // 15.0, etc.). However, this battle-tested impl is predicated on the tick base being
+               // a multiple of the tick unit. So, to avoid changing key battle-tested logic, let's
+               // do the following:
+               //    1) Round the value to the nearest multiple of '2.5' (the tick size)
+               //    2) Format the rounded value with normal tick factors (unit 5, base 10),
+               //       (ticks of the sort this method was designed to expect).
+               var tuf = tick.tick_unit_frac<T>();
+               var total_ticks = value / tuf;
+               long total_ticks_l = NumericCastConverter<T, long>.convert(value / tuf);
+               var rounded_total_ticks = NumericCastConverter<long, T>.convert(total_ticks_l);
+               var rem = value - (rounded_total_ticks * tuf);
+               var unit_rem = rem / tuf;
+               Log.assert(unit_rem <= T.One && unit_rem >= T.Zero);
+               // Check if unit_rem closer to T.One, or T.Zero. Round accordingly
+               if (T.One - unit_rem <= unit_rem - T.Zero) {
+                  // round up
+                  rounded_total_ticks++;
+               }
+               var rounded_val = tuf * rounded_total_ticks;
+               return _format_tick(new TickFactors(10, 5), rounded_val, settings);
+            } catch (OverflowException ex) {
+               Log.Ui.warn($"Arithmetic overflow exception in format_tick (tick:{tick}, value:{value}):{ex}");
+               return "Overflow";
+            }
+         }
       }
 
       static T _round_tick<T>(T tick_unit_frac, T input) where T : IFloatingPointIeee754<T> =>

src/client/spark_api/data/ticks/TickFactorsFactory.cs

@@ -134,8 +134,20 @@ namespace bts.data.ticks {
       static TickFactors _inflate_to_base(TickFactors tick_factors, double new_base) =>
          new TickFactors(new_base, tick_factors.tick_unit * new_base / tick_factors.tick_base);
 
-      static bool _check_tick_size(double tick_size) =>
-         tick_size > 0 && (tick_size <= 0.5 || MathUtils.is_int(tick_size));
+      static bool _check_tick_size(double tick_size) {
+         if (!tick_size.is_real_positive()) {
+            return false;
+         }
+         if (tick_size <= 0.5 || tick_size.is_int()) {
+            return true;
+         }
+         // Check if tick size represents quarters (i.e., `tick_size=0.25 * 10^c` for some int c).
+         // Note: this check is designed specifically to support (non-perp) futures at Deribit
+         if (Math.Log10(tick_size / 0.25).is_int()) {
+            return true;
+         }
+         return false;
+      }
 
       struct SpecKey : IEquatable<SpecKey> {
          public readonly uint denominator, display_digits;

src/client/spark_api/data/ticks/TickUtils.cs

@@ -209,6 +209,17 @@ namespace mbts.spark {
       }
 
       public static decimal round_to_tick(decimal tick_size, decimal value) {
+         if (tick_size == (decimal)2.5) {
+            // This is a special case for [non-perp] futures at Deribit, which tick in 25/10
+            // (unit/base). While this code looks harmless enough, and should work for all ticks in
+            // theory, let's be extra safe and only apply the change to a specific case (when
+            // tick_size == 2.5).
+            // Note: Making the ticks code work for such a wide range of tick factors was no easy
+            // task, and involved overcoming a plethora of obscure rounding errors along the way.
+            var n_ticks = value / tick_size;
+            var simple_round = tick_size * n_ticks;
+            return simple_round;
+         }
          decimal intpart = (long)value;
          decimal floatpart = value % 1;
          floatpart = ((int)(floatpart / tick_size)) * tick_size;
@@ -293,6 +304,11 @@ namespace mbts.spark {
          if (!tf.low_tick.Equals(tf.high_tick) || !tf.is_valid() || tf.low_tick.is_integer())
             return null;
          var ticks_per_dollar = (uint)(tf.low_tick.tick_base / tf.low_tick.tick_unit);
+         // Note: this check was introduced specifically to account for [non-perp] Futures at
+         // Deribit, which use TickFactors 25/10 (e.g., 10.0, 12.5, 15.0, etc.).
+         if (ticks_per_dollar == 0) {
+            return null;
+         }
          // if ticks_per_row is greater than ticks_per_dollar, round to a multiple of ticks_per_dollar
          if (ticks_per_row > ticks_per_dollar) {
             return (uint)(ticks_per_dollar * Math.Floor((double)(ticks_per_row / ticks_per_dollar)));

src/client/spark_core/Collections/Extensions/EnumerableExtensions.cs

@@ -1190,7 +1190,6 @@ namespace System.Collections.Generic {
       /// A tuple whose items each represent the sum of their repsective items in
       /// <paramref name="source"/>
       /// </returns>
-      [RequiresPreviewFeatures]
       public static (T1, T2) Sum<T1, T2>(this IEnumerable<(T1, T2)> source)
          where T1 : INumber<T1>
          where T2 : INumber<T2> =>
@@ -1203,7 +1202,6 @@ namespace System.Collections.Generic {
       /// A tuple whose items each represent the sum of their repsective items in
       /// <paramref name="source"/>
       /// </returns>
-      [RequiresPreviewFeatures]
       public static (T1, T2) Sum<T1, T2>(this IEnumerable<(T1, T2)> source, T1 seed1, T2 seed2)
          where T1 : IAdditionOperators<T1, T1, T1>
          where T2 : IAdditionOperators<T2, T2, T2> =>
@@ -1217,7 +1215,6 @@ namespace System.Collections.Generic {
       /// A tuple whose items each represent the sum of their repsective items in
       /// <paramref name="source"/>
       /// </returns>
-      [RequiresPreviewFeatures]
       public static (T1, T2, T3) Sum<T1, T2, T3>(this IEnumerable<(T1, T2, T3)> source)
          where T1 : INumber<T1>
          where T2 : INumber<T2>
@@ -1231,7 +1228,6 @@ namespace System.Collections.Generic {
       /// A tuple whose items each represent the sum of their repsective items in
       /// <paramref name="source"/>
       /// </returns>
-      [RequiresPreviewFeatures]
       public static (T1, T2, T3) Sum<T1, T2, T3>(
          this IEnumerable<(T1, T2, T3)> source, T1 seed1, T2 seed2, T3 seed3)
          where T1 : IAdditionOperators<T1, T1, T1>
@@ -1248,7 +1244,6 @@ namespace System.Collections.Generic {
       /// A tuple whose items each represent the sum of their repsective items in
       /// <paramref name="source"/>
       /// </returns>
-      [RequiresPreviewFeatures]
       public static (T1, T2, T3, T4) Sum<T1, T2, T3, T4>(this IEnumerable<(T1, T2, T3, T4)> source)
          where T1 : INumber<T1>
          where T2 : INumber<T2>
@@ -1263,7 +1258,6 @@ namespace System.Collections.Generic {
       /// A tuple whose items each represent the sum of their repsective items in
       /// <paramref name="source"/>
       /// </returns>
-      [RequiresPreviewFeatures]
       public static (T1, T2, T3, T4) Sum<T1, T2, T3, T4>(
          this IEnumerable<(T1, T2, T3, T4)> source, T1 seed1, T2 seed2, T3 seed3, T4 seed4)
          where T1 : IAdditionOperators<T1, T1, T1>

src/data_files/CMakeDeps.cmake

@@ -1 +0,0 @@
-set(sources files/audit_ilink3.py;files/audit_upload.sh;files/audit_uploader.py;files/cert_secure_logon.txt;files/example.conf;files/futures.coconut.xz;files/onixs_ice/ConnectivityConfiguration.APITest.xml;files/onixs_ice/ConnectivityConfiguration.PerfTest.xml;files/onixs_ice/ConnectivityConfiguration.Production.xml;files/perf_trace/compiled.css;files/perf_trace/index.html;files/perf_trace/jquery-3.3.1.min.js;files/perf_trace/main.js;files/perf_trace/perf_log.sql;files/perf_trace/perf_trace_analysis.sql;files/perf_trace/plotly-latest.min.js;files/small_es.pcap.xz;files/spartan/create_polygon_books.sql;files/spartan/log.sql;files/spartan/md.sql;files/spartan/sim_db.sql)

src/lib/CMakeDeps.cmake

@@ -1 +0,0 @@
-set(sources api/analytics/ExpirationDriver.proto;api/analytics/TimeWeightedInterpolatedDriver.proto;api/apicore.proto;api/apigateway.proto;api/apioptions.proto;api/core/ApiClientInterop.cpp;api/core/ApiServerInterop.cpp;api/domain/Currency.proto;api/domain/SecurityCriteria.proto;api/domain/SecurityCriteriaClause.proto;api/domain/SecurityExpiration.proto;api/domain/SecurityMetadata.proto;api/domain/SecurityMetadataCollection.proto;api/domain/SecurityMetadataItem.proto;api/domain/SecurityType.proto;api/domain/TickDisplaySettings.proto;api/domain/TickFactors.proto;api/domain/Venue.proto;api/io/URL.proto;api/md/MarketDataSubscriptionParams.proto;api/md/binance/BinanceDepthStreamParams.proto;api/md/binance/BinanceMarketDataSubscriptionParams.proto;api/md/deribit/DeribitMarketDataSubscriptionParams.proto;api/md/ftx/FTXDepthStreamParams.proto;api/md/ftx/FTXMarketDataSubscriptionParams.proto;api/md/glbx/GlbxMarketDataSubscriptionParams.proto;api/md/huobi/HuobiMarketDataSubscriptionParams.proto;api/public/ApiServer.cpp;api/public/ClientChannel.cpp;api/public/QuickbusClientChannel.cpp;api/public/ServerChannel.cpp;api/public/ServiceContext.cpp;api/service_helpers/AdminServiceUtils.cpp;api/service_helpers/AlgoServiceUtils.cpp;api/service_helpers/PositionUtilities.cpp;api/services/AdminService.proto;api/services/AlgoDefinitionService.proto;api/services/AlgoService.proto;api/services/AnalyticsService.proto;api/services/ApiUnitTestService.proto;api/services/AuthService.proto;api/services/BasePositionService.proto;api/services/ErrorReportService.proto;api/services/HelloService.proto;api/services/MBOService.proto;api/services/MarginService.proto;api/services/MarketDataAuxiliaryService.proto;api/services/MarketDataRelay.proto;api/services/MarketDataService.proto;api/services/MoneyFileService.proto;api/services/OrderService.proto;api/services/OutOfProcessURLListener.proto;api/services/PersistenceService.proto;api/services/PositionService.proto;api/services/ShellDiagnosticService.proto;api/services/ShellLatencyService.proto;api/services/SyntheticSecurityService.proto;api/test/ApiTests.cpp;api/test/HelloServiceServer.cpp;communincation/websockets/tests/WebsocketSyncClientTests.cpp;config/ConfigCheck.cpp;config/ConfigDownloader.cpp;config/ConfigErrorCollector.cpp;config/ConfigLog.cpp;config/ConfigPropertyEvaluator.cpp;config/ConfigValueQueue.cpp;config/ConfigurationException.cpp;config/ConfigurationState.cpp;config/PreprocessInputStream.cpp;config/config.cpp;config/defaults/MarketData.conf;config/defaults/Ports.conf;config/defaults/SecurityMetadataCollection.conf;config/defaults/Symbols.conf;config/defaults/cme_reference_data.conf;config/defaults/edge.conf;config/defaults/jetfix.conf;config/defaults/keycode.conf;config/defaults/logging.conf;config/defaults/orderman_cfg.conf;config/defaults/orders_fills_cleanup.conf;config/defaults/position_cfg.conf;config/defaults/span.conf;config/defaults/spreader.conf;config/defaults/testkeys.conf;config/defaults/threads.conf;config/defaults/toggles.conf;config/extensions/extensions.proto;config/invalid/invalid.proto;config/protos/algo_types.proto;config/protos/cme_reference_data.proto;config/protos/exchange_sim_cfg.proto;config/protos/jetfix.proto;config/protos/keycode.proto;config/protos/logging.proto;config/protos/market_data.proto;config/protos/mercury_cfg.proto;config/protos/orderman_cfg.proto;config/protos/orders_fills_cleanup.proto;config/protos/perflog_cfg.proto;config/protos/ports.proto;config/protos/position_cfg.proto;config/protos/quickbus_cfg.proto;config/protos/span.proto;config/protos/spartan.proto;config/protos/spreader.proto;config/protos/subprocess.proto;config/protos/symbols.proto;config/protos/system.proto;config/protos/testkeys.proto;config/protos/threads.proto;config/protos/toggles.proto;config/protos/ui.proto;config/tests/HTTPServer.cpp;config/tests/config_ref_tests.cpp;config/tests/config_tests.cpp;config/text_format/text_format.cpp;domain/AggressorSide.cpp;domain/NewOrder.cpp;domain/OrderDefinition.cpp;domain/OrderStatus.cpp;domain/OrderUpdate.cpp;domain/Side.cpp;domain/TimeInForce.cpp;domain_model/Security.cpp;domain_model/Security.proto;domain_model/SecurityTests.cpp;domain_model/sod.cpp;domain_model/tests/domain_tests.cpp;expression/BooleanGrammar.cpp;expression/ExpressionEvaluator.cpp;expression/ParserExceptions.cpp;expression/Value.cpp;expression/tests/expression_parser_tests.cpp;io/QueryParams.cpp;io/URL.cpp;io/http/CurlHandle.cpp;io/http/HTTPClient.cpp;io/http/HTTPHeaders.cpp;io/http/HTTPRequest.cpp;io/http/HTTPResponse.cpp;io/websocket/WebsocketClient.cpp;io/websocket/WebsocketClientConfiguration.cpp;io/websocket/WebsocketError.cpp;io/websocket/WebsocketMessage.cpp;io/websocket/WebsocketSSLClientInternal.cpp;io/websocket/tests/WebsocketClientTests.cpp;jetfix/codec/FIXArray.cpp;jetfix/common/OutFIX.cpp;jetfix/connection/AsyncReader.cpp;jetfix/connection/IConnection.cpp;jetfix/connection/SocketConnection.cpp;jetfix/gateway/CoreGateway.cpp;jetfix/gateway/JetFIXGatLine truncated

src/lib/CMakeDeps_win.cmake

@@ -1 +0,0 @@
-set(sources api/analytics/ExpirationDriver.proto;api/analytics/TimeWeightedInterpolatedDriver.proto;api/apicore.proto;api/apigateway.proto;api/apioptions.proto;api/core/ApiClientInterop.cpp;api/core/ApiServerInterop.cpp;api/domain/Currency.proto;api/domain/SecurityCriteria.proto;api/domain/SecurityCriteriaClause.proto;api/domain/SecurityExpiration.proto;api/domain/SecurityMetadata.proto;api/domain/SecurityMetadataCollection.proto;api/domain/SecurityMetadataItem.proto;api/domain/SecurityType.proto;api/domain/TickDisplayRule.proto;api/domain/TickDisplaySettings.proto;api/domain/TickFactors.proto;api/domain/Venue.proto;api/io/URL.proto;api/md/MarketDataSubscriptionParams.proto;api/md/binance/BinanceDepthStreamParams.proto;api/md/binance/BinanceMarketDataSubscriptionParams.proto;api/md/deribit/DeribitMarketDataSubscriptionParams.proto;api/md/ftx/FTXDepthStreamParams.proto;api/md/ftx/FTXMarketDataSubscriptionParams.proto;api/md/glbx/GlbxMarketDataSubscriptionParams.proto;api/md/huobi/HuobiMarketDataSubscriptionParams.proto;api/public/ApiServer.cpp;api/public/ClientChannel.cpp;api/public/QuickbusClientChannel.cpp;api/public/ServerChannel.cpp;api/public/ServiceContext.cpp;api/service_helpers/AdminServiceUtils.cpp;api/service_helpers/AlgoServiceUtils.cpp;api/service_helpers/PositionUtilities.cpp;api/services/AdminService.proto;api/services/AlgoDefinitionService.proto;api/services/AlgoService.proto;api/services/AnalyticsService.proto;api/services/ApiUnitTestService.proto;api/services/AuthService.proto;api/services/BasePositionService.proto;api/services/ErrorReportService.proto;api/services/HelloService.proto;api/services/MBOService.proto;api/services/MarginService.proto;api/services/MarketDataAuxiliaryService.proto;api/services/MarketDataRelay.proto;api/services/MarketDataService.proto;api/services/MoneyFileService.proto;api/services/OrderService.proto;api/services/OutOfProcessURLListener.proto;api/services/PersistenceService.proto;api/services/PositionService.proto;api/services/ShellDiagnosticService.proto;api/services/ShellLatencyService.proto;api/services/SyntheticSecurityService.proto;api/test/ApiTests.cpp;api/test/HelloServiceServer.cpp;config/ConfigCheck.cpp;config/ConfigDownloader.cpp;config/ConfigErrorCollector.cpp;config/ConfigLog.cpp;config/ConfigPropertyEvaluator.cpp;config/ConfigValueQueue.cpp;config/ConfigurationException.cpp;config/ConfigurationState.cpp;config/PreprocessInputStream.cpp;config/config.cpp;config/defaults/MarketData.conf;config/defaults/Ports.conf;config/defaults/SecurityMetadataCollection.conf;config/defaults/Symbols.conf;config/defaults/cme_reference_data.conf;config/defaults/edge.conf;config/defaults/jetfix.conf;config/defaults/keycode.conf;config/defaults/logging.conf;config/defaults/orderman_cfg.conf;config/defaults/orders_fills_cleanup.conf;config/defaults/position_cfg.conf;config/defaults/span.conf;config/defaults/spreader.conf;config/defaults/testkeys.conf;config/defaults/threads.conf;config/defaults/toggles.conf;config/extensions/extensions.proto;config/invalid/invalid.proto;config/protos/algo_types.proto;config/protos/cme_reference_data.proto;config/protos/exchange_sim_cfg.proto;config/protos/jetfix.proto;config/protos/keycode.proto;config/protos/logging.proto;config/protos/market_data.proto;config/protos/mercury_cfg.proto;config/protos/orderman_cfg.proto;config/protos/orders_fills_cleanup.proto;config/protos/perflog_cfg.proto;config/protos/ports.proto;config/protos/position_cfg.proto;config/protos/quickbus_cfg.proto;config/protos/span.proto;config/protos/spartan.proto;config/protos/spreader.proto;config/protos/subprocess.proto;config/protos/symbols.proto;config/protos/system.proto;config/protos/testkeys.proto;config/protos/threads.proto;config/protos/toggles.proto;config/protos/ui.proto;config/tests/HTTPServer.cpp;config/tests/config_ref_tests.cpp;config/tests/config_tests.cpp;config/text_format/text_format.cpp;domain_model/Security.cpp;domain_model/Security.proto;domain_model/SecurityTests.cpp;domain_model/sod.cpp;domain_model/tests/domain_tests.cpp;expression/BooleanGrammar.cpp;expression/ExpressionEvaluator.cpp;expression/ParserExceptions.cpp;expression/Value.cpp;expression/tests/expression_parser_tests.cpp;io/QueryParams.cpp;io/URL.cpp;io/http/CurlHandle.cpp;io/http/HTTPClient.cpp;io/http/HTTPHeaders.cpp;io/http/HTTPRequest.cpp;io/http/HTTPResponse.cpp;io/websocket/WebsocketClient.cpp;io/websocket/WebsocketClientConfiguration.cpp;io/websocket/WebsocketError.cpp;io/websocket/WebsocketMessage.cpp;io/websocket/WebsocketSSLClientInternal.cpp;io/websocket/tests/WebsocketClientTests.cpp;logging/LoggerHub.cpp;logging/Sink.cpp;logging/logging.cpp;logging/tests/logging_tests.cpp;md/CCApiDepthWalker.cpp;md/CCApiListenerAdaptor.cpp;md/CCApiRequestor.cpp;md/Diag/MDPeriodicDumper.cpp;md/Diag/ShmDumper.cpp;md/MBOProcessor.cpp;md/MBOServiceServer.cpp;md/MDClient.cpp;md/MDConsumer.cpp;md/MDConsumerHandle.cpp;md/MDConsumerInterop.cpp;md/MDProducer.cpp;md/MDProducerHandle.cpp;md/MarketDataLine truncated

src/lib/CMakeLists.txt

@@ -30,6 +30,7 @@ if(WIN32)
     RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
     api/*.cpp
     config/*.cpp
+    core/*.cpp
     domain_model/*.cpp
     expression/*.cpp
     logging/*.cpp
@@ -108,13 +109,11 @@ target_include_directories(
    ${CMAKE_BINARY_DIR}/src)
 
 if(WIN32)
+  find_package(fmt CONFIG REQUIRED)
   target_link_libraries(
     libspark
     libcurl
-    debug
-    fmtd
-    optimized
-    fmt
+    fmt::fmt-header-only
     libtbb
     libboost_system
     libboost_thread

src/lib/algorithm/PlusX.h

@@ -0,0 +1,13 @@
+#pragma once
+
+#include <functional>
+
+namespace bts::algorithm {
+
+template <typename _Tp, typename _TAddend, _Tp _Addend>
+struct PlusX {
+   /// Returns the sum
+   _Tp operator()(const _Tp &__x) const { return __x + _Addend; }
+};
+
+} // namespace bts::algorithm

src/lib/algorithm/StringCaseConverter.h

@@ -0,0 +1,49 @@
+#pragma once
+#include <string>
+
+namespace bts::algorithm {
+
+class StringCaseConverter {
+ public:
+   static void to_upper(const std::string &str, std::string &dest) noexcept;
+   static std::string to_upper(const std::string &str) noexcept;
+
+   static void to_lower(const std::string &str, std::string &dest) noexcept;
+   static std::string to_lower(const std::string &str) noexcept;
+};
+
+//////////////////////////////////////////////////
+// Definition
+//////////////////////////////////////////////////
+
+inline std::string StringCaseConverter::to_upper(const std::string &str) noexcept {
+   std::string rv{str};
+   for (auto &c : rv) {
+      c = std::toupper(c);
+   }
+   return rv;
+}
+
+inline void StringCaseConverter::to_upper(const std::string &str, std::string &dest) noexcept {
+   dest = str;
+   for (auto &c : dest) {
+      c = std::toupper(c);
+   }
+}
+
+inline std::string StringCaseConverter::to_lower(const std::string &str) noexcept {
+   std::string rv{str};
+   for (auto &c : rv) {
+      c = std::tolower(c);
+   }
+   return rv;
+}
+
+inline void StringCaseConverter::to_lower(const std::string &str, std::string &dest) noexcept {
+   dest = str;
+   for (auto &c : dest) {
+      c = std::tolower(c);
+   }
+}
+
+} // namespace bts::algorithm

src/lib/algorithm/TryCatch.h

@@ -0,0 +1,28 @@
+#pragma once
+
+#define CATCH_AND_LOG(subsys, level, label)                                    \
+   catch (const std::exception &ex) {                                          \
+      LOG(subsys, level, label ": {}", std::string{ex.what()});                \
+   }                                                                           \
+   catch (...) {                                                               \
+      LOG(subsys, level, label ": Unknown exception");                         \
+   }
+
+#define CATCH_AND_HANDLE(err_var_name, impl)                                   \
+   catch (const std::exception &err_var_name##_ex) {                           \
+      const auto &err_var_name = err_var_name##_ex.what();                     \
+      impl;                                                                    \
+   }                                                                           \
+   catch (...) {                                                               \
+      const std::string err_var_name{"Unknown exception"};                     \
+      impl;                                                                    \
+   }
+
+// Normalizes 'std::exception' and '...' handlers into single callback
+#define CATCH_AND_HANDLE_MSG(impl)                                             \
+   catch (const std::exception &ex) {                                          \
+      impl(std::string{ex.what()});                                            \
+   }                                                                           \
+   catch (...) {                                                               \
+      impl(std::string{"Unknown exception"});                                  \
+   }

src/lib/api/services/OrderService.h

@@ -0,0 +1,31 @@
+#pragma once
+
+#include "lib/api/services/OrderService.pb.h"
+#include "lib/core/EnumMacros.h"
+#include "lib/core/FMT_IMPL.h"
+#include "lib/io/JsonReader.h"
+#include "lib/logging/logging.h"
+
+#include <nlohmann/json.hpp>
+
+namespace bts::api::orderman {
+
+PB_ENUM_FROM_JSON(OrderStatus)
+PB_ENUM_TO_JSON(OrderStatus)
+
+PB_ENUM_FROM_JSON(TriggerPriceType)
+PB_ENUM_TO_JSON(TriggerPriceType)
+
+} // namespace bts::api::orderman
+
+FMT_IMPL(
+    bts::api::orderman::OrderStatus,
+    s,
+    "{}",
+    bts::api::orderman::OrderStatus_Name(s));
+
+FMT_IMPL(
+    bts::api::orderman::TriggerPriceType,
+    t,
+    "{}",
+    bts::api::orderman::TriggerPriceType_Name(t));

src/lib/config/ConfigValueQueue.cpp

@@ -141,7 +141,7 @@ void ConfigValueQueue::assign(
       if (field->is_repeated()) {
          for (vector<Value>::iterator i = v.begin(); i != v.end(); ++i) {
             int size = reflection->FieldSize(*message, field);
-            add<TProto>(
+            add(
                 message, reflection, field, (TProto)boost::get<TValue>(*i));
             if (index != size) {
                for (int i = index; i < size; ++i) {
@@ -182,7 +182,7 @@ void ConfigValueQueue::lexical_assign(
       for (vector<Value>::iterator i = v.begin(); i != v.end(); ++i) {
          int size = reflection->FieldSize(*message, field);
          TProto result = boost::lexical_cast<TProto>(boost::get<TValue>(*i));
-         add<TProto>(message, reflection, field, result);
+         add(message, reflection, field, result);
          if (index != size) {
             for (int i = index; i < size; ++i) {
                reflection->SwapElements(message, field, i, size);

src/lib/config/defaults/vol_surface_config.conf

@@ -0,0 +1,22 @@
+VolSurface {
+    value {
+        default_abs_moves: -600
+        default_abs_moves: -300
+        default_abs_moves: -200
+        default_abs_moves: -125
+        default_abs_moves: -100 
+        default_abs_moves: -75
+        default_abs_moves: -50
+        default_abs_moves: -25 
+        default_abs_moves: 0
+        default_abs_moves: 25 
+        default_abs_moves: 50 
+        default_abs_moves: 75 
+        default_abs_moves: 100
+        default_abs_moves: 125
+        default_abs_moves: 200
+        default_abs_moves: 300
+        default_abs_moves: 600
+
+    }
+}

src/lib/config/protos/vol_surface_config.proto

@@ -0,0 +1,6 @@
+syntax = 'proto2';
+package bts.config;
+
+message VolSurface {
+    repeated double default_abs_moves = 1;
+}

src/lib/core/EnumBitwiseOperators.h

@@ -1,15 +0,0 @@
-#pragma once
-
-#define BITWISE_OP(enum_type, numeric_type, op)                                \
-   enum_type operator op(enum_type lhs, enum_type rhs) noexcept {              \
-      return static_cast<enum_type>(static_cast<numeric_type>(lhs)             \
-                                        op static_cast<numeric_type>(rhs));    \
-   }
-
-#define BITWISE_OPS(enum_type, numeric_type)                                   \
-   BITWISE_OP(enum_type, numeric_type, |)                                      \
-   BITWISE_OP(enum_type, numeric_type, &)                                      \
-   BITWISE_OP(enum_type, numeric_type, ^)
-
-#define HAS_FLAG(enum_value, flag) (enum_value & flag) == flag
-#define STRIP_FLAG(enum_value, flag) (enum_value | flag) ^ flag

src/lib/core/EnumMacros.h

@@ -0,0 +1,154 @@
+#pragma once
+
+#define ENUM_BASIC_MEMBERS(type)                                               \
+   Value value;                                                                \
+                                                                               \
+   type() = default;                                                           \
+   /*NOLINTNEXTLINE*/                                                          \
+   constexpr type(Value value) noexcept : value(value) {}                      \
+                                                                               \
+   type(const type &) noexcept = default;                                      \
+   type(type &&) noexcept = default;                                           \
+   type &operator=(const type &) noexcept = default;                           \
+   type &operator=(type &&) noexcept = default;                                \
+                                                                               \
+   ~type() = default;                                                          \
+                                                                               \
+   static constexpr bool parse(const char *name, type *value) noexcept;        \
+   static bool parse(const std::string &name, type *value) noexcept;           \
+                                                                               \
+   [[nodiscard]] constexpr const char *name() const noexcept;
+
+#define ENUM_VALUE_TO_NAME_CASE(type, v)                                       \
+   case type::v:                                                               \
+      return #v;
+
+#define ENUM_NAME_TO_VALUE_CLAUSE(type, v)                                     \
+   if (name == #v) {                                                           \
+      *value = type::v;                                                        \
+      return true;                                                             \
+   }
+
+#define ENUM_NAMEC_TO_VALUE_CLAUSE(type, v)                                    \
+   if (std::string_view(name) == #v) {                                         \
+      *value = type::v;                                                        \
+      return true;                                                             \
+   }
+
+// clang-format off
+#define ENUM_NAME_DEF(type, for_all)                                           \
+   constexpr const char *type::name() const noexcept {                         \
+      switch (value) {                                                         \
+         for_all(ENUM_VALUE_TO_NAME_CASE)                                      \
+         default: return "ENUM_UNKNOWN";                                       \
+      }                                                                        \
+   }
+// clang-format on
+
+#define ENUM_PARSE_DEF(type, for_all)                                          \
+   constexpr bool type::parse(const char *name, type *value) noexcept {        \
+      for_all(ENUM_NAMEC_TO_VALUE_CLAUSE) return false;                        \
+   }                                                                           \
+   inline bool type::parse(const std::string &name, type *value) noexcept {    \
+      for_all(ENUM_NAME_TO_VALUE_CLAUSE) return false;                         \
+   }
+
+#define ENUM_EQUALITY_OPS(type)                                                \
+   constexpr bool operator==(type lhs, type rhs) noexcept {                    \
+      return lhs.value == rhs.value;                                           \
+   }                                                                           \
+   constexpr bool operator!=(type lhs, type rhs) noexcept {                    \
+      return lhs.value != rhs.value;                                           \
+   }                                                                           \
+   constexpr bool operator==(type lhs, type::Value rhs) noexcept {             \
+      return lhs.value == rhs;                                                 \
+   }                                                                           \
+   constexpr bool operator!=(type lhs, type::Value rhs) noexcept {             \
+      return lhs.value != rhs;                                                 \
+   }
+
+#define ENUM_OUTPUT_OPS(type)                                                  \
+   inline std::ostream &operator<<(std::ostream &out, type value) noexcept {   \
+      out << value.name();                                                     \
+      return out;                                                              \
+   }                                                                           \
+                                                                               \
+   inline std::ostream &operator<<(                                            \
+       std::ostream &out, type::Value value) noexcept {                        \
+      out << type{value}.name();                                               \
+      return out;                                                              \
+   }
+
+#define ENUM_FROM_JSON(type)                                                   \
+   inline void from_json(const nlohmann::json &j, type &value) {               \
+      std::string value_str;                                                   \
+      j.get_to(value_str);                                                     \
+      type::parse(value_str, &value);                                          \
+   }
+
+#define ENUM_TO_JSON(type)                                                     \
+   inline void to_json(nlohmann::json &j, const type &value) {                 \
+      j = value.name();                                                        \
+   }
+
+#define ENUM_DECLARE_VALUE(t, x) x,
+
+#define ENUM_CLASS_DEF(type, for_all)                                          \
+   class type {                                                                \
+    public:                                                                    \
+      enum Value { for_all(ENUM_DECLARE_VALUE) };                              \
+                                                                               \
+      ENUM_BASIC_MEMBERS(type)                                                 \
+   };                                                                          \
+                                                                               \
+   ENUM_OUTPUT_OPS(type)                                                       \
+   ENUM_NAME_DEF(type, for_all)                                                \
+   ENUM_PARSE_DEF(type, for_all)                                               \
+   ENUM_FROM_JSON(type)                                                        \
+   ENUM_TO_JSON(type)                                                          \
+   ENUM_EQUALITY_OPS(type)
+
+#define ENUM_BITWISE_COMPARISON_OP(enum_type, numeric_type, op)                \
+   constexpr enum_type operator op(enum_type lhs, enum_type rhs) noexcept {    \
+      return static_cast<enum_type>(static_cast<numeric_type>(lhs)             \
+                                        op static_cast<numeric_type>(rhs));    \
+   }
+
+#define ENUM_BITWISE_COMPARISON_OPS(enum_type, numeric_type)                   \
+   ENUM_BITWISE_COMPARISON_OP(enum_type, numeric_type, |)                      \
+   ENUM_BITWISE_COMPARISON_OP(enum_type, numeric_type, &)                      \
+   ENUM_BITWISE_COMPARISON_OP(enum_type, numeric_type, ^)
+
+#define ENUM_BITWISE_ASSIGNMENT_OP(enum_type, op)                              \
+   inline enum_type &operator op##=(enum_type &lhs, enum_type rhs) noexcept {  \
+      lhs = lhs op rhs;                                                        \
+      return lhs;                                                              \
+   }
+
+#define ENUM_BITWISE_ASSIGNMENT_OPS(enum_type)                                 \
+   ENUM_BITWISE_ASSIGNMENT_OP(enum_type, |)                                    \
+   ENUM_BITWISE_ASSIGNMENT_OP(enum_type, &)                                    \
+   ENUM_BITWISE_ASSIGNMENT_OP(enum_type, ^)
+
+#define ENUM_BITWISE_OPS(enum_type, numeric_type)                              \
+   ENUM_BITWISE_COMPARISON_OPS(enum_type, numeric_type)                        \
+   ENUM_BITWISE_ASSIGNMENT_OPS(enum_type)
+
+#define ENUM_HAS_FLAG(enum_value, flag) (enum_value & flag) == flag
+#define ENUM_STRIP_FLAG(enum_value, flag) (enum_value | flag) ^ flag
+
+#define PB_ENUM_FROM_JSON(enum_type)                                           \
+   inline void from_json(const nlohmann::json &j, enum_type &value) {          \
+      io::JsonReader jr{&j};                                                   \
+      std::string name;                                                        \
+      bool success = jr.read(name) && enum_type##_Parse(name, &value);         \
+      if (!success) {                                                          \
+         const auto j_str = j.dump(2);                                         \
+         LOG(Gen, Diag, "Failed to parse json to " #enum_type ":\n{}", j_str); \
+      }                                                                        \
+   }
+
+#define PB_ENUM_TO_JSON(enum_type)                                             \
+   inline void to_json(nlohmann::json &j, const enum_type &value) {            \
+      j = enum_type##_Name(value);                                             \
+   }

src/lib/core/FMT.h

@@ -0,0 +1,34 @@
+#pragma once
+
+#include <fmt/core.h>
+#include <fmt/format.h>
+#ifndef _WIN32
+#include <range/v3/view.hpp>
+#endif
+#include <vector>
+
+#define FMT(template_str, ...) fmt::format(template_str, __VA_ARGS__)
+#define FMT_JOIN(range, delimiter)                                             \
+   fmt::to_string(fmt::join(range.begin(), range.end(), delimiter))
+
+namespace bts {
+
+template <class Range>
+std::string _fmt_join(Range range, const char *delimiter) {
+   return fmt::join(range.begin(), range.end(), delimiter);
+}
+
+#ifndef _WIN32
+template <class Range, class Op>
+std::string _fmt_join_fn(Range range, const char *delimiter, Op op) {
+   return fmt::to_string(
+       fmt::join(range | ranges::views::transform(op), delimiter));
+}
+#endif
+
+} // namespace bts
+
+#ifndef _WIN32
+#define FMT_JOIN_FN(range, delimiter, fn)                                      \
+   bts::_fmt_join_fn(range, delimiter, fn)
+#endif

src/lib/core/FMT_IMPL.h

@@ -0,0 +1,65 @@
+#pragma once
+
+#include <fmt/format.h>
+
+#define FMT_IMPL(Type, var_name, ...)                                          \
+   template <>                                                                 \
+   struct fmt::formatter<Type> {                                               \
+      constexpr auto parse(fmt::format_parse_context &ctx)                     \
+          -> decltype(ctx.end()) {                                             \
+         return ctx.end();                                                     \
+      }                                                                        \
+                                                                               \
+      template <typename FormatContext>                                        \
+      auto format(const Type &var_name, FormatContext &ctx) const              \
+          -> decltype(ctx.out()) {                                             \
+         return fmt::format_to(ctx.out(), __VA_ARGS__);                        \
+      }                                                                        \
+   }
+
+#define FMT_IMPL_T(Type, var_name, ...)                                        \
+   struct fmt::formatter<Type> {                                               \
+      constexpr auto parse(fmt::format_parse_context &ctx)                     \
+          -> decltype(ctx.end()) {                                             \
+         return ctx.end();                                                     \
+      }                                                                        \
+                                                                               \
+      template <typename FormatContext>                                        \
+      auto format(const Type &var_name, FormatContext &ctx) const              \
+          -> decltype(ctx.out()) {                                             \
+         return fmt::format_to(ctx.out(), __VA_ARGS__);                        \
+      }                                                                        \
+   }
+
+#define FMT_IMPL_OSTREAM(T)                                                    \
+   template <>                                                                 \
+   struct fmt::formatter<T> {                                                  \
+      constexpr auto parse(fmt::format_parse_context &ctx)                     \
+          -> decltype(ctx.end()) {                                             \
+         return ctx.end();                                                     \
+      }                                                                        \
+                                                                               \
+      template <typename FormatContext>                                        \
+      auto format(const T &t, FormatContext &ctx) const                        \
+          -> decltype(ctx.out()) {                                             \
+         std::stringstream ss;                                                 \
+         ss << t;                                                              \
+         return fmt::format_to(ctx.out(), "{}", ss.str());                     \
+      }                                                                        \
+   }
+
+#define FMT_IMPL_OSTREAM_T(T, ...)                                             \
+   struct fmt::formatter<T<__VA_ARGS__>> {                                     \
+      constexpr auto parse(fmt::format_parse_context &ctx)                     \
+          -> decltype(ctx.end()) {                                             \
+         return ctx.end();                                                     \
+      }                                                                        \
+                                                                               \
+      template <typename FormatContext>                                        \
+      auto format(const T<__VA_ARGS__> &t, FormatContext &ctx) const           \
+          -> decltype(ctx.out()) {                                             \
+         std::stringstream ss;                                                 \
+         ss << t;                                                              \
+         return fmt::format_to(ctx.out(), "{}", ss.str());                     \
+      }                                                                        \
+   }

src/lib/core/HashPhaser.h

@@ -132,7 +132,7 @@ std::size_t HashPhaser::auto_phase_range(Iter first, Iter last) {
    std::uint16_t el_count =
        num_vals > (std::numeric_limits<std::uint16_t>::max)()
            ? max_els
-           : std::max(static_cast<std::uint16_t>(num_vals), max_els);
+           : (static_cast<std::uint16_t>(num_vals) > max_els ? static_cast<std::uint16_t>(num_vals) : max_els);
    auto bits_per_hash = (sizeof(std::size_t) * __CHAR_BIT__) / el_count;
    std::size_t phase = 1U << (bits_per_hash - 1);
    std::size_t composite_hash = 0;
@@ -157,7 +157,7 @@ std::size_t HashPhaser::auto_phase_range(Iter first, Iter last, Hash hash) {
    std::uint16_t el_count =
        num_vals > (std::numeric_limits<std::uint16_t>::max)()
            ? max_els
-           : std::max(static_cast<std::uint16_t>(num_vals), max_els);
+           : (static_cast<std::uint16_t>(num_vals) > max_els ? static_cast<std::uint16_t>(num_vals) : max_els);
    auto bits_per_hash = (sizeof(std::size_t) * __CHAR_BIT__) / el_count;
    std::size_t phase = 1U << (bits_per_hash - 1);
    std::size_t composite_hash = 0;

src/lib/core/TimeSpan.cpp

@@ -0,0 +1,26 @@
+#include "TimeSpan.h"
+#include <boost/date_time/posix_time/time_formatters.hpp>
+#include <boost/date_time/posix_time/time_parsers.hpp>
+
+namespace bts::core {
+
+TimeSpan TimeSpan::from_iso_string(const std::string &iso_string) {
+   const auto pdur = boost::posix_time::duration_from_string(iso_string);
+   return from_posix_duration(pdur);
+}
+
+std::string TimeSpan::to_iso_string() const {
+   const auto pdur = to_posix_duration();
+   return boost::posix_time::to_iso_string(pdur);
+}
+
+std::ostream &operator<<(std::ostream &out, const TimeSpan &ts) noexcept {
+   out << ts.to_iso_string();
+   return out;
+}
+
+TimeSpan TimeSpan::from_posix_duration(boost::posix_time::time_duration d) {
+   return TimeSpan{d.total_nanoseconds()};
+}
+
+} // namespace bts::core

src/lib/core/TimeSpan.h

@@ -0,0 +1,125 @@
+#pragma once
+#include "lib/core/FMT_IMPL.h"
+#include "lib/utils/bts_assert.h"
+#include "lib/utils/datetime.h"
+#include <boost/date_time/posix_time/conversion.hpp>
+#include <boost/date_time/posix_time/posix_time_config.hpp>
+#include <boost/date_time/posix_time/posix_time_duration.hpp>
+#include <boost/date_time/posix_time/ptime.hpp>
+#include <boost/date_time/time.hpp>
+#include <chrono>
+#include <cstdint>
+#include <fmt/format.h>
+#include <string>
+
+namespace bts::core {
+
+/**
+ * @brief Represents a duration of time
+ */
+class TimeSpan {
+ public:
+   constexpr TimeSpan() = default;
+   explicit constexpr TimeSpan(std::int64_t ticks);
+
+   static constexpr TimeSpan zero();
+
+   static constexpr TimeSpan from_nanos(std::int64_t value);
+   static constexpr TimeSpan from_micros(std::int64_t value);
+   static constexpr TimeSpan from_millis(std::int64_t value);
+   static constexpr TimeSpan from_seconds(std::int64_t value);
+   static TimeSpan from_posix_duration(boost::posix_time::time_duration t);
+   static TimeSpan from_iso_string(const std::string &iso_string);
+
+   constexpr std::int64_t nanos() const;
+   constexpr std::int64_t micros() const;
+   constexpr std::int64_t millis() const;
+   constexpr std::int64_t seconds() const;
+
+   constexpr double total_millis() const;
+   constexpr double total_seconds() const;
+
+   constexpr boost::posix_time::time_duration to_posix_duration() const;
+   std::string to_iso_string() const;
+
+ private:
+   // nanos
+   std::int64_t ticks_{};
+};
+
+constexpr bool operator==(TimeSpan lhs, TimeSpan rhs) noexcept;
+constexpr bool operator!=(TimeSpan lhs, TimeSpan rhs) noexcept;
+constexpr bool operator<(TimeSpan lhs, TimeSpan rhs) noexcept;
+constexpr bool operator>(TimeSpan lhs, TimeSpan rhs) noexcept;
+
+std::ostream &operator<<(std::ostream &out, const TimeSpan &ts) noexcept;
+
+//////////////////////////////////////////////////
+// Definition
+//////////////////////////////////////////////////
+
+constexpr TimeSpan::TimeSpan(std::int64_t ticks) : ticks_(ticks) {}
+
+constexpr TimeSpan TimeSpan::zero() { return TimeSpan{0}; }
+
+constexpr TimeSpan TimeSpan::from_nanos(std::int64_t value) {
+   return TimeSpan{value};
+}
+
+constexpr TimeSpan TimeSpan::from_micros(std::int64_t value) {
+   constexpr const auto nanos_per_micro = 1'000;
+   return TimeSpan{value * nanos_per_micro};
+}
+
+constexpr TimeSpan TimeSpan::from_millis(std::int64_t value) {
+   constexpr const auto nanos_per_milli = 1'000'000;
+   return TimeSpan{value * nanos_per_milli};
+}
+
+constexpr TimeSpan TimeSpan::from_seconds(std::int64_t value) {
+   constexpr const auto nanos_per_second = 1'000'000'000;
+   return TimeSpan{value * nanos_per_second};
+}
+
+constexpr std::int64_t TimeSpan::nanos() const { return ticks_; }
+
+constexpr std::int64_t TimeSpan::micros() const { return ticks_ / 1'000; }
+
+constexpr std::int64_t TimeSpan::millis() const { return ticks_ / 1'000'000; }
+
+constexpr std::int64_t TimeSpan::seconds() const {
+   return ticks_ / 1'000'000'000;
+}
+
+constexpr double TimeSpan::total_millis() const {
+   return static_cast<double>(ticks_) / 1000'000;
+}
+
+constexpr double TimeSpan::total_seconds() const {
+   return static_cast<double>(ticks_) / 1000'000'000;
+}
+
+constexpr boost::posix_time::time_duration TimeSpan::to_posix_duration() const {
+   return boost::date_time::subsecond_duration<
+       boost::posix_time::time_duration, 1000000000>{nanos()};
+}
+
+constexpr bool operator==(TimeSpan lhs, TimeSpan rhs) noexcept {
+   return lhs.nanos() == rhs.nanos();
+}
+
+constexpr bool operator!=(TimeSpan lhs, TimeSpan rhs) noexcept {
+   return lhs.nanos() != rhs.nanos();
+}
+
+constexpr bool operator<(TimeSpan lhs, TimeSpan rhs) noexcept {
+   return lhs.nanos() < rhs.nanos();
+}
+
+constexpr bool operator>(TimeSpan lhs, TimeSpan rhs) noexcept {
+   return lhs.nanos() > rhs.nanos();
+}
+
+} // namespace bts::core
+
+FMT_IMPL(bts::core::TimeSpan, t, "{}", t.to_iso_string());

src/lib/core/TimeStamp.cpp

@@ -0,0 +1,58 @@
+#include "TimeStamp.h"
+#include "lib/utils/datetime.h"
+#include <boost/date_time/posix_time/time_formatters.hpp>
+
+namespace bts::core {
+
+using std::uint32_t;
+
+static bool is_valid_ymd(uint32_t year, uint32_t month, uint32_t day) {
+   return year >= 1970 && month >= 1 && month <= 12 && day >= 1 && day <= 31;
+}
+
+TimeStamp::TimeStamp(uint32_t year, uint32_t month, uint32_t day) {
+   bts_assert_msgf(
+       is_valid_ymd(year, month, day), "Invalid timestamp {}/{}/{} (y/m/d)",
+       year, month, day);
+   std::tm date = {};
+   date.tm_year = year - 1900;
+   date.tm_mon = month - 1;
+   date.tm_mday = day;
+   auto time = std::chrono::system_clock::from_time_t(mktime(&date));
+   auto duration = time.time_since_epoch();
+   value_ =
+       std::chrono::duration_cast<std::chrono::nanoseconds>(duration).count();
+}
+
+bool TimeStamp::from_ymd(
+    std::uint32_t year, std::uint32_t month, std::uint32_t day, TimeStamp *result) {
+   if (!is_valid_ymd(year, month, day)) {
+      return false;
+   }
+   *result = TimeStamp{year, month, day};
+   return true;
+}
+
+TimeStamp TimeStamp::from_iso_string(const std::string &iso_string) {
+   auto ptime = datetime::parse_iso_date_time_string(iso_string);
+   return from_ptime(ptime);
+}
+
+std::string TimeStamp::to_iso_string() const {
+   return boost::posix_time::to_iso_string(to_ptime());
+}
+
+std::ostream &operator<<(std::ostream &out, const TimeStamp &ts) noexcept {
+   out << ts.to_iso_string();
+   return out;
+}
+
+TimeStamp TimeStamp::from_ptime(boost::posix_time::ptime t) {
+   // time_t is Unix time: number of seconds since epoch Jan 1, 1970
+   auto epoch_pt =
+       t - boost::posix_time::ptime(boost::gregorian::date(1970, 1, 1));
+   auto epoch_nanos = epoch_pt.total_nanoseconds();
+   return TimeStamp{static_cast<std::uint64_t>(epoch_nanos)};
+}
+
+} // namespace bts::core

src/lib/core/TimeStamp.h

@@ -1,27 +1,211 @@
 #pragma once
+#include "lib/core/FMT_IMPL.h"
+#include "lib/core/TimeSpan.h"
+#include "lib/utils/bts_assert.h"
+#include "lib/utils/datetime.h"
+
+#include <boost/date_time/posix_time/conversion.hpp>
+#include <boost/date_time/posix_time/posix_time_duration.hpp>
+#include <boost/date_time/posix_time/ptime.hpp>
+#include <boost/date_time/time.hpp>
+#include <chrono>
 #include <cstdint>
+#include <fmt/format.h>
+#include <limits>
+#include <string>
 
 namespace bts::core {
 
-// I can send this whole class later but its a nice wrapper around the linux
-// posix time
+/**
+ * @brief Represents time as nano-seconds since the unix epoch (Jan 1, 1970)
+ */
 struct TimeStamp {
  public:
    constexpr TimeStamp() = default;
    explicit constexpr TimeStamp(std::uint64_t value);
+   // constructs a TimeStamp from a UTC date
+   explicit TimeStamp(uint32_t year, uint32_t month, uint32_t day);
+
+   static constexpr TimeStamp min();
+   static constexpr TimeStamp max();
+
+   static constexpr TimeStamp from_int64(std::int64_t value);
+   static constexpr TimeStamp from_nanos(std::int64_t value);
+   static constexpr TimeStamp from_micros(std::int64_t value);
+   static constexpr TimeStamp from_millis(std::int64_t value);
+   static constexpr TimeStamp from_seconds(std::int64_t value);
+   static TimeStamp from_ptime(boost::posix_time::ptime t);
+   static TimeStamp from_iso_string(const std::string &iso_string);
+   static bool from_ymd(
+       std::uint32_t year,
+       std::uint32_t month,
+       std::uint32_t day,
+       TimeStamp *result);
+
    constexpr std::uint64_t nanos() const;
+   constexpr std::uint64_t micros() const;
+   constexpr std::uint64_t millis() const;
+   constexpr std::uint64_t seconds() const;
+
+   constexpr double total_millis() const;
+   constexpr double total_seconds() const;
+
+   constexpr std::int64_t to_int64();
+   inline void to_date(uint32_t &year, uint32_t &month, uint32_t &day);
+   constexpr boost::posix_time::ptime to_ptime() const;
+   std::string to_iso_string() const;
+   // current epoch ns
+   static uint64_t now_ns() {
+#ifndef _WIN32
+      return datetime::unix_nano_now();
+#else
+      auto now = std::chrono::system_clock::now();
+      auto ns = std::chrono::duration_cast<std::chrono::nanoseconds>(
+                    now.time_since_epoch())
+                    .count();
+      return ns;
+#endif
+   }
+   static TimeStamp now() { return from_nanos(now_ns()); }
 
  private:
    // posix time
    std::uint64_t value_{};
 };
 
+constexpr bool operator==(TimeStamp lhs, TimeStamp rhs) noexcept;
+constexpr bool operator!=(TimeStamp lhs, TimeStamp rhs) noexcept;
+constexpr bool operator<(TimeStamp lhs, TimeStamp rhs) noexcept;
+constexpr bool operator<=(TimeStamp lhs, TimeStamp rhs) noexcept;
+constexpr bool operator>(TimeStamp lhs, TimeStamp rhs) noexcept;
+constexpr bool operator>=(TimeStamp lhs, TimeStamp rhs) noexcept;
+
+constexpr TimeSpan operator-(TimeStamp lhs, TimeStamp rhs) noexcept;
+constexpr TimeStamp operator-(TimeStamp lhs, TimeSpan rhs) noexcept;
+constexpr TimeStamp operator+(TimeStamp lhs, TimeSpan rhs) noexcept;
+
+std::ostream &operator<<(std::ostream &out, const TimeStamp &ts) noexcept;
+
 //////////////////////////////////////////////////
 // Definition
 //////////////////////////////////////////////////
 
 constexpr TimeStamp::TimeStamp(std::uint64_t value) : value_(value) {}
 
+constexpr TimeStamp TimeStamp::min() { return TimeStamp{0}; }
+
+constexpr TimeStamp TimeStamp::max() {
+   return TimeStamp{std::numeric_limits<std::uint64_t>::max()};
+}
+
+constexpr TimeStamp TimeStamp::from_int64(std::int64_t value) {
+   bts_assert(value >= 0);
+   return TimeStamp{static_cast<std::uint64_t>(value)};
+}
+
+constexpr TimeStamp TimeStamp::from_nanos(std::int64_t value) {
+   bts_assert(value >= 0);
+   return TimeStamp{static_cast<std::uint64_t>(value)};
+}
+
+constexpr TimeStamp TimeStamp::from_micros(std::int64_t value) {
+   bts_assert(value >= 0);
+   constexpr const auto nanos_per_micro = 1'000;
+   return TimeStamp{static_cast<std::uint64_t>(value * nanos_per_micro)};
+}
+
+constexpr TimeStamp TimeStamp::from_millis(std::int64_t value) {
+   bts_assert(value >= 0);
+   constexpr const auto nanos_per_milli = 1'000'000;
+   return TimeStamp{static_cast<std::uint64_t>(value * nanos_per_milli)};
+}
+
+constexpr TimeStamp TimeStamp::from_seconds(std::int64_t value) {
+   bts_assert(value >= 0);
+   constexpr const auto nanos_per_second = 1'000'000'000;
+   return TimeStamp{static_cast<std::uint64_t>(value * nanos_per_second)};
+}
+
 constexpr std::uint64_t TimeStamp::nanos() const { return value_; }
 
+constexpr std::uint64_t TimeStamp::micros() const { return value_ / 1'000; }
+
+constexpr std::uint64_t TimeStamp::millis() const { return value_ / 1'000'000; }
+
+constexpr std::uint64_t TimeStamp::seconds() const {
+   return value_ / 1'000'000'000;
+}
+
+constexpr double TimeStamp::total_millis() const {
+   return static_cast<double>(value_) / 1000'000;
+}
+
+constexpr double TimeStamp::total_seconds() const {
+   return static_cast<double>(value_) / 1000'000'000;
+}
+
+constexpr std::int64_t TimeStamp::to_int64() {
+   bts_assert(
+       value_ <=
+       static_cast<std::uint64_t>(std::numeric_limits<std::int64_t>::max()));
+   return static_cast<std::int64_t>(value_);
+}
+void TimeStamp::to_date(uint32_t &year, uint32_t &month, uint32_t &day) {
+   std::uint64_t epoch_secs = value_ / (1000 * 1000 * 1000);
+   time_t ts_tt(epoch_secs);
+   struct tm *ptm = gmtime(&ts_tt);
+   year = ptm->tm_year + 1900;
+   month = ptm->tm_mon + 1;
+   day = ptm->tm_mday;
+}
+
+constexpr boost::posix_time::ptime TimeStamp::to_ptime() const {
+   constexpr const auto nanos_per_sec = 1'000'000'000;
+   constexpr const auto nanos_per_micro = 1'000;
+   // time_t is Unix time: number of seconds since epoch Jan 1, 1970
+   std::time_t ts = value_ / nanos_per_sec;
+   const auto nanos_rem = value_ % nanos_per_sec;
+   return boost::posix_time::ptime(boost::gregorian::date(1970, 1, 1)) +
+          boost::posix_time::seconds(static_cast<long>(ts)) +
+          boost::posix_time::microseconds(nanos_rem / nanos_per_micro);
+}
+
+constexpr bool operator==(TimeStamp lhs, TimeStamp rhs) noexcept {
+   return lhs.nanos() == rhs.nanos();
+}
+
+constexpr bool operator!=(TimeStamp lhs, TimeStamp rhs) noexcept {
+   return lhs.nanos() != rhs.nanos();
+}
+
+constexpr bool operator<(TimeStamp lhs, TimeStamp rhs) noexcept {
+   return lhs.nanos() < rhs.nanos();
+}
+
+constexpr bool operator<=(TimeStamp lhs, TimeStamp rhs) noexcept {
+   return lhs.nanos() <= rhs.nanos();
+}
+
+constexpr bool operator>(TimeStamp lhs, TimeStamp rhs) noexcept {
+   return lhs.nanos() > rhs.nanos();
+}
+
+constexpr bool operator>=(TimeStamp lhs, TimeStamp rhs) noexcept {
+   return lhs.nanos() >= rhs.nanos();
+}
+
+constexpr TimeSpan operator-(TimeStamp lhs, TimeStamp rhs) noexcept {
+   return TimeSpan::from_nanos(lhs.nanos() - rhs.nanos());
+}
+
+constexpr TimeStamp operator-(TimeStamp lhs, TimeSpan rhs) noexcept {
+   return TimeStamp{lhs.nanos() - rhs.nanos()};
+}
+
+constexpr TimeStamp operator+(TimeStamp lhs, TimeSpan rhs) noexcept {
+   return TimeStamp{lhs.nanos() + rhs.nanos()};
+}
+
 } // namespace bts::core
+
+FMT_IMPL(bts::core::TimeStamp, t, "{}", t.to_iso_string());

src/lib/domain/Price.h

@@ -1,10 +1,14 @@
 #pragma once
+
 #include "Side.h"
+#include "lib/core/FMT_IMPL.h"
 #include "lib/utils/bts_assert.h"
+
 #include <cmath>
 #include <cstdint>
 #include <limits>
 #include <ostream>
+#include <string>
 
 #if bts_assert_msg
 #define __ASSERT_MSG(EXP, msg) bts_assert_msg(EXP, msg)
@@ -490,6 +494,8 @@ inline std::ostream &operator<<(std::ostream &out, Price p) noexcept {
 
 } // namespace bts::domain
 
+FMT_IMPL(bts::domain::Price, p, "{}", p.to_double());
+
 namespace std { // NOLINT
 
 constexpr size_t hash<bts::domain::Price>::operator()(

src/lib/domain/Quantity.h

@@ -1,5 +1,9 @@
 #pragma once
+
+#include "lib/core/FMT_IMPL.h"
+#include "lib/logging/logging.h"
 #include "lib/utils/ValueWrapper.h"
+
 #include <cstdint>
 
 namespace bts::domain {
@@ -110,3 +114,5 @@ QTY_UINT32_BIN_REF_OP(%=)
 #undef QTY_UINT32_BIN_REF_OP
 
 } // namespace bts::domain
+
+FMT_IMPL(bts::domain::Quantity, q, "{}", *q);

src/lib/domain/SecurityID.h

@@ -1,4 +1,6 @@
 #pragma once
+
+#include "lib/core/FMT_IMPL.h"
 #include "lib/utils/ValueWrapper.h"
 #include <cstdint>
 
@@ -7,3 +9,5 @@ namespace bts::domain {
 using SecurityID = utils::ValueWrapper<struct SecuirtyIDTag, std::uint64_t>;
 
 } // namespace bts::domain
+
+FMT_IMPL(bts::domain::SecurityID, id, "{}", *id);

src/lib/domain_model/Security.h

@@ -1,12 +1,15 @@
 #pragma once
 
-#include "lib/utils/datetime.h"
-#include <cmath>
-#include <memory>
-
+#include "lib/core/FMT_IMPL.h"
 #include "lib/domain_model/Security.pb.h"
+#include "lib/domain_model/Side.h"
+#include "lib/domain_model/Venue.h"
 #include "lib/domain_model/common.h"
 #include "lib/utils/bts_assert.h"
+#include "lib/utils/datetime.h"
+
+#include <cmath>
+#include <memory>
 
 namespace boost {
 namespace gregorian {

src/lib/domain_model/Side.h

@@ -0,0 +1,18 @@
+#pragma once
+
+#include "lib/core/EnumMacros.h"
+#include "lib/core/FMT_IMPL.h"
+#include "lib/domain_model/Security.pb.h"
+#include "lib/io/JsonReader.h"
+#include "lib/logging/logging.h"
+
+#include <nlohmann/json.hpp>
+
+namespace bts::api {
+
+PB_ENUM_FROM_JSON(Side)
+PB_ENUM_TO_JSON(Side)
+
+} // namespace bts::api
+
+FMT_IMPL(bts::api::Side, s, "{}", bts::api::Side_Name(s));

src/lib/domain_model/Venue.h

@@ -0,0 +1,18 @@
+#pragma once
+
+#include "lib/core/EnumMacros.h"
+#include "lib/core/FMT_IMPL.h"
+#include "lib/domain_model/Security.pb.h"
+#include "lib/io/JsonReader.h"
+#include "lib/logging/logging.h"
+
+#include <nlohmann/json.hpp>
+
+namespace bts::api {
+
+PB_ENUM_FROM_JSON(Venue)
+PB_ENUM_TO_JSON(Venue)
+
+} // namespace bts::api
+
+FMT_IMPL(bts::api::Venue, v, "{}", bts::api::Venue_Name(v));

src/lib/domain_model/common.h

@@ -6,6 +6,7 @@
 #include <cstdint>
 #include <string>
 
+#include "lib/core/FMT_IMPL.h"
 #include "lib/domain_model/SecurityKey.h"
 #include "lib/utils/ValueWrapper.h"
 
@@ -102,3 +103,6 @@ using Price = utils::ValueWrapper<struct OrderPriceTag, double>;
 using Quantity = CQuantity;
 using UserLogin = utils::ValueWrapper<struct UserLoginTag, std::string>;
 } // namespace bts
+
+FMT_IMPL(bts::Price, p, "{}", p.value());
+FMT_IMPL(bts::Quantity, q, "{}", q.value());

src/lib/io/JsonDumper.h

@@ -0,0 +1,48 @@
+#pragma once
+#include <nlohmann/json.hpp>
+#include <sstream>
+#include <string>
+
+namespace bts::io {
+
+class JsonDumper {
+ public:
+   template <class T>
+   static std::string dump(const T &value, int indentation = 2);
+
+   template <class Iter>
+   static std::string dump(
+       Iter first,
+       Iter last,
+       int indentation = 2,
+       const char *separator = ",\n");
+};
+
+//////////////////////////////////////////////////
+// Definition
+//////////////////////////////////////////////////
+
+template <class T>
+std::string JsonDumper::dump(const T &value, int indentation) {
+   nlohmann::json j;
+   to_json(j, value);
+   return j.dump(indentation);
+}
+
+template <class Iter>
+std::string JsonDumper::dump(
+    Iter first, Iter last, int indentation, const char *separator) {
+   if (first == last) {
+      return "";
+   }
+   std::stringstream str;
+   auto it = first;
+   str << "[" << dump(*it, indentation);
+   while (++it != last) {
+      str << separator << dump(*it, indentation);
+   }
+   str << "]";
+   return str.str();
+}
+
+} // namespace bts::io

src/lib/io/JsonMacros.h

@@ -0,0 +1,22 @@
+#pragma once
+
+#define _TO_JSON_FIELD(field) j[#field] = t.field;
+#define _FROM_JSON_FIELD(field) bts::io::JsonReader{&j, #field}.read(t.field);
+
+#define SPARK_TO_JSON_IMPL(type, for_each)                                     \
+   inline void to_json(nlohmann::json &j, const type &t) {                     \
+      FOR_EACH(_TO_JSON_FIELD)                                                 \
+   }
+
+#define SPARK_FROM_JSON_IMPL(type, for_each)                                   \
+   inline void from_json(const nlohmann::json &j, type &t) {                   \
+      FOR_EACH(_FROM_JSON_FIELD)                                               \
+   }
+
+#define OSTREAM_JSON_IMPL(type)                                                \
+   inline std::ostream &operator<<(std::ostream &out, const type &t) {         \
+      nlohmann::json j;                                                        \
+      to_json(j, t);                                                           \
+      out << j.dump(2);                                                        \
+      return out;                                                              \
+   }

src/lib/io/JsonReader.cpp

@@ -0,0 +1,520 @@
+#include "JsonReader.h"
+#include "lib/algorithm/TryCatch.h"
+#include "lib/core/TimeStamp.h"
+#include "lib/logging/logging.h"
+#include "lib/utils/TimeStamp.h"
+#include "lib/utils/datetime.h"
+#include <boost/date_time/posix_time/ptime.hpp>
+#include <exception>
+#include <limits>
+#include <string>
+
+namespace bts::io {
+
+using core::TimeStamp;
+using nlohmann::json;
+using std::int32_t;
+using std::int64_t;
+using std::numeric_limits;
+using std::stoull;
+using std::string;
+using std::uint32_t;
+using std::uint64_t;
+
+// 'read' fn bodies should be wrapped between BEGIN/END_READ_JSON tags.
+// These macros abstract out repetative exception handling code (and a nullptr check).
+
+#define BEGIN_READ_JSON                                                        \
+   if (json_ == nullptr) {                                                     \
+      return false;                                                            \
+   }                                                                           \
+   try
+
+#define END_READ_JSON(type_name)                                               \
+   catch (const std::exception &e) {                                           \
+      LOG(Gen, Diag, "Failed to parse json to '{}'. Exception: {}",            \
+          std::string{type_name}, std::string{e.what()});                      \
+   }                                                                           \
+   catch (...) {                                                               \
+      bts_assert_msgf(                                                         \
+          false, "Failed to parse json to '{}'. Exception: unknown",           \
+          std::string{type_name});                                             \
+   }                                                                           \
+   return false;
+
+// The idea here is we parse numbers to wide types (either int64, uint64, or double),
+// then cast that value to the (potentially narrower) output type. This macro evaluates
+// to true if 'wide_val' is within the bounds of the result type (otherwise false).
+#define NUM_WITHIN_BOUNDS(wide_val, result_t)                                  \
+   (wide_val >=                                                                \
+        static_cast<decltype(wide_val)>(numeric_limits<result_t>::min()) &&    \
+    wide_val <=                                                                \
+        static_cast<decltype(wide_val)>(numeric_limits<result_t>::max()))
+
+static const nlohmann::json *get_field_json(
+    const nlohmann::json *json, const std::string &name) {
+   if (!json || !json->is_object()) {
+      return nullptr;
+   }
+   auto it = json->find(name);
+   if (it == json->end()) {
+      return nullptr;
+   }
+   return &it.value();
+}
+
+JsonReader::JsonReader(const nlohmann::json *json) : json_(json) {}
+
+JsonReader::JsonReader(const nlohmann::json *json, const std::string &name)
+    : json_(get_field_json(json, name)) {}
+
+bool JsonReader::try_parse(
+    const std::string &str, nlohmann::json &result, std::string *err) {
+   try {
+      result = nlohmann::json::parse(str);
+      return true;
+   }
+   CATCH_AND_HANDLE_MSG([err](const auto &msg) {
+      if (err != nullptr) {
+         *err = msg;
+      }
+   })
+   return false;
+}
+
+bool JsonReader::read(const string &name, string &result) const {
+   return JsonReader{json_, name}.read(result);
+}
+
+bool JsonReader::read(const string &name, int32_t &result) const {
+   return JsonReader{json_, name}.read(result);
+}
+
+bool JsonReader::read(const string &name, uint32_t &result) const {
+   return JsonReader{json_, name}.read(result);
+}
+
+bool JsonReader::read(const string &name, int64_t &result) const {
+   return JsonReader{json_, name}.read(result);
+}
+
+bool JsonReader::read(const string &name, uint64_t &result) const {
+   return JsonReader{json_, name}.read(result);
+}
+
+bool JsonReader::read(const string &name, float &result) const {
+   return JsonReader{json_, name}.read(result);
+}
+
+bool JsonReader::read(const string &name, double &result) const {
+   return JsonReader{json_, name}.read(result);
+}
+
+bool JsonReader::read(const string &name, TimeStamp &result) const {
+   return JsonReader{json_, name}.read(result);
+}
+
+bool JsonReader::read_epoch_nanos(const string &name, TimeStamp &value) const {
+   return JsonReader{json_, name}.read_epoch_nanos(value);
+}
+
+bool JsonReader::read_epoch_micros(const string &name, TimeStamp &value) const {
+   return JsonReader{json_, name}.read_epoch_micros(value);
+}
+
+bool JsonReader::read_epoch_millis(const string &name, TimeStamp &value) const {
+   return JsonReader{json_, name}.read_epoch_millis(value);
+}
+
+bool JsonReader::read(const string &name, nlohmann::json &result) const {
+   return JsonReader{json_, name}.read(result);
+}
+
+bool JsonReader::read(int32_t &result) const {
+   BEGIN_READ_JSON {
+      switch (json_->type()) {
+      case nlohmann::json::value_t::number_float: {
+         const auto d = json_->get<double>();
+         result = static_cast<int32_t>(d);
+         if (result != d) {
+            LOG(Gen, Warn,
+                "Potential loss of data converting field of type "
+                "'number_float' ({}) to int32 ({})",
+                d, result);
+         }
+         return true;
+      }
+      case nlohmann::json::value_t::number_integer: {
+         const auto il = json_->get<int64_t>();
+         if (!NUM_WITHIN_BOUNDS(il, int32_t)) {
+            LOG(Gen, Error,
+                "Failed to convert field of type 'number_integer' ({}) to "
+                "uint32: value is negative.",
+                il);
+         }
+         result = static_cast<int32_t>(il);
+         return true;
+      }
+      case nlohmann::json::value_t::number_unsigned: {
+         const auto ul = json_->get<uint64_t>();
+         if (ul > static_cast<uint64_t>(numeric_limits<int32_t>::max())) {
+            LOG(Gen, Error,
+                "Failed to convert field of type 'number_unsigned' ({}) to "
+                "int32: value is too large.",
+                ul);
+            return false;
+         }
+         result = static_cast<int32_t>(ul);
+         return true;
+      }
+      case nlohmann::json::value_t::string: {
+         const auto str = json_->get<string>();
+         if (!str.empty()) {
+            result = std::stoi(str);
+            return true;
+         }
+         break;
+      }
+      default:
+         return false;
+      }
+   }
+   END_READ_JSON("int32")
+}
+
+bool JsonReader::read(uint32_t &result) const {
+   BEGIN_READ_JSON {
+      switch (json_->type()) {
+      case nlohmann::json::value_t::number_float: {
+         const auto d = json_->get<double>();
+         result = static_cast<uint32_t>(d);
+         if (result != d) {
+            LOG(Gen, Warn,
+                "Potential loss of data converting field of type "
+                "'number_float' ({}) to uint32 ({})",
+                d, result);
+         }
+         return true;
+      }
+      case nlohmann::json::value_t::number_integer: {
+         const auto i = json_->get<int64_t>();
+         if (!NUM_WITHIN_BOUNDS(i, uint32_t)) {
+            LOG(Gen, Error,
+                "Failed to convert field of type 'number_integer' ({}) to "
+                "uint32: value is negative.",
+                i);
+            return false;
+         }
+         result = static_cast<uint32_t>(i);
+         return true;
+      }
+      case nlohmann::json::value_t::number_unsigned: {
+         const auto ul = json_->get<uint64_t>();
+         if (ul > static_cast<uint64_t>(numeric_limits<int32_t>::max())) {
+            LOG(Gen, Error,
+                "Failed to convert field of type 'number_unsigned' ({}) to "
+                "int32: value is too large.",
+                ul);
+            return false;
+         }
+         result = static_cast<int32_t>(ul);
+         return true;
+      }
+      case nlohmann::json::value_t::string: {
+         const auto str = json_->get<string>();
+         if (!str.empty()) {
+            result = std::stoul(str);
+            return true;
+         }
+         break;
+      }
+      default:
+         return false;
+      }
+   }
+   END_READ_JSON("uint32")
+}
+
+bool JsonReader::read(int64_t &result) const {
+   BEGIN_READ_JSON {
+      switch (json_->type()) {
+      case nlohmann::json::value_t::number_float: {
+         const auto d = json_->get<double>();
+         if (result != d) {
+            LOG(Gen, Warn,
+                "Potential loss of data converting field of type "
+                "'number_float' ({}) to int64 ({})",
+                d, result);
+         }
+         result = static_cast<int64_t>(d);
+         return true;
+      }
+      case nlohmann::json::value_t::number_integer: {
+         result = json_->get<int64_t>();
+         return true;
+      }
+      case nlohmann::json::value_t::number_unsigned: {
+         const auto ul = json_->get<uint64_t>();
+         if (NUM_WITHIN_BOUNDS(ul, int64_t)) {
+            LOG(Gen, Error,
+                "Potential loss of data converting field of type "
+                "'number_unsigned' ({}) to int64 ({})",
+                ul, result);
+            return false;
+         }
+         result = static_cast<int64_t>(ul);
+         return true;
+      }
+      case nlohmann::json::value_t::string: {
+         const auto str = json_->get<string>();
+         if (!str.empty()) {
+            result = std::stol(str);
+            return true;
+         }
+         break;
+      }
+      default:
+         return false;
+      }
+   }
+   END_READ_JSON("int64")
+}
+
+bool JsonReader::read(uint64_t &result) const {
+   BEGIN_READ_JSON {
+      switch (json_->type()) {
+      case nlohmann::json::value_t::number_float: {
+         const auto d = json_->get<double>();
+         if (result != d) {
+            LOG(Gen, Warn,
+                "Potential loss of data converting field of type "
+                "'number_float' ({}) to uint64 ({})",
+                d, result);
+         }
+         result = static_cast<uint64_t>(d);
+         return true;
+      }
+      case nlohmann::json::value_t::number_integer: {
+         const auto il = json_->get<int64_t>();
+         if (NUM_WITHIN_BOUNDS(il, uint64_t)) {
+            LOG(Gen, Error,
+                "Potential loss of data converting field of type "
+                "'number_integer' ({}) to uint64 ({})",
+                il, result);
+            return false;
+         }
+         result = static_cast<uint64_t>(il);
+         return true;
+      }
+      case nlohmann::json::value_t::number_unsigned: {
+         result = json_->get<uint64_t>();
+         return true;
+      }
+      case nlohmann::json::value_t::string: {
+         const auto str = json_->get<string>();
+         if (!str.empty()) {
+            result = std::stoul(str);
+            return true;
+         }
+         break;
+      }
+      default:
+         return false;
+      }
+   }
+   END_READ_JSON("uint64")
+}
+
+bool JsonReader::read(float &result) const {
+   BEGIN_READ_JSON {
+      switch (json_->type()) {
+      case nlohmann::json::value_t::number_float:
+      case nlohmann::json::value_t::number_integer:
+      case nlohmann::json::value_t::number_unsigned:
+         json_->get_to(result);
+         return true;
+      case nlohmann::json::value_t::string: {
+         const auto str = json_->get<string>();
+         if (!str.empty()) {
+            result = std::stof(str);
+            return true;
+         }
+         break;
+      }
+      default:
+         return false;
+      }
+   }
+   END_READ_JSON("float")
+}
+
+bool JsonReader::read(double &result) const {
+   BEGIN_READ_JSON {
+      switch (json_->type()) {
+      case nlohmann::json::value_t::number_float:
+      case nlohmann::json::value_t::number_integer:
+      case nlohmann::json::value_t::number_unsigned:
+         json_->get_to(result);
+         return true;
+      case nlohmann::json::value_t::string: {
+         const auto str = json_->get<string>();
+         if (!str.empty()) {
+            result = std::stod(str);
+            return true;
+         }
+         break;
+      }
+      default:
+         return false;
+      }
+   }
+   END_READ_JSON("double")
+}
+
+bool JsonReader::read(string &result) const {
+   BEGIN_READ_JSON {
+      json_->get_to(result);
+      return true;
+   }
+   END_READ_JSON("string")
+}
+
+bool JsonReader::read(nlohmann::json &result) const {
+   BEGIN_READ_JSON {
+      json_->get_to(result);
+      return true;
+   }
+   END_READ_JSON("json-object")
+}
+
+bool JsonReader::read_flat(std::vector<nlohmann::json> &result) const {
+   if (json_ == nullptr) {
+      return false;
+   }
+   switch (json_->type()) {
+   case json::value_t::array:
+      return std::all_of(json_->begin(), json_->end(), [&](const auto &j) {
+         return JsonReader{&j}.read_flat(result);
+      });
+   case json::value_t::null:
+      return false;
+   default:
+      read(result.emplace_back());
+      return true;
+   }
+}
+
+bool JsonReader::parse_time_stamp_(const string &str, TimeStamp &value) const {
+   return parse_iso_time_stamp_(str, value) ||
+          parse_epoch_nanos_time_stamp_(str, value);
+}
+
+bool JsonReader::parse_iso_time_stamp_(
+    const string &str, TimeStamp &value) const {
+   value = TimeStamp::from_iso_string(str);
+   return true;
+}
+
+bool JsonReader::parse_epoch_nanos_time_stamp_(
+    const string &str, TimeStamp &value) const {
+   uint64_t nanos = stoull(str);
+   value = TimeStamp{nanos};
+   return true;
+}
+
+bool JsonReader::read(TimeStamp &result) const {
+   BEGIN_READ_JSON {
+      switch (json_->type()) {
+      case nlohmann::json::value_t::null:
+      case nlohmann::json::value_t::object:
+      case nlohmann::json::value_t::array:
+      case nlohmann::json::value_t::boolean:
+      case nlohmann::json::value_t::binary:
+      case nlohmann::json::value_t::discarded:
+         return false;
+      case nlohmann::json::value_t::string: {
+         const auto str = json_->get<string>();
+         return parse_iso_time_stamp_(str, result) ||
+                parse_epoch_nanos_time_stamp_(str, result);
+      }
+      case nlohmann::json::value_t::number_integer:
+      case nlohmann::json::value_t::number_unsigned:
+      case nlohmann::json::value_t::number_float:
+         // Maybe we could assume seconds here, idk though...when in doubt, prob
+         // better to be cautious than cavalier
+         return false;
+      }
+   }
+   END_READ_JSON("TimeStamp")
+}
+
+bool JsonReader::read_iso(
+    const std::string &name, core::TimeStamp &result) const {
+   return JsonReader{json_, name}.read_iso(result);
+}
+
+bool JsonReader::read_iso(core::TimeStamp &result) const {
+   BEGIN_READ_JSON {
+      switch (json_->type()) {
+      case nlohmann::json::value_t::string: {
+         return parse_iso_time_stamp_(json_->get<string>(), result);
+      }
+      default:
+         return false;
+      }
+   }
+   END_READ_JSON("TimeStamp")
+}
+
+bool JsonReader::read_epoch_nanos(TimeStamp &value) const {
+   int64_t nanos;
+   if (!read(nanos)) {
+      return false;
+   }
+   value = TimeStamp::from_nanos(nanos);
+   return true;
+}
+
+bool JsonReader::read_epoch_micros(TimeStamp &value) const {
+   int64_t micros;
+   if (!read(micros)) {
+      return false;
+   }
+   value = TimeStamp::from_micros(micros);
+   return true;
+}
+
+bool JsonReader::read_epoch_millis(TimeStamp &value) const {
+   int64_t millis;
+   if (!read(millis)) {
+      return false;
+   }
+   value = TimeStamp::from_millis(millis);
+   return true;
+}
+
+bool JsonReader::read_optional_epoch_millis(
+    std::optional<TimeStamp> &result) const {
+   if (json_ == nullptr) {
+      return false;
+   }
+   if (json_->is_null()) {
+      result = std::nullopt;
+      return true;
+   }
+   return read_epoch_millis(result.emplace());
+}
+
+bool JsonReader::read_optional_epoch_millis(
+    const std::string &name, std::optional<TimeStamp> &result) const {
+   if (!json_ || !json_->is_object()) {
+      return false;
+   }
+   auto it = json_->find(name);
+   if (it == json_->end()) {
+      return false;
+   }
+   return JsonReader{&*it}.read_epoch_millis(result.emplace());
+}
+
+} // namespace bts::io

src/lib/io/JsonReader.h

@@ -0,0 +1,291 @@
+#pragma once
+#include "lib/core/TimeStamp.h"
+#include "lib/utils/bts_assert.h"
+
+#include <cstdint>
+#include <nlohmann/json.hpp>
+#include <optional>
+#include <string>
+
+namespace bts::io {
+
+class JsonReader {
+ public:
+   explicit JsonReader(const nlohmann::json *json);
+
+   JsonReader(const nlohmann::json *json, const std::string &name);
+
+   static bool try_parse(
+       const std::string &str,
+       nlohmann::json &result,
+       std::string *err = nullptr);
+
+   bool read(const std::string &name, std::string &result) const;
+   bool read(const std::string &name, std::int32_t &result) const;
+   bool read(const std::string &name, std::uint32_t &result) const;
+   bool read(const std::string &name, std::int64_t &result) const;
+   bool read(const std::string &name, std::uint64_t &result) const;
+   bool read(const std::string &name, float &result) const;
+   bool read(const std::string &name, double &result) const;
+   bool read(const std::string &name, nlohmann::json &result) const;
+   bool read(const std::string &name, core::TimeStamp &result) const;
+   bool read_iso(const std::string &name, core::TimeStamp &result) const;
+   bool read_epoch_nanos(
+       const std::string &name, core::TimeStamp &result) const;
+   bool read_epoch_micros(
+       const std::string &name, core::TimeStamp &result) const;
+   bool read_epoch_millis(
+       const std::string &name, core::TimeStamp &result) const;
+
+   template <class T>
+   bool read(const std::string &name, T &result) const;
+
+   template <class T>
+   bool read_one_or_more(
+       const std::string &name,
+       std::vector<T> &result,
+       bool append = true) const;
+
+   template <class T>
+   T read(const std::string &name) const;
+
+   bool read(std::string &result) const;
+   bool read(std::int32_t &result) const;
+   bool read(std::uint32_t &result) const;
+   bool read(std::int64_t &result) const;
+   bool read(std::uint64_t &result) const;
+   bool read(float &result) const;
+   bool read(double &result) const;
+   bool read(nlohmann::json &result) const;
+   bool read(core::TimeStamp &result) const;
+   bool read_iso(core::TimeStamp &result) const;
+   bool read_epoch_nanos(core::TimeStamp &result) const;
+   bool read_epoch_micros(core::TimeStamp &result) const;
+   bool read_epoch_millis(core::TimeStamp &result) const;
+
+   template <class T>
+   bool read_one_or_more(std::vector<T> &result, bool append = true) const;
+
+   template <class T>
+   static bool read_flat(
+       const std::vector<nlohmann::json> &items, std::vector<T> &result);
+
+   bool read_flat(std::vector<nlohmann::json> &result) const;
+
+   template <class T>
+   bool read_optional(std::optional<T> &result) const;
+
+   template <class T>
+   bool read_optional(const std::string &name, std::optional<T> &result) const;
+
+   bool read_optional_epoch_millis(
+       std::optional<core::TimeStamp> &result) const;
+   bool read_optional_epoch_millis(
+       const std::string &name, std::optional<core::TimeStamp> &result) const;
+
+   template <class T>
+   bool read(T &result) const;
+
+   template <class T>
+   T read() const;
+
+   /**
+    * @brief Reads a json field into \p result
+    *
+    * @details If the read fails, assigns \p default_value to \p result
+    * @return True if the read succeeds, otherwise false
+    */
+   template <class T>
+   bool read_or(const std::string &name, T &result, T default_value) const;
+
+ private:
+   const nlohmann::json *json_;
+
+   bool parse_time_stamp_(
+       const std::string &str, core::TimeStamp &result) const;
+   bool parse_iso_time_stamp_(
+       const std::string &str, core::TimeStamp &result) const;
+   bool parse_epoch_nanos_time_stamp_(
+       const std::string &str, core::TimeStamp &result) const;
+};
+
+//////////////////////////////////////////////////
+// Definition
+//////////////////////////////////////////////////
+
+#define CATCH_PARSE_ERRORS(type_name)                                          \
+   catch (const std::exception &e) {                                           \
+      LOG(Gen, Diag, "Failed to parse json to '{}'. Exception: {}",            \
+          std::string{type_name}, std::string{e.what()});                      \
+   }                                                                           \
+   catch (...) {                                                               \
+      bts_assert_msgf(                                                         \
+          false, "Failed to parse json to '{}'. Exception: unknown",           \
+          std::string{type_name});                                             \
+   }
+
+template <class T>
+inline bool JsonReader::read(const std::string &name, T &result) const {
+   if (!json_ || !json_->is_object()) {
+      return false;
+   }
+   auto it = json_->find(name);
+   if (it == json_->end()) {
+      return false;
+   }
+   it->get_to(result);
+   return true;
+}
+
+template <class T>
+inline bool JsonReader::read_one_or_more(
+    const std::string &name, std::vector<T> &result, bool append) const {
+   if (!json_ || !json_->is_object()) {
+      return false;
+   }
+   auto it = json_->find(name);
+   if (it == json_->end()) {
+      return false;
+   }
+   try {
+      switch (it->type()) {
+      case nlohmann::json::value_t::array:
+         if (append) {
+            for (const auto &item : it.value()) {
+               item.get_to(result.emplace_back());
+            }
+         } else {
+            it.value().get_to(result);
+         }
+         return true;
+      default:
+         if (append) {
+            it.value().get_to(result.emplace_back());
+         } else {
+            if (result.empty()) {
+               result.emplace_back();
+            }
+            it.value().get_to(result.at(0));
+            result.erase(result.begin() + 1, result.end());
+         }
+         return true;
+      }
+   }
+   CATCH_PARSE_ERRORS(typeid(T).name())
+   return false;
+}
+
+template <class T>
+inline bool JsonReader::read_one_or_more(
+    std::vector<T> &result, bool append) const {
+   if (!json_) {
+      return false;
+   }
+   try {
+      switch (json_->type()) {
+      case nlohmann::json::value_t::array:
+         if (append) {
+            for (const auto &item : json_->array()) {
+               item.get_to(result.emplace_back());
+            }
+         } else {
+            json_->get_to(result);
+         }
+         return true;
+      default:
+         if (append) {
+            json_->get_to(result.emplace_back());
+         } else {
+            if (result.empty()) {
+               result.emplace_back();
+            }
+            json_->get_to(result.at(0));
+            result.erase(result.begin() + 1, result.end());
+         }
+         return true;
+      }
+   }
+   CATCH_PARSE_ERRORS(typeid(T).name());
+   return false;
+}
+
+template <class T>
+inline bool JsonReader::read(T &result) const {
+   try {
+      json_->get_to(result);
+      return true;
+   } catch (...) {
+      const auto j_str = json_->dump(2);
+      LOG(md, Error, "Exception parsing json:\n{}", j_str);
+      return false;
+   }
+}
+
+template <class T>
+inline T JsonReader::read() const {
+   T result;
+   read(result);
+   return result;
+}
+
+template <class T>
+inline bool JsonReader::read_optional(std::optional<T> &result) const {
+   if (json_ == nullptr) {
+      return false;
+   }
+   if (json_->is_null()) {
+      result = std::nullopt;
+      return true;
+   }
+   return read(result.emplace());
+}
+
+template <class T>
+inline bool JsonReader::read_optional(
+    const std::string &name, std::optional<T> &result) const {
+   if (!json_ || !json_->is_object()) {
+      return false;
+   }
+   auto it = json_->find(name);
+   if (it == json_->end()) {
+      return false;
+   }
+   return JsonReader{&it.value()}.read_optional(result);
+}
+
+template <class T>
+inline T JsonReader::read(const std::string &name) const {
+   bts_assert(json_ && json_->is_object());
+   auto it = json_->find(name);
+   bts_assert_msgf(it != json_->end(), "json doesn't contain field '{}'", name);
+   return JsonReader{&it.value()}.read<T>();
+}
+
+template <class T>
+inline bool JsonReader::read_or(
+    const std::string &name, T &result, T default_value) const {
+   if (read(name, result)) {
+      return true;
+   }
+   result = default_value;
+   return false;
+}
+
+template <class T>
+bool JsonReader::read_flat(
+    const std::vector<nlohmann::json> &items, std::vector<T> &result) {
+   std::vector<nlohmann::json> flat_js;
+   for (const auto &item_j : items) {
+      if (!JsonReader{&item_j}.read_flat(flat_js)) {
+         return false;
+      }
+   }
+   for (const auto &el_j : flat_js) {
+      JsonReader{&el_j}.read(result.emplace_back());
+   }
+   return true;
+}
+
+#undef CATCH_PARSE_ERRORS
+
+} // namespace bts::io

src/lib/io/websocket/WebsocketSSLClientInternal.cpp

@@ -89,7 +89,7 @@ void WebsocketSSLClientInternal::_handle_error(
    str << "Error_code:" << error_code << " Text:" << err_msg;
    LOG(OrderMan, Info,
        "WebsocketSSLClientInternal::_handle_error error_code:{} err_msg:{}",
-       error_code, err_msg);
+       error_code.message(), err_msg);
    _listener->on_error({err_type, str.str()});
    close_connection();
 }
@@ -131,7 +131,8 @@ void WebsocketSSLClientInternal::_on_connect(
       return;
    }
    LOG(OrderMan, Info,
-       "WebsocketSSLClientInternal::_on_connect successfull ep:{}", ep);
+       "WebsocketSSLClientInternal::_on_connect successfull ep: {}:{}",
+       ep.address().to_string(), ep.port());
    ip_port = _conf.ip + ':' + std::to_string(ep.port());
    // Set a timeout on the operation
    boost::beast::get_lowest_layer(_ws).expires_after(std::chrono::seconds(30));

src/lib/logging/MsgPacking.h

@@ -4,6 +4,7 @@
 #include <fmt/ostream.h>
 #include <iostream>
 #include <type_traits>
+#include <vector>
 
 namespace bts {
 namespace log {
@@ -12,7 +13,12 @@ template <class Tuple, std::size_t... Is>
 std::string format_tuple(
     const char *msg_template, Tuple const &t, std::index_sequence<Is...>) {
    try {
-      return fmt::format(msg_template, std::get<Is>(t)...);
+      using ctx = fmt::format_context;
+      std::vector<fmt::basic_format_arg<ctx>> fmt_args;
+      (fmt_args.push_back(fmt::detail::make_arg<ctx>(std::get<Is>(t))), ...);
+      return fmt::vformat(
+          msg_template,
+          fmt::basic_format_args<ctx>(fmt_args.data(), fmt_args.size()));
    } catch (fmt::format_error const &e) {
       return fmt::format(
           "Bad log message template: '{}' Error: '{}'", msg_template, e.what());

src/lib/logging/tests/logging_tests.cpp

@@ -1,8 +1,11 @@
 #include "catch/catch.hpp"
 #include "lib/config/config.h"
 #include "lib/config/protos/logging.pb.h"
-#include "lib/utils/bts_chrono.h"
 #include "lib/logging/logging.h"
+#include "lib/utils/bts_chrono.h"
+
+#include <fmt/format.h>
+#include <fmt/chrono.h>
 
 namespace bts {
 int logging_tests;
@@ -22,7 +25,8 @@ TEST_CASE("Logging Tests", "[logging] [benchmark]") {
    SECTION("timestamp_interpolation") {
       auto start = bts::Timer::now();
       for (int i = 0; i < sz; i++)
-         LOG(Gen, Info, "Hello World. It is now: {0}", Timer::now());
+         LOG(Gen, Info, "Hello World. It is now: {}",
+             Timer::now().time_since_epoch());
       auto stop = Timer::now();
       LOG(Testing, Info, "Logging: ns/msg for log with timestamps: {} tt: {} ",
           bts::ns_diff(stop, start) / sz, bts::ns_diff(stop, start));

src/lib/md/MBOProcessor.h

@@ -2,6 +2,7 @@
 #include "ExchangeOrderKey.h"
 #include "IntPrice.h"
 #include "MDProducerHandle.h"
+#include "lib/core/FMT_IMPL.h"
 #include "lib/domain_model/Security.h"
 #include "lib/domain_model/SecurityKey.h"
 #include "lib/utils/TimeStamp.h"
@@ -169,3 +170,5 @@ class MBOProcessor {
 namespace std {
 std::ostream &operator<<(std::ostream &os, const bts::md::inc::Update &);
 }
+
+FMT_IMPL_OSTREAM(bts::md::inc::Update);

src/lib/md/MarketDataError.h

@@ -1,5 +1,9 @@
 #pragma once
+
+#include "lib/core/FMT_IMPL.h"
+
 #include <ostream>
+#include <sstream>
 #include <string>
 
 namespace bts::md {
@@ -30,3 +34,5 @@ constexpr const char *MarketDataError::type_name() const noexcept {
 }
 
 } // namespace bts::md
+
+FMT_IMPL_OSTREAM(bts::md::MarketDataError);

src/lib/md/MarketDataListenerID.h

@@ -1,4 +1,7 @@
 #pragma once
+#ifndef _WIN32
+
+#include "lib/core/FMT_IMPL.h"
 #include "lib/core/ValueWrapper.h"
 
 namespace bts::md {
@@ -7,3 +10,6 @@ using MarketDataListenerID =
     core::ValueWrapper<struct MarketDataListenedIDTag, std::uint32_t>;
 
 } // namespace bts::md
+
+FMT_IMPL(bts::md::MarketDataListenerID, id, "{}", *id);
+#endif

src/lib/md/MarketDataRegistrationError.h

@@ -1,4 +1,6 @@
 #pragma once
+#ifndef _WIN32
+
 #include <optional>
 
 namespace bts::md {
@@ -22,3 +24,4 @@ struct MarketDataRegstrationError {
 };
 
 } // namespace bts::md
+#endif

src/lib/md/MarketDataRegistrationFactory.cpp

@@ -1,3 +1,4 @@
+#ifndef _WIN32
 #include "MarketDataRegistrationFactory.h"
 #include "MarketDataRegistration.h"
 
@@ -8,3 +9,4 @@ MarketDataRegistration MarketDataRegistrationFactory::create() {
 }
 
 } // namespace bts::md
+#endif

src/lib/md/MarketDataRegistrationFactory.h

@@ -1,4 +1,5 @@
 #pragma once
+#ifndef _WIN32
 #include <atomic>
 #include <cstdint>
 
@@ -15,3 +16,4 @@ class MarketDataRegistrationFactory {
 };
 
 } // namespace bts::md
+#endif

src/lib/md/Shm/FixedSecurity.h

@@ -1,6 +1,7 @@
 #pragma once
 
 #include "../common.h"
+#include "lib/core/FMT_IMPL.h"
 #include "lib/utils/ValueWrapper.h"
 #include "lib/utils/bts_array_size.h"
 #include "lib/utils/bts_assert.h"
@@ -284,3 +285,5 @@ void with_security_like(const api::Security &security, C cb) {
 
 } // namespace md
 } // namespace bts
+
+FMT_IMPL(bts::md::SecurityKey, k, "{}", *k);

src/lib/md/SyntheticSecurityService.cpp

@@ -152,7 +152,7 @@ void SyntheticSecurityService::create_synthetic(
                        } else {
                           LOG(Api, Info,
                               "Failed to un-delete security {}. Error {}", id,
-                              result.error());
+                              UpdateResult_Error_Name(result.error()));
                           send_reply(
                               reply,
                               CreateResult::ERROR_SECURITY_ALREADY_EXISTS, id);

src/lib/md/vendor/OnixsUtils.cpp

@@ -58,7 +58,7 @@ OnixsTemplLicense onixs_create_temporary_license() {
          LOG(md, Warn,
              "Licensing error. Failed to write to '{}'. Please ensure it is "
              "writeable.",
-             linfo.path);
+             linfo.path.string());
       }
    }
    return linfo;

src/lib/mercury/broker/Broker.cpp

@@ -369,14 +369,13 @@ void Broker::notify(
    if (srec->is_local() || srec->sub_id == ex_subid) {
 
       LOG(Latency, Diag, "Started handling event {} on broker {}",
-          log::CharPtrWrapper{srec->topic_str}, me->my_id.value());
+          std::string{srec->topic_str}, me->my_id);
       auto start = Timer::now();
       srec->notify(payload);
       auto stop = Timer::now();
 
       LOG(Latency, Diag, "Finished handling event {} on broker {}. Time: {}us",
-          log::CharPtrWrapper{srec->topic_str}, me->my_id.value(),
-          us_diff(stop, start));
+          std::string{srec->topic_str}, me->my_id, us_diff(stop, start));
    }
 }
 

src/lib/mercury/common/mercury_common.h

@@ -1,5 +1,6 @@
 #pragma once
 
+#include "lib/core/FMT_IMPL.h"
 #include "lib/mercury/common/FatWrapper.h"
 #include "lib/utils/ValueWrapper.h"
 #include <array>
@@ -30,3 +31,6 @@ using TimeStampStr = utils::ValueWrapper<
 
 } // namespace mercury
 } // namespace bts
+
+FMT_IMPL(bts::mercury::BrokerID, id, "{}", *id);
+FMT_IMPL(bts::mercury::LocalSubID, id, "{}", *id);

src/lib/mercury/topics/Topic.h

@@ -1,8 +1,10 @@
 #pragma once
 
+#include "lib/core/FMT_IMPL.h"
 #include "lib/mercury/events/Event.h"
 #include "lib/mercury/topics/ITopic.h"
 #include "lib/utils/type_name.h"
+
 #include <fmt/format.h>
 #include <memory>
 
@@ -52,3 +54,13 @@ template <class EV> class Topic<EV, false> : public ITopic {
 
 } // namespace mercury
 } // namespace bts
+
+#define _TOPIC(...) bts::mercury::Topic<__VA_ARGS__>
+
+template <class EV>
+FMT_IMPL_T(_TOPIC(EV, true), t, "{}", t.to_str());
+
+template <class EV>
+FMT_IMPL_T(_TOPIC(EV, false), t, "{}", t.to_str());
+
+#undef _TOPIC

src/lib/orderman/CancelOrderMsg.h

@@ -1,7 +1,11 @@
 #pragma once
+
 #include "lib/api/services/OrderService.pb.h"
+#include "lib/core/FMT_IMPL.h"
 #include "lib/orderman/orderman_utils/orderman_common.h"
 
+#include <sstream>
+
 namespace bts::orderman {
 
 struct CancelOrderMsg {
@@ -14,3 +18,5 @@ struct CancelOrderMsg {
 };
 
 } // namespace bts::orderman
+
+FMT_IMPL_OSTREAM(bts::orderman::CancelOrderMsg);

src/lib/orderman/CancelReplaceOrderMsg.h

@@ -1,7 +1,11 @@
 #pragma once
+
+#include "lib/core/FMT_IMPL.h"
 #include "lib/orderman/CreateOrderMsg.h"
 #include "lib/orderman/orderman_utils/orderman_common.h"
 
+#include <sstream>
+
 namespace bts::orderman {
 
 struct CancelReplaceOrderMsg {
@@ -10,3 +14,5 @@ struct CancelReplaceOrderMsg {
 };
 
 } // namespace bts::orderman
+
+FMT_IMPL_OSTREAM(bts::orderman::CancelReplaceOrderMsg);

src/lib/orderman/CreateOrderMsg.h

@@ -1,6 +1,12 @@
 #pragma once
+
+#include "lib/core/FMT_IMPL.h"
+#include "lib/domain_model/Security.h"
+#include "lib/domain_model/common.h"
 #include "lib/orderman/orderman_utils/orderman_common.h"
 
+#include <sstream>
+
 namespace bts::orderman {
 
 struct CreateOrderMsg {
@@ -18,3 +24,5 @@ struct CreateOrderMsg {
 };
 
 } // namespace bts::orderman
+
+FMT_IMPL_OSTREAM(bts::orderman::CreateOrderMsg);

src/lib/orderman/OrderMsg.h

@@ -1,9 +1,12 @@
 #pragma once
+
 #include "lib/domain_model/Security.pb.h"
 #include "lib/orderman/CancelOrderMsg.h"
 #include "lib/orderman/CancelReplaceOrderMsg.h"
 #include "lib/orderman/CreateOrderMsg.h"
 #include "lib/orderman/RFQMsg.h"
+#include "lib/utils/overloaded.h"
+
 #include <variant>
 
 namespace bts::orderman {
@@ -32,3 +35,32 @@ inline std::ostream &operator<<(std::ostream &out, OrderMsg const &msg) {
    return out;
 }
 } // namespace bts::orderman
+
+template <>
+struct fmt ::formatter<bts ::orderman ::OrderMsg> {
+   constexpr auto parse(fmt ::format_parse_context &ctx)
+       -> decltype(ctx.end()) {
+      return ctx.end();
+   }
+   template <typename FormatContext>
+   auto format(const bts ::orderman ::OrderMsg &m, FormatContext &ctx) const
+       -> decltype(ctx.out()) {
+      std::visit(
+          bts::overloaded{
+              [&ctx](const bts::orderman::CreateOrderMsg &m) {
+                 fmt::format_to(ctx.out(), "{}", m);
+              },
+              [&ctx](const bts::orderman::CancelReplaceOrderMsg &m) {
+                 fmt::format_to(ctx.out(), "{}", m);
+              },
+              [&ctx](const bts::orderman::CancelOrderMsg &m) {
+                 fmt::format_to(ctx.out(), "{}", m);
+              },
+              [&ctx](const bts::orderman::RFQMsg &r) {
+                 fmt::format_to(ctx.out(), "{}", r);
+              },
+              []() {}},
+          m);
+      return fmt::format_to(ctx.out(), "{}", m);
+   }
+};

src/lib/orderman/RFQMsg.h

@@ -1,8 +1,11 @@
 #pragma once
+
 #include "lib/api/services/OrderService.pb.h"
+#include "lib/core/FMT_IMPL.h"
 #include "lib/domain_model/Security.pb.h"
 #include "lib/orderman/events/RFQEvent.h"
 #include "lib/orderman/orderman_utils/orderman_common.h"
+
 #include <optional>
 
 namespace bts::orderman {
@@ -13,3 +16,5 @@ struct RFQMsg {
 };
 
 } // namespace bts::orderman
+
+FMT_IMPL(bts::orderman::RFQMsg, m, "{}", m.rfq.DebugString());

src/lib/orderman/actors/order_manager/OrderManager.cpp

@@ -642,7 +642,7 @@ void OrderManager::after_event() {
             LOG(Latency, Info,
                 "entry_type:after_event channel:{} sn:{} app_arrival:{} now:{} "
                 "delta:{}",
-                log::CharPtrWrapper{corid.channel}, corid.sn,
+                std::string{corid.channel}, corid.sn,
                 corid.app_arrival.to_uint64(), now.to_uint64(),
                 (now - corid.app_arrival).nanoseconds());
             osd->gw->send_msgs(queued_ops, osd->current_trace);

src/lib/orderman/events/RFQEvent.h

@@ -1,6 +1,7 @@
 #pragma once
 
 #include "lib/api/services/OrderService.pb.h"
+#include "lib/core/FMT_IMPL.h"
 #include "lib/domain_model/Security.pb.h"
 #include "lib/mercury/events/Event.h"
 #include "lib/tracing/PerfLog.h"
@@ -20,3 +21,5 @@ class RFQEvent : public mercury::Event<RFQEvent> {
 
 } // namespace orderman
 } // namespace bts
+
+FMT_IMPL(bts::orderman::RFQEvent, e, "{}", e.rfq.DebugString());

src/lib/orderman/gateways/GLBXiLink3ExecutionGateway.cpp

@@ -4,6 +4,7 @@
 #include "lib/domain_model/Security.h"
 #include "lib/jetfix/common/OutFIX.h"
 #include "lib/jetfix/store/session_name.h"
+#include "lib/logging/logging.h"
 #include "lib/orderman/events/NewSecuritiesEvent.h"
 #include "lib/orderman/gateways/GLBXExecutionGateway.h"
 #include "lib/orderman/ilink3/IL3GW.h"
@@ -24,14 +25,6 @@ struct OrderStats {
    size_t n_cxls = 0;
 };
 
-enum class GLBXiLink3ExecutionGateway::State {
-
-   WAITING_FOR_SECURITIES,
-   PAUSED,
-   CONNECTING,
-   READY
-};
-
 struct GLBXiLink3ExecutionGateway::PImpl {
    PImpl(
        config::JetFIX::Connection const &cfg,
@@ -92,7 +85,7 @@ void GLBXiLink3ExecutionGateway::start() {
 
 void GLBXiLink3ExecutionGateway::dispatch_incoming_fix_messages() {
    if (me->started) {
-      switch (me->state) {
+      switch (me->state.value) {
       case State::WAITING_FOR_SECURITIES:
          break;
       case State::PAUSED:
@@ -181,9 +174,8 @@ void GLBXiLink3ExecutionGateway::connect(jetfix::SessionName const &sn) {
 
    me->gw->subscribe_state([this](auto gw_state) {
       LOG(iLink3, Info, "Internal gateway state change in {}. State is now {}",
-          me->gw_name.value(),
-          log::CharPtrWrapper{ilink3::state_string(gw_state)});
-      switch (me->state) {
+          me->gw_name.value(), gw_state);
+      switch (me->state.value) {
       case State::CONNECTING:
          if (gw_state == ilink3::State::READY) {
             transition(State::READY);
@@ -262,7 +254,7 @@ void GLBXiLink3ExecutionGateway::send_msgs(
          auto cancel_replace_ptr = std::get_if<CancelReplaceOrderMsg>(&m);
          if (cancel_replace_ptr) {
             LOG(Latency, Info, "Cancel Replace. CorID:{}.{} Order:{}",
-                log::CharPtrWrapper{corid.channel}, corid.sn,
+                std::string{corid.channel}, corid.sn,
                 cancel_replace_ptr->exchange_order_id);
 
             me->gw->send_cancel_replace(*cancel_replace_ptr, corid);
@@ -307,24 +299,9 @@ GLBXiLink3ExecutionGateway::on_exchange_open_orders() {
    return me->on_exchange_open_orders;
 }
 
-log::CharPtrWrapper GLBXiLink3ExecutionGateway::state_name(State st) {
-   switch (st) {
-   case State::WAITING_FOR_SECURITIES:
-      return {"WAITING_FOR_SECURITIES"};
-   case State::PAUSED:
-      return {"PAUSED"};
-   case State::CONNECTING:
-      return {"CONNECTING"};
-   case State::READY:
-      return {"READY"};
-   }
-   bts_assert(false);
-   return {""};
-}
-
 void GLBXiLink3ExecutionGateway::transition(State new_state) {
    LOG(iLink3, Info, "Transitioning {} from {} to {}", me->gw_name.value(),
-       state_name(me->state), state_name(new_state));
+       me->state, new_state);
    me->state = new_state;
 }
 

src/lib/orderman/gateways/GLBXiLink3ExecutionGateway.h

@@ -1,7 +1,7 @@
 #pragma once
 
 #include "IExecutionGateway.h"
-#include "lib/logging/logging.h"
+#include "lib/orderman/gateways/GLBXiLink3ExecutionGatewayState.h"
 
 namespace bts {
 namespace jetfix {
@@ -36,7 +36,7 @@ class GLBXiLink3ExecutionGateway : public IExecutionGateway {
    OnExchangeOpenOrdersState &on_exchange_open_orders() override;
 
  private:
-   enum class State;
+   using State = GLBXiLink3ExecutionGatewayState;
    struct PImpl;
    std::unique_ptr<PImpl> me;
 
@@ -44,7 +44,6 @@ class GLBXiLink3ExecutionGateway : public IExecutionGateway {
    void on_time();
    bool pause_in_effect() const;
    bool max_failures_reached() const;
-   static log::CharPtrWrapper state_name(State);
    void transition(State);
 
    void pause();

src/lib/orderman/gateways/GLBXiLink3ExecutionGatewayState.h

@@ -0,0 +1,32 @@
+#pragma once
+#ifdef SPARK_ENABLE_CCAPI
+
+#include "lib/core/EnumMacros.h"
+#include "lib/core/FMT_IMPL.h"
+
+#include <fmt/format.h>
+#include <nlohmann/json.hpp>
+#include <ostream>
+#include <string>
+
+namespace bts::orderman::ilink3 {
+
+// clang-format off
+#define FOR_EACH_GLBX_ILINK3_EXECUTION_GATEWAY_STATE(per_value)       \
+   per_value(GLBXiLink3ExecutionGatewayState, WAITING_FOR_SECURITIES) \
+   per_value(GLBXiLink3ExecutionGatewayState, PAUSED)                 \
+   per_value(GLBXiLink3ExecutionGatewayState, CONNECTING)             \
+   per_value(GLBXiLink3ExecutionGatewayState, READY)
+// clang-format on
+
+ENUM_CLASS_DEF(
+    GLBXiLink3ExecutionGatewayState,
+    FOR_EACH_GLBX_ILINK3_EXECUTION_GATEWAY_STATE)
+
+#undef FOR_EACH_GLBX_ILINK3_EXECUTION_GATEWAY_STATE
+
+} // namespace bts::orderman::ilink3
+#endif
+
+FMT_IMPL(
+    bts::orderman::ilink3::GLBXiLink3ExecutionGatewayState, s, "{}", s.name());

src/lib/orderman/ilink3/IL3GW.cpp

@@ -33,7 +33,7 @@ namespace bts {
 namespace orderman {
 namespace ilink3 {
 
-log::CharPtrWrapper wrap(const char *str) { return log::CharPtrWrapper{str}; }
+using std::string;
 
 IL3GW::IL3GW(
     IL3Connection &con,
@@ -233,11 +233,9 @@ void IL3GW::send_order(
    _current_corid = corid;
    auto now = utils::TimeStamp::now();
    LOG(Latency, Info,
-       "entry_type:send_order channel:{} sn:{} app_arrival:{} now:{} "
-       "delta:{}",
-       log::CharPtrWrapper{corid.channel}, corid.sn,
-       corid.app_arrival.to_uint64(), now.to_uint64(),
-       (now - corid.app_arrival).nanoseconds());
+       "entry_type:send_order channel:{} sn:{} app_arrival:{} now:{} delta:{}",
+       string{corid.channel}, corid.sn, corid.app_arrival.to_uint64(),
+       now.to_uint64(), (now - corid.app_arrival).nanoseconds());
    bts_assert_d(create.routing_record);
    bts_assert_d(create.security);
    bts_assert_d(create.security->has_glbx());
@@ -386,31 +384,9 @@ void IL3GW::flush(bool publish) {
       publish_session_update();
 }
 
-const char *state_string(State s) {
-   switch (s) {
-   case State::INIT:
-      return "INIT";
-   case State::CONNECTING:
-      return "CONNECTING";
-   case State::NEGOTIATING:
-      return "NEGOTIATING";
-   case State::ESTABLISHING:
-      return "ESTABLISHING";
-   case State::RETRANSMITTING:
-      return "RETRANSMITTING";
-   case State::READY:
-      return "READY";
-   case State::ERROR:
-      return "ERROR";
-   }
-   return "UNKNOWN";
-}
-log::CharPtrWrapper wstate_string(State s) { return wrap(state_string(s)); }
-
 void IL3GW::set_state(State s) {
    if (s != _x_state) {
-      LOG(iLink3, Info, "ILinkGW {}: {}->{}", _session.uuid,
-          wstate_string(_x_state), wstate_string(s));
+      LOG(iLink3, Info, "ILinkGW {}: {}->{}", _session.uuid, _x_state, s);
       _x_state = s;
       if (_on_state_change)
          _on_state_change(s);
@@ -572,8 +548,8 @@ void IL3GW::handle(const iLinkBinary::Sequence506 &msg) {
 void IL3GW::handle(const iLinkBinary::Terminate507 &msg) {
    // https://www.cmegroup.com/confluence/display/EPICSANDBOX/Terminate
    LOG(iLink3, Error, "Terminated by exchange. Code: {} Reason: {} Text: {}",
-       msg.errorCodes(), wrap(msg.reason()),
-       wrap(translate_terminate(msg.errorCodes())));
+       msg.errorCodes(), std::string{msg.reason()},
+       std::string{translate_terminate(msg.errorCodes())});
    if (msg.errorCodes() == 11 || msg.errorCodes() == 10) {
       // This indicates our sequence number is too low. We will jump
       // forward one at a time. That should eventually fix it.
@@ -607,7 +583,7 @@ void IL3GW::handle(
 }
 void IL3GW::handle(const iLinkBinary::BusinessReject521 &msg) {
    LOG(iLink3, Error, "Business Reject: {} '{}'.", msg.businessRejectReason(),
-       wrap(translate_business_reject_reason(msg.businessRejectReason())));
+       string{translate_business_reject_reason(msg.businessRejectReason())});
    handle_common(msg);
    // A business reject can de-sync the sequence number because it rejects
    // the sequence number of the rejected message. Send a sequence message
@@ -619,7 +595,7 @@ void IL3GW::handle(const iLinkBinary::ExecutionReportNew522 &msg) {
    handle_exe_report(msg);
 }
 void IL3GW::handle(const iLinkBinary::ExecutionReportReject523 &msg) {
-   LOG(iLink3, Error, "Execution report reject: {}", wrap(msg.text()));
+   LOG(iLink3, Error, "Execution report reject: {}", string{msg.text()});
    handle_exe_report(msg);
 }
 void IL3GW::handle(const iLinkBinary::ExecutionReportElimination524 &msg) {

src/lib/orderman/ilink3/IL3GW.h

@@ -6,6 +6,7 @@
 #include "TimeKeeper.h"
 #include "generated/MessageHeader.h"
 #include "lib/config/protos/jetfix.pb.h"
+#include "lib/orderman/ilink3/IL3State.h"
 #include "lib/orderman/ilink3/RawMsg.h"
 #include "lib/orderman/gateways/IExecutionGateway.h"
 #include "lib/orderman/orderman_utils/CancelReject.h"
@@ -42,16 +43,8 @@ struct Config {
        raw_message_logger;
 };
 
-enum class State {
-   INIT,
-   CONNECTING,
-   NEGOTIATING,
-   ESTABLISHING,
-   RETRANSMITTING,
-   READY,
-   ERROR
-};
-const char *state_string(State s);
+using State = IL3State;
+
 typedef std::function<void(State)> OnStateChange;
 typedef std::function<void(const ExecutionReport *)> OnExecutionReport;
 typedef std::function<void(const Session &s)> OnSessionChange;

src/lib/orderman/ilink3/IL3Reactor.cpp

@@ -130,7 +130,7 @@ struct IL3Reactor::Connection : public IL3Connection {
       LOG(Latency, Info,
           "entry_type:socket_write channel:{} sn:{} app_arrival:{} now:{} "
           "delta:{}",
-          log::CharPtrWrapper{corid.channel}, corid.sn,
+          std::string{corid.channel}, corid.sn,
           corid.app_arrival.to_uint64(), before_write.to_uint64(),
           (before_write - corid.app_arrival).nanoseconds());
 

src/lib/orderman/ilink3/IL3State.h

@@ -0,0 +1,32 @@
+#pragma once
+#ifdef SPARK_ENABLE_CCAPI
+
+#include "lib/core/EnumMacros.h"
+#include "lib/core/FMT_IMPL.h"
+
+#include <fmt/format.h>
+#include <nlohmann/json.hpp>
+#include <ostream>
+#include <string>
+
+namespace bts::orderman::ilink3 {
+
+// clang-format off
+#define FOR_EACH_IL3_STATE(per_value)                            \
+   per_value(IL3State, INIT)                                     \
+   per_value(IL3State, CONNECTING)                               \
+   per_value(IL3State, NEGOTIATING)                              \
+   per_value(IL3State, ESTABLISHING)                             \
+   per_value(IL3State, RETRANSMITTING)                           \
+   per_value(IL3State, READY)                                    \
+   per_value(IL3State, ERROR)
+// clang-format on
+
+ENUM_CLASS_DEF(IL3State, FOR_EACH_IL3_STATE)
+
+#undef FOR_EACH_IL3_STATE
+
+} // namespace bts::orderman::ilink3
+#endif
+
+FMT_IMPL(bts::orderman::ilink3::IL3State, s, "{}", s.name());

src/lib/orderman/orderman_utils/orderman_common.h

@@ -8,6 +8,7 @@
 #include <variant>
 
 #include "lib/api/services/OrderService.pb.h"
+#include "lib/core/FMT_IMPL.h"
 #include "lib/domain_model/common.h"
 #include "lib/utils/ValueWrapper.h"
 #include "lib/utils/bts_assert.h"
@@ -289,3 +290,8 @@ inline void set_update_quantity(
 
 } // namespace orderman
 } // namespace bts
+
+FMT_IMPL(bts::orderman::AccountID, id, "{}", *id);
+FMT_IMPL(bts::orderman::ExchangeOrderID64, id, "{}", *id);
+FMT_IMPL(bts::orderman::GatewayName, n, "{}", *n);
+FMT_IMPL(bts::orderman::OrderKey, k, "{}", *k);

src/lib/orderman/orders/GLBXNativeOrder.cpp

@@ -27,6 +27,8 @@ namespace orderman {
 
 using namespace api::orderman;
 
+using std::string;
+
 OrderStatus map_external_status(NativeOrderStatus);
 
 bool use_ilink3(std::string const &for_session) {
@@ -458,9 +460,8 @@ OrderOpResult GLBXNativeOrder::on_activate(
       LOG(Latency, Info,
           "entry_type:on_activate_1 channel:{} sn:{} app_arrival:{} now:{} "
           "delta:{}",
-          log::CharPtrWrapper{corid.channel}, corid.sn,
-          corid.app_arrival.to_uint64(), now.to_uint64(),
-          (now - corid.app_arrival).nanoseconds());
+          string{corid.channel}, corid.sn, corid.app_arrival.to_uint64(),
+          now.to_uint64(), (now - corid.app_arrival).nanoseconds());
 
       result = _on_activate(Warn, Info, p, q, services);
       if (result.ok()) {
@@ -468,9 +469,8 @@ OrderOpResult GLBXNativeOrder::on_activate(
          LOG(Latency, Info,
              "entry_type:on_activate_2 channel:{} sn:{} app_arrival:{} now:{} "
              "delta:{}",
-             log::CharPtrWrapper{corid.channel}, corid.sn,
-             corid.app_arrival.to_uint64(), now.to_uint64(),
-             (now - corid.app_arrival).nanoseconds());
+             string{corid.channel}, corid.sn, corid.app_arrival.to_uint64(),
+             now.to_uint64(), (now - corid.app_arrival).nanoseconds());
 
          me->encoder->send_create(*me->services, *me->session_services);
          me->services->governor(*me->session_services).record_sent();
@@ -1264,8 +1264,8 @@ bool GLBXNativeOrder::terminal_state(api::orderman::OrderStatus status) const {
    return false;
 }
 
-api::orderman::NativeOrderState &
-GLBXNativeOrder::get_native_state(api::orderman::OrderState &st) {
+api::orderman::NativeOrderState &GLBXNativeOrder::get_native_state(
+    api::orderman::OrderState &st) {
    switch (st.custom_case()) {
    case OrderState::kMarket:
       return *st.mutable_market()->mutable_native_state();

src/lib/orderman/orders/TriggerOrder.cpp

@@ -1,4 +1,5 @@
 #include "TriggerOrder.h"
+#include "lib/api/services/OrderService.h"
 #include "lib/domain_model/Security.h"
 #include "lib/logging/logging.h"
 #include "lib/orderman/OrderBuilder.h"
@@ -62,7 +63,7 @@ void TriggerOrder::on_init_new(
     const PreActivationOrderServices &pre_os) {
    LOG(OrderMan, Info, "TriggerOrder::on_init_new key:{} state:{}", key(),
        _state);
-   switch (_state) {
+   switch (_state.value) {
    case State::START:
       me->is_succ_inited = _on_init_new(order_def, pre_os);
       break;
@@ -81,7 +82,7 @@ void TriggerOrder::on_init_recovered(
     const PreActivationOrderServices &pre_os) {
    LOG(OrderMan, Info, "TriggerOrder::on_init_recovered key:{} state:{}", key(),
        _state);
-   switch (_state) {
+   switch (_state.value) {
    case State::START:
       me->is_succ_inited = _on_init_new(rec.def(), pre_os);
       // get the state from what was recovered (we will only use
@@ -109,7 +110,7 @@ TriggerOrder::on_activate(Price price, Quantity qty, OrderServices &services) {
    LOG(OrderMan, Info,
        "TriggerOrder::on_activate key:{} state:{} price:{} qty:{}", key(),
        _state, price, qty);
-   switch (_state) {
+   switch (_state.value) {
    case State::START:
       me->services = &services;
       me->state.set_price(price.value());
@@ -150,7 +151,7 @@ OrderOpResult TriggerOrder::on_update(Price price, Quantity qty) {
    LOG(OrderMan, Info,
        "TriggerOrder::on_update key:{} state:{} price:{} qty:{}", key(), _state,
        price, qty);
-   switch (_state) {
+   switch (_state.value) {
    case State::START:
       me->state.set_price(price.value());
       me->state.set_quantity(qty.value());
@@ -177,7 +178,7 @@ OrderOpResult TriggerOrder::on_cancel() {
    LOG(OrderMan, Info, "TriggerOrder::on_cancel key:{} state:{}", key(),
        _state);
    me->cancel_mode = CancelMode::external;
-   switch (_state) {
+   switch (_state.value) {
    case State::START:
       _switch_state(State::END);
       break;
@@ -200,7 +201,7 @@ OrderOpResult TriggerOrder::on_cancel() {
    return OrderOpResult{};
 }
 void TriggerOrder::_on_book_update(const BookUpdateEvent &b_event) {
-   switch (_state) {
+   switch (_state.value) {
    case State::START:
       break;
    case State::WAITING_FOR_TRIGGER:
@@ -227,7 +228,7 @@ void TriggerOrder::_on_child_order_update(const OrderStateUpdateEvent &o_ev) {
    if (o_ev.key == me->child_order.key()) {
       auto ev_state = o_ev.rebuild_state();
       me->child_state = ev_state;
-      switch (_state) {
+      switch (_state.value) {
       case State::START:
       case State::WAITING_FOR_TRIGGER:
          bts_assert(false);
@@ -251,7 +252,7 @@ void TriggerOrder::_on_child_order_update(const OrderStateUpdateEvent &o_ev) {
    }
 }
 void TriggerOrder::_on_time(const bts::mercury::TimerEvent &) {
-   switch (_state) {
+   switch (_state.value) {
    case State::START:
    case State::WAITING_FOR_TRIGGER:
    case State::WAITING_FOR_FIRST_ACK:
@@ -342,7 +343,7 @@ void TriggerOrder::_report_state() {
    }
 }
 void TriggerOrder::_switch_state(State new_state) {
-   switch (new_state) {
+   switch (new_state.value) {
    case State::START:
       break;
    case State::WAITING_FOR_TRIGGER:
@@ -415,7 +416,7 @@ void TriggerOrder::_analyze_trigger_and_cancel() {
    }
 }
 api::orderman::OrderStatus TriggerOrder::_status() const {
-   switch (_state) {
+   switch (_state.value) {
    // will only be called once the order
    // has been activated
    case State::START:
@@ -440,7 +441,7 @@ api::orderman::OrderStatus TriggerOrder::_status() const {
                   return api::orderman::OrderStatus::ORDER_STATUS_COMPLETED;
                else if (SpreaderUtils::order_in_cancelled_state(
                             me->child_state.status())) {
-                  switch (me->cancel_mode) {
+                  switch (me->cancel_mode.value) {
                   case CancelMode::none: // this means an unscolicted child cxl
                   case CancelMode::external:
                      return api::orderman::OrderStatus::ORDER_STATUS_CANCELED;

src/lib/orderman/orders/TriggerOrder.h

@@ -2,6 +2,8 @@
 
 #include "lib/api/services/OrderService.server.pb.h"
 #include "lib/orderman/orders/Order.h"
+#include "lib/orderman/orders/TriggerOrderCancelMode.h"
+#include "lib/orderman/orders/TriggerOrderState.h"
 #include <memory>
 
 namespace bts {
@@ -24,34 +26,8 @@ class PreActivationOrderServices;
 
 class TriggerOrder : public Order {
  private:
-   enum State {
-      // remains here till on_actiavte is called
-      START = 0,
-      // waiting for trigger conditions, sends
-      // order and waits to steady
-      WAITING_FOR_TRIGGER = 1,
-      // once the order is submitted it gets into
-      // this state until getting a callback
-      // from the child order
-      WAITING_FOR_FIRST_ACK = 2,
-      // order is out
-      STEADY = 3,
-      END = 4
-   };
-   // this is the ways a child order
-   // can be cancelled
-   enum CancelMode {
-      // not cancelled, initial value
-      none = 0,
-      // meaning the trigger order
-      // cancelled the child order because
-      // of time or conditional reasons
-      trigger,
-      // the trigger order is itself
-      // externally cancelled
-      // through the cancel call
-      external
-   };
+   using State = TriggerOrderState;
+   using CancelMode = TriggerOrderCancelMode;
 
  public:
    TriggerOrder();

src/lib/orderman/orders/TriggerOrderCancelMode.h

@@ -0,0 +1,34 @@
+#pragma once
+
+#include "lib/core/EnumMacros.h"
+#include "lib/core/FMT_IMPL.h"
+
+#include <fmt/format.h>
+#include <nlohmann/json.hpp>
+#include <ostream>
+#include <string>
+
+namespace bts::orderman {
+
+// clang-format off
+#define FOR_ALL_VALUES(per_value)                                            \
+   /* not cancelled, initial value */                                        \
+   per_value(TriggerOrderCancelMode, none)                                   \
+   /* meaning the trigger order cancelled the child order because of time */ \
+   /* or conditional reasons */                                              \
+   per_value(TriggerOrderCancelMode, trigger)                                \
+   /* the trigger order is itself externally cancelled through the cancel */ \
+   /* call */                                                                \
+   per_value(TriggerOrderCancelMode, external)
+// clang-format on
+
+/**
+ * @brief Represents the ways a TriggerOrder child order can be cancelled
+ */
+ENUM_CLASS_DEF(TriggerOrderCancelMode, FOR_ALL_VALUES)
+
+#undef FOR_ALL_VALUES
+
+} // namespace bts::orderman
+
+FMT_IMPL(bts::orderman::TriggerOrderCancelMode, cm, "{}", cm.name());

src/lib/orderman/orders/TriggerOrderState.h

@@ -0,0 +1,33 @@
+#pragma once
+
+#include "lib/core/EnumMacros.h"
+#include "lib/core/FMT_IMPL.h"
+
+#include <fmt/format.h>
+#include <nlohmann/json.hpp>
+#include <ostream>
+#include <string>
+
+namespace bts::orderman {
+
+// clang-format off
+#define FOR_EACH_TRIGGER_ORDER_STATE(per_value)                               \
+   /* remains here till on_actiavte is called */                              \
+   per_value(TriggerOrderState, START)                                        \
+   /* waiting for trigger conditions, sends order and waits to steady */      \
+   per_value(TriggerOrderState, WAITING_FOR_TRIGGER)                          \
+   /* once the order is submitted it gets into this state until getting a */  \
+   /* callback from the child order */                                        \
+   per_value(TriggerOrderState, WAITING_FOR_FIRST_ACK)                        \
+   /* order is out */                                                         \
+   per_value(TriggerOrderState, STEADY)                                       \
+   per_value(TriggerOrderState, END)
+// clang-format on
+
+ENUM_CLASS_DEF(TriggerOrderState, FOR_EACH_TRIGGER_ORDER_STATE)
+
+#undef FOR_EACH_TRIGGER_ORDER_STATE
+
+} // namespace bts::orderman
+
+FMT_IMPL(bts::orderman::TriggerOrderState, s, "{}", s.name());

src/lib/orderman/orders/algo/AlgoWrapperOrder.cpp

@@ -1129,7 +1129,7 @@ bool AlgoWrapperOrder::is_working() const {
    return me->status == Status::WORKING;
 }
 
-log::CharPtrWrapper AlgoWrapperOrder::state_name(Status st) {
+std::string AlgoWrapperOrder::state_name(Status st) {
    switch (st) {
    case Status::COMPLETED:
       return {"COMPLETED"};

src/lib/orderman/orders/algo/AlgoWrapperOrder.h

@@ -105,7 +105,7 @@ class AlgoWrapperOrder : public Order, private IAlgoServices {
    OrderKey index_and_launch_child(ChildOrder &&, Price, Quantity, bool);
    bool is_working() const;
    size_t working_children_count() const;
-   log::CharPtrWrapper state_name(Status);
+   std::string state_name(Status);
    ChildStatus map_child_status(api::orderman::OrderStatus);
    bool _is_attaching(api::orderman::OrderDefinition const &) const;
 };

src/lib/orderman/orders/algo/algo_orders/CoverAlgo.cpp

@@ -10,6 +10,8 @@ namespace orderman {
 
 using namespace api::orderman;
 
+using std::string;
+
 enum class CoverAlgo::Status { WORKING, WAITING_FOR_HEDGES, DONE, CANCELING };
 
 struct CoverAlgo::PImpl {
@@ -262,8 +264,7 @@ void CoverAlgo::update_definition(
 void CoverAlgo::terminate_if_hedges_complete() {
    LOG(OrderMan, Diag,
        "CoverAlgo {} check_terminate. State: {} Hedge count: {}",
-       algo().order_key(), log::CharPtrWrapper{status_name(me->status)},
-       me->hedges.size());
+       algo().order_key(), string{status_name(me->status)}, me->hedges.size());
 
    if (me->status == Status::WAITING_FOR_HEDGES && me->hedges.empty()) {
       LOG(OrderMan, Info, "CoverAlgo {}. Last hedge completed, terminating",

src/lib/orderman/orders/algo/algo_orders/ProRataAlgo.cpp

@@ -13,6 +13,8 @@ namespace orderman {
 
 using namespace api::orderman;
 
+using std::string;
+
 enum class ProRataAlgo::Status {
    WAITING_FOR_MD,
    WORKING,
@@ -373,8 +375,7 @@ void ProRataAlgo::set_parameters(
 void ProRataAlgo::check_terminate() {
    LOG(OrderMan, Diag,
        "ProRataAlgo {} check_terminate. State: {} Hedge count: {}",
-       algo().order_key(), log::CharPtrWrapper{status_name(me->status)},
-       me->hedges.size());
+       algo().order_key(), string{status_name(me->status)}, me->hedges.size());
 
    if (me->status == Status::DONE_WORKING && me->hedges.empty() &&
        me->overfill_hedges.empty()) {

src/lib/orderman/orders/algo/algo_orders/RegisterDorman.cpp

@@ -300,6 +300,9 @@ extern "C" void init_execution_tester_plugin(bts::SparkServices &srv) {
 extern "C" void init_task_eye_plugin(bts::SparkServices &srv) {
    DynamicOrderDescriptor tasking_eye_algo;
    tasking_eye_algo.set_dynamic_order_type("Tasking Eye");
+   // if this is set to true, it waits for the existance of Signal File
+   *tasking_eye_algo.add_param_desc() = param_bool("Wait For Signal");
+   *tasking_eye_algo.add_param_desc() = param_string("Signal File");
    *tasking_eye_algo.add_param_desc() = param_string("Account");
    *tasking_eye_algo.add_param_desc() = param_double("Edge");
    *tasking_eye_algo.add_param_desc() = param_double("Shot Size");