build: Upgrade (Windows) c++ to stdlib 20

fix build: Remove a few RequiresPreviewFeatures attrs
Features which are now standard in .Net
2023-12-23 13:23:52 -05:00 · 2023-12-23 13:23:33 -05:00 · 2023-12-23 13:23:29 -05:00 · 2023-12-23 13:12:30 -05:00 · 2023-12-23 13:12:30 -05:00 · 2023-12-23 13:12:30 -05:00
146 changed files with 4559 additions and 582 deletions
@@ -51,3 +51,5 @@ trace_data/
 /.venv*/*
 .cache/
 src/web/api/tsconfig.*.g.json
+**/CMakeDeps.cmake
+**/CMakeDeps_win.cmake
@@ -7,3 +7,6 @@
 [submodule "docker/windows"]
 	path = docker/windows
 	url = git@bitbucket.org:bluetradesys/windows_docker.git
+[submodule "vcpkg"]
+	path = vcpkg
+	url = https://github.com/microsoft/vcpkg.git
@@ -1,6 +1,9 @@
 cmake_minimum_required(VERSION 3.12)
 project(spark)

+include("${CMAKE_SOURCE_DIR}/vcpkg/scripts/buildsystems/vcpkg.cmake")
+set(CMAKE_CXX_STANDARD 20)
+
 if(WIN32)
  set(platform "windows")
 else()
@@ -35,7 +38,7 @@ if(WIN32)
    /D_WINSOCK_DEPRECATED_NO_WARNINGS \
    /D__TBB_NO_IMPLICIT_LINKAGE \
    /DBOOST_ALL_NO_LIB \
-    /std:c++17 \
+    /std:c++20 \
    /D_SCL_SECURE_NO_WARNINGS /D_CRT_SECURE_NO_WARNINGS /D_WIN32_WINNT=0x0600")
  set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /MTd")
  set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_DEBUG} /MT")
@@ -1 +0,0 @@
-set(sources /home/taha/spark/doc/user/build.sh;content/_index.md;content/bts_spark/Glossary/_index.md;content/bts_spark/Help/_index.md;content/bts_spark/Help/about_spark.png;content/bts_spark/Help/image39.png;content/bts_spark/Help/image58.png;content/bts_spark/Launcher/_index.md;content/bts_spark/Launcher/image38.png;content/bts_spark/Launcher/image68.png;content/bts_spark/Logging-In-to-the-System/_index.md;content/bts_spark/Logging-In-to-the-System/image3.png;content/bts_spark/Logging-In-to-the-System/image64.png;content/bts_spark/Market/_index.md;content/bts_spark/Market/change_securities.png;content/bts_spark/Market/image11.png;content/bts_spark/Market/image14.png;content/bts_spark/Market/image20.png;content/bts_spark/Market/image21.png;content/bts_spark/Market/image23.png;content/bts_spark/Market/image24.png;content/bts_spark/Market/image34.png;content/bts_spark/Market/image49.png;content/bts_spark/Market/image53.png;content/bts_spark/Market/image66.png;content/bts_spark/Market/image69.png;content/bts_spark/Market/market_view.png;content/bts_spark/Market/settings_icon.png;content/bts_spark/Market/time_and_sales.png;content/bts_spark/My-Windows/_index.md;content/bts_spark/My-Windows/image15.png;content/bts_spark/My-Windows/image16.png;content/bts_spark/My-Windows/image37.png;content/bts_spark/Settings/_index.md;content/bts_spark/Settings/image51.png;content/bts_spark/Settings/image9.png;content/bts_spark/Spreader/_index.md;content/bts_spark/Spreader/image10.png;content/bts_spark/Spreader/image13.png;content/bts_spark/Spreader/image18.png;content/bts_spark/Spreader/image19.png;content/bts_spark/Spreader/image28.png;content/bts_spark/Spreader/image29.png;content/bts_spark/Spreader/image31.png;content/bts_spark/Spreader/image35.png;content/bts_spark/Spreader/image40.png;content/bts_spark/Spreader/image43.png;content/bts_spark/Spreader/image54.png;content/bts_spark/Spreader/image55.png;content/bts_spark/Spreader/image56.png;content/bts_spark/Spreader/image6.png;content/bts_spark/Spreader/image7.png;content/bts_spark/Spreader/image70.png;content/bts_spark/Spreader/image72.png;content/bts_spark/Spreader/image8.png;content/bts_spark/Trading/OrderTypes/index.md;content/bts_spark/Trading/_index.md;content/bts_spark/Trading/algo_studio.png;content/bts_spark/Trading/collapse_all_icon.png;content/bts_spark/Trading/cover_algo_control.png;content/bts_spark/Trading/expand_all_icon.png;content/bts_spark/Trading/image1.png;content/bts_spark/Trading/image17.png;content/bts_spark/Trading/image2.png;content/bts_spark/Trading/image25.png;content/bts_spark/Trading/image27.png;content/bts_spark/Trading/image33.png;content/bts_spark/Trading/image42.png;content/bts_spark/Trading/image44.png;content/bts_spark/Trading/image47.png;content/bts_spark/Trading/image48.png;content/bts_spark/Trading/image5.png;content/bts_spark/Trading/image50.png;content/bts_spark/Trading/image52.png;content/bts_spark/Trading/image57.png;content/bts_spark/Trading/image59.png;content/bts_spark/Trading/image60.png;content/bts_spark/Trading/image61.png;content/bts_spark/Trading/image62.png;content/bts_spark/Trading/image73.png;content/bts_spark/Trading/image74.png;content/bts_spark/Trading/join_algo_control.png;content/bts_spark/Trading/ladder.png;content/bts_spark/Trading/ladder_badges.png;content/bts_spark/Trading/ladder_main_part.png;content/bts_spark/Trading/lean_algo_control.png;content/bts_spark/Trading/mini_orderbook.png;content/bts_spark/Trading/oco_algo_control.png;content/bts_spark/Trading/offset_algo_control.png;content/bts_spark/Trading/pnl_calculation.png;content/bts_spark/Trading/pnl_window.png;content/bts_spark/Trading/recreate_icon.png;content/bts_spark/Trading/settings_icon.png;content/bts_spark/Trading/spark_menu_order_book.png;content/bts_spark/Trading/top_algo_control.png;content/bts_spark/Trading/trigger_algo_control.png;content/bts_spark/_index.md;content/release_notes/_index.md;content/system_requirements/_index.md;static/book.js;static/favicon.ico;static/image.js;templates/index.html;templates/page.html;templates/section.html;templates/shortcodes/back_button.html)
@@ -0,0 +1,135 @@
+FROM ubuntu:latest
+
+ARG DEBIAN_FRONTEND=noninteractive
+ENV TZ=America/New_York
+
+# light deps (should be quick to download/install)
+RUN apt-get update && apt-get install -y \
+    ca-certificates \
+    curl \
+    gnupg \
+    liblttng-ust-dev \
+    libpcap0.8-dev \
+    locales \
+    ninja-build \
+    software-properties-common \
+    sqlite3 \
+    tzdata
+
+# clang
+RUN apt-get update && apt-get install -y \
+    clang-13 \
+    clang-format-13 \
+    clangd-13 \
+    clang-15 \
+    clang-format-15 \
+    clangd-15
+
+# g++
+RUN apt-get update && apt-get install -y \
+    g++-11 \
+    g++-12
+
+# python
+RUN apt-get update && apt-get install -y \
+    python3-pip \
+    python3-venv \
+    && python3 -m pip install --upgrade pip
+
+# dotnet
+RUN apt-get update && apt-get install -y \
+    dotnet7
+
+# ssh
+RUN apt-get update && apt-get install -y \
+    libssh2-1-dev \
+    openssh-server
+
+# cmake
+RUN \
+    curl -fsSL https://apt.kitware.com/keys/kitware-archive-latest.asc 2> /dev/null \
+    | gpg --dearmor - | tee /etc/apt/trusted.gpg.d/kitware.gpg > /dev/null \
+    && apt-add-repository "deb https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" \
+    && apt-get update \
+    && apt-get install -y cmake
+
+# node
+ARG node_major=18
+RUN mkdir -p /etc/apt/keyrings \
+    && curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key \
+    | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg \
+    && echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$node_major.x nodistro main" \
+    | tee /etc/apt/sources.list.d/nodesource.list \
+    && apt-get update \
+    && apt-get install -y nodejs
+
+# ccache is done here so that ccache sees clang++ installed, not clang-10
+RUN apt-get update && apt-get install -y \
+    ccache 
+
+# emumba (light deps)
+RUN apt-get update && apt-get install -y \
+    build-essential \
+    doxygen \
+    python3-sphinx \
+    graphviz \
+    libgtest-dev \
+    libssl-dev \
+    rapidjson-dev \
+    zlib1g-dev \
+    pkg-config
+
+# emumba (boost)
+RUN apt-get update && apt-get install -y \
+    libboost-filesystem-dev \
+    libboost-program-options-dev \
+    libboost-test-dev \
+    libboost-date-time-dev
+
+# dev utils
+RUN apt-get update && apt-get install -y \ 
+    fish \
+    git-all \
+    sudo \
+    vim \
+    wget
+
+# set up default executables (update-alternatives)
+RUN \
+    update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-13 100; \
+    update-alternatives --install /usr/bin/clang clang /usr/bin/clang-13 100; \
+    update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-13 100; \
+    update-alternatives --install /usr/bin/clangd clangd /usr/bin/clangd-13 100; \
+    update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-11 100; \
+    update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 100
+
+# Add compiler symlinks for ccache 
+RUN \
+    ln -s /usr/bin/ccache /usr/local/bin/clang; \
+    ln -s /usr/bin/ccache /usr/local/bin/clang++; \
+    ln -s /usr/bin/ccache /usr/local/bin/clang-13; \
+    ln -s /usr/bin/ccache /usr/local/bin/clang++-13; \
+    ln -s /usr/bin/ccache /usr/local/bin/clang-15; \
+    ln -s /usr/bin/ccache /usr/local/bin/clang++-15; \
+    ln -s /usr/bin/ccache /usr/local/bin/gcc; \
+    ln -s /usr/bin/ccache /usr/local/bin/g++; \
+    ln -s /usr/bin/ccache /usr/local/bin/gcc-11; \
+    ln -s /usr/bin/ccache /usr/local/bin/g++-11; \
+    ln -s /usr/bin/ccache /usr/local/bin/gcc-12; \
+    ln -s /usr/bin/ccache /usr/local/bin/g++-12
+
+RUN locale-gen en_US.UTF-8
+
+ARG docker_user="bts"
+ARG docker_uid="1000"
+ARG docker_pass="bts"
+
+RUN \
+    groupadd -g "$docker_uid" "$docker_user" \
+    && useradd \
+    -m \
+    -u "$docker_uid" \
+    -g "$docker_uid" \
+    -G sudo \
+    --password "$(openssl passwd -1 "$docker_pass")" \
+    "$docker_user"
@@ -0,0 +1,64 @@
+[dev containers]: https://code.visualstudio.com/docs/devcontainers/containers
+[dev containers ext]: https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-containers
+[docker repo sparkd-dev]: https://hub.docker.com/repository/docker/bluetradingsystems/sparkd-dev/general
+# Sparkd Dev Container
+## How to create a VS Code [Dev Container][dev containers]
+
+1. Make sure the **[Dev Containers][dev containers ext]** extension is installed
+1. Make sure your Docker image is up to date
+   ```
+   docker pull bluetradingsystems/sparkd-dev:latest
+   ```
+1. Copy **devcontainer.json** from this folder to **$workspaceFolder/.devcontainer/**.
+
+1. In the VS Code Command Palette, run **Dev Containers: Reopen in Container**
+
+---
+## How to create a Sparkd Container in the terminal (without VS Code)
+
+1. Make sure your Docker image is up to date
+   ```
+   docker pull bluetradingsystems/sparkd-dev:latest
+   ```
+2. Start the container
+   ```
+   # Replace <cont_name> with a name for the container. The container's
+   # "$HOME" dir will be mounted to "$HOME/.sparkd-dev/mounts/<cont_name> on
+   # the host
+   
+   ./start_docker.sh --container <cont_name> --mount-existing
+   ```
+   * *Replace <cont_name> with a name for the container*
+   * *The container's `$HOME` dir will be mounted to
+   `$HOME/.sparkd-dev/mounts<cont_name>` on the host*<br>
+   * *Run: `./start_docker.sh --help` to see additional options*
+3. *(Optional) Attach VS Code to the running container.*<br>
+Run **Dev Containers: Attach to Running Container** from the command palette,
+then select your container in the dropdown
+
+---
+## How to update the Docker image
+
+1. Change the image by modifying the Dockerfile (don't tag arbitrary
+  images on your machine and push them to our repo on Docker Hub)
+
+1. After modifying the Dockerfile, (re)build the image
+   ```
+   # Run in this dir
+   docker build -t bluetradingsystems/sparkd-dev:latest .
+   ```
+   ***Note:** Your docker login must have write permissions to push to the
+   [sparkd-dev][docker repo sparkd-dev] repository*
+
+2. Commit the changes to Dockerfile in git
+
+3. Push the new image to Docker Hub
+
+   ```
+   docker push bluetradingsystems/sparkd-dev:latest
+   ```
+   ***Note:** The image doesn't have to be on Docker Hub, we can simply build
+   it from scratch on our local machines (with the Dockerfile). However, this
+   takes time. It is significantly quicker to download a prebuilt image, so we
+   store it on a remote repository to save time when setting up a new dev
+   machine.*
@@ -0,0 +1,123 @@
+# ~/.bashrc: executed by bash(1) for non-login shells.
+# see /usr/share/doc/bash/examples/startup-files (in the package bash-doc)
+# for examples
+
+# If not running interactively, don't do anything
+case $- in
+    *i*) ;;
+      *) return;;
+esac
+
+# don't put duplicate lines or lines starting with space in the history.
+# See bash(1) for more options
+HISTCONTROL=ignoreboth
+
+# append to the history file, don't overwrite it
+shopt -s histappend
+
+# for setting history length see HISTSIZE and HISTFILESIZE in bash(1)
+HISTSIZE=1000
+HISTFILESIZE=2000
+
+# check the window size after each command and, if necessary,
+# update the values of LINES and COLUMNS.
+shopt -s checkwinsize
+
+# If set, the pattern "**" used in a pathname expansion context will
+# match all files and zero or more directories and subdirectories.
+#shopt -s globstar
+
+# make less more friendly for non-text input files, see lesspipe(1)
+[ -x /usr/bin/lesspipe ] && eval "$(SHELL=/bin/sh lesspipe)"
+
+# set variable identifying the chroot you work in (used in the prompt below)
+if [ -z "${debian_chroot:-}" ] && [ -r /etc/debian_chroot ]; then
+    debian_chroot=$(cat /etc/debian_chroot)
+fi
+
+# set a fancy prompt (non-color, unless we know we "want" color)
+case "$TERM" in
+    xterm-color|*-256color) color_prompt=yes;;
+esac
+
+# uncomment for a colored prompt, if the terminal has the capability; turned
+# off by default to not distract the user: the focus in a terminal window
+# should be on the output of commands, not on the prompt
+#force_color_prompt=yes
+
+if [ -n "$force_color_prompt" ]; then
+    if [ -x /usr/bin/tput ] && tput setaf 1 >&/dev/null; then
+	# We have color support; assume it's compliant with Ecma-48
+	# (ISO/IEC-6429). (Lack of such support is extremely rare, and such
+	# a case would tend to support setf rather than setaf.)
+	color_prompt=yes
+    else
+	color_prompt=
+    fi
+fi
+
+if [ "$color_prompt" = yes ]; then
+    PS1='${debian_chroot:+($debian_chroot)}\[\033[01;32m\]\u@\h\[\033[00m\]:\[\033[01;34m\]\w\[\033[00m\]\$ '
+else
+    PS1='${debian_chroot:+($debian_chroot)}\u@\h:\w\$ '
+fi
+unset color_prompt force_color_prompt
+
+# If this is an xterm set the title to user@host:dir
+case "$TERM" in
+xterm*|rxvt*)
+    PS1="\[\e]0;${debian_chroot:+($debian_chroot)}\u@\h: \w\a\]$PS1"
+    ;;
+*)
+    ;;
+esac
+
+# enable color support of ls and also add handy aliases
+if [ -x /usr/bin/dircolors ]; then
+    test -r ~/.dircolors && eval "$(dircolors -b ~/.dircolors)" || eval "$(dircolors -b)"
+    alias ls='ls --color=auto'
+    #alias dir='dir --color=auto'
+    #alias vdir='vdir --color=auto'
+
+    alias grep='grep --color=auto'
+    alias fgrep='fgrep --color=auto'
+    alias egrep='egrep --color=auto'
+fi
+
+# colored GCC warnings and errors
+#export GCC_COLORS='error=01;31:warning=01;35:note=01;36:caret=01;32:locus=01:quote=01'
+
+# some more ls aliases
+alias ll='ls -alF'
+alias la='ls -A'
+alias l='ls -CF'
+
+# Add an "alert" alias for long running commands.  Use like so:
+#   sleep 10; alert
+alias alert='notify-send --urgency=low -i "$([ $? = 0 ] && echo terminal || echo error)" "$(history|tail -n1|sed -e '\''s/^\s*[0-9]\+\s*//;s/[;&|]\s*alert$//'\'')"'
+
+# Alias definitions.
+# You may want to put all your additions into a separate file like
+# ~/.bash_aliases, instead of adding them here directly.
+# See /usr/share/doc/bash-doc/examples in the bash-doc package.
+
+if [ -f ~/.bash_aliases ]; then
+    . ~/.bash_aliases
+fi
+
+# enable programmable completion features (you don't need to enable
+# this, if it's already enabled in /etc/bash.bashrc and /etc/profile
+# sources /etc/bash.bashrc).
+if ! shopt -oq posix; then
+  if [ -f /usr/share/bash-completion/bash_completion ]; then
+    . /usr/share/bash-completion/bash_completion
+  elif [ -f /etc/bash_completion ]; then
+    . /etc/bash_completion
+  fi
+fi
+
+if [[ ! -v XDG_CONFIG_HOME ]]; then
+  export XDG_CONFIG_HOME="$HOME/.config"
+fi
+
+export PATH="$HOME/.local/bin:$PATH"
@@ -0,0 +1,25 @@
+if ! set -q XDG_CONFIG_HOME
+    set -g XDG_CONFIG_HOME "$HOME/.config"
+end
+set fish_config_home $XDG_CONFIG_HOME/fish
+if test -f "$fish_config_home/aliases.fish"
+    source "$fish_config_home/aliases.fish"
+end
+# enable general colors and case insensitive search in 'less'
+set -x LESS "$LESS -R -Q -I"
+set -x LC_ALL en_US.UTF-8
+# enable syntax highlighting in 'less'
+# set highlight color scheme in 'less' (fg)(bg)
+set -g LESS_TERMCAP_md (tput bold)(tput setaf 36)
+set -g LESS_TERMCAP_me (tput sgr0)
+set -g LESS_TERMCAP_so (tput setaf 232)(tput setab 49)
+
+set -q TERM
+if test $status != 0; or test $TERM = 'xterm'
+    set -g TERM 'xterm-256color'
+end
+
+# Fix impossible-to-see navy blue command text color
+set fish_color_command F8F8F2
+
+set -g PATH "$HOME/.local/bin" "$PATH"
@@ -0,0 +1,32 @@
+{
+  "image": "bluetradingsystems/sparkd-dev:latest",
+  "initializeCommand": "${localWorkspaceFolder}/docker/sparkd/dev/devcontainer_initialize.sh",
+  "hostRequirements": {
+    "memory": "4gb",
+    "storage": "6gb"
+  },
+  "postCreateCommand": "/home/bts/spark/docker/sparkd/dev/devcontainer_post_create.sh",
+  "containerUser": "bts",
+  "workspaceMount": "source=${localWorkspaceFolder},target=/home/bts/spark,type=bind",
+  "workspaceFolder": "/home/bts/spark",
+  "updateRemoteUserUID": true,
+  "shutdownAction": "none",
+  // Note: If running multiple containers on the same host, each must use a
+  // unique set of ports. Usually we just add 1000 to each port as needed. For
+  // example, if 8000-8003 is taken, go with 9000-9003, then 10000-10003, etc.
+  "runArgs": [
+    "--shm-size",
+    "2gb",
+    "-p",
+    "8000:8000",
+    "-p",
+    "8001:8001",
+    "-p",
+    "8002:8002",
+    "-p",
+    "8003:8003"
+  ],
+  "mounts": [
+    "source=${localEnv:HOME}/.sparkd-dev/mounts/sparkd-dev,target=/home/bts,type=bind"
+  ]
+}
@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+mkdir -p "$HOME/.sparkd-dev/mounts/sparkd-dev"
@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+
+this_dir="$(realpath "$(dirname "${BASH_SOURCE[0]}")")"
+repo_dir="$(cd "$this_dir" && git rev-parse --show-toplevel)"
+tools_dir="$repo_dir/tools"
+gen_tag="GENERATED BY $(basename "${BASH_SOURCE[0]}")"
+
+function copy_if_not_exist() {
+  src="$1"
+  dest="$2"
+  if [[ ! -e "$src" ]]; then
+    echo "Unable to copy $src to $dest. Source file doesn't exist"
+    exit 1
+  fi
+  if [[ ! -e "$dest" ]]; then
+    cp "$src" "$dest"
+  fi
+}
+
+# Init bashrc
+echo "Initializing bashrc"
+copy_if_not_exist "$this_dir/bashrc" "$HOME/.bashrc"
+
+# Init fish config
+echo "Initializing fish config"
+fish_cfg_dir="$HOME/.config/fish/"
+mkdir -p "$fish_cfg_dir"
+exec "$tools_dir/write_tag_block.sh" \
+  --tag "$gen_tag" \
+  --replace \
+  --in-place \
+  --file "$fish_cfg_dir/config.fish" \
+  --text "$(cat "$this_dir/config.fish")"
@@ -0,0 +1,250 @@
+#!/usr/bin/env bash
+
+function _exit() {
+  if [[ "$#" -gt 1 ]]; then
+    # When there are multiple args, the last arg is the exit code.
+    echo "${@:1:($# - 1)}"
+    exit "${@: -1}"
+  fi
+  echo "$1"
+  exit 0
+}
+
+help_text="\
+Add or update a user's uid
+
+This script is intended to be run in a Docker container as a means to obtain
+file permissions in mounted directories owned by the specified UID.
+
+Usage: set_user.sh [OPTIONS] USER UID
+
+Positionals:
+  USER               The name of the user
+
+Options:
+  --user USER        The name of the user (defaults to 'bts')
+  --uid UID          The new uid to give USER (defaults to '1000')
+  --group GROUP      The primary group to give USER (defaults to USER)
+  --gid GID          The primary gid to give USER (defaults to UID)
+  --pass PASSWORD    The password to give USER (defaults to USER, only applies
+                     if USER hasn't been created yet) 
+  --modify-existing  Update UID/GID even if USER already exists
+  -h,--help          Print this help message and exit
+
+"
+
+function _add_group() {
+  if [[ "$#" -ne 2 ]]; then
+    _exit "'_add_group' requires exactly 2 args (group, gid). Received $#." 1
+  fi
+  local group="$1"
+  local gid="$2"
+  # Check if gid already exists
+  local existing_group=""
+  existing_group="$(getent group "$gid" | cut -d : -f 1)"
+  if [[ -n "$existing_group" ]]; then
+    if [[ "$existing_group" != "$group" ]]; then
+      echo "\
+The gid '$gid' is already in use by group '$existing_group' (will not rename \
+the group)"
+    fi
+    return
+  fi
+
+  # Check if group with same name already exists
+  local existing_gid=""
+  existing_gid="$(getent group "$group" | cut -d : -f 3)"
+  if [[ -n "$existing_gid" ]]; then
+    # The group already exists. Only modify if --modify-existing was specified.
+    if [[ "$modify_existing" -eq 0 ]]; then
+      _exit "\
+The group '$group' already exists with gid '$existing_gid'.
+Specify --modify-existing to change the gid anyways." 1
+    fi
+    echo "Changing gid for group '$group' from '$existing_gid' to '$gid'"
+    groupmod -g "$gid" "$group"
+  else
+    # The group doesn't exist. Add it
+    echo "Adding group '$group:$gid'"
+    groupadd -g "$gid" "$group"
+  fi
+}
+
+function _add_user() {
+  local user="$1"
+  local uid="$2"
+  local group="$3"
+  local gid="$4"
+  local pass="$5"
+  if [[ "$#" -ne 5 ]]; then
+    _exit "'_add_user' requires exactly 5 args (user, uid, group, gid, pass). \
+Received $#." 1
+  fi
+
+  # Check if uid is being used by another user
+  local existing_user=""
+  existing_user="$(getent passwd "$uid" | cut -d : -f 1)"
+  if [[ -n "$existing_user" ]]; then
+    if [[ "$existing_user" != "$user" ]]; then
+      _exit "The uid '$uid' is already in use by user '$existing_user'." 1
+    fi
+
+    echo "The user '$user' already has the specified uid '$uid'."
+    # Modify gid if necessary
+    local existing_gid=""
+    existing_gid="$(id -g "$user" 2> /dev/null)"
+    if [[ "$existing_gid" = "$gid" ]]; then
+      echo "The user '$user' already has the specified gid '$gid'."
+      return
+    fi
+
+    if [[ "$modify_existing" -eq 0 ]]; then
+      _exit "\
+The user '$user' is associated with the primary group '$existing_gid'.
+Specify --modify-existing to change the gid anyways." 1
+    fi
+
+    echo "Changing primary group for '$user' from '$existing_gid' to '$gid'."
+    usermod -g "$gid" "$user"
+    return
+  fi
+
+  # Check if user with same name already exists
+  local existing_uid=""
+  existing_uid="$(id -u "$user" 2> /dev/null)"
+  if [[ -n "$existing_uid" ]]; then
+    if [[ "$existing_uid" = "$uid" ]]; then
+      _exit "The user '$user' already has the specified uid '$uid'" 0
+    fi
+
+    if [[ "$modify_existing" -eq 0 ]]; then
+      _exit "\
+User '$user' already exists with uid '$existing_uid'.
+Specify --modify-existing to change the uid anyways." 1
+    fi
+
+    echo "Changing uid for user '$user' from '$existing_uid' to '$uid'"
+    if ! usermod -u "$uid" -g "$gid" "$user"; then
+      _exit "usermod exited with code $?" 1
+    fi
+    return
+  fi
+
+  # The user doesn't exist. Add it
+  echo "Adding user '$user:$uid' (group='$group:$gid', password='$pass')"
+  useradd -m -u "$uid" -g "$gid" -G sudo --password "$(openssl passwd -1 "$pass")" "$user"
+}
+
+function _update_fish_dir_perms() {
+  local user="$1"
+  local uid="$2"
+  local gid="$3"
+  # When a fish shell session starts, a temp dir named '/tmp/fish.<user>' is
+  # created with the user's current uid/gid. If such a dir exists for the
+  # specified USER, make sure the uid/gid match the new values.
+  fish_dir="/tmp/fish.$user"
+  if [[ -d "$fish_dir" ]]; then
+    perms="$uid:$gid"
+    existing_perms="$(stat -c %u:%g "$fish_dir")"
+    if [[ "$existing_perms" != "$perms" ]]; then
+      echo "Changing uid:gid for dir '$fish_dir' from '$existing_perms' to '$perms'"
+      chown -R "$perms" "$fish_dir"
+    fi
+  fi
+}
+
+user=""
+group=""
+pass=""
+declare -i uid=0
+declare -i gid=0
+declare -i modify_existing=0
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --user)
+      if [[ -z "$2" ]]; then
+        _exit "USER (--user) must be non-empty" 1
+      elif [[ "$2" = "root" ]]; then
+        _exit "USER (--user) cannot be 'root'" 1
+      fi
+      user="$2"
+      shift 2
+      ;;
+    --group)
+      if [[ -z "$2" ]]; then
+        _exit "GROUP (--group) must be non-empty" 1
+      elif [[ "$2" = "root" ]]; then
+        _exit "GROUP (--group) cannot be 'root'" 1
+      fi
+      group="$2"
+      shift 2
+      ;;
+    --uid)
+      if [[ -z "$2" ]]; then
+        _exit "UID (--uid) must be non-empty" 1
+      fi
+      uid="$2"
+      shift 2
+      ;;
+    --gid)
+      if [[ -z "$2" ]]; then
+        _exit "GID (--gid) must be non-empty" 1
+      fi
+      gid="$2"
+      shift 2
+      ;;
+    --pass)
+      if [[ -z "$2" ]]; then
+        _exit "PASS (--pass) must be non-empty" 1
+      fi
+      pass="$2"
+      shift 2
+      ;;
+    --modify-existing)
+      modify_existing=1
+      shift 1
+      ;;
+    -h)
+      ;&
+    --help)
+      _exit "$help_text" 0
+      ;;
+    *)
+      if [[ -z "$1" ]]; then
+        _exit "Invalid positional argument '$1'" 1
+      elif [[ -z "$user" ]]; then
+        user="$1"
+      elif [[ -z "$uid" ]]; then
+        uid="$1"
+      else
+        _exit "Unexpected argument '$1'" 1
+      fi
+      shift 1
+      ;;
+  esac
+done
+
+if [[ -z "$user" ]]; then
+  user="bts"
+fi
+
+if [[ $uid -eq 0 ]]; then
+  uid=1000
+fi
+
+# In general, a user's primary group/gid matches the user's name/uid.
+# If group/gid args weren't specified, set them to name/uid.
+if [[ -z "$group" ]]; then
+  group="$user"
+fi
+if [[ $gid -eq 0 ]]; then
+  gid=$uid
+fi
+if [[ -z "$pass" ]]; then
+  pass="$user"
+fi
+
+_add_group "$group" "$gid"
+_add_user "$user" "$uid" "$group" "$gid" "$pass"
+_update_fish_dir_perms "$user" "$uid" "$gid"
@@ -0,0 +1,230 @@
+#!/usr/bin/env bash
+
+function _exit() {
+  if [[ "$#" -gt 1 ]]; then
+    # When there are multiple args, the last arg is the exit code.
+    echo "${@:1:($# - 1)}"
+    exit "${@: -1}"
+  fi
+  echo "$1"
+  exit 0
+}
+
+help_text="\
+Start a docker dev container
+Usage: start_docker.sh [OPTIONS] image container
+
+Positionals:
+  image             The image to instantiate
+  container         The name for the container
+
+Options:
+  -u,--user USER      The user to add in the container (defaults to 'bts')
+  -p,--pass PASS      The password for USER in the container (defaults to USER)
+  -i,--image IMAGE    The image to instantiate (defaults to 'bluetradingsystems/sparkd-dev:latest')
+  -c,--container CONT The name for the container
+  -m,--mount MOUNT    The name for the folder (outside the container) to create/mount to the home
+                      directory in the container. Mount folders are added here: $mount_prefix
+  --mount-existing    Mount the specified dir even if it already exists
+  --port <cont_port>:<host_port> OR [<cont_port>:<host_port>,...] Port mapping(s). May be
+                      specified multiple times.
+  -h,--help           Print this help message and exit
+
+"
+
+mount_prefix="$HOME/.sparkd-dev/mounts"
+declare docker_user=""
+declare docker_pass=""
+declare image_name=""
+declare cont_name=""
+declare mount_name=""
+declare -i mount_existing=0
+declare -a port_mappings=()
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    -u)
+      ;&
+    --user)
+      if [[ -z "$2" ]]; then
+        _exit "User (-u,--user) must be non-empty" 1
+      fi
+      docker_user="$2"
+      shift 2
+      ;;
+    -p)
+      ;&
+    --pass)
+      if [[ -z "$2" ]]; then
+        _exit "Password (-p,--pass) must be non-empty" 1
+      fi
+      docker_pass="$2"
+      shift 2
+      ;;
+    -i)
+      ;&
+    --image)
+      if [[ -n "$image_name" ]]; then
+        _exit "Unexpected argument '$1'" 1
+      elif [[ -z "$2" ]]; then
+        _exit "Image (-i,--image) must be non-empty" 1
+      fi
+      image_name="$2"
+      shift 2
+      ;;
+    -c)
+      ;&
+    --container)
+      if [[ -n "$cont_name" ]]; then
+        _exit "Unexpected argument '$1'" 1
+      elif [[ -z "$2" ]]; then
+        _exit "Container (-c,--container) must be non-empty" 1
+      fi
+      cont_name="$2"
+      shift 2
+      ;;
+    --port)
+      # Parse port(s) arg, and append the result to 'port_mappings'
+      # Note: port(s) may be specified in bracketed notation:
+      #   [a:b,c:d,...,y:z]
+      next_ports="$(echo "$2" | sed -e 's/\[//g' -e 's/\]//g' -e 's/,/ /g')"
+      if [[ "${#port_mappings[@]}" -gt 0 ]]; then
+        next_ports=" $next_ports"
+      fi
+      port_mappings+=("$next_ports")
+      shift 2
+      ;;
+    -m)
+      ;&
+    --mount)
+      if [[ -z "$2" ]]; then
+        echo "Mount (-m,--mount) must be non-empty"
+      fi
+      mount_name="$2"
+      shift 2
+      ;;
+    --mount-existing)
+      mount_existing=1
+      shift 1
+      ;;
+    -h)
+      ;&
+    --help)
+      _exit "$help_text" 0
+      ;;
+    *)
+      if [[ -z "$1" ]]; then
+        _exit "Invalid positional argument '$1'" "$help_text" 1
+      elif [[ -z "$image_name" ]]; then
+        image_name="$1"
+      elif [[ -z "$cont_name" ]]; then
+        cont_name="$1"
+      else
+        _exit "Unexpected argument '$1'" "$help_text" 1
+      fi
+      shift 1
+      ;;
+  esac
+done
+
+
+if [[ -z "$image_name" ]]; then
+  image_name="bluetradingsystems/sparkd-dev:latest"
+fi
+
+host_name="$(hostname)"
+if [[ -n "$cont_name" ]]; then
+  host_name="$host_name.$cont_name"
+fi
+
+if [[ -z "$mount_name" ]]; then
+  mount_name="$cont_name"
+fi
+
+if [[ -z "$mount_name" && -z "$cont_name" ]]; then
+  _exit "Please specify mount (-m,--mount) and/or container (-c,--container)" 1
+fi
+
+if [[ -z "$docker_user" ]]; then
+  docker_user="bts"
+fi
+
+if [[ "$docker_user" = "root" ]]; then
+  _exit "USER (--user) cannot be 'root'" 1
+fi
+
+cont_home="/home/$docker_user"
+home_src="$mount_prefix/$mount_name"
+home_tgt="$cont_home"
+this_dir="$(dirname "$(realpath "${BASH_SOURCE[0]}")")"
+repo_src="$(cd "$this_dir" && git rev-parse --show-toplevel)"
+repo_tgt="$cont_home/spark"
+working_dir="$cont_home"
+
+if [[ $mount_existing -eq 0 && -e "$home_src" ]]; then
+  # Don't overwrite existing dir (might be mounted in another container)
+  _exit "Mount dir '$home_src' already exists (use --mount-existing to mount \
+an existing directory)" 1
+fi
+
+mkdir -p "$home_src"
+
+if [[ "${#port_mappings[@]}" -eq 0 ]]; then
+  port_mappings=("8000:8000" "8001:8001" "8002:8002" "8003:8003")
+fi
+
+# Adapt port_mappings into args for `docker run` by prepending "-p " to each
+# item
+declare -a docker_port_args=()
+for port_mapping in "${port_mappings[@]}"; do
+  docker_port_args+=("-p $port_mapping")
+done
+
+echo "Starting container '$cont_name'
+image_name  : $image_name
+cont_name   : $cont_name
+home_src    : $home_src
+home_tgt    : $home_tgt
+repo_src    : $repo_src
+repo_tgt    : $repo_tgt
+working dir : $working_dir
+ports       : ${port_mappings[*]}
+"
+
+# DON'T wrap $docker_port_args in quotes. We want bash to split args by
+# whitespace
+docker run \
+  --net bridge \
+  -h "$host_name" \
+  -e "TZ=America/New_York" \
+  --cap-add=SYS_PTRACE \
+  --security-opt seccomp=unconfined \
+  --name "$cont_name" \
+  -v "$home_src:$home_tgt" \
+  -v "$repo_src:$repo_tgt" \
+  -w "$working_dir" \
+  ${docker_port_args[*]} \
+  --shm-size=4024m \
+  -t \
+  -i \
+  -d \
+  -e DISPLAY="$DISPLAY" \
+  "$image_name"
+
+# Add user w/ same uid and gid as the current user outside of docker, so the
+# user inside the docker has file permissions in the mounted dir.
+docker_group="$docker_user"
+docker_uid="$(id -u "$USER")"
+docker_gid="$(id -g "$USER")"
+if [[ -z "$docker_pass" ]]; then
+  docker_pass="$docker_user"
+fi
+docker exec -u root -it "$cont_name" "$repo_tgt/docker/sparkd/dev/set_user.sh" \
+--user "$docker_user" \
+--uid "$docker_uid" \
+--group "$docker_group" \
+--gid "$docker_gid" \
+--pass "$docker_pass"
+
+# Run same script VS Code runs to initialize the container
+docker exec -u "$docker_user" -it "$cont_name" "$repo_tgt/docker/sparkd/dev/devcontainer_post_create.sh"
@@ -0,0 +1,64 @@
+#!/usr/bin/env bash
+
+function print_help_and_exit() {
+  cat <<EOF
+Open a shell inside the docker container
+Usage: docker_shell.sh [OPTIONS] [CONTAINER]
+
+Options:
+  -u,--user         The user to login with in the container (defaults to the current user)
+  -c,--container    The container name (may also be specified as positional arg)
+  -h,--help         Print this help message and exit
+
+EOF
+  exit "$1"
+}
+
+
+declare docker_user=""
+declare cont_name=""
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    -u)
+      ;&
+    --user)
+      if [[ -z "$2" ]]; then
+        echo "User (-u,--user) must be non-empty"
+        print_help_and_exit 1
+      fi
+      docker_user="$2"
+      shift 2
+      ;;
+    -h)
+      ;&
+    --help)
+      print_help_and_exit 0
+      ;;
+    -c)
+      ;&
+    --container)
+      shift 1
+      ;&
+    *)
+      if [[ ! -z "$cont_name" ]]; then
+        echo "Unexpected argument '$1'"
+        print_help_and_exit 1
+      fi
+      cont_name="$1"
+      shift 1
+      ;;
+  esac
+done
+
+
+if [[ -z "$cont_name" ]]; then
+  echo "Please specify the container ([-c,--container])"
+  print_help_and_exit 1
+fi
+
+if [[ -z "$docker_user" ]]; then
+  docker_user="$USER"
+fi
+
+docker exec -u "$docker_user" -it "$cont_name" bash
@@ -0,0 +1,233 @@
+#!/usr/bin/env bash
+
+mount_prefix="$HOME/.spark-dev/mounts"
+
+function print_help_and_exit() {
+  cat <<EOF
+Start a docker dev container
+Usage: start_docker.sh [OPTIONS] image container
+
+Positionals:
+  image             The image to instantiate
+  container         The name for the container
+
+Options:
+  -u,--user USER      The user to add in the container (defaults to the current user)
+  -p,--pass PASS      The password for USER in the container (defaults to USER)
+  -i,--image IMAGE    The image to instantiate
+  -c,--container CONT The name for the container
+  -m,--mount MOUNT    The name for the folder (outside the container) to create/mount to the home
+                      directory in the container. Mount folders are added here: $mount_prefix
+  --mount-existing    Mount the specified dir even if it already exists
+  --port <cont_port>:<host_port> OR [<cont_port>:<host_port>,...] Port mapping(s). May be
+                      specified multiple times.
+  -h,--help           Print this help message and exit
+
+EOF
+  exit "$1"
+}
+
+
+declare docker_user=""
+declare docker_pass=""
+declare image_name=""
+declare cont_name=""
+declare mount_name=""
+declare -i mount_existing=0
+declare -a port_mappings=()
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    -u)
+      ;&
+    --user)
+      if [[ -z "$2" ]]; then
+        echo "User (-u,--user) must be non-empty"
+        print_help_and_exit 1
+      fi
+      docker_user="$2"
+      shift 2
+      ;;
+    -p)
+      ;&
+    --pass)
+      if [[ -z "$2" ]]; then
+        echo "Password (-p,--pass) must be non-empty"
+        print_help_and_exit 1
+      fi
+      docker_pass="$2"
+      shift 2
+      ;;
+    -i)
+      ;&
+    --image)
+      if [[ -n "$image_name" ]]; then
+        echo "Unexpected argument '$1'"
+        print_help_and_exit 1
+      elif [[ -z "$2" ]]; then
+        echo "Image (-i,--image) must be non-empty"
+        print_help_and_exit 1
+      fi
+      image_name="$2"
+      shift 2
+      ;;
+    -c)
+      ;&
+    --container)
+      if [[ -n "$cont_name" ]]; then
+        echo "Unexpected argument '$1'"
+        print_help_and_exit 1
+      elif [[ -z "$2" ]]; then
+        echo "Container (-c,--container) must be non-empty"
+        print_help_and_exit 1
+      fi
+      cont_name="$2"
+      shift 2
+      ;;
+    --port)
+      # Parse port(s) arg, and append the result to 'port_mappings'
+      # Note: port(s) may be specified in bracketed notation:
+      #   [a:b,c:d,...,y:z]
+      next_ports="$(echo "$2" | sed -e 's/\[//g' -e 's/\]//g' -e 's/,/ /g')"
+      if [[ "${#port_mappings[@]}" -gt 0 ]]; then
+        next_ports=" $next_ports"
+      fi
+      port_mappings+=("$next_ports")
+      shift 2
+      ;;
+    -m)
+      ;&
+    --mount)
+      if [[ -z "$2" ]]; then
+        echo "Mount (-m,--mount) must be non-empty"
+      fi
+      mount_name="$2"
+      shift 2
+      ;;
+    --mount-existing)
+      mount_existing=1
+      shift 1
+      ;;
+    -h)
+      ;&
+    --help)
+      print_help_and_exit 0
+      ;;
+    *)
+      if [[ -z "$1" ]]; then
+        echo "Invalid positional argument '$1'"
+        print_help_and_exit 1
+      elif [[ -z "$image_name" ]]; then
+        image_name="$1"
+      elif [[ -z "$cont_name" ]]; then
+        cont_name="$1"
+      else
+        echo "Unexpected argument '$1'"
+        print_help_and_exit 1
+      fi
+      shift 1
+      ;;
+  esac
+done
+
+
+if [[ -z "$image_name" ]]; then
+  image_name="bluetradingsystems/spark-crypto-dev"
+fi
+
+host_name="$(hostname)"
+if [[ -n "$cont_name" ]]; then
+  host_name="$host_name.$cont_name"
+fi
+
+if [[ -z "$mount_name" ]]; then
+  mount_name="$cont_name"
+fi
+
+if [[ -z "$mount_name" && -z "$cont_name" ]]; then
+  echo "Please specify mount (-m,--mount) and/or container (-c,--container)"
+  print_help_and_exit 1
+fi
+
+if [[ -z "$docker_user" ]]; then
+  docker_user="$USER"
+fi
+
+home_src=$mount_prefix/$mount_name
+home_tgt=$HOME
+repo_src=$(git rev-parse --show-superproject-working-tree)
+repo_tgt=$HOME/$(basename "$repo_src")
+working_dir=$HOME
+
+if [[ $mount_existing -eq 0 && -e "$home_src" ]]; then
+  # Don't overwrite existing dir (might be mounted in another container)
+  echo "Mount dir '$home_src' already exists (use --mount-existing to mount an existing directory)"
+  print_help_and_exit 1
+fi
+
+mkdir -p "$home_src"
+
+if [[ "${#port_mappings[@]}" -eq 0 ]]; then
+  port_mappings=("8000:8000" "8001:8001" "8002:8002" "8003:8003")
+fi
+
+# Adapt port_mappings into args for `docker run` by prepending "-p " to each
+# item
+declare -a docker_port_args=()
+for port_mapping in "${port_mappings[@]}"; do
+  docker_port_args+=("-p $port_mapping")
+done
+
+echo "Starting container '$cont_name'
+image_name  : $image_name
+cont_name   : $cont_name
+home_src    : $home_src
+home_tgt    : $home_tgt
+repo_src    : $repo_src
+repo_tgt    : $repo_tgt
+working dir : $working_dir
+ports       : ${port_mappings[*]}
+"
+
+# DON'T wrap $docker_port_args in quotes. We want bash to split args by
+# whitespace
+docker run \
+  --net bridge \
+  -h "$host_name" \
+  -e "TZ=America/New_York" \
+  --cap-add=SYS_PTRACE \
+  --security-opt seccomp=unconfined \
+  --name "$cont_name" \
+  -v "$home_src:$home_tgt" \
+  -v "$repo_src:$repo_tgt" \
+  -w "$working_dir" \
+  ${docker_port_args[*]} \
+  --shm-size=4024m \
+  -t \
+  -i \
+  -d \
+  -e DISPLAY="$DISPLAY" \
+  "$image_name" \
+  bash
+
+# Add user w/ same uid and gid as the current user outside of docker, so the
+# user inside the docker has file permissions in the mounted dir.
+docker_group="$docker_user"
+docker_user_uid="$(id -u "$USER")"
+docker_user_gid="$(id -g "$USER")"
+
+echo "Adding user in container '$cont_name'
+name : $docker_user
+uid  : $docker_user_uid
+gid  : $docker_user_gid"
+
+if [[ -z "$docker_pass" ]]; then
+  docker_pass="$docker_user"
+  echo "Giving user '$docker_user' a secure password: $docker_pass"
+fi
+
+docker exec -u root -it "$cont_name" groupadd -g "$docker_user_gid" "$docker_group"
+docker exec -u root -it "$cont_name" useradd -u "$docker_user_uid" -g "$docker_user_gid" "$docker_user"
+docker exec -u root -it "$cont_name" usermod --password "$(openssl passwd -1 "$docker_pass")" "$docker_user"
+docker exec -u root -it "$cont_name" usermod -aG sudo "$docker_user"
+docker exec -u root -it "$cont_name" "$repo_tgt/externals/src/emumba/install_emumba-deribit-connector.sh"
@@ -116,7 +116,7 @@
                  _timer.start();
                  _analytics_service.archive_theos(archived_theos);
                  _timer.stop();
-                  Log.Gen.info($"SingleVolSurfaceGateway._on_bb_data archiving theos size: " +
+                  Log.Gen.info($"SingleVolSurfaceGateway._on_bb_data archiving theos vs_id: {_volsurface_id} size: " +
                               $"{archived_theos.SecTheo.Count} time: {_timer.lastIntervalTime} msecs");
               }
            } catch (Exception ex) {
@@ -160,8 +160,8 @@ namespace mbts.spark {

      public static IEnumerable<bts.api.SecurityTuple> get_vs_sectups(
         int vs_id, ThreadedBlackboard bb) {
-         VolSurfaceUsageListImmutable vs_usage_list = bb.read_state(SparkTags.VolSurfaceUsageTag);
-         return vs_usage_list.get_sectups(vs_id);
+         var vs_usage_list = bb.read_state(SparkTags.VolSurfaceUsageTag);
+         return (vs_usage_list?.get_sectups(vs_id)).empty_if_null();
      }


@@ -242,4 +242,4 @@ namespace mbts.spark {
      }
      #endregion
   }
-}
+}
@@ -60,16 +60,18 @@ namespace mbts.spark {

      private IReadOnlyDictionary<string, OptionTheoInterpolationData>
         _filter_for_publish(IReadOnlyDictionary<string, OptionTheoInterpolationData> all_theos) {
+         // find the symbol vs vs_id, get the committed vs, see if it is
+         // owned by the current user, and only then publish it
         HashSet<string> syms = new HashSet<string>();
-         VolSurfaceUsageListImmutable usage = _bb.read_state(SparkTags.VolSurfaceUsageTag);
+         var usage = _bb.read_state(SparkTags.VolSurfaceUsageTag);
         string current_user = _bb.read_state(SparkTags.CurrentUser).name;
-         foreach (var u in usage.UsageList) {
+         foreach (var u in (usage?.UsageList).empty_if_null()) {
            VolSurfaceInput vs_input =
               _bb.read_state(SparkTags.VolSurfaceCommittedInputTag(u.VsId));
            if (vs_input != null) {
               if (vs_input.surface_settings.core_settings.vs_manager ==
                  current_user) {
-                  syms.Add(u.SymbolName);
+                  syms.Add(u.SecTup.Symbol);
               }
            }
         }
@@ -81,14 +83,16 @@ namespace mbts.spark {
         SystemContractData sc_data = _bb.read_state(SparkTags.SystemContractsTag, null);
         IReadOnlyDictionary<string, OptionTheoInterpolationData> all_theos =
            _bb.read_state(SparkTags.AllInterpGreeks, null);
-         all_theos = _filter_for_publish(all_theos);
-         // find the symbol vs vs_id, get the committed vs, see if it is
-         // owned by the current user, and only then publish it
         IReadOnlyDictionary<string, OptionDriverPriceMap> all_drivers =
            _bb.read_state(SparkTags.AllOptionDriverBBOTag, null);
         if (sc_data == null || all_theos == null || all_drivers == null)
            return;
         _timer.start();
+         all_theos = _filter_for_publish(all_theos);
+         if (all_theos.Count == 0) {
+            bts.Log.Gen.info($"InterpTheosServerPublisher._publish_data all_theos.Count == 0");
+            return;
+         }
         IDictionary<UInt32, DriverMD> driver_map = _get_drivers(all_drivers, sc_data);
         IEnumerable<Tuple<SecurityImmutable, TheoPoint>> theo_map = _get_theos(all_theos, sc_data);
         // assemble it together and send it to the server
@@ -111,6 +115,7 @@ namespace mbts.spark {
            ptp.Vega = is_call ? tp.cvega : tp.pvega;
            ptp.Theta = is_call ? tp.c_theta : tp.p_theta;
            ptp.Rho = Double.NaN;
+            ptp.Vol = tp.vol;
            ptp_list.Add(ptp);
         }
         bts.api.orderman.PrecomputedTheos server_theos = new PrecomputedTheos();
@@ -121,7 +126,7 @@ namespace mbts.spark {
         _order_service.set_precomputed_theos(server_theos);
         _timer.stop();
         var pub_time = _timer.lastIntervalTime;
-         bts.Log.Gen.info($"InterpTheosServerPublisher._publish_data " +
+         bts.Log.Gen.info($"InterpTheosServerPublisher._publish_data symbols: {all_theos.Count} " +
                          $"size: {server_theos.Theos.Count} comp_time: {comp_time} msecs " +
                          $"pub_time: {pub_time}");
      }
@@ -130,4 +135,4 @@ namespace mbts.spark {
         _bb.data_callback -= _on_data;
      }
   }
-}
+}
@@ -14,7 +14,7 @@ namespace mbts.spark {
   class VolSurfaceAdvancedSettingsEditorVM : ViewModelBase {
      readonly VolSurfaceAdvancedSettings _settings;

-      public VolSurfaceAdvancedSettingsEditorVM(VolSurfaceAdvancedSettings settings, string vs_name, Action<VolSurfaceAdvancedSettings> commit) {
+      public VolSurfaceAdvancedSettingsEditorVM(VolSurfaceAdvancedSettings settings, string vs_name) {
         _settings = settings;
         _settings.theo_handle_measure = VolSurfaceEnums.GeneralChangeMeasure.Absolute;
         _settings.slide_handle_measure = VolSurfaceEnums.GeneralChangeMeasure.Percentage;
@@ -23,7 +23,7 @@ namespace mbts.spark {
         if (_settings.precomputed_theo_handles == null)
            _settings.precomputed_theo_handles = new List<double>();
         vol_surface_name = vs_name;
-         this.commit = new DelegateCommand(() => request_commit?.Invoke(ProtobufSerializationHelper.clone(_settings)));
+         commit = new DelegateCommand(() => request_commit?.Invoke(ProtobufSerializationHelper.clone(_settings)));
      }

      public event Action<VolSurfaceAdvancedSettings> request_commit;
@@ -121,21 +121,24 @@ namespace mbts.spark {
            if (settings == null) {
               return null;
            }
-            // return new VolSurfaceAdvancedSettingsEditor {
-            //    DataContext = new VolSurfaceAdvancedSettingsEditorVM(
-            //       VolSurfaceUtilityFunctions.clone_proto(settings.advanced_settings),
-            //       settings.core_settings.vol_surface_name,
-            //       advanced_settings => {
-            //          var current_settings = _get_volsurface_settings();
-            //          if (current_settings == null) BTSMessageBox.Show("Error changing settings", "VS Settings", System.Windows.MessageBoxButton.OK, System.Windows.MessageBoxImage.Error, true, true, true);
-            //          else {
-            //             current_settings.advanced_settings = advanced_settings;
-            //             _model.set_candidate_settings(current_settings);
-            //          }
-            //       }
-            //    )
-            // };
-            return null;
+            var vm = new VolSurfaceAdvancedSettingsEditorVM(
+               VolSurfaceUtilityFunctions.clone_proto(settings.advanced_settings),
+               settings.core_settings.vol_surface_name);
+            vm.request_commit += commit_advanced_settings;
+            return new VolSurfaceAdvancedSettingsEditor {
+               DataContext = vm
+            };
+
+            async void commit_advanced_settings(VolSurfaceAdvancedSettings value) {
+               var current_settings = await _get_volsurface_settings();
+               if (current_settings == null) {
+                  BTSMessageBox.Show("Error changing settings", "VS Settings");
+               }
+               else {
+                  current_settings.advanced_settings = value;
+                  _model.set_candidate_settings(current_settings);
+               }
+            }
         }

         Window? mk_grid_settings_win() {
@@ -22,7 +22,7 @@ namespace mbts.spark {
   // ii. IntepGreeks for each vs
   public class VolSurfaceInterpGreeksMultiplexer : IDisposable {
      private readonly ThreadedBlackboard _bb;
-      private readonly ITag<VolSurfaceUsageListImmutable> _in_bb_vs_usage_tag;
+      private readonly ITag<VolSurfaceUsageListImmutable?> _in_bb_vs_usage_tag;
      private IDictionary<int, ITag<VSTheos>> _vs_theo_tags =
         new Dictionary<int, ITag<VSTheos>>();
      private VolSurfaceUsageListImmutable _vs_usage_map =
@@ -48,8 +48,8 @@ namespace mbts.spark {
      }

      private void _analyze_usages() {
-         _vs_usage_map = _bb.read_state(_in_bb_vs_usage_tag,
-            new VolSurfaceUsageListImmutable(new VolSurfaceUsageList()));
+         _vs_usage_map = _bb.read_state(_in_bb_vs_usage_tag) ??
+            new VolSurfaceUsageList().immutable();
         bts.Log.Gen.info($"VolSurfaceInterpGreeksMultiplexer._analyze_usages " +
                          $"count: {_vs_usage_map.UsageList.Count()}");
         foreach (int vs_id in _vs_usage_map.vss()) {
@@ -9,26 +9,34 @@ namespace mbts.analytics
   [ProtoBuf.ProtoContract(Name = "VolSurfaceAdvancedSettings")]
   public class VolSurfaceAdvancedSettings
   {
+      public static VolSurfaceAdvancedSettings Default =>
+         new() {
+            number_of_cashed_saved_days = double.NaN,
+            precomputed_theo_handles = new(0),
+            theo_handle_measure = VolSurfaceEnums.GeneralChangeMeasure.Absolute,
+            precomputed_slide_handles = new(0),
+            slide_handle_measure = VolSurfaceEnums.GeneralChangeMeasure.Percentage,
+            publish_from_client = false,
+            publish_from_client_user = string.Empty,
+            strike_normalization_axis = VolSurfaceEnums.VolCurveStrikeAxis.RawStrike,
+         };
+
      [ProtoBuf.ProtoMember(1)]
-      [System.ComponentModel.DefaultValue(Double.NaN)]
      public double number_of_cashed_saved_days;
      
      [ProtoBuf.ProtoMember(10)]      
      public List<double> precomputed_theo_handles;
      
      [ProtoBuf.ProtoMember(15)]
-      [System.ComponentModel.DefaultValue(VolSurfaceEnums.GeneralChangeMeasure.Absolute)]
      public VolSurfaceEnums.GeneralChangeMeasure theo_handle_measure;
      
      [ProtoBuf.ProtoMember(20)]      
      public List<double> precomputed_slide_handles;
      
      [ProtoBuf.ProtoMember(25)]
-      [System.ComponentModel.DefaultValue(VolSurfaceEnums.GeneralChangeMeasure.Percentage)]
      public VolSurfaceEnums.GeneralChangeMeasure slide_handle_measure;

      [ProtoBuf.ProtoMember(30)]
-      //[System.ComponentModel.DefaultValue(false)]
      public bool publish_from_client;

      [ProtoBuf.ProtoMember(40)]
@@ -38,5 +46,11 @@ namespace mbts.analytics
      [ProtoBuf.ProtoMember(50)]
      [System.ComponentModel.DefaultValue(VolSurfaceEnums.VolCurveStrikeAxis.RawStrike)]
      public VolSurfaceEnums.VolCurveStrikeAxis strike_normalization_axis;
+      
+      public void ensure_no_nulls() {
+         precomputed_theo_handles ??= new(0);
+         precomputed_slide_handles ??= new(0);
+         publish_from_client_user ??= string.Empty;
+      }
   }
 }
@@ -25,7 +25,8 @@ namespace mbts.analytics {
            volpath_settings = new(),
            safety_limits = new(),
         },
-         polynomial_model_settings = new()
+         polynomial_model_settings = new(),
+         advanced_settings = VolSurfaceAdvancedSettings.Default
      };

      [ProtoMember(10)]
@@ -79,6 +80,8 @@ namespace mbts.analytics {
         if (core_settings.time_settings == null) {
            core_settings.time_settings = new VolSurfaceTimeSettings();
         }
+         advanced_settings ??= VolSurfaceAdvancedSettings.Default;
+         advanced_settings.ensure_no_nulls();
         return this;
      }
   }
@@ -40,10 +40,8 @@ namespace mbts.analytics {
      /// last computed min/max driver for each exp
      private IDictionary<VolSurfaceExpiration, Tuple<double, double>>
      _computed_driver_range_map = new Dictionary<VolSurfaceExpiration, Tuple<double, double>>();
-      private readonly IEnumerable<double> _default_abs_moves =
-         new double[] { 300, -200, -125, -100, -75, -50, -25, 0, 25, 50, 75, 100, 125, 200, 300 };
+      private readonly IEnumerable<double> _default_abs_moves =  config.Configuration.get<bts.config.VolSurface>().default_abs_moves;
      private readonly double _cushion_percentage = 25.0;
-
      public event
      Action<IDictionary<VolSurfaceExpiration, IEnumerable<Theos>>, double, double> on_theos =
         (o, vt, tt) => { };
@@ -52,7 +50,7 @@ namespace mbts.analytics {
         (o, vt, tt) => { };

      public VolSurfaceInterpGreeksProducer(int vs_id) {
-         Log.Gen.info($"VolSurfaceInterpGreeksProducer.cons vs_id: {vs_id}");
+        Log.Gen.info($"VolSurfaceInterpGreeksProducer.cons vs_id: {vs_id}, _default_abs_moves: {string.Join(", ", _default_abs_moves)}");
         _vs_id = vs_id;
      }

@@ -72,7 +70,7 @@ namespace mbts.analytics {
      }

      public void set_vol_data(VolSurfaceInput vs_input) {
-         Log.Gen.info($"VolSurfaceInterpGreeksProducer.set_vol_data vs_id: {_vs_id}");
+         Log.Gen.info($"VolSurfaceInterpGreeksProducer.set_vol_data vs_id: {_vs_id} precomputed_theo_handles?: {string.Join(", ", (vs_input?.surface_settings?.advanced_settings?.precomputed_theo_handles).empty_if_null())}");
         _vs_input = vs_input != null ? VolSurfaceUtilityFunctions.clone_proto(vs_input) : null;
         _compute_and_fire(InputType.Vol);
      }
@@ -0,0 +1,58 @@
+<Window
+  x:Class="mbts.ui.shell.VolSurfaceAdvancedSettingsEditor"
+  x:ClassModifier="internal"
+  xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation"
+  xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
+  xmlns:c="clr-namespace:mbts.ui.controls"
+  xmlns:cv="clr-namespace:mbts.ui.converters"
+  Title="{Binding vol_surface_name, StringFormat='{}Advanced Settings \&apos;{0}\&apos;'}"
+  Style="{StaticResource BTSDialog}"
+  SizeToContent="WidthAndHeight"
+  TextElement.FontSize="10">
+  <StackPanel>
+    <Grid Margin="5" Grid.IsSharedSizeScope="True">
+      <Grid.RowDefinitions>
+        <RowDefinition Height="Auto"/>
+        <RowDefinition Height="Auto"/>
+        <RowDefinition Height="Auto"/>
+        <RowDefinition Height="Auto"/>
+        <RowDefinition Height="Auto"/>
+      </Grid.RowDefinitions>
+      <Grid.ColumnDefinitions>
+        <ColumnDefinition Width="Auto" SharedSizeGroup="label"/>
+        <ColumnDefinition Width="*"/>
+      </Grid.ColumnDefinitions>
+      <GroupBox Grid.ColumnSpan="2" Padding="3">
+        <GroupBox.Header>
+          <TextBlock Text="Theos" FontWeight="Bold"/>
+        </GroupBox.Header>
+        <Grid>
+          <Grid.RowDefinitions>
+            <RowDefinition Height="Auto"/>
+            <RowDefinition Height="Auto"/>
+          </Grid.RowDefinitions>
+          <Grid.ColumnDefinitions>
+            <ColumnDefinition Width="Auto" SharedSizeGroup="label"/>
+            <ColumnDefinition Width="Auto"/>
+          </Grid.ColumnDefinitions>
+          <c:RadioButtonsControl Grid.ColumnSpan="2" Margin="0,0,0,5" ItemsSource="{Binding general_change_measures}" SelectedItem="{Binding theo_handle_measure}" IsEnabled="False"/>
+          <TextBlock Grid.Row="1" Text="Precomputed Handles" VerticalAlignment="Center"/>
+          <TextBox
+            x:Name="PART_Handles"
+            Validation.ErrorTemplate="{x:Null}"
+            Grid.Row="1"
+            Grid.Column="1"
+            Width="160"
+            Margin="5,0,0,0"
+            Text="{Binding precomputed_theo_handles, Converter={x:Static cv:EnumerableDoubleStringConverter.Instance}}" />
+        </Grid>
+      </GroupBox>
+      <TextBlock Visibility="Collapsed" Grid.Row="1" VerticalAlignment="Center" Margin="5,0,0,0" Text="Num Cash Saved Days" />
+      <c:TickUpDownDouble Visibility="Collapsed" Grid.Row="1" Grid.Column="1" HorizontalAlignment="Stretch" Margin="14,5,5,5" Value="{Binding number_of_cash_saved_days}"/>
+      <TextBlock Grid.Row="2" VerticalAlignment="Center" Margin="5,0,0,0" Text="Strike Normalization Axis"/>
+      <ComboBox Grid.Row="2" Grid.Column="1" HorizontalAlignment="Stretch" Margin="14,5,5,5" ItemsSource="{Binding strike_normalization_axes}" SelectedValue="{Binding selected_strike_normalization_axis}"/>
+      <CheckBox Grid.Row="3" Margin="7,0,0,0" Content="Publish From Client" IsChecked="{Binding publish_from_client}"/>
+    </Grid>
+    <c:DialogSkirtOKCancel OKCommand="{Binding commit}" />
+  </StackPanel>
+</Window>
@@ -0,0 +1,11 @@
+namespace mbts.ui.shell {
+   using System.Windows;
+
+   /// <summary>
+   /// Interaction logic for VolSurfaceAdvancedSettingsEditor.xaml
+   /// </summary>
+   partial class VolSurfaceAdvancedSettingsEditor : Window {
+      public VolSurfaceAdvancedSettingsEditor() =>
+         InitializeComponent();
+   }
+}
@@ -87,7 +87,7 @@ namespace mbts.spark {
         length = _initial_row_count,
      };
      LadderPriceList _pending_price_list = null;
-      IComparer<double> _price_comparer;
+      IComparer<double> _price_comparer = MathUtils.epsilon_comparer.reverse();
      double _last_origin_price;
      readonly MarketBreakLines _mkt_break_lines;
      NativePriceSize _last_trade;
@@ -1281,7 +1281,7 @@ namespace mbts.spark {
         if (!_settings.piq_column.show)
            return;
         int origin_index = LadderRender.index_of(_price_list, _mkt_break_lines.origin, _price_comparer);
-         LadderRender.render_piqs(_piq_vis_tup.visual, _last_vis_tup.glyph_context, origin_index, _price_list, _vm.price_vs_piq, _settings.row_height, _spec, _settings.piq_column, _fmt_mkt_quantity);
+         LadderRender.render_piqs(_piq_vis_tup.visual, _last_vis_tup.glyph_context, _price_list, _vm.price_vs_piq, _settings.row_height, _spec, _settings.piq_column, _fmt_mkt_quantity);
      }

      void _refresh_sizes() {
@@ -368,30 +368,32 @@ namespace mbts.spark {

      public delegate void RenderCell<T>(DrawingContext dc, int row, T data_item);

-      static void _render_cells<T>(DrawingVisual dv, double row_height, IEnumerable<(double price, T value)> data, LadderColumnSettings settings, IList<double> prices, RegionalTickFactors spec, RenderCell<T> render_cell) {
+      static void _render_cells<T>(DrawingVisual dv, double row_height, IEnumerable<(double price, T value)> cells, LadderColumnSettings settings, IList<double> prices, RegionalTickFactors spec, RenderCell<T> render_cell) {
         using (var dc = dv.RenderOpen())
         using (_draw_with_clip(dc, settings, row_height, prices.Count)) {
-            if (data.is_null_or_empty())
+            if (prices.is_null_or_empty() || cells.is_null_or_empty()) {
               return;
-            int curr_index = 0;
-            double curr_price = prices[curr_index];
-            foreach (var (price, value) in data) {
-               if (double.IsNaN(price))
-                  continue;
-               // prices are in descending order, so get diff from curr_price - price
-               var tick_diff = spec.tickdiff(curr_price, price) ?? 0;
-               Log.assert(
-                  tick_diff > (int.MaxValue - curr_index),
-                  $"Unable to index ladder row for price {price}: tick count exceeds int.MaxValue");
-               curr_index += (int)tick_diff;
-               curr_price = price;
-               if (curr_index >= 0 && curr_index < prices.Count) {
-                  render_cell(dc, curr_index, value);
-               } else break;
+            }
+            var min_price = prices[prices.Count - 1];
+            var max_price = prices[0];
+            Log.assert(min_price.is_real() && max_price.is_real() && min_price <= max_price);
+            var indexed_cells = cells
+               .Where(item => item.price.is_real() && item.price.is_within_range(min_price, max_price))
+               .Select(row => {
+                  checked {
+                     return (row.price, row.value, (int)spec.tickdiff(max_price, row.price));
+                  }
+               });
+            foreach (var (price, value, idx) in indexed_cells) {
+               var check_price = spec.tickdown(max_price, idx);
+               Log.assert(spec.tick_equals(price, check_price));
+               Log.assert(idx >= 0 && idx < prices.Count);
+               render_cell(dc, idx, value);
            }
         }
      }

+
      internal static void render_last(DrawingVisual dv, GlyphContext glyph_context, NativePriceSize last, int index, double row_height, LadderColumnSettings settings, Func<uint, string> fmt_value) {
         if (fmt_value is null)
            fmt_value = FormattingDelegates.default_string<uint>();
@@ -405,38 +407,18 @@ namespace mbts.spark {
      internal static void render_piqs(
         DrawingVisual dv,
         GlyphContext glyph_context,
-         int origin_index,
         IList<double> prices,
         IEnumerable<(double price, uint piq)> pvp,
         double row_height,
         RegionalTickFactors spec,
         LadderColumnSettings settings,
-         Func<uint, string> fmt_value) {
-         //
-         using (var dc = dv.RenderOpen()) {
-            using (_draw_with_clip(dc, settings, row_height, prices.Count)) {
-               if (!prices.within_range(origin_index) || pvp.is_null_or_empty())
-                  return;
-               if (fmt_value is null)
-                  fmt_value = FormattingDelegates.default_string<uint>();
-               var foreground = settings.foreground.rgb_brush();
-               int curr_index = origin_index;
-               double curr_price = prices[curr_index];
-               foreach (var entry in pvp) {
-                  double price = entry.price;
-                  // prices are in descending order, so get diff from curr_price - price
-                  var tick_diff = spec.tickdiff(curr_price, price).assert_value();
-                  checked {
-                     curr_index += (int)tick_diff;
-                  }
-                  curr_price = price;
-                  if (curr_index >= 0 && curr_index < prices.Count) {
-                     var gr = format_text(fmt_value(entry.piq), curr_index, row_height, settings, glyph_context);
-                     dc.DrawGlyphRun(foreground, gr);
-                  } else break;
-               }
-            }
-         }
+         Func<uint, string> fmt_value
+      ) {
+         fmt_value ??= FormattingDelegates.default_string<uint>();
+         var fg = settings.foreground.rgb_brush();
+         _render_cells(dv, row_height, pvp, settings, prices, spec, (dc, row, piq) =>
+            dc.DrawGlyphRun(
+               fg, format_text(fmt_value(piq), row, row_height, settings, glyph_context)));
      }

      public static void render_mkt_break(DrawingVisual dv, int index, double row_height, uint color) =>
@@ -59,7 +59,9 @@ namespace bts.shell.manual_fills {
            nameof(quantity_display),
            () => {
               var sec_meta = security.HasValue ? _sec_meta_store.get(security.Value) : null;
-               return sec_meta?.QuantityDisplay ?? Security.Defaults.quantity_display_settingsi;
+               return sec_meta?.HasQuantityDisplay ?? false ?
+                  sec_meta.Value.QuantityDisplay :
+                  Security.Defaults.quantity_display_settingsi;
         });
         // TODO (prob): remove conversion_data.  Replace get_qty_display_divisor delegates w/
         // TickDisplaySettingsImmutable for qty, obtained via ISecurityMetadataStore.
@@ -293,7 +293,10 @@ namespace bts.shell.pnl {
               return _quantity_display_backing.Value;
            }
            if (security.HasValue) {
-               _quantity_display_backing = _sec_meta_store.get(security.Value)?.QuantityDisplay;
+               var sec_meta = _sec_meta_store.get(security.Value);
+               if (sec_meta?.HasQuantityDisplay ?? false) {
+                  _quantity_display_backing = sec_meta.Value.QuantityDisplay;
+               }
            }
            return _quantity_display_backing ??= Security.Defaults.quantity_display_settingsi;
         }
@@ -121,7 +121,7 @@ namespace bts.shell {

      void _on_orders(TableUpdate<(SecurityImmutable? sec, OrderRecordImmutable order), uint> update) {
         // must update notification_service_data_cache first, some of the predicate/triggers (esp. ToastNotifications) will use the notification_service_data_cache to lookup order-history
-         _data_cache.update(update.as_iincremental());
+         _data_cache.update(update);
         foreach (var o in update.added.Concat(update.changed))
            _order_notifications.process_item(o);
      }
@@ -1,89 +1,157 @@
-using bts.api;
-using bts.api.orderman;
-
-using mbts.spark;
-using mbts.utils;
-
-using System;
-using System.Collections.Generic;
-
-namespace bts.shell {
-   // Any predicate-sound/notifications that need to reference previous data (i.e. notifications based on change/time, or state changes) should use the INotificationServiceDataCache.
-   // The INotificationService updates these caches in its IObservable.OnNext callbacks, just before calling all of the predicate-trigger methods.
-   // Therfore, data referenced through this interface is guarenteed to be up-to-date, and thread safe.  It also reduces performance overhead by avoiding redundant bookeeping.
-   // Note: last_order_reset_time can be used by sound/toast-notification sources to avoid sending false-positive notifications for old orders (i.e. all the orders that arrive in the first update on startup/server restart)
-   // --------------------------------------------------------------------------------------------------------------------------------------------
-   // IMPOARTANT!!! The current implementation of this interface is NOT INTENDED FOR GENERAL USE - order_history is NOT THREAD-SAFE
-   interface INotificationServiceDataCache {
-      DateTime last_order_reset_time { get; }
-      bool is_first_order_update { get; }
-      IReadOnlyDictionary<uint, List<TimeStampedData<OrderSecurity>>> order_history { get; }
-   }
-
-   static class NotificationServiceDataCacheExtensions {
-      public static OrderSecurity? get_previous_order_state(this INotificationServiceDataCache notification_service_data_cache, uint key) {
-         if (notification_service_data_cache.order_history.TryGetValue(key, out List<TimeStampedData<OrderSecurity>> l)) {
-            if (l.Count > 1) {
-               return l[l.Count - 2].data;
-            }
-         }
-         return null;
-      }
-   }
-
-   struct TimeStampedData<T> {
-      public readonly T data;
-      public readonly DateTime time_stamp;
-
-      public TimeStampedData(T data, DateTime time_stamp) {
-         this.data = data;
-         this.time_stamp = time_stamp;
-      }
-   }
-
-   partial class NotificationService : INotificationService, IDisposable {
-      class DataCache : INotificationServiceDataCache {
-         const int _buffer_size = 10;
-         readonly Dictionary<uint, OrderSecurity> _order_cache = new Dictionary<uint, OrderSecurity>();
-         readonly Dictionary<uint, List<TimeStampedData<OrderSecurity>>> _order_history = new Dictionary<uint, List<TimeStampedData<OrderSecurity>>>();
-
-         public IReadOnlyDictionary<uint, List<TimeStampedData<OrderSecurity>>> order_history => _order_history;
-
-         public DateTime last_order_reset_time { get; private set; }
-         public bool is_first_order_update { get; private set; }
-
-         internal void update(IIncrementalUpdate<(SecurityImmutable? sec, OrderRecordImmutable order), uint> data) {
-            DateTime now = DateTime.Now;
-            if (data.removed.Count == _order_cache.Count) {
-               last_order_reset_time = now;
-               _order_cache.Clear();
-            }
-            is_first_order_update = _order_cache.Count == 0 && data.added.Count > 0;
-            foreach (var kv in data.removed) {
-               if (data.added.ContainsKey(kv.Key)) {
-                  _order_cache[kv.Key] = kv.Value;
-                  _order_history[kv.Key].Add(new TimeStampedData<OrderSecurity>(kv.Value, now));
-               } else {
-                  _order_cache.Remove(kv.Key);
-                  // TODO: maybe these shouldn't be removed?
-                  // if we add delta/time predicates that need to refer to order history, we might need to keep them around
-                  // although, if the server is telling us to remove them, perhaps we shouldn't consider such data.
-                  _order_history.Remove(kv.Key);
-               }
-            }
-            foreach (var kv in data.added) {
-               if (data.removed.ContainsKey(kv.Key))
-                  continue;
-               _order_cache.Add(kv.Key, kv.Value);
-               _order_history.AddOrUpdate(kv.Key,
-                  k => new List<TimeStampedData<OrderSecurity>> { new TimeStampedData<OrderSecurity>(kv.Value, now) },
-                  (k, l) => {
-                     l.Add(new TimeStampedData<OrderSecurity>(kv.Value, now));
-                     if (l.Count >= 10)
-                        l.RemoveAt(0);
-                  });
-            }
-         }
-      }
-   }
-}
+using bts.api;
+using bts.api.core;
+using bts.api.orderman;
+
+using mbts.spark;
+using mbts.utils;
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+
+namespace bts.shell {
+   // Any predicate-sound/notifications that need to reference previous data (i.e. notifications based on change/time, or state changes) should use the INotificationServiceDataCache.
+   // The INotificationService updates these caches in its IObservable.OnNext callbacks, just before calling all of the predicate-trigger methods.
+   // Therfore, data referenced through this interface is guarenteed to be up-to-date, and thread safe.  It also reduces performance overhead by avoiding redundant bookeeping.
+   // Note: last_order_reset_time can be used by sound/toast-notification sources to avoid sending false-positive notifications for old orders (i.e. all the orders that arrive in the first update on startup/server restart)
+   // --------------------------------------------------------------------------------------------------------------------------------------------
+   // IMPOARTANT!!! The current implementation of this interface is NOT INTENDED FOR GENERAL USE - order_history is NOT THREAD-SAFE
+   interface INotificationServiceDataCache {
+      bool is_first_order_update { get; }
+      IReadOnlyDictionary<uint, List<TimeStampedData<OrderSecurity>>> order_history { get; }
+   }
+
+   static class NotificationServiceDataCacheExtensions {
+      public static OrderSecurity? get_previous_order_state(this INotificationServiceDataCache notification_service_data_cache, uint key) {
+         if (notification_service_data_cache.order_history.TryGetValue(key, out List<TimeStampedData<OrderSecurity>> l)) {
+            if (l.Count > 1) {
+               return l[l.Count - 2].data;
+            }
+         }
+         return null;
+      }
+   }
+
+   struct TimeStampedData<T> {
+      public readonly T data;
+      public readonly DateTime time_stamp;
+
+      public TimeStampedData(T data, DateTime time_stamp) {
+         this.data = data;
+         this.time_stamp = time_stamp;
+      }
+   }
+
+   partial class NotificationService : INotificationService, IDisposable {
+      class DataCache : INotificationServiceDataCache {
+         const int _ORDER_HISTORY_BUFFER_SIZE = 10;
+         const int _TABLE_HISTORY_BUFFER_SIZE = 5;
+
+         readonly Dictionary<uint, OrderSecurity> _order_cache = new Dictionary<uint, OrderSecurity>();
+         readonly Dictionary<uint, List<TimeStampedData<OrderSecurity>>> _order_history = new Dictionary<uint, List<TimeStampedData<OrderSecurity>>>();
+         readonly List<(TableUpdate<(SecurityImmutable?, OrderRecordImmutable), uint>, DateTime)> _table_history = new(_TABLE_HISTORY_BUFFER_SIZE);
+         readonly object _lock = new();
+
+         public IReadOnlyDictionary<uint, List<TimeStampedData<OrderSecurity>>> order_history => _order_history;
+
+         public bool is_first_order_update { get; private set; }
+
+         internal void update(TableUpdate<(SecurityImmutable? sec, OrderRecordImmutable order), uint> table) {
+            // First, try processing the update per usual (incrementally). If it throws,
+            // then log the exception and reset the state.
+            DateTime now = DateTime.Now;
+            lock (_lock) {
+               _circular_buffer_add(_table_history, (table, now), _TABLE_HISTORY_BUFFER_SIZE);
+               try {
+                  impl();
+                  return;
+               } catch (Exception ex) {
+                  var table_history_desc = _fmt_order_table_history_safe();
+                  Log.Ui.error(
+                     $"Error processing order update in NotificationService: {ex}.\n" +
+                     $"***Last table updates below***\n" +
+                     table_history_desc);
+               }
+               // This is the 'escape' path. It really shouldn't throw.
+               reset();
+            }
+
+            void impl() {
+               if (table.state.Count == 0) {
+                  _order_cache.Clear();
+               }
+               is_first_order_update = _order_cache.Count == 0 && table.state.Count > 0;
+               foreach (var order in table.removed) {
+                  var k = table.key(order);
+                  _order_cache.Remove(k);
+                  _order_history.Remove(k);
+               }
+               foreach (var ch in table.changes) {
+                  var k = table.key(ch.New);
+                  _order_cache[k] = ch.New;
+                  _order_history[k].Add(new(ch.New, now));
+               }
+               foreach (var order in table.added) {
+                  var k = table.key(order);
+                  _order_cache[k] = order;
+                  var hist = _order_history.GetOrAdd(k, _ => new());
+                  _circular_buffer_add(hist, new(order, now), _ORDER_HISTORY_BUFFER_SIZE);
+               }
+            }
+
+            void reset() {
+               _order_cache.Clear();
+               _order_history.Clear();
+               _table_history.Clear();
+               foreach (var (k, order) in table.state) {
+                  _order_cache[k] = order;
+                  _order_history.GetOrAdd(k, _ => new()).Add(new(order, now));
+               }
+               is_first_order_update = true;
+            }
+         }
+
+         static void _circular_buffer_add<T>(List<T> list, T item, int buffer_size) {
+            Log.assert(buffer_size > 0);
+            if (list.Count > buffer_size - 1) {
+               // Remove oldest items (trim front)
+               list.RemoveRange(
+                  0, list.Count - (buffer_size - 1));
+            }
+            list.Add(item);
+         }
+
+         string _fmt_order_table_history_safe() {
+            return _fmt_table_history_safe(_table_history, fmt_order_tup);
+
+            static string fmt_order_tup((SecurityImmutable?, OrderRecordImmutable) order_tup) {
+               var (sec, order) = order_tup;
+               var order_desc = OrderUtils.short_description(order, sec.spec_or_default());
+               var sec_desc = sec?.ToString() ?? ("sid=" + order.security_key().ToString());
+               return $"{sec_desc} {order_desc}";
+            }
+         }
+
+         static string _fmt_table_history_safe<T, TKey>(
+            List<(TableUpdate<T, TKey>, DateTime)> table_history, Func<T, string> fmt_value) where TKey : IEquatable<TKey> {
+            var table_history_desc = new System.Text.StringBuilder();
+            for (int i = 0; i < table_history.Count; i++) {
+               var (table, time_stamp) = table_history[i];
+               table_history_desc.AppendLine(
+                  $"Update - {table_history.Count - i}, at {time_stamp:yyyy-MM-dd hh:mm:ss.ffffff}");
+               table_history_desc.AppendLine(_fmt_table_safe(table, fmt_value));
+            }
+            return table_history_desc.ToString();
+         }
+
+         static string _fmt_table_safe<T, TKey>(
+            TableUpdate<T, TKey> table, Func<T, string> fmt_value) where TKey : IEquatable<TKey> {
+            try {
+               return table.description(fmt_value);
+            } catch (Exception ex) {
+               return $"Exception while formatting TableUpdate: ${ex}";
+            }
+         }
+      }
+   }
+}
@@ -90,7 +90,8 @@ namespace mbts {
            }
            var sec = order.security;
            var sec_meta = sec.HasValue ? _sec_meta_store.get(sec.Value) : null;
-            var qty_display = sec_meta?.QuantityDisplay ??
+            var qty_display = sec_meta?.HasQuantityDisplay ?? false ?
+               sec_meta.Value.QuantityDisplay :
               Security.Defaults.quantity_display_settingsi;
            return ToastNotifications.order_completed(order, qty_display);
         }
@@ -266,8 +266,8 @@ namespace mbts.spark {

      // List of symbol to volsurface mapping.
      // Publisher: VolSurfaceBlackboardManager
-      public static ITag<VolSurfaceUsageListImmutable> VolSurfaceUsageTag { get; } =
-         _mk_tag<VolSurfaceUsageListImmutable>();
+      public static ITag<VolSurfaceUsageListImmutable?> VolSurfaceUsageTag { get; } =
+         _mk_tag<VolSurfaceUsageListImmutable?>();

      // The AW2 machinary can compute and publish the theos
      // right back locally(vs the server doing that) if configured that way.
@@ -10,8 +10,8 @@
  </ResourceDictionary.MergedDictionaries>

  <Style x:Key="LogViewer.TextBox" TargetType="TextBox">
-    <Setter Property="Background" Value="{StaticResource TextBox.BackgroundBrush}" />
-    <Setter Property="Foreground" Value="{StaticResource TextBox.ForegroundBrush}" />
+    <Setter Property="Background" Value="{u:BTSColor UI.TextBox.default_background}" />
+    <Setter Property="Foreground" Value="{u:BTSColor UI.TextBox.default_foreground}" />
    <Setter Property="BorderThickness" Value="0" />
  </Style>

@@ -14,6 +14,9 @@ namespace mbts.spark {
      uint _bid_size, _ask_size;
      ulong _time;

+      public BBOVM() =>
+         _dispose = Disposable.Create(() => Disposable.dispose(ref _md_poll));
+
      public SecurityImmutable? security => _security_md?.security;
      public SecurityMD? security_md {
         get => _security_md;
@@ -63,7 +63,11 @@ namespace mbts {
         if (ticks_per_base.is_int()) {
            return true;
         }
-         return tick.tick_unit > tick.tick_base && tick.tick_base == 1;
+         // The Log10 check accounts for [non-perp] futures at Deribit, which use TickFactors 25/10
+         // (e.g., 10.0, 12.5, 15.0, etc.). It was added 11/02/2023, when it replaced the
+         // commented out line below.
+         // return tick.tick_unit > tick.tick_base && tick.tick_base == 1;
+         return tick.tick_unit > tick.tick_base && Math.Log10(tick.tick_base).is_int();
      }

      public static bool is_valid(this RegionalTickFactors regional_tick) =>
@@ -667,7 +671,8 @@ namespace mbts {
         if (negate) {
            value = -value;
         }
-         var text = core_impl();
+         var text = tick.tick_unit == 25 && tick.tick_base == 10 ?
+            deribit_future_impl() : core_impl();
         if (!settings.show_zero_integer) {
            // Hide the integer if its zero.  This could be better optimized,
            // but speed shouldn't be an issue, so it doesn't seem worth the time/effort.
@@ -721,6 +726,37 @@ namespace mbts {
               return "Overflow";
            }
         }
+
+         string deribit_future_impl() {
+            try {
+               // TODO: Replace this hack with a suitable long term solution
+               //
+               // The hack: [non-perp] futures at Deribit tick in units of '2.5' (e.g., 10.0, 12.5,
+               // 15.0, etc.). However, this battle-tested impl is predicated on the tick base being
+               // a multiple of the tick unit. So, to avoid changing key battle-tested logic, let's
+               // do the following:
+               //    1) Round the value to the nearest multiple of '2.5' (the tick size)
+               //    2) Format the rounded value with normal tick factors (unit 5, base 10),
+               //       (ticks of the sort this method was designed to expect).
+               var tuf = tick.tick_unit_frac<T>();
+               var total_ticks = value / tuf;
+               long total_ticks_l = NumericCastConverter<T, long>.convert(value / tuf);
+               var rounded_total_ticks = NumericCastConverter<long, T>.convert(total_ticks_l);
+               var rem = value - (rounded_total_ticks * tuf);
+               var unit_rem = rem / tuf;
+               Log.assert(unit_rem <= T.One && unit_rem >= T.Zero);
+               // Check if unit_rem closer to T.One, or T.Zero. Round accordingly
+               if (T.One - unit_rem <= unit_rem - T.Zero) {
+                  // round up
+                  rounded_total_ticks++;
+               }
+               var rounded_val = tuf * rounded_total_ticks;
+               return _format_tick(new TickFactors(10, 5), rounded_val, settings);
+            } catch (OverflowException ex) {
+               Log.Ui.warn($"Arithmetic overflow exception in format_tick (tick:{tick}, value:{value}):{ex}");
+               return "Overflow";
+            }
+         }
      }

      static T _round_tick<T>(T tick_unit_frac, T input) where T : IFloatingPointIeee754<T> =>
@@ -134,8 +134,20 @@ namespace bts.data.ticks {
      static TickFactors _inflate_to_base(TickFactors tick_factors, double new_base) =>
         new TickFactors(new_base, tick_factors.tick_unit * new_base / tick_factors.tick_base);

-      static bool _check_tick_size(double tick_size) =>
-         tick_size > 0 && (tick_size <= 0.5 || MathUtils.is_int(tick_size));
+      static bool _check_tick_size(double tick_size) {
+         if (!tick_size.is_real_positive()) {
+            return false;
+         }
+         if (tick_size <= 0.5 || tick_size.is_int()) {
+            return true;
+         }
+         // Check if tick size represents quarters (i.e., `tick_size=0.25 * 10^c` for some int c).
+         // Note: this check is designed specifically to support (non-perp) futures at Deribit
+         if (Math.Log10(tick_size / 0.25).is_int()) {
+            return true;
+         }
+         return false;
+      }

      struct SpecKey : IEquatable<SpecKey> {
         public readonly uint denominator, display_digits;
@@ -209,6 +209,17 @@ namespace mbts.spark {
      }

      public static decimal round_to_tick(decimal tick_size, decimal value) {
+         if (tick_size == (decimal)2.5) {
+            // This is a special case for [non-perp] futures at Deribit, which tick in 25/10
+            // (unit/base). While this code looks harmless enough, and should work for all ticks in
+            // theory, let's be extra safe and only apply the change to a specific case (when
+            // tick_size == 2.5).
+            // Note: Making the ticks code work for such a wide range of tick factors was no easy
+            // task, and involved overcoming a plethora of obscure rounding errors along the way.
+            var n_ticks = value / tick_size;
+            var simple_round = tick_size * n_ticks;
+            return simple_round;
+         }
         decimal intpart = (long)value;
         decimal floatpart = value % 1;
         floatpart = ((int)(floatpart / tick_size)) * tick_size;
@@ -293,6 +304,11 @@ namespace mbts.spark {
         if (!tf.low_tick.Equals(tf.high_tick) || !tf.is_valid() || tf.low_tick.is_integer())
            return null;
         var ticks_per_dollar = (uint)(tf.low_tick.tick_base / tf.low_tick.tick_unit);
+         // Note: this check was introduced specifically to account for [non-perp] Futures at
+         // Deribit, which use TickFactors 25/10 (e.g., 10.0, 12.5, 15.0, etc.).
+         if (ticks_per_dollar == 0) {
+            return null;
+         }
         // if ticks_per_row is greater than ticks_per_dollar, round to a multiple of ticks_per_dollar
         if (ticks_per_row > ticks_per_dollar) {
            return (uint)(ticks_per_dollar * Math.Floor((double)(ticks_per_row / ticks_per_dollar)));
@@ -1190,7 +1190,6 @@ namespace System.Collections.Generic {
      /// A tuple whose items each represent the sum of their repsective items in
      /// <paramref name="source"/>
      /// </returns>
-      [RequiresPreviewFeatures]
      public static (T1, T2) Sum<T1, T2>(this IEnumerable<(T1, T2)> source)
         where T1 : INumber<T1>
         where T2 : INumber<T2> =>
@@ -1203,7 +1202,6 @@ namespace System.Collections.Generic {
      /// A tuple whose items each represent the sum of their repsective items in
      /// <paramref name="source"/>
      /// </returns>
-      [RequiresPreviewFeatures]
      public static (T1, T2) Sum<T1, T2>(this IEnumerable<(T1, T2)> source, T1 seed1, T2 seed2)
         where T1 : IAdditionOperators<T1, T1, T1>
         where T2 : IAdditionOperators<T2, T2, T2> =>
@@ -1217,7 +1215,6 @@ namespace System.Collections.Generic {
      /// A tuple whose items each represent the sum of their repsective items in
      /// <paramref name="source"/>
      /// </returns>
-      [RequiresPreviewFeatures]
      public static (T1, T2, T3) Sum<T1, T2, T3>(this IEnumerable<(T1, T2, T3)> source)
         where T1 : INumber<T1>
         where T2 : INumber<T2>
@@ -1231,7 +1228,6 @@ namespace System.Collections.Generic {
      /// A tuple whose items each represent the sum of their repsective items in
      /// <paramref name="source"/>
      /// </returns>
-      [RequiresPreviewFeatures]
      public static (T1, T2, T3) Sum<T1, T2, T3>(
         this IEnumerable<(T1, T2, T3)> source, T1 seed1, T2 seed2, T3 seed3)
         where T1 : IAdditionOperators<T1, T1, T1>
@@ -1248,7 +1244,6 @@ namespace System.Collections.Generic {
      /// A tuple whose items each represent the sum of their repsective items in
      /// <paramref name="source"/>
      /// </returns>
-      [RequiresPreviewFeatures]
      public static (T1, T2, T3, T4) Sum<T1, T2, T3, T4>(this IEnumerable<(T1, T2, T3, T4)> source)
         where T1 : INumber<T1>
         where T2 : INumber<T2>
@@ -1263,7 +1258,6 @@ namespace System.Collections.Generic {
      /// A tuple whose items each represent the sum of their repsective items in
      /// <paramref name="source"/>
      /// </returns>
-      [RequiresPreviewFeatures]
      public static (T1, T2, T3, T4) Sum<T1, T2, T3, T4>(
         this IEnumerable<(T1, T2, T3, T4)> source, T1 seed1, T2 seed2, T3 seed3, T4 seed4)
         where T1 : IAdditionOperators<T1, T1, T1>
@@ -1 +0,0 @@
-set(sources files/audit_ilink3.py;files/audit_upload.sh;files/audit_uploader.py;files/cert_secure_logon.txt;files/example.conf;files/futures.coconut.xz;files/onixs_ice/ConnectivityConfiguration.APITest.xml;files/onixs_ice/ConnectivityConfiguration.PerfTest.xml;files/onixs_ice/ConnectivityConfiguration.Production.xml;files/perf_trace/compiled.css;files/perf_trace/index.html;files/perf_trace/jquery-3.3.1.min.js;files/perf_trace/main.js;files/perf_trace/perf_log.sql;files/perf_trace/perf_trace_analysis.sql;files/perf_trace/plotly-latest.min.js;files/small_es.pcap.xz;files/spartan/create_polygon_books.sql;files/spartan/log.sql;files/spartan/md.sql;files/spartan/sim_db.sql)
@@ -1 +0,0 @@
-set(sources api/analytics/ExpirationDriver.proto;api/analytics/TimeWeightedInterpolatedDriver.proto;api/apicore.proto;api/apigateway.proto;api/apioptions.proto;api/core/ApiClientInterop.cpp;api/core/ApiServerInterop.cpp;api/domain/Currency.proto;api/domain/SecurityCriteria.proto;api/domain/SecurityCriteriaClause.proto;api/domain/SecurityExpiration.proto;api/domain/SecurityMetadata.proto;api/domain/SecurityMetadataCollection.proto;api/domain/SecurityMetadataItem.proto;api/domain/SecurityType.proto;api/domain/TickDisplaySettings.proto;api/domain/TickFactors.proto;api/domain/Venue.proto;api/io/URL.proto;api/md/MarketDataSubscriptionParams.proto;api/md/binance/BinanceDepthStreamParams.proto;api/md/binance/BinanceMarketDataSubscriptionParams.proto;api/md/deribit/DeribitMarketDataSubscriptionParams.proto;api/md/ftx/FTXDepthStreamParams.proto;api/md/ftx/FTXMarketDataSubscriptionParams.proto;api/md/glbx/GlbxMarketDataSubscriptionParams.proto;api/md/huobi/HuobiMarketDataSubscriptionParams.proto;api/public/ApiServer.cpp;api/public/ClientChannel.cpp;api/public/QuickbusClientChannel.cpp;api/public/ServerChannel.cpp;api/public/ServiceContext.cpp;api/service_helpers/AdminServiceUtils.cpp;api/service_helpers/AlgoServiceUtils.cpp;api/service_helpers/PositionUtilities.cpp;api/services/AdminService.proto;api/services/AlgoDefinitionService.proto;api/services/AlgoService.proto;api/services/AnalyticsService.proto;api/services/ApiUnitTestService.proto;api/services/AuthService.proto;api/services/BasePositionService.proto;api/services/ErrorReportService.proto;api/services/HelloService.proto;api/services/MBOService.proto;api/services/MarginService.proto;api/services/MarketDataAuxiliaryService.proto;api/services/MarketDataRelay.proto;api/services/MarketDataService.proto;api/services/MoneyFileService.proto;api/services/OrderService.proto;api/services/OutOfProcessURLListener.proto;api/services/PersistenceService.proto;api/services/PositionService.proto;api/services/ShellDiagnosticService.proto;api/services/ShellLatencyService.proto;api/services/SyntheticSecurityService.proto;api/test/ApiTests.cpp;api/test/HelloServiceServer.cpp;communincation/websockets/tests/WebsocketSyncClientTests.cpp;config/ConfigCheck.cpp;config/ConfigDownloader.cpp;config/ConfigErrorCollector.cpp;config/ConfigLog.cpp;config/ConfigPropertyEvaluator.cpp;config/ConfigValueQueue.cpp;config/ConfigurationException.cpp;config/ConfigurationState.cpp;config/PreprocessInputStream.cpp;config/config.cpp;config/defaults/MarketData.conf;config/defaults/Ports.conf;config/defaults/SecurityMetadataCollection.conf;config/defaults/Symbols.conf;config/defaults/cme_reference_data.conf;config/defaults/edge.conf;config/defaults/jetfix.conf;config/defaults/keycode.conf;config/defaults/logging.conf;config/defaults/orderman_cfg.conf;config/defaults/orders_fills_cleanup.conf;config/defaults/position_cfg.conf;config/defaults/span.conf;config/defaults/spreader.conf;config/defaults/testkeys.conf;config/defaults/threads.conf;config/defaults/toggles.conf;config/extensions/extensions.proto;config/invalid/invalid.proto;config/protos/algo_types.proto;config/protos/cme_reference_data.proto;config/protos/exchange_sim_cfg.proto;config/protos/jetfix.proto;config/protos/keycode.proto;config/protos/logging.proto;config/protos/market_data.proto;config/protos/mercury_cfg.proto;config/protos/orderman_cfg.proto;config/protos/orders_fills_cleanup.proto;config/protos/perflog_cfg.proto;config/protos/ports.proto;config/protos/position_cfg.proto;config/protos/quickbus_cfg.proto;config/protos/span.proto;config/protos/spartan.proto;config/protos/spreader.proto;config/protos/subprocess.proto;config/protos/symbols.proto;config/protos/system.proto;config/protos/testkeys.proto;config/protos/threads.proto;config/protos/toggles.proto;config/protos/ui.proto;config/tests/HTTPServer.cpp;config/tests/config_ref_tests.cpp;config/tests/config_tests.cpp;config/text_format/text_format.cpp;domain/AggressorSide.cpp;domain/NewOrder.cpp;domain/OrderDefinition.cpp;domain/OrderStatus.cpp;domain/OrderUpdate.cpp;domain/Side.cpp;domain/TimeInForce.cpp;domain_model/Security.cpp;domain_model/Security.proto;domain_model/SecurityTests.cpp;domain_model/sod.cpp;domain_model/tests/domain_tests.cpp;expression/BooleanGrammar.cpp;expression/ExpressionEvaluator.cpp;expression/ParserExceptions.cpp;expression/Value.cpp;expression/tests/expression_parser_tests.cpp;io/QueryParams.cpp;io/URL.cpp;io/http/CurlHandle.cpp;io/http/HTTPClient.cpp;io/http/HTTPHeaders.cpp;io/http/HTTPRequest.cpp;io/http/HTTPResponse.cpp;io/websocket/WebsocketClient.cpp;io/websocket/WebsocketClientConfiguration.cpp;io/websocket/WebsocketError.cpp;io/websocket/WebsocketMessage.cpp;io/websocket/WebsocketSSLClientInternal.cpp;io/websocket/tests/WebsocketClientTests.cpp;jetfix/codec/FIXArray.cpp;jetfix/common/OutFIX.cpp;jetfix/connection/AsyncReader.cpp;jetfix/connection/IConnection.cpp;jetfix/connection/SocketConnection.cpp;jetfix/gateway/CoreGateway.cpp;jetfix/gateway/JetFIXGatLine truncated
@@ -1 +0,0 @@
-set(sources api/analytics/ExpirationDriver.proto;api/analytics/TimeWeightedInterpolatedDriver.proto;api/apicore.proto;api/apigateway.proto;api/apioptions.proto;api/core/ApiClientInterop.cpp;api/core/ApiServerInterop.cpp;api/domain/Currency.proto;api/domain/SecurityCriteria.proto;api/domain/SecurityCriteriaClause.proto;api/domain/SecurityExpiration.proto;api/domain/SecurityMetadata.proto;api/domain/SecurityMetadataCollection.proto;api/domain/SecurityMetadataItem.proto;api/domain/SecurityType.proto;api/domain/TickDisplaySettings.proto;api/domain/TickFactors.proto;api/domain/Venue.proto;api/io/URL.proto;api/md/MarketDataSubscriptionParams.proto;api/md/binance/BinanceDepthStreamParams.proto;api/md/binance/BinanceMarketDataSubscriptionParams.proto;api/md/deribit/DeribitMarketDataSubscriptionParams.proto;api/md/ftx/FTXDepthStreamParams.proto;api/md/ftx/FTXMarketDataSubscriptionParams.proto;api/md/glbx/GlbxMarketDataSubscriptionParams.proto;api/md/huobi/HuobiMarketDataSubscriptionParams.proto;api/public/ApiServer.cpp;api/public/ClientChannel.cpp;api/public/QuickbusClientChannel.cpp;api/public/ServerChannel.cpp;api/public/ServiceContext.cpp;api/service_helpers/AdminServiceUtils.cpp;api/service_helpers/AlgoServiceUtils.cpp;api/service_helpers/PositionUtilities.cpp;api/services/AdminService.proto;api/services/AlgoDefinitionService.proto;api/services/AlgoService.proto;api/services/AnalyticsService.proto;api/services/ApiUnitTestService.proto;api/services/AuthService.proto;api/services/BasePositionService.proto;api/services/ErrorReportService.proto;api/services/HelloService.proto;api/services/MBOService.proto;api/services/MarginService.proto;api/services/MarketDataAuxiliaryService.proto;api/services/MarketDataRelay.proto;api/services/MarketDataService.proto;api/services/MoneyFileService.proto;api/services/OrderService.proto;api/services/OutOfProcessURLListener.proto;api/services/PersistenceService.proto;api/services/PositionService.proto;api/services/ShellDiagnosticService.proto;api/services/ShellLatencyService.proto;api/services/SyntheticSecurityService.proto;api/test/ApiTests.cpp;api/test/HelloServiceServer.cpp;config/ConfigCheck.cpp;config/ConfigDownloader.cpp;config/ConfigErrorCollector.cpp;config/ConfigLog.cpp;config/ConfigPropertyEvaluator.cpp;config/ConfigValueQueue.cpp;config/ConfigurationException.cpp;config/ConfigurationState.cpp;config/PreprocessInputStream.cpp;config/config.cpp;config/defaults/MarketData.conf;config/defaults/Ports.conf;config/defaults/SecurityMetadataCollection.conf;config/defaults/Symbols.conf;config/defaults/cme_reference_data.conf;config/defaults/edge.conf;config/defaults/jetfix.conf;config/defaults/keycode.conf;config/defaults/logging.conf;config/defaults/orderman_cfg.conf;config/defaults/orders_fills_cleanup.conf;config/defaults/position_cfg.conf;config/defaults/span.conf;config/defaults/spreader.conf;config/defaults/testkeys.conf;config/defaults/threads.conf;config/defaults/toggles.conf;config/extensions/extensions.proto;config/invalid/invalid.proto;config/protos/algo_types.proto;config/protos/cme_reference_data.proto;config/protos/exchange_sim_cfg.proto;config/protos/jetfix.proto;config/protos/keycode.proto;config/protos/logging.proto;config/protos/market_data.proto;config/protos/mercury_cfg.proto;config/protos/orderman_cfg.proto;config/protos/orders_fills_cleanup.proto;config/protos/perflog_cfg.proto;config/protos/ports.proto;config/protos/position_cfg.proto;config/protos/quickbus_cfg.proto;config/protos/span.proto;config/protos/spartan.proto;config/protos/spreader.proto;config/protos/subprocess.proto;config/protos/symbols.proto;config/protos/system.proto;config/protos/testkeys.proto;config/protos/threads.proto;config/protos/toggles.proto;config/protos/ui.proto;config/tests/HTTPServer.cpp;config/tests/config_ref_tests.cpp;config/tests/config_tests.cpp;config/text_format/text_format.cpp;domain_model/Security.cpp;domain_model/Security.proto;domain_model/SecurityTests.cpp;domain_model/sod.cpp;domain_model/tests/domain_tests.cpp;expression/BooleanGrammar.cpp;expression/ExpressionEvaluator.cpp;expression/ParserExceptions.cpp;expression/Value.cpp;expression/tests/expression_parser_tests.cpp;io/QueryParams.cpp;io/URL.cpp;io/http/CurlHandle.cpp;io/http/HTTPClient.cpp;io/http/HTTPHeaders.cpp;io/http/HTTPRequest.cpp;io/http/HTTPResponse.cpp;io/websocket/WebsocketClient.cpp;io/websocket/WebsocketClientConfiguration.cpp;io/websocket/WebsocketError.cpp;io/websocket/WebsocketMessage.cpp;io/websocket/WebsocketSSLClientInternal.cpp;io/websocket/tests/WebsocketClientTests.cpp;logging/LoggerHub.cpp;logging/Sink.cpp;logging/logging.cpp;logging/tests/logging_tests.cpp;md/CCApiDepthWalker.cpp;md/CCApiListenerAdaptor.cpp;md/CCApiRequestor.cpp;md/Diag/MDPeriodicDumper.cpp;md/Diag/ShmDumper.cpp;md/MBOProcessor.cpp;md/MBOServiceServer.cpp;md/MDClient.cpp;md/MDConsumer.cpp;md/MDConsumerHandle.cpp;md/MDConsumerInterop.cpp;md/MDProducer.cpp;md/MDProducerHandle.cpp;md/MarketDataError.cpp;md/MarketDataGateway.cpLine truncated
@@ -30,6 +30,7 @@ if(WIN32)
    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
    api/*.cpp
    config/*.cpp
+    core/*.cpp
    domain_model/*.cpp
    expression/*.cpp
    logging/*.cpp
@@ -108,13 +109,11 @@ target_include_directories(
   ${CMAKE_BINARY_DIR}/src)

 if(WIN32)
+  find_package(fmt CONFIG REQUIRED)
  target_link_libraries(
    libspark
    libcurl
-    debug
-    fmtd
-    optimized
-    fmt
+    fmt::fmt-header-only
    libtbb
    libboost_system
    libboost_thread
@@ -0,0 +1,13 @@
+#pragma once
+
+#include <functional>
+
+namespace bts::algorithm {
+
+template <typename _Tp, typename _TAddend, _Tp _Addend>
+struct PlusX {
+   /// Returns the sum
+   _Tp operator()(const _Tp &__x) const { return __x + _Addend; }
+};
+
+} // namespace bts::algorithm
@@ -0,0 +1,49 @@
+#pragma once
+#include <string>
+
+namespace bts::algorithm {
+
+class StringCaseConverter {
+ public:
+   static void to_upper(const std::string &str, std::string &dest) noexcept;
+   static std::string to_upper(const std::string &str) noexcept;
+
+   static void to_lower(const std::string &str, std::string &dest) noexcept;
+   static std::string to_lower(const std::string &str) noexcept;
+};
+
+//////////////////////////////////////////////////
+// Definition
+//////////////////////////////////////////////////
+
+inline std::string StringCaseConverter::to_upper(const std::string &str) noexcept {
+   std::string rv{str};
+   for (auto &c : rv) {
+      c = std::toupper(c);
+   }
+   return rv;
+}
+
+inline void StringCaseConverter::to_upper(const std::string &str, std::string &dest) noexcept {
+   dest = str;
+   for (auto &c : dest) {
+      c = std::toupper(c);
+   }
+}
+
+inline std::string StringCaseConverter::to_lower(const std::string &str) noexcept {
+   std::string rv{str};
+   for (auto &c : rv) {
+      c = std::tolower(c);
+   }
+   return rv;
+}
+
+inline void StringCaseConverter::to_lower(const std::string &str, std::string &dest) noexcept {
+   dest = str;
+   for (auto &c : dest) {
+      c = std::tolower(c);
+   }
+}
+
+} // namespace bts::algorithm
@@ -0,0 +1,28 @@
+#pragma once
+
+#define CATCH_AND_LOG(subsys, level, label)                                    \
+   catch (const std::exception &ex) {                                          \
+      LOG(subsys, level, label ": {}", std::string{ex.what()});                \
+   }                                                                           \
+   catch (...) {                                                               \
+      LOG(subsys, level, label ": Unknown exception");                         \
+   }
+
+#define CATCH_AND_HANDLE(err_var_name, impl)                                   \
+   catch (const std::exception &err_var_name##_ex) {                           \
+      const auto &err_var_name = err_var_name##_ex.what();                     \
+      impl;                                                                    \
+   }                                                                           \
+   catch (...) {                                                               \
+      const std::string err_var_name{"Unknown exception"};                     \
+      impl;                                                                    \
+   }
+
+// Normalizes 'std::exception' and '...' handlers into single callback
+#define CATCH_AND_HANDLE_MSG(impl)                                             \
+   catch (const std::exception &ex) {                                          \
+      impl(std::string{ex.what()});                                            \
+   }                                                                           \
+   catch (...) {                                                               \
+      impl(std::string{"Unknown exception"});                                  \
+   }
@@ -0,0 +1,31 @@
+#pragma once
+
+#include "lib/api/services/OrderService.pb.h"
+#include "lib/core/EnumMacros.h"
+#include "lib/core/FMT_IMPL.h"
+#include "lib/io/JsonReader.h"
+#include "lib/logging/logging.h"
+
+#include <nlohmann/json.hpp>
+
+namespace bts::api::orderman {
+
+PB_ENUM_FROM_JSON(OrderStatus)
+PB_ENUM_TO_JSON(OrderStatus)
+
+PB_ENUM_FROM_JSON(TriggerPriceType)
+PB_ENUM_TO_JSON(TriggerPriceType)
+
+} // namespace bts::api::orderman
+
+FMT_IMPL(
+    bts::api::orderman::OrderStatus,
+    s,
+    "{}",
+    bts::api::orderman::OrderStatus_Name(s));
+
+FMT_IMPL(
+    bts::api::orderman::TriggerPriceType,
+    t,
+    "{}",
+    bts::api::orderman::TriggerPriceType_Name(t));
@@ -44,6 +44,7 @@ message PrecomputedTheoPoint {
  double vega = 60;
  double theta = 70;
  double rho = 80;
+  double vol = 90;
 }

 message PrecomputedTheos {
@@ -141,7 +141,7 @@ void ConfigValueQueue::assign(
      if (field->is_repeated()) {
         for (vector<Value>::iterator i = v.begin(); i != v.end(); ++i) {
            int size = reflection->FieldSize(*message, field);
-            add<TProto>(
+            add(
                message, reflection, field, (TProto)boost::get<TValue>(*i));
            if (index != size) {
               for (int i = index; i < size; ++i) {
@@ -182,7 +182,7 @@ void ConfigValueQueue::lexical_assign(
      for (vector<Value>::iterator i = v.begin(); i != v.end(); ++i) {
         int size = reflection->FieldSize(*message, field);
         TProto result = boost::lexical_cast<TProto>(boost::get<TValue>(*i));
-         add<TProto>(message, reflection, field, result);
+         add(message, reflection, field, result);
         if (index != size) {
            for (int i = index; i < size; ++i) {
               reflection->SwapElements(message, field, i, size);
@@ -0,0 +1,22 @@
+VolSurface {
+    value {
+        default_abs_moves: -600
+        default_abs_moves: -300
+        default_abs_moves: -200
+        default_abs_moves: -125
+        default_abs_moves: -100 
+        default_abs_moves: -75
+        default_abs_moves: -50
+        default_abs_moves: -25 
+        default_abs_moves: 0
+        default_abs_moves: 25 
+        default_abs_moves: 50 
+        default_abs_moves: 75 
+        default_abs_moves: 100
+        default_abs_moves: 125
+        default_abs_moves: 200
+        default_abs_moves: 300
+        default_abs_moves: 600
+
+    }
+}
@@ -0,0 +1,6 @@
+syntax = 'proto2';
+package bts.config;
+
+message VolSurface {
+    repeated double default_abs_moves = 1;
+}
@@ -1,15 +0,0 @@
-#pragma once
-
-#define BITWISE_OP(enum_type, numeric_type, op)                                \
-   enum_type operator op(enum_type lhs, enum_type rhs) noexcept {              \
-      return static_cast<enum_type>(static_cast<numeric_type>(lhs)             \
-                                        op static_cast<numeric_type>(rhs));    \
-   }
-
-#define BITWISE_OPS(enum_type, numeric_type)                                   \
-   BITWISE_OP(enum_type, numeric_type, |)                                      \
-   BITWISE_OP(enum_type, numeric_type, &)                                      \
-   BITWISE_OP(enum_type, numeric_type, ^)
-
-#define HAS_FLAG(enum_value, flag) (enum_value & flag) == flag
-#define STRIP_FLAG(enum_value, flag) (enum_value | flag) ^ flag
@@ -0,0 +1,154 @@
+#pragma once
+
+#define ENUM_BASIC_MEMBERS(type)                                               \
+   Value value;                                                                \
+                                                                               \
+   type() = default;                                                           \
+   /*NOLINTNEXTLINE*/                                                          \
+   constexpr type(Value value) noexcept : value(value) {}                      \
+                                                                               \
+   type(const type &) noexcept = default;                                      \
+   type(type &&) noexcept = default;                                           \
+   type &operator=(const type &) noexcept = default;                           \
+   type &operator=(type &&) noexcept = default;                                \
+                                                                               \
+   ~type() = default;                                                          \
+                                                                               \
+   static constexpr bool parse(const char *name, type *value) noexcept;        \
+   static bool parse(const std::string &name, type *value) noexcept;           \
+                                                                               \
+   [[nodiscard]] constexpr const char *name() const noexcept;
+
+#define ENUM_VALUE_TO_NAME_CASE(type, v)                                       \
+   case type::v:                                                               \
+      return #v;
+
+#define ENUM_NAME_TO_VALUE_CLAUSE(type, v)                                     \
+   if (name == #v) {                                                           \
+      *value = type::v;                                                        \
+      return true;                                                             \
+   }
+
+#define ENUM_NAMEC_TO_VALUE_CLAUSE(type, v)                                    \
+   if (std::string_view(name) == #v) {                                         \
+      *value = type::v;                                                        \
+      return true;                                                             \
+   }
+
+// clang-format off
+#define ENUM_NAME_DEF(type, for_all)                                           \
+   constexpr const char *type::name() const noexcept {                         \
+      switch (value) {                                                         \
+         for_all(ENUM_VALUE_TO_NAME_CASE)                                      \
+         default: return "ENUM_UNKNOWN";                                       \
+      }                                                                        \
+   }
+// clang-format on
+
+#define ENUM_PARSE_DEF(type, for_all)                                          \
+   constexpr bool type::parse(const char *name, type *value) noexcept {        \
+      for_all(ENUM_NAMEC_TO_VALUE_CLAUSE) return false;                        \
+   }                                                                           \
+   inline bool type::parse(const std::string &name, type *value) noexcept {    \
+      for_all(ENUM_NAME_TO_VALUE_CLAUSE) return false;                         \
+   }
+
+#define ENUM_EQUALITY_OPS(type)                                                \
+   constexpr bool operator==(type lhs, type rhs) noexcept {                    \
+      return lhs.value == rhs.value;                                           \
+   }                                                                           \
+   constexpr bool operator!=(type lhs, type rhs) noexcept {                    \
+      return lhs.value != rhs.value;                                           \
+   }                                                                           \
+   constexpr bool operator==(type lhs, type::Value rhs) noexcept {             \
+      return lhs.value == rhs;                                                 \
+   }                                                                           \
+   constexpr bool operator!=(type lhs, type::Value rhs) noexcept {             \
+      return lhs.value != rhs;                                                 \
+   }
+
+#define ENUM_OUTPUT_OPS(type)                                                  \
+   inline std::ostream &operator<<(std::ostream &out, type value) noexcept {   \
+      out << value.name();                                                     \
+      return out;                                                              \
+   }                                                                           \
+                                                                               \
+   inline std::ostream &operator<<(                                            \
+       std::ostream &out, type::Value value) noexcept {                        \
+      out << type{value}.name();                                               \
+      return out;                                                              \
+   }
+
+#define ENUM_FROM_JSON(type)                                                   \
+   inline void from_json(const nlohmann::json &j, type &value) {               \
+      std::string value_str;                                                   \
+      j.get_to(value_str);                                                     \
+      type::parse(value_str, &value);                                          \
+   }
+
+#define ENUM_TO_JSON(type)                                                     \
+   inline void to_json(nlohmann::json &j, const type &value) {                 \
+      j = value.name();                                                        \
+   }
+
+#define ENUM_DECLARE_VALUE(t, x) x,
+
+#define ENUM_CLASS_DEF(type, for_all)                                          \
+   class type {                                                                \
+    public:                                                                    \
+      enum Value { for_all(ENUM_DECLARE_VALUE) };                              \
+                                                                               \
+      ENUM_BASIC_MEMBERS(type)                                                 \
+   };                                                                          \
+                                                                               \
+   ENUM_OUTPUT_OPS(type)                                                       \
+   ENUM_NAME_DEF(type, for_all)                                                \
+   ENUM_PARSE_DEF(type, for_all)                                               \
+   ENUM_FROM_JSON(type)                                                        \
+   ENUM_TO_JSON(type)                                                          \
+   ENUM_EQUALITY_OPS(type)
+
+#define ENUM_BITWISE_COMPARISON_OP(enum_type, numeric_type, op)                \
+   constexpr enum_type operator op(enum_type lhs, enum_type rhs) noexcept {    \
+      return static_cast<enum_type>(static_cast<numeric_type>(lhs)             \
+                                        op static_cast<numeric_type>(rhs));    \
+   }
+
+#define ENUM_BITWISE_COMPARISON_OPS(enum_type, numeric_type)                   \
+   ENUM_BITWISE_COMPARISON_OP(enum_type, numeric_type, |)                      \
+   ENUM_BITWISE_COMPARISON_OP(enum_type, numeric_type, &)                      \
+   ENUM_BITWISE_COMPARISON_OP(enum_type, numeric_type, ^)
+
+#define ENUM_BITWISE_ASSIGNMENT_OP(enum_type, op)                              \
+   inline enum_type &operator op##=(enum_type &lhs, enum_type rhs) noexcept {  \
+      lhs = lhs op rhs;                                                        \
+      return lhs;                                                              \
+   }
+
+#define ENUM_BITWISE_ASSIGNMENT_OPS(enum_type)                                 \
+   ENUM_BITWISE_ASSIGNMENT_OP(enum_type, |)                                    \
+   ENUM_BITWISE_ASSIGNMENT_OP(enum_type, &)                                    \
+   ENUM_BITWISE_ASSIGNMENT_OP(enum_type, ^)
+
+#define ENUM_BITWISE_OPS(enum_type, numeric_type)                              \
+   ENUM_BITWISE_COMPARISON_OPS(enum_type, numeric_type)                        \
+   ENUM_BITWISE_ASSIGNMENT_OPS(enum_type)
+
+#define ENUM_HAS_FLAG(enum_value, flag) (enum_value & flag) == flag
+#define ENUM_STRIP_FLAG(enum_value, flag) (enum_value | flag) ^ flag
+
+#define PB_ENUM_FROM_JSON(enum_type)                                           \
+   inline void from_json(const nlohmann::json &j, enum_type &value) {          \
+      io::JsonReader jr{&j};                                                   \
+      std::string name;                                                        \
+      bool success = jr.read(name) && enum_type##_Parse(name, &value);         \
+      if (!success) {                                                          \
+         const auto j_str = j.dump(2);                                         \
+         LOG(Gen, Diag, "Failed to parse json to " #enum_type ":\n{}", j_str); \
+      }                                                                        \
+   }
+
+#define PB_ENUM_TO_JSON(enum_type)                                             \
+   inline void to_json(nlohmann::json &j, const enum_type &value) {            \
+      j = enum_type##_Name(value);                                             \
+   }
@@ -0,0 +1,34 @@
+#pragma once
+
+#include <fmt/core.h>
+#include <fmt/format.h>
+#ifndef _WIN32
+#include <range/v3/view.hpp>
+#endif
+#include <vector>
+
+#define FMT(template_str, ...) fmt::format(template_str, __VA_ARGS__)
+#define FMT_JOIN(range, delimiter)                                             \
+   fmt::to_string(fmt::join(range.begin(), range.end(), delimiter))
+
+namespace bts {
+
+template <class Range>
+std::string _fmt_join(Range range, const char *delimiter) {
+   return fmt::join(range.begin(), range.end(), delimiter);
+}
+
+#ifndef _WIN32
+template <class Range, class Op>
+std::string _fmt_join_fn(Range range, const char *delimiter, Op op) {
+   return fmt::to_string(
+       fmt::join(range | ranges::views::transform(op), delimiter));
+}
+#endif
+
+} // namespace bts
+
+#ifndef _WIN32
+#define FMT_JOIN_FN(range, delimiter, fn)                                      \
+   bts::_fmt_join_fn(range, delimiter, fn)
+#endif
@@ -0,0 +1,65 @@
+#pragma once
+
+#include <fmt/format.h>
+
+#define FMT_IMPL(Type, var_name, ...)                                          \
+   template <>                                                                 \
+   struct fmt::formatter<Type> {                                               \
+      constexpr auto parse(fmt::format_parse_context &ctx)                     \
+          -> decltype(ctx.end()) {                                             \
+         return ctx.end();                                                     \
+      }                                                                        \
+                                                                               \
+      template <typename FormatContext>                                        \
+      auto format(const Type &var_name, FormatContext &ctx) const              \
+          -> decltype(ctx.out()) {                                             \
+         return fmt::format_to(ctx.out(), __VA_ARGS__);                        \
+      }                                                                        \
+   }
+
+#define FMT_IMPL_T(Type, var_name, ...)                                        \
+   struct fmt::formatter<Type> {                                               \
+      constexpr auto parse(fmt::format_parse_context &ctx)                     \
+          -> decltype(ctx.end()) {                                             \
+         return ctx.end();                                                     \
+      }                                                                        \
+                                                                               \
+      template <typename FormatContext>                                        \
+      auto format(const Type &var_name, FormatContext &ctx) const              \
+          -> decltype(ctx.out()) {                                             \
+         return fmt::format_to(ctx.out(), __VA_ARGS__);                        \
+      }                                                                        \
+   }
+
+#define FMT_IMPL_OSTREAM(T)                                                    \
+   template <>                                                                 \
+   struct fmt::formatter<T> {                                                  \
+      constexpr auto parse(fmt::format_parse_context &ctx)                     \
+          -> decltype(ctx.end()) {                                             \
+         return ctx.end();                                                     \
+      }                                                                        \
+                                                                               \
+      template <typename FormatContext>                                        \
+      auto format(const T &t, FormatContext &ctx) const                        \
+          -> decltype(ctx.out()) {                                             \
+         std::stringstream ss;                                                 \
+         ss << t;                                                              \
+         return fmt::format_to(ctx.out(), "{}", ss.str());                     \
+      }                                                                        \
+   }
+
+#define FMT_IMPL_OSTREAM_T(T, ...)                                             \
+   struct fmt::formatter<T<__VA_ARGS__>> {                                     \
+      constexpr auto parse(fmt::format_parse_context &ctx)                     \
+          -> decltype(ctx.end()) {                                             \
+         return ctx.end();                                                     \
+      }                                                                        \
+                                                                               \
+      template <typename FormatContext>                                        \
+      auto format(const T<__VA_ARGS__> &t, FormatContext &ctx) const           \
+          -> decltype(ctx.out()) {                                             \
+         std::stringstream ss;                                                 \
+         ss << t;                                                              \
+         return fmt::format_to(ctx.out(), "{}", ss.str());                     \
+      }                                                                        \
+   }
@@ -132,7 +132,7 @@ std::size_t HashPhaser::auto_phase_range(Iter first, Iter last) {
   std::uint16_t el_count =
       num_vals > (std::numeric_limits<std::uint16_t>::max)()
           ? max_els
-           : std::max(static_cast<std::uint16_t>(num_vals), max_els);
+           : (static_cast<std::uint16_t>(num_vals) > max_els ? static_cast<std::uint16_t>(num_vals) : max_els);
   auto bits_per_hash = (sizeof(std::size_t) * __CHAR_BIT__) / el_count;
   std::size_t phase = 1U << (bits_per_hash - 1);
   std::size_t composite_hash = 0;
@@ -157,7 +157,7 @@ std::size_t HashPhaser::auto_phase_range(Iter first, Iter last, Hash hash) {
   std::uint16_t el_count =
       num_vals > (std::numeric_limits<std::uint16_t>::max)()
           ? max_els
-           : std::max(static_cast<std::uint16_t>(num_vals), max_els);
+           : (static_cast<std::uint16_t>(num_vals) > max_els ? static_cast<std::uint16_t>(num_vals) : max_els);
   auto bits_per_hash = (sizeof(std::size_t) * __CHAR_BIT__) / el_count;
   std::size_t phase = 1U << (bits_per_hash - 1);
   std::size_t composite_hash = 0;
@@ -0,0 +1,26 @@
+#include "TimeSpan.h"
+#include <boost/date_time/posix_time/time_formatters.hpp>
+#include <boost/date_time/posix_time/time_parsers.hpp>
+
+namespace bts::core {
+
+TimeSpan TimeSpan::from_iso_string(const std::string &iso_string) {
+   const auto pdur = boost::posix_time::duration_from_string(iso_string);
+   return from_posix_duration(pdur);
+}
+
+std::string TimeSpan::to_iso_string() const {
+   const auto pdur = to_posix_duration();
+   return boost::posix_time::to_iso_string(pdur);
+}
+
+std::ostream &operator<<(std::ostream &out, const TimeSpan &ts) noexcept {
+   out << ts.to_iso_string();
+   return out;
+}
+
+TimeSpan TimeSpan::from_posix_duration(boost::posix_time::time_duration d) {
+   return TimeSpan{d.total_nanoseconds()};
+}
+
+} // namespace bts::core
@@ -0,0 +1,125 @@
+#pragma once
+#include "lib/core/FMT_IMPL.h"
+#include "lib/utils/bts_assert.h"
+#include "lib/utils/datetime.h"
+#include <boost/date_time/posix_time/conversion.hpp>
+#include <boost/date_time/posix_time/posix_time_config.hpp>
+#include <boost/date_time/posix_time/posix_time_duration.hpp>
+#include <boost/date_time/posix_time/ptime.hpp>
+#include <boost/date_time/time.hpp>
+#include <chrono>
+#include <cstdint>
+#include <fmt/format.h>
+#include <string>
+
+namespace bts::core {
+
+/**
+ * @brief Represents a duration of time
+ */
+class TimeSpan {
+ public:
+   constexpr TimeSpan() = default;
+   explicit constexpr TimeSpan(std::int64_t ticks);
+
+   static constexpr TimeSpan zero();
+
+   static constexpr TimeSpan from_nanos(std::int64_t value);
+   static constexpr TimeSpan from_micros(std::int64_t value);
+   static constexpr TimeSpan from_millis(std::int64_t value);
+   static constexpr TimeSpan from_seconds(std::int64_t value);
+   static TimeSpan from_posix_duration(boost::posix_time::time_duration t);
+   static TimeSpan from_iso_string(const std::string &iso_string);
+
+   constexpr std::int64_t nanos() const;
+   constexpr std::int64_t micros() const;
+   constexpr std::int64_t millis() const;
+   constexpr std::int64_t seconds() const;
+
+   constexpr double total_millis() const;
+   constexpr double total_seconds() const;
+
+   constexpr boost::posix_time::time_duration to_posix_duration() const;
+   std::string to_iso_string() const;
+
+ private:
+   // nanos
+   std::int64_t ticks_{};
+};
+
+constexpr bool operator==(TimeSpan lhs, TimeSpan rhs) noexcept;
+constexpr bool operator!=(TimeSpan lhs, TimeSpan rhs) noexcept;
+constexpr bool operator<(TimeSpan lhs, TimeSpan rhs) noexcept;
+constexpr bool operator>(TimeSpan lhs, TimeSpan rhs) noexcept;
+
+std::ostream &operator<<(std::ostream &out, const TimeSpan &ts) noexcept;
+
+//////////////////////////////////////////////////
+// Definition
+//////////////////////////////////////////////////
+
+constexpr TimeSpan::TimeSpan(std::int64_t ticks) : ticks_(ticks) {}
+
+constexpr TimeSpan TimeSpan::zero() { return TimeSpan{0}; }
+
+constexpr TimeSpan TimeSpan::from_nanos(std::int64_t value) {
+   return TimeSpan{value};
+}
+
+constexpr TimeSpan TimeSpan::from_micros(std::int64_t value) {
+   constexpr const auto nanos_per_micro = 1'000;
+   return TimeSpan{value * nanos_per_micro};
+}
+
+constexpr TimeSpan TimeSpan::from_millis(std::int64_t value) {
+   constexpr const auto nanos_per_milli = 1'000'000;
+   return TimeSpan{value * nanos_per_milli};
+}
+
+constexpr TimeSpan TimeSpan::from_seconds(std::int64_t value) {
+   constexpr const auto nanos_per_second = 1'000'000'000;
+   return TimeSpan{value * nanos_per_second};
+}
+
+constexpr std::int64_t TimeSpan::nanos() const { return ticks_; }
+
+constexpr std::int64_t TimeSpan::micros() const { return ticks_ / 1'000; }
+
+constexpr std::int64_t TimeSpan::millis() const { return ticks_ / 1'000'000; }
+
+constexpr std::int64_t TimeSpan::seconds() const {
+   return ticks_ / 1'000'000'000;
+}
+
+constexpr double TimeSpan::total_millis() const {
+   return static_cast<double>(ticks_) / 1000'000;
+}
+
+constexpr double TimeSpan::total_seconds() const {
+   return static_cast<double>(ticks_) / 1000'000'000;
+}
+
+constexpr boost::posix_time::time_duration TimeSpan::to_posix_duration() const {
+   return boost::date_time::subsecond_duration<
+       boost::posix_time::time_duration, 1000000000>{nanos()};
+}
+
+constexpr bool operator==(TimeSpan lhs, TimeSpan rhs) noexcept {
+   return lhs.nanos() == rhs.nanos();
+}
+
+constexpr bool operator!=(TimeSpan lhs, TimeSpan rhs) noexcept {
+   return lhs.nanos() != rhs.nanos();
+}
+
+constexpr bool operator<(TimeSpan lhs, TimeSpan rhs) noexcept {
+   return lhs.nanos() < rhs.nanos();
+}
+
+constexpr bool operator>(TimeSpan lhs, TimeSpan rhs) noexcept {
+   return lhs.nanos() > rhs.nanos();
+}
+
+} // namespace bts::core
+
+FMT_IMPL(bts::core::TimeSpan, t, "{}", t.to_iso_string());
@@ -0,0 +1,58 @@
+#include "TimeStamp.h"
+#include "lib/utils/datetime.h"
+#include <boost/date_time/posix_time/time_formatters.hpp>
+
+namespace bts::core {
+
+using std::uint32_t;
+
+static bool is_valid_ymd(uint32_t year, uint32_t month, uint32_t day) {
+   return year >= 1970 && month >= 1 && month <= 12 && day >= 1 && day <= 31;
+}
+
+TimeStamp::TimeStamp(uint32_t year, uint32_t month, uint32_t day) {
+   bts_assert_msgf(
+       is_valid_ymd(year, month, day), "Invalid timestamp {}/{}/{} (y/m/d)",
+       year, month, day);
+   std::tm date = {};
+   date.tm_year = year - 1900;
+   date.tm_mon = month - 1;
+   date.tm_mday = day;
+   auto time = std::chrono::system_clock::from_time_t(mktime(&date));
+   auto duration = time.time_since_epoch();
+   value_ =
+       std::chrono::duration_cast<std::chrono::nanoseconds>(duration).count();
+}
+
+bool TimeStamp::from_ymd(
+    std::uint32_t year, std::uint32_t month, std::uint32_t day, TimeStamp *result) {
+   if (!is_valid_ymd(year, month, day)) {
+      return false;
+   }
+   *result = TimeStamp{year, month, day};
+   return true;
+}
+
+TimeStamp TimeStamp::from_iso_string(const std::string &iso_string) {
+   auto ptime = datetime::parse_iso_date_time_string(iso_string);
+   return from_ptime(ptime);
+}
+
+std::string TimeStamp::to_iso_string() const {
+   return boost::posix_time::to_iso_string(to_ptime());
+}
+
+std::ostream &operator<<(std::ostream &out, const TimeStamp &ts) noexcept {
+   out << ts.to_iso_string();
+   return out;
+}
+
+TimeStamp TimeStamp::from_ptime(boost::posix_time::ptime t) {
+   // time_t is Unix time: number of seconds since epoch Jan 1, 1970
+   auto epoch_pt =
+       t - boost::posix_time::ptime(boost::gregorian::date(1970, 1, 1));
+   auto epoch_nanos = epoch_pt.total_nanoseconds();
+   return TimeStamp{static_cast<std::uint64_t>(epoch_nanos)};
+}
+
+} // namespace bts::core
@@ -1,27 +1,211 @@
 #pragma once
+#include "lib/core/FMT_IMPL.h"
+#include "lib/core/TimeSpan.h"
+#include "lib/utils/bts_assert.h"
+#include "lib/utils/datetime.h"
+
+#include <boost/date_time/posix_time/conversion.hpp>
+#include <boost/date_time/posix_time/posix_time_duration.hpp>
+#include <boost/date_time/posix_time/ptime.hpp>
+#include <boost/date_time/time.hpp>
+#include <chrono>
 #include <cstdint>
+#include <fmt/format.h>
+#include <limits>
+#include <string>

 namespace bts::core {

-// I can send this whole class later but its a nice wrapper around the linux
-// posix time
+/**
+ * @brief Represents time as nano-seconds since the unix epoch (Jan 1, 1970)
+ */
 struct TimeStamp {
 public:
   constexpr TimeStamp() = default;
   explicit constexpr TimeStamp(std::uint64_t value);
+   // constructs a TimeStamp from a UTC date
+   explicit TimeStamp(uint32_t year, uint32_t month, uint32_t day);
+
+   static constexpr TimeStamp min();
+   static constexpr TimeStamp max();
+
+   static constexpr TimeStamp from_int64(std::int64_t value);
+   static constexpr TimeStamp from_nanos(std::int64_t value);
+   static constexpr TimeStamp from_micros(std::int64_t value);
+   static constexpr TimeStamp from_millis(std::int64_t value);
+   static constexpr TimeStamp from_seconds(std::int64_t value);
+   static TimeStamp from_ptime(boost::posix_time::ptime t);
+   static TimeStamp from_iso_string(const std::string &iso_string);
+   static bool from_ymd(
+       std::uint32_t year,
+       std::uint32_t month,
+       std::uint32_t day,
+       TimeStamp *result);
+
   constexpr std::uint64_t nanos() const;
+   constexpr std::uint64_t micros() const;
+   constexpr std::uint64_t millis() const;
+   constexpr std::uint64_t seconds() const;
+
+   constexpr double total_millis() const;
+   constexpr double total_seconds() const;
+
+   constexpr std::int64_t to_int64();
+   inline void to_date(uint32_t &year, uint32_t &month, uint32_t &day);
+   constexpr boost::posix_time::ptime to_ptime() const;
+   std::string to_iso_string() const;
+   // current epoch ns
+   static uint64_t now_ns() {
+#ifndef _WIN32
+      return datetime::unix_nano_now();
+#else
+      auto now = std::chrono::system_clock::now();
+      auto ns = std::chrono::duration_cast<std::chrono::nanoseconds>(
+                    now.time_since_epoch())
+                    .count();
+      return ns;
+#endif
+   }
+   static TimeStamp now() { return from_nanos(now_ns()); }

 private:
   // posix time
   std::uint64_t value_{};
 };

+constexpr bool operator==(TimeStamp lhs, TimeStamp rhs) noexcept;
+constexpr bool operator!=(TimeStamp lhs, TimeStamp rhs) noexcept;
+constexpr bool operator<(TimeStamp lhs, TimeStamp rhs) noexcept;
+constexpr bool operator<=(TimeStamp lhs, TimeStamp rhs) noexcept;
+constexpr bool operator>(TimeStamp lhs, TimeStamp rhs) noexcept;
+constexpr bool operator>=(TimeStamp lhs, TimeStamp rhs) noexcept;
+
+constexpr TimeSpan operator-(TimeStamp lhs, TimeStamp rhs) noexcept;
+constexpr TimeStamp operator-(TimeStamp lhs, TimeSpan rhs) noexcept;
+constexpr TimeStamp operator+(TimeStamp lhs, TimeSpan rhs) noexcept;
+
+std::ostream &operator<<(std::ostream &out, const TimeStamp &ts) noexcept;
+
 //////////////////////////////////////////////////
 // Definition
 //////////////////////////////////////////////////

 constexpr TimeStamp::TimeStamp(std::uint64_t value) : value_(value) {}

+constexpr TimeStamp TimeStamp::min() { return TimeStamp{0}; }
+
+constexpr TimeStamp TimeStamp::max() {
+   return TimeStamp{std::numeric_limits<std::uint64_t>::max()};
+}
+
+constexpr TimeStamp TimeStamp::from_int64(std::int64_t value) {
+   bts_assert(value >= 0);
+   return TimeStamp{static_cast<std::uint64_t>(value)};
+}
+
+constexpr TimeStamp TimeStamp::from_nanos(std::int64_t value) {
+   bts_assert(value >= 0);
+   return TimeStamp{static_cast<std::uint64_t>(value)};
+}
+
+constexpr TimeStamp TimeStamp::from_micros(std::int64_t value) {
+   bts_assert(value >= 0);
+   constexpr const auto nanos_per_micro = 1'000;
+   return TimeStamp{static_cast<std::uint64_t>(value * nanos_per_micro)};
+}
+
+constexpr TimeStamp TimeStamp::from_millis(std::int64_t value) {
+   bts_assert(value >= 0);
+   constexpr const auto nanos_per_milli = 1'000'000;
+   return TimeStamp{static_cast<std::uint64_t>(value * nanos_per_milli)};
+}
+
+constexpr TimeStamp TimeStamp::from_seconds(std::int64_t value) {
+   bts_assert(value >= 0);
+   constexpr const auto nanos_per_second = 1'000'000'000;
+   return TimeStamp{static_cast<std::uint64_t>(value * nanos_per_second)};
+}
+
 constexpr std::uint64_t TimeStamp::nanos() const { return value_; }

+constexpr std::uint64_t TimeStamp::micros() const { return value_ / 1'000; }
+
+constexpr std::uint64_t TimeStamp::millis() const { return value_ / 1'000'000; }
+
+constexpr std::uint64_t TimeStamp::seconds() const {
+   return value_ / 1'000'000'000;
+}
+
+constexpr double TimeStamp::total_millis() const {
+   return static_cast<double>(value_) / 1000'000;
+}
+
+constexpr double TimeStamp::total_seconds() const {
+   return static_cast<double>(value_) / 1000'000'000;
+}
+
+constexpr std::int64_t TimeStamp::to_int64() {
+   bts_assert(
+       value_ <=
+       static_cast<std::uint64_t>(std::numeric_limits<std::int64_t>::max()));
+   return static_cast<std::int64_t>(value_);
+}
+void TimeStamp::to_date(uint32_t &year, uint32_t &month, uint32_t &day) {
+   std::uint64_t epoch_secs = value_ / (1000 * 1000 * 1000);
+   time_t ts_tt(epoch_secs);
+   struct tm *ptm = gmtime(&ts_tt);
+   year = ptm->tm_year + 1900;
+   month = ptm->tm_mon + 1;
+   day = ptm->tm_mday;
+}
+
+constexpr boost::posix_time::ptime TimeStamp::to_ptime() const {
+   constexpr const auto nanos_per_sec = 1'000'000'000;
+   constexpr const auto nanos_per_micro = 1'000;
+   // time_t is Unix time: number of seconds since epoch Jan 1, 1970
+   std::time_t ts = value_ / nanos_per_sec;
+   const auto nanos_rem = value_ % nanos_per_sec;
+   return boost::posix_time::ptime(boost::gregorian::date(1970, 1, 1)) +
+          boost::posix_time::seconds(static_cast<long>(ts)) +
+          boost::posix_time::microseconds(nanos_rem / nanos_per_micro);
+}
+
+constexpr bool operator==(TimeStamp lhs, TimeStamp rhs) noexcept {
+   return lhs.nanos() == rhs.nanos();
+}
+
+constexpr bool operator!=(TimeStamp lhs, TimeStamp rhs) noexcept {
+   return lhs.nanos() != rhs.nanos();
+}
+
+constexpr bool operator<(TimeStamp lhs, TimeStamp rhs) noexcept {
+   return lhs.nanos() < rhs.nanos();
+}
+
+constexpr bool operator<=(TimeStamp lhs, TimeStamp rhs) noexcept {
+   return lhs.nanos() <= rhs.nanos();
+}
+
+constexpr bool operator>(TimeStamp lhs, TimeStamp rhs) noexcept {
+   return lhs.nanos() > rhs.nanos();
+}
+
+constexpr bool operator>=(TimeStamp lhs, TimeStamp rhs) noexcept {
+   return lhs.nanos() >= rhs.nanos();
+}
+
+constexpr TimeSpan operator-(TimeStamp lhs, TimeStamp rhs) noexcept {
+   return TimeSpan::from_nanos(lhs.nanos() - rhs.nanos());
+}
+
+constexpr TimeStamp operator-(TimeStamp lhs, TimeSpan rhs) noexcept {
+   return TimeStamp{lhs.nanos() - rhs.nanos()};
+}
+
+constexpr TimeStamp operator+(TimeStamp lhs, TimeSpan rhs) noexcept {
+   return TimeStamp{lhs.nanos() + rhs.nanos()};
+}
+
 } // namespace bts::core
+
+FMT_IMPL(bts::core::TimeStamp, t, "{}", t.to_iso_string());
@@ -1,10 +1,14 @@
 #pragma once
+
 #include "Side.h"
+#include "lib/core/FMT_IMPL.h"
 #include "lib/utils/bts_assert.h"
+
 #include <cmath>
 #include <cstdint>
 #include <limits>
 #include <ostream>
+#include <string>

 #if bts_assert_msg
 #define __ASSERT_MSG(EXP, msg) bts_assert_msg(EXP, msg)
@@ -490,6 +494,8 @@ inline std::ostream &operator<<(std::ostream &out, Price p) noexcept {

 } // namespace bts::domain

+FMT_IMPL(bts::domain::Price, p, "{}", p.to_double());
+
 namespace std { // NOLINT

 constexpr size_t hash<bts::domain::Price>::operator()(
@@ -1,5 +1,9 @@
 #pragma once
+
+#include "lib/core/FMT_IMPL.h"
+#include "lib/logging/logging.h"
 #include "lib/utils/ValueWrapper.h"
+
 #include <cstdint>

 namespace bts::domain {
@@ -110,3 +114,5 @@ QTY_UINT32_BIN_REF_OP(%=)
 #undef QTY_UINT32_BIN_REF_OP

 } // namespace bts::domain
+
+FMT_IMPL(bts::domain::Quantity, q, "{}", *q);
@@ -1,4 +1,6 @@
 #pragma once
+
+#include "lib/core/FMT_IMPL.h"
 #include "lib/utils/ValueWrapper.h"
 #include <cstdint>

@@ -7,3 +9,5 @@ namespace bts::domain {
 using SecurityID = utils::ValueWrapper<struct SecuirtyIDTag, std::uint64_t>;

 } // namespace bts::domain
+
+FMT_IMPL(bts::domain::SecurityID, id, "{}", *id);
@@ -1,12 +1,15 @@
 #pragma once

-#include "lib/utils/datetime.h"
-#include <cmath>
-#include <memory>
-
+#include "lib/core/FMT_IMPL.h"
 #include "lib/domain_model/Security.pb.h"
+#include "lib/domain_model/Side.h"
+#include "lib/domain_model/Venue.h"
 #include "lib/domain_model/common.h"
 #include "lib/utils/bts_assert.h"
+#include "lib/utils/datetime.h"
+
+#include <cmath>
+#include <memory>

 namespace boost {
 namespace gregorian {
@@ -114,6 +117,10 @@ inline bool is_same(Price p1, Price p2) {
   return std::abs(p1.value() - p2.value()) < market_epsilon();
 }

+inline bool is_same(double d1, double d2) {
+   return std::abs(d1 - d2) < market_epsilon();
+}
+
 inline bool is_equal_or_better(api::Side side, Price p1, Price p2) {
   return is_same(p1, p2) || is_better(side, p1, p2);
 }
@@ -0,0 +1,18 @@
+#pragma once
+
+#include "lib/core/EnumMacros.h"
+#include "lib/core/FMT_IMPL.h"
+#include "lib/domain_model/Security.pb.h"
+#include "lib/io/JsonReader.h"
+#include "lib/logging/logging.h"
+
+#include <nlohmann/json.hpp>
+
+namespace bts::api {
+
+PB_ENUM_FROM_JSON(Side)
+PB_ENUM_TO_JSON(Side)
+
+} // namespace bts::api
+
+FMT_IMPL(bts::api::Side, s, "{}", bts::api::Side_Name(s));
@@ -0,0 +1,18 @@
+#pragma once
+
+#include "lib/core/EnumMacros.h"
+#include "lib/core/FMT_IMPL.h"
+#include "lib/domain_model/Security.pb.h"
+#include "lib/io/JsonReader.h"
+#include "lib/logging/logging.h"
+
+#include <nlohmann/json.hpp>
+
+namespace bts::api {
+
+PB_ENUM_FROM_JSON(Venue)
+PB_ENUM_TO_JSON(Venue)
+
+} // namespace bts::api
+
+FMT_IMPL(bts::api::Venue, v, "{}", bts::api::Venue_Name(v));
@@ -6,6 +6,7 @@
 #include <cstdint>
 #include <string>

+#include "lib/core/FMT_IMPL.h"
 #include "lib/domain_model/SecurityKey.h"
 #include "lib/utils/ValueWrapper.h"

@@ -102,3 +103,6 @@ using Price = utils::ValueWrapper<struct OrderPriceTag, double>;
 using Quantity = CQuantity;
 using UserLogin = utils::ValueWrapper<struct UserLoginTag, std::string>;
 } // namespace bts
+
+FMT_IMPL(bts::Price, p, "{}", p.value());
+FMT_IMPL(bts::Quantity, q, "{}", q.value());
@@ -0,0 +1,48 @@
+#pragma once
+#include <nlohmann/json.hpp>
+#include <sstream>
+#include <string>
+
+namespace bts::io {
+
+class JsonDumper {
+ public:
+   template <class T>
+   static std::string dump(const T &value, int indentation = 2);
+
+   template <class Iter>
+   static std::string dump(
+       Iter first,
+       Iter last,
+       int indentation = 2,
+       const char *separator = ",\n");
+};
+
+//////////////////////////////////////////////////
+// Definition
+//////////////////////////////////////////////////
+
+template <class T>
+std::string JsonDumper::dump(const T &value, int indentation) {
+   nlohmann::json j;
+   to_json(j, value);
+   return j.dump(indentation);
+}
+
+template <class Iter>
+std::string JsonDumper::dump(
+    Iter first, Iter last, int indentation, const char *separator) {
+   if (first == last) {
+      return "";
+   }
+   std::stringstream str;
+   auto it = first;
+   str << "[" << dump(*it, indentation);
+   while (++it != last) {
+      str << separator << dump(*it, indentation);
+   }
+   str << "]";
+   return str.str();
+}
+
+} // namespace bts::io
@@ -0,0 +1,22 @@
+#pragma once
+
+#define _TO_JSON_FIELD(field) j[#field] = t.field;
+#define _FROM_JSON_FIELD(field) bts::io::JsonReader{&j, #field}.read(t.field);
+
+#define SPARK_TO_JSON_IMPL(type, for_each)                                     \
+   inline void to_json(nlohmann::json &j, const type &t) {                     \
+      FOR_EACH(_TO_JSON_FIELD)                                                 \
+   }
+
+#define SPARK_FROM_JSON_IMPL(type, for_each)                                   \
+   inline void from_json(const nlohmann::json &j, type &t) {                   \
+      FOR_EACH(_FROM_JSON_FIELD)                                               \
+   }
+
+#define OSTREAM_JSON_IMPL(type)                                                \
+   inline std::ostream &operator<<(std::ostream &out, const type &t) {         \
+      nlohmann::json j;                                                        \
+      to_json(j, t);                                                           \
+      out << j.dump(2);                                                        \
+      return out;                                                              \
+   }
@@ -0,0 +1,520 @@
+#include "JsonReader.h"
+#include "lib/algorithm/TryCatch.h"
+#include "lib/core/TimeStamp.h"
+#include "lib/logging/logging.h"
+#include "lib/utils/TimeStamp.h"
+#include "lib/utils/datetime.h"
+#include <boost/date_time/posix_time/ptime.hpp>
+#include <exception>
+#include <limits>
+#include <string>
+
+namespace bts::io {
+
+using core::TimeStamp;
+using nlohmann::json;
+using std::int32_t;
+using std::int64_t;
+using std::numeric_limits;
+using std::stoull;
+using std::string;
+using std::uint32_t;
+using std::uint64_t;
+
+// 'read' fn bodies should be wrapped between BEGIN/END_READ_JSON tags.
+// These macros abstract out repetative exception handling code (and a nullptr check).
+
+#define BEGIN_READ_JSON                                                        \
+   if (json_ == nullptr) {                                                     \
+      return false;                                                            \
+   }                                                                           \
+   try
+
+#define END_READ_JSON(type_name)                                               \
+   catch (const std::exception &e) {                                           \
+      LOG(Gen, Diag, "Failed to parse json to '{}'. Exception: {}",            \
+          std::string{type_name}, std::string{e.what()});                      \
+   }                                                                           \
+   catch (...) {                                                               \
+      bts_assert_msgf(                                                         \
+          false, "Failed to parse json to '{}'. Exception: unknown",           \
+          std::string{type_name});                                             \
+   }                                                                           \
+   return false;
+
+// The idea here is we parse numbers to wide types (either int64, uint64, or double),
+// then cast that value to the (potentially narrower) output type. This macro evaluates
+// to true if 'wide_val' is within the bounds of the result type (otherwise false).
+#define NUM_WITHIN_BOUNDS(wide_val, result_t)                                  \
+   (wide_val >=                                                                \
+        static_cast<decltype(wide_val)>(numeric_limits<result_t>::min()) &&    \
+    wide_val <=                                                                \
+        static_cast<decltype(wide_val)>(numeric_limits<result_t>::max()))
+
+static const nlohmann::json *get_field_json(
+    const nlohmann::json *json, const std::string &name) {
+   if (!json || !json->is_object()) {
+      return nullptr;
+   }
+   auto it = json->find(name);
+   if (it == json->end()) {
+      return nullptr;
+   }
+   return &it.value();
+}
+
+JsonReader::JsonReader(const nlohmann::json *json) : json_(json) {}
+
+JsonReader::JsonReader(const nlohmann::json *json, const std::string &name)
+    : json_(get_field_json(json, name)) {}
+
+bool JsonReader::try_parse(
+    const std::string &str, nlohmann::json &result, std::string *err) {
+   try {
+      result = nlohmann::json::parse(str);
+      return true;
+   }
+   CATCH_AND_HANDLE_MSG([err](const auto &msg) {
+      if (err != nullptr) {
+         *err = msg;
+      }
+   })
+   return false;
+}
+
+bool JsonReader::read(const string &name, string &result) const {
+   return JsonReader{json_, name}.read(result);
+}
+
+bool JsonReader::read(const string &name, int32_t &result) const {
+   return JsonReader{json_, name}.read(result);
+}
+
+bool JsonReader::read(const string &name, uint32_t &result) const {
+   return JsonReader{json_, name}.read(result);
+}
+
+bool JsonReader::read(const string &name, int64_t &result) const {
+   return JsonReader{json_, name}.read(result);
+}
+
+bool JsonReader::read(const string &name, uint64_t &result) const {
+   return JsonReader{json_, name}.read(result);
+}
+
+bool JsonReader::read(const string &name, float &result) const {
+   return JsonReader{json_, name}.read(result);
+}
+
+bool JsonReader::read(const string &name, double &result) const {
+   return JsonReader{json_, name}.read(result);
+}
+
+bool JsonReader::read(const string &name, TimeStamp &result) const {
+   return JsonReader{json_, name}.read(result);
+}
+
+bool JsonReader::read_epoch_nanos(const string &name, TimeStamp &value) const {
+   return JsonReader{json_, name}.read_epoch_nanos(value);
+}
+
+bool JsonReader::read_epoch_micros(const string &name, TimeStamp &value) const {
+   return JsonReader{json_, name}.read_epoch_micros(value);
+}
+
+bool JsonReader::read_epoch_millis(const string &name, TimeStamp &value) const {
+   return JsonReader{json_, name}.read_epoch_millis(value);
+}
+
+bool JsonReader::read(const string &name, nlohmann::json &result) const {
+   return JsonReader{json_, name}.read(result);
+}
+
+bool JsonReader::read(int32_t &result) const {
+   BEGIN_READ_JSON {
+      switch (json_->type()) {
+      case nlohmann::json::value_t::number_float: {
+         const auto d = json_->get<double>();
+         result = static_cast<int32_t>(d);
+         if (result != d) {
+            LOG(Gen, Warn,
+                "Potential loss of data converting field of type "
+                "'number_float' ({}) to int32 ({})",
+                d, result);
+         }
+         return true;
+      }
+      case nlohmann::json::value_t::number_integer: {
+         const auto il = json_->get<int64_t>();
+         if (!NUM_WITHIN_BOUNDS(il, int32_t)) {
+            LOG(Gen, Error,
+                "Failed to convert field of type 'number_integer' ({}) to "
+                "uint32: value is negative.",
+                il);
+         }
+         result = static_cast<int32_t>(il);
+         return true;
+      }
+      case nlohmann::json::value_t::number_unsigned: {
+         const auto ul = json_->get<uint64_t>();
+         if (ul > static_cast<uint64_t>(numeric_limits<int32_t>::max())) {
+            LOG(Gen, Error,
+                "Failed to convert field of type 'number_unsigned' ({}) to "
+                "int32: value is too large.",
+                ul);
+            return false;
+         }
+         result = static_cast<int32_t>(ul);
+         return true;
+      }
+      case nlohmann::json::value_t::string: {
+         const auto str = json_->get<string>();
+         if (!str.empty()) {
+            result = std::stoi(str);
+            return true;
+         }
+         break;
+      }
+      default:
+         return false;
+      }
+   }
+   END_READ_JSON("int32")
+}
+
+bool JsonReader::read(uint32_t &result) const {
+   BEGIN_READ_JSON {
+      switch (json_->type()) {
+      case nlohmann::json::value_t::number_float: {
+         const auto d = json_->get<double>();
+         result = static_cast<uint32_t>(d);
+         if (result != d) {
+            LOG(Gen, Warn,
+                "Potential loss of data converting field of type "
+                "'number_float' ({}) to uint32 ({})",
+                d, result);
+         }
+         return true;
+      }
+      case nlohmann::json::value_t::number_integer: {
+         const auto i = json_->get<int64_t>();
+         if (!NUM_WITHIN_BOUNDS(i, uint32_t)) {
+            LOG(Gen, Error,
+                "Failed to convert field of type 'number_integer' ({}) to "
+                "uint32: value is negative.",
+                i);
+            return false;
+         }
+         result = static_cast<uint32_t>(i);
+         return true;
+      }
+      case nlohmann::json::value_t::number_unsigned: {
+         const auto ul = json_->get<uint64_t>();
+         if (ul > static_cast<uint64_t>(numeric_limits<int32_t>::max())) {
+            LOG(Gen, Error,
+                "Failed to convert field of type 'number_unsigned' ({}) to "
+                "int32: value is too large.",
+                ul);
+            return false;
+         }
+         result = static_cast<int32_t>(ul);
+         return true;
+      }
+      case nlohmann::json::value_t::string: {
+         const auto str = json_->get<string>();
+         if (!str.empty()) {
+            result = std::stoul(str);
+            return true;
+         }
+         break;
+      }
+      default:
+         return false;
+      }
+   }
+   END_READ_JSON("uint32")
+}
+
+bool JsonReader::read(int64_t &result) const {
+   BEGIN_READ_JSON {
+      switch (json_->type()) {
+      case nlohmann::json::value_t::number_float: {
+         const auto d = json_->get<double>();
+         if (result != d) {
+            LOG(Gen, Warn,
+                "Potential loss of data converting field of type "
+                "'number_float' ({}) to int64 ({})",
+                d, result);
+         }
+         result = static_cast<int64_t>(d);
+         return true;
+      }
+      case nlohmann::json::value_t::number_integer: {
+         result = json_->get<int64_t>();
+         return true;
+      }
+      case nlohmann::json::value_t::number_unsigned: {
+         const auto ul = json_->get<uint64_t>();
+         if (NUM_WITHIN_BOUNDS(ul, int64_t)) {
+            LOG(Gen, Error,
+                "Potential loss of data converting field of type "
+                "'number_unsigned' ({}) to int64 ({})",
+                ul, result);
+            return false;
+         }
+         result = static_cast<int64_t>(ul);
+         return true;
+      }
+      case nlohmann::json::value_t::string: {
+         const auto str = json_->get<string>();
+         if (!str.empty()) {
+            result = std::stol(str);
+            return true;
+         }
+         break;
+      }
+      default:
+         return false;
+      }
+   }
+   END_READ_JSON("int64")
+}
+
+bool JsonReader::read(uint64_t &result) const {
+   BEGIN_READ_JSON {
+      switch (json_->type()) {
+      case nlohmann::json::value_t::number_float: {
+         const auto d = json_->get<double>();
+         if (result != d) {
+            LOG(Gen, Warn,
+                "Potential loss of data converting field of type "
+                "'number_float' ({}) to uint64 ({})",
+                d, result);
+         }
+         result = static_cast<uint64_t>(d);
+         return true;
+      }
+      case nlohmann::json::value_t::number_integer: {
+         const auto il = json_->get<int64_t>();
+         if (NUM_WITHIN_BOUNDS(il, uint64_t)) {
+            LOG(Gen, Error,
+                "Potential loss of data converting field of type "
+                "'number_integer' ({}) to uint64 ({})",
+                il, result);
+            return false;
+         }
+         result = static_cast<uint64_t>(il);
+         return true;
+      }
+      case nlohmann::json::value_t::number_unsigned: {
+         result = json_->get<uint64_t>();
+         return true;
+      }
+      case nlohmann::json::value_t::string: {
+         const auto str = json_->get<string>();
+         if (!str.empty()) {
+            result = std::stoul(str);
+            return true;
+         }
+         break;
+      }
+      default:
+         return false;
+      }
+   }
+   END_READ_JSON("uint64")
+}
+
+bool JsonReader::read(float &result) const {
+   BEGIN_READ_JSON {
+      switch (json_->type()) {
+      case nlohmann::json::value_t::number_float:
+      case nlohmann::json::value_t::number_integer:
+      case nlohmann::json::value_t::number_unsigned:
+         json_->get_to(result);
+         return true;
+      case nlohmann::json::value_t::string: {
+         const auto str = json_->get<string>();
+         if (!str.empty()) {
+            result = std::stof(str);
+            return true;
+         }
+         break;
+      }
+      default:
+         return false;
+      }
+   }
+   END_READ_JSON("float")
+}
+
+bool JsonReader::read(double &result) const {
+   BEGIN_READ_JSON {
+      switch (json_->type()) {
+      case nlohmann::json::value_t::number_float:
+      case nlohmann::json::value_t::number_integer:
+      case nlohmann::json::value_t::number_unsigned:
+         json_->get_to(result);
+         return true;
+      case nlohmann::json::value_t::string: {
+         const auto str = json_->get<string>();
+         if (!str.empty()) {
+            result = std::stod(str);
+            return true;
+         }
+         break;
+      }
+      default:
+         return false;
+      }
+   }
+   END_READ_JSON("double")
+}
+
+bool JsonReader::read(string &result) const {
+   BEGIN_READ_JSON {
+      json_->get_to(result);
+      return true;
+   }
+   END_READ_JSON("string")
+}
+
+bool JsonReader::read(nlohmann::json &result) const {
+   BEGIN_READ_JSON {
+      json_->get_to(result);
+      return true;
+   }
+   END_READ_JSON("json-object")
+}
+
+bool JsonReader::read_flat(std::vector<nlohmann::json> &result) const {
+   if (json_ == nullptr) {
+      return false;
+   }
+   switch (json_->type()) {
+   case json::value_t::array:
+      return std::all_of(json_->begin(), json_->end(), [&](const auto &j) {
+         return JsonReader{&j}.read_flat(result);
+      });
+   case json::value_t::null:
+      return false;
+   default:
+      read(result.emplace_back());
+      return true;
+   }
+}
+
+bool JsonReader::parse_time_stamp_(const string &str, TimeStamp &value) const {
+   return parse_iso_time_stamp_(str, value) ||
+          parse_epoch_nanos_time_stamp_(str, value);
+}
+
+bool JsonReader::parse_iso_time_stamp_(
+    const string &str, TimeStamp &value) const {
+   value = TimeStamp::from_iso_string(str);
+   return true;
+}
+
+bool JsonReader::parse_epoch_nanos_time_stamp_(
+    const string &str, TimeStamp &value) const {
+   uint64_t nanos = stoull(str);
+   value = TimeStamp{nanos};
+   return true;
+}
+
+bool JsonReader::read(TimeStamp &result) const {
+   BEGIN_READ_JSON {
+      switch (json_->type()) {
+      case nlohmann::json::value_t::null:
+      case nlohmann::json::value_t::object:
+      case nlohmann::json::value_t::array:
+      case nlohmann::json::value_t::boolean:
+      case nlohmann::json::value_t::binary:
+      case nlohmann::json::value_t::discarded:
+         return false;
+      case nlohmann::json::value_t::string: {
+         const auto str = json_->get<string>();
+         return parse_iso_time_stamp_(str, result) ||
+                parse_epoch_nanos_time_stamp_(str, result);
+      }
+      case nlohmann::json::value_t::number_integer:
+      case nlohmann::json::value_t::number_unsigned:
+      case nlohmann::json::value_t::number_float:
+         // Maybe we could assume seconds here, idk though...when in doubt, prob
+         // better to be cautious than cavalier
+         return false;
+      }
+   }
+   END_READ_JSON("TimeStamp")
+}
+
+bool JsonReader::read_iso(
+    const std::string &name, core::TimeStamp &result) const {
+   return JsonReader{json_, name}.read_iso(result);
+}
+
+bool JsonReader::read_iso(core::TimeStamp &result) const {
+   BEGIN_READ_JSON {
+      switch (json_->type()) {
+      case nlohmann::json::value_t::string: {
+         return parse_iso_time_stamp_(json_->get<string>(), result);
+      }
+      default:
+         return false;
+      }
+   }
+   END_READ_JSON("TimeStamp")
+}
+
+bool JsonReader::read_epoch_nanos(TimeStamp &value) const {
+   int64_t nanos;
+   if (!read(nanos)) {
+      return false;
+   }
+   value = TimeStamp::from_nanos(nanos);
+   return true;
+}
+
+bool JsonReader::read_epoch_micros(TimeStamp &value) const {
+   int64_t micros;
+   if (!read(micros)) {
+      return false;
+   }
+   value = TimeStamp::from_micros(micros);
+   return true;
+}
+
+bool JsonReader::read_epoch_millis(TimeStamp &value) const {
+   int64_t millis;
+   if (!read(millis)) {
+      return false;
+   }
+   value = TimeStamp::from_millis(millis);
+   return true;
+}
+
+bool JsonReader::read_optional_epoch_millis(
+    std::optional<TimeStamp> &result) const {
+   if (json_ == nullptr) {
+      return false;
+   }
+   if (json_->is_null()) {
+      result = std::nullopt;
+      return true;
+   }
+   return read_epoch_millis(result.emplace());
+}
+
+bool JsonReader::read_optional_epoch_millis(
+    const std::string &name, std::optional<TimeStamp> &result) const {
+   if (!json_ || !json_->is_object()) {
+      return false;
+   }
+   auto it = json_->find(name);
+   if (it == json_->end()) {
+      return false;
+   }
+   return JsonReader{&*it}.read_epoch_millis(result.emplace());
+}
+
+} // namespace bts::io
@@ -0,0 +1,291 @@
+#pragma once
+#include "lib/core/TimeStamp.h"
+#include "lib/utils/bts_assert.h"
+
+#include <cstdint>
+#include <nlohmann/json.hpp>
+#include <optional>
+#include <string>
+
+namespace bts::io {
+
+class JsonReader {
+ public:
+   explicit JsonReader(const nlohmann::json *json);
+
+   JsonReader(const nlohmann::json *json, const std::string &name);
+
+   static bool try_parse(
+       const std::string &str,
+       nlohmann::json &result,
+       std::string *err = nullptr);
+
+   bool read(const std::string &name, std::string &result) const;
+   bool read(const std::string &name, std::int32_t &result) const;
+   bool read(const std::string &name, std::uint32_t &result) const;
+   bool read(const std::string &name, std::int64_t &result) const;
+   bool read(const std::string &name, std::uint64_t &result) const;
+   bool read(const std::string &name, float &result) const;
+   bool read(const std::string &name, double &result) const;
+   bool read(const std::string &name, nlohmann::json &result) const;
+   bool read(const std::string &name, core::TimeStamp &result) const;
+   bool read_iso(const std::string &name, core::TimeStamp &result) const;
+   bool read_epoch_nanos(
+       const std::string &name, core::TimeStamp &result) const;
+   bool read_epoch_micros(
+       const std::string &name, core::TimeStamp &result) const;
+   bool read_epoch_millis(
+       const std::string &name, core::TimeStamp &result) const;
+
+   template <class T>
+   bool read(const std::string &name, T &result) const;
+
+   template <class T>
+   bool read_one_or_more(
+       const std::string &name,
+       std::vector<T> &result,
+       bool append = true) const;
+
+   template <class T>
+   T read(const std::string &name) const;
+
+   bool read(std::string &result) const;
+   bool read(std::int32_t &result) const;
+   bool read(std::uint32_t &result) const;
+   bool read(std::int64_t &result) const;
+   bool read(std::uint64_t &result) const;
+   bool read(float &result) const;
+   bool read(double &result) const;
+   bool read(nlohmann::json &result) const;
+   bool read(core::TimeStamp &result) const;
+   bool read_iso(core::TimeStamp &result) const;
+   bool read_epoch_nanos(core::TimeStamp &result) const;
+   bool read_epoch_micros(core::TimeStamp &result) const;
+   bool read_epoch_millis(core::TimeStamp &result) const;
+
+   template <class T>
+   bool read_one_or_more(std::vector<T> &result, bool append = true) const;
+
+   template <class T>
+   static bool read_flat(
+       const std::vector<nlohmann::json> &items, std::vector<T> &result);
+
+   bool read_flat(std::vector<nlohmann::json> &result) const;
+
+   template <class T>
+   bool read_optional(std::optional<T> &result) const;
+
+   template <class T>
+   bool read_optional(const std::string &name, std::optional<T> &result) const;
+
+   bool read_optional_epoch_millis(
+       std::optional<core::TimeStamp> &result) const;
+   bool read_optional_epoch_millis(
+       const std::string &name, std::optional<core::TimeStamp> &result) const;
+
+   template <class T>
+   bool read(T &result) const;
+
+   template <class T>
+   T read() const;
+
+   /**
+    * @brief Reads a json field into \p result
+    *
+    * @details If the read fails, assigns \p default_value to \p result
+    * @return True if the read succeeds, otherwise false
+    */
+   template <class T>
+   bool read_or(const std::string &name, T &result, T default_value) const;
+
+ private:
+   const nlohmann::json *json_;
+
+   bool parse_time_stamp_(
+       const std::string &str, core::TimeStamp &result) const;
+   bool parse_iso_time_stamp_(
+       const std::string &str, core::TimeStamp &result) const;
+   bool parse_epoch_nanos_time_stamp_(
+       const std::string &str, core::TimeStamp &result) const;
+};
+
+//////////////////////////////////////////////////
+// Definition
+//////////////////////////////////////////////////
+
+#define CATCH_PARSE_ERRORS(type_name)                                          \
+   catch (const std::exception &e) {                                           \
+      LOG(Gen, Diag, "Failed to parse json to '{}'. Exception: {}",            \
+          std::string{type_name}, std::string{e.what()});                      \
+   }                                                                           \
+   catch (...) {                                                               \
+      bts_assert_msgf(                                                         \
+          false, "Failed to parse json to '{}'. Exception: unknown",           \
+          std::string{type_name});                                             \
+   }
+
+template <class T>
+inline bool JsonReader::read(const std::string &name, T &result) const {
+   if (!json_ || !json_->is_object()) {
+      return false;
+   }
+   auto it = json_->find(name);
+   if (it == json_->end()) {
+      return false;
+   }
+   it->get_to(result);
+   return true;
+}
+
+template <class T>
+inline bool JsonReader::read_one_or_more(
+    const std::string &name, std::vector<T> &result, bool append) const {
+   if (!json_ || !json_->is_object()) {
+      return false;
+   }
+   auto it = json_->find(name);
+   if (it == json_->end()) {
+      return false;
+   }
+   try {
+      switch (it->type()) {
+      case nlohmann::json::value_t::array:
+         if (append) {
+            for (const auto &item : it.value()) {
+               item.get_to(result.emplace_back());
+            }
+         } else {
+            it.value().get_to(result);
+         }
+         return true;
+      default:
+         if (append) {
+            it.value().get_to(result.emplace_back());
+         } else {
+            if (result.empty()) {
+               result.emplace_back();
+            }
+            it.value().get_to(result.at(0));
+            result.erase(result.begin() + 1, result.end());
+         }
+         return true;
+      }
+   }
+   CATCH_PARSE_ERRORS(typeid(T).name())
+   return false;
+}
+
+template <class T>
+inline bool JsonReader::read_one_or_more(
+    std::vector<T> &result, bool append) const {
+   if (!json_) {
+      return false;
+   }
+   try {
+      switch (json_->type()) {
+      case nlohmann::json::value_t::array:
+         if (append) {
+            for (const auto &item : json_->array()) {
+               item.get_to(result.emplace_back());
+            }
+         } else {
+            json_->get_to(result);
+         }
+         return true;
+      default:
+         if (append) {
+            json_->get_to(result.emplace_back());
+         } else {
+            if (result.empty()) {
+               result.emplace_back();
+            }
+            json_->get_to(result.at(0));
+            result.erase(result.begin() + 1, result.end());
+         }
+         return true;
+      }
+   }
+   CATCH_PARSE_ERRORS(typeid(T).name());
+   return false;
+}
+
+template <class T>
+inline bool JsonReader::read(T &result) const {
+   try {
+      json_->get_to(result);
+      return true;
+   } catch (...) {
+      const auto j_str = json_->dump(2);
+      LOG(md, Error, "Exception parsing json:\n{}", j_str);
+      return false;
+   }
+}
+
+template <class T>
+inline T JsonReader::read() const {
+   T result;
+   read(result);
+   return result;
+}
+
+template <class T>
+inline bool JsonReader::read_optional(std::optional<T> &result) const {
+   if (json_ == nullptr) {
+      return false;
+   }
+   if (json_->is_null()) {
+      result = std::nullopt;
+      return true;
+   }
+   return read(result.emplace());
+}
+
+template <class T>
+inline bool JsonReader::read_optional(
+    const std::string &name, std::optional<T> &result) const {
+   if (!json_ || !json_->is_object()) {
+      return false;
+   }
+   auto it = json_->find(name);
+   if (it == json_->end()) {
+      return false;
+   }
+   return JsonReader{&it.value()}.read_optional(result);
+}
+
+template <class T>
+inline T JsonReader::read(const std::string &name) const {
+   bts_assert(json_ && json_->is_object());
+   auto it = json_->find(name);
+   bts_assert_msgf(it != json_->end(), "json doesn't contain field '{}'", name);
+   return JsonReader{&it.value()}.read<T>();
+}
+
+template <class T>
+inline bool JsonReader::read_or(
+    const std::string &name, T &result, T default_value) const {
+   if (read(name, result)) {
+      return true;
+   }
+   result = default_value;
+   return false;
+}
+
+template <class T>
+bool JsonReader::read_flat(
+    const std::vector<nlohmann::json> &items, std::vector<T> &result) {
+   std::vector<nlohmann::json> flat_js;
+   for (const auto &item_j : items) {
+      if (!JsonReader{&item_j}.read_flat(flat_js)) {
+         return false;
+      }
+   }
+   for (const auto &el_j : flat_js) {
+      JsonReader{&el_j}.read(result.emplace_back());
+   }
+   return true;
+}
+
+#undef CATCH_PARSE_ERRORS
+
+} // namespace bts::io
@@ -89,7 +89,7 @@ void WebsocketSSLClientInternal::_handle_error(
   str << "Error_code:" << error_code << " Text:" << err_msg;
   LOG(OrderMan, Info,
       "WebsocketSSLClientInternal::_handle_error error_code:{} err_msg:{}",
-       error_code, err_msg);
+       error_code.message(), err_msg);
   _listener->on_error({err_type, str.str()});
   close_connection();
 }
@@ -131,7 +131,8 @@ void WebsocketSSLClientInternal::_on_connect(
      return;
   }
   LOG(OrderMan, Info,
-       "WebsocketSSLClientInternal::_on_connect successfull ep:{}", ep);
+       "WebsocketSSLClientInternal::_on_connect successfull ep: {}:{}",
+       ep.address().to_string(), ep.port());
   ip_port = _conf.ip + ':' + std::to_string(ep.port());
   // Set a timeout on the operation
   boost::beast::get_lowest_layer(_ws).expires_after(std::chrono::seconds(30));
@@ -4,6 +4,7 @@
 #include <fmt/ostream.h>
 #include <iostream>
 #include <type_traits>
+#include <vector>

 namespace bts {
 namespace log {
@@ -12,7 +13,12 @@ template <class Tuple, std::size_t... Is>
 std::string format_tuple(
    const char *msg_template, Tuple const &t, std::index_sequence<Is...>) {
   try {
-      return fmt::format(msg_template, std::get<Is>(t)...);
+      using ctx = fmt::format_context;
+      std::vector<fmt::basic_format_arg<ctx>> fmt_args;
+      (fmt_args.push_back(fmt::detail::make_arg<ctx>(std::get<Is>(t))), ...);
+      return fmt::vformat(
+          msg_template,
+          fmt::basic_format_args<ctx>(fmt_args.data(), fmt_args.size()));
   } catch (fmt::format_error const &e) {
      return fmt::format(
          "Bad log message template: '{}' Error: '{}'", msg_template, e.what());
@@ -1,8 +1,11 @@
 #include "catch/catch.hpp"
 #include "lib/config/config.h"
 #include "lib/config/protos/logging.pb.h"
-#include "lib/utils/bts_chrono.h"
 #include "lib/logging/logging.h"
+#include "lib/utils/bts_chrono.h"
+
+#include <fmt/format.h>
+#include <fmt/chrono.h>

 namespace bts {
 int logging_tests;
@@ -22,7 +25,8 @@ TEST_CASE("Logging Tests", "[logging] [benchmark]") {
   SECTION("timestamp_interpolation") {
      auto start = bts::Timer::now();
      for (int i = 0; i < sz; i++)
-         LOG(Gen, Info, "Hello World. It is now: {0}", Timer::now());
+         LOG(Gen, Info, "Hello World. It is now: {}",
+             Timer::now().time_since_epoch());
      auto stop = Timer::now();
      LOG(Testing, Info, "Logging: ns/msg for log with timestamps: {} tt: {} ",
          bts::ns_diff(stop, start) / sz, bts::ns_diff(stop, start));
@@ -2,6 +2,7 @@
 #include "ExchangeOrderKey.h"
 #include "IntPrice.h"
 #include "MDProducerHandle.h"
+#include "lib/core/FMT_IMPL.h"
 #include "lib/domain_model/Security.h"
 #include "lib/domain_model/SecurityKey.h"
 #include "lib/utils/TimeStamp.h"
@@ -169,3 +170,5 @@ class MBOProcessor {
 namespace std {
 std::ostream &operator<<(std::ostream &os, const bts::md::inc::Update &);
 }
+
+FMT_IMPL_OSTREAM(bts::md::inc::Update);
@@ -1,5 +1,9 @@
 #pragma once
+
+#include "lib/core/FMT_IMPL.h"
+
 #include <ostream>
+#include <sstream>
 #include <string>

 namespace bts::md {
@@ -30,3 +34,5 @@ constexpr const char *MarketDataError::type_name() const noexcept {
 }

 } // namespace bts::md
+
+FMT_IMPL_OSTREAM(bts::md::MarketDataError);
@@ -1,4 +1,7 @@
 #pragma once
+#ifndef _WIN32
+
+#include "lib/core/FMT_IMPL.h"
 #include "lib/core/ValueWrapper.h"

 namespace bts::md {
@@ -7,3 +10,6 @@ using MarketDataListenerID =
    core::ValueWrapper<struct MarketDataListenedIDTag, std::uint32_t>;

 } // namespace bts::md
+
+FMT_IMPL(bts::md::MarketDataListenerID, id, "{}", *id);
+#endif
@@ -1,4 +1,6 @@
 #pragma once
+#ifndef _WIN32
+
 #include <optional>

 namespace bts::md {
@@ -22,3 +24,4 @@ struct MarketDataRegstrationError {
 };

 } // namespace bts::md
+#endif
@@ -1,3 +1,4 @@
+#ifndef _WIN32
 #include "MarketDataRegistrationFactory.h"
 #include "MarketDataRegistration.h"

@@ -8,3 +9,4 @@ MarketDataRegistration MarketDataRegistrationFactory::create() {
 }

 } // namespace bts::md
+#endif
@@ -1,4 +1,5 @@
 #pragma once
+#ifndef _WIN32
 #include <atomic>
 #include <cstdint>

@@ -15,3 +16,4 @@ class MarketDataRegistrationFactory {
 };

 } // namespace bts::md
+#endif
@@ -1,6 +1,7 @@
 #pragma once

 #include "../common.h"
+#include "lib/core/FMT_IMPL.h"
 #include "lib/utils/ValueWrapper.h"
 #include "lib/utils/bts_array_size.h"
 #include "lib/utils/bts_assert.h"
@@ -284,3 +285,5 @@ void with_security_like(const api::Security &security, C cb) {

 } // namespace md
 } // namespace bts
+
+FMT_IMPL(bts::md::SecurityKey, k, "{}", *k);
@@ -152,7 +152,7 @@ void SyntheticSecurityService::create_synthetic(
                       } else {
                          LOG(Api, Info,
                              "Failed to un-delete security {}. Error {}", id,
-                              result.error());
+                              UpdateResult_Error_Name(result.error()));
                          send_reply(
                              reply,
                              CreateResult::ERROR_SECURITY_ALREADY_EXISTS, id);
@@ -58,7 +58,7 @@ OnixsTemplLicense onixs_create_temporary_license() {
         LOG(md, Warn,
             "Licensing error. Failed to write to '{}'. Please ensure it is "
             "writeable.",
-             linfo.path);
+             linfo.path.string());
      }
   }
   return linfo;
@@ -369,14 +369,13 @@ void Broker::notify(
   if (srec->is_local() || srec->sub_id == ex_subid) {

      LOG(Latency, Diag, "Started handling event {} on broker {}",
-          log::CharPtrWrapper{srec->topic_str}, me->my_id.value());
+          std::string{srec->topic_str}, me->my_id);
      auto start = Timer::now();
      srec->notify(payload);
      auto stop = Timer::now();

      LOG(Latency, Diag, "Finished handling event {} on broker {}. Time: {}us",
-          log::CharPtrWrapper{srec->topic_str}, me->my_id.value(),
-          us_diff(stop, start));
+          std::string{srec->topic_str}, me->my_id, us_diff(stop, start));
   }
 }

@@ -1,5 +1,6 @@
 #pragma once

+#include "lib/core/FMT_IMPL.h"
 #include "lib/mercury/common/FatWrapper.h"
 #include "lib/utils/ValueWrapper.h"
 #include <array>
@@ -30,3 +31,6 @@ using TimeStampStr = utils::ValueWrapper<

 } // namespace mercury
 } // namespace bts
+
+FMT_IMPL(bts::mercury::BrokerID, id, "{}", *id);
+FMT_IMPL(bts::mercury::LocalSubID, id, "{}", *id);
@@ -1,8 +1,10 @@
 #pragma once

+#include "lib/core/FMT_IMPL.h"
 #include "lib/mercury/events/Event.h"
 #include "lib/mercury/topics/ITopic.h"
 #include "lib/utils/type_name.h"
+
 #include <fmt/format.h>
 #include <memory>

@@ -52,3 +54,13 @@ template <class EV> class Topic<EV, false> : public ITopic {

 } // namespace mercury
 } // namespace bts
+
+#define _TOPIC(...) bts::mercury::Topic<__VA_ARGS__>
+
+template <class EV>
+FMT_IMPL_T(_TOPIC(EV, true), t, "{}", t.to_str());
+
+template <class EV>
+FMT_IMPL_T(_TOPIC(EV, false), t, "{}", t.to_str());
+
+#undef _TOPIC
@@ -1,7 +1,11 @@
 #pragma once
+
 #include "lib/api/services/OrderService.pb.h"
+#include "lib/core/FMT_IMPL.h"
 #include "lib/orderman/orderman_utils/orderman_common.h"

+#include <sstream>
+
 namespace bts::orderman {

 struct CancelOrderMsg {
@@ -14,3 +18,5 @@ struct CancelOrderMsg {
 };

 } // namespace bts::orderman
+
+FMT_IMPL_OSTREAM(bts::orderman::CancelOrderMsg);
@@ -1,7 +1,11 @@
 #pragma once
+
+#include "lib/core/FMT_IMPL.h"
 #include "lib/orderman/CreateOrderMsg.h"
 #include "lib/orderman/orderman_utils/orderman_common.h"

+#include <sstream>
+
 namespace bts::orderman {

 struct CancelReplaceOrderMsg {
@@ -10,3 +14,5 @@ struct CancelReplaceOrderMsg {
 };

 } // namespace bts::orderman
+
+FMT_IMPL_OSTREAM(bts::orderman::CancelReplaceOrderMsg);
@@ -1,6 +1,12 @@
 #pragma once
+
+#include "lib/core/FMT_IMPL.h"
+#include "lib/domain_model/Security.h"
+#include "lib/domain_model/common.h"
 #include "lib/orderman/orderman_utils/orderman_common.h"

+#include <sstream>
+
 namespace bts::orderman {

 struct CreateOrderMsg {
@@ -18,3 +24,5 @@ struct CreateOrderMsg {
 };

 } // namespace bts::orderman
+
+FMT_IMPL_OSTREAM(bts::orderman::CreateOrderMsg);
@@ -1,9 +1,12 @@
 #pragma once
+
 #include "lib/domain_model/Security.pb.h"
 #include "lib/orderman/CancelOrderMsg.h"
 #include "lib/orderman/CancelReplaceOrderMsg.h"
 #include "lib/orderman/CreateOrderMsg.h"
 #include "lib/orderman/RFQMsg.h"
+#include "lib/utils/overloaded.h"
+
 #include <variant>

 namespace bts::orderman {
@@ -32,3 +35,32 @@ inline std::ostream &operator<<(std::ostream &out, OrderMsg const &msg) {
   return out;
 }
 } // namespace bts::orderman
+
+template <>
+struct fmt ::formatter<bts ::orderman ::OrderMsg> {
+   constexpr auto parse(fmt ::format_parse_context &ctx)
+       -> decltype(ctx.end()) {
+      return ctx.end();
+   }
+   template <typename FormatContext>
+   auto format(const bts ::orderman ::OrderMsg &m, FormatContext &ctx) const
+       -> decltype(ctx.out()) {
+      std::visit(
+          bts::overloaded{
+              [&ctx](const bts::orderman::CreateOrderMsg &m) {
+                 fmt::format_to(ctx.out(), "{}", m);
+              },
+              [&ctx](const bts::orderman::CancelReplaceOrderMsg &m) {
+                 fmt::format_to(ctx.out(), "{}", m);
+              },
+              [&ctx](const bts::orderman::CancelOrderMsg &m) {
+                 fmt::format_to(ctx.out(), "{}", m);
+              },
+              [&ctx](const bts::orderman::RFQMsg &r) {
+                 fmt::format_to(ctx.out(), "{}", r);
+              },
+              []() {}},
+          m);
+      return fmt::format_to(ctx.out(), "{}", m);
+   }
+};
@@ -1,8 +1,11 @@
 #pragma once
+
 #include "lib/api/services/OrderService.pb.h"
+#include "lib/core/FMT_IMPL.h"
 #include "lib/domain_model/Security.pb.h"
 #include "lib/orderman/events/RFQEvent.h"
 #include "lib/orderman/orderman_utils/orderman_common.h"
+
 #include <optional>

 namespace bts::orderman {
@@ -13,3 +16,5 @@ struct RFQMsg {
 };

 } // namespace bts::orderman
+
+FMT_IMPL(bts::orderman::RFQMsg, m, "{}", m.rfq.DebugString());
@@ -642,7 +642,7 @@ void OrderManager::after_event() {
            LOG(Latency, Info,
                "entry_type:after_event channel:{} sn:{} app_arrival:{} now:{} "
                "delta:{}",
-                log::CharPtrWrapper{corid.channel}, corid.sn,
+                std::string{corid.channel}, corid.sn,
                corid.app_arrival.to_uint64(), now.to_uint64(),
                (now - corid.app_arrival).nanoseconds());
            osd->gw->send_msgs(queued_ops, osd->current_trace);
@@ -25,5 +25,9 @@ class ITheoCalculator {
       bts::api::orderman::PrecomputedTheoPoint &result) const = 0;
   // gets the driver security for a security with id sec_id
   virtual std::optional<uint32_t> get_driver_sec(uint32_t sec_id) const = 0;
+   // given a vol for an option and the current driver, returns the theo that
+   // corresponds to that vol.
+   virtual double get_theo_correspoding_to_vol(
+       uint32_t sec_id, double driver, double vol) const = 0;
 };
 } // namespace bts::orderman::analytics
@@ -50,12 +50,15 @@ void interpolate(
       lower.driver(), lower.theta(), higher.driver(), higher.theta(), driver);
   double rho = lin_interp_exterp(
       lower.driver(), lower.rho(), higher.driver(), higher.rho(), driver);
+   double vol = lin_interp_exterp(
+       lower.driver(), lower.vol(), higher.driver(), higher.vol(), driver);
   result.set_theo(theo);
   result.set_delta(delta);
   result.set_gamma(gamma);
   result.set_vega(vega);
   result.set_theta(theta);
   result.set_rho(rho);
+   result.set_vol(vol);
 }

 struct DriverNeighbors {
@@ -213,6 +216,24 @@ std::optional<uint32_t> TheoCalculator::get_driver_sec(uint32_t sec_id) const {
   return me->total_theo_data.get_driver_sec(sec_id);
 }

+double TheoCalculator::get_theo_correspoding_to_vol(
+    uint32_t sec_id, double driver, double vol) const {
+   PrecomputedTheoPoint rv;
+   if (!get_theo(sec_id, driver, rv)) {
+      return bts::os::nan();
+   }
+   // use the vega to compute the theo
+   double d_vol = vol - rv.vol();
+   double d_theo = rv.vega() * d_vol;
+   double new_theo = rv.theo() + d_theo;
+   LOG(Gen, Info,
+       "TheoCalculator::get_theo_correspoding_to_vol sec_id: {} driver: {} "
+       "vol: {} rv.theo: {} rv.vol: {} rv.vega: {} "
+       "new_theo: {}",
+       sec_id, driver, vol, rv.theo(), rv.vol(), rv.vega(), new_theo);
+   return new_theo;
+}
+
 void TheoCalculator::set_data(const PrecomputedTheos &theos) {
   LOG(Gen, Info, "TheoCalculator::set_data size: {}", theos.theos_size());
   WriteLock write_lock(me->lock);
@@ -23,6 +23,8 @@ class TheoCalculator : public ITheoCalculator {
       double driver,
       bts::api::orderman::PrecomputedTheoPoint &result) const override;
   std::optional<uint32_t> get_driver_sec(uint32_t sec_id) const override;
+   double get_theo_correspoding_to_vol(
+       uint32_t sec_id, double driver, double vol) const override;
   void set_data(const bts::api::orderman::PrecomputedTheos &theos);

 private:
@@ -1,6 +1,7 @@
 #pragma once

 #include "lib/api/services/OrderService.pb.h"
+#include "lib/core/FMT_IMPL.h"
 #include "lib/domain_model/Security.pb.h"
 #include "lib/mercury/events/Event.h"
 #include "lib/tracing/PerfLog.h"
@@ -20,3 +21,5 @@ class RFQEvent : public mercury::Event<RFQEvent> {

 } // namespace orderman
 } // namespace bts
+
+FMT_IMPL(bts::orderman::RFQEvent, e, "{}", e.rfq.DebugString());
@@ -4,6 +4,7 @@
 #include "lib/domain_model/Security.h"
 #include "lib/jetfix/common/OutFIX.h"
 #include "lib/jetfix/store/session_name.h"
+#include "lib/logging/logging.h"
 #include "lib/orderman/events/NewSecuritiesEvent.h"
 #include "lib/orderman/gateways/GLBXExecutionGateway.h"
 #include "lib/orderman/ilink3/IL3GW.h"
@@ -24,14 +25,6 @@ struct OrderStats {
   size_t n_cxls = 0;
 };

-enum class GLBXiLink3ExecutionGateway::State {
-
-   WAITING_FOR_SECURITIES,
-   PAUSED,
-   CONNECTING,
-   READY
-};
-
 struct GLBXiLink3ExecutionGateway::PImpl {
   PImpl(
       config::JetFIX::Connection const &cfg,
@@ -92,7 +85,7 @@ void GLBXiLink3ExecutionGateway::start() {

 void GLBXiLink3ExecutionGateway::dispatch_incoming_fix_messages() {
   if (me->started) {
-      switch (me->state) {
+      switch (me->state.value) {
      case State::WAITING_FOR_SECURITIES:
         break;
      case State::PAUSED:
@@ -181,9 +174,8 @@ void GLBXiLink3ExecutionGateway::connect(jetfix::SessionName const &sn) {

   me->gw->subscribe_state([this](auto gw_state) {
      LOG(iLink3, Info, "Internal gateway state change in {}. State is now {}",
-          me->gw_name.value(),
-          log::CharPtrWrapper{ilink3::state_string(gw_state)});
-      switch (me->state) {
+          me->gw_name.value(), gw_state);
+      switch (me->state.value) {
      case State::CONNECTING:
         if (gw_state == ilink3::State::READY) {
            transition(State::READY);
@@ -262,7 +254,7 @@ void GLBXiLink3ExecutionGateway::send_msgs(
         auto cancel_replace_ptr = std::get_if<CancelReplaceOrderMsg>(&m);
         if (cancel_replace_ptr) {
            LOG(Latency, Info, "Cancel Replace. CorID:{}.{} Order:{}",
-                log::CharPtrWrapper{corid.channel}, corid.sn,
+                std::string{corid.channel}, corid.sn,
                cancel_replace_ptr->exchange_order_id);

            me->gw->send_cancel_replace(*cancel_replace_ptr, corid);
@@ -307,24 +299,9 @@ GLBXiLink3ExecutionGateway::on_exchange_open_orders() {
   return me->on_exchange_open_orders;
 }

-log::CharPtrWrapper GLBXiLink3ExecutionGateway::state_name(State st) {
-   switch (st) {
-   case State::WAITING_FOR_SECURITIES:
-      return {"WAITING_FOR_SECURITIES"};
-   case State::PAUSED:
-      return {"PAUSED"};
-   case State::CONNECTING:
-      return {"CONNECTING"};
-   case State::READY:
-      return {"READY"};
-   }
-   bts_assert(false);
-   return {""};
-}
-
 void GLBXiLink3ExecutionGateway::transition(State new_state) {
   LOG(iLink3, Info, "Transitioning {} from {} to {}", me->gw_name.value(),
-       state_name(me->state), state_name(new_state));
+       me->state, new_state);
   me->state = new_state;
 }

@@ -1,7 +1,7 @@
 #pragma once

 #include "IExecutionGateway.h"
-#include "lib/logging/logging.h"
+#include "lib/orderman/gateways/GLBXiLink3ExecutionGatewayState.h"

 namespace bts {
 namespace jetfix {
@@ -36,7 +36,7 @@ class GLBXiLink3ExecutionGateway : public IExecutionGateway {
   OnExchangeOpenOrdersState &on_exchange_open_orders() override;

 private:
-   enum class State;
+   using State = GLBXiLink3ExecutionGatewayState;
   struct PImpl;
   std::unique_ptr<PImpl> me;

@@ -44,7 +44,6 @@ class GLBXiLink3ExecutionGateway : public IExecutionGateway {
   void on_time();
   bool pause_in_effect() const;
   bool max_failures_reached() const;
-   static log::CharPtrWrapper state_name(State);
   void transition(State);

   void pause();
Author	SHA1	Message	Date
John Colvin	55dba0f50a	build: Upgrade (Windows) c++ to stdlib 20	2023-12-23 13:23:52 -05:00
John Colvin	ec7457b04c	fix build: Remove a few RequiresPreviewFeatures attrs Features which are now standard in .Net	2023-12-23 13:23:33 -05:00
John Colvin	822aa051d8	Add vcpkg submodule to update fmt to v10 (build still broken)	2023-12-23 13:23:29 -05:00
John Colvin	a84f29ba38	build: Stop tracking CMakeDeps files Not sure why these files were tracked in the first place, but cmake automatically detects changes to the source file glob. Perhaps they were originally added because it didn't work on some older version of CMake.	2023-12-23 13:12:30 -05:00
John Colvin	a4f8c331e6	cleanup: Get rid of old custom optional.h It's was integrated into the standard lib on windows some time ago	2023-12-23 13:12:30 -05:00
John Colvin	d5e6b00534	Port a few common utils from development branch This is a prereq to cherry-picking the commit to upgrade fmt to v10.0.0	2023-12-23 13:12:30 -05:00
John Colvin	d24becd052	fix: Ensure LadderGridHost's price_comparer isn't null Initialize it w/ reasonable default when LadderGridHost is constructed. Bugs (fixed): * Sometimes the OrderBook's ladder panel throws a null ref ex when the orders collection changes Cause: The orders collection changed callback references the ladder's price_comparer, which may be null since it isn't initialized until the DataContextChanged event fires (the relative order of these events is non-deterministic)	2023-11-02 14:15:58 -04:00
John Colvin	13251f5c06	fix: Add special-case tick logic for [non-perp] futs at Deribit Unlike any other securities supported by our system, these use TickFactors of 25/10 (e.g., 10.0, 12.5, 15.0, etc.)	2023-11-02 13:55:36 -04:00
John Colvin	03b9aaa0e8	fix: Dispose BBOVM's md poll loop Bugs (fixed): * Memory Leak! The md poll loop (instance var) in BBOVM is never disposed	2023-11-02 12:47:58 -04:00
Mohammad Arham	32d127fd84	added log to chec user input	2023-10-18 09:34:00 -07:00
John Colvin	3cb69c7520	fix build: Pass 'shell=true' to subprocess.run Otherwise it doesn't evaluate the command string properly	2023-10-17 13:50:38 -04:00
John Colvin	47e897d038	feat: Resurrect VolSurfaceAdvancedSettings	2023-10-17 13:50:38 -04:00
John Colvin	126ec169ea	refactor: Remove useless param	2023-10-17 13:49:13 -04:00
John Colvin	84953df12c	dev: Copy docker/sparkd dir from development branch So we can build prod in devcontainer	2023-10-17 13:34:23 -04:00
John Colvin	e8322183ec	Update externals to get emumba-deribit deb w/out LICENSE check	2023-10-17 13:33:01 -04:00
Mohammad Arham	30ba7ecd19	added logs for _default_abs_moves	2023-10-10 15:14:28 -07:00
Mohammad Arham	db6b34c6a8	changd default_abs_moves from hard coded to config	2023-10-05 23:17:02 -07:00
Taha Afzal	4dbafcab91	TaskingEye can shoot deep in the book as well	2023-08-18 05:44:12 +00:00
Taha Afzal	9fd7f8a0a6	Adding security list for TaskingEye Algo	2023-08-17 21:33:40 +00:00
John Colvin	04e0297e86	refactor: Simplify 'render_piq' by using common code No need to re-invent the 'render_cells' wheel	2023-07-31 14:50:44 -04:00
John Colvin	562706b968	fix: Don't assume ladder row-data is sorted by price Bugs (fixed): * Ladder assert-crashes when rendering volume-at-price Cause: The render routine expected items to be sorted by price, and asserted as much (indirectly). In hindsight, this expectation placed an unnecessary burden on the caller.	2023-07-31 14:20:08 -04:00
John Colvin	4195207a71	fix: Replace non-existent StaticResource w/ BTSColor Bugs (fixed): * The Activity Log window crashes immediately opened. Cause: WPF throws an Exception because it fails to resolve the StaticResources 'TextBox.BackgroundBrush', and 'TextBox.ForegroundBrush', as they are defined in Colors.xaml, which is not imported by LogViewer.xaml. I replaced the StaticResources w/ BTSColor extension (rather than include Colors.xaml), because it is less error prone, and allows us to keep colors organized in 1 place: BTSColors.cs. I'm not sure how long the bug was around. It likely slipped through the cracks because the ActivityLog window isn't used often.	2023-07-28 14:16:46 -04:00
John Colvin	71eb89b6a1	fix: Check 'HasQuantityDisplay' before accessing prop Bugs (fixed): * The PNL Summary window throws a null ref ex when the first row is populated. Cause: The SecurityMetadataImmutable's internal QuantityDisplay is null. In other words, the mutable SecurityMetadata obj around which it wraps has a null QuantityDisplay, which is not allowed since Immutable protos (such as 'QuantityDisplayImmutable') must be constructed w/ a non-null instance of their mutable counterpart. * Implement same fix (pre-emptively) in ManualFillRow, and OrderCompletedEventPredicate.	2023-07-28 14:01:57 -04:00
John Colvin	94e4a0b725	fix: Change bus data type to nullable immutable Bugs (fixed): * The InterpTheosServerPublisher throws a null ref during startup when it accesses 'UsageList' from the VolSurfaceUsageTag data. Cause: Immutable protos instantiated w/ the default ctr have an invalid state, because the mutable (reference type) instance around which they wrap is null. Since an immutable type's props are trivial wrappers for this mutable instance, they throw a null ref if the mutable instance is null.	2023-07-28 12:02:50 -04:00
John Colvin	40d1fe20e3	diag: Add detailed error logs to notification serv order handler Maintain a circular buffer of the last N updates. Then if/when an exception occurs, log the update sequence that led up to it.	2023-07-26 15:30:55 -04:00
John Colvin	2ed23159ca	style: Clean up jenkins build script * Get rid of double backslashes in quoted paths to improve readability * Call 'activate.bat' instead of 'activate' (official python docs say to use the former) * Add docs	2023-07-26 15:26:30 -04:00
John Colvin	faa8438adf	build: Make jenkins build script independent of working dir This alleviates an unneccessary burden on callers (previously, it had to be run from the repo root dir).	2023-07-26 11:59:03 -04:00
John Colvin	0b35c2ccde	fix: Add lock, try/catch in notification serv order handler Brandon has been experiencing a crash on startup. The stack trace shows this handler is throwing a duplicate-key ex on 'dictionary.Add' (unclear why).	2023-07-26 00:55:50 -04:00
John Colvin	98eea0ef09	chore: Remove unused iface member	2023-07-26 00:13:22 -04:00
John Colvin	4bf237aebe	fix (pre-emptive): Guard against null/empty prices in LadderRender	2023-07-25 14:27:13 -04:00
John Colvin	da927adef4	fix: Check price is within ladder's range before rendering Bugs (fixed): The render routine was indirectly asserting that each item's price was >= the ladder's min price. That seems like an inappropriate burden to put on the caller. Now we filter these items out in the routine.	2023-07-25 14:24:16 -04:00
John Colvin	cf587e26d5	fix: Remove bad assert abt tickdiff in LadderRender * The test condition was wrong in the first place * Instead of assert/crashing, log a warning	2023-07-25 14:22:15 -04:00
John Colvin	9e4354fb55	fix buid: Don't goto ERROR just bc venv already exists This seems like a typo	2023-07-18 23:40:18 -04:00
John Colvin	e6bb4c73a8	fix build (jenkins): Upgrade pip via 'python -m pip' For some reason 'pip install --upgrade pip' works on my laptop, but fails on the Jenkins box.	2023-07-18 22:53:32 -04:00
John Colvin	e43a2a4818	build: Add build script for new Jenkins (on aws)	2023-07-18 22:25:56 -04:00
John Colvin	cde0f44b7d	build: Default to Release cfg for 'publish' target	2023-07-18 22:25:06 -04:00
Taha	b07a1e2c74	Fixing the issues, where the theos are only published out by the vol surface owner	2023-07-18 20:40:59 -04:00
Taha Afzal	8376fce5fe	Merge branch 'spark_crypto.prod.vol_order' into spark_crypto.prod	2023-07-14 17:01:29 +00:00
Taha Afzal	d9e5c846c2	Adding more logs to VolAlgo	2023-07-14 16:57:55 +00:00
Taha Afzal	65736c13ef	VolAlgo completed	2023-07-14 03:54:14 +00:00
Taha Afzal	071c28e925	VolAlgo reflects its state correctly now	2023-07-10 03:20:05 +00:00
Taha Afzal	8355e3e1be	VolAlgo code complete - need to test	2023-07-09 04:21:42 +00:00
Taha	3704d1ed16	InterptheosServerPublisher is now opulated the newly added vol field in the PrecomputedTheoPoint proto	2023-07-05 14:41:31 -04:00
Taha Afzal	f2c3ff4be6	Adding vol to PrecomputedTheoPoint	2023-07-05 15:16:45 +00:00
				`@@ -1 +0,0 @@`
				set(sources /home/taha/spark/doc/user/build.sh;content/_index.md;content/bts_spark/Glossary/_index.md;content/bts_spark/Help/_index.md;content/bts_spark/Help/about_spark.png;content/bts_spark/Help/image39.png;content/bts_spark/Help/image58.png;content/bts_spark/Launcher/_index.md;content/bts_spark/Launcher/image38.png;content/bts_spark/Launcher/image68.png;content/bts_spark/Logging-In-to-the-System/_index.md;content/bts_spark/Logging-In-to-the-System/image3.png;content/bts_spark/Logging-In-to-the-System/image64.png;content/bts_spark/Market/_index.md;content/bts_spark/Market/change_securities.png;content/bts_spark/Market/image11.png;content/bts_spark/Market/image14.png;content/bts_spark/Market/image20.png;content/bts_spark/Market/image21.png;content/bts_spark/Market/image23.png;content/bts_spark/Market/image24.png;content/bts_spark/Market/image34.png;content/bts_spark/Market/image49.png;content/bts_spark/Market/image53.png;content/bts_spark/Market/image66.png;content/bts_spark/Market/image69.png;content/bts_spark/Market/market_view.png;content/bts_spark/Market/settings_icon.png;content/bts_spark/Market/time_and_sales.png;content/bts_spark/My-Windows/_index.md;content/bts_spark/My-Windows/image15.png;content/bts_spark/My-Windows/image16.png;content/bts_spark/My-Windows/image37.png;content/bts_spark/Settings/_index.md;content/bts_spark/Settings/image51.png;content/bts_spark/Settings/image9.png;content/bts_spark/Spreader/_index.md;content/bts_spark/Spreader/image10.png;content/bts_spark/Spreader/image13.png;content/bts_spark/Spreader/image18.png;content/bts_spark/Spreader/image19.png;content/bts_spark/Spreader/image28.png;content/bts_spark/Spreader/image29.png;content/bts_spark/Spreader/image31.png;content/bts_spark/Spreader/image35.png;content/bts_spark/Spreader/image40.png;content/bts_spark/Spreader/image43.png;content/bts_spark/Spreader/image54.png;content/bts_spark/Spreader/image55.png;content/bts_spark/Spreader/image56.png;content/bts_spark/Spreader/image6.png;content/bts_spark/Spreader/image7.png;content/bts_spark/Spreader/image70.png;content/bts_spark/Spreader/image72.png;content/bts_spark/Spreader/image8.png;content/bts_spark/Trading/OrderTypes/index.md;content/bts_spark/Trading/_index.md;content/bts_spark/Trading/algo_studio.png;content/bts_spark/Trading/collapse_all_icon.png;content/bts_spark/Trading/cover_algo_control.png;content/bts_spark/Trading/expand_all_icon.png;content/bts_spark/Trading/image1.png;content/bts_spark/Trading/image17.png;content/bts_spark/Trading/image2.png;content/bts_spark/Trading/image25.png;content/bts_spark/Trading/image27.png;content/bts_spark/Trading/image33.png;content/bts_spark/Trading/image42.png;content/bts_spark/Trading/image44.png;content/bts_spark/Trading/image47.png;content/bts_spark/Trading/image48.png;content/bts_spark/Trading/image5.png;content/bts_spark/Trading/image50.png;content/bts_spark/Trading/image52.png;content/bts_spark/Trading/image57.png;content/bts_spark/Trading/image59.png;content/bts_spark/Trading/image60.png;content/bts_spark/Trading/image61.png;content/bts_spark/Trading/image62.png;content/bts_spark/Trading/image73.png;content/bts_spark/Trading/image74.png;content/bts_spark/Trading/join_algo_control.png;content/bts_spark/Trading/ladder.png;content/bts_spark/Trading/ladder_badges.png;content/bts_spark/Trading/ladder_main_part.png;content/bts_spark/Trading/lean_algo_control.png;content/bts_spark/Trading/mini_orderbook.png;content/bts_spark/Trading/oco_algo_control.png;content/bts_spark/Trading/offset_algo_control.png;content/bts_spark/Trading/pnl_calculation.png;content/bts_spark/Trading/pnl_window.png;content/bts_spark/Trading/recreate_icon.png;content/bts_spark/Trading/settings_icon.png;content/bts_spark/Trading/spark_menu_order_book.png;content/bts_spark/Trading/top_algo_control.png;content/bts_spark/Trading/trigger_algo_control.png;content/bts_spark/_index.md;content/release_notes/_index.md;content/system_requirements/_index.md;static/book.js;static/favicon.ico;static/image.js;templates/index.html;templates/page.html;templates/section.html;templates/shortcodes/back_button.html)
				`@@ -1 +0,0 @@`
				set(sources files/audit_ilink3.py;files/audit_upload.sh;files/audit_uploader.py;files/cert_secure_logon.txt;files/example.conf;files/futures.coconut.xz;files/onixs_ice/ConnectivityConfiguration.APITest.xml;files/onixs_ice/ConnectivityConfiguration.PerfTest.xml;files/onixs_ice/ConnectivityConfiguration.Production.xml;files/perf_trace/compiled.css;files/perf_trace/index.html;files/perf_trace/jquery-3.3.1.min.js;files/perf_trace/main.js;files/perf_trace/perf_log.sql;files/perf_trace/perf_trace_analysis.sql;files/perf_trace/plotly-latest.min.js;files/small_es.pcap.xz;files/spartan/create_polygon_books.sql;files/spartan/log.sql;files/spartan/md.sql;files/spartan/sim_db.sql)
				`@@ -1 +0,0 @@`
				set(sources api/analytics/ExpirationDriver.proto;api/analytics/TimeWeightedInterpolatedDriver.proto;api/apicore.proto;api/apigateway.proto;api/apioptions.proto;api/core/ApiClientInterop.cpp;api/core/ApiServerInterop.cpp;api/domain/Currency.proto;api/domain/SecurityCriteria.proto;api/domain/SecurityCriteriaClause.proto;api/domain/SecurityExpiration.proto;api/domain/SecurityMetadata.proto;api/domain/SecurityMetadataCollection.proto;api/domain/SecurityMetadataItem.proto;api/domain/SecurityType.proto;api/domain/TickDisplaySettings.proto;api/domain/TickFactors.proto;api/domain/Venue.proto;api/io/URL.proto;api/md/MarketDataSubscriptionParams.proto;api/md/binance/BinanceDepthStreamParams.proto;api/md/binance/BinanceMarketDataSubscriptionParams.proto;api/md/deribit/DeribitMarketDataSubscriptionParams.proto;api/md/ftx/FTXDepthStreamParams.proto;api/md/ftx/FTXMarketDataSubscriptionParams.proto;api/md/glbx/GlbxMarketDataSubscriptionParams.proto;api/md/huobi/HuobiMarketDataSubscriptionParams.proto;api/public/ApiServer.cpp;api/public/ClientChannel.cpp;api/public/QuickbusClientChannel.cpp;api/public/ServerChannel.cpp;api/public/ServiceContext.cpp;api/service_helpers/AdminServiceUtils.cpp;api/service_helpers/AlgoServiceUtils.cpp;api/service_helpers/PositionUtilities.cpp;api/services/AdminService.proto;api/services/AlgoDefinitionService.proto;api/services/AlgoService.proto;api/services/AnalyticsService.proto;api/services/ApiUnitTestService.proto;api/services/AuthService.proto;api/services/BasePositionService.proto;api/services/ErrorReportService.proto;api/services/HelloService.proto;api/services/MBOService.proto;api/services/MarginService.proto;api/services/MarketDataAuxiliaryService.proto;api/services/MarketDataRelay.proto;api/services/MarketDataService.proto;api/services/MoneyFileService.proto;api/services/OrderService.proto;api/services/OutOfProcessURLListener.proto;api/services/PersistenceService.proto;api/services/PositionService.proto;api/services/ShellDiagnosticService.proto;api/services/ShellLatencyService.proto;api/services/SyntheticSecurityService.proto;api/test/ApiTests.cpp;api/test/HelloServiceServer.cpp;communincation/websockets/tests/WebsocketSyncClientTests.cpp;config/ConfigCheck.cpp;config/ConfigDownloader.cpp;config/ConfigErrorCollector.cpp;config/ConfigLog.cpp;config/ConfigPropertyEvaluator.cpp;config/ConfigValueQueue.cpp;config/ConfigurationException.cpp;config/ConfigurationState.cpp;config/PreprocessInputStream.cpp;config/config.cpp;config/defaults/MarketData.conf;config/defaults/Ports.conf;config/defaults/SecurityMetadataCollection.conf;config/defaults/Symbols.conf;config/defaults/cme_reference_data.conf;config/defaults/edge.conf;config/defaults/jetfix.conf;config/defaults/keycode.conf;config/defaults/logging.conf;config/defaults/orderman_cfg.conf;config/defaults/orders_fills_cleanup.conf;config/defaults/position_cfg.conf;config/defaults/span.conf;config/defaults/spreader.conf;config/defaults/testkeys.conf;config/defaults/threads.conf;config/defaults/toggles.conf;config/extensions/extensions.proto;config/invalid/invalid.proto;config/protos/algo_types.proto;config/protos/cme_reference_data.proto;config/protos/exchange_sim_cfg.proto;config/protos/jetfix.proto;config/protos/keycode.proto;config/protos/logging.proto;config/protos/market_data.proto;config/protos/mercury_cfg.proto;config/protos/orderman_cfg.proto;config/protos/orders_fills_cleanup.proto;config/protos/perflog_cfg.proto;config/protos/ports.proto;config/protos/position_cfg.proto;config/protos/quickbus_cfg.proto;config/protos/span.proto;config/protos/spartan.proto;config/protos/spreader.proto;config/protos/subprocess.proto;config/protos/symbols.proto;config/protos/system.proto;config/protos/testkeys.proto;config/protos/threads.proto;config/protos/toggles.proto;config/protos/ui.proto;config/tests/HTTPServer.cpp;config/tests/config_ref_tests.cpp;config/tests/config_tests.cpp;config/text_format/text_format.cpp;domain/AggressorSide.cpp;domain/NewOrder.cpp;domain/OrderDefinition.cpp;domain/OrderStatus.cpp;domain/OrderUpdate.cpp;domain/Side.cpp;domain/TimeInForce.cpp;domain_model/Security.cpp;domain_model/Security.proto;domain_model/SecurityTests.cpp;domain_model/sod.cpp;domain_model/tests/domain_tests.cpp;expression/BooleanGrammar.cpp;expression/ExpressionEvaluator.cpp;expression/ParserExceptions.cpp;expression/Value.cpp;expression/tests/expression_parser_tests.cpp;io/QueryParams.cpp;io/URL.cpp;io/http/CurlHandle.cpp;io/http/HTTPClient.cpp;io/http/HTTPHeaders.cpp;io/http/HTTPRequest.cpp;io/http/HTTPResponse.cpp;io/websocket/WebsocketClient.cpp;io/websocket/WebsocketClientConfiguration.cpp;io/websocket/WebsocketError.cpp;io/websocket/WebsocketMessage.cpp;io/websocket/WebsocketSSLClientInternal.cpp;io/websocket/tests/WebsocketClientTests.cpp;jetfix/codec/FIXArray.cpp;jetfix/common/OutFIX.cpp;jetfix/connection/AsyncReader.cpp;jetfix/connection/IConnection.cpp;jetfix/connection/SocketConnection.cpp;jetfix/gateway/CoreGateway.cpp;jetfix/gateway/JetFIXGatLine truncated