added log to chec user input

Otherwise it doesn't evaluate the command string properly

docker/sparkd/dev/Dockerfile

@@ -0,0 +1,135 @@
+FROM ubuntu:latest
+
+ARG DEBIAN_FRONTEND=noninteractive
+ENV TZ=America/New_York
+
+# light deps (should be quick to download/install)
+RUN apt-get update && apt-get install -y \
+    ca-certificates \
+    curl \
+    gnupg \
+    liblttng-ust-dev \
+    libpcap0.8-dev \
+    locales \
+    ninja-build \
+    software-properties-common \
+    sqlite3 \
+    tzdata
+
+# clang
+RUN apt-get update && apt-get install -y \
+    clang-13 \
+    clang-format-13 \
+    clangd-13 \
+    clang-15 \
+    clang-format-15 \
+    clangd-15
+
+# g++
+RUN apt-get update && apt-get install -y \
+    g++-11 \
+    g++-12
+
+# python
+RUN apt-get update && apt-get install -y \
+    python3-pip \
+    python3-venv \
+    && python3 -m pip install --upgrade pip
+
+# dotnet
+RUN apt-get update && apt-get install -y \
+    dotnet7
+
+# ssh
+RUN apt-get update && apt-get install -y \
+    libssh2-1-dev \
+    openssh-server
+
+# cmake
+RUN \
+    curl -fsSL https://apt.kitware.com/keys/kitware-archive-latest.asc 2> /dev/null \
+    | gpg --dearmor - | tee /etc/apt/trusted.gpg.d/kitware.gpg > /dev/null \
+    && apt-add-repository "deb https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" \
+    && apt-get update \
+    && apt-get install -y cmake
+
+# node
+ARG node_major=18
+RUN mkdir -p /etc/apt/keyrings \
+    && curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key \
+    | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg \
+    && echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$node_major.x nodistro main" \
+    | tee /etc/apt/sources.list.d/nodesource.list \
+    && apt-get update \
+    && apt-get install -y nodejs
+
+# ccache is done here so that ccache sees clang++ installed, not clang-10
+RUN apt-get update && apt-get install -y \
+    ccache 
+
+# emumba (light deps)
+RUN apt-get update && apt-get install -y \
+    build-essential \
+    doxygen \
+    python3-sphinx \
+    graphviz \
+    libgtest-dev \
+    libssl-dev \
+    rapidjson-dev \
+    zlib1g-dev \
+    pkg-config
+
+# emumba (boost)
+RUN apt-get update && apt-get install -y \
+    libboost-filesystem-dev \
+    libboost-program-options-dev \
+    libboost-test-dev \
+    libboost-date-time-dev
+
+# dev utils
+RUN apt-get update && apt-get install -y \ 
+    fish \
+    git-all \
+    sudo \
+    vim \
+    wget
+
+# set up default executables (update-alternatives)
+RUN \
+    update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-13 100; \
+    update-alternatives --install /usr/bin/clang clang /usr/bin/clang-13 100; \
+    update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-13 100; \
+    update-alternatives --install /usr/bin/clangd clangd /usr/bin/clangd-13 100; \
+    update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-11 100; \
+    update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 100
+
+# Add compiler symlinks for ccache 
+RUN \
+    ln -s /usr/bin/ccache /usr/local/bin/clang; \
+    ln -s /usr/bin/ccache /usr/local/bin/clang++; \
+    ln -s /usr/bin/ccache /usr/local/bin/clang-13; \
+    ln -s /usr/bin/ccache /usr/local/bin/clang++-13; \
+    ln -s /usr/bin/ccache /usr/local/bin/clang-15; \
+    ln -s /usr/bin/ccache /usr/local/bin/clang++-15; \
+    ln -s /usr/bin/ccache /usr/local/bin/gcc; \
+    ln -s /usr/bin/ccache /usr/local/bin/g++; \
+    ln -s /usr/bin/ccache /usr/local/bin/gcc-11; \
+    ln -s /usr/bin/ccache /usr/local/bin/g++-11; \
+    ln -s /usr/bin/ccache /usr/local/bin/gcc-12; \
+    ln -s /usr/bin/ccache /usr/local/bin/g++-12
+
+RUN locale-gen en_US.UTF-8
+
+ARG docker_user="bts"
+ARG docker_uid="1000"
+ARG docker_pass="bts"
+
+RUN \
+    groupadd -g "$docker_uid" "$docker_user" \
+    && useradd \
+    -m \
+    -u "$docker_uid" \
+    -g "$docker_uid" \
+    -G sudo \
+    --password "$(openssl passwd -1 "$docker_pass")" \
+    "$docker_user"

docker/sparkd/dev/README.md

@@ -0,0 +1,64 @@
+[dev containers]: https://code.visualstudio.com/docs/devcontainers/containers
+[dev containers ext]: https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-containers
+[docker repo sparkd-dev]: https://hub.docker.com/repository/docker/bluetradingsystems/sparkd-dev/general
+# Sparkd Dev Container
+## How to create a VS Code [Dev Container][dev containers]
+
+1. Make sure the **[Dev Containers][dev containers ext]** extension is installed
+1. Make sure your Docker image is up to date
+   ```
+   docker pull bluetradingsystems/sparkd-dev:latest
+   ```
+1. Copy **devcontainer.json** from this folder to **$workspaceFolder/.devcontainer/**.
+
+1. In the VS Code Command Palette, run **Dev Containers: Reopen in Container**
+
+---
+## How to create a Sparkd Container in the terminal (without VS Code)
+
+1. Make sure your Docker image is up to date
+   ```
+   docker pull bluetradingsystems/sparkd-dev:latest
+   ```
+2. Start the container
+   ```
+   # Replace <cont_name> with a name for the container. The container's
+   # "$HOME" dir will be mounted to "$HOME/.sparkd-dev/mounts/<cont_name> on
+   # the host
+   
+   ./start_docker.sh --container <cont_name> --mount-existing
+   ```
+   * *Replace <cont_name> with a name for the container*
+   * *The container's `$HOME` dir will be mounted to
+   `$HOME/.sparkd-dev/mounts<cont_name>` on the host*<br>
+   * *Run: `./start_docker.sh --help` to see additional options*
+3. *(Optional) Attach VS Code to the running container.*<br>
+Run **Dev Containers: Attach to Running Container** from the command palette,
+then select your container in the dropdown
+
+---
+## How to update the Docker image
+
+1. Change the image by modifying the Dockerfile (don't tag arbitrary
+  images on your machine and push them to our repo on Docker Hub)
+
+1. After modifying the Dockerfile, (re)build the image
+   ```
+   # Run in this dir
+   docker build -t bluetradingsystems/sparkd-dev:latest .
+   ```
+   ***Note:** Your docker login must have write permissions to push to the
+   [sparkd-dev][docker repo sparkd-dev] repository*
+
+2. Commit the changes to Dockerfile in git
+
+3. Push the new image to Docker Hub
+
+   ```
+   docker push bluetradingsystems/sparkd-dev:latest
+   ```
+   ***Note:** The image doesn't have to be on Docker Hub, we can simply build
+   it from scratch on our local machines (with the Dockerfile). However, this
+   takes time. It is significantly quicker to download a prebuilt image, so we
+   store it on a remote repository to save time when setting up a new dev
+   machine.*

docker/sparkd/dev/bashrc

@@ -0,0 +1,123 @@
+# ~/.bashrc: executed by bash(1) for non-login shells.
+# see /usr/share/doc/bash/examples/startup-files (in the package bash-doc)
+# for examples
+
+# If not running interactively, don't do anything
+case $- in
+    *i*) ;;
+      *) return;;
+esac
+
+# don't put duplicate lines or lines starting with space in the history.
+# See bash(1) for more options
+HISTCONTROL=ignoreboth
+
+# append to the history file, don't overwrite it
+shopt -s histappend
+
+# for setting history length see HISTSIZE and HISTFILESIZE in bash(1)
+HISTSIZE=1000
+HISTFILESIZE=2000
+
+# check the window size after each command and, if necessary,
+# update the values of LINES and COLUMNS.
+shopt -s checkwinsize
+
+# If set, the pattern "**" used in a pathname expansion context will
+# match all files and zero or more directories and subdirectories.
+#shopt -s globstar
+
+# make less more friendly for non-text input files, see lesspipe(1)
+[ -x /usr/bin/lesspipe ] && eval "$(SHELL=/bin/sh lesspipe)"
+
+# set variable identifying the chroot you work in (used in the prompt below)
+if [ -z "${debian_chroot:-}" ] && [ -r /etc/debian_chroot ]; then
+    debian_chroot=$(cat /etc/debian_chroot)
+fi
+
+# set a fancy prompt (non-color, unless we know we "want" color)
+case "$TERM" in
+    xterm-color|*-256color) color_prompt=yes;;
+esac
+
+# uncomment for a colored prompt, if the terminal has the capability; turned
+# off by default to not distract the user: the focus in a terminal window
+# should be on the output of commands, not on the prompt
+#force_color_prompt=yes
+
+if [ -n "$force_color_prompt" ]; then
+    if [ -x /usr/bin/tput ] && tput setaf 1 >&/dev/null; then
+	# We have color support; assume it's compliant with Ecma-48
+	# (ISO/IEC-6429). (Lack of such support is extremely rare, and such
+	# a case would tend to support setf rather than setaf.)
+	color_prompt=yes
+    else
+	color_prompt=
+    fi
+fi
+
+if [ "$color_prompt" = yes ]; then
+    PS1='${debian_chroot:+($debian_chroot)}\[\033[01;32m\]\u@\h\[\033[00m\]:\[\033[01;34m\]\w\[\033[00m\]\$ '
+else
+    PS1='${debian_chroot:+($debian_chroot)}\u@\h:\w\$ '
+fi
+unset color_prompt force_color_prompt
+
+# If this is an xterm set the title to user@host:dir
+case "$TERM" in
+xterm*|rxvt*)
+    PS1="\[\e]0;${debian_chroot:+($debian_chroot)}\u@\h: \w\a\]$PS1"
+    ;;
+*)
+    ;;
+esac
+
+# enable color support of ls and also add handy aliases
+if [ -x /usr/bin/dircolors ]; then
+    test -r ~/.dircolors && eval "$(dircolors -b ~/.dircolors)" || eval "$(dircolors -b)"
+    alias ls='ls --color=auto'
+    #alias dir='dir --color=auto'
+    #alias vdir='vdir --color=auto'
+
+    alias grep='grep --color=auto'
+    alias fgrep='fgrep --color=auto'
+    alias egrep='egrep --color=auto'
+fi
+
+# colored GCC warnings and errors
+#export GCC_COLORS='error=01;31:warning=01;35:note=01;36:caret=01;32:locus=01:quote=01'
+
+# some more ls aliases
+alias ll='ls -alF'
+alias la='ls -A'
+alias l='ls -CF'
+
+# Add an "alert" alias for long running commands.  Use like so:
+#   sleep 10; alert
+alias alert='notify-send --urgency=low -i "$([ $? = 0 ] && echo terminal || echo error)" "$(history|tail -n1|sed -e '\''s/^\s*[0-9]\+\s*//;s/[;&|]\s*alert$//'\'')"'
+
+# Alias definitions.
+# You may want to put all your additions into a separate file like
+# ~/.bash_aliases, instead of adding them here directly.
+# See /usr/share/doc/bash-doc/examples in the bash-doc package.
+
+if [ -f ~/.bash_aliases ]; then
+    . ~/.bash_aliases
+fi
+
+# enable programmable completion features (you don't need to enable
+# this, if it's already enabled in /etc/bash.bashrc and /etc/profile
+# sources /etc/bash.bashrc).
+if ! shopt -oq posix; then
+  if [ -f /usr/share/bash-completion/bash_completion ]; then
+    . /usr/share/bash-completion/bash_completion
+  elif [ -f /etc/bash_completion ]; then
+    . /etc/bash_completion
+  fi
+fi
+
+if [[ ! -v XDG_CONFIG_HOME ]]; then
+  export XDG_CONFIG_HOME="$HOME/.config"
+fi
+
+export PATH="$HOME/.local/bin:$PATH"

docker/sparkd/dev/config.fish

@@ -0,0 +1,25 @@
+if ! set -q XDG_CONFIG_HOME
+    set -g XDG_CONFIG_HOME "$HOME/.config"
+end
+set fish_config_home $XDG_CONFIG_HOME/fish
+if test -f "$fish_config_home/aliases.fish"
+    source "$fish_config_home/aliases.fish"
+end
+# enable general colors and case insensitive search in 'less'
+set -x LESS "$LESS -R -Q -I"
+set -x LC_ALL en_US.UTF-8
+# enable syntax highlighting in 'less'
+# set highlight color scheme in 'less' (fg)(bg)
+set -g LESS_TERMCAP_md (tput bold)(tput setaf 36)
+set -g LESS_TERMCAP_me (tput sgr0)
+set -g LESS_TERMCAP_so (tput setaf 232)(tput setab 49)
+
+set -q TERM
+if test $status != 0; or test $TERM = 'xterm'
+    set -g TERM 'xterm-256color'
+end
+
+# Fix impossible-to-see navy blue command text color
+set fish_color_command F8F8F2
+
+set -g PATH "$HOME/.local/bin" "$PATH"

docker/sparkd/dev/devcontainer.json

@@ -0,0 +1,32 @@
+{
+  "image": "bluetradingsystems/sparkd-dev:latest",
+  "initializeCommand": "${localWorkspaceFolder}/docker/sparkd/dev/devcontainer_initialize.sh",
+  "hostRequirements": {
+    "memory": "4gb",
+    "storage": "6gb"
+  },
+  "postCreateCommand": "/home/bts/spark/docker/sparkd/dev/devcontainer_post_create.sh",
+  "containerUser": "bts",
+  "workspaceMount": "source=${localWorkspaceFolder},target=/home/bts/spark,type=bind",
+  "workspaceFolder": "/home/bts/spark",
+  "updateRemoteUserUID": true,
+  "shutdownAction": "none",
+  // Note: If running multiple containers on the same host, each must use a
+  // unique set of ports. Usually we just add 1000 to each port as needed. For
+  // example, if 8000-8003 is taken, go with 9000-9003, then 10000-10003, etc.
+  "runArgs": [
+    "--shm-size",
+    "2gb",
+    "-p",
+    "8000:8000",
+    "-p",
+    "8001:8001",
+    "-p",
+    "8002:8002",
+    "-p",
+    "8003:8003"
+  ],
+  "mounts": [
+    "source=${localEnv:HOME}/.sparkd-dev/mounts/sparkd-dev,target=/home/bts,type=bind"
+  ]
+}

docker/sparkd/dev/devcontainer_initialize.sh

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+mkdir -p "$HOME/.sparkd-dev/mounts/sparkd-dev"

docker/sparkd/dev/devcontainer_post_create.sh

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+
+this_dir="$(realpath "$(dirname "${BASH_SOURCE[0]}")")"
+repo_dir="$(cd "$this_dir" && git rev-parse --show-toplevel)"
+tools_dir="$repo_dir/tools"
+gen_tag="GENERATED BY $(basename "${BASH_SOURCE[0]}")"
+
+function copy_if_not_exist() {
+  src="$1"
+  dest="$2"
+  if [[ ! -e "$src" ]]; then
+    echo "Unable to copy $src to $dest. Source file doesn't exist"
+    exit 1
+  fi
+  if [[ ! -e "$dest" ]]; then
+    cp "$src" "$dest"
+  fi
+}
+
+# Init bashrc
+echo "Initializing bashrc"
+copy_if_not_exist "$this_dir/bashrc" "$HOME/.bashrc"
+
+# Init fish config
+echo "Initializing fish config"
+fish_cfg_dir="$HOME/.config/fish/"
+mkdir -p "$fish_cfg_dir"
+exec "$tools_dir/write_tag_block.sh" \
+  --tag "$gen_tag" \
+  --replace \
+  --in-place \
+  --file "$fish_cfg_dir/config.fish" \
+  --text "$(cat "$this_dir/config.fish")"

docker/sparkd/dev/set_user.sh

@@ -0,0 +1,250 @@
+#!/usr/bin/env bash
+
+function _exit() {
+  if [[ "$#" -gt 1 ]]; then
+    # When there are multiple args, the last arg is the exit code.
+    echo "${@:1:($# - 1)}"
+    exit "${@: -1}"
+  fi
+  echo "$1"
+  exit 0
+}
+
+help_text="\
+Add or update a user's uid
+
+This script is intended to be run in a Docker container as a means to obtain
+file permissions in mounted directories owned by the specified UID.
+
+Usage: set_user.sh [OPTIONS] USER UID
+
+Positionals:
+  USER               The name of the user
+
+Options:
+  --user USER        The name of the user (defaults to 'bts')
+  --uid UID          The new uid to give USER (defaults to '1000')
+  --group GROUP      The primary group to give USER (defaults to USER)
+  --gid GID          The primary gid to give USER (defaults to UID)
+  --pass PASSWORD    The password to give USER (defaults to USER, only applies
+                     if USER hasn't been created yet) 
+  --modify-existing  Update UID/GID even if USER already exists
+  -h,--help          Print this help message and exit
+
+"
+
+function _add_group() {
+  if [[ "$#" -ne 2 ]]; then
+    _exit "'_add_group' requires exactly 2 args (group, gid). Received $#." 1
+  fi
+  local group="$1"
+  local gid="$2"
+  # Check if gid already exists
+  local existing_group=""
+  existing_group="$(getent group "$gid" | cut -d : -f 1)"
+  if [[ -n "$existing_group" ]]; then
+    if [[ "$existing_group" != "$group" ]]; then
+      echo "\
+The gid '$gid' is already in use by group '$existing_group' (will not rename \
+the group)"
+    fi
+    return
+  fi
+
+  # Check if group with same name already exists
+  local existing_gid=""
+  existing_gid="$(getent group "$group" | cut -d : -f 3)"
+  if [[ -n "$existing_gid" ]]; then
+    # The group already exists. Only modify if --modify-existing was specified.
+    if [[ "$modify_existing" -eq 0 ]]; then
+      _exit "\
+The group '$group' already exists with gid '$existing_gid'.
+Specify --modify-existing to change the gid anyways." 1
+    fi
+    echo "Changing gid for group '$group' from '$existing_gid' to '$gid'"
+    groupmod -g "$gid" "$group"
+  else
+    # The group doesn't exist. Add it
+    echo "Adding group '$group:$gid'"
+    groupadd -g "$gid" "$group"
+  fi
+}
+
+function _add_user() {
+  local user="$1"
+  local uid="$2"
+  local group="$3"
+  local gid="$4"
+  local pass="$5"
+  if [[ "$#" -ne 5 ]]; then
+    _exit "'_add_user' requires exactly 5 args (user, uid, group, gid, pass). \
+Received $#." 1
+  fi
+
+  # Check if uid is being used by another user
+  local existing_user=""
+  existing_user="$(getent passwd "$uid" | cut -d : -f 1)"
+  if [[ -n "$existing_user" ]]; then
+    if [[ "$existing_user" != "$user" ]]; then
+      _exit "The uid '$uid' is already in use by user '$existing_user'." 1
+    fi
+
+    echo "The user '$user' already has the specified uid '$uid'."
+    # Modify gid if necessary
+    local existing_gid=""
+    existing_gid="$(id -g "$user" 2> /dev/null)"
+    if [[ "$existing_gid" = "$gid" ]]; then
+      echo "The user '$user' already has the specified gid '$gid'."
+      return
+    fi
+
+    if [[ "$modify_existing" -eq 0 ]]; then
+      _exit "\
+The user '$user' is associated with the primary group '$existing_gid'.
+Specify --modify-existing to change the gid anyways." 1
+    fi
+
+    echo "Changing primary group for '$user' from '$existing_gid' to '$gid'."
+    usermod -g "$gid" "$user"
+    return
+  fi
+
+  # Check if user with same name already exists
+  local existing_uid=""
+  existing_uid="$(id -u "$user" 2> /dev/null)"
+  if [[ -n "$existing_uid" ]]; then
+    if [[ "$existing_uid" = "$uid" ]]; then
+      _exit "The user '$user' already has the specified uid '$uid'" 0
+    fi
+
+    if [[ "$modify_existing" -eq 0 ]]; then
+      _exit "\
+User '$user' already exists with uid '$existing_uid'.
+Specify --modify-existing to change the uid anyways." 1
+    fi
+
+    echo "Changing uid for user '$user' from '$existing_uid' to '$uid'"
+    if ! usermod -u "$uid" -g "$gid" "$user"; then
+      _exit "usermod exited with code $?" 1
+    fi
+    return
+  fi
+
+  # The user doesn't exist. Add it
+  echo "Adding user '$user:$uid' (group='$group:$gid', password='$pass')"
+  useradd -m -u "$uid" -g "$gid" -G sudo --password "$(openssl passwd -1 "$pass")" "$user"
+}
+
+function _update_fish_dir_perms() {
+  local user="$1"
+  local uid="$2"
+  local gid="$3"
+  # When a fish shell session starts, a temp dir named '/tmp/fish.<user>' is
+  # created with the user's current uid/gid. If such a dir exists for the
+  # specified USER, make sure the uid/gid match the new values.
+  fish_dir="/tmp/fish.$user"
+  if [[ -d "$fish_dir" ]]; then
+    perms="$uid:$gid"
+    existing_perms="$(stat -c %u:%g "$fish_dir")"
+    if [[ "$existing_perms" != "$perms" ]]; then
+      echo "Changing uid:gid for dir '$fish_dir' from '$existing_perms' to '$perms'"
+      chown -R "$perms" "$fish_dir"
+    fi
+  fi
+}
+
+user=""
+group=""
+pass=""
+declare -i uid=0
+declare -i gid=0
+declare -i modify_existing=0
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --user)
+      if [[ -z "$2" ]]; then
+        _exit "USER (--user) must be non-empty" 1
+      elif [[ "$2" = "root" ]]; then
+        _exit "USER (--user) cannot be 'root'" 1
+      fi
+      user="$2"
+      shift 2
+      ;;
+    --group)
+      if [[ -z "$2" ]]; then
+        _exit "GROUP (--group) must be non-empty" 1
+      elif [[ "$2" = "root" ]]; then
+        _exit "GROUP (--group) cannot be 'root'" 1
+      fi
+      group="$2"
+      shift 2
+      ;;
+    --uid)
+      if [[ -z "$2" ]]; then
+        _exit "UID (--uid) must be non-empty" 1
+      fi
+      uid="$2"
+      shift 2
+      ;;
+    --gid)
+      if [[ -z "$2" ]]; then
+        _exit "GID (--gid) must be non-empty" 1
+      fi
+      gid="$2"
+      shift 2
+      ;;
+    --pass)
+      if [[ -z "$2" ]]; then
+        _exit "PASS (--pass) must be non-empty" 1
+      fi
+      pass="$2"
+      shift 2
+      ;;
+    --modify-existing)
+      modify_existing=1
+      shift 1
+      ;;
+    -h)
+      ;&
+    --help)
+      _exit "$help_text" 0
+      ;;
+    *)
+      if [[ -z "$1" ]]; then
+        _exit "Invalid positional argument '$1'" 1
+      elif [[ -z "$user" ]]; then
+        user="$1"
+      elif [[ -z "$uid" ]]; then
+        uid="$1"
+      else
+        _exit "Unexpected argument '$1'" 1
+      fi
+      shift 1
+      ;;
+  esac
+done
+
+if [[ -z "$user" ]]; then
+  user="bts"
+fi
+
+if [[ $uid -eq 0 ]]; then
+  uid=1000
+fi
+
+# In general, a user's primary group/gid matches the user's name/uid.
+# If group/gid args weren't specified, set them to name/uid.
+if [[ -z "$group" ]]; then
+  group="$user"
+fi
+if [[ $gid -eq 0 ]]; then
+  gid=$uid
+fi
+if [[ -z "$pass" ]]; then
+  pass="$user"
+fi
+
+_add_group "$group" "$gid"
+_add_user "$user" "$uid" "$group" "$gid" "$pass"
+_update_fish_dir_perms "$user" "$uid" "$gid"

docker/sparkd/dev/start_docker.sh

@@ -0,0 +1,230 @@
+#!/usr/bin/env bash
+
+function _exit() {
+  if [[ "$#" -gt 1 ]]; then
+    # When there are multiple args, the last arg is the exit code.
+    echo "${@:1:($# - 1)}"
+    exit "${@: -1}"
+  fi
+  echo "$1"
+  exit 0
+}
+
+help_text="\
+Start a docker dev container
+Usage: start_docker.sh [OPTIONS] image container
+
+Positionals:
+  image             The image to instantiate
+  container         The name for the container
+
+Options:
+  -u,--user USER      The user to add in the container (defaults to 'bts')
+  -p,--pass PASS      The password for USER in the container (defaults to USER)
+  -i,--image IMAGE    The image to instantiate (defaults to 'bluetradingsystems/sparkd-dev:latest')
+  -c,--container CONT The name for the container
+  -m,--mount MOUNT    The name for the folder (outside the container) to create/mount to the home
+                      directory in the container. Mount folders are added here: $mount_prefix
+  --mount-existing    Mount the specified dir even if it already exists
+  --port <cont_port>:<host_port> OR [<cont_port>:<host_port>,...] Port mapping(s). May be
+                      specified multiple times.
+  -h,--help           Print this help message and exit
+
+"
+
+mount_prefix="$HOME/.sparkd-dev/mounts"
+declare docker_user=""
+declare docker_pass=""
+declare image_name=""
+declare cont_name=""
+declare mount_name=""
+declare -i mount_existing=0
+declare -a port_mappings=()
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    -u)
+      ;&
+    --user)
+      if [[ -z "$2" ]]; then
+        _exit "User (-u,--user) must be non-empty" 1
+      fi
+      docker_user="$2"
+      shift 2
+      ;;
+    -p)
+      ;&
+    --pass)
+      if [[ -z "$2" ]]; then
+        _exit "Password (-p,--pass) must be non-empty" 1
+      fi
+      docker_pass="$2"
+      shift 2
+      ;;
+    -i)
+      ;&
+    --image)
+      if [[ -n "$image_name" ]]; then
+        _exit "Unexpected argument '$1'" 1
+      elif [[ -z "$2" ]]; then
+        _exit "Image (-i,--image) must be non-empty" 1
+      fi
+      image_name="$2"
+      shift 2
+      ;;
+    -c)
+      ;&
+    --container)
+      if [[ -n "$cont_name" ]]; then
+        _exit "Unexpected argument '$1'" 1
+      elif [[ -z "$2" ]]; then
+        _exit "Container (-c,--container) must be non-empty" 1
+      fi
+      cont_name="$2"
+      shift 2
+      ;;
+    --port)
+      # Parse port(s) arg, and append the result to 'port_mappings'
+      # Note: port(s) may be specified in bracketed notation:
+      #   [a:b,c:d,...,y:z]
+      next_ports="$(echo "$2" | sed -e 's/\[//g' -e 's/\]//g' -e 's/,/ /g')"
+      if [[ "${#port_mappings[@]}" -gt 0 ]]; then
+        next_ports=" $next_ports"
+      fi
+      port_mappings+=("$next_ports")
+      shift 2
+      ;;
+    -m)
+      ;&
+    --mount)
+      if [[ -z "$2" ]]; then
+        echo "Mount (-m,--mount) must be non-empty"
+      fi
+      mount_name="$2"
+      shift 2
+      ;;
+    --mount-existing)
+      mount_existing=1
+      shift 1
+      ;;
+    -h)
+      ;&
+    --help)
+      _exit "$help_text" 0
+      ;;
+    *)
+      if [[ -z "$1" ]]; then
+        _exit "Invalid positional argument '$1'" "$help_text" 1
+      elif [[ -z "$image_name" ]]; then
+        image_name="$1"
+      elif [[ -z "$cont_name" ]]; then
+        cont_name="$1"
+      else
+        _exit "Unexpected argument '$1'" "$help_text" 1
+      fi
+      shift 1
+      ;;
+  esac
+done
+
+
+if [[ -z "$image_name" ]]; then
+  image_name="bluetradingsystems/sparkd-dev:latest"
+fi
+
+host_name="$(hostname)"
+if [[ -n "$cont_name" ]]; then
+  host_name="$host_name.$cont_name"
+fi
+
+if [[ -z "$mount_name" ]]; then
+  mount_name="$cont_name"
+fi
+
+if [[ -z "$mount_name" && -z "$cont_name" ]]; then
+  _exit "Please specify mount (-m,--mount) and/or container (-c,--container)" 1
+fi
+
+if [[ -z "$docker_user" ]]; then
+  docker_user="bts"
+fi
+
+if [[ "$docker_user" = "root" ]]; then
+  _exit "USER (--user) cannot be 'root'" 1
+fi
+
+cont_home="/home/$docker_user"
+home_src="$mount_prefix/$mount_name"
+home_tgt="$cont_home"
+this_dir="$(dirname "$(realpath "${BASH_SOURCE[0]}")")"
+repo_src="$(cd "$this_dir" && git rev-parse --show-toplevel)"
+repo_tgt="$cont_home/spark"
+working_dir="$cont_home"
+
+if [[ $mount_existing -eq 0 && -e "$home_src" ]]; then
+  # Don't overwrite existing dir (might be mounted in another container)
+  _exit "Mount dir '$home_src' already exists (use --mount-existing to mount \
+an existing directory)" 1
+fi
+
+mkdir -p "$home_src"
+
+if [[ "${#port_mappings[@]}" -eq 0 ]]; then
+  port_mappings=("8000:8000" "8001:8001" "8002:8002" "8003:8003")
+fi
+
+# Adapt port_mappings into args for `docker run` by prepending "-p " to each
+# item
+declare -a docker_port_args=()
+for port_mapping in "${port_mappings[@]}"; do
+  docker_port_args+=("-p $port_mapping")
+done
+
+echo "Starting container '$cont_name'
+image_name  : $image_name
+cont_name   : $cont_name
+home_src    : $home_src
+home_tgt    : $home_tgt
+repo_src    : $repo_src
+repo_tgt    : $repo_tgt
+working dir : $working_dir
+ports       : ${port_mappings[*]}
+"
+
+# DON'T wrap $docker_port_args in quotes. We want bash to split args by
+# whitespace
+docker run \
+  --net bridge \
+  -h "$host_name" \
+  -e "TZ=America/New_York" \
+  --cap-add=SYS_PTRACE \
+  --security-opt seccomp=unconfined \
+  --name "$cont_name" \
+  -v "$home_src:$home_tgt" \
+  -v "$repo_src:$repo_tgt" \
+  -w "$working_dir" \
+  ${docker_port_args[*]} \
+  --shm-size=4024m \
+  -t \
+  -i \
+  -d \
+  -e DISPLAY="$DISPLAY" \
+  "$image_name"
+
+# Add user w/ same uid and gid as the current user outside of docker, so the
+# user inside the docker has file permissions in the mounted dir.
+docker_group="$docker_user"
+docker_uid="$(id -u "$USER")"
+docker_gid="$(id -g "$USER")"
+if [[ -z "$docker_pass" ]]; then
+  docker_pass="$docker_user"
+fi
+docker exec -u root -it "$cont_name" "$repo_tgt/docker/sparkd/dev/set_user.sh" \
+--user "$docker_user" \
+--uid "$docker_uid" \
+--group "$docker_group" \
+--gid "$docker_gid" \
+--pass "$docker_pass"
+
+# Run same script VS Code runs to initialize the container
+docker exec -u "$docker_user" -it "$cont_name" "$repo_tgt/docker/sparkd/dev/devcontainer_post_create.sh"

docker/sparkd/docker_shell.sh

@@ -0,0 +1,64 @@
+#!/usr/bin/env bash
+
+function print_help_and_exit() {
+  cat <<EOF
+Open a shell inside the docker container
+Usage: docker_shell.sh [OPTIONS] [CONTAINER]
+
+Options:
+  -u,--user         The user to login with in the container (defaults to the current user)
+  -c,--container    The container name (may also be specified as positional arg)
+  -h,--help         Print this help message and exit
+
+EOF
+  exit "$1"
+}
+
+
+declare docker_user=""
+declare cont_name=""
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    -u)
+      ;&
+    --user)
+      if [[ -z "$2" ]]; then
+        echo "User (-u,--user) must be non-empty"
+        print_help_and_exit 1
+      fi
+      docker_user="$2"
+      shift 2
+      ;;
+    -h)
+      ;&
+    --help)
+      print_help_and_exit 0
+      ;;
+    -c)
+      ;&
+    --container)
+      shift 1
+      ;&
+    *)
+      if [[ ! -z "$cont_name" ]]; then
+        echo "Unexpected argument '$1'"
+        print_help_and_exit 1
+      fi
+      cont_name="$1"
+      shift 1
+      ;;
+  esac
+done
+
+
+if [[ -z "$cont_name" ]]; then
+  echo "Please specify the container ([-c,--container])"
+  print_help_and_exit 1
+fi
+
+if [[ -z "$docker_user" ]]; then
+  docker_user="$USER"
+fi
+
+docker exec -u "$docker_user" -it "$cont_name" bash

docker/sparkd/start_docker.sh

@@ -0,0 +1,233 @@
+#!/usr/bin/env bash
+
+mount_prefix="$HOME/.spark-dev/mounts"
+
+function print_help_and_exit() {
+  cat <<EOF
+Start a docker dev container
+Usage: start_docker.sh [OPTIONS] image container
+
+Positionals:
+  image             The image to instantiate
+  container         The name for the container
+
+Options:
+  -u,--user USER      The user to add in the container (defaults to the current user)
+  -p,--pass PASS      The password for USER in the container (defaults to USER)
+  -i,--image IMAGE    The image to instantiate
+  -c,--container CONT The name for the container
+  -m,--mount MOUNT    The name for the folder (outside the container) to create/mount to the home
+                      directory in the container. Mount folders are added here: $mount_prefix
+  --mount-existing    Mount the specified dir even if it already exists
+  --port <cont_port>:<host_port> OR [<cont_port>:<host_port>,...] Port mapping(s). May be
+                      specified multiple times.
+  -h,--help           Print this help message and exit
+
+EOF
+  exit "$1"
+}
+
+
+declare docker_user=""
+declare docker_pass=""
+declare image_name=""
+declare cont_name=""
+declare mount_name=""
+declare -i mount_existing=0
+declare -a port_mappings=()
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    -u)
+      ;&
+    --user)
+      if [[ -z "$2" ]]; then
+        echo "User (-u,--user) must be non-empty"
+        print_help_and_exit 1
+      fi
+      docker_user="$2"
+      shift 2
+      ;;
+    -p)
+      ;&
+    --pass)
+      if [[ -z "$2" ]]; then
+        echo "Password (-p,--pass) must be non-empty"
+        print_help_and_exit 1
+      fi
+      docker_pass="$2"
+      shift 2
+      ;;
+    -i)
+      ;&
+    --image)
+      if [[ -n "$image_name" ]]; then
+        echo "Unexpected argument '$1'"
+        print_help_and_exit 1
+      elif [[ -z "$2" ]]; then
+        echo "Image (-i,--image) must be non-empty"
+        print_help_and_exit 1
+      fi
+      image_name="$2"
+      shift 2
+      ;;
+    -c)
+      ;&
+    --container)
+      if [[ -n "$cont_name" ]]; then
+        echo "Unexpected argument '$1'"
+        print_help_and_exit 1
+      elif [[ -z "$2" ]]; then
+        echo "Container (-c,--container) must be non-empty"
+        print_help_and_exit 1
+      fi
+      cont_name="$2"
+      shift 2
+      ;;
+    --port)
+      # Parse port(s) arg, and append the result to 'port_mappings'
+      # Note: port(s) may be specified in bracketed notation:
+      #   [a:b,c:d,...,y:z]
+      next_ports="$(echo "$2" | sed -e 's/\[//g' -e 's/\]//g' -e 's/,/ /g')"
+      if [[ "${#port_mappings[@]}" -gt 0 ]]; then
+        next_ports=" $next_ports"
+      fi
+      port_mappings+=("$next_ports")
+      shift 2
+      ;;
+    -m)
+      ;&
+    --mount)
+      if [[ -z "$2" ]]; then
+        echo "Mount (-m,--mount) must be non-empty"
+      fi
+      mount_name="$2"
+      shift 2
+      ;;
+    --mount-existing)
+      mount_existing=1
+      shift 1
+      ;;
+    -h)
+      ;&
+    --help)
+      print_help_and_exit 0
+      ;;
+    *)
+      if [[ -z "$1" ]]; then
+        echo "Invalid positional argument '$1'"
+        print_help_and_exit 1
+      elif [[ -z "$image_name" ]]; then
+        image_name="$1"
+      elif [[ -z "$cont_name" ]]; then
+        cont_name="$1"
+      else
+        echo "Unexpected argument '$1'"
+        print_help_and_exit 1
+      fi
+      shift 1
+      ;;
+  esac
+done
+
+
+if [[ -z "$image_name" ]]; then
+  image_name="bluetradingsystems/spark-crypto-dev"
+fi
+
+host_name="$(hostname)"
+if [[ -n "$cont_name" ]]; then
+  host_name="$host_name.$cont_name"
+fi
+
+if [[ -z "$mount_name" ]]; then
+  mount_name="$cont_name"
+fi
+
+if [[ -z "$mount_name" && -z "$cont_name" ]]; then
+  echo "Please specify mount (-m,--mount) and/or container (-c,--container)"
+  print_help_and_exit 1
+fi
+
+if [[ -z "$docker_user" ]]; then
+  docker_user="$USER"
+fi
+
+home_src=$mount_prefix/$mount_name
+home_tgt=$HOME
+repo_src=$(git rev-parse --show-superproject-working-tree)
+repo_tgt=$HOME/$(basename "$repo_src")
+working_dir=$HOME
+
+if [[ $mount_existing -eq 0 && -e "$home_src" ]]; then
+  # Don't overwrite existing dir (might be mounted in another container)
+  echo "Mount dir '$home_src' already exists (use --mount-existing to mount an existing directory)"
+  print_help_and_exit 1
+fi
+
+mkdir -p "$home_src"
+
+if [[ "${#port_mappings[@]}" -eq 0 ]]; then
+  port_mappings=("8000:8000" "8001:8001" "8002:8002" "8003:8003")
+fi
+
+# Adapt port_mappings into args for `docker run` by prepending "-p " to each
+# item
+declare -a docker_port_args=()
+for port_mapping in "${port_mappings[@]}"; do
+  docker_port_args+=("-p $port_mapping")
+done
+
+echo "Starting container '$cont_name'
+image_name  : $image_name
+cont_name   : $cont_name
+home_src    : $home_src
+home_tgt    : $home_tgt
+repo_src    : $repo_src
+repo_tgt    : $repo_tgt
+working dir : $working_dir
+ports       : ${port_mappings[*]}
+"
+
+# DON'T wrap $docker_port_args in quotes. We want bash to split args by
+# whitespace
+docker run \
+  --net bridge \
+  -h "$host_name" \
+  -e "TZ=America/New_York" \
+  --cap-add=SYS_PTRACE \
+  --security-opt seccomp=unconfined \
+  --name "$cont_name" \
+  -v "$home_src:$home_tgt" \
+  -v "$repo_src:$repo_tgt" \
+  -w "$working_dir" \
+  ${docker_port_args[*]} \
+  --shm-size=4024m \
+  -t \
+  -i \
+  -d \
+  -e DISPLAY="$DISPLAY" \
+  "$image_name" \
+  bash
+
+# Add user w/ same uid and gid as the current user outside of docker, so the
+# user inside the docker has file permissions in the mounted dir.
+docker_group="$docker_user"
+docker_user_uid="$(id -u "$USER")"
+docker_user_gid="$(id -g "$USER")"
+
+echo "Adding user in container '$cont_name'
+name : $docker_user
+uid  : $docker_user_uid
+gid  : $docker_user_gid"
+
+if [[ -z "$docker_pass" ]]; then
+  docker_pass="$docker_user"
+  echo "Giving user '$docker_user' a secure password: $docker_pass"
+fi
+
+docker exec -u root -it "$cont_name" groupadd -g "$docker_user_gid" "$docker_group"
+docker exec -u root -it "$cont_name" useradd -u "$docker_user_uid" -g "$docker_user_gid" "$docker_user"
+docker exec -u root -it "$cont_name" usermod --password "$(openssl passwd -1 "$docker_pass")" "$docker_user"
+docker exec -u root -it "$cont_name" usermod -aG sudo "$docker_user"
+docker exec -u root -it "$cont_name" "$repo_tgt/externals/src/emumba/install_emumba-deribit-connector.sh"

src/client/spark/AW2/Bus/SingleVolSurfaceGateway.cs

@@ -55,8 +55,10 @@
       private void _on_vss(bts.api.core.Update<VolSurfaceEntryImmutable, UInt64> update) {
          IEnumerable<VolSurfaceEntryImmutable> all_vss = update.all;
          var this_vs_vss = all_vss.Where(vs => vs.VsId == _volsurface_id);
-         bts.Log.Gen.info(String.Format("SingleVolSurfaceGateway._on_vss count:{0} vs_count:{1}",
-            all_vss.Count(), this_vs_vss.Count()));
+         bts.Log.Gen.info(
+            $"SingleVolSurfaceGateway._on_vss vs_count: {this_vs_vss.Count()} " +
+            $"all: {update.all.Count()} added: {update.added.Count()} " +
+            $"updated: {update.changed.Count()} removed: {update.removed.Count()}");
          if (this_vs_vss.Count() == 0) {
             return;
          }
@@ -110,8 +112,13 @@
                   "SingleVolSurfaceGateway._on_bb_data Time to commit volsurface {0} {1} msecs",
                   _volsurface_id, _timer.lastIntervalTime));
                var archived_theos = _get_theos();
-               if (archived_theos != null)
+               if (archived_theos != null) {
+                  _timer.start();
                   _analytics_service.archive_theos(archived_theos);
+                  _timer.stop();
+                  Log.Gen.info($"SingleVolSurfaceGateway._on_bb_data archiving theos vs_id: {_volsurface_id} size: " +
+                               $"{archived_theos.SecTheo.Count} time: {_timer.lastIntervalTime} msecs");
+               }
             } catch (Exception ex) {
                Log.Gen.warn(ex.ToString());
                _bb.post(SparkTags.MessageDisplayTag,

src/client/spark/AW2/Bus/SingleVolSurfaceTheoSource.cs

@@ -160,8 +160,8 @@ namespace mbts.spark {
 
       public static IEnumerable<bts.api.SecurityTuple> get_vs_sectups(
          int vs_id, ThreadedBlackboard bb) {
-         VolSurfaceUsageListImmutable vs_usage_list = bb.read_state(SparkTags.VolSurfaceUsageTag);
-         return vs_usage_list.get_sectups(vs_id);
+         var vs_usage_list = bb.read_state(SparkTags.VolSurfaceUsageTag);
+         return (vs_usage_list?.get_sectups(vs_id)).empty_if_null();
       }
 
 
@@ -242,4 +242,4 @@ namespace mbts.spark {
       }
       #endregion
    }
-}
+}

src/client/spark/AW2/Bus/VolSurfaceBlackboardManager.cs

@@ -270,6 +270,7 @@ namespace mbts.spark {
                                           new mbts.analytics.VolSurfaceNonVolInput(exp_param_map.exp_map.ToDictionary()));
          } else if (_in_bb_req_vol.Equals(tag)) {
             ClientArchivedVolRequest client_req = _bb.read_state(_in_bb_req_vol);
+            bts.Log.Ui.info($"VolSurfaceBusManager._on_bb_update ClientArchivedVolRequest: {client_req.ToString()}");
             ArchivedTheoRetrieval ret = client_req.ret;
             if (ret == null || ret.SecTup == null) {
                bts.Log.Ui.warn($"VolSurfaceBusManager malformed ArchivedTheoRetrieval");

src/client/spark/AW2/InterpTheosServerPublisher.cs

@@ -5,6 +5,7 @@ using bts.api;
 using bts.collections;
 using System.Linq;
 using bts.api.orderman;
+using bts.api.analytics;
 using bts;
 
 namespace mbts.spark {
@@ -23,6 +24,8 @@ namespace mbts.spark {
          _order_service = order_service;
          _bb.subscribe(SparkTags.AllInterpGreeks);
          _bb.subscribe(SparkTags.SystemContractsTag);
+         _bb.subscribe(SparkTags.CurrentUser);
+         _bb.subscribe(SparkTags.VolSurfaceUsageTag);
          _bb.data_callback += _on_data;
       }
 
@@ -55,6 +58,27 @@ namespace mbts.spark {
          return rv;
       }
 
+      private IReadOnlyDictionary<string, OptionTheoInterpolationData>
+         _filter_for_publish(IReadOnlyDictionary<string, OptionTheoInterpolationData> all_theos) {
+         // find the symbol vs vs_id, get the committed vs, see if it is
+         // owned by the current user, and only then publish it
+         HashSet<string> syms = new HashSet<string>();
+         var usage = _bb.read_state(SparkTags.VolSurfaceUsageTag);
+         string current_user = _bb.read_state(SparkTags.CurrentUser).name;
+         foreach (var u in (usage?.UsageList).empty_if_null()) {
+            VolSurfaceInput vs_input =
+               _bb.read_state(SparkTags.VolSurfaceCommittedInputTag(u.VsId));
+            if (vs_input != null) {
+               if (vs_input.surface_settings.core_settings.vs_manager ==
+                  current_user) {
+                  syms.Add(u.SecTup.Symbol);
+               }
+            }
+         }
+         return
+            all_theos.Where(kv => syms.Contains(kv.Key)).ToDictionary(kv => kv.Key, kv => kv.Value);
+      }
+
       private void _publish_data() {
          SystemContractData sc_data = _bb.read_state(SparkTags.SystemContractsTag, null);
          IReadOnlyDictionary<string, OptionTheoInterpolationData> all_theos =
@@ -64,6 +88,11 @@ namespace mbts.spark {
          if (sc_data == null || all_theos == null || all_drivers == null)
             return;
          _timer.start();
+         all_theos = _filter_for_publish(all_theos);
+         if (all_theos.Count == 0) {
+            bts.Log.Gen.info($"InterpTheosServerPublisher._publish_data all_theos.Count == 0");
+            return;
+         }
          IDictionary<UInt32, DriverMD> driver_map = _get_drivers(all_drivers, sc_data);
          IEnumerable<Tuple<SecurityImmutable, TheoPoint>> theo_map = _get_theos(all_theos, sc_data);
          // assemble it together and send it to the server
@@ -86,6 +115,7 @@ namespace mbts.spark {
             ptp.Vega = is_call ? tp.cvega : tp.pvega;
             ptp.Theta = is_call ? tp.c_theta : tp.p_theta;
             ptp.Rho = Double.NaN;
+            ptp.Vol = tp.vol;
             ptp_list.Add(ptp);
          }
          bts.api.orderman.PrecomputedTheos server_theos = new PrecomputedTheos();
@@ -96,7 +126,7 @@ namespace mbts.spark {
          _order_service.set_precomputed_theos(server_theos);
          _timer.stop();
          var pub_time = _timer.lastIntervalTime;
-         bts.Log.Gen.info($"InterpTheosServerPublisher._publish_data " +
+         bts.Log.Gen.info($"InterpTheosServerPublisher._publish_data symbols: {all_theos.Count} " +
                           $"size: {server_theos.Theos.Count} comp_time: {comp_time} msecs " +
                           $"pub_time: {pub_time}");
       }
@@ -105,4 +135,4 @@ namespace mbts.spark {
          _bb.data_callback -= _on_data;
       }
    }
-}
+}

src/client/spark/AW2/ViewModel/SettingsEditors/VolSurfaceAdvancedSettingsEditorVM.cs

@@ -14,7 +14,7 @@ namespace mbts.spark {
    class VolSurfaceAdvancedSettingsEditorVM : ViewModelBase {
       readonly VolSurfaceAdvancedSettings _settings;
 
-      public VolSurfaceAdvancedSettingsEditorVM(VolSurfaceAdvancedSettings settings, string vs_name, Action<VolSurfaceAdvancedSettings> commit) {
+      public VolSurfaceAdvancedSettingsEditorVM(VolSurfaceAdvancedSettings settings, string vs_name) {
          _settings = settings;
          _settings.theo_handle_measure = VolSurfaceEnums.GeneralChangeMeasure.Absolute;
          _settings.slide_handle_measure = VolSurfaceEnums.GeneralChangeMeasure.Percentage;
@@ -23,7 +23,7 @@ namespace mbts.spark {
          if (_settings.precomputed_theo_handles == null)
             _settings.precomputed_theo_handles = new List<double>();
          vol_surface_name = vs_name;
-         this.commit = new DelegateCommand(() => request_commit?.Invoke(ProtobufSerializationHelper.clone(_settings)));
+         commit = new DelegateCommand(() => request_commit?.Invoke(ProtobufSerializationHelper.clone(_settings)));
       }
 
       public event Action<VolSurfaceAdvancedSettings> request_commit;

src/client/spark/AW2/ViewModel/VolSurfaceVM.cs

@@ -121,21 +121,24 @@ namespace mbts.spark {
             if (settings == null) {
                return null;
             }
-            // return new VolSurfaceAdvancedSettingsEditor {
-            //    DataContext = new VolSurfaceAdvancedSettingsEditorVM(
-            //       VolSurfaceUtilityFunctions.clone_proto(settings.advanced_settings),
-            //       settings.core_settings.vol_surface_name,
-            //       advanced_settings => {
-            //          var current_settings = _get_volsurface_settings();
-            //          if (current_settings == null) BTSMessageBox.Show("Error changing settings", "VS Settings", System.Windows.MessageBoxButton.OK, System.Windows.MessageBoxImage.Error, true, true, true);
-            //          else {
-            //             current_settings.advanced_settings = advanced_settings;
-            //             _model.set_candidate_settings(current_settings);
-            //          }
-            //       }
-            //    )
-            // };
-            return null;
+            var vm = new VolSurfaceAdvancedSettingsEditorVM(
+               VolSurfaceUtilityFunctions.clone_proto(settings.advanced_settings),
+               settings.core_settings.vol_surface_name);
+            vm.request_commit += commit_advanced_settings;
+            return new VolSurfaceAdvancedSettingsEditor {
+               DataContext = vm
+            };
+
+            async void commit_advanced_settings(VolSurfaceAdvancedSettings value) {
+               var current_settings = await _get_volsurface_settings();
+               if (current_settings == null) {
+                  BTSMessageBox.Show("Error changing settings", "VS Settings");
+               }
+               else {
+                  current_settings.advanced_settings = value;
+                  _model.set_candidate_settings(current_settings);
+               }
+            }
          }
 
          Window? mk_grid_settings_win() {

src/client/spark/AW2/VolSurfaceInterpGreeksMultiplexer.cs

@@ -22,7 +22,7 @@ namespace mbts.spark {
    // ii. IntepGreeks for each vs
    public class VolSurfaceInterpGreeksMultiplexer : IDisposable {
       private readonly ThreadedBlackboard _bb;
-      private readonly ITag<VolSurfaceUsageListImmutable> _in_bb_vs_usage_tag;
+      private readonly ITag<VolSurfaceUsageListImmutable?> _in_bb_vs_usage_tag;
       private IDictionary<int, ITag<VSTheos>> _vs_theo_tags =
          new Dictionary<int, ITag<VSTheos>>();
       private VolSurfaceUsageListImmutable _vs_usage_map =
@@ -48,8 +48,8 @@ namespace mbts.spark {
       }
 
       private void _analyze_usages() {
-         _vs_usage_map = _bb.read_state(_in_bb_vs_usage_tag,
-            new VolSurfaceUsageListImmutable(new VolSurfaceUsageList()));
+         _vs_usage_map = _bb.read_state(_in_bb_vs_usage_tag) ??
+            new VolSurfaceUsageList().immutable();
          bts.Log.Gen.info($"VolSurfaceInterpGreeksMultiplexer._analyze_usages " +
                           $"count: {_vs_usage_map.UsageList.Count()}");
          foreach (int vs_id in _vs_usage_map.vss()) {

src/client/spark/Analytics/Core/Settings/VolSurfaceAdvancedSettings.cs

@@ -9,26 +9,34 @@ namespace mbts.analytics
    [ProtoBuf.ProtoContract(Name = "VolSurfaceAdvancedSettings")]
    public class VolSurfaceAdvancedSettings
    {
+      public static VolSurfaceAdvancedSettings Default =>
+         new() {
+            number_of_cashed_saved_days = double.NaN,
+            precomputed_theo_handles = new(0),
+            theo_handle_measure = VolSurfaceEnums.GeneralChangeMeasure.Absolute,
+            precomputed_slide_handles = new(0),
+            slide_handle_measure = VolSurfaceEnums.GeneralChangeMeasure.Percentage,
+            publish_from_client = false,
+            publish_from_client_user = string.Empty,
+            strike_normalization_axis = VolSurfaceEnums.VolCurveStrikeAxis.RawStrike,
+         };
+
       [ProtoBuf.ProtoMember(1)]
-      [System.ComponentModel.DefaultValue(Double.NaN)]
       public double number_of_cashed_saved_days;
       
       [ProtoBuf.ProtoMember(10)]      
       public List<double> precomputed_theo_handles;
       
       [ProtoBuf.ProtoMember(15)]
-      [System.ComponentModel.DefaultValue(VolSurfaceEnums.GeneralChangeMeasure.Absolute)]
       public VolSurfaceEnums.GeneralChangeMeasure theo_handle_measure;
       
       [ProtoBuf.ProtoMember(20)]      
       public List<double> precomputed_slide_handles;
       
       [ProtoBuf.ProtoMember(25)]
-      [System.ComponentModel.DefaultValue(VolSurfaceEnums.GeneralChangeMeasure.Percentage)]
       public VolSurfaceEnums.GeneralChangeMeasure slide_handle_measure;
 
       [ProtoBuf.ProtoMember(30)]
-      //[System.ComponentModel.DefaultValue(false)]
       public bool publish_from_client;
 
       [ProtoBuf.ProtoMember(40)]
@@ -38,5 +46,11 @@ namespace mbts.analytics
       [ProtoBuf.ProtoMember(50)]
       [System.ComponentModel.DefaultValue(VolSurfaceEnums.VolCurveStrikeAxis.RawStrike)]
       public VolSurfaceEnums.VolCurveStrikeAxis strike_normalization_axis;
+      
+      public void ensure_no_nulls() {
+         precomputed_theo_handles ??= new(0);
+         precomputed_slide_handles ??= new(0);
+         publish_from_client_user ??= string.Empty;
+      }
    }
 }

src/client/spark/Analytics/Core/Settings/VolSurfaceSettings.cs

@@ -25,7 +25,8 @@ namespace mbts.analytics {
             volpath_settings = new(),
             safety_limits = new(),
          },
-         polynomial_model_settings = new()
+         polynomial_model_settings = new(),
+         advanced_settings = VolSurfaceAdvancedSettings.Default
       };
 
       [ProtoMember(10)]
@@ -79,6 +80,8 @@ namespace mbts.analytics {
          if (core_settings.time_settings == null) {
             core_settings.time_settings = new VolSurfaceTimeSettings();
          }
+         advanced_settings ??= VolSurfaceAdvancedSettings.Default;
+         advanced_settings.ensure_no_nulls();
          return this;
       }
    }

src/client/spark/Analytics/Pricing/VolSurfaceInterpGreeksProducer.cs

@@ -40,10 +40,8 @@ namespace mbts.analytics {
       /// last computed min/max driver for each exp
       private IDictionary<VolSurfaceExpiration, Tuple<double, double>>
       _computed_driver_range_map = new Dictionary<VolSurfaceExpiration, Tuple<double, double>>();
-      private readonly IEnumerable<double> _default_abs_moves =
-         new double[] { 300, -200, -125, -100, -75, -50, -25, 0, 25, 50, 75, 100, 125, 200, 300 };
+      private readonly IEnumerable<double> _default_abs_moves =  config.Configuration.get<bts.config.VolSurface>().default_abs_moves;
       private readonly double _cushion_percentage = 25.0;
-
       public event
       Action<IDictionary<VolSurfaceExpiration, IEnumerable<Theos>>, double, double> on_theos =
          (o, vt, tt) => { };
@@ -52,7 +50,7 @@ namespace mbts.analytics {
          (o, vt, tt) => { };
 
       public VolSurfaceInterpGreeksProducer(int vs_id) {
-         Log.Gen.info($"VolSurfaceInterpGreeksProducer.cons vs_id: {vs_id}");
+        Log.Gen.info($"VolSurfaceInterpGreeksProducer.cons vs_id: {vs_id}, _default_abs_moves: {string.Join(", ", _default_abs_moves)}");
          _vs_id = vs_id;
       }
 
@@ -72,7 +70,7 @@ namespace mbts.analytics {
       }
 
       public void set_vol_data(VolSurfaceInput vs_input) {
-         Log.Gen.info($"VolSurfaceInterpGreeksProducer.set_vol_data vs_id: {_vs_id}");
+         Log.Gen.info($"VolSurfaceInterpGreeksProducer.set_vol_data vs_id: {_vs_id} precomputed_theo_handles?: {string.Join(", ", (vs_input?.surface_settings?.advanced_settings?.precomputed_theo_handles).empty_if_null())}");
          _vs_input = vs_input != null ? VolSurfaceUtilityFunctions.clone_proto(vs_input) : null;
          _compute_and_fire(InputType.Vol);
       }

src/client/spark/Modules/Analytics/VolSurface/View/VolSurfaceAdvancedSettingsEditor.xaml

@@ -0,0 +1,58 @@
+<Window
+  x:Class="mbts.ui.shell.VolSurfaceAdvancedSettingsEditor"
+  x:ClassModifier="internal"
+  xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation"
+  xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
+  xmlns:c="clr-namespace:mbts.ui.controls"
+  xmlns:cv="clr-namespace:mbts.ui.converters"
+  Title="{Binding vol_surface_name, StringFormat='{}Advanced Settings \&apos;{0}\&apos;'}"
+  Style="{StaticResource BTSDialog}"
+  SizeToContent="WidthAndHeight"
+  TextElement.FontSize="10">
+  <StackPanel>
+    <Grid Margin="5" Grid.IsSharedSizeScope="True">
+      <Grid.RowDefinitions>
+        <RowDefinition Height="Auto"/>
+        <RowDefinition Height="Auto"/>
+        <RowDefinition Height="Auto"/>
+        <RowDefinition Height="Auto"/>
+        <RowDefinition Height="Auto"/>
+      </Grid.RowDefinitions>
+      <Grid.ColumnDefinitions>
+        <ColumnDefinition Width="Auto" SharedSizeGroup="label"/>
+        <ColumnDefinition Width="*"/>
+      </Grid.ColumnDefinitions>
+      <GroupBox Grid.ColumnSpan="2" Padding="3">
+        <GroupBox.Header>
+          <TextBlock Text="Theos" FontWeight="Bold"/>
+        </GroupBox.Header>
+        <Grid>
+          <Grid.RowDefinitions>
+            <RowDefinition Height="Auto"/>
+            <RowDefinition Height="Auto"/>
+          </Grid.RowDefinitions>
+          <Grid.ColumnDefinitions>
+            <ColumnDefinition Width="Auto" SharedSizeGroup="label"/>
+            <ColumnDefinition Width="Auto"/>
+          </Grid.ColumnDefinitions>
+          <c:RadioButtonsControl Grid.ColumnSpan="2" Margin="0,0,0,5" ItemsSource="{Binding general_change_measures}" SelectedItem="{Binding theo_handle_measure}" IsEnabled="False"/>
+          <TextBlock Grid.Row="1" Text="Precomputed Handles" VerticalAlignment="Center"/>
+          <TextBox
+            x:Name="PART_Handles"
+            Validation.ErrorTemplate="{x:Null}"
+            Grid.Row="1"
+            Grid.Column="1"
+            Width="160"
+            Margin="5,0,0,0"
+            Text="{Binding precomputed_theo_handles, Converter={x:Static cv:EnumerableDoubleStringConverter.Instance}}" />
+        </Grid>
+      </GroupBox>
+      <TextBlock Visibility="Collapsed" Grid.Row="1" VerticalAlignment="Center" Margin="5,0,0,0" Text="Num Cash Saved Days" />
+      <c:TickUpDownDouble Visibility="Collapsed" Grid.Row="1" Grid.Column="1" HorizontalAlignment="Stretch" Margin="14,5,5,5" Value="{Binding number_of_cash_saved_days}"/>
+      <TextBlock Grid.Row="2" VerticalAlignment="Center" Margin="5,0,0,0" Text="Strike Normalization Axis"/>
+      <ComboBox Grid.Row="2" Grid.Column="1" HorizontalAlignment="Stretch" Margin="14,5,5,5" ItemsSource="{Binding strike_normalization_axes}" SelectedValue="{Binding selected_strike_normalization_axis}"/>
+      <CheckBox Grid.Row="3" Margin="7,0,0,0" Content="Publish From Client" IsChecked="{Binding publish_from_client}"/>
+    </Grid>
+    <c:DialogSkirtOKCancel OKCommand="{Binding commit}" />
+  </StackPanel>
+</Window>

src/client/spark/Modules/Analytics/VolSurface/View/VolSurfaceAdvancedSettingsEditor.xaml.cs

@@ -0,0 +1,11 @@
+namespace mbts.ui.shell {
+   using System.Windows;
+
+   /// <summary>
+   /// Interaction logic for VolSurfaceAdvancedSettingsEditor.xaml
+   /// </summary>
+   partial class VolSurfaceAdvancedSettingsEditor : Window {
+      public VolSurfaceAdvancedSettingsEditor() =>
+         InitializeComponent();
+   }
+}

src/client/spark/Modules/Trading/Ladder/View/LadderGridHost.cs

@@ -1281,7 +1281,7 @@ namespace mbts.spark {
          if (!_settings.piq_column.show)
             return;
          int origin_index = LadderRender.index_of(_price_list, _mkt_break_lines.origin, _price_comparer);
-         LadderRender.render_piqs(_piq_vis_tup.visual, _last_vis_tup.glyph_context, origin_index, _price_list, _vm.price_vs_piq, _settings.row_height, _spec, _settings.piq_column, _fmt_mkt_quantity);
+         LadderRender.render_piqs(_piq_vis_tup.visual, _last_vis_tup.glyph_context, _price_list, _vm.price_vs_piq, _settings.row_height, _spec, _settings.piq_column, _fmt_mkt_quantity);
       }
 
       void _refresh_sizes() {

src/client/spark/Modules/Trading/Ladder/View/LadderRender.cs

@@ -368,30 +368,32 @@ namespace mbts.spark {
 
       public delegate void RenderCell<T>(DrawingContext dc, int row, T data_item);
 
-      static void _render_cells<T>(DrawingVisual dv, double row_height, IEnumerable<(double price, T value)> data, LadderColumnSettings settings, IList<double> prices, RegionalTickFactors spec, RenderCell<T> render_cell) {
+      static void _render_cells<T>(DrawingVisual dv, double row_height, IEnumerable<(double price, T value)> cells, LadderColumnSettings settings, IList<double> prices, RegionalTickFactors spec, RenderCell<T> render_cell) {
          using (var dc = dv.RenderOpen())
          using (_draw_with_clip(dc, settings, row_height, prices.Count)) {
-            if (data.is_null_or_empty())
+            if (prices.is_null_or_empty() || cells.is_null_or_empty()) {
                return;
-            int curr_index = 0;
-            double curr_price = prices[curr_index];
-            foreach (var (price, value) in data) {
-               if (double.IsNaN(price))
-                  continue;
-               // prices are in descending order, so get diff from curr_price - price
-               var tick_diff = spec.tickdiff(curr_price, price) ?? 0;
-               Log.assert(
-                  tick_diff > (int.MaxValue - curr_index),
-                  $"Unable to index ladder row for price {price}: tick count exceeds int.MaxValue");
-               curr_index += (int)tick_diff;
-               curr_price = price;
-               if (curr_index >= 0 && curr_index < prices.Count) {
-                  render_cell(dc, curr_index, value);
-               } else break;
+            }
+            var min_price = prices[prices.Count - 1];
+            var max_price = prices[0];
+            Log.assert(min_price.is_real() && max_price.is_real() && min_price <= max_price);
+            var indexed_cells = cells
+               .Where(item => item.price.is_real() && item.price.is_within_range(min_price, max_price))
+               .Select(row => {
+                  checked {
+                     return (row.price, row.value, (int)spec.tickdiff(max_price, row.price));
+                  }
+               });
+            foreach (var (price, value, idx) in indexed_cells) {
+               var check_price = spec.tickdown(max_price, idx);
+               Log.assert(spec.tick_equals(price, check_price));
+               Log.assert(idx >= 0 && idx < prices.Count);
+               render_cell(dc, idx, value);
             }
          }
       }
 
+
       internal static void render_last(DrawingVisual dv, GlyphContext glyph_context, NativePriceSize last, int index, double row_height, LadderColumnSettings settings, Func<uint, string> fmt_value) {
          if (fmt_value is null)
             fmt_value = FormattingDelegates.default_string<uint>();
@@ -405,38 +407,18 @@ namespace mbts.spark {
       internal static void render_piqs(
          DrawingVisual dv,
          GlyphContext glyph_context,
-         int origin_index,
          IList<double> prices,
          IEnumerable<(double price, uint piq)> pvp,
          double row_height,
          RegionalTickFactors spec,
          LadderColumnSettings settings,
-         Func<uint, string> fmt_value) {
-         //
-         using (var dc = dv.RenderOpen()) {
-            using (_draw_with_clip(dc, settings, row_height, prices.Count)) {
-               if (!prices.within_range(origin_index) || pvp.is_null_or_empty())
-                  return;
-               if (fmt_value is null)
-                  fmt_value = FormattingDelegates.default_string<uint>();
-               var foreground = settings.foreground.rgb_brush();
-               int curr_index = origin_index;
-               double curr_price = prices[curr_index];
-               foreach (var entry in pvp) {
-                  double price = entry.price;
-                  // prices are in descending order, so get diff from curr_price - price
-                  var tick_diff = spec.tickdiff(curr_price, price).assert_value();
-                  checked {
-                     curr_index += (int)tick_diff;
-                  }
-                  curr_price = price;
-                  if (curr_index >= 0 && curr_index < prices.Count) {
-                     var gr = format_text(fmt_value(entry.piq), curr_index, row_height, settings, glyph_context);
-                     dc.DrawGlyphRun(foreground, gr);
-                  } else break;
-               }
-            }
-         }
+         Func<uint, string> fmt_value
+      ) {
+         fmt_value ??= FormattingDelegates.default_string<uint>();
+         var fg = settings.foreground.rgb_brush();
+         _render_cells(dv, row_height, pvp, settings, prices, spec, (dc, row, piq) =>
+            dc.DrawGlyphRun(
+               fg, format_text(fmt_value(piq), row, row_height, settings, glyph_context)));
       }
 
       public static void render_mkt_break(DrawingVisual dv, int index, double row_height, uint color) =>

src/client/spark/Modules/Trading/ManualFills/ManualFillRow.cs

@@ -59,7 +59,9 @@ namespace bts.shell.manual_fills {
             nameof(quantity_display),
             () => {
                var sec_meta = security.HasValue ? _sec_meta_store.get(security.Value) : null;
-               return sec_meta?.QuantityDisplay ?? Security.Defaults.quantity_display_settingsi;
+               return sec_meta?.HasQuantityDisplay ?? false ?
+                  sec_meta.Value.QuantityDisplay :
+                  Security.Defaults.quantity_display_settingsi;
          });
          // TODO (prob): remove conversion_data.  Replace get_qty_display_divisor delegates w/
          // TickDisplaySettingsImmutable for qty, obtained via ISecurityMetadataStore.

src/client/spark/Modules/Trading/PNL/VM/PNLRowVM.cs

@@ -293,7 +293,10 @@ namespace bts.shell.pnl {
                return _quantity_display_backing.Value;
             }
             if (security.HasValue) {
-               _quantity_display_backing = _sec_meta_store.get(security.Value)?.QuantityDisplay;
+               var sec_meta = _sec_meta_store.get(security.Value);
+               if (sec_meta?.HasQuantityDisplay ?? false) {
+                  _quantity_display_backing = sec_meta.Value.QuantityDisplay;
+               }
             }
             return _quantity_display_backing ??= Security.Defaults.quantity_display_settingsi;
          }

src/client/spark/Notifications/INotificationService.cs

@@ -121,7 +121,7 @@ namespace bts.shell {
 
       void _on_orders(TableUpdate<(SecurityImmutable? sec, OrderRecordImmutable order), uint> update) {
          // must update notification_service_data_cache first, some of the predicate/triggers (esp. ToastNotifications) will use the notification_service_data_cache to lookup order-history
-         _data_cache.update(update.as_iincremental());
+         _data_cache.update(update);
          foreach (var o in update.added.Concat(update.changed))
             _order_notifications.process_item(o);
       }

src/client/spark/Notifications/INotificationServiceDataCache.cs

@@ -1,89 +1,157 @@
-using bts.api;
-using bts.api.orderman;
-
-using mbts.spark;
-using mbts.utils;
-
-using System;
-using System.Collections.Generic;
-
-namespace bts.shell {
-   // Any predicate-sound/notifications that need to reference previous data (i.e. notifications based on change/time, or state changes) should use the INotificationServiceDataCache.
-   // The INotificationService updates these caches in its IObservable.OnNext callbacks, just before calling all of the predicate-trigger methods.
-   // Therfore, data referenced through this interface is guarenteed to be up-to-date, and thread safe.  It also reduces performance overhead by avoiding redundant bookeeping.
-   // Note: last_order_reset_time can be used by sound/toast-notification sources to avoid sending false-positive notifications for old orders (i.e. all the orders that arrive in the first update on startup/server restart)
-   // --------------------------------------------------------------------------------------------------------------------------------------------
-   // IMPOARTANT!!! The current implementation of this interface is NOT INTENDED FOR GENERAL USE - order_history is NOT THREAD-SAFE
-   interface INotificationServiceDataCache {
-      DateTime last_order_reset_time { get; }
-      bool is_first_order_update { get; }
-      IReadOnlyDictionary<uint, List<TimeStampedData<OrderSecurity>>> order_history { get; }
-   }
-
-   static class NotificationServiceDataCacheExtensions {
-      public static OrderSecurity? get_previous_order_state(this INotificationServiceDataCache notification_service_data_cache, uint key) {
-         if (notification_service_data_cache.order_history.TryGetValue(key, out List<TimeStampedData<OrderSecurity>> l)) {
-            if (l.Count > 1) {
-               return l[l.Count - 2].data;
-            }
-         }
-         return null;
-      }
-   }
-
-   struct TimeStampedData<T> {
-      public readonly T data;
-      public readonly DateTime time_stamp;
-
-      public TimeStampedData(T data, DateTime time_stamp) {
-         this.data = data;
-         this.time_stamp = time_stamp;
-      }
-   }
-
-   partial class NotificationService : INotificationService, IDisposable {
-      class DataCache : INotificationServiceDataCache {
-         const int _buffer_size = 10;
-         readonly Dictionary<uint, OrderSecurity> _order_cache = new Dictionary<uint, OrderSecurity>();
-         readonly Dictionary<uint, List<TimeStampedData<OrderSecurity>>> _order_history = new Dictionary<uint, List<TimeStampedData<OrderSecurity>>>();
-
-         public IReadOnlyDictionary<uint, List<TimeStampedData<OrderSecurity>>> order_history => _order_history;
-
-         public DateTime last_order_reset_time { get; private set; }
-         public bool is_first_order_update { get; private set; }
-
-         internal void update(IIncrementalUpdate<(SecurityImmutable? sec, OrderRecordImmutable order), uint> data) {
-            DateTime now = DateTime.Now;
-            if (data.removed.Count == _order_cache.Count) {
-               last_order_reset_time = now;
-               _order_cache.Clear();
-            }
-            is_first_order_update = _order_cache.Count == 0 && data.added.Count > 0;
-            foreach (var kv in data.removed) {
-               if (data.added.ContainsKey(kv.Key)) {
-                  _order_cache[kv.Key] = kv.Value;
-                  _order_history[kv.Key].Add(new TimeStampedData<OrderSecurity>(kv.Value, now));
-               } else {
-                  _order_cache.Remove(kv.Key);
-                  // TODO: maybe these shouldn't be removed?
-                  // if we add delta/time predicates that need to refer to order history, we might need to keep them around
-                  // although, if the server is telling us to remove them, perhaps we shouldn't consider such data.
-                  _order_history.Remove(kv.Key);
-               }
-            }
-            foreach (var kv in data.added) {
-               if (data.removed.ContainsKey(kv.Key))
-                  continue;
-               _order_cache.Add(kv.Key, kv.Value);
-               _order_history.AddOrUpdate(kv.Key,
-                  k => new List<TimeStampedData<OrderSecurity>> { new TimeStampedData<OrderSecurity>(kv.Value, now) },
-                  (k, l) => {
-                     l.Add(new TimeStampedData<OrderSecurity>(kv.Value, now));
-                     if (l.Count >= 10)
-                        l.RemoveAt(0);
-                  });
-            }
-         }
-      }
-   }
-}
+using bts.api;
+using bts.api.core;
+using bts.api.orderman;
+
+using mbts.spark;
+using mbts.utils;
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+
+namespace bts.shell {
+   // Any predicate-sound/notifications that need to reference previous data (i.e. notifications based on change/time, or state changes) should use the INotificationServiceDataCache.
+   // The INotificationService updates these caches in its IObservable.OnNext callbacks, just before calling all of the predicate-trigger methods.
+   // Therfore, data referenced through this interface is guarenteed to be up-to-date, and thread safe.  It also reduces performance overhead by avoiding redundant bookeeping.
+   // Note: last_order_reset_time can be used by sound/toast-notification sources to avoid sending false-positive notifications for old orders (i.e. all the orders that arrive in the first update on startup/server restart)
+   // --------------------------------------------------------------------------------------------------------------------------------------------
+   // IMPOARTANT!!! The current implementation of this interface is NOT INTENDED FOR GENERAL USE - order_history is NOT THREAD-SAFE
+   interface INotificationServiceDataCache {
+      bool is_first_order_update { get; }
+      IReadOnlyDictionary<uint, List<TimeStampedData<OrderSecurity>>> order_history { get; }
+   }
+
+   static class NotificationServiceDataCacheExtensions {
+      public static OrderSecurity? get_previous_order_state(this INotificationServiceDataCache notification_service_data_cache, uint key) {
+         if (notification_service_data_cache.order_history.TryGetValue(key, out List<TimeStampedData<OrderSecurity>> l)) {
+            if (l.Count > 1) {
+               return l[l.Count - 2].data;
+            }
+         }
+         return null;
+      }
+   }
+
+   struct TimeStampedData<T> {
+      public readonly T data;
+      public readonly DateTime time_stamp;
+
+      public TimeStampedData(T data, DateTime time_stamp) {
+         this.data = data;
+         this.time_stamp = time_stamp;
+      }
+   }
+
+   partial class NotificationService : INotificationService, IDisposable {
+      class DataCache : INotificationServiceDataCache {
+         const int _ORDER_HISTORY_BUFFER_SIZE = 10;
+         const int _TABLE_HISTORY_BUFFER_SIZE = 5;
+
+         readonly Dictionary<uint, OrderSecurity> _order_cache = new Dictionary<uint, OrderSecurity>();
+         readonly Dictionary<uint, List<TimeStampedData<OrderSecurity>>> _order_history = new Dictionary<uint, List<TimeStampedData<OrderSecurity>>>();
+         readonly List<(TableUpdate<(SecurityImmutable?, OrderRecordImmutable), uint>, DateTime)> _table_history = new(_TABLE_HISTORY_BUFFER_SIZE);
+         readonly object _lock = new();
+
+         public IReadOnlyDictionary<uint, List<TimeStampedData<OrderSecurity>>> order_history => _order_history;
+
+         public bool is_first_order_update { get; private set; }
+
+         internal void update(TableUpdate<(SecurityImmutable? sec, OrderRecordImmutable order), uint> table) {
+            // First, try processing the update per usual (incrementally). If it throws,
+            // then log the exception and reset the state.
+            DateTime now = DateTime.Now;
+            lock (_lock) {
+               _circular_buffer_add(_table_history, (table, now), _TABLE_HISTORY_BUFFER_SIZE);
+               try {
+                  impl();
+                  return;
+               } catch (Exception ex) {
+                  var table_history_desc = _fmt_order_table_history_safe();
+                  Log.Ui.error(
+                     $"Error processing order update in NotificationService: {ex}.\n" +
+                     $"***Last table updates below***\n" +
+                     table_history_desc);
+               }
+               // This is the 'escape' path. It really shouldn't throw.
+               reset();
+            }
+
+            void impl() {
+               if (table.state.Count == 0) {
+                  _order_cache.Clear();
+               }
+               is_first_order_update = _order_cache.Count == 0 && table.state.Count > 0;
+               foreach (var order in table.removed) {
+                  var k = table.key(order);
+                  _order_cache.Remove(k);
+                  _order_history.Remove(k);
+               }
+               foreach (var ch in table.changes) {
+                  var k = table.key(ch.New);
+                  _order_cache[k] = ch.New;
+                  _order_history[k].Add(new(ch.New, now));
+               }
+               foreach (var order in table.added) {
+                  var k = table.key(order);
+                  _order_cache[k] = order;
+                  var hist = _order_history.GetOrAdd(k, _ => new());
+                  _circular_buffer_add(hist, new(order, now), _ORDER_HISTORY_BUFFER_SIZE);
+               }
+            }
+
+            void reset() {
+               _order_cache.Clear();
+               _order_history.Clear();
+               _table_history.Clear();
+               foreach (var (k, order) in table.state) {
+                  _order_cache[k] = order;
+                  _order_history.GetOrAdd(k, _ => new()).Add(new(order, now));
+               }
+               is_first_order_update = true;
+            }
+         }
+
+         static void _circular_buffer_add<T>(List<T> list, T item, int buffer_size) {
+            Log.assert(buffer_size > 0);
+            if (list.Count > buffer_size - 1) {
+               // Remove oldest items (trim front)
+               list.RemoveRange(
+                  0, list.Count - (buffer_size - 1));
+            }
+            list.Add(item);
+         }
+
+         string _fmt_order_table_history_safe() {
+            return _fmt_table_history_safe(_table_history, fmt_order_tup);
+
+            static string fmt_order_tup((SecurityImmutable?, OrderRecordImmutable) order_tup) {
+               var (sec, order) = order_tup;
+               var order_desc = OrderUtils.short_description(order, sec.spec_or_default());
+               var sec_desc = sec?.ToString() ?? ("sid=" + order.security_key().ToString());
+               return $"{sec_desc} {order_desc}";
+            }
+         }
+
+         static string _fmt_table_history_safe<T, TKey>(
+            List<(TableUpdate<T, TKey>, DateTime)> table_history, Func<T, string> fmt_value) where TKey : IEquatable<TKey> {
+            var table_history_desc = new System.Text.StringBuilder();
+            for (int i = 0; i < table_history.Count; i++) {
+               var (table, time_stamp) = table_history[i];
+               table_history_desc.AppendLine(
+                  $"Update - {table_history.Count - i}, at {time_stamp:yyyy-MM-dd hh:mm:ss.ffffff}");
+               table_history_desc.AppendLine(_fmt_table_safe(table, fmt_value));
+            }
+            return table_history_desc.ToString();
+         }
+
+         static string _fmt_table_safe<T, TKey>(
+            TableUpdate<T, TKey> table, Func<T, string> fmt_value) where TKey : IEquatable<TKey> {
+            try {
+               return table.description(fmt_value);
+            } catch (Exception ex) {
+               return $"Exception while formatting TableUpdate: ${ex}";
+            }
+         }
+      }
+   }
+}

src/client/spark/Predicates/Data/Events/OrderCompletedEventPredicate.cs

@@ -90,7 +90,8 @@ namespace mbts {
             }
             var sec = order.security;
             var sec_meta = sec.HasValue ? _sec_meta_store.get(sec.Value) : null;
-            var qty_display = sec_meta?.QuantityDisplay ??
+            var qty_display = sec_meta?.HasQuantityDisplay ?? false ?
+               sec_meta.Value.QuantityDisplay :
                Security.Defaults.quantity_display_settingsi;
             return ToastNotifications.order_completed(order, qty_display);
          }

src/client/spark/core/Bus/SparkTags.cs

@@ -266,8 +266,8 @@ namespace mbts.spark {
 
       // List of symbol to volsurface mapping.
       // Publisher: VolSurfaceBlackboardManager
-      public static ITag<VolSurfaceUsageListImmutable> VolSurfaceUsageTag { get; } =
-         _mk_tag<VolSurfaceUsageListImmutable>();
+      public static ITag<VolSurfaceUsageListImmutable?> VolSurfaceUsageTag { get; } =
+         _mk_tag<VolSurfaceUsageListImmutable?>();
 
       // The AW2 machinary can compute and publish the theos
       // right back locally(vs the server doing that) if configured that way.

src/client/spark/view/Controls/LogViewer.xaml

@@ -10,8 +10,8 @@
   </ResourceDictionary.MergedDictionaries>
 
   <Style x:Key="LogViewer.TextBox" TargetType="TextBox">
-    <Setter Property="Background" Value="{StaticResource TextBox.BackgroundBrush}" />
-    <Setter Property="Foreground" Value="{StaticResource TextBox.ForegroundBrush}" />
+    <Setter Property="Background" Value="{u:BTSColor UI.TextBox.default_background}" />
+    <Setter Property="Foreground" Value="{u:BTSColor UI.TextBox.default_foreground}" />
     <Setter Property="BorderThickness" Value="0" />
   </Style>
 

src/lib/CMakeDeps_win.cmake

@@ -1 +1 @@
-set(sources api/analytics/ExpirationDriver.proto;api/analytics/TimeWeightedInterpolatedDriver.proto;api/apicore.proto;api/apigateway.proto;api/apioptions.proto;api/core/ApiClientInterop.cpp;api/core/ApiServerInterop.cpp;api/domain/Currency.proto;api/domain/SecurityCriteria.proto;api/domain/SecurityCriteriaClause.proto;api/domain/SecurityExpiration.proto;api/domain/SecurityMetadata.proto;api/domain/SecurityMetadataCollection.proto;api/domain/SecurityMetadataItem.proto;api/domain/SecurityType.proto;api/domain/TickDisplaySettings.proto;api/domain/TickFactors.proto;api/domain/Venue.proto;api/io/URL.proto;api/md/MarketDataSubscriptionParams.proto;api/md/binance/BinanceDepthStreamParams.proto;api/md/binance/BinanceMarketDataSubscriptionParams.proto;api/md/deribit/DeribitMarketDataSubscriptionParams.proto;api/md/ftx/FTXDepthStreamParams.proto;api/md/ftx/FTXMarketDataSubscriptionParams.proto;api/md/glbx/GlbxMarketDataSubscriptionParams.proto;api/md/huobi/HuobiMarketDataSubscriptionParams.proto;api/public/ApiServer.cpp;api/public/ClientChannel.cpp;api/public/QuickbusClientChannel.cpp;api/public/ServerChannel.cpp;api/public/ServiceContext.cpp;api/service_helpers/AdminServiceUtils.cpp;api/service_helpers/AlgoServiceUtils.cpp;api/service_helpers/PositionUtilities.cpp;api/services/AdminService.proto;api/services/AlgoDefinitionService.proto;api/services/AlgoService.proto;api/services/AnalyticsService.proto;api/services/ApiUnitTestService.proto;api/services/AuthService.proto;api/services/BasePositionService.proto;api/services/ErrorReportService.proto;api/services/HelloService.proto;api/services/MBOService.proto;api/services/MarginService.proto;api/services/MarketDataAuxiliaryService.proto;api/services/MarketDataRelay.proto;api/services/MarketDataService.proto;api/services/MoneyFileService.proto;api/services/OrderService.proto;api/services/OutOfProcessURLListener.proto;api/services/PersistenceService.proto;api/services/PositionService.proto;api/services/ShellDiagnosticService.proto;api/services/ShellLatencyService.proto;api/services/SyntheticSecurityService.proto;api/test/ApiTests.cpp;api/test/HelloServiceServer.cpp;config/ConfigCheck.cpp;config/ConfigDownloader.cpp;config/ConfigErrorCollector.cpp;config/ConfigLog.cpp;config/ConfigPropertyEvaluator.cpp;config/ConfigValueQueue.cpp;config/ConfigurationException.cpp;config/ConfigurationState.cpp;config/PreprocessInputStream.cpp;config/config.cpp;config/defaults/MarketData.conf;config/defaults/Ports.conf;config/defaults/SecurityMetadataCollection.conf;config/defaults/Symbols.conf;config/defaults/cme_reference_data.conf;config/defaults/edge.conf;config/defaults/jetfix.conf;config/defaults/keycode.conf;config/defaults/logging.conf;config/defaults/orderman_cfg.conf;config/defaults/orders_fills_cleanup.conf;config/defaults/position_cfg.conf;config/defaults/span.conf;config/defaults/spreader.conf;config/defaults/testkeys.conf;config/defaults/threads.conf;config/defaults/toggles.conf;config/extensions/extensions.proto;config/invalid/invalid.proto;config/protos/algo_types.proto;config/protos/cme_reference_data.proto;config/protos/exchange_sim_cfg.proto;config/protos/jetfix.proto;config/protos/keycode.proto;config/protos/logging.proto;config/protos/market_data.proto;config/protos/mercury_cfg.proto;config/protos/orderman_cfg.proto;config/protos/orders_fills_cleanup.proto;config/protos/perflog_cfg.proto;config/protos/ports.proto;config/protos/position_cfg.proto;config/protos/quickbus_cfg.proto;config/protos/span.proto;config/protos/spartan.proto;config/protos/spreader.proto;config/protos/subprocess.proto;config/protos/symbols.proto;config/protos/system.proto;config/protos/testkeys.proto;config/protos/threads.proto;config/protos/toggles.proto;config/protos/ui.proto;config/tests/HTTPServer.cpp;config/tests/config_ref_tests.cpp;config/tests/config_tests.cpp;config/text_format/text_format.cpp;domain_model/Security.cpp;domain_model/Security.proto;domain_model/SecurityTests.cpp;domain_model/sod.cpp;domain_model/tests/domain_tests.cpp;expression/BooleanGrammar.cpp;expression/ExpressionEvaluator.cpp;expression/ParserExceptions.cpp;expression/Value.cpp;expression/tests/expression_parser_tests.cpp;io/QueryParams.cpp;io/URL.cpp;io/http/CurlHandle.cpp;io/http/HTTPClient.cpp;io/http/HTTPHeaders.cpp;io/http/HTTPRequest.cpp;io/http/HTTPResponse.cpp;io/websocket/WebsocketClient.cpp;io/websocket/WebsocketClientConfiguration.cpp;io/websocket/WebsocketError.cpp;io/websocket/WebsocketMessage.cpp;io/websocket/WebsocketSSLClientInternal.cpp;io/websocket/tests/WebsocketClientTests.cpp;logging/LoggerHub.cpp;logging/Sink.cpp;logging/logging.cpp;logging/tests/logging_tests.cpp;md/CCApiDepthWalker.cpp;md/CCApiListenerAdaptor.cpp;md/CCApiRequestor.cpp;md/Diag/MDPeriodicDumper.cpp;md/Diag/ShmDumper.cpp;md/MBOProcessor.cpp;md/MBOServiceServer.cpp;md/MDClient.cpp;md/MDConsumer.cpp;md/MDConsumerHandle.cpp;md/MDConsumerInterop.cpp;md/MDProducer.cpp;md/MDProducerHandle.cpp;md/MarketDataError.cpp;md/MarketDataGateway.cpLine truncated
+set(sources api/analytics/ExpirationDriver.proto;api/analytics/TimeWeightedInterpolatedDriver.proto;api/apicore.proto;api/apigateway.proto;api/apioptions.proto;api/core/ApiClientInterop.cpp;api/core/ApiServerInterop.cpp;api/domain/Currency.proto;api/domain/SecurityCriteria.proto;api/domain/SecurityCriteriaClause.proto;api/domain/SecurityExpiration.proto;api/domain/SecurityMetadata.proto;api/domain/SecurityMetadataCollection.proto;api/domain/SecurityMetadataItem.proto;api/domain/SecurityType.proto;api/domain/TickDisplayRule.proto;api/domain/TickDisplaySettings.proto;api/domain/TickFactors.proto;api/domain/Venue.proto;api/io/URL.proto;api/md/MarketDataSubscriptionParams.proto;api/md/binance/BinanceDepthStreamParams.proto;api/md/binance/BinanceMarketDataSubscriptionParams.proto;api/md/deribit/DeribitMarketDataSubscriptionParams.proto;api/md/ftx/FTXDepthStreamParams.proto;api/md/ftx/FTXMarketDataSubscriptionParams.proto;api/md/glbx/GlbxMarketDataSubscriptionParams.proto;api/md/huobi/HuobiMarketDataSubscriptionParams.proto;api/public/ApiServer.cpp;api/public/ClientChannel.cpp;api/public/QuickbusClientChannel.cpp;api/public/ServerChannel.cpp;api/public/ServiceContext.cpp;api/service_helpers/AdminServiceUtils.cpp;api/service_helpers/AlgoServiceUtils.cpp;api/service_helpers/PositionUtilities.cpp;api/services/AdminService.proto;api/services/AlgoDefinitionService.proto;api/services/AlgoService.proto;api/services/AnalyticsService.proto;api/services/ApiUnitTestService.proto;api/services/AuthService.proto;api/services/BasePositionService.proto;api/services/ErrorReportService.proto;api/services/HelloService.proto;api/services/MBOService.proto;api/services/MarginService.proto;api/services/MarketDataAuxiliaryService.proto;api/services/MarketDataRelay.proto;api/services/MarketDataService.proto;api/services/MoneyFileService.proto;api/services/OrderService.proto;api/services/OutOfProcessURLListener.proto;api/services/PersistenceService.proto;api/services/PositionService.proto;api/services/ShellDiagnosticService.proto;api/services/ShellLatencyService.proto;api/services/SyntheticSecurityService.proto;api/test/ApiTests.cpp;api/test/HelloServiceServer.cpp;config/ConfigCheck.cpp;config/ConfigDownloader.cpp;config/ConfigErrorCollector.cpp;config/ConfigLog.cpp;config/ConfigPropertyEvaluator.cpp;config/ConfigValueQueue.cpp;config/ConfigurationException.cpp;config/ConfigurationState.cpp;config/PreprocessInputStream.cpp;config/config.cpp;config/defaults/MarketData.conf;config/defaults/Ports.conf;config/defaults/SecurityMetadataCollection.conf;config/defaults/Symbols.conf;config/defaults/cme_reference_data.conf;config/defaults/edge.conf;config/defaults/jetfix.conf;config/defaults/keycode.conf;config/defaults/logging.conf;config/defaults/orderman_cfg.conf;config/defaults/orders_fills_cleanup.conf;config/defaults/position_cfg.conf;config/defaults/span.conf;config/defaults/spreader.conf;config/defaults/testkeys.conf;config/defaults/threads.conf;config/defaults/toggles.conf;config/extensions/extensions.proto;config/invalid/invalid.proto;config/protos/algo_types.proto;config/protos/cme_reference_data.proto;config/protos/exchange_sim_cfg.proto;config/protos/jetfix.proto;config/protos/keycode.proto;config/protos/logging.proto;config/protos/market_data.proto;config/protos/mercury_cfg.proto;config/protos/orderman_cfg.proto;config/protos/orders_fills_cleanup.proto;config/protos/perflog_cfg.proto;config/protos/ports.proto;config/protos/position_cfg.proto;config/protos/quickbus_cfg.proto;config/protos/span.proto;config/protos/spartan.proto;config/protos/spreader.proto;config/protos/subprocess.proto;config/protos/symbols.proto;config/protos/system.proto;config/protos/testkeys.proto;config/protos/threads.proto;config/protos/toggles.proto;config/protos/ui.proto;config/tests/HTTPServer.cpp;config/tests/config_ref_tests.cpp;config/tests/config_tests.cpp;config/text_format/text_format.cpp;domain_model/Security.cpp;domain_model/Security.proto;domain_model/SecurityTests.cpp;domain_model/sod.cpp;domain_model/tests/domain_tests.cpp;expression/BooleanGrammar.cpp;expression/ExpressionEvaluator.cpp;expression/ParserExceptions.cpp;expression/Value.cpp;expression/tests/expression_parser_tests.cpp;io/QueryParams.cpp;io/URL.cpp;io/http/CurlHandle.cpp;io/http/HTTPClient.cpp;io/http/HTTPHeaders.cpp;io/http/HTTPRequest.cpp;io/http/HTTPResponse.cpp;io/websocket/WebsocketClient.cpp;io/websocket/WebsocketClientConfiguration.cpp;io/websocket/WebsocketError.cpp;io/websocket/WebsocketMessage.cpp;io/websocket/WebsocketSSLClientInternal.cpp;io/websocket/tests/WebsocketClientTests.cpp;logging/LoggerHub.cpp;logging/Sink.cpp;logging/logging.cpp;logging/tests/logging_tests.cpp;md/CCApiDepthWalker.cpp;md/CCApiListenerAdaptor.cpp;md/CCApiRequestor.cpp;md/Diag/MDPeriodicDumper.cpp;md/Diag/ShmDumper.cpp;md/MBOProcessor.cpp;md/MBOServiceServer.cpp;md/MDClient.cpp;md/MDConsumer.cpp;md/MDConsumerHandle.cpp;md/MDConsumerInterop.cpp;md/MDProducer.cpp;md/MDProducerHandle.cpp;md/MarketDataLine truncated

src/lib/api/public/StatefulPublisher.h

@@ -4,6 +4,7 @@
 #include <set>
 #include <unordered_set>
 #include <vector>
+#include <map>
 
 namespace bts {
 namespace api {
@@ -55,8 +56,14 @@ class StatefulPublisher {
       }
    }
    void erase(TKey m) {
+      // there is an assumption that we are not dealing with that within a 
+      // transaction, add_or_update and erase is called just once a key
       if (_has_key(m)) {
          _erased.insert(m);
+         if (_keep_state)
+            _state_data.erase(m);
+         else
+            _state.erase(m);
       }
    }
    void flush() {

src/lib/api/services/OrderService.proto

@@ -44,6 +44,7 @@ message PrecomputedTheoPoint {
   double vega = 60;
   double theta = 70;
   double rho = 80;
+  double vol = 90;
 }
 
 message PrecomputedTheos {

src/lib/config/defaults/vol_surface_config.conf

@@ -0,0 +1,22 @@
+VolSurface {
+    value {
+        default_abs_moves: -600
+        default_abs_moves: -300
+        default_abs_moves: -200
+        default_abs_moves: -125
+        default_abs_moves: -100 
+        default_abs_moves: -75
+        default_abs_moves: -50
+        default_abs_moves: -25 
+        default_abs_moves: 0
+        default_abs_moves: 25 
+        default_abs_moves: 50 
+        default_abs_moves: 75 
+        default_abs_moves: 100
+        default_abs_moves: 125
+        default_abs_moves: 200
+        default_abs_moves: 300
+        default_abs_moves: 600
+
+    }
+}

src/lib/config/protos/vol_surface_config.proto

@@ -0,0 +1,6 @@
+syntax = 'proto2';
+package bts.config;
+
+message VolSurface {
+    repeated double default_abs_moves = 1;
+}

src/lib/domain_model/Security.h

@@ -114,6 +114,10 @@ inline bool is_same(Price p1, Price p2) {
    return std::abs(p1.value() - p2.value()) < market_epsilon();
 }
 
+inline bool is_same(double d1, double d2) {
+   return std::abs(d1 - d2) < market_epsilon();
+}
+
 inline bool is_equal_or_better(api::Side side, Price p1, Price p2) {
    return is_same(p1, p2) || is_better(side, p1, p2);
 }

src/lib/orderman/calculations/ITheoCalculator.h

@@ -25,5 +25,9 @@ class ITheoCalculator {
        bts::api::orderman::PrecomputedTheoPoint &result) const = 0;
    // gets the driver security for a security with id sec_id
    virtual std::optional<uint32_t> get_driver_sec(uint32_t sec_id) const = 0;
+   // given a vol for an option and the current driver, returns the theo that
+   // corresponds to that vol.
+   virtual double get_theo_correspoding_to_vol(
+       uint32_t sec_id, double driver, double vol) const = 0;
 };
 } // namespace bts::orderman::analytics

src/lib/orderman/calculations/TheoCalculator.cpp

@@ -50,12 +50,15 @@ void interpolate(
        lower.driver(), lower.theta(), higher.driver(), higher.theta(), driver);
    double rho = lin_interp_exterp(
        lower.driver(), lower.rho(), higher.driver(), higher.rho(), driver);
+   double vol = lin_interp_exterp(
+       lower.driver(), lower.vol(), higher.driver(), higher.vol(), driver);
    result.set_theo(theo);
    result.set_delta(delta);
    result.set_gamma(gamma);
    result.set_vega(vega);
    result.set_theta(theta);
    result.set_rho(rho);
+   result.set_vol(vol);
 }
 
 struct DriverNeighbors {
@@ -213,6 +216,24 @@ std::optional<uint32_t> TheoCalculator::get_driver_sec(uint32_t sec_id) const {
    return me->total_theo_data.get_driver_sec(sec_id);
 }
 
+double TheoCalculator::get_theo_correspoding_to_vol(
+    uint32_t sec_id, double driver, double vol) const {
+   PrecomputedTheoPoint rv;
+   if (!get_theo(sec_id, driver, rv)) {
+      return bts::os::nan();
+   }
+   // use the vega to compute the theo
+   double d_vol = vol - rv.vol();
+   double d_theo = rv.vega() * d_vol;
+   double new_theo = rv.theo() + d_theo;
+   LOG(Gen, Info,
+       "TheoCalculator::get_theo_correspoding_to_vol sec_id: {} driver: {} "
+       "vol: {} rv.theo: {} rv.vol: {} rv.vega: {} "
+       "new_theo: {}",
+       sec_id, driver, vol, rv.theo(), rv.vol(), rv.vega(), new_theo);
+   return new_theo;
+}
+
 void TheoCalculator::set_data(const PrecomputedTheos &theos) {
    LOG(Gen, Info, "TheoCalculator::set_data size: {}", theos.theos_size());
    WriteLock write_lock(me->lock);

src/lib/orderman/calculations/TheoCalculator.h

@@ -23,6 +23,8 @@ class TheoCalculator : public ITheoCalculator {
        double driver,
        bts::api::orderman::PrecomputedTheoPoint &result) const override;
    std::optional<uint32_t> get_driver_sec(uint32_t sec_id) const override;
+   double get_theo_correspoding_to_vol(
+       uint32_t sec_id, double driver, double vol) const override;
    void set_data(const bts::api::orderman::PrecomputedTheos &theos);
 
  private:

src/lib/orderman/orderman_utils/orderman_common.h

@@ -279,5 +279,13 @@ inline Quantity get_update_quantity(const bts::api::orderman::Update &u) {
    return rv;
 }
 
+inline void set_update_quantity(
+    bts::api::orderman::Update &state, Quantity cqty) {
+   uint32_t int_qty = cqty.int_part();
+   double fractional_qty = cqty.fractional_part();
+   state.set_quantity(int_qty);
+   state.mutable_fractional_quantity()->set_fraction(fractional_qty);
+}
+
 } // namespace orderman
 } // namespace bts

src/lib/orderman/orders/algo/algo_orders/RegisterDorman.cpp

@@ -12,6 +12,7 @@
 #include "lib/orderman/orderman_utils/orderman_utils.h"
 #include "lib/orderman/orders/algo/algo_orders/LimitCoverAlgo.h"
 #include "lib/orderman/orders/algo/algo_orders/TaskingEyeAlgo.h"
+#include "lib/orderman/orders/algo/algo_orders/VolAlgo.h"
 #include "lib/orderman/orders/algo/algo_orders/limit_cover/LimitCoverAdapter.h"
 #include "lib/orderman/orders/algo/algo_orders/tests/ExecutionTesterAlgo.h"
 #include "lib/plugin/SparkServices.h"
@@ -299,6 +300,9 @@ extern "C" void init_execution_tester_plugin(bts::SparkServices &srv) {
 extern "C" void init_task_eye_plugin(bts::SparkServices &srv) {
    DynamicOrderDescriptor tasking_eye_algo;
    tasking_eye_algo.set_dynamic_order_type("Tasking Eye");
+   // if this is set to true, it waits for the existance of Signal File
+   *tasking_eye_algo.add_param_desc() = param_bool("Wait For Signal");
+   *tasking_eye_algo.add_param_desc() = param_string("Signal File");
    *tasking_eye_algo.add_param_desc() = param_string("Account");
    *tasking_eye_algo.add_param_desc() = param_double("Edge");
    *tasking_eye_algo.add_param_desc() = param_double("Shot Size");
@@ -325,6 +329,19 @@ extern "C" void init_task_eye_plugin(bts::SparkServices &srv) {
    });
 }
 
+extern "C" void init_vol_algo_plugin(bts::SparkServices &srv) {
+   DynamicOrderDescriptor vol_algo;
+   vol_algo.set_dynamic_order_type("Vol");
+   //*vol_algo.add_param_desc() = param_string("Account Override");
+   *vol_algo.add_param_desc() = param_double("Vol");
+   *vol_algo.add_param_desc() = param_bool("Cross At Entry?");
+   *vol_algo.add_param_desc() = param_bool("Requote?");
+   *vol_algo.add_param_desc() = param_bool("Improve Price?");
+   *vol_algo.add_param_desc() = param_int32("Distance From Top");
+   srv.register_dynamic_algo_order(
+       vol_algo, []() { return std::make_unique<orderman::VolAlgo>(); });
+}
+
 extern "C" void init_spark_plugin(bts::SparkServices &srv) {
    auto &dyn_algo_config = config::get<config::toggles::DynamicAlgos>();
    if (dyn_algo_config.enable_dorman())
@@ -345,6 +362,7 @@ extern "C" void init_spark_plugin(bts::SparkServices &srv) {
       init_execution_tester_plugin(srv);
    // init_sniper_plugin(srv);
    init_task_eye_plugin(srv);
+   init_vol_algo_plugin(srv);
 }
 
 } // namespace dorman

src/lib/orderman/orders/algo/algo_orders/TaskingEyeAlgo.cpp

@@ -23,10 +23,17 @@
 #include "lib/utils/bts_assert.h"
 #include <bits/stdint-uintn.h>
 #include <boost/algorithm/string.hpp>
+#include <chrono>
+#include <elf.h>
+#include <filesystem>
 #include <google/protobuf/timestamp.pb.h>
 #include <google/protobuf/util/time_util.h>
+#include <iostream>
 #include <optional>
+#include <sstream>
 #include <string>
+#include <thread>
+#include <tuple>
 #include <unordered_map>
 
 using namespace bts::api::orderman;
@@ -69,6 +76,11 @@ double eps = 0.0000001;
 bool eq(double d1, double d2) { return std::abs(d1 - d2) < eps; }
 bool greater_then_eq(double d1, double d2) { return eq(d1, d2) || d2 > d1; }
 
+void sleep_log() {
+   return;
+   std::this_thread::sleep_for(std::chrono::seconds(5));
+}
+
 // Returns the number of shots that are within the Shots Window
 size_t n_recent_shots(
     const std::vector<TEAShot> &shots, uint64_t window_nanos) {
@@ -100,7 +112,91 @@ bool in_flight_orders(const std::vector<TEAShot> &shots) {
    return false;
 }
 
-std::set<SecurityKey> parse_securities(const std::string &secs) {
+// returns the securities that match the criteria from the 2nd - 8th arg
+std::set<SecurityKey> match_securities(
+    const std::vector<api::Security const *> &secs,
+    api::Venue venue,
+    const std::string &symbol,
+    uint32_t exp_month,
+    uint32_t exp_day,
+    uint32_t exp_yr,
+    std::tuple<double, double> strike_range,
+    api::PutOrCall cp) {
+   std::set<SecurityKey> rv = {};
+   double strike_min = std::get<0>(strike_range);
+   double strike_max = std::get<1>(strike_range);
+   for (const auto &s : secs) {
+      if (s->exchange() == venue && s->has_option() && s->symbol() == symbol) {
+         const auto &oc = s->option();
+         if (oc.call_put() == cp && oc.strike() >= strike_min &&
+             oc.strike() <= strike_max && oc.expiration_year() == exp_yr &&
+             oc.expiration_month() == exp_month &&
+             oc.expiration_day() == exp_day)
+            rv.insert(SecurityKey(s->id()));
+      }
+   }
+   return rv;
+}
+
+// Parses the sec expressed as text in the secs string.
+// the format is symbol:exp:call/put:strike range, ex
+// BTC:08/14/2023:Call:25000-30000
+std::set<SecurityKey> parse_securities_from_range(
+    const std::string &secs,
+    const std::vector<api::Security const *> &sec_vec) {
+   LOG(OrderMan, Info,
+       "TaskingEyeAlgo::parse_securities_from_range secs: {} sec_vec: {}", secs,
+       sec_vec.size());
+   sleep_log();
+   if (secs.empty())
+      return {};
+   auto parse_exp = [](const std::string &exp_str) {
+      LOG(OrderMan, Info,
+          "TaskingEyeAlgo::parse_securities_from_range parse_exp exp: {}",
+          exp_str);
+      std::vector<std::string> exp_parts;
+      boost::split(exp_parts, exp_str, boost::is_any_of("/"));
+      LOG(OrderMan, Info,
+          "TaskingEyeAlgo::parse_securities_from_range exp_parts: {}",
+          exp_parts.size());
+      uint32_t mm = static_cast<std::uint32_t>(std::stoul(exp_parts[0]));
+      uint32_t dd = static_cast<std::uint32_t>(std::stoul(exp_parts[1]));
+      uint32_t yy = static_cast<std::uint32_t>(std::stoul(exp_parts[2]));
+      LOG(OrderMan, Info,
+          "TaskingEyeAlgo::parse_securities_from_range {} mm: {} dd: {} yy: {}",
+          mm, dd, yy);
+      return std::tuple<uint32_t, uint32_t, uint32_t>(mm, dd, yy);
+   };
+   auto parse_strike_range = [](const std::string &strike_range_str) {
+      std::vector<std::string> strike_parts;
+      boost::split(strike_parts, strike_range_str, boost::is_any_of("-"));
+      double k_min = std::stod(strike_parts[0]);
+      double k_max = std::stod(strike_parts[1]);
+      return std::tuple<double, double>(k_min, k_max);
+   };
+   std::vector<std::string> parts;
+   boost::split(parts, secs, boost::is_any_of(":"));
+   const std::string &symbol = parts[0];
+   const std::string &exp_str = parts[1];
+   const std::string &cp_str = parts[2];
+   const auto &strike_range_str = parts[3];
+   LOG(OrderMan, Info,
+       "TaskingEyeAlgo::parse_securities_from_range symbol: {} exp: {} cp: {} "
+       "stike_range: {}",
+       symbol, exp_str, cp_str, strike_range_str);
+   // debugging
+   sleep_log();
+   auto [mm, dd, yy] = parse_exp(exp_str);
+   auto strike_range = parse_strike_range(strike_range_str);
+   api::PutOrCall cp = (cp_str[0] == 'C' || cp_str[0] == 'c')
+                           ? api::PutOrCall::PUTORCALL_CALL
+                           : api::PutOrCall::PUTORCALL_PUT;
+
+   return match_securities(
+       sec_vec, api::Venue::VENUE_DERIBIT, symbol, mm, dd, yy, strike_range,
+       cp);
+}
+std::set<SecurityKey> parse_securities_from_keys(const std::string &secs) {
    std::set<SecurityKey> rv = {};
    std::vector<std::string> parts;
    boost::split(parts, secs, boost::is_any_of(","));
@@ -113,10 +209,34 @@ std::set<SecurityKey> parse_securities(const std::string &secs) {
    return rv;
 }
 
+std::set<SecurityKey> parse_securities(
+    const std::string &secs,
+    const std::vector<api::Security const *> &sec_vec) {
+   bts_assert(!secs.empty());
+   LOG(OrderMan, Info, "TaskingEyeAlgo::parse_securities sec: {}", secs);
+   sleep_log();
+   std::set<SecurityKey> rv = {};
+   if (secs.find("/") != std::string::npos &&
+       secs.find(":") != std::string::npos) {
+      rv = parse_securities_from_range(secs, sec_vec);
+   } else {
+      rv = parse_securities_from_keys(secs);
+   }
+   std::stringstream ss;
+   for (auto k : rv) {
+      ss << k << " ";
+   }
+   LOG(OrderMan, Info, "TaskingEyeAlgo::parse_securities sec: {} rv: {}", secs,
+       ss.str());
+   return rv;
+}
+
 struct TaskingEyeAlgo ::PImpl {
    IAlgoServices *service_ptr = nullptr;
    analytics::ITheoCalculator const *theo_calc = nullptr;
    TEAState state = TEAState::ACTIVE;
+   bool wait_for_signal = false;
+   std::string signal_file = "";
    double edge = 100000;
    double shot_size = 0;
    double target_qty = 0;
@@ -150,31 +270,62 @@ TaskingEyeAlgo::~TaskingEyeAlgo() {}
 OrderOpResult TaskingEyeAlgo::do_before_launch(
     IBeforeLaunchAlgoServices &serv) {
    auto is_data_valid = [&]() {
-      return greater_then_eq(0.1, me->shot_size) &&
-             greater_then_eq(1, me->target_qty) &&
-             greater_then_eq(1, me->max_shots_per_window) &&
-             greater_then_eq(1000000000, me->window_size_nanos) &&
-             me->side != api::SIDE_UNDEFINED;
+      bool rv = greater_then_eq(0.1, me->shot_size) &&
+                greater_then_eq(1, me->target_qty) &&
+                greater_then_eq(1, me->max_shots_per_window) &&
+                greater_then_eq(1000000000, me->window_size_nanos) &&
+                me->side != api::SIDE_UNDEFINED;
+      bool signal_correct = !me->wait_for_signal || !me->signal_file.empty();
+      rv = rv && signal_correct;
+      return rv;
    };
    LOG(OrderMan, Info,
        "TaskingEyeAlgo::do_before_launch order_key: {} security: {} "
        "definition: {}",
        serv.order_key().value(), utils::pb::to_string(serv.security()),
        utils::pb::to_string(serv.definition()));
+   sleep_log();
    auto &def = serv.definition();
    if (def.has_custom_def() && def.custom_def().has_dynamic()) {
       const auto &d_params = def.custom_def().dynamic().order_params();
+      auto wait_for_signal = get_if<bool>(d_params, "Wait For Signal");
+      me->wait_for_signal =
+          wait_for_signal.has_value() ? wait_for_signal.value() : false;
+      auto signal_file = get_if<std::string>(d_params, "Signal File");
+      me->signal_file = signal_file.has_value() ? signal_file.value() : "";
       me->edge = get_if<double>(d_params, "Edge").value();
+      LOG(OrderMan, Info, "TaskingEyeAlgo::do_before_launch edge: {}",
+          me->edge);
+      sleep_log();
       me->shot_size = get_if<double>(d_params, "Shot Size").value();
+      LOG(OrderMan, Info, "TaskingEyeAlgo::do_before_launch shot_size: {}",
+          me->shot_size);
+      sleep_log();
       me->target_qty = get_if<double>(d_params, "Target Quantity").value();
+      LOG(OrderMan, Info, "TaskingEyeAlgo::do_before_launch target_qty: {}",
+          me->target_qty);
+      sleep_log();
+      // TODO, make sure the side if right
       me->side = def.side();
       me->max_shots_per_window =
           get_if<uint32_t>(d_params, "Max Shots Per Window").value();
+      LOG(OrderMan, Info,
+          "TaskingEyeAlgo::do_before_launch max_shots_per_window: {}",
+          me->max_shots_per_window);
+      sleep_log();
       uint32_t window_size_secs =
           get_if<uint32_t>(d_params, "Window Size (secs)").value();
+      LOG(OrderMan, Info,
+          "TaskingEyeAlgo::do_before_launch window_size_secs: {}",
+          window_size_secs);
+      sleep_log();
       me->window_size_nanos = seconds_to_nanos(window_size_secs);
       auto sec_str = get_if<std::string>(d_params, "Securities").value();
-      auto parsed_secs = parse_securities(sec_str);
+      LOG(OrderMan, Info, "TaskingEyeAlgo::do_before_launch Securities: {}",
+          sec_str);
+      sleep_log();
+      auto parsed_secs = parse_securities(sec_str, serv.known_securities());
+      std::stringstream ss;
       std::vector<SecurityKey> final_secs;
       if (parsed_secs.empty()) {
          final_secs.push_back(SecurityKey{serv.security().id()});
@@ -186,18 +337,21 @@ OrderOpResult TaskingEyeAlgo::do_before_launch(
             auto ct = get_tea_child_template(serv, sec_key, me->side);
             me->sec_map.insert({sec_key, {ct}});
             LOG(OrderMan, Info,
-                "TaskingEyeAlgo::do_before_launch order_key: {} successfully "
+                "TaskingEyeAlgo::do_before_launch order_key: {} "
+                "successfully "
                 "created ChildTemplate for sec_key: {}",
                 serv.order_key(), sec_key);
          }
          LOG(OrderMan, Info,
-             "TaskingEyeAlgo::do_before_launch algo constructed order_key: {} "
-             "edge: {} shot_size: {} is_buy: {} target_qty: {} "
-             "max_shots_per_window: {} window_size_nanos: {}  n_secs: {}",
+             "TaskingEyeAlgo::do_before_launch algo constructed order_key: "
+             "{} "
+             "edge: {} shot_size: {} side: {} target_qty: {} "
+             "max_shots_per_window: {} window_size_nanos: {}  n_secs: {} "
+             "wait_for_signal: {} signal_file: {}",
              serv.order_key(), me->edge, me->shot_size,
-             me->side == api::SIDE_BUY, me->target_qty,
+             bts::api::Side_Name(me->side), me->target_qty,
              me->max_shots_per_window, me->window_size_nanos,
-             me->sec_map.size());
+             me->sec_map.size(), me->wait_for_signal, me->signal_file);
          return {};
       }
    }
@@ -213,7 +367,8 @@ void TaskingEyeAlgo::do_launch(IAlgoServices &serv) {
    for (const auto &kv : me->sec_map) {
       auto sec_key = kv.first;
       LOG(OrderMan, Info,
-          "TaskingEyeAlgo::do_before_launch subscribing book for sec_key: {}",
+          "TaskingEyeAlgo::do_before_launch subscribing book for sec_key: "
+          "{}",
           sec_key);
       me->service_ptr->subscribe_book(sec_key);
    }
@@ -315,9 +470,9 @@ void TaskingEyeAlgo::on_theos(
       if (!sec_data.driver_sec_key.has_value() ||
           sec_data.driver_sec_key->value() != driver.value()) {
          LOG(OrderMan, Info,
-             "TaskingEyeAlgo::on_theos order_key: {} subscribing to driver "
-             "sec_key: {}",
-             me->service_ptr->order_key(), driver.value());
+             "TaskingEyeAlgo::on_theos order_key: {} subscribing to driver for "
+             "sec_key: {} driver_key: {}",
+             me->service_ptr->order_key(), sec_key, driver.value());
          sec_data.driver_sec_key = SecurityKey(driver.value());
          sub_to_driver(SecurityKey(driver.value()));
       }
@@ -331,8 +486,14 @@ void TaskingEyeAlgo::on_position(PositionUpdate const &pu) {
 }
 
 void TaskingEyeAlgo::_evaluate_and_shoot() {
-   if (me->state != TEAState::ACTIVE)
+   if (me->state != TEAState::ACTIVE) {
+      LOG(OrderMan, Warn,
+          "TaskingEyeAlgo::_evaluate_and_shoot order_key: {} state is not "
+          "active state: {} ",
+          me->service_ptr->order_key(), me->state);
       return;
+   }
+
    auto max_shots_per_win_reached = [&]() {
       return n_recent_shots(me->shots, me->window_size_nanos) >=
              me->max_shots_per_window;
@@ -343,12 +504,36 @@ void TaskingEyeAlgo::_evaluate_and_shoot() {
             return true;
       return false;
    };
+   auto signal_condition_met = [&]() {
+      if (me->wait_for_signal) {
+         return std::filesystem::exists(me->signal_file);
+      } else
+         return true;
+   };
+   if (!signal_condition_met()) {
+      // Log ?
+      LOG(OrderMan, Warn,
+          "TaskingEyeAlgo::_evaluate_and_shoot order_key: {} Waiting for "
+          "Signal",
+          me->service_ptr->order_key());
+      return;
+   }
    for (auto &kv : me->sec_map) {
       auto sec_key = kv.first;
-      if (max_shots_per_win_reached())
+      if (max_shots_per_win_reached()) {
+         LOG(OrderMan, Warn,
+             "TaskingEyeAlgo::_evaluate_and_shoot order_key: {} "
+             "max_shots_per_win_reached",
+             me->service_ptr->order_key());
          return;
-      if (sec_has_active_order(sec_key))
+      }
+      if (sec_has_active_order(sec_key)) {
+         LOG(OrderMan, Warn,
+             "TaskingEyeAlgo::_evaluate_and_shoot sec_key: {} has working "
+             "order, skipping",
+             sec_key.value());
          continue;
+      }
       Price price;
       Quantity qty;
       if (_in_shooting_zone(sec_key, price, qty)) {
@@ -357,7 +542,8 @@ void TaskingEyeAlgo::_evaluate_and_shoot() {
          auto ch_ord_key =
              me->service_ptr->launch_child(*sec_data.order, price, qty);
          LOG(OrderMan, Info,
-             "TakingEyeAlgo::_evaluate_and_shoot ord_key: {} child_ord_key: {} "
+             "TakingEyeAlgo::_evaluate_and_shoot ord_key: {} child_ord_key: "
+             "{} "
              "price: {} qty: {}",
              me->service_ptr->order_key(), ch_ord_key, price, qty.full_value());
          me->shots.push_back(TEAShot{sec_key, ch_ord_key});
@@ -375,8 +561,13 @@ bool TaskingEyeAlgo::_in_shooting_zone(
    TEASecData const &sec_data = it->second;
    const md::PriceSize &driver_bid = sec_data.driver_book.bid().at(0);
    const md::PriceSize &driver_ask = sec_data.driver_book.ask().at(0);
-   if (eq(driver_bid.size(), 0) || eq(driver_ask.size(), 0))
+   if (eq(driver_bid.size(), 0) || eq(driver_ask.size(), 0)) {
+      LOG(OrderMan, Warn,
+          "TaskingEyeAlgo::_in_shooting_zone no driver for sec_key: {} "
+          "driver_bid_qty: {} driver_ask_qty: {}",
+          sec_key, driver_bid.size(), driver_ask.size());
       return false;
+   }
    // finding weighted mean (TODO change it to mark price later)
    double driver = (driver_bid.price() * driver_bid.size() +
                     driver_ask.price() * driver_ask.size()) /
@@ -386,31 +577,55 @@ bool TaskingEyeAlgo::_in_shooting_zone(
    // TODO normalize theo
    double theo = ptp.theo();
    theo = theo / driver;
-   if (me->side == api::SIDE_BUY) {
+   // auto analyze_buy =
+   auto analyze_buy = [&](const md::PriceSize &md_ask, double theo,
+                          size_t level, Price &price, Quantity &qty) -> bool {
       double ee_buy_pr = theo - me->edge;
-      const md::PriceSize &md_ask = sec_data.sec_book.ask().at(0);
-      LOG(OrderMan, Diag,
-          "TaskingEyeAlgo::_in_shooting_zone sec_key: {} driver: {} theo: {} "
-          "ee_price: {} md: {}/{}",
-          sec_key, driver, theo, ee_buy_pr, md_ask.price(), md_ask.size());
+      bool rv = false;
       if (greater_then_eq(1, md_ask.size()) &&
           greater_then_eq(md_ask.price(), ee_buy_pr)) {
          price = Price(md_ask.price());
-         qty = std::min(static_cast<double>(md_ask.size()), me->shot_size);
-         return true;
+         qty = me->shot_size;
+         rv = true;
       }
-   } else {
-      double ee_sell_pr = theo + me->edge;
-      const md::PriceSize &md_bid = sec_data.sec_book.bid().at(0);
       LOG(OrderMan, Diag,
-          "TaskingEyeAlgo::_in_shooting_zone sec_key: {} driver: {} theo: {} "
-          "ee_price: {} md: {}/{}",
-          sec_key, driver, theo, ee_sell_pr, md_bid.price(), md_bid.size());
+          "TaskingEyeAlgo::_in_shooting_zone::analyze_buy sec_key: {}  "
+          "theo: {} level: {}"
+          "ee_buy_pr: {} rv: {}",
+          sec_key, level, theo, ee_buy_pr, rv);
+      return rv;
+   };
+   auto analyze_sell = [&](const md::PriceSize &md_bid, double theo,
+                           size_t level, Price &price, Quantity &qty) -> bool {
+      double ee_sell_pr = theo + me->edge;
+      bool rv = false;
       if (greater_then_eq(1, md_bid.size()) &&
           greater_then_eq(ee_sell_pr, md_bid.price())) {
          price = Price(md_bid.price());
-         qty = std::min(static_cast<double>(md_bid.size()), me->shot_size);
-         return true;
+         qty = me->shot_size;
+         rv = true;
+      }
+      LOG(OrderMan, Diag,
+          "TaskingEyeAlgo::_in_shooting_zone::analyze_sell sec_key: {} theo:"
+          "{} level: {} ee_sell_pr: {} rv: {}",
+          sec_key, theo, level, ee_sell_pr, rv);
+      return rv;
+   };
+   if (me->side == api::SIDE_BUY) {
+      const auto &ask_book = sec_data.sec_book.ask();
+      for (int i = ask_book.size() - 1; i >= 0; i--) {
+         const auto &md_ask = ask_book.at(i);
+         if (analyze_buy(md_ask, theo, i, price, qty)) {
+            return true;
+         }
+      }
+   } else {
+      const auto &bid_book = sec_data.sec_book.bid();
+      for (int i = bid_book.size() - 1; i >= 0; i--) {
+         const md::PriceSize &md_bid = bid_book.at(i);
+         if (analyze_sell(md_bid, theo, i, price, qty)) {
+            return true;
+         }
       }
    }
    return false;

src/lib/orderman/orders/algo/algo_orders/VolAlgo.cpp

@@ -0,0 +1,354 @@
+#include "VolAlgo.h"
+#include "lib/api/services/OrderService.pb.h"
+#include "lib/domain_model/Security.h"
+#include "lib/domain_model/Security.pb.h"
+#include "lib/domain_model/SecurityKey.h"
+#include "lib/domain_model/common.h"
+#include "lib/logging/logging.h"
+#include "lib/md/PriceSize.h"
+#include "lib/orderman/ChildOrder.h"
+#include "lib/orderman/OrderBuilder.h"
+#include "lib/orderman/limiter/SecurityLimitNode.h"
+#include "lib/orderman/order_factory/IOrderFactory.h"
+#include "lib/orderman/orderman_utils/Book.h"
+#include "lib/orderman/orderman_utils/orderman_common.h"
+#include "lib/orderman/orderman_utils/orderman_utils.h"
+#include "lib/orderman/orders/OrderOpResult.h"
+#include "lib/orderman/orders/algo/ChildTemplate.h"
+#include "lib/orderman/orders/algo/IAlgoOrder.h"
+#include "lib/orderman/orders/algo/IAlgoServices.h"
+#include "lib/orderman/orders/algo/IBeforeLaunchAlgoServices.h"
+#include "lib/os/os.h"
+#include "lib/utils/PBUtils.h"
+#include "lib/utils/bts_assert.h"
+#include <bits/stdint-uintn.h>
+#include <google/protobuf/util/time_util.h>
+#include <optional>
+#include <string>
+
+using namespace bts::api::orderman;
+
+namespace bts::orderman {
+
+ChildTemplate *get_vol_algo_child_template(
+    IBeforeLaunchAlgoServices &serv,
+    SecurityKey key,
+    api::Side side,
+    std::optional<std::string> account_override) {
+   OrderDefinition order_def = serv.definition();
+   order_def.set_security_key(key.value());
+   order_def.set_side(side);
+   order_def.set_time_in_force(order_def.time_in_force());
+   if (account_override.has_value()) {
+      // no need for it now
+   }
+   order_def.mutable_custom_def()->mutable_limit()->MergeFrom(
+       api::orderman::LimitOrderDefinition());
+   return &(serv.make_child_template(order_def));
+}
+
+struct VolAlgoParams {
+   std::optional<std::string> account_override;
+   double vol_perc;
+   bool cross_at_entry = false;
+   bool requote = false;
+   bool improve_price = false;
+   int distance_from_top = 2;
+};
+
+std::optional<VolAlgoParams> parse_parameters(
+    const DynamicOrderDefinition &def) {
+   const auto &d_params = def.order_params();
+   VolAlgoParams rv = {};
+   // vol
+   auto vol_perc = get_if<double>(d_params, "Vol");
+   if (!vol_perc.has_value())
+      return std::nullopt;
+   rv.vol_perc = vol_perc.value();
+   // cross at entry
+   auto cross_at_entry = get_if<bool>(d_params, "Cross At Entry?");
+   rv.cross_at_entry =
+       cross_at_entry.has_value() ? cross_at_entry.value() : false;
+   // requote
+   auto requote = get_if<bool>(d_params, "Requote?");
+   rv.requote = requote.has_value() ? requote.value() : false;
+   // improve price
+   auto improve_price = get_if<bool>(d_params, "Improve Price?");
+   rv.improve_price = improve_price.has_value() ? improve_price.value() : false;
+   // Distance from Top
+   auto distance_from_top = get_if<int>(d_params, "Distance From Top");
+   if (!distance_from_top.has_value())
+      return std::nullopt;
+   rv.distance_from_top = distance_from_top.value();
+   return rv;
+}
+
+struct VolAlgo ::PImpl {
+   IAlgoServices *service_ptr = nullptr;
+   analytics::ITheoCalculator const *theo_calc = nullptr;
+   OrderKey order_key;
+   uint32_t sec_key;
+   api::Side side;
+   std::optional<std::string> account_override;
+   double qty = 0;
+   VolAlgoParams params = {};
+   uint32_t driver_sec_key = 0;
+   bool subbed_to_driver = false;
+   std::optional<Book> driver_book = std::nullopt;
+   std::optional<Book> sec_book = std::nullopt;
+   // out
+   ChildTemplate *child_order = nullptr;
+   std::optional<OrderKey> child_order_key = std::nullopt;
+   Price current_price = Price{os::nan()};
+   bool inited() const {
+      // once we have receved both books we are ready to go
+      return driver_book.has_value() && sec_book.has_value();
+   }
+   bool order_fired() const { return child_order_key.has_value(); }
+};
+
+VolAlgo::VolAlgo() : me(new PImpl()) { LOG(OrderMan, Info, "VolAlgo created"); }
+
+VolAlgo::~VolAlgo() {}
+
+OrderOpResult VolAlgo::do_before_launch(IBeforeLaunchAlgoServices &serv) {
+   LOG(OrderMan, Info,
+       "VolAlgo::do_before_launch order_key: {} security: {} "
+       "definition: {}",
+       serv.order_key().value(), utils::pb::to_string(serv.security()),
+       utils::pb::to_string(serv.definition()));
+   auto &def = serv.definition();
+   me->order_key = serv.order_key();
+   me->sec_key = def.security_key();
+   me->side = def.side();
+   if (def.has_custom_def() && def.custom_def().has_dynamic()) {
+      auto va_params = parse_parameters(def.custom_def().dynamic());
+      if (!va_params.has_value()) {
+         return {
+             bts::api::orderman::OrderError::ORDER_ERROR_INVALID_PARAMETER,
+             std::string("Parameters are not well formed")};
+      }
+      me->params = va_params.value();
+      me->child_order = get_vol_algo_child_template(
+          serv, SecurityKey(me->sec_key), me->side,
+          me->params.account_override);
+      LOG(OrderMan, Info,
+          "VolAlgo::do_before_launch order_key: {} sec_key: {} side: {} vol: "
+          "{} cross_at_entry: {} re_quote: {} improve_price: {} "
+          "distance_from_top: {}",
+          me->order_key, me->sec_key, bts::api::Side_Name(me->side),
+          me->params.vol_perc, me->params.cross_at_entry, me->params.requote,
+          me->params.improve_price, me->params.distance_from_top);
+      return {};
+   }
+   return {
+       bts::api::orderman::OrderError::ORDER_ERROR_INVALID_PARAMETER,
+       std::string("Parameters are not well formed")};
+}
+
+void VolAlgo::do_launch(IAlgoServices &serv) {
+   LOG(OrderMan, Info, "VolAlgo::do_launch order_key: {} qty: {}",
+       serv.order_key().value(), serv.quantity());
+   me->service_ptr = &serv;
+   me->qty = serv.quantity().full_value();
+   me->service_ptr->subscribe_theos();
+   me->service_ptr->subscribe_timer();
+   me->service_ptr->subscribe_book(SecurityKey(me->sec_key));
+}
+
+void VolAlgo::do_launch_attached(IAlgoServices &serv, OrderKey, ChildState) {
+   me->service_ptr = &serv;
+   _terminate_algo(true, "Vol Algo can not be attached to working orders");
+}
+
+void VolAlgo::on_book(SecurityKey sec_key, Book const &book) {
+   LOG(OrderMan, Diag, "VolAlgo::on_book order_key: {} sec_key: {}",
+       me->order_key, sec_key);
+   if (sec_key.value() == me->sec_key) {
+      me->sec_book = book;
+      _analyze_and_act();
+   } else if (sec_key.value() == me->driver_sec_key) {
+      me->driver_book = book;
+      _analyze_and_act();
+   }
+}
+void VolAlgo::do_update(api::orderman::Update const &u) {
+   LOG(OrderMan, Info,
+       "VolAlgo::do_update order_key: {} sec_key: {} "
+       "update: {}",
+       me->order_key, me->sec_key, utils::pb::to_string(u));
+   const auto &p = u.custom().dynamic();
+   auto params = parse_parameters(p);
+   if (!params.has_value()) {
+      LOG(Gen, Error,
+          "VolAlgo::do_update order_key: {} parameters are not well formed",
+          me->order_key);
+      return;
+   }
+   me->qty = u.quantity();
+   me->params = params.value();
+   _analyze_and_act(true);
+   // price changes are ignored the price is the vol
+}
+
+void VolAlgo::on_child_change(OrderKey, ChildState const &state) {
+   // price / qty is reflected from the child
+   me->service_ptr->update_fill_state(state.total_fills, state.last_price);
+   switch (state.status) {
+   case ChildStatus::WORKING:
+      break;
+   case ChildStatus::COMPLETED:
+      me->service_ptr->complete();
+      break;
+   case ChildStatus::CANCELED:
+      me->service_ptr->cancel("Child Order Cancelled");
+      break;
+   }
+}
+
+void VolAlgo::on_status(SecurityKey, co2::TradingStatus) {}
+void VolAlgo::on_timer() {}
+
+void VolAlgo::on_theos(analytics::ITheoCalculator const *theo_calculator) {
+   LOG(OrderMan, Info, "VolAlgo::on_theos order_key: {}", me->order_key);
+   me->theo_calc = theo_calculator;
+   if (me->subbed_to_driver) {
+      _analyze_and_act();
+      return;
+   }
+   auto driver_sec = me->theo_calc->get_driver_sec(me->sec_key);
+   if (driver_sec.has_value()) {
+      LOG(OrderMan, Info,
+          "VolAlgo::on_theos order_key: {} subscribing to driver sec_key: {}",
+          me->order_key, driver_sec.value());
+      me->subbed_to_driver = true;
+      me->driver_sec_key = driver_sec.value();
+      me->service_ptr->subscribe_book(SecurityKey(me->driver_sec_key));
+   } else {
+      LOG(OrderMan, Warn,
+          "VolAlgo::on_theos order_key: {} could not find driver",
+          me->order_key);
+   }
+}
+void VolAlgo::on_position(PositionUpdate const &) {}
+
+// this is where all the action happens, both new order (just once) as well as
+// order updates
+void VolAlgo::_analyze_and_act(bool force_reprice) {
+   // check for period (to stop frequent updates)
+   auto get_driver = [&]() {
+      const md::PriceSize &driver_bid = me->driver_book->bid().at(0);
+      const md::PriceSize &driver_ask = me->driver_book->ask().at(0);
+      if (bts::securities::is_same(driver_bid.size(), 0) ||
+          bts::securities::is_same(driver_ask.size(), 0))
+         return os::nan();
+      // finding weighted mean (TODO change it to mark price later)
+      double driver = (driver_bid.price() * driver_bid.size() +
+                       driver_ask.price() * driver_ask.size()) /
+                      (driver_bid.size() + driver_ask.size());
+      return driver;
+   };
+   auto crosses_opp_market = [&](Price order_price) {
+      if (me->side == api::SIDE_BUY) {
+         double top = me->sec_book->ask().at(0).price();
+         return securities::gr_than_eq(order_price.value(), top);
+      } else {
+         double top = me->sec_book->bid().at(0).price();
+         return securities::gr_than_eq(top, order_price.value());
+      }
+   };
+   if (me->order_fired()) {
+      // repricing if parameters (and arg) allows
+      if (!me->params.requote && !force_reprice)
+         return;
+      // working order
+      double driver = get_driver();
+      if (os::is_nan(driver)) {
+         // keep working it
+         return;
+      }
+      double theo_unnorm = me->theo_calc->get_theo_correspoding_to_vol(
+          me->sec_key, driver, me->params.vol_perc / 100.0);
+      // TODO: normalize
+      double theo = theo_unnorm / driver;
+      Price new_price = _get_order_price(theo);
+      LOG(OrderMan, Info,
+          "VolAlgo::_analyze_and_act (calc updating) order_key: {} "
+          "theo_unnorm: {} driver: {} theo: {} order_price: {}",
+          me->order_key, theo_unnorm, driver, theo, new_price);
+      if (securities::is_same(me->current_price, new_price)) {
+         return;
+      }
+      bool improving =
+          securities::is_better(me->side, new_price, me->current_price);
+      if (!improving || me->params.improve_price || force_reprice) {
+         bts::api::orderman::Update update;
+         update.set_price(new_price.value());
+         set_update_quantity(update, Quantity::create_new(me->qty));
+         me->current_price = new_price;
+         LOG(Gen, Info,
+             "VolAlgo::_analyze_and_act updating order order_key: {} price: {} "
+             "qty: {}",
+             me->order_key, me->current_price, me->qty);
+         me->service_ptr->update_child(me->child_order_key.value(), update);
+         return;
+      }
+   } else {
+      // new order path
+      if (me->inited()) {
+         // time to fire
+         double driver = get_driver();
+         if (os::is_nan(driver)) {
+            return;
+         }
+         // entering order
+         double theo_unnorm = me->theo_calc->get_theo_correspoding_to_vol(
+             me->sec_key, driver, me->params.vol_perc / 100.0);
+         double theo = theo_unnorm / driver;
+         Price order_price = _get_order_price(theo);
+         LOG(OrderMan, Info,
+             "VolAlgo::_analyze_and_act (calc new) order_key: {} theo_unnorm: "
+             "{} driver: {} theo: {} order_price: {}",
+             me->order_key, theo_unnorm, driver, theo, order_price);
+         if (!me->params.cross_at_entry && crosses_opp_market(order_price))
+            return;
+         me->current_price = order_price;
+         LOG(Gen, Info,
+             "VolAlgo::_analyze_and_act placing order order_key: {} price: {} "
+             "qty: {}",
+             me->order_key, me->current_price, me->qty);
+         me->child_order_key = me->service_ptr->launch_child(
+             *me->child_order, me->current_price,
+             Quantity::create_new(me->qty));
+      }
+   }
+}
+
+void VolAlgo::_terminate_algo(bool is_error, const std::string &err_msg) {
+   // TODO unsub ?
+   if (is_error) {
+      me->service_ptr->terminate_error(err_msg);
+   } else {
+      me->service_ptr->complete();
+   }
+}
+
+Price VolAlgo::_get_order_price(double theo) const {
+   // BTC Opts (p <=  0.005) : price = 0.0001 qty = 0.1
+   // BTC Opts (p > 0.005)   : price = 0.0005 qty = 0.1
+   // -------------------------------------------------
+   // ETH Opts (p <=  0.005) : price = 0.0001 qty = 1
+   // ETH Opts (p > 0.005)   : price = 0.0005 qty = 1
+   // TODO: get the above from the security itself
+   double rv = 0;
+   if (theo > 0.005) {
+      rv = securities::raw_round_to_quote_price(theo, 0.0005, me->side);
+   } else {
+      rv = securities::raw_round_to_quote_price(theo, 0.0001, me->side);
+   }
+   LOG(Gen, Info,
+       "VolAlgo::_get_order_price order_key: {} theo: {} order_price: {}",
+       me->order_key, theo, rv);
+   return Price{rv};
+}
+} // namespace bts::orderman

src/lib/orderman/orders/algo/algo_orders/VolAlgo.h

@@ -0,0 +1,38 @@
+#pragma once
+
+#include "lib/domain_model/SecurityKey.h"
+#include "lib/domain_model/common.h"
+#include "lib/orderman/calculations/ITheoCalculator.h"
+#include "lib/orderman/orders/algo/IAlgoOrder.h"
+#include <openssl/ossl_typ.h>
+
+namespace bts::orderman {
+
+// This is a test algo, not to be used in prod, that sends an order and modifies
+// it by moving the price up and down at a given period
+class VolAlgo : public IAlgoOrder {
+ public:
+   VolAlgo();
+   ~VolAlgo();
+   OrderOpResult do_before_launch(IBeforeLaunchAlgoServices &) override;
+   void on_book(SecurityKey, Book const &) override;
+   void do_launch(IAlgoServices &) override;
+   void do_launch_attached(IAlgoServices &, OrderKey, ChildState) override;
+   void do_update(api::orderman::Update const &) override;
+   void on_child_change(OrderKey, ChildState const &) override;
+   void on_status(SecurityKey, co2::TradingStatus) override;
+   void on_timer() override;
+   void on_theos(analytics::ITheoCalculator const *theo_calculator) override;
+   void on_position(PositionUpdate const &) override;
+
+ private:
+   struct PImpl;
+   std::unique_ptr<PImpl> me;
+
+ private:
+   void _analyze_and_act(bool force_reprice = false);
+   void _terminate_algo(bool is_error, const std::string &err_msg);
+   Price _get_order_price(double theo) const;
+};
+
+} // namespace bts::orderman

src/server/sparkd/analytics/AnalyticsServiceServer.cpp

@@ -8,6 +8,7 @@
 #include "lib/persistence/SqliteStore.h"
 #include "lib/utils/PBUtils.h"
 #include "lib/utils/Puid.h"
+#include "lib/utils/TimeStamp.h"
 #include "lib/utils/bts_assert.h"
 #include "server/sparkd/algo/AlgoDefinitionServiceServer.h"
 #include <boost/algorithm/string.hpp>
@@ -90,6 +91,14 @@ struct AnalyticsServiceServer::PImpl {
    const std::string vs_usage_ns = "volsurface_usage";
    const std::string reports_ns = "reports";
    const std::string archived_theos_ns = "archived_theos";
+   // this is the total number of vs data that will be stored in the db,
+   // beyond that, old values will be recycled
+   const size_t vs_history_sz = 500;
+   // anologous to above for archived theos (however unlike the above this is
+   // recycled periodically, and is not published out)
+   const size_t archived_theos_history_sz = 5000;
+   const size_t n_theos_archives_before_cleanup = 100;
+   size_t n_theos_archives = 0;
    std::unique_ptr<persistence::IStore> vs_store;
    std::unique_ptr<persistence::IStore> exp_params_store;
    std::unique_ptr<persistence::IStore> vs_usage_store;
@@ -126,24 +135,36 @@ AnalyticsServiceServer::AnalyticsServiceServer(
     : AnalyticsServiceBase(context),
       me(new PImpl(
           context,
-          make_volsurfaces_stateful_publisher(),
+          bts::api::StatefulPublisher<VolSurfaceEntry, uint64_t>(
+              make_volsurfaces_publisher(),
+              [](const VolSurfaceEntry &v) { return v.save_time(); },
+              true),
+          // make_volsurfaces_stateful_publisher(),
           make_exp_parameters_stateful_publisher(),
           make_volsurface_usage_stateful_publisher(),
           make_reports_stateful_publisher())) {
    // publishing all vol surfaces
    LOG(Gen, Info, "Starting AnalyticsServiceServer");
    _clean_up_vss();
+   _clean_up_archived_theos();
    auto vs_table = me->vs_store->retrieve_all();
    for (auto const &ns_vals : vs_table) {
       const std::string &vs_name = ns_vals.first;
       const std::map<std::string, std::string> &key_vals = ns_vals.second;
-      LOG(OrderMan, Info, "vs:{} entries:{}", vs_name, key_vals.size());
+      LOG(OrderMan, Info, "vs: {} entries: {}", vs_name, key_vals.size());
       for (const auto &kvp : key_vals) {
          const std::string &key = kvp.first;
          const std::string &val = kvp.second;
-         LOG(OrderMan, Diag, "vs:{} key:{} val:{}", vs_name, key, val);
+         LOG(OrderMan, Info,
+             "AnalyticsServiceServer::cons loaded from db vs: {} key: {}",
+             vs_name, key);
          VolSurfaceEntry vs_entry;
          utils::pb::from_string(val, &vs_entry);
+         LOG(Gen, Diag,
+             "AnalyticsServiceServer::cons loading to vs_publisher ns: {} "
+             "key: {} vs_name: {} vs_id: {} vs.save_time: {}",
+             vs_name, key, vs_entry.vs_name(), vs_entry.vs_id(),
+             vs_entry.save_time());
          me->vs_publisher.add_or_update(vs_entry);
       }
    }
@@ -213,6 +234,36 @@ void AnalyticsServiceServer::get_new_volsurface_id(
 void AnalyticsServiceServer::save_volsurface(
     const bts::api::analytics::VolSurfaceEntry &vs_entry_arg,
     const api::Response<::bts::api::ResultMessage> &response) {
+   auto t0 = utils::TimeStamp::now().to_uint64();
+   auto remove_oldest = [&](const std::string &vs_ns) {
+      // return;
+      int vs_id = vs_entry_arg.vs_id();
+      // find all the keys belonging to this vs
+      std::vector<uint64_t> vs_keys = {};
+      for (const auto &kv : me->vs_publisher.state_data()) {
+         if (kv.second.vs_id() == vs_id) {
+            vs_keys.push_back(kv.first);
+         }
+      }
+      LOG(Gen, Info,
+          "AnalyticsServiceServer::save_volsurface vs_id: {} n_entries: {}",
+          vs_entry_arg.vs_id(), vs_keys.size());
+      if (vs_keys.size() <= me->vs_history_sz)
+         return;
+      // since we are already truncating it at the the start, we'll just
+      // find the earliest key and delete it         
+      uint64_t key_to_delete = std::numeric_limits<uint64_t>::max();
+      for (uint64_t k : vs_keys) {
+         if (k < key_to_delete) {
+            key_to_delete = k;
+         }
+      }
+      LOG(Gen, Info,
+          "AnalyticsServiceServer::save_volsurface removing row ns: {} key: {}",
+          vs_ns, key_to_delete);
+      me->vs_store->remove(vs_ns, {std::to_string(key_to_delete)});
+      me->vs_publisher.erase(key_to_delete);
+   };
    uint64_t ms_u = get_epoch();
    VolSurfaceEntry vs_entry = vs_entry_arg;
    vs_entry.set_save_time(ms_u);
@@ -226,9 +277,15 @@ void AnalyticsServiceServer::save_volsurface(
    std::string value = utils::pb::to_string(vs_entry);
    me->vs_store->insert(ns, key, value);
    me->vs_publisher.add_or_update(vs_entry);
+   remove_oldest(ns);
    me->vs_publisher.flush();
    api::ResultMessage ret;
    ret.set_success(true);
+   auto t1 = utils::TimeStamp::now().to_uint64();
+   LOG(Gen, Info,
+       "AnalyticsServiceServer::save_volsurface ns: {} key: {} n_entries: {} "
+       "time: {} ns",
+       ns, key, me->vs_publisher.state_data().size(), t1 - t0);
    response(ret);
 }
 
@@ -370,9 +427,8 @@ void AnalyticsServiceServer::delete_report(
 void AnalyticsServiceServer::archive_theos(
     const bts::api::analytics::ArchivedTheoList &theo_list,
     const api::Response<::bts::api::ResultMessage> &resp) {
-   LOG(Gen, Info,
-       "AnalyticsServiceServer::archive_theos sec_tups: {} sec_theo: {}",
-       theo_list.sec_tups().size(), theo_list.sec_theo().size());
+   me->n_theos_archives++;
+   uint64_t t0 = utils::TimeStamp::now().to_uint64();
    if (theo_list.sec_theo().empty()) {
       std::string err_msg = "Empty archived theo list, call failed";
       LOG(Gen, Error, "AnalyticsServiceServer::archive_theos {}", err_msg);
@@ -383,12 +439,23 @@ void AnalyticsServiceServer::archive_theos(
    const std::string key = std::to_string(theo_list.save_time());
    std::string value = utils::pb::to_string(theo_list);
    me->theos_store->insert(ns, key, value);
+   uint64_t t1 = utils::TimeStamp::now().to_uint64();
+   uint64_t t_diff = t1 - t0;
+   LOG(Gen, Info,
+       "AnalyticsServiceServer::archive_theos sec_tups: {} sec_theo: {} time: "
+       "{} ns",
+       theo_list.sec_tups().size(), theo_list.sec_theo().size(), t_diff);
+   if (me->n_theos_archives >= me->n_theos_archives_before_cleanup) {
+      me->n_theos_archives = 0;
+      _clean_up_archived_theos();
+   }
    fire_cb(resp, true);
 }
 
 void AnalyticsServiceServer::get_archived_theos(
     const bts::api::analytics::ArchivedTheoRetrieval &ret,
     const bts::api::Response<bts::api::analytics::ArchivedTheoList> &resp) {
+   uint64_t t0 = utils::TimeStamp::now().to_uint64();
    LOG(Gen, Info, "AnalyticsServiceServer::get_archived_theos ret: {}",
        utils::pb::to_string(ret));
    auto contains_st = [&](const ArchivedTheoList &tl) {
@@ -422,54 +489,80 @@ void AnalyticsServiceServer::get_archived_theos(
          }
       }
    }
-   LOG(Gen, Info, "AnalyticsServiceServer::get_archived_theos ret: {} key: {}",
-       utils::pb::to_string(ret), key);
+   uint64_t t1 = utils::TimeStamp::now().to_uint64();
+   uint64_t t_diff = t1 - t0;
+   LOG(Gen, Info,
+       "AnalyticsServiceServer::get_archived_theos ret: {} key: {} time: {} ns",
+       utils::pb::to_string(ret), key, t_diff);
    resp(rv);
 }
 
 AnalyticsServiceServer::~AnalyticsServiceServer() {}
 
+// for a map where the key is the epoch timestamp, this returns the keys
+// to delete such that final size is final_sz
+std::set<std::string> get_keys_to_delete(
+    const std::map<std::string, std::string> &time_keyed_map, size_t final_sz) {
+   if (time_keyed_map.size() <= final_sz) {
+      return {};
+   }
+   std::vector<uint64_t> keys_uint;
+   for (const auto &kv : time_keyed_map) {
+      try {
+         keys_uint.push_back(std::stoull(kv.first));
+      } catch (int) {
+         LOG(Gen, Error,
+             "AnalyticsServiceServer::get_keys_to_delete could not "
+             "convert "
+             "key: {} to uint",
+             kv.first);
+      }
+   }
+   // this is sorted lowest to highest
+   std::sort(keys_uint.begin(), keys_uint.end());
+   size_t n_del_rows = keys_uint.size() - final_sz;
+   std::vector<uint64_t> keys_uint_to_del =
+       std::vector<uint64_t>(keys_uint.begin(), keys_uint.begin() + n_del_rows);
+   std::set<std::string> keys_to_del;
+   for (auto k : keys_uint_to_del) {
+      keys_to_del.insert(std::to_string(k));
+   }
+   return keys_to_del;
+}
+
 void AnalyticsServiceServer::_clean_up_vss() {
-   const size_t final_sz = 500;
+   const size_t final_sz = me->vs_history_sz;
    auto all_rows = me->vs_store->retrieve_all();
    for (const auto &ns_k : all_rows) {
+      auto t0 = utils::TimeStamp::now().to_uint64();
       const auto &vs = ns_k.first;
       const auto &vs_row_map = ns_k.second;
-      if (vs_row_map.size() <= final_sz) {
-         LOG(Gen, Info,
-             "AnalyticsServiceServer::_clean_up_vss ignoring vs: {} because it "
-             "has only {} rows",
-             vs, vs_row_map.size());
-         continue;
-      }
-      std::vector<uint64_t> keys_uint;
-      for (const auto &r : vs_row_map) {
-         const auto &k = r.first;
-         try {
-            keys_uint.push_back(std::stoull(k));
-         } catch (int) {
-            LOG(Gen, Error,
-                "AnalyticsServiceServer::_clean_up_vss could not convert "
-                "key: "
-                "{} to uint",
-                k);
-         }
-      }
-      // sort and remove
-      std::sort(keys_uint.begin(), keys_uint.end());
-      size_t n_del_rows = keys_uint.size() - final_sz;
-      std::vector<uint64_t> keys_uint_to_del = std::vector<uint64_t>(
-          keys_uint.begin(), keys_uint.begin() + n_del_rows);
-      std::set<std::string> keys_to_del;
-      for (auto k : keys_uint_to_del) {
-         keys_to_del.insert(std::to_string(k));
-      }
-      LOG(Gen, Info,
-          "AnalyticsServiceServer::_clean_up_vss vs: {} n_rows: {} removing {} "
-          "rows",
-          vs, keys_uint.size(), keys_to_del.size());
+      auto keys_to_del = get_keys_to_delete(vs_row_map, final_sz);
       me->vs_store->remove(vs, keys_to_del);
+      auto t1 = utils::TimeStamp::now().to_uint64();
+      LOG(Gen, Info,
+          "AnalyticsServiceServer::_clean_up_vss vs: {} n_rows: {} "
+          "n_dels {} time: {} ns",
+          vs, vs_row_map.size(), keys_to_del.size(), t1 - t0);
    }
 }
 
+void AnalyticsServiceServer::_clean_up_archived_theos() {
+   uint64_t t0 = utils::TimeStamp::now().to_uint64();
+   std::map<std::string, std::map<std::string, std::string>> all_data =
+       me->theos_store->retrieve_all();
+   auto it = all_data.find(me->archived_theos_ns);
+   if (it == all_data.end())
+      return;
+   const std::map<std::string, std::string> &theo_data = it->second;
+   std::set<std::string> keys_to_del =
+       get_keys_to_delete(theo_data, me->archived_theos_history_sz);
+   me->theos_store->remove(me->archived_theos_ns, keys_to_del);
+   uint64_t t1 = utils::TimeStamp::now().to_uint64();
+   uint64_t t_diff = t1 - t0;
+   LOG(Gen, Info,
+       "AnalyticsServiceServer::_clean_up_archived_theos n_dels: {} "
+       "time: {} ns",
+       keys_to_del.size(), t_diff);
+}
 } // namespace bts::analytics

src/server/sparkd/analytics/AnalyticsServiceServer.h

@@ -72,5 +72,6 @@ class AnalyticsServiceServer
 
  private:
    void _clean_up_vss();
+   void _clean_up_archived_theos();
 };
 } // namespace bts::analytics

src/server/sparkd/cmds/md/cmds/Run.cpp

@@ -115,6 +115,7 @@ void Run::run(
    auto impl = [&mbo_writer, &tss_dispatcher, &type, &cleanup]() {
       LOG(md, Info, "Run::impl lambda t_id:{} t_name", os::get_thread_id(),
           get_thread_name());
+      return;
       if (cleanup)
          md::cmds::Cleanup::execute();
       if (type == Run::MDType::Configured) {
@@ -148,6 +149,7 @@ void Run::run(
          threading::make_thread("Emumba", []() { run_emumba(); }).detach();
       }
       if (bts::config::get<bts::md::MarketDataConfig>().use_sim()) {
+         return;
          threading::make_thread("ExchangeSimGLBX", [&mbo_writer]() {
             sim::start_exchange_sim(api::Venue::VENUE_SIM, mbo_writer);
          }).detach();

tools/ensure_venv.cmd

@@ -8,17 +8,18 @@ if exist %venv_dir% (
       echo Removing python virtual environment 
       rmdir /S /Q %venv_dir%
    ) else (
-      GOTO ERROR
+      GOTO END
    )
 )
 
 echo Creating python virtual environment
 @call python -m venv %venv_dir%
 @call %venv_dir%\Scripts\activate.bat
-@call pip install --upgrade pip
+@call %venv_dir%\Scripts\python -m pip install --upgrade pip
 @call %venv_dir%\Scripts\pip install -r %~dp0..\requirements.txt
 @call deactivate
 
+:END
 EXIT /B %ERRORLEVEL%
 
 :ERROR

tools/jenkins_build_spark.cmd

@@ -0,0 +1,13 @@
+@echo off
+
+@REM Build script designed for our jenkins build job(s). It assumes the py
+@REM venv already exists.
+
+setlocal
+set tools_dir=%~dp0
+set venv_dir=%~dp0..\.venv\
+call "C:\Program Files\Microsoft Visual Studio\2022\Community\Common7\Tools\VsDevCmd.bat"
+call "%venv_dir%Scripts\activate.bat"
+call python "%tools_dir%py\build_spark.py" %*
+
+EXIT /B %ERRORLEVEL%

tools/py/build_spark.py

@@ -24,7 +24,8 @@ class VisualStudioContext:
     def configure_libspark(self, build_dir) -> subprocess.CompletedProcess:
         with CWD(build_dir):
             res = subprocess.run(
-                f'cmake.exe -G "Visual Studio {self.version} {self.year}" {SparkPaths.root()}'
+                f'cmake.exe -G "Visual Studio {self.version} {self.year}" {SparkPaths.root()}',
+                shell=True,
             )
             shutil.copy(
                 SparkPaths.tools().joinpath('cpp.build.props'),
@@ -173,8 +174,7 @@ def _get_arg_parser():
         metavar='debug|release',
         type=str,
         nargs='?',
-        help='The build configuration',
-        default='Debug')
+        help='The build configuration')
 
     parser.add_argument(
         '-o',
@@ -336,9 +336,10 @@ def _main():
     link_build_dir = arg_to_path(get_arg(args_dict, 'link_output_dir'))
     targets = flatten_arglist(get_arg(args_dict, 'target'))
     verbosity = get_arg(args_dict, 'verbosity') or None
-
+    configuration = args.configuration or 'Release' if targets is not None and any(
+        filter(lambda t: t.lower() == 'publish', targets)) else 'Debug'
     build(
-        configuration=args.configuration,
+        configuration=configuration,
         clean=args.clean,
         restore=args.restore,
         build_dir=build_dir,