Compare commits

...
6 Commits
Author SHA1 Message Date
John Colvin 3cb69c7520 fix build: Pass 'shell=true' to subprocess.run
Otherwise it doesn't evaluate the command string properly
2023-10-17 13:50:38 -04:00
John Colvin 47e897d038 feat: Resurrect VolSurfaceAdvancedSettings 2023-10-17 13:50:38 -04:00
John Colvin 126ec169ea refactor: Remove useless param 2023-10-17 13:49:13 -04:00
John Colvin 84953df12c dev: Copy docker/sparkd dir from development branch
So we can build prod in devcontainer
2023-10-17 13:34:23 -04:00
John Colvin e8322183ec Update externals to get emumba-deribit deb w/out LICENSE check 2023-10-17 13:33:01 -04:00
Mohammad Arham 30ba7ecd19 added logs for _default_abs_moves 2023-10-10 15:14:28 -07:00
20 changed files with 1307 additions and 25 deletions

No files matched your search

+135
View File
@@ -0,0 +1,135 @@
FROM ubuntu:latest
ARG DEBIAN_FRONTEND=noninteractive
ENV TZ=America/New_York
# light deps (should be quick to download/install)
RUN apt-get update && apt-get install -y \
ca-certificates \
curl \
gnupg \
liblttng-ust-dev \
libpcap0.8-dev \
locales \
ninja-build \
software-properties-common \
sqlite3 \
tzdata
# clang
RUN apt-get update && apt-get install -y \
clang-13 \
clang-format-13 \
clangd-13 \
clang-15 \
clang-format-15 \
clangd-15
# g++
RUN apt-get update && apt-get install -y \
g++-11 \
g++-12
# python
RUN apt-get update && apt-get install -y \
python3-pip \
python3-venv \
&& python3 -m pip install --upgrade pip
# dotnet
RUN apt-get update && apt-get install -y \
dotnet7
# ssh
RUN apt-get update && apt-get install -y \
libssh2-1-dev \
openssh-server
# cmake
RUN \
curl -fsSL https://apt.kitware.com/keys/kitware-archive-latest.asc 2> /dev/null \
| gpg --dearmor - | tee /etc/apt/trusted.gpg.d/kitware.gpg > /dev/null \
&& apt-add-repository "deb https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" \
&& apt-get update \
&& apt-get install -y cmake
# node
ARG node_major=18
RUN mkdir -p /etc/apt/keyrings \
&& curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key \
| gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg \
&& echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$node_major.x nodistro main" \
| tee /etc/apt/sources.list.d/nodesource.list \
&& apt-get update \
&& apt-get install -y nodejs
# ccache is done here so that ccache sees clang++ installed, not clang-10
RUN apt-get update && apt-get install -y \
ccache
# emumba (light deps)
RUN apt-get update && apt-get install -y \
build-essential \
doxygen \
python3-sphinx \
graphviz \
libgtest-dev \
libssl-dev \
rapidjson-dev \
zlib1g-dev \
pkg-config
# emumba (boost)
RUN apt-get update && apt-get install -y \
libboost-filesystem-dev \
libboost-program-options-dev \
libboost-test-dev \
libboost-date-time-dev
# dev utils
RUN apt-get update && apt-get install -y \
fish \
git-all \
sudo \
vim \
wget
# set up default executables (update-alternatives)
RUN \
update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-13 100; \
update-alternatives --install /usr/bin/clang clang /usr/bin/clang-13 100; \
update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-13 100; \
update-alternatives --install /usr/bin/clangd clangd /usr/bin/clangd-13 100; \
update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-11 100; \
update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 100
# Add compiler symlinks for ccache
RUN \
ln -s /usr/bin/ccache /usr/local/bin/clang; \
ln -s /usr/bin/ccache /usr/local/bin/clang++; \
ln -s /usr/bin/ccache /usr/local/bin/clang-13; \
ln -s /usr/bin/ccache /usr/local/bin/clang++-13; \
ln -s /usr/bin/ccache /usr/local/bin/clang-15; \
ln -s /usr/bin/ccache /usr/local/bin/clang++-15; \
ln -s /usr/bin/ccache /usr/local/bin/gcc; \
ln -s /usr/bin/ccache /usr/local/bin/g++; \
ln -s /usr/bin/ccache /usr/local/bin/gcc-11; \
ln -s /usr/bin/ccache /usr/local/bin/g++-11; \
ln -s /usr/bin/ccache /usr/local/bin/gcc-12; \
ln -s /usr/bin/ccache /usr/local/bin/g++-12
RUN locale-gen en_US.UTF-8
ARG docker_user="bts"
ARG docker_uid="1000"
ARG docker_pass="bts"
RUN \
groupadd -g "$docker_uid" "$docker_user" \
&& useradd \
-m \
-u "$docker_uid" \
-g "$docker_uid" \
-G sudo \
--password "$(openssl passwd -1 "$docker_pass")" \
"$docker_user"
+64
View File
@@ -0,0 +1,64 @@
[dev containers]: https://code.visualstudio.com/docs/devcontainers/containers
[dev containers ext]: https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-containers
[docker repo sparkd-dev]: https://hub.docker.com/repository/docker/bluetradingsystems/sparkd-dev/general
# Sparkd Dev Container
## How to create a VS Code [Dev Container][dev containers]
1. Make sure the **[Dev Containers][dev containers ext]** extension is installed
1. Make sure your Docker image is up to date
```
docker pull bluetradingsystems/sparkd-dev:latest
```
1. Copy **devcontainer.json** from this folder to **$workspaceFolder/.devcontainer/**.
1. In the VS Code Command Palette, run **Dev Containers: Reopen in Container**
---
## How to create a Sparkd Container in the terminal (without VS Code)
1. Make sure your Docker image is up to date
```
docker pull bluetradingsystems/sparkd-dev:latest
```
2. Start the container
```
# Replace <cont_name> with a name for the container. The container's
# "$HOME" dir will be mounted to "$HOME/.sparkd-dev/mounts/<cont_name> on
# the host
./start_docker.sh --container <cont_name> --mount-existing
```
* *Replace <cont_name> with a name for the container*
* *The container's `$HOME` dir will be mounted to
`$HOME/.sparkd-dev/mounts<cont_name>` on the host*<br>
* *Run: `./start_docker.sh --help` to see additional options*
3. *(Optional) Attach VS Code to the running container.*<br>
Run **Dev Containers: Attach to Running Container** from the command palette,
then select your container in the dropdown
---
## How to update the Docker image
1. Change the image by modifying the Dockerfile (don't tag arbitrary
images on your machine and push them to our repo on Docker Hub)
1. After modifying the Dockerfile, (re)build the image
```
# Run in this dir
docker build -t bluetradingsystems/sparkd-dev:latest .
```
***Note:** Your docker login must have write permissions to push to the
[sparkd-dev][docker repo sparkd-dev] repository*
2. Commit the changes to Dockerfile in git
3. Push the new image to Docker Hub
```
docker push bluetradingsystems/sparkd-dev:latest
```
***Note:** The image doesn't have to be on Docker Hub, we can simply build
it from scratch on our local machines (with the Dockerfile). However, this
takes time. It is significantly quicker to download a prebuilt image, so we
store it on a remote repository to save time when setting up a new dev
machine.*
+123
View File
@@ -0,0 +1,123 @@
# ~/.bashrc: executed by bash(1) for non-login shells.
# see /usr/share/doc/bash/examples/startup-files (in the package bash-doc)
# for examples
# If not running interactively, don't do anything
case $- in
*i*) ;;
*) return;;
esac
# don't put duplicate lines or lines starting with space in the history.
# See bash(1) for more options
HISTCONTROL=ignoreboth
# append to the history file, don't overwrite it
shopt -s histappend
# for setting history length see HISTSIZE and HISTFILESIZE in bash(1)
HISTSIZE=1000
HISTFILESIZE=2000
# check the window size after each command and, if necessary,
# update the values of LINES and COLUMNS.
shopt -s checkwinsize
# If set, the pattern "**" used in a pathname expansion context will
# match all files and zero or more directories and subdirectories.
#shopt -s globstar
# make less more friendly for non-text input files, see lesspipe(1)
[ -x /usr/bin/lesspipe ] && eval "$(SHELL=/bin/sh lesspipe)"
# set variable identifying the chroot you work in (used in the prompt below)
if [ -z "${debian_chroot:-}" ] && [ -r /etc/debian_chroot ]; then
debian_chroot=$(cat /etc/debian_chroot)
fi
# set a fancy prompt (non-color, unless we know we "want" color)
case "$TERM" in
xterm-color|*-256color) color_prompt=yes;;
esac
# uncomment for a colored prompt, if the terminal has the capability; turned
# off by default to not distract the user: the focus in a terminal window
# should be on the output of commands, not on the prompt
#force_color_prompt=yes
if [ -n "$force_color_prompt" ]; then
if [ -x /usr/bin/tput ] && tput setaf 1 >&/dev/null; then
# We have color support; assume it's compliant with Ecma-48
# (ISO/IEC-6429). (Lack of such support is extremely rare, and such
# a case would tend to support setf rather than setaf.)
color_prompt=yes
else
color_prompt=
fi
fi
if [ "$color_prompt" = yes ]; then
PS1='${debian_chroot:+($debian_chroot)}\[\033[01;32m\]\u@\h\[\033[00m\]:\[\033[01;34m\]\w\[\033[00m\]\$ '
else
PS1='${debian_chroot:+($debian_chroot)}\u@\h:\w\$ '
fi
unset color_prompt force_color_prompt
# If this is an xterm set the title to user@host:dir
case "$TERM" in
xterm*|rxvt*)
PS1="\[\e]0;${debian_chroot:+($debian_chroot)}\u@\h: \w\a\]$PS1"
;;
*)
;;
esac
# enable color support of ls and also add handy aliases
if [ -x /usr/bin/dircolors ]; then
test -r ~/.dircolors && eval "$(dircolors -b ~/.dircolors)" || eval "$(dircolors -b)"
alias ls='ls --color=auto'
#alias dir='dir --color=auto'
#alias vdir='vdir --color=auto'
alias grep='grep --color=auto'
alias fgrep='fgrep --color=auto'
alias egrep='egrep --color=auto'
fi
# colored GCC warnings and errors
#export GCC_COLORS='error=01;31:warning=01;35:note=01;36:caret=01;32:locus=01:quote=01'
# some more ls aliases
alias ll='ls -alF'
alias la='ls -A'
alias l='ls -CF'
# Add an "alert" alias for long running commands. Use like so:
# sleep 10; alert
alias alert='notify-send --urgency=low -i "$([ $? = 0 ] && echo terminal || echo error)" "$(history|tail -n1|sed -e '\''s/^\s*[0-9]\+\s*//;s/[;&|]\s*alert$//'\'')"'
# Alias definitions.
# You may want to put all your additions into a separate file like
# ~/.bash_aliases, instead of adding them here directly.
# See /usr/share/doc/bash-doc/examples in the bash-doc package.
if [ -f ~/.bash_aliases ]; then
. ~/.bash_aliases
fi
# enable programmable completion features (you don't need to enable
# this, if it's already enabled in /etc/bash.bashrc and /etc/profile
# sources /etc/bash.bashrc).
if ! shopt -oq posix; then
if [ -f /usr/share/bash-completion/bash_completion ]; then
. /usr/share/bash-completion/bash_completion
elif [ -f /etc/bash_completion ]; then
. /etc/bash_completion
fi
fi
if [[ ! -v XDG_CONFIG_HOME ]]; then
export XDG_CONFIG_HOME="$HOME/.config"
fi
export PATH="$HOME/.local/bin:$PATH"
+25
View File
@@ -0,0 +1,25 @@
if ! set -q XDG_CONFIG_HOME
set -g XDG_CONFIG_HOME "$HOME/.config"
end
set fish_config_home $XDG_CONFIG_HOME/fish
if test -f "$fish_config_home/aliases.fish"
source "$fish_config_home/aliases.fish"
end
# enable general colors and case insensitive search in 'less'
set -x LESS "$LESS -R -Q -I"
set -x LC_ALL en_US.UTF-8
# enable syntax highlighting in 'less'
# set highlight color scheme in 'less' (fg)(bg)
set -g LESS_TERMCAP_md (tput bold)(tput setaf 36)
set -g LESS_TERMCAP_me (tput sgr0)
set -g LESS_TERMCAP_so (tput setaf 232)(tput setab 49)
set -q TERM
if test $status != 0; or test $TERM = 'xterm'
set -g TERM 'xterm-256color'
end
# Fix impossible-to-see navy blue command text color
set fish_color_command F8F8F2
set -g PATH "$HOME/.local/bin" "$PATH"
+32
View File
@@ -0,0 +1,32 @@
{
"image": "bluetradingsystems/sparkd-dev:latest",
"initializeCommand": "${localWorkspaceFolder}/docker/sparkd/dev/devcontainer_initialize.sh",
"hostRequirements": {
"memory": "4gb",
"storage": "6gb"
},
"postCreateCommand": "/home/bts/spark/docker/sparkd/dev/devcontainer_post_create.sh",
"containerUser": "bts",
"workspaceMount": "source=${localWorkspaceFolder},target=/home/bts/spark,type=bind",
"workspaceFolder": "/home/bts/spark",
"updateRemoteUserUID": true,
"shutdownAction": "none",
// Note: If running multiple containers on the same host, each must use a
// unique set of ports. Usually we just add 1000 to each port as needed. For
// example, if 8000-8003 is taken, go with 9000-9003, then 10000-10003, etc.
"runArgs": [
"--shm-size",
"2gb",
"-p",
"8000:8000",
"-p",
"8001:8001",
"-p",
"8002:8002",
"-p",
"8003:8003"
],
"mounts": [
"source=${localEnv:HOME}/.sparkd-dev/mounts/sparkd-dev,target=/home/bts,type=bind"
]
}
+3
View File
@@ -0,0 +1,3 @@
#!/usr/bin/env bash
mkdir -p "$HOME/.sparkd-dev/mounts/sparkd-dev"
+33
View File
@@ -0,0 +1,33 @@
#!/usr/bin/env bash
this_dir="$(realpath "$(dirname "${BASH_SOURCE[0]}")")"
repo_dir="$(cd "$this_dir" && git rev-parse --show-toplevel)"
tools_dir="$repo_dir/tools"
gen_tag="GENERATED BY $(basename "${BASH_SOURCE[0]}")"
function copy_if_not_exist() {
src="$1"
dest="$2"
if [[ ! -e "$src" ]]; then
echo "Unable to copy $src to $dest. Source file doesn't exist"
exit 1
fi
if [[ ! -e "$dest" ]]; then
cp "$src" "$dest"
fi
}
# Init bashrc
echo "Initializing bashrc"
copy_if_not_exist "$this_dir/bashrc" "$HOME/.bashrc"
# Init fish config
echo "Initializing fish config"
fish_cfg_dir="$HOME/.config/fish/"
mkdir -p "$fish_cfg_dir"
exec "$tools_dir/write_tag_block.sh" \
--tag "$gen_tag" \
--replace \
--in-place \
--file "$fish_cfg_dir/config.fish" \
--text "$(cat "$this_dir/config.fish")"
+250
View File
@@ -0,0 +1,250 @@
#!/usr/bin/env bash
function _exit() {
if [[ "$#" -gt 1 ]]; then
# When there are multiple args, the last arg is the exit code.
echo "${@:1:($# - 1)}"
exit "${@: -1}"
fi
echo "$1"
exit 0
}
help_text="\
Add or update a user's uid
This script is intended to be run in a Docker container as a means to obtain
file permissions in mounted directories owned by the specified UID.
Usage: set_user.sh [OPTIONS] USER UID
Positionals:
USER The name of the user
Options:
--user USER The name of the user (defaults to 'bts')
--uid UID The new uid to give USER (defaults to '1000')
--group GROUP The primary group to give USER (defaults to USER)
--gid GID The primary gid to give USER (defaults to UID)
--pass PASSWORD The password to give USER (defaults to USER, only applies
if USER hasn't been created yet)
--modify-existing Update UID/GID even if USER already exists
-h,--help Print this help message and exit
"
function _add_group() {
if [[ "$#" -ne 2 ]]; then
_exit "'_add_group' requires exactly 2 args (group, gid). Received $#." 1
fi
local group="$1"
local gid="$2"
# Check if gid already exists
local existing_group=""
existing_group="$(getent group "$gid" | cut -d : -f 1)"
if [[ -n "$existing_group" ]]; then
if [[ "$existing_group" != "$group" ]]; then
echo "\
The gid '$gid' is already in use by group '$existing_group' (will not rename \
the group)"
fi
return
fi
# Check if group with same name already exists
local existing_gid=""
existing_gid="$(getent group "$group" | cut -d : -f 3)"
if [[ -n "$existing_gid" ]]; then
# The group already exists. Only modify if --modify-existing was specified.
if [[ "$modify_existing" -eq 0 ]]; then
_exit "\
The group '$group' already exists with gid '$existing_gid'.
Specify --modify-existing to change the gid anyways." 1
fi
echo "Changing gid for group '$group' from '$existing_gid' to '$gid'"
groupmod -g "$gid" "$group"
else
# The group doesn't exist. Add it
echo "Adding group '$group:$gid'"
groupadd -g "$gid" "$group"
fi
}
function _add_user() {
local user="$1"
local uid="$2"
local group="$3"
local gid="$4"
local pass="$5"
if [[ "$#" -ne 5 ]]; then
_exit "'_add_user' requires exactly 5 args (user, uid, group, gid, pass). \
Received $#." 1
fi
# Check if uid is being used by another user
local existing_user=""
existing_user="$(getent passwd "$uid" | cut -d : -f 1)"
if [[ -n "$existing_user" ]]; then
if [[ "$existing_user" != "$user" ]]; then
_exit "The uid '$uid' is already in use by user '$existing_user'." 1
fi
echo "The user '$user' already has the specified uid '$uid'."
# Modify gid if necessary
local existing_gid=""
existing_gid="$(id -g "$user" 2> /dev/null)"
if [[ "$existing_gid" = "$gid" ]]; then
echo "The user '$user' already has the specified gid '$gid'."
return
fi
if [[ "$modify_existing" -eq 0 ]]; then
_exit "\
The user '$user' is associated with the primary group '$existing_gid'.
Specify --modify-existing to change the gid anyways." 1
fi
echo "Changing primary group for '$user' from '$existing_gid' to '$gid'."
usermod -g "$gid" "$user"
return
fi
# Check if user with same name already exists
local existing_uid=""
existing_uid="$(id -u "$user" 2> /dev/null)"
if [[ -n "$existing_uid" ]]; then
if [[ "$existing_uid" = "$uid" ]]; then
_exit "The user '$user' already has the specified uid '$uid'" 0
fi
if [[ "$modify_existing" -eq 0 ]]; then
_exit "\
User '$user' already exists with uid '$existing_uid'.
Specify --modify-existing to change the uid anyways." 1
fi
echo "Changing uid for user '$user' from '$existing_uid' to '$uid'"
if ! usermod -u "$uid" -g "$gid" "$user"; then
_exit "usermod exited with code $?" 1
fi
return
fi
# The user doesn't exist. Add it
echo "Adding user '$user:$uid' (group='$group:$gid', password='$pass')"
useradd -m -u "$uid" -g "$gid" -G sudo --password "$(openssl passwd -1 "$pass")" "$user"
}
function _update_fish_dir_perms() {
local user="$1"
local uid="$2"
local gid="$3"
# When a fish shell session starts, a temp dir named '/tmp/fish.<user>' is
# created with the user's current uid/gid. If such a dir exists for the
# specified USER, make sure the uid/gid match the new values.
fish_dir="/tmp/fish.$user"
if [[ -d "$fish_dir" ]]; then
perms="$uid:$gid"
existing_perms="$(stat -c %u:%g "$fish_dir")"
if [[ "$existing_perms" != "$perms" ]]; then
echo "Changing uid:gid for dir '$fish_dir' from '$existing_perms' to '$perms'"
chown -R "$perms" "$fish_dir"
fi
fi
}
user=""
group=""
pass=""
declare -i uid=0
declare -i gid=0
declare -i modify_existing=0
while [[ $# -gt 0 ]]; do
case "$1" in
--user)
if [[ -z "$2" ]]; then
_exit "USER (--user) must be non-empty" 1
elif [[ "$2" = "root" ]]; then
_exit "USER (--user) cannot be 'root'" 1
fi
user="$2"
shift 2
;;
--group)
if [[ -z "$2" ]]; then
_exit "GROUP (--group) must be non-empty" 1
elif [[ "$2" = "root" ]]; then
_exit "GROUP (--group) cannot be 'root'" 1
fi
group="$2"
shift 2
;;
--uid)
if [[ -z "$2" ]]; then
_exit "UID (--uid) must be non-empty" 1
fi
uid="$2"
shift 2
;;
--gid)
if [[ -z "$2" ]]; then
_exit "GID (--gid) must be non-empty" 1
fi
gid="$2"
shift 2
;;
--pass)
if [[ -z "$2" ]]; then
_exit "PASS (--pass) must be non-empty" 1
fi
pass="$2"
shift 2
;;
--modify-existing)
modify_existing=1
shift 1
;;
-h)
;&
--help)
_exit "$help_text" 0
;;
*)
if [[ -z "$1" ]]; then
_exit "Invalid positional argument '$1'" 1
elif [[ -z "$user" ]]; then
user="$1"
elif [[ -z "$uid" ]]; then
uid="$1"
else
_exit "Unexpected argument '$1'" 1
fi
shift 1
;;
esac
done
if [[ -z "$user" ]]; then
user="bts"
fi
if [[ $uid -eq 0 ]]; then
uid=1000
fi
# In general, a user's primary group/gid matches the user's name/uid.
# If group/gid args weren't specified, set them to name/uid.
if [[ -z "$group" ]]; then
group="$user"
fi
if [[ $gid -eq 0 ]]; then
gid=$uid
fi
if [[ -z "$pass" ]]; then
pass="$user"
fi
_add_group "$group" "$gid"
_add_user "$user" "$uid" "$group" "$gid" "$pass"
_update_fish_dir_perms "$user" "$uid" "$gid"
+230
View File
@@ -0,0 +1,230 @@
#!/usr/bin/env bash
function _exit() {
if [[ "$#" -gt 1 ]]; then
# When there are multiple args, the last arg is the exit code.
echo "${@:1:($# - 1)}"
exit "${@: -1}"
fi
echo "$1"
exit 0
}
help_text="\
Start a docker dev container
Usage: start_docker.sh [OPTIONS] image container
Positionals:
image The image to instantiate
container The name for the container
Options:
-u,--user USER The user to add in the container (defaults to 'bts')
-p,--pass PASS The password for USER in the container (defaults to USER)
-i,--image IMAGE The image to instantiate (defaults to 'bluetradingsystems/sparkd-dev:latest')
-c,--container CONT The name for the container
-m,--mount MOUNT The name for the folder (outside the container) to create/mount to the home
directory in the container. Mount folders are added here: $mount_prefix
--mount-existing Mount the specified dir even if it already exists
--port <cont_port>:<host_port> OR [<cont_port>:<host_port>,...] Port mapping(s). May be
specified multiple times.
-h,--help Print this help message and exit
"
mount_prefix="$HOME/.sparkd-dev/mounts"
declare docker_user=""
declare docker_pass=""
declare image_name=""
declare cont_name=""
declare mount_name=""
declare -i mount_existing=0
declare -a port_mappings=()
while [[ $# -gt 0 ]]; do
case "$1" in
-u)
;&
--user)
if [[ -z "$2" ]]; then
_exit "User (-u,--user) must be non-empty" 1
fi
docker_user="$2"
shift 2
;;
-p)
;&
--pass)
if [[ -z "$2" ]]; then
_exit "Password (-p,--pass) must be non-empty" 1
fi
docker_pass="$2"
shift 2
;;
-i)
;&
--image)
if [[ -n "$image_name" ]]; then
_exit "Unexpected argument '$1'" 1
elif [[ -z "$2" ]]; then
_exit "Image (-i,--image) must be non-empty" 1
fi
image_name="$2"
shift 2
;;
-c)
;&
--container)
if [[ -n "$cont_name" ]]; then
_exit "Unexpected argument '$1'" 1
elif [[ -z "$2" ]]; then
_exit "Container (-c,--container) must be non-empty" 1
fi
cont_name="$2"
shift 2
;;
--port)
# Parse port(s) arg, and append the result to 'port_mappings'
# Note: port(s) may be specified in bracketed notation:
# [a:b,c:d,...,y:z]
next_ports="$(echo "$2" | sed -e 's/\[//g' -e 's/\]//g' -e 's/,/ /g')"
if [[ "${#port_mappings[@]}" -gt 0 ]]; then
next_ports=" $next_ports"
fi
port_mappings+=("$next_ports")
shift 2
;;
-m)
;&
--mount)
if [[ -z "$2" ]]; then
echo "Mount (-m,--mount) must be non-empty"
fi
mount_name="$2"
shift 2
;;
--mount-existing)
mount_existing=1
shift 1
;;
-h)
;&
--help)
_exit "$help_text" 0
;;
*)
if [[ -z "$1" ]]; then
_exit "Invalid positional argument '$1'" "$help_text" 1
elif [[ -z "$image_name" ]]; then
image_name="$1"
elif [[ -z "$cont_name" ]]; then
cont_name="$1"
else
_exit "Unexpected argument '$1'" "$help_text" 1
fi
shift 1
;;
esac
done
if [[ -z "$image_name" ]]; then
image_name="bluetradingsystems/sparkd-dev:latest"
fi
host_name="$(hostname)"
if [[ -n "$cont_name" ]]; then
host_name="$host_name.$cont_name"
fi
if [[ -z "$mount_name" ]]; then
mount_name="$cont_name"
fi
if [[ -z "$mount_name" && -z "$cont_name" ]]; then
_exit "Please specify mount (-m,--mount) and/or container (-c,--container)" 1
fi
if [[ -z "$docker_user" ]]; then
docker_user="bts"
fi
if [[ "$docker_user" = "root" ]]; then
_exit "USER (--user) cannot be 'root'" 1
fi
cont_home="/home/$docker_user"
home_src="$mount_prefix/$mount_name"
home_tgt="$cont_home"
this_dir="$(dirname "$(realpath "${BASH_SOURCE[0]}")")"
repo_src="$(cd "$this_dir" && git rev-parse --show-toplevel)"
repo_tgt="$cont_home/spark"
working_dir="$cont_home"
if [[ $mount_existing -eq 0 && -e "$home_src" ]]; then
# Don't overwrite existing dir (might be mounted in another container)
_exit "Mount dir '$home_src' already exists (use --mount-existing to mount \
an existing directory)" 1
fi
mkdir -p "$home_src"
if [[ "${#port_mappings[@]}" -eq 0 ]]; then
port_mappings=("8000:8000" "8001:8001" "8002:8002" "8003:8003")
fi
# Adapt port_mappings into args for `docker run` by prepending "-p " to each
# item
declare -a docker_port_args=()
for port_mapping in "${port_mappings[@]}"; do
docker_port_args+=("-p $port_mapping")
done
echo "Starting container '$cont_name'
image_name : $image_name
cont_name : $cont_name
home_src : $home_src
home_tgt : $home_tgt
repo_src : $repo_src
repo_tgt : $repo_tgt
working dir : $working_dir
ports : ${port_mappings[*]}
"
# DON'T wrap $docker_port_args in quotes. We want bash to split args by
# whitespace
docker run \
--net bridge \
-h "$host_name" \
-e "TZ=America/New_York" \
--cap-add=SYS_PTRACE \
--security-opt seccomp=unconfined \
--name "$cont_name" \
-v "$home_src:$home_tgt" \
-v "$repo_src:$repo_tgt" \
-w "$working_dir" \
${docker_port_args[*]} \
--shm-size=4024m \
-t \
-i \
-d \
-e DISPLAY="$DISPLAY" \
"$image_name"
# Add user w/ same uid and gid as the current user outside of docker, so the
# user inside the docker has file permissions in the mounted dir.
docker_group="$docker_user"
docker_uid="$(id -u "$USER")"
docker_gid="$(id -g "$USER")"
if [[ -z "$docker_pass" ]]; then
docker_pass="$docker_user"
fi
docker exec -u root -it "$cont_name" "$repo_tgt/docker/sparkd/dev/set_user.sh" \
--user "$docker_user" \
--uid "$docker_uid" \
--group "$docker_group" \
--gid "$docker_gid" \
--pass "$docker_pass"
# Run same script VS Code runs to initialize the container
docker exec -u "$docker_user" -it "$cont_name" "$repo_tgt/docker/sparkd/dev/devcontainer_post_create.sh"
+64
View File
@@ -0,0 +1,64 @@
#!/usr/bin/env bash
function print_help_and_exit() {
cat <<EOF
Open a shell inside the docker container
Usage: docker_shell.sh [OPTIONS] [CONTAINER]
Options:
-u,--user The user to login with in the container (defaults to the current user)
-c,--container The container name (may also be specified as positional arg)
-h,--help Print this help message and exit
EOF
exit "$1"
}
declare docker_user=""
declare cont_name=""
while [[ $# -gt 0 ]]; do
case "$1" in
-u)
;&
--user)
if [[ -z "$2" ]]; then
echo "User (-u,--user) must be non-empty"
print_help_and_exit 1
fi
docker_user="$2"
shift 2
;;
-h)
;&
--help)
print_help_and_exit 0
;;
-c)
;&
--container)
shift 1
;&
*)
if [[ ! -z "$cont_name" ]]; then
echo "Unexpected argument '$1'"
print_help_and_exit 1
fi
cont_name="$1"
shift 1
;;
esac
done
if [[ -z "$cont_name" ]]; then
echo "Please specify the container ([-c,--container])"
print_help_and_exit 1
fi
if [[ -z "$docker_user" ]]; then
docker_user="$USER"
fi
docker exec -u "$docker_user" -it "$cont_name" bash
+233
View File
@@ -0,0 +1,233 @@
#!/usr/bin/env bash
mount_prefix="$HOME/.spark-dev/mounts"
function print_help_and_exit() {
cat <<EOF
Start a docker dev container
Usage: start_docker.sh [OPTIONS] image container
Positionals:
image The image to instantiate
container The name for the container
Options:
-u,--user USER The user to add in the container (defaults to the current user)
-p,--pass PASS The password for USER in the container (defaults to USER)
-i,--image IMAGE The image to instantiate
-c,--container CONT The name for the container
-m,--mount MOUNT The name for the folder (outside the container) to create/mount to the home
directory in the container. Mount folders are added here: $mount_prefix
--mount-existing Mount the specified dir even if it already exists
--port <cont_port>:<host_port> OR [<cont_port>:<host_port>,...] Port mapping(s). May be
specified multiple times.
-h,--help Print this help message and exit
EOF
exit "$1"
}
declare docker_user=""
declare docker_pass=""
declare image_name=""
declare cont_name=""
declare mount_name=""
declare -i mount_existing=0
declare -a port_mappings=()
while [[ $# -gt 0 ]]; do
case "$1" in
-u)
;&
--user)
if [[ -z "$2" ]]; then
echo "User (-u,--user) must be non-empty"
print_help_and_exit 1
fi
docker_user="$2"
shift 2
;;
-p)
;&
--pass)
if [[ -z "$2" ]]; then
echo "Password (-p,--pass) must be non-empty"
print_help_and_exit 1
fi
docker_pass="$2"
shift 2
;;
-i)
;&
--image)
if [[ -n "$image_name" ]]; then
echo "Unexpected argument '$1'"
print_help_and_exit 1
elif [[ -z "$2" ]]; then
echo "Image (-i,--image) must be non-empty"
print_help_and_exit 1
fi
image_name="$2"
shift 2
;;
-c)
;&
--container)
if [[ -n "$cont_name" ]]; then
echo "Unexpected argument '$1'"
print_help_and_exit 1
elif [[ -z "$2" ]]; then
echo "Container (-c,--container) must be non-empty"
print_help_and_exit 1
fi
cont_name="$2"
shift 2
;;
--port)
# Parse port(s) arg, and append the result to 'port_mappings'
# Note: port(s) may be specified in bracketed notation:
# [a:b,c:d,...,y:z]
next_ports="$(echo "$2" | sed -e 's/\[//g' -e 's/\]//g' -e 's/,/ /g')"
if [[ "${#port_mappings[@]}" -gt 0 ]]; then
next_ports=" $next_ports"
fi
port_mappings+=("$next_ports")
shift 2
;;
-m)
;&
--mount)
if [[ -z "$2" ]]; then
echo "Mount (-m,--mount) must be non-empty"
fi
mount_name="$2"
shift 2
;;
--mount-existing)
mount_existing=1
shift 1
;;
-h)
;&
--help)
print_help_and_exit 0
;;
*)
if [[ -z "$1" ]]; then
echo "Invalid positional argument '$1'"
print_help_and_exit 1
elif [[ -z "$image_name" ]]; then
image_name="$1"
elif [[ -z "$cont_name" ]]; then
cont_name="$1"
else
echo "Unexpected argument '$1'"
print_help_and_exit 1
fi
shift 1
;;
esac
done
if [[ -z "$image_name" ]]; then
image_name="bluetradingsystems/spark-crypto-dev"
fi
host_name="$(hostname)"
if [[ -n "$cont_name" ]]; then
host_name="$host_name.$cont_name"
fi
if [[ -z "$mount_name" ]]; then
mount_name="$cont_name"
fi
if [[ -z "$mount_name" && -z "$cont_name" ]]; then
echo "Please specify mount (-m,--mount) and/or container (-c,--container)"
print_help_and_exit 1
fi
if [[ -z "$docker_user" ]]; then
docker_user="$USER"
fi
home_src=$mount_prefix/$mount_name
home_tgt=$HOME
repo_src=$(git rev-parse --show-superproject-working-tree)
repo_tgt=$HOME/$(basename "$repo_src")
working_dir=$HOME
if [[ $mount_existing -eq 0 && -e "$home_src" ]]; then
# Don't overwrite existing dir (might be mounted in another container)
echo "Mount dir '$home_src' already exists (use --mount-existing to mount an existing directory)"
print_help_and_exit 1
fi
mkdir -p "$home_src"
if [[ "${#port_mappings[@]}" -eq 0 ]]; then
port_mappings=("8000:8000" "8001:8001" "8002:8002" "8003:8003")
fi
# Adapt port_mappings into args for `docker run` by prepending "-p " to each
# item
declare -a docker_port_args=()
for port_mapping in "${port_mappings[@]}"; do
docker_port_args+=("-p $port_mapping")
done
echo "Starting container '$cont_name'
image_name : $image_name
cont_name : $cont_name
home_src : $home_src
home_tgt : $home_tgt
repo_src : $repo_src
repo_tgt : $repo_tgt
working dir : $working_dir
ports : ${port_mappings[*]}
"
# DON'T wrap $docker_port_args in quotes. We want bash to split args by
# whitespace
docker run \
--net bridge \
-h "$host_name" \
-e "TZ=America/New_York" \
--cap-add=SYS_PTRACE \
--security-opt seccomp=unconfined \
--name "$cont_name" \
-v "$home_src:$home_tgt" \
-v "$repo_src:$repo_tgt" \
-w "$working_dir" \
${docker_port_args[*]} \
--shm-size=4024m \
-t \
-i \
-d \
-e DISPLAY="$DISPLAY" \
"$image_name" \
bash
# Add user w/ same uid and gid as the current user outside of docker, so the
# user inside the docker has file permissions in the mounted dir.
docker_group="$docker_user"
docker_user_uid="$(id -u "$USER")"
docker_user_gid="$(id -g "$USER")"
echo "Adding user in container '$cont_name'
name : $docker_user
uid : $docker_user_uid
gid : $docker_user_gid"
if [[ -z "$docker_pass" ]]; then
docker_pass="$docker_user"
echo "Giving user '$docker_user' a secure password: $docker_pass"
fi
docker exec -u root -it "$cont_name" groupadd -g "$docker_user_gid" "$docker_group"
docker exec -u root -it "$cont_name" useradd -u "$docker_user_uid" -g "$docker_user_gid" "$docker_user"
docker exec -u root -it "$cont_name" usermod --password "$(openssl passwd -1 "$docker_pass")" "$docker_user"
docker exec -u root -it "$cont_name" usermod -aG sudo "$docker_user"
docker exec -u root -it "$cont_name" "$repo_tgt/externals/src/emumba/install_emumba-deribit-connector.sh"
@@ -14,7 +14,7 @@ namespace mbts.spark {
class VolSurfaceAdvancedSettingsEditorVM : ViewModelBase {
readonly VolSurfaceAdvancedSettings _settings;
public VolSurfaceAdvancedSettingsEditorVM(VolSurfaceAdvancedSettings settings, string vs_name, Action<VolSurfaceAdvancedSettings> commit) {
public VolSurfaceAdvancedSettingsEditorVM(VolSurfaceAdvancedSettings settings, string vs_name) {
_settings = settings;
_settings.theo_handle_measure = VolSurfaceEnums.GeneralChangeMeasure.Absolute;
_settings.slide_handle_measure = VolSurfaceEnums.GeneralChangeMeasure.Percentage;
@@ -23,7 +23,7 @@ namespace mbts.spark {
if (_settings.precomputed_theo_handles == null)
_settings.precomputed_theo_handles = new List<double>();
vol_surface_name = vs_name;
this.commit = new DelegateCommand(() => request_commit?.Invoke(ProtobufSerializationHelper.clone(_settings)));
commit = new DelegateCommand(() => request_commit?.Invoke(ProtobufSerializationHelper.clone(_settings)));
}
public event Action<VolSurfaceAdvancedSettings> request_commit;
+18 -15
View File
@@ -121,21 +121,24 @@ namespace mbts.spark {
if (settings == null) {
return null;
}
// return new VolSurfaceAdvancedSettingsEditor {
// DataContext = new VolSurfaceAdvancedSettingsEditorVM(
// VolSurfaceUtilityFunctions.clone_proto(settings.advanced_settings),
// settings.core_settings.vol_surface_name,
// advanced_settings => {
// var current_settings = _get_volsurface_settings();
// if (current_settings == null) BTSMessageBox.Show("Error changing settings", "VS Settings", System.Windows.MessageBoxButton.OK, System.Windows.MessageBoxImage.Error, true, true, true);
// else {
// current_settings.advanced_settings = advanced_settings;
// _model.set_candidate_settings(current_settings);
// }
// }
// )
// };
return null;
var vm = new VolSurfaceAdvancedSettingsEditorVM(
VolSurfaceUtilityFunctions.clone_proto(settings.advanced_settings),
settings.core_settings.vol_surface_name);
vm.request_commit += commit_advanced_settings;
return new VolSurfaceAdvancedSettingsEditor {
DataContext = vm
};
async void commit_advanced_settings(VolSurfaceAdvancedSettings value) {
var current_settings = await _get_volsurface_settings();
if (current_settings == null) {
BTSMessageBox.Show("Error changing settings", "VS Settings");
}
else {
current_settings.advanced_settings = value;
_model.set_candidate_settings(current_settings);
}
}
}
Window? mk_grid_settings_win() {
@@ -9,26 +9,34 @@ namespace mbts.analytics
[ProtoBuf.ProtoContract(Name = "VolSurfaceAdvancedSettings")]
public class VolSurfaceAdvancedSettings
{
public static VolSurfaceAdvancedSettings Default =>
new() {
number_of_cashed_saved_days = double.NaN,
precomputed_theo_handles = new(0),
theo_handle_measure = VolSurfaceEnums.GeneralChangeMeasure.Absolute,
precomputed_slide_handles = new(0),
slide_handle_measure = VolSurfaceEnums.GeneralChangeMeasure.Percentage,
publish_from_client = false,
publish_from_client_user = string.Empty,
strike_normalization_axis = VolSurfaceEnums.VolCurveStrikeAxis.RawStrike,
};
[ProtoBuf.ProtoMember(1)]
[System.ComponentModel.DefaultValue(Double.NaN)]
public double number_of_cashed_saved_days;
[ProtoBuf.ProtoMember(10)]
public List<double> precomputed_theo_handles;
[ProtoBuf.ProtoMember(15)]
[System.ComponentModel.DefaultValue(VolSurfaceEnums.GeneralChangeMeasure.Absolute)]
public VolSurfaceEnums.GeneralChangeMeasure theo_handle_measure;
[ProtoBuf.ProtoMember(20)]
public List<double> precomputed_slide_handles;
[ProtoBuf.ProtoMember(25)]
[System.ComponentModel.DefaultValue(VolSurfaceEnums.GeneralChangeMeasure.Percentage)]
public VolSurfaceEnums.GeneralChangeMeasure slide_handle_measure;
[ProtoBuf.ProtoMember(30)]
//[System.ComponentModel.DefaultValue(false)]
public bool publish_from_client;
[ProtoBuf.ProtoMember(40)]
@@ -38,5 +46,11 @@ namespace mbts.analytics
[ProtoBuf.ProtoMember(50)]
[System.ComponentModel.DefaultValue(VolSurfaceEnums.VolCurveStrikeAxis.RawStrike)]
public VolSurfaceEnums.VolCurveStrikeAxis strike_normalization_axis;
public void ensure_no_nulls() {
precomputed_theo_handles ??= new(0);
precomputed_slide_handles ??= new(0);
publish_from_client_user ??= string.Empty;
}
}
}
@@ -25,7 +25,8 @@ namespace mbts.analytics {
volpath_settings = new(),
safety_limits = new(),
},
polynomial_model_settings = new()
polynomial_model_settings = new(),
advanced_settings = VolSurfaceAdvancedSettings.Default
};
[ProtoMember(10)]
@@ -79,6 +80,8 @@ namespace mbts.analytics {
if (core_settings.time_settings == null) {
core_settings.time_settings = new VolSurfaceTimeSettings();
}
advanced_settings ??= VolSurfaceAdvancedSettings.Default;
advanced_settings.ensure_no_nulls();
return this;
}
}
@@ -50,7 +50,7 @@ namespace mbts.analytics {
(o, vt, tt) => { };
public VolSurfaceInterpGreeksProducer(int vs_id) {
Log.Gen.info($"VolSurfaceInterpGreeksProducer.cons vs_id: {vs_id}");
Log.Gen.info($"VolSurfaceInterpGreeksProducer.cons vs_id: {vs_id}, _default_abs_moves: {string.Join(", ", _default_abs_moves)}");
_vs_id = vs_id;
}
@@ -0,0 +1,58 @@
<Window
x:Class="mbts.ui.shell.VolSurfaceAdvancedSettingsEditor"
x:ClassModifier="internal"
xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation"
xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
xmlns:c="clr-namespace:mbts.ui.controls"
xmlns:cv="clr-namespace:mbts.ui.converters"
Title="{Binding vol_surface_name, StringFormat='{}Advanced Settings \&apos;{0}\&apos;'}"
Style="{StaticResource BTSDialog}"
SizeToContent="WidthAndHeight"
TextElement.FontSize="10">
<StackPanel>
<Grid Margin="5" Grid.IsSharedSizeScope="True">
<Grid.RowDefinitions>
<RowDefinition Height="Auto"/>
<RowDefinition Height="Auto"/>
<RowDefinition Height="Auto"/>
<RowDefinition Height="Auto"/>
<RowDefinition Height="Auto"/>
</Grid.RowDefinitions>
<Grid.ColumnDefinitions>
<ColumnDefinition Width="Auto" SharedSizeGroup="label"/>
<ColumnDefinition Width="*"/>
</Grid.ColumnDefinitions>
<GroupBox Grid.ColumnSpan="2" Padding="3">
<GroupBox.Header>
<TextBlock Text="Theos" FontWeight="Bold"/>
</GroupBox.Header>
<Grid>
<Grid.RowDefinitions>
<RowDefinition Height="Auto"/>
<RowDefinition Height="Auto"/>
</Grid.RowDefinitions>
<Grid.ColumnDefinitions>
<ColumnDefinition Width="Auto" SharedSizeGroup="label"/>
<ColumnDefinition Width="Auto"/>
</Grid.ColumnDefinitions>
<c:RadioButtonsControl Grid.ColumnSpan="2" Margin="0,0,0,5" ItemsSource="{Binding general_change_measures}" SelectedItem="{Binding theo_handle_measure}" IsEnabled="False"/>
<TextBlock Grid.Row="1" Text="Precomputed Handles" VerticalAlignment="Center"/>
<TextBox
x:Name="PART_Handles"
Validation.ErrorTemplate="{x:Null}"
Grid.Row="1"
Grid.Column="1"
Width="160"
Margin="5,0,0,0"
Text="{Binding precomputed_theo_handles, Converter={x:Static cv:EnumerableDoubleStringConverter.Instance}}" />
</Grid>
</GroupBox>
<TextBlock Visibility="Collapsed" Grid.Row="1" VerticalAlignment="Center" Margin="5,0,0,0" Text="Num Cash Saved Days" />
<c:TickUpDownDouble Visibility="Collapsed" Grid.Row="1" Grid.Column="1" HorizontalAlignment="Stretch" Margin="14,5,5,5" Value="{Binding number_of_cash_saved_days}"/>
<TextBlock Grid.Row="2" VerticalAlignment="Center" Margin="5,0,0,0" Text="Strike Normalization Axis"/>
<ComboBox Grid.Row="2" Grid.Column="1" HorizontalAlignment="Stretch" Margin="14,5,5,5" ItemsSource="{Binding strike_normalization_axes}" SelectedValue="{Binding selected_strike_normalization_axis}"/>
<CheckBox Grid.Row="3" Margin="7,0,0,0" Content="Publish From Client" IsChecked="{Binding publish_from_client}"/>
</Grid>
<c:DialogSkirtOKCancel OKCommand="{Binding commit}" />
</StackPanel>
</Window>
@@ -0,0 +1,11 @@
namespace mbts.ui.shell {
using System.Windows;
/// <summary>
/// Interaction logic for VolSurfaceAdvancedSettingsEditor.xaml
/// </summary>
partial class VolSurfaceAdvancedSettingsEditor : Window {
public VolSurfaceAdvancedSettingsEditor() =>
InitializeComponent();
}
}
+2 -1
View File
@@ -24,7 +24,8 @@ class VisualStudioContext:
def configure_libspark(self, build_dir) -> subprocess.CompletedProcess:
with CWD(build_dir):
res = subprocess.run(
f'cmake.exe -G "Visual Studio {self.version} {self.year}" {SparkPaths.root()}'
f'cmake.exe -G "Visual Studio {self.version} {self.year}" {SparkPaths.root()}',
shell=True,
)
shutil.copy(
SparkPaths.tools().joinpath('cpp.build.props'),