diff --git a/.gitmodules b/.gitmodules index e1458df02..3afa2cc94 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,9 +1,3 @@ [submodule "externals"] path = externals url = git@bitbucket.org:bluetradesys/externals.git -[submodule "docker/linux"] - path = docker/linux - url = git@bitbucket.org:bluetradesys/linux_docker.git -[submodule "docker/windows"] - path = docker/windows - url = git@bitbucket.org:bluetradesys/windows_docker.git diff --git a/docker/linux b/docker/linux deleted file mode 160000 index 81567b1f8..000000000 --- a/docker/linux +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 81567b1f82b8824aecfc376ffc0c4c881fbb39df diff --git a/docker/sparkd/Dockerfile b/docker/sparkd/Dockerfile new file mode 100644 index 000000000..17ab3c810 --- /dev/null +++ b/docker/sparkd/Dockerfile @@ -0,0 +1,89 @@ +FROM ubuntu:22.10 + +RUN \ + apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y \ + clang-13 \ + clang-format-13 \ + clangd-13 \ + clang-15 \ + clang-format-15 \ + clangd-15 \ + cmake \ + curl \ + dotnet7 \ + fish \ + g++-11 \ + g++-12 \ + gnupg \ + liblttng-ust-dev \ + libssh2-1-dev \ + libpcap0.8-dev \ + locales \ + ninja-build \ + nodejs \ + npm \ + openssh-server \ + python3-pip \ + python3-venv \ + rapidjson-dev \ + software-properties-common \ + sqlite3 \ + sudo \ + tar \ + vim \ + wget + +# this is done here so that ccache sees clang++ installed, not clang-10 +RUN \ + apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y ccache + +# emumba deps +RUN \ + apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y \ + tzdata \ + build-essential \ + cmake-curses-gui \ + git-all \ + doxygen \ + python3-sphinx \ + graphviz \ + libgtest-dev \ + libssl-dev \ + libboost-filesystem-dev \ + libboost-program-options-dev \ + libboost-test-dev \ + libboost-date-time-dev \ + zlib1g-dev \ + netcat-traditional \ + net-tools \ + gdb gdbserver \ + pkg-config + +# set up default executables (update-alternatives) +RUN \ + update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-13 100; \ + update-alternatives --install /usr/bin/clang clang /usr/bin/clang-13 100; \ + update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-13 100; \ + update-alternatives --install /usr/bin/clangd clangd /usr/bin/clangd-13 100; \ + update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-11 100; \ + update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 100 + +# Add compiler symlinks for ccache +RUN \ + ln -s /usr/bin/ccache /usr/local/bin/clang; \ + ln -s /usr/bin/ccache /usr/local/bin/clang++; \ + ln -s /usr/bin/ccache /usr/local/bin/clang-13; \ + ln -s /usr/bin/ccache /usr/local/bin/clang++-13; \ + ln -s /usr/bin/ccache /usr/local/bin/clang-15; \ + ln -s /usr/bin/ccache /usr/local/bin/clang++-15; \ + ln -s /usr/bin/ccache /usr/local/bin/gcc; \ + ln -s /usr/bin/ccache /usr/local/bin/g++; \ + ln -s /usr/bin/ccache /usr/local/bin/gcc-11; \ + ln -s /usr/bin/ccache /usr/local/bin/g++-11; \ + ln -s /usr/bin/ccache /usr/local/bin/gcc-12; \ + ln -s /usr/bin/ccache /usr/local/bin/g++-12 + +RUN locale-gen en_US.UTF-8 diff --git a/docker/sparkd/build.sh b/docker/sparkd/build.sh new file mode 100755 index 000000000..2c8839ec1 --- /dev/null +++ b/docker/sparkd/build.sh @@ -0,0 +1,95 @@ +#!/usr/bin/env bash + +function print_help_and_exit() { + cat < in the image (defaults to the username in the image) + -i,--image The name to give the image + -h,--help Print this help message and exit + +EOF + exit $1 +} + + +declare docker_user="" +declare docker_pass="" +declare image_name="" + +while [[ $# -gt 0 ]]; do + case "$1" in + -i) + ;& + --image) + if [[ -z "$2" ]]; then + echo "Image (-i,--image) must be non-empty" + print_help_and_exit 1 + fi + image_name="$2" + shift 2 + ;; + -u) + ;& + --user) + if [[ -z "$2" ]]; then + echo "User (-u,--user) must be non-empty" + print_help_and_exit 1 + fi + docker_user="$2" + shift 2 + ;; + -p) + ;& + --pass) + if [[ -z "$2" ]]; then + echo "Password (-p,--pass) must be non-empty" + print_help_and_exit 1 + fi + docker_pass="$2" + shift 2 + ;; + -h) + ;& + --help) + print_help_and_exit 0 + ;; + *) + echo "Unexpected argument '$1'" + print_help_and_exit 1 + ;; + esac +done + +if [[ -z "$image_name" ]]; then + image_name="bluetradingsystems/spark-crypto-dev" + echo "Pulling image '$image_name' from Dockerhub (use --image to build a new image)" + docker pull "$image_name" + declare -n pull_status=$? + if [[ $pull_status -ne 0 ]]; then + echo "docker pull exited with code $pull_status" + fi + exit $? +fi + +if [[ -z "$docker_user" ]]; then + docker_user="$USER" +fi + +if [[ -z "$docker_pass" ]]; then + docker_pass="$docker_user" + echo "Giving user '$docker_user' a secure password: $docker_pass" +fi + +echo "Building image +image_name : $image_name +user : $docker_user" + +docker build \ + --build-arg docker_user=$docker_user \ + --build-arg docker_password=$docker_pass \ + -t "$image_name" \ + . diff --git a/docker/sparkd/dev/Dockerfile b/docker/sparkd/dev/Dockerfile new file mode 100644 index 000000000..6e639f233 --- /dev/null +++ b/docker/sparkd/dev/Dockerfile @@ -0,0 +1,14 @@ +FROM johncolvin/bluetradingsystems:spark-dev + +ARG docker_user +ARG docker_uid +ARG docker_pass + +RUN \ + groupadd -g "$docker_uid" "$docker_user" \ + && useradd -u "$docker_uid" -g "$docker_uid" "$docker_user" \ + && usermod --password "$(openssl passwd -1 "$docker_pass")" "$docker_user" \ + && usermod -aG sudo "$docker_user" + +# Set the default user +USER $docker_user:$docker_user diff --git a/docker/sparkd/dev/README.md b/docker/sparkd/dev/README.md new file mode 100644 index 000000000..52a8adb58 --- /dev/null +++ b/docker/sparkd/dev/README.md @@ -0,0 +1,40 @@ +# Spark Dev Container +## How to create a Spark VS Code Devcontainer + + +## 1. Build the Docker image +Run the script `build.sh` located in this directory to build the image +``` +build.sh +``` +***Notes*** +* Builds an image named `$USER:spark-dev` +* Adds a user to the image with the same name/uid/gid as the current user so + it has proper file permissions in mounted directories +* Run `build.sh --help` to see available options + + +## 2. Generate a devcontainer.json +Run the script `gen_devcontainer.sh` located in this directory to create a devcontainer.json file.
+***Note:** The file path must be `/.devcontainer/devcontainer.json`* + +``` +# Dump `gen_devcontainer.sh` output to devcontainer.json +gen_devcontainer.sh > $workspaceFolder/.devcontainer/devcontainer.json +``` + +***Notes*** +* Mounts the following directories (`:`): + - `/home/:/home//.spark-dev/mounts/` + - `/home//spark:/path/to/this/repo` +* Run `gen_devcontainer.sh --help` to see available options + + +## 3. Open VS Code in Container +1. Launch **VS Code** and open the repo folder on the host machine. +1. Run **Dev Containers: Open Folder in Container** from the command palette.
+ +***Notes** + * The container stops when you close VS Code. All processes, including + `sparkd`, will be suspened until you open the workspace again (or manually + restart the container). diff --git a/docker/sparkd/dev/bashrc b/docker/sparkd/dev/bashrc new file mode 100644 index 000000000..b488fcc4c --- /dev/null +++ b/docker/sparkd/dev/bashrc @@ -0,0 +1,117 @@ +# ~/.bashrc: executed by bash(1) for non-login shells. +# see /usr/share/doc/bash/examples/startup-files (in the package bash-doc) +# for examples + +# If not running interactively, don't do anything +case $- in + *i*) ;; + *) return;; +esac + +# don't put duplicate lines or lines starting with space in the history. +# See bash(1) for more options +HISTCONTROL=ignoreboth + +# append to the history file, don't overwrite it +shopt -s histappend + +# for setting history length see HISTSIZE and HISTFILESIZE in bash(1) +HISTSIZE=1000 +HISTFILESIZE=2000 + +# check the window size after each command and, if necessary, +# update the values of LINES and COLUMNS. +shopt -s checkwinsize + +# If set, the pattern "**" used in a pathname expansion context will +# match all files and zero or more directories and subdirectories. +#shopt -s globstar + +# make less more friendly for non-text input files, see lesspipe(1) +[ -x /usr/bin/lesspipe ] && eval "$(SHELL=/bin/sh lesspipe)" + +# set variable identifying the chroot you work in (used in the prompt below) +if [ -z "${debian_chroot:-}" ] && [ -r /etc/debian_chroot ]; then + debian_chroot=$(cat /etc/debian_chroot) +fi + +# set a fancy prompt (non-color, unless we know we "want" color) +case "$TERM" in + xterm-color|*-256color) color_prompt=yes;; +esac + +# uncomment for a colored prompt, if the terminal has the capability; turned +# off by default to not distract the user: the focus in a terminal window +# should be on the output of commands, not on the prompt +#force_color_prompt=yes + +if [ -n "$force_color_prompt" ]; then + if [ -x /usr/bin/tput ] && tput setaf 1 >&/dev/null; then + # We have color support; assume it's compliant with Ecma-48 + # (ISO/IEC-6429). (Lack of such support is extremely rare, and such + # a case would tend to support setf rather than setaf.) + color_prompt=yes + else + color_prompt= + fi +fi + +if [ "$color_prompt" = yes ]; then + PS1='${debian_chroot:+($debian_chroot)}\[\033[01;32m\]\u@\h\[\033[00m\]:\[\033[01;34m\]\w\[\033[00m\]\$ ' +else + PS1='${debian_chroot:+($debian_chroot)}\u@\h:\w\$ ' +fi +unset color_prompt force_color_prompt + +# If this is an xterm set the title to user@host:dir +case "$TERM" in +xterm*|rxvt*) + PS1="\[\e]0;${debian_chroot:+($debian_chroot)}\u@\h: \w\a\]$PS1" + ;; +*) + ;; +esac + +# enable color support of ls and also add handy aliases +if [ -x /usr/bin/dircolors ]; then + test -r ~/.dircolors && eval "$(dircolors -b ~/.dircolors)" || eval "$(dircolors -b)" + alias ls='ls --color=auto' + #alias dir='dir --color=auto' + #alias vdir='vdir --color=auto' + + alias grep='grep --color=auto' + alias fgrep='fgrep --color=auto' + alias egrep='egrep --color=auto' +fi + +# colored GCC warnings and errors +#export GCC_COLORS='error=01;31:warning=01;35:note=01;36:caret=01;32:locus=01:quote=01' + +# some more ls aliases +alias ll='ls -alF' +alias la='ls -A' +alias l='ls -CF' + +# Add an "alert" alias for long running commands. Use like so: +# sleep 10; alert +alias alert='notify-send --urgency=low -i "$([ $? = 0 ] && echo terminal || echo error)" "$(history|tail -n1|sed -e '\''s/^\s*[0-9]\+\s*//;s/[;&|]\s*alert$//'\'')"' + +# Alias definitions. +# You may want to put all your additions into a separate file like +# ~/.bash_aliases, instead of adding them here directly. +# See /usr/share/doc/bash-doc/examples in the bash-doc package. + +if [ -f ~/.bash_aliases ]; then + . ~/.bash_aliases +fi + +# enable programmable completion features (you don't need to enable +# this, if it's already enabled in /etc/bash.bashrc and /etc/profile +# sources /etc/bash.bashrc). +if ! shopt -oq posix; then + if [ -f /usr/share/bash-completion/bash_completion ]; then + . /usr/share/bash-completion/bash_completion + elif [ -f /etc/bash_completion ]; then + . /etc/bash_completion + fi +fi diff --git a/docker/sparkd/dev/build.sh b/docker/sparkd/dev/build.sh new file mode 100755 index 000000000..8135d7e14 --- /dev/null +++ b/docker/sparkd/dev/build.sh @@ -0,0 +1,122 @@ +#!/usr/bin/env bash + +function print_help_and_exit() { + cat <> "$HOME/.bashrc" + fi +} + +append_path_cmd="export PATH=\$PATH:$HOME/.local/bin" +declare -i num_append_path_cmds="$(wc -l "$(grep "$append_path_cmd")")" +if [[ "$num_append_path_cmds" -eq 0 ]]; then + echo "$append_path_cmd" >> "$HOME/.bashrc" +fi diff --git a/docker/sparkd/docker_container_name.sh b/docker/sparkd/docker_container_name.sh new file mode 100755 index 000000000..9be627cea --- /dev/null +++ b/docker/sparkd/docker_container_name.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +if [ $# -gt 1 ]; then + echo $2 + exit 0 +fi + +declare image_name="$1" +if [ $# -eq 0 ]; then + declare image_name_cmd="$(realpath $(dirname "$0"))/docker_image_name.fish" + image_name="$(eval $image_name_cmd)" +fi +echo "${USER}s_$image_name" diff --git a/docker/sparkd/docker_shell.sh b/docker/sparkd/docker_shell.sh new file mode 100755 index 000000000..1054d0dc6 --- /dev/null +++ b/docker/sparkd/docker_shell.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash + +function print_help_and_exit() { + cat <: OR [:,...] Port mapping(s). May be + specified multiple times. + -h,--help Print this help message and exit + +EOF + exit "$1" +} + + +declare docker_user="" +declare docker_pass="" +declare image_name="" +declare cont_name="" +declare mount_name="" +declare -i mount_existing=0 +declare -a port_mappings=() + +while [[ $# -gt 0 ]]; do + case "$1" in + -u) + ;& + --user) + if [[ -z "$2" ]]; then + echo "User (-u,--user) must be non-empty" + print_help_and_exit 1 + fi + docker_user="$2" + shift 2 + ;; + -p) + ;& + --pass) + if [[ -z "$2" ]]; then + echo "Password (-p,--pass) must be non-empty" + print_help_and_exit 1 + fi + docker_pass="$2" + shift 2 + ;; + -i) + ;& + --image) + if [[ -n "$image_name" ]]; then + echo "Unexpected argument '$1'" + print_help_and_exit 1 + elif [[ -z "$2" ]]; then + echo "Image (-i,--image) must be non-empty" + print_help_and_exit 1 + fi + image_name="$2" + shift 2 + ;; + -c) + ;& + --container) + if [[ -n "$cont_name" ]]; then + echo "Unexpected argument '$1'" + print_help_and_exit 1 + elif [[ -z "$2" ]]; then + echo "Container (-c,--container) must be non-empty" + print_help_and_exit 1 + fi + cont_name="$2" + shift 2 + ;; + --port) + # Parse port(s) arg, and append the result to 'port_mappings' + # Note: port(s) may be specified in bracketed notation: + # [a:b,c:d,...,y:z] + next_ports="$(echo "$2" | sed -e 's/\[//g' -e 's/\]//g' -e 's/,/ /g')" + if [[ "${#port_mappings[@]}" -gt 0 ]]; then + next_ports=" $next_ports" + fi + port_mappings+=("$next_ports") + shift 2 + ;; + -m) + ;& + --mount) + if [[ -z "$2" ]]; then + echo "Mount (-m,--mount) must be non-empty" + fi + mount_name="$2" + shift 2 + ;; + --mount-existing) + mount_existing=1 + shift 1 + ;; + -h) + ;& + --help) + print_help_and_exit 0 + ;; + *) + if [[ -z "$1" ]]; then + echo "Invalid positional argument '$1'" + print_help_and_exit 1 + elif [[ -z "$image_name" ]]; then + image_name="$1" + elif [[ -z "$cont_name" ]]; then + cont_name="$1" + else + echo "Unexpected argument '$1'" + print_help_and_exit 1 + fi + shift 1 + ;; + esac +done + + +if [[ -z "$image_name" ]]; then + image_name="bluetradingsystems/spark-crypto-dev" +fi + +host_name="$(hostname)" +if [[ -n "$cont_name" ]]; then + host_name="$host_name.$cont_name" +fi + +if [[ -z "$mount_name" ]]; then + mount_name="$cont_name" +fi + +if [[ -z "$mount_name" && -z "$cont_name" ]]; then + echo "Please specify mount (-m,--mount) and/or container (-c,--container)" + print_help_and_exit 1 +fi + +if [[ -z "$docker_user" ]]; then + docker_user="$USER" +fi + +home_src=$mount_prefix/$mount_name +home_tgt=$HOME +repo_src=$(git rev-parse --show-superproject-working-tree) +repo_tgt=$HOME/$(basename "$repo_src") +working_dir=$HOME + +if [[ $mount_existing -eq 0 && -e "$home_src" ]]; then + # Don't overwrite existing dir (might be mounted in another container) + echo "Mount dir '$home_src' already exists (use --mount-existing to mount an existing directory)" + print_help_and_exit 1 +fi + +mkdir -p "$home_src" + +if [[ "${#port_mappings[@]}" -eq 0 ]]; then + port_mappings=("8000:8000" "8001:8001" "8002:8002" "8003:8003") +fi + +# Adapt port_mappings into args for `docker run` by prepending "-p " to each +# item +declare -a docker_port_args=() +for port_mapping in "${port_mappings[@]}"; do + docker_port_args+=("-p $port_mapping") +done + +echo "Starting container '$cont_name' +image_name : $image_name +cont_name : $cont_name +home_src : $home_src +home_tgt : $home_tgt +repo_src : $repo_src +repo_tgt : $repo_tgt +working dir : $working_dir +ports : ${port_mappings[*]} +" + +# DON'T wrap $docker_port_args in quotes. We want bash to split args by +# whitespace +docker run \ + --net bridge \ + -h "$host_name" \ + -e "TZ=America/New_York" \ + --cap-add=SYS_PTRACE \ + --security-opt seccomp=unconfined \ + --name "$cont_name" \ + -v "$home_src:$home_tgt" \ + -v "$repo_src:$repo_tgt" \ + -w "$working_dir" \ + ${docker_port_args[*]} \ + --shm-size=4024m \ + -t \ + -i \ + -d \ + -e DISPLAY="$DISPLAY" \ + "$image_name" \ + bash + +# Add user w/ same uid and gid as the current user outside of docker, so the +# user inside the docker has file permissions in the mounted dir. +docker_group="$docker_user" +docker_user_uid="$(id -u "$USER")" +docker_user_gid="$(id -g "$USER")" + +echo "Adding user in container '$cont_name' +name : $docker_user +uid : $docker_user_uid +gid : $docker_user_gid" + +if [[ -z "$docker_pass" ]]; then + docker_pass="$docker_user" + echo "Giving user '$docker_user' a secure password: $docker_pass" +fi + +docker exec -u root -it "$cont_name" groupadd -g "$docker_user_gid" "$docker_group" +docker exec -u root -it "$cont_name" useradd -u "$docker_user_uid" -g "$docker_user_gid" "$docker_user" +docker exec -u root -it "$cont_name" usermod --password "$(openssl passwd -1 "$docker_pass")" "$docker_user" +docker exec -u root -it "$cont_name" usermod -aG sudo "$docker_user" +docker exec -u root -it "$cont_name" "$repo_tgt/externals/src/emumba/install_emumba-deribit-connector.sh" diff --git a/docker/windows b/docker/windows deleted file mode 160000 index de7d40c09..000000000 --- a/docker/windows +++ /dev/null @@ -1 +0,0 @@ -Subproject commit de7d40c09de473e1b2802c4b27e03dc7f0c360d6