diff --git a/Dockerfile b/Dockerfile index 1a54fe1..c45e7df 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,120 +1,39 @@ -FROM ubuntu:bionic +FROM ubuntu:22.10 RUN \ apt-get update \ && DEBIAN_FRONTEND=noninteractive apt-get install -y \ - gnupg \ - software-properties-common \ - wget - -RUN \ - echo "deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-9 main" >> /etc/apt/sources.list && \ - wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key > llvm_key && apt-key add llvm_key - -# Add Kitware repo to get latest cmake. The ccapi lib relies on -# 'add_compile_definitions', which was introduced in cmake 3.12. -RUN \ - wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | apt-key add - && \ - apt-add-repository 'deb https://apt.kitware.com/ubuntu/ bionic main' && \ - apt update - -# Add ubuntu-toolchain repo to get g++-11 -RUN \ - add-apt-repository ppa:ubuntu-toolchain-r/test && \ - apt-get update - -# add the latest node repository to apt sources (npm in the standard bionic apt -# repostitory depends on libssl1.0-dev, but emumba depends on libssl-dev, so a -# the latest npm must be installed to avoid this dependency conflict -RUN \ - wget -O - https://deb.nodesource.com/setup_16.x > install_node.sh && chmod +x install_node.sh && ./install_node.sh - -RUN \ - apt-get update \ - && DEBIAN_FRONTEND=noninteractive apt-get install -y \ - libllvm-10-ocaml-dev libllvm10 llvm-10 llvm-10-dev llvm-10-doc llvm-10-examples llvm-10-runtime \ - clang-10 clang-tools-10 clang-10-doc libclang-common-10-dev libclang-10-dev libclang1-10 clang-format-10 python3-clang-10 clangd-10 \ - libfuzzer-9-dev \ - lldb-9 \ - lld-9 \ - libc++-9-dev libc++abi-9-dev \ - autoconf \ - automake \ - bsdmainutils \ + clang-15 \ + clang-format-15 \ + clangd-15 \ cmake \ curl \ - libunwind8 \ - flex \ - bison \ - autopoint \ - gettext \ - g++-11 \ - gdb \ - git \ - glibc-source \ - htop \ - iputils-ping \ - libc6-dbg \ + dotnet-sdk-6.0 \ + fish \ + gcc-12 \ + gnupg \ liblttng-ust-dev \ - libncurses5-dev \ - libreadline-dev \ - libtool \ - libz-dev \ + libssh2-1-dev \ + libpcap0.8-dev \ locales \ - lttng-modules-dkms \ - lttng-tools \ - make \ - nano \ - net-tools \ ninja-build \ + nodejs \ + npm \ openssh-server \ - pkg-config \ - python-pip \ python3-pip \ python3-venv \ rapidjson-dev \ - screen \ - silversearcher-ag \ + software-properties-common \ + sqlite3 \ sudo \ - tcl \ - tcsh \ - texinfo \ - tmux \ - tree \ - unzip \ - uuid-dev \ + tar \ vim \ - tcpdump \ - valgrind \ - tzdata \ - nodejs \ - rsync \ - datamash \ - hwloc \ - strace \ - sshpass \ - libssh2-1-dev + wget -# temporary mono dependency until .net core 2.1 +# this is done here so that ccache sees clang++ installed, not clang-10 RUN \ apt-get update \ - && DEBIAN_FRONTEND=noninteractive apt-get install -y \ - mono-runtime \ - libfsharp-core4.3-cil \ - libmono-system-net-http4.0-cil \ - mono-reference-assemblies-4.0 \ - bash-completion \ - libpq-dev \ - libpqxx-dev \ - zip \ - parallel \ - patchelf - -# aws tools -RUN \ - apt-get update \ - && DEBIAN_FRONTEND=noninteractive apt-get install -y \ - awscli + && DEBIAN_FRONTEND=noninteractive apt-get install -y ccache # emumba deps RUN \ @@ -129,178 +48,32 @@ RUN \ graphviz \ libgtest-dev \ libssl-dev \ - libboost-filesystem-dev libboost-program-options-dev libboost-test-dev libboost-date-time-dev \ + libboost-filesystem-dev \ + libboost-program-options-dev \ + libboost-test-dev \ + libboost-date-time-dev \ zlib1g-dev \ - netcat \ + netcat-traditional \ net-tools \ gdb gdbserver \ pkg-config -# clean up apt lists -RUN rm -rf /var/lig/apt/lists/* - -# set default executables (update-alternatives) +# set up default executables (update-alternatives) RUN \ - update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-10 100; \ - update-alternatives --install /usr/bin/clang clang /usr/bin/clang-10 100; \ - update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-10 100; \ - update-alternatives --install /usr/bin/clangd clangd /usr/bin/clangd-10 100; \ - update-alternatives --config clang; \ - update-alternatives --config clang++; \ - update-alternatives --config clang-format; \ - update-alternatives --config clangd + update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-15 100; \ + update-alternatives --install /usr/bin/clang clang /usr/bin/clang-15 100; \ + update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-15 100; \ + update-alternatives --install /usr/bin/clangd clangd /usr/bin/clangd-15 100; \ + update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-12 100; \ + update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 100 -# this is done here so that ccache sees clang++ installed, not clang-10 +# dotnet sdk RUN \ - apt-get update \ - && DEBIAN_FRONTEND=noninteractive apt-get install -y ccache + wget https://download.visualstudio.microsoft.com/download/pr/8159607a-e686-4ead-ac99-b4c97290a5fd/ec6070b1b2cc0651ebe57cf1bd411315/dotnet-sdk-6.0.401-linux-x64.tar.gz \ + && tar -xf dotnet-sdk-6.0.401-linux-x64.tar.gz -C /usr/local/bin \ + && rm dotnet-sdk-6.0.401-linux-x64.tar.gz -# install python packages -RUN pip install --upgrade pip -RUN pip install pybars3 paramiko mkdocs -RUN pip3 install --upgrade pip -RUN pip3 install paramiko cryptography - -# fix locale so we can run tmux -RUN sed -i 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen -ENV LANG='en_US.utf8' LANGUAGE='en_US:en' LC_ALL='en_US.utf8' TZ='America/Chicago' -RUN locale-gen - -# expand libc sources -RUN cd /usr/src/glibc ; tar xvf glibc-*.tar.xz - -# setup ssh -RUN \ - mkdir /var/run/sshd \ - && echo 'root:root' \ - | chpasswd \ - && sed -i 's/#PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config \ - && sed -i 's/#PasswordAuthentication yes/PasswordAuthentication yes/' /etc/ssh/sshd_config - -# SSH login fix. Otherwise user is kicked off after login -RUN sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd - -# vim friendlier defaults -RUN echo 'set background=dark \n\ -set showcmd \n\ -set showmatch \n\ -set ignorecase \n\ -set smartcase \n\ -set incsearch \n\ -set autowrite \n\ -set hidden \n\ -set mouse= \n\ -imap jj \n\ -' >> /etc/vim/vimrc - -# region NETCORE -# for netcore2 reference https://hub.docker.com/_/microsoft-dotnet-core-sdk -# for netcore3 reference https://github.com/dotnet/dotnet-docker/blob/master/3.0/sdk/bionic/amd64/Dockerfile -# taken from https://dotnet.microsoft.com/learn/dotnet/hello-world-tutorial/install - -RUN \ - wget -q https://packages.microsoft.com/config/ubuntu/18.04/packages-microsoft-prod.deb -O packages-microsoft-prod.deb \ - && sudo dpkg -i packages-microsoft-prod.deb \ - && sudo apt-get update \ - && sudo apt-get install apt-transport-https -y \ - && sudo apt-get update \ - && sudo apt-get install dotnet-sdk-6.0 -y - -# Enable detection of running in a container -ENV DOTNET_RUNNING_IN_CONTAINER=true \ - # Enable correct mode for dotnet watch (only mode supported in a container) - DOTNET_USE_POLLING_FILE_WATCHER=true \ - # Skip extraction of XML docs - generally not useful within an image/container - helps performance - NUGET_XMLDOC_MODE=skip - -# Trigger first run experience by running arbitrary cmd -RUN dotnet help - -# Install PowerShell global tool -ENV POWERSHELL_VERSION=7.0.0-preview.4 \ - POWERSHELL_DISTRIBUTION_CHANNEL=PSDocker-DotnetCoreSDK-Ubuntu-18.04 - -RUN \ - curl -SL --output PowerShell.Linux.x64.$POWERSHELL_VERSION.nupkg https://pwshtool.blob.core.windows.net/tool/$POWERSHELL_VERSION/PowerShell.Linux.x64.$POWERSHELL_VERSION.nupkg \ - && powershell_sha512='0fb0167e13560371bffec38a4a87bf39377fa1a5cc39b3a078ddec8803212bede73e5821861036ba5c345bd55c74703134c9b55c49385f87dae9e2de9239f5d9' \ - && echo "$powershell_sha512 PowerShell.Linux.x64.$POWERSHELL_VERSION.nupkg" | sha512sum -c - \ - && mkdir -p /usr/share/powershell \ - && dotnet tool install --add-source / --tool-path /usr/share/powershell --version $POWERSHELL_VERSION PowerShell.Linux.x64 \ - && rm PowerShell.Linux.x64.$POWERSHELL_VERSION.nupkg \ - && ln -s /usr/share/powershell/pwsh /usr/bin/pwsh \ - # To reduce image size, remove the copy nupkg that nuget keeps. - && find /usr/share/powershell -print | grep -i '.*[.]nupkg$' | xargs rm - -#vsdbg for remote .net core debugging -RUN \ - wget https://aka.ms/getvsdbgsh -O - 2>/dev/null \ - | /bin/bash /dev/stdin -v latest -l ~/vsdbg - -# endregion NETCORE - -RUN DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1 dotnet tool install --tool-path /usr/local/bin paket -RUN chmod +X /usr/local/bin/paket - -COPY sqlite-version-3.26.0.zip extension-functions.c build_sqlite.sh / -RUN ./build_sqlite.sh && rm -rf /sqlite-version-3.26.0/ - - -#git-lfs -COPY git-lfs /usr/bin/ -RUN chmod +x /usr/bin/git-lfs && git lfs install - -##### Onload Stuff ##### - -# Onload version and its md5sum -ARG ONLOAD_VERSION="201811-u1" -ARG ONLOAD_MD5SUM="357e64862aa4145e49d218fd04e63407" - -# When ONLOAD_WITHZF is non-empty, the build includes Solarflare's TCPDirect library. -# Default is to exclude it. -ARG ONLOAD_WITHZF - -# 1) Install OpenOnload build dependencies -# 2) Download and verify OpenOnload from Solarflare's site -# 3) Extract, build, and install onload -# 4) Cleanup - -RUN \ - DEBIAN_FRONTEND=noninteractive apt-get update -y \ - && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ - ethtool \ - gcc-multilib \ - libc6-dev-i386 \ - libpcap0.8-dev \ - libtool-bin \ - && cd /tmp \ - ## && curl -fSL https://www.openonload.org/download/openonload-${ONLOAD_VERSION}.tgz -o /tmp/openonload-${ONLOAD_VERSION}.tgz \ - && curl -fSL https://github.com/rigtorp/openonload/archive/master.zip -o /tmp/openonload-${ONLOAD_VERSION}.zip \ - ## && echo "${ONLOAD_MD5SUM} openonload-${ONLOAD_VERSION}.tgz" | md5sum --check \ - ## && tar -zxf openonload-${ONLOAD_VERSION}.tgz \ - && unzip -a openonload-${ONLOAD_VERSION}.zip \ - ## && cd /tmp/openonload-${ONLOAD_VERSION} \ - && cd /tmp/openonload-master \ - && cd scripts \ - && ./onload_build --user \ - && ./onload_install --userfiles --nobuild \ - && cd /tmp \ - && rm -rf openonload* \ - && if [ -z ${ONLOAD_WITHZF} ] ; then rm -rf /usr/include/zf /usr/bin/zf_stackdump /usr/bin/zf_debug /usr/lib/x86_64-linux-gnu/zf /usr/lib/x86_64-linux-gnu/libonload_zf* ; fi \ - && DEBIAN_FRONTEND=noninteractive apt-get remove -y --purge \ - curl \ - patch \ - && DEBIAN_FRONTEND=noninteractive apt-get autoremove -y - -# One of the onload dependencies installed above causes apt to remove -# pkg-config (presumably due to a dependency conflict). Re-installing -# pkg-config seems to resolve the issue. -# TODO (when we have time): investigate this further - -RUN \ - apt-get update \ - && DEBIAN_FRONTEND=noninteractive apt-get install -y \ - pkg-config -######################## +RUN locale-gen en_US.UTF-8 ARG docker_user ARG docker_password @@ -319,4 +92,4 @@ RUN USER=$docker_user && \ printf "user: $USER\ngroup: $GROUP\n" > /etc/fixuid/config.yml USER $docker_user:$docker_user -ENTRYPOINT [fixuid] +ENTRYPOINT ["fixuid"] diff --git a/build.sh b/build.sh index d565420..2c8839e 100755 --- a/build.sh +++ b/build.sh @@ -1,8 +1,95 @@ #!/usr/bin/env bash +function print_help_and_exit() { + cat < in the image (defaults to the username in the image) + -i,--image The name to give the image + -h,--help Print this help message and exit + +EOF + exit $1 +} + + +declare docker_user="" +declare docker_pass="" +declare image_name="" + +while [[ $# -gt 0 ]]; do + case "$1" in + -i) + ;& + --image) + if [[ -z "$2" ]]; then + echo "Image (-i,--image) must be non-empty" + print_help_and_exit 1 + fi + image_name="$2" + shift 2 + ;; + -u) + ;& + --user) + if [[ -z "$2" ]]; then + echo "User (-u,--user) must be non-empty" + print_help_and_exit 1 + fi + docker_user="$2" + shift 2 + ;; + -p) + ;& + --pass) + if [[ -z "$2" ]]; then + echo "Password (-p,--pass) must be non-empty" + print_help_and_exit 1 + fi + docker_pass="$2" + shift 2 + ;; + -h) + ;& + --help) + print_help_and_exit 0 + ;; + *) + echo "Unexpected argument '$1'" + print_help_and_exit 1 + ;; + esac +done + +if [[ -z "$image_name" ]]; then + image_name="bluetradingsystems/spark-crypto-dev" + echo "Pulling image '$image_name' from Dockerhub (use --image to build a new image)" + docker pull "$image_name" + declare -n pull_status=$? + if [[ $pull_status -ne 0 ]]; then + echo "docker pull exited with code $pull_status" + fi + exit $? +fi + +if [[ -z "$docker_user" ]]; then + docker_user="$USER" +fi + +if [[ -z "$docker_pass" ]]; then + docker_pass="$docker_user" + echo "Giving user '$docker_user' a secure password: $docker_pass" +fi + +echo "Building image +image_name : $image_name +user : $docker_user" + docker build \ - --build-arg docker_user=$USER \ - --build-arg docker_user_id=$(id -u) \ - --build-arg docker_password=$USER \ - -t spark-crypto_18.04 \ - . + --build-arg docker_user=$docker_user \ + --build-arg docker_password=$docker_pass \ + -t "$image_name" \ + . diff --git a/docker_image_name.sh b/docker_image_name.sh deleted file mode 100755 index dc573a7..0000000 --- a/docker_image_name.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env bash - -echo "spark-crypto_18.04" diff --git a/start_docker.fish b/start_docker.fish new file mode 100755 index 0000000..2afcb3c --- /dev/null +++ b/start_docker.fish @@ -0,0 +1,100 @@ +#!/usr/bin/env fish + +function print_help_and_exit + echo "\ +Start a docker dev container +Usage: start_docker.fish [OPTIONS] + +Options: + -i,--image The image to use + -c,--container The name to give the container + -h,--help Print this help message and exit +" + return $argv +end + +argparse 'h/help' 'i/image' 'c/container' -- $argv + +if set -q _flag_help + print_help_and_exit 0 + return 0 +end + +set pos 1 + +# Set 'image_name' +set image_name "" +if set -q _flag_image + if test (count $argv) -ge $pos + set image_name "$argv[$pos]" + set pos (math $pos + 1) + if test "$image_name" = "" + echo "Image (-i,--image) must be non-empty" + print_help_and_exit 1 + return 1 + end + else + echo "Invalid arguments" + print_help_and_exit 1 + return 1 + end +else + set image_name "bluetradingsystems/spark-crypto-dev" +end + +# Set 'cont_name' +set cont_name "" +if set -q _flag_container + if test (count $argv) -ge $pos + set cont_name "$argv[$pos]" + set pos (math $pos + 1) + if test "$cont_name" = "" + echo "Container (-c,--container) must be non-empty" + print_help_and_exit 1 + return 1 + end + else + echo "Invalid arguments" + print_help_and_exit 1 + return 1 + end +end + +set home_src $HOME/.spark-dev/mounts/$cont_name +set home_tgt $HOME +set repo_src (git rev-parse --show-superproject-working-tree) +set repo_tgt $HOME/(basename $repo_src) +set working_dir $HOME + +# Make sure the $home_src dir exists before executing 'docker run' (otherwise +# Docker will automatically create the directory with root ownership). +mkdir -p $home_src + +echo "\ +Starting container +image_name : $image_name +cont_name : $cont_name +home_src : $home_src +home_tgt : $home_tgt +repo_src : $repo_src +repo_tgt : $repo_tgt +working dir : $working_dir" + +docker run \ + --net host \ + -h (hostname) \ + -e "TZ=America/New_York" \ + --cap-add=SYS_PTRACE \ + --security-opt seccomp=unconfined \ + --name $cont_name \ + -v $home_src:$home_tgt \ + -v $repo_src:$repo_tgt \ + -u (id -u $USER):(id -g $USER) \ + -w $working_dir \ + --shm-size=4024m \ + -t \ + -i \ + -d \ + -e DISPLAY="$DISPLAY" \ + "$image_name" \ + fixuid bash diff --git a/start_docker.sh b/start_docker.sh index fc5b455..7183545 100755 --- a/start_docker.sh +++ b/start_docker.sh @@ -1,40 +1,119 @@ #!/usr/bin/env bash -declare image_name="$1" -declare cont_name="$2" -declare this_dir="$(realpath $(dirname "$0"))" +function print_help_and_exit() { + cat < bash`. -declare working_dir=$repo_path_out +declare host_name="$(hostname)" +if [[ ! -z "$cont_name" ]]; then + host_name="$host_name.$cont_name" +fi + +if [[ -z "$mount_name" ]]; then + mount_name="$cont_name" +fi + +if [[ -z "$mount_name" && -z "$cont_name" ]]; then + echo "Please specify mount (-m,--mount) and/or container (-c,--container)" + print_help_and_exit 1 +fi + +declare home_src=$HOME/.spark-dev/mounts/$mount_name +declare home_tgt=$HOME +declare repo_src=$(git rev-parse --show-superproject-working-tree) +declare repo_tgt=$HOME/$(basename $repo_src) +declare working_dir=$HOME + +if [[ -e $home_src ]]; then + # Don't overwrite existing dir (might be mounted in another container) + echo "Mount dir '$home_src' already exists" + print_help_and_exit 1 +fi + +mkdir -p $home_src + +echo "Starting container +image_name : $image_name +cont_name : $cont_name +home_src : $home_src +home_tgt : $home_tgt +repo_src : $repo_src +repo_tgt : $repo_tgt +working dir : $working_dir" docker run \ - --rm \ --net host \ - -h $(hostname) \ + -h "$host_name" \ -e "TZ=America/New_York" \ --cap-add=SYS_PTRACE \ --security-opt seccomp=unconfined \ - --name $cont_name \ - -v $repo_path:$repo_path_out \ - -v $spark_data_path:$spark_data_path_out \ - -u $(id -u $USER) \ - -w $working_dir \ + --name "$cont_name" \ + -v "$home_src:$home_tgt" \ + -v "$repo_src:$repo_tgt" \ + -u $(id -u $USER):$(id -g $USER) \ + -w "$working_dir" \ --shm-size=4024m \ -t \ -i \