Upgrade gcc to v11

It's unnecessary, and furthermore, the sudo cmd isn't installed until after cmake

Dockerfile

@@ -1,97 +1,182 @@
 FROM ubuntu:bionic
 
-RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
- autoconf \
- automake \
- bsdmainutils \
- ccache \
- clang-format \
- clang \
- cmake \
- curl \
- libunwind8 \
- flex \
- bison \
- autopoint \
- gettext \
- g++ \
- gcc \
- gdb \
- git \
- glibc-source \
- htop  \
- iputils-ping \
- libc6-dbg \
- libclang-dev \
- liblttng-ust-dev \
- libncurses5-dev \
- libreadline-dev \
- libtool \
- libz-dev \
- llvm \
- locales \
- lttng-modules-dkms \
- lttng-tools \
- make \
- nano \
- net-tools \
- ninja-build \
- openssh-server \
- pkg-config \
- python-pip \
- screen \
- silversearcher-ag \
- sudo \
- tcl \
- tcsh \
- texinfo \
- tmux \
- tree \
- unzip \
- uuid-dev \
- vim \
- tcpdump \
- valgrind \
- tzdata \
- npm \
- nodejs \
- rsync \
- datamash \
- hwloc \
- strace \
- sshpass \
- libssh2-1-dev \
- # temporary mono dependency until .net core 2.1
- mono-runtime \
- libfsharp-core4.3-cil \
- libmono-system-net-http4.0-cil \
- mono-reference-assemblies-4.0 \
- bash-completion \
- libpq-dev \
- libpqxx-dev \
- zip \
- && rm -rf /var/lig/apt/lists/*
+RUN \
+    apt-get update \
+    && DEBIAN_FRONTEND=noninteractive apt-get install -y \
+    gnupg \
+    software-properties-common \
+    wget 
 
+RUN \
+    echo "deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-9 main" >> /etc/apt/sources.list && \
+    wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key > llvm_key && apt-key add llvm_key
 
-run pip install pybars3 paramiko mkdocs
-run npm install -g npx
+# Add Kitware repo to get latest cmake.  The ccapi lib relies on
+# 'add_compile_definitions', which was introduced in cmake 3.12.
+RUN \
+    wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | apt-key add - && \
+    apt-add-repository 'deb https://apt.kitware.com/ubuntu/ bionic main' && \
+    apt update
+
+# Add ubuntu-toolchain repo to get g++-11
+RUN \
+    add-apt-repository ppa:ubuntu-toolchain-r/test && \
+    apt-get update
+
+# add the latest node repository to apt sources (npm in the standard bionic apt
+# repostitory depends on libssl1.0-dev, but emumba depends on libssl-dev, so a
+# the latest npm must be installed to avoid this dependency conflict
+RUN \
+    wget -O - https://deb.nodesource.com/setup_16.x > install_node.sh && chmod +x install_node.sh && ./install_node.sh
+
+RUN \
+    apt-get update \
+    && DEBIAN_FRONTEND=noninteractive apt-get install -y \
+    libllvm-10-ocaml-dev libllvm10 llvm-10 llvm-10-dev llvm-10-doc llvm-10-examples llvm-10-runtime \
+    clang-10 clang-tools-10 clang-10-doc libclang-common-10-dev libclang-10-dev libclang1-10 clang-format-10 python3-clang-10 clangd-10 \
+    libfuzzer-9-dev \
+    lldb-9 \
+    lld-9 \
+    libc++-9-dev libc++abi-9-dev \
+    autoconf \
+    automake \
+    bsdmainutils \
+    cmake \
+    curl \
+    libunwind8 \
+    flex \
+    bison \
+    autopoint \
+    gettext \
+    g++-11 \
+    gdb \
+    git \
+    glibc-source \
+    htop  \
+    iputils-ping \
+    libc6-dbg \
+    liblttng-ust-dev \
+    libncurses5-dev \
+    libreadline-dev \
+    libtool \
+    libz-dev \
+    locales \
+    lttng-modules-dkms \
+    lttng-tools \
+    make \
+    nano \
+    net-tools \
+    ninja-build \
+    openssh-server \
+    pkg-config \
+    python-pip \
+    python3-pip \
+    python3-venv \
+    rapidjson-dev \
+    screen \
+    silversearcher-ag \
+    sudo \
+    tcl \
+    tcsh \
+    texinfo \
+    tmux \
+    tree \
+    unzip \
+    uuid-dev \
+    vim \
+    tcpdump \
+    valgrind \
+    tzdata \
+    nodejs \
+    rsync \
+    datamash \
+    hwloc \
+    strace \
+    sshpass \
+    libssh2-1-dev
+
+# temporary mono dependency until .net core 2.1
+RUN \
+    apt-get update \
+    && DEBIAN_FRONTEND=noninteractive apt-get install -y \
+    mono-runtime \
+    libfsharp-core4.3-cil \
+    libmono-system-net-http4.0-cil \
+    mono-reference-assemblies-4.0 \
+    bash-completion \
+    libpq-dev \
+    libpqxx-dev \
+    zip \
+    parallel \
+    patchelf
+
+# aws tools
+RUN \
+    apt-get update \
+    && DEBIAN_FRONTEND=noninteractive apt-get install -y \
+    awscli
+
+# emumba deps
+RUN \
+    apt-get update \
+    && DEBIAN_FRONTEND=noninteractive apt-get install -y \
+    tzdata \
+    build-essential \
+    cmake-curses-gui \
+    git-all \
+    doxygen \
+    python3-sphinx \
+    graphviz \
+    libgtest-dev \
+    libssl-dev \
+    libboost-filesystem-dev libboost-program-options-dev libboost-test-dev libboost-date-time-dev \
+    zlib1g-dev \
+    netcat \
+    net-tools \
+    gdb gdbserver \
+    pkg-config
+
+# clean up apt lists
+RUN rm -rf /var/lig/apt/lists/*
+
+# set default executables (update-alternatives)
+RUN \
+    update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-10 100; \
+    update-alternatives --install /usr/bin/clang clang /usr/bin/clang-10 100; \
+    update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-10 100; \
+    update-alternatives --install /usr/bin/clangd clangd /usr/bin/clangd-10 100; \
+    update-alternatives --config clang; \
+    update-alternatives --config clang++; \
+    update-alternatives --config clang-format; \
+    update-alternatives --config clangd
+
+# this is done here so that ccache sees clang++ installed, not clang-10
+RUN \
+    apt-get update \
+    && DEBIAN_FRONTEND=noninteractive apt-get install -y ccache 
+
+# install python packages
+RUN pip install --upgrade pip
+RUN pip install pybars3 paramiko mkdocs
+RUN pip3 install --upgrade pip
+RUN pip3 install paramiko cryptography
 
 # fix locale so we can run tmux
-run sed -i 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen
+RUN sed -i 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen
 ENV LANG='en_US.utf8' LANGUAGE='en_US:en' LC_ALL='en_US.utf8' TZ='America/Chicago'
-run locale-gen
+RUN locale-gen
 
 # expand libc sources
-run cd /usr/src/glibc ; tar xvf glibc-*.tar.xz
-
-# build cgdb (the version shipped with ubuntu is too old)
-copy cgdb-0.7.0.tar.gz /externals/
-run cd /externals/ && tar -xvzf cgdb-0.7.0.tar.gz && cd cgdb-0.7.0 && ./configure && make -j$(nproc) && make install
+RUN cd /usr/src/glibc ; tar xvf glibc-*.tar.xz
 
 # setup ssh
-RUN mkdir /var/run/sshd && echo 'root:root' | chpasswd && \
-sed -i 's/#PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config && \
-sed -i 's/#PasswordAuthentication yes/PasswordAuthentication yes/' /etc/ssh/sshd_config
+RUN \
+    mkdir /var/run/sshd \
+    && echo 'root:root' \
+    | chpasswd \
+    && sed -i 's/#PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config \
+    && sed -i 's/#PasswordAuthentication yes/PasswordAuthentication yes/' /etc/ssh/sshd_config
+
 # SSH login fix. Otherwise user is kicked off after login
 RUN sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd
 
@@ -113,13 +198,13 @@ imap jj <Esc> \n\
 # for netcore3 reference https://github.com/dotnet/dotnet-docker/blob/master/3.0/sdk/bionic/amd64/Dockerfile
 # taken from https://dotnet.microsoft.com/learn/dotnet/hello-world-tutorial/install
 
-RUN wget -q https://packages.microsoft.com/config/ubuntu/18.04/packages-microsoft-prod.deb -O packages-microsoft-prod.deb \
+RUN \
+    wget -q https://packages.microsoft.com/config/ubuntu/18.04/packages-microsoft-prod.deb -O packages-microsoft-prod.deb \
     && sudo dpkg -i packages-microsoft-prod.deb \
     && sudo apt-get update \
     && sudo apt-get install apt-transport-https -y \
     && sudo apt-get update \
-    && sudo apt-get install dotnet-sdk-2.1 -y \
-    && sudo apt-get install dotnet-sdk-3.0 -y     
+    && sudo apt-get install dotnet-sdk-6.0 -y
 
 # Enable detection of running in a container
 ENV DOTNET_RUNNING_IN_CONTAINER=true \
@@ -127,7 +212,7 @@ ENV DOTNET_RUNNING_IN_CONTAINER=true \
     DOTNET_USE_POLLING_FILE_WATCHER=true \
     # Skip extraction of XML docs - generally not useful within an image/container - helps performance
     NUGET_XMLDOC_MODE=skip
-	
+
 # Trigger first run experience by running arbitrary cmd
 RUN dotnet help
 
@@ -135,7 +220,8 @@ RUN dotnet help
 ENV POWERSHELL_VERSION=7.0.0-preview.4 \
     POWERSHELL_DISTRIBUTION_CHANNEL=PSDocker-DotnetCoreSDK-Ubuntu-18.04
 
-RUN curl -SL --output PowerShell.Linux.x64.$POWERSHELL_VERSION.nupkg https://pwshtool.blob.core.windows.net/tool/$POWERSHELL_VERSION/PowerShell.Linux.x64.$POWERSHELL_VERSION.nupkg \
+RUN \
+    curl -SL --output PowerShell.Linux.x64.$POWERSHELL_VERSION.nupkg https://pwshtool.blob.core.windows.net/tool/$POWERSHELL_VERSION/PowerShell.Linux.x64.$POWERSHELL_VERSION.nupkg \
     && powershell_sha512='0fb0167e13560371bffec38a4a87bf39377fa1a5cc39b3a078ddec8803212bede73e5821861036ba5c345bd55c74703134c9b55c49385f87dae9e2de9239f5d9' \
     && echo "$powershell_sha512  PowerShell.Linux.x64.$POWERSHELL_VERSION.nupkg" | sha512sum -c - \
     && mkdir -p /usr/share/powershell \
@@ -146,36 +232,22 @@ RUN curl -SL --output PowerShell.Linux.x64.$POWERSHELL_VERSION.nupkg https://pws
     && find /usr/share/powershell -print | grep -i '.*[.]nupkg$' | xargs rm	
 
 #vsdbg for remote .net core debugging	
-RUN wget https://aka.ms/getvsdbgsh -O - 2>/dev/null | /bin/sh /dev/stdin -v latest -l ~/vsdbg
-	
+RUN \
+    wget https://aka.ms/getvsdbgsh -O - 2>/dev/null \
+    | /bin/bash /dev/stdin -v latest -l ~/vsdbg
+
 # endregion NETCORE
 
-run DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1 dotnet tool install --tool-path /usr/local/bin paket 
-run chmod +X /usr/local/bin/paket
+RUN DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1 dotnet tool install --tool-path /usr/local/bin paket 
+RUN chmod +X /usr/local/bin/paket
 
-copy sqlite-version-3.26.0.zip extension-functions.c build_sqlite.sh /
-run ./build_sqlite.sh && rm -rf /sqlite-version-3.26.0/
+COPY sqlite-version-3.26.0.zip extension-functions.c build_sqlite.sh /
+RUN ./build_sqlite.sh && rm -rf /sqlite-version-3.26.0/
 
 
 #git-lfs
-copy git-lfs /usr/bin/
-run chmod +x /usr/bin/git-lfs && git lfs install
-
-#postgres
-RUN sudo apt-get update \
-	&& wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add - \
-	&& echo "deb http://apt.postgresql.org/pub/repos/apt/ `lsb_release -cs`-pgdg main" |sudo tee  /etc/apt/sources.list.d/pgdg.list \
-	&& sudo apt-get update \
-	&& sudo apt-get install -y postgresql-12 postgresql-client-12
-	
-RUN mkdir -p /var/lib/postgresql/ssl ;\
-    chown postgres:postgres /var/lib/postgresql/ssl ;\
-    cp /etc/ssl/certs/ssl-cert-snakeoil.pem /var/lib/postgresql/ssl/ ;\
-    cp /etc/ssl/private/ssl-cert-snakeoil.key /var/lib/postgresql/ssl/ ;\
-    chown -R postgres:postgres /var/lib/postgresql/ssl/ssl-cert-snakeoil.* ;\
-    chmod 600 /var/lib/postgresql/ssl/* ;\
-    chmod 700 /var/lib/postgresql/ssl
-#	
+COPY git-lfs /usr/bin/
+RUN chmod +x /usr/bin/git-lfs && git lfs install
 
 ##### Onload Stuff #####
 
@@ -195,25 +267,45 @@ ARG ONLOAD_WITHZF
 RUN \
     DEBIAN_FRONTEND=noninteractive apt-get update -y \
     && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
-        ethtool \
-        gcc-multilib \
-        libc6-dev-i386 \
-        libpcap0.8-dev \
-        libtool-bin \
+    ethtool \
+    gcc-multilib \
+    libc6-dev-i386 \
+    libpcap0.8-dev \
+    libtool-bin \
     && cd /tmp \
-    && curl -fSL https://www.openonload.org/download/openonload-${ONLOAD_VERSION}.tgz -o /tmp/openonload-${ONLOAD_VERSION}.tgz \
-    && echo "${ONLOAD_MD5SUM} openonload-${ONLOAD_VERSION}.tgz" | md5sum --check \
-    && tar -zxf openonload-${ONLOAD_VERSION}.tgz \
-    && cd /tmp/openonload-${ONLOAD_VERSION} \
+    ##    && curl -fSL https://www.openonload.org/download/openonload-${ONLOAD_VERSION}.tgz -o /tmp/openonload-${ONLOAD_VERSION}.tgz \
+    && curl -fSL https://github.com/rigtorp/openonload/archive/master.zip -o /tmp/openonload-${ONLOAD_VERSION}.zip \
+    ##    && echo "${ONLOAD_MD5SUM} openonload-${ONLOAD_VERSION}.tgz" | md5sum --check \
+    ##    && tar -zxf openonload-${ONLOAD_VERSION}.tgz \
+    && unzip -a openonload-${ONLOAD_VERSION}.zip \
+    ##    && cd /tmp/openonload-${ONLOAD_VERSION} \
+    && cd /tmp/openonload-master \
     && cd scripts \
     && ./onload_build --user \
     && ./onload_install --userfiles --nobuild \
     && cd /tmp \
-    && rm -rf openonload-${ONLOAD_VERSION}.tgz openonload-${ONLOAD_VERSION} \
+    && rm -rf openonload* \
     && if [ -z ${ONLOAD_WITHZF} ] ; then rm -rf /usr/include/zf /usr/bin/zf_stackdump /usr/bin/zf_debug /usr/lib/x86_64-linux-gnu/zf /usr/lib/x86_64-linux-gnu/libonload_zf* ; fi \
     && DEBIAN_FRONTEND=noninteractive apt-get remove -y --purge \
-        curl \
-        patch \
+    curl \
+    patch \
     && DEBIAN_FRONTEND=noninteractive apt-get autoremove -y
 
+# One of the onload dependencies installed above causes apt to remove
+# pkg-config (presumably due to a dependency conflict).  Re-installing
+# pkg-config seems to resolve the issue.
+# TODO (when we have time): investigate this further
+
+RUN \
+    apt-get update \
+    && DEBIAN_FRONTEND=noninteractive apt-get install -y \
+    pkg-config
 ########################
+
+RUN USER=docker && \
+    GROUP=docker && \
+    curl -SsL https://github.com/boxboat/fixuid/releases/download/v0.5.1/fixuid-0.5.1-linux-amd64.tar.gz | tar -C /usr/local/bin -xzf - && \
+    chown root:root /usr/local/bin/fixuid && \
+    chmod 4755 /usr/local/bin/fixuid && \
+    mkdir -p /etc/fixuid && \
+    printf "user: $USER\ngroup: $GROUP\n" > /etc/fixuid/config.yml

build.sh

@@ -1 +1,8 @@
-docker build -t bluetradingsystems/linux_docker .
+#!/usr/bin/env bash
+
+docker build \
+   --build-arg docker_user=$USER \
+   --build-arg docker_user_id=$(id -u) \
+   --build-arg docker_password=$USER \
+   -t spark-crypto_18.04 \
+   .

docker_container_name.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+if [ $# -gt 1 ]; then
+  echo $2
+  exit 0
+fi
+
+declare image_name="$1"
+if [ $# -eq 0 ]; then
+  declare image_name_cmd="$(realpath $(dirname "$0"))/docker_image_name.fish"
+  image_name="$(eval $image_name_cmd)"
+fi
+echo "${USER}s_$image_name"

docker_image_name.sh

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+echo "spark-crypto_18.04"

start_docker.sh

@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+
+declare image_name="$1"
+declare cont_name="$2"
+declare this_dir="$(realpath $(dirname "$0"))"
+
+if [ $# -lt 2 ]; then
+  if [ $# -lt 1 ]; then
+    declare image_name_cmd="$this_dir/docker_image_name.sh"
+    image_name="$(eval $image_name_cmd)"
+  fi
+  declare cont_name_cmd="$this_dir/docker_container_name.sh"
+  cont_name="$(eval $cont_name_cmd $image_name)"
+fi
+
+# The root repo path (`repo_path`) is mounted to `repo_path_out`. The latter is
+# stored in a separate variable for convenience.
+declare repo_path="$(cd .. && git rev-parse --show-toplevel && cd $this_dir)"
+declare repo_path_out=$repo_path
+declare spark_data_path="$HOME/sparkdata"
+declare spark_data_path_out=$spark_data_path
+# `working_dir` is the starting dir for all docker shells launched via
+# `docker exec -it <container> bash`.
+declare working_dir=$repo_path_out
+
+docker run \
+  --rm \
+  --net host \
+  -h $(hostname) \
+  -e "TZ=America/New_York" \
+  --cap-add=SYS_PTRACE \
+  --security-opt seccomp=unconfined \
+  --name $cont_name \
+  -v $repo_path:$repo_path_out \
+  -v $spark_data_path:$spark_data_path_out \
+  -u $(id -u $USER) \
+  -w $working_dir \
+  --shm-size=4024m \
+  -t \
+  -i \
+  -d \
+  -e DISPLAY="$DISPLAY" \
+  "$image_name" \
+  bash