Upgrade ubuntu to latest LTS (22.10)

This is a big upgrade, from 18.04!
This commit is contained in:
John Colvin committed 2022-10-04 13:49:25 -05:00
1 parent 6537812262
commit 56c61231f3
5 files changed
+333 -297

No files matched your search

+36 -263
View File
@@ -1,120 +1,39 @@
FROM ubuntu:bionic
FROM ubuntu:22.10
RUN \
apt-get update \
&& DEBIAN_FRONTEND=noninteractive apt-get install -y \
gnupg \
software-properties-common \
wget
RUN \
echo "deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-9 main" >> /etc/apt/sources.list && \
wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key > llvm_key && apt-key add llvm_key
# Add Kitware repo to get latest cmake. The ccapi lib relies on
# 'add_compile_definitions', which was introduced in cmake 3.12.
RUN \
wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | apt-key add - && \
apt-add-repository 'deb https://apt.kitware.com/ubuntu/ bionic main' && \
apt update
# Add ubuntu-toolchain repo to get g++-11
RUN \
add-apt-repository ppa:ubuntu-toolchain-r/test && \
apt-get update
# add the latest node repository to apt sources (npm in the standard bionic apt
# repostitory depends on libssl1.0-dev, but emumba depends on libssl-dev, so a
# the latest npm must be installed to avoid this dependency conflict
RUN \
wget -O - https://deb.nodesource.com/setup_16.x > install_node.sh && chmod +x install_node.sh && ./install_node.sh
RUN \
apt-get update \
&& DEBIAN_FRONTEND=noninteractive apt-get install -y \
libllvm-10-ocaml-dev libllvm10 llvm-10 llvm-10-dev llvm-10-doc llvm-10-examples llvm-10-runtime \
clang-10 clang-tools-10 clang-10-doc libclang-common-10-dev libclang-10-dev libclang1-10 clang-format-10 python3-clang-10 clangd-10 \
libfuzzer-9-dev \
lldb-9 \
lld-9 \
libc++-9-dev libc++abi-9-dev \
autoconf \
automake \
bsdmainutils \
clang-15 \
clang-format-15 \
clangd-15 \
cmake \
curl \
libunwind8 \
flex \
bison \
autopoint \
gettext \
g++-11 \
gdb \
git \
glibc-source \
htop \
iputils-ping \
libc6-dbg \
dotnet-sdk-6.0 \
fish \
gcc-12 \
gnupg \
liblttng-ust-dev \
libncurses5-dev \
libreadline-dev \
libtool \
libz-dev \
libssh2-1-dev \
libpcap0.8-dev \
locales \
lttng-modules-dkms \
lttng-tools \
make \
nano \
net-tools \
ninja-build \
nodejs \
npm \
openssh-server \
pkg-config \
python-pip \
python3-pip \
python3-venv \
rapidjson-dev \
screen \
silversearcher-ag \
software-properties-common \
sqlite3 \
sudo \
tcl \
tcsh \
texinfo \
tmux \
tree \
unzip \
uuid-dev \
tar \
vim \
tcpdump \
valgrind \
tzdata \
nodejs \
rsync \
datamash \
hwloc \
strace \
sshpass \
libssh2-1-dev
wget
# temporary mono dependency until .net core 2.1
# this is done here so that ccache sees clang++ installed, not clang-10
RUN \
apt-get update \
&& DEBIAN_FRONTEND=noninteractive apt-get install -y \
mono-runtime \
libfsharp-core4.3-cil \
libmono-system-net-http4.0-cil \
mono-reference-assemblies-4.0 \
bash-completion \
libpq-dev \
libpqxx-dev \
zip \
parallel \
patchelf
# aws tools
RUN \
apt-get update \
&& DEBIAN_FRONTEND=noninteractive apt-get install -y \
awscli
&& DEBIAN_FRONTEND=noninteractive apt-get install -y ccache
# emumba deps
RUN \
@@ -129,178 +48,32 @@ RUN \
graphviz \
libgtest-dev \
libssl-dev \
libboost-filesystem-dev libboost-program-options-dev libboost-test-dev libboost-date-time-dev \
libboost-filesystem-dev \
libboost-program-options-dev \
libboost-test-dev \
libboost-date-time-dev \
zlib1g-dev \
netcat \
netcat-traditional \
net-tools \
gdb gdbserver \
pkg-config
# clean up apt lists
RUN rm -rf /var/lig/apt/lists/*
# set default executables (update-alternatives)
# set up default executables (update-alternatives)
RUN \
update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-10 100; \
update-alternatives --install /usr/bin/clang clang /usr/bin/clang-10 100; \
update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-10 100; \
update-alternatives --install /usr/bin/clangd clangd /usr/bin/clangd-10 100; \
update-alternatives --config clang; \
update-alternatives --config clang++; \
update-alternatives --config clang-format; \
update-alternatives --config clangd
update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-15 100; \
update-alternatives --install /usr/bin/clang clang /usr/bin/clang-15 100; \
update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-15 100; \
update-alternatives --install /usr/bin/clangd clangd /usr/bin/clangd-15 100; \
update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-12 100; \
update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 100
# this is done here so that ccache sees clang++ installed, not clang-10
# dotnet sdk
RUN \
apt-get update \
&& DEBIAN_FRONTEND=noninteractive apt-get install -y ccache
wget https://download.visualstudio.microsoft.com/download/pr/8159607a-e686-4ead-ac99-b4c97290a5fd/ec6070b1b2cc0651ebe57cf1bd411315/dotnet-sdk-6.0.401-linux-x64.tar.gz \
&& tar -xf dotnet-sdk-6.0.401-linux-x64.tar.gz -C /usr/local/bin \
&& rm dotnet-sdk-6.0.401-linux-x64.tar.gz
# install python packages
RUN pip install --upgrade pip
RUN pip install pybars3 paramiko mkdocs
RUN pip3 install --upgrade pip
RUN pip3 install paramiko cryptography
# fix locale so we can run tmux
RUN sed -i 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen
ENV LANG='en_US.utf8' LANGUAGE='en_US:en' LC_ALL='en_US.utf8' TZ='America/Chicago'
RUN locale-gen
# expand libc sources
RUN cd /usr/src/glibc ; tar xvf glibc-*.tar.xz
# setup ssh
RUN \
mkdir /var/run/sshd \
&& echo 'root:root' \
| chpasswd \
&& sed -i 's/#PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config \
&& sed -i 's/#PasswordAuthentication yes/PasswordAuthentication yes/' /etc/ssh/sshd_config
# SSH login fix. Otherwise user is kicked off after login
RUN sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd
# vim friendlier defaults
RUN echo 'set background=dark \n\
set showcmd \n\
set showmatch \n\
set ignorecase \n\
set smartcase \n\
set incsearch \n\
set autowrite \n\
set hidden \n\
set mouse= \n\
imap jj <Esc> \n\
' >> /etc/vim/vimrc
# region NETCORE
# for netcore2 reference https://hub.docker.com/_/microsoft-dotnet-core-sdk
# for netcore3 reference https://github.com/dotnet/dotnet-docker/blob/master/3.0/sdk/bionic/amd64/Dockerfile
# taken from https://dotnet.microsoft.com/learn/dotnet/hello-world-tutorial/install
RUN \
wget -q https://packages.microsoft.com/config/ubuntu/18.04/packages-microsoft-prod.deb -O packages-microsoft-prod.deb \
&& sudo dpkg -i packages-microsoft-prod.deb \
&& sudo apt-get update \
&& sudo apt-get install apt-transport-https -y \
&& sudo apt-get update \
&& sudo apt-get install dotnet-sdk-6.0 -y
# Enable detection of running in a container
ENV DOTNET_RUNNING_IN_CONTAINER=true \
# Enable correct mode for dotnet watch (only mode supported in a container)
DOTNET_USE_POLLING_FILE_WATCHER=true \
# Skip extraction of XML docs - generally not useful within an image/container - helps performance
NUGET_XMLDOC_MODE=skip
# Trigger first run experience by running arbitrary cmd
RUN dotnet help
# Install PowerShell global tool
ENV POWERSHELL_VERSION=7.0.0-preview.4 \
POWERSHELL_DISTRIBUTION_CHANNEL=PSDocker-DotnetCoreSDK-Ubuntu-18.04
RUN \
curl -SL --output PowerShell.Linux.x64.$POWERSHELL_VERSION.nupkg https://pwshtool.blob.core.windows.net/tool/$POWERSHELL_VERSION/PowerShell.Linux.x64.$POWERSHELL_VERSION.nupkg \
&& powershell_sha512='0fb0167e13560371bffec38a4a87bf39377fa1a5cc39b3a078ddec8803212bede73e5821861036ba5c345bd55c74703134c9b55c49385f87dae9e2de9239f5d9' \
&& echo "$powershell_sha512 PowerShell.Linux.x64.$POWERSHELL_VERSION.nupkg" | sha512sum -c - \
&& mkdir -p /usr/share/powershell \
&& dotnet tool install --add-source / --tool-path /usr/share/powershell --version $POWERSHELL_VERSION PowerShell.Linux.x64 \
&& rm PowerShell.Linux.x64.$POWERSHELL_VERSION.nupkg \
&& ln -s /usr/share/powershell/pwsh /usr/bin/pwsh \
# To reduce image size, remove the copy nupkg that nuget keeps.
&& find /usr/share/powershell -print | grep -i '.*[.]nupkg$' | xargs rm
#vsdbg for remote .net core debugging
RUN \
wget https://aka.ms/getvsdbgsh -O - 2>/dev/null \
| /bin/bash /dev/stdin -v latest -l ~/vsdbg
# endregion NETCORE
RUN DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1 dotnet tool install --tool-path /usr/local/bin paket
RUN chmod +X /usr/local/bin/paket
COPY sqlite-version-3.26.0.zip extension-functions.c build_sqlite.sh /
RUN ./build_sqlite.sh && rm -rf /sqlite-version-3.26.0/
#git-lfs
COPY git-lfs /usr/bin/
RUN chmod +x /usr/bin/git-lfs && git lfs install
##### Onload Stuff #####
# Onload version and its md5sum
ARG ONLOAD_VERSION="201811-u1"
ARG ONLOAD_MD5SUM="357e64862aa4145e49d218fd04e63407"
# When ONLOAD_WITHZF is non-empty, the build includes Solarflare's TCPDirect library.
# Default is to exclude it.
ARG ONLOAD_WITHZF
# 1) Install OpenOnload build dependencies
# 2) Download and verify OpenOnload from Solarflare's site
# 3) Extract, build, and install onload
# 4) Cleanup
RUN \
DEBIAN_FRONTEND=noninteractive apt-get update -y \
&& DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
ethtool \
gcc-multilib \
libc6-dev-i386 \
libpcap0.8-dev \
libtool-bin \
&& cd /tmp \
## && curl -fSL https://www.openonload.org/download/openonload-${ONLOAD_VERSION}.tgz -o /tmp/openonload-${ONLOAD_VERSION}.tgz \
&& curl -fSL https://github.com/rigtorp/openonload/archive/master.zip -o /tmp/openonload-${ONLOAD_VERSION}.zip \
## && echo "${ONLOAD_MD5SUM} openonload-${ONLOAD_VERSION}.tgz" | md5sum --check \
## && tar -zxf openonload-${ONLOAD_VERSION}.tgz \
&& unzip -a openonload-${ONLOAD_VERSION}.zip \
## && cd /tmp/openonload-${ONLOAD_VERSION} \
&& cd /tmp/openonload-master \
&& cd scripts \
&& ./onload_build --user \
&& ./onload_install --userfiles --nobuild \
&& cd /tmp \
&& rm -rf openonload* \
&& if [ -z ${ONLOAD_WITHZF} ] ; then rm -rf /usr/include/zf /usr/bin/zf_stackdump /usr/bin/zf_debug /usr/lib/x86_64-linux-gnu/zf /usr/lib/x86_64-linux-gnu/libonload_zf* ; fi \
&& DEBIAN_FRONTEND=noninteractive apt-get remove -y --purge \
curl \
patch \
&& DEBIAN_FRONTEND=noninteractive apt-get autoremove -y
# One of the onload dependencies installed above causes apt to remove
# pkg-config (presumably due to a dependency conflict). Re-installing
# pkg-config seems to resolve the issue.
# TODO (when we have time): investigate this further
RUN \
apt-get update \
&& DEBIAN_FRONTEND=noninteractive apt-get install -y \
pkg-config
########################
RUN locale-gen en_US.UTF-8
ARG docker_user
ARG docker_password
@@ -319,4 +92,4 @@ RUN USER=$docker_user && \
printf "user: $USER\ngroup: $GROUP\n" > /etc/fixuid/config.yml
USER $docker_user:$docker_user
ENTRYPOINT [fixuid]
ENTRYPOINT ["fixuid"]
+92 -5
View File
@@ -1,8 +1,95 @@
#!/usr/bin/env bash
function print_help_and_exit() {
cat <<EOF
Build the (dev) docker image
Usage: build.sh [OPTIONS]
Options:
-u,--user The user (name) to add to the image (defaults to the current user)
-p,--pass The password for the <--user> in the image (defaults to the username in the image)
-i,--image The name to give the image
-h,--help Print this help message and exit
EOF
exit $1
}
declare docker_user=""
declare docker_pass=""
declare image_name=""
while [[ $# -gt 0 ]]; do
case "$1" in
-i)
;&
--image)
if [[ -z "$2" ]]; then
echo "Image (-i,--image) must be non-empty"
print_help_and_exit 1
fi
image_name="$2"
shift 2
;;
-u)
;&
--user)
if [[ -z "$2" ]]; then
echo "User (-u,--user) must be non-empty"
print_help_and_exit 1
fi
docker_user="$2"
shift 2
;;
-p)
;&
--pass)
if [[ -z "$2" ]]; then
echo "Password (-p,--pass) must be non-empty"
print_help_and_exit 1
fi
docker_pass="$2"
shift 2
;;
-h)
;&
--help)
print_help_and_exit 0
;;
*)
echo "Unexpected argument '$1'"
print_help_and_exit 1
;;
esac
done
if [[ -z "$image_name" ]]; then
image_name="bluetradingsystems/spark-crypto-dev"
echo "Pulling image '$image_name' from Dockerhub (use --image <name> to build a new image)"
docker pull "$image_name"
declare -n pull_status=$?
if [[ $pull_status -ne 0 ]]; then
echo "docker pull exited with code $pull_status"
fi
exit $?
fi
if [[ -z "$docker_user" ]]; then
docker_user="$USER"
fi
if [[ -z "$docker_pass" ]]; then
docker_pass="$docker_user"
echo "Giving user '$docker_user' a secure password: $docker_pass"
fi
echo "Building image
image_name : $image_name
user : $docker_user"
docker build \
--build-arg docker_user=$USER \
--build-arg docker_user_id=$(id -u) \
--build-arg docker_password=$USER \
-t spark-crypto_18.04 \
.
--build-arg docker_user=$docker_user \
--build-arg docker_password=$docker_pass \
-t "$image_name" \
.
-3
View File
@@ -1,3 +0,0 @@
#!/usr/bin/env bash
echo "spark-crypto_18.04"
+100
View File
@@ -0,0 +1,100 @@
#!/usr/bin/env fish
function print_help_and_exit
echo "\
Start a docker dev container
Usage: start_docker.fish [OPTIONS]
Options:
-i,--image The image to use
-c,--container The name to give the container
-h,--help Print this help message and exit
"
return $argv
end
argparse 'h/help' 'i/image' 'c/container' -- $argv
if set -q _flag_help
print_help_and_exit 0
return 0
end
set pos 1
# Set 'image_name'
set image_name ""
if set -q _flag_image
if test (count $argv) -ge $pos
set image_name "$argv[$pos]"
set pos (math $pos + 1)
if test "$image_name" = ""
echo "Image (-i,--image) must be non-empty"
print_help_and_exit 1
return 1
end
else
echo "Invalid arguments"
print_help_and_exit 1
return 1
end
else
set image_name "bluetradingsystems/spark-crypto-dev"
end
# Set 'cont_name'
set cont_name ""
if set -q _flag_container
if test (count $argv) -ge $pos
set cont_name "$argv[$pos]"
set pos (math $pos + 1)
if test "$cont_name" = ""
echo "Container (-c,--container) must be non-empty"
print_help_and_exit 1
return 1
end
else
echo "Invalid arguments"
print_help_and_exit 1
return 1
end
end
set home_src $HOME/.spark-dev/mounts/$cont_name
set home_tgt $HOME
set repo_src (git rev-parse --show-superproject-working-tree)
set repo_tgt $HOME/(basename $repo_src)
set working_dir $HOME
# Make sure the $home_src dir exists before executing 'docker run' (otherwise
# Docker will automatically create the directory with root ownership).
mkdir -p $home_src
echo "\
Starting container
image_name : $image_name
cont_name : $cont_name
home_src : $home_src
home_tgt : $home_tgt
repo_src : $repo_src
repo_tgt : $repo_tgt
working dir : $working_dir"
docker run \
--net host \
-h (hostname) \
-e "TZ=America/New_York" \
--cap-add=SYS_PTRACE \
--security-opt seccomp=unconfined \
--name $cont_name \
-v $home_src:$home_tgt \
-v $repo_src:$repo_tgt \
-u (id -u $USER):(id -g $USER) \
-w $working_dir \
--shm-size=4024m \
-t \
-i \
-d \
-e DISPLAY="$DISPLAY" \
"$image_name" \
fixuid bash
+105 -26
View File
@@ -1,40 +1,119 @@
#!/usr/bin/env bash
declare image_name="$1"
declare cont_name="$2"
declare this_dir="$(realpath $(dirname "$0"))"
function print_help_and_exit() {
cat <<EOF
Start a docker dev container
Usage: start_docker.sh [OPTIONS]
if [ $# -lt 2 ]; then
if [ $# -lt 1 ]; then
declare image_name_cmd="$this_dir/docker_image_name.sh"
image_name="$(eval $image_name_cmd)"
fi
declare cont_name_cmd="$this_dir/docker_container_name.sh"
cont_name="$(eval $cont_name_cmd $image_name)"
Options:
-i,--image The image to instantiate
-c,--container The name to give the container
-m,--mount The name for the folder (on the host) that is mounted to container's 'home' dir
-h,--help Print this help message and exit
EOF
exit $1
}
declare image_name=""
declare cont_name=""
declare mount_name=""
while [[ $# -gt 0 ]]; do
case "$1" in
-i)
;&
--image)
if [[ -z "$2" ]]; then
echo "Image (-i,--image) must be non-empty"
fi
image_name="$2"
shift 2
;;
-c)
;&
--container)
if [[ -z "$2" ]]; then
echo "Container (-c,--container) must be non-empty"
fi
cont_name="$2"
shift 2
;;
-m)
;&
--mount)
if [[ -z "$2" ]]; then
echo "Mount (-m,--mount) must be non-empty"
fi
mount_name="$2"
shift 2
;;
-h)
;&
--help)
print_help_and_exit 0
;;
*)
echo "Unexpected argument '$1'"
print_help_and_exit 1
;;
esac
done
if [[ -z "$image_name" ]]; then
image_name="bluetradingsystems/spark-crypto-dev"
fi
# The root repo path (`repo_path`) is mounted to `repo_path_out`. The latter is
# stored in a separate variable for convenience.
declare repo_path="$(cd .. && git rev-parse --show-toplevel && cd $this_dir)"
declare repo_path_out=$repo_path
declare spark_data_path="$HOME/sparkdata"
declare spark_data_path_out=$spark_data_path
# `working_dir` is the starting dir for all docker shells launched via
# `docker exec -it <container> bash`.
declare working_dir=$repo_path_out
declare host_name="$(hostname)"
if [[ ! -z "$cont_name" ]]; then
host_name="$host_name.$cont_name"
fi
if [[ -z "$mount_name" ]]; then
mount_name="$cont_name"
fi
if [[ -z "$mount_name" && -z "$cont_name" ]]; then
echo "Please specify mount (-m,--mount) and/or container (-c,--container)"
print_help_and_exit 1
fi
declare home_src=$HOME/.spark-dev/mounts/$mount_name
declare home_tgt=$HOME
declare repo_src=$(git rev-parse --show-superproject-working-tree)
declare repo_tgt=$HOME/$(basename $repo_src)
declare working_dir=$HOME
if [[ -e $home_src ]]; then
# Don't overwrite existing dir (might be mounted in another container)
echo "Mount dir '$home_src' already exists"
print_help_and_exit 1
fi
mkdir -p $home_src
echo "Starting container
image_name : $image_name
cont_name : $cont_name
home_src : $home_src
home_tgt : $home_tgt
repo_src : $repo_src
repo_tgt : $repo_tgt
working dir : $working_dir"
docker run \
--rm \
--net host \
-h $(hostname) \
-h "$host_name" \
-e "TZ=America/New_York" \
--cap-add=SYS_PTRACE \
--security-opt seccomp=unconfined \
--name $cont_name \
-v $repo_path:$repo_path_out \
-v $spark_data_path:$spark_data_path_out \
-u $(id -u $USER) \
-w $working_dir \
--name "$cont_name" \
-v "$home_src:$home_tgt" \
-v "$repo_src:$repo_tgt" \
-u $(id -u $USER):$(id -g $USER) \
-w "$working_dir" \
--shm-size=4024m \
-t \
-i \