20 Commits
Author SHA1 Message Date
John Colvin 6537812262 Upgrade gcc to v11 2022-09-21 00:09:12 -05:00
John Colvin 823bdd85e4 Upgrade libllvm pkgs to v10 (same as clang) 2022-09-21 00:09:12 -05:00
johncolvin-og 22f9b7bc9a Upgrade pip before installing py pkgs 2022-09-21 00:09:12 -05:00
johncolvin-og 73e1a3a596 Install cmake without 'sudo' cmd
It's unnecessary, and furthermore, the sudo cmd isn't installed until after cmake
2022-09-21 00:09:12 -05:00
johncolvin-og 03c84954ff Install software-properties-common before running its cmds
e.g., apt-add-repository, apt-key
2022-09-21 00:09:12 -05:00
John Colvin 42916b0142 Use fixuid to resolve user perm conflicts in mounted dirs
It matches the user permissions in the container to those in the host. This has
clear advantages to our previous solution, which matched the permissions in the
image. One immediate benefit is that the jenkins server-build node can pull the
image from Docker Hub, rather than build it on the host (for each build, to
ensure the image is up-to-date).
2022-09-21 00:09:12 -05:00
John Colvin 191d74b404 Add ccapi_cpp deps to docker
* Add rapidjson-dev
* Add Kitware repo to get latest cmake (the ccapi_cpp lib relies on
  'add_compile_definitions', which was introduced in cmake 3.12)
2022-06-08 14:40:44 -04:00
John Colvin c3630582cc Upgrade dotnet-sdk to 6.0 2022-04-21 13:41:25 -04:00
John Colvin 05967d0f18 Mount host's sparkdata dir in docker 2021-12-29 21:58:21 -05:00
John Colvin eee74d82b2 fix: Change update-alternative calls to use clang-10 (not 9)
This fixes the docker image build (all clang tools were recently
upgraded from v9 to 10)
2021-12-29 16:05:19 -05:00
John Colvin 4744ddc633 Add script to start docker container 2021-12-29 16:05:19 -05:00
John Colvin 713cdf165e Upgrade compiler to clang-10 2021-12-11 15:50:41 -05:00
John Colvin 8fbd35654e Re-install pkg-config after installing onload
This is a workaround for what appears to be a dependency conflict
related to onload
2021-12-10 14:33:32 -06:00
John Colvin f2e2963489 Add python3-venv to docker 2021-12-10 14:30:32 -06:00
John Colvin 54c0c16d3b Install latest npm (outside of apt repo)
This resolves a dependency conflict between libssl-dev and the npm
available in apt standard repos
2021-11-06 10:42:50 -05:00
John Colvin 4000d62710 Add emumba deps to Dockerfile 2021-11-02 13:54:36 -05:00
John Colvin 117143d9e1 style: Clean up Dockerfile
* Add labels
* Indent command chains
2021-11-02 12:54:26 -05:00
John Colvin f1d398ba81 Add docker_user and password args to Dockerfile
Symlinking /etc/passwd files leads to invalid cross-device link errors
when running apt commands.

Internally, apt uses dpkg, which adds a user "irc" when performing apt
installs/upgrades.  This subprocess leads to the error.

The solution implemented here creates a user with the same name and id
as the current user.  This gives both users the the same file permissions,
without having to symlink /etc/passwd.
2021-10-10 14:20:07 -04:00
John Colvin fc316243c2 Use /bin/bash instead of /bin/sh
The docker build was failing because /bin/sh doesn't exist.
Did the base image change?
2021-10-10 14:03:53 -04:00
John Colvin dd4ca356d4 Get rid of rtags and cgdb
I don't think anyone actually uses this
2021-10-10 14:00:25 -04:00
5 changed files with 276 additions and 150 deletions

No files matched your search

+208 -149
View File
@@ -1,134 +1,182 @@
FROM ubuntu:bionic
RUN apt-get update && \
DEBIAN_FRONTEND=noninteractive apt-get install -y \
gnupg \
software-properties-common \
wget
RUN \
apt-get update \
&& DEBIAN_FRONTEND=noninteractive apt-get install -y \
gnupg \
software-properties-common \
wget
RUN \
echo "deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-9 main" >> /etc/apt/sources.list && \
wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key > llvm_key && apt-key add llvm_key
echo "deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-9 main" >> /etc/apt/sources.list && \
wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key > llvm_key && apt-key add llvm_key
# Add Kitware repo to get latest cmake
# Add Kitware repo to get latest cmake. The ccapi lib relies on
# 'add_compile_definitions', which was introduced in cmake 3.12.
RUN \
wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | apt-key add - && \
apt-add-repository 'deb https://apt.kitware.com/ubuntu/ bionic main' && \
apt update
# Add ubuntu-toolchain repo to get g++-11
RUN \
apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
libllvm-9-ocaml-dev libllvm9 llvm-9 llvm-9-dev llvm-9-doc llvm-9-examples llvm-9-runtime \
clang-9 clang-tools-9 clang-9-doc libclang-common-9-dev libclang-9-dev libclang1-9 clang-format-9 python-clang-9 clangd-9 \
libfuzzer-9-dev \
lldb-9 \
lld-9 \
libc++-9-dev libc++abi-9-dev \
autoconf \
automake \
bsdmainutils \
cmake \
curl \
libunwind8 \
flex \
bison \
autopoint \
gettext \
g++ \
gcc \
gdb \
git \
glibc-source \
htop \
iputils-ping \
libc6-dbg \
liblttng-ust-dev \
libncurses5-dev \
libreadline-dev \
libtool \
libz-dev \
locales \
lttng-modules-dkms \
lttng-tools \
make \
nano \
net-tools \
ninja-build \
openssh-server \
pkg-config \
python-pip \
python3-pip \
python3-venv \
screen \
silversearcher-ag \
sudo \
tcl \
tcsh \
texinfo \
tmux \
tree \
unzip \
uuid-dev \
vim \
tcpdump \
valgrind \
tzdata \
npm \
nodejs \
rsync \
datamash \
hwloc \
strace \
sshpass \
libssh2-1-dev \
# temporary mono dependency until .net core 2.1
mono-runtime \
libfsharp-core4.3-cil \
libmono-system-net-http4.0-cil \
mono-reference-assemblies-4.0 \
bash-completion \
libpq-dev \
libpqxx-dev \
zip \
parallel \
patchelf \
add-apt-repository ppa:ubuntu-toolchain-r/test && \
apt-get update
# add the latest node repository to apt sources (npm in the standard bionic apt
# repostitory depends on libssl1.0-dev, but emumba depends on libssl-dev, so a
# the latest npm must be installed to avoid this dependency conflict
RUN \
wget -O - https://deb.nodesource.com/setup_16.x > install_node.sh && chmod +x install_node.sh && ./install_node.sh
RUN \
apt-get update \
&& DEBIAN_FRONTEND=noninteractive apt-get install -y \
libllvm-10-ocaml-dev libllvm10 llvm-10 llvm-10-dev llvm-10-doc llvm-10-examples llvm-10-runtime \
clang-10 clang-tools-10 clang-10-doc libclang-common-10-dev libclang-10-dev libclang1-10 clang-format-10 python3-clang-10 clangd-10 \
libfuzzer-9-dev \
lldb-9 \
lld-9 \
libc++-9-dev libc++abi-9-dev \
autoconf \
automake \
bsdmainutils \
cmake \
curl \
libunwind8 \
flex \
bison \
autopoint \
gettext \
g++-11 \
gdb \
git \
glibc-source \
htop \
iputils-ping \
libc6-dbg \
liblttng-ust-dev \
libncurses5-dev \
libreadline-dev \
libtool \
libz-dev \
locales \
lttng-modules-dkms \
lttng-tools \
make \
nano \
net-tools \
ninja-build \
openssh-server \
pkg-config \
python-pip \
python3-pip \
python3-venv \
rapidjson-dev \
screen \
silversearcher-ag \
sudo \
tcl \
tcsh \
texinfo \
tmux \
tree \
unzip \
uuid-dev \
vim \
tcpdump \
valgrind \
tzdata \
nodejs \
rsync \
datamash \
hwloc \
strace \
sshpass \
libssh2-1-dev
# temporary mono dependency until .net core 2.1
RUN \
apt-get update \
&& DEBIAN_FRONTEND=noninteractive apt-get install -y \
mono-runtime \
libfsharp-core4.3-cil \
libmono-system-net-http4.0-cil \
mono-reference-assemblies-4.0 \
bash-completion \
libpq-dev \
libpqxx-dev \
zip \
parallel \
patchelf
# aws tools
awscli \
&& rm -rf /var/lig/apt/lists/*
RUN \
apt-get update \
&& DEBIAN_FRONTEND=noninteractive apt-get install -y \
awscli
# emumba deps
RUN \
apt-get update \
&& DEBIAN_FRONTEND=noninteractive apt-get install -y \
tzdata \
build-essential \
cmake-curses-gui \
git-all \
doxygen \
python3-sphinx \
graphviz \
libgtest-dev \
libssl-dev \
libboost-filesystem-dev libboost-program-options-dev libboost-test-dev libboost-date-time-dev \
zlib1g-dev \
netcat \
net-tools \
gdb gdbserver \
pkg-config
run \
update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-9 100; \
update-alternatives --install /usr/bin/clang clang /usr/bin/clang-9 100; \
update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-9 100; \
update-alternatives --install /usr/bin/clangd clangd /usr/bin/clangd-9 100; \
update-alternatives --config clang; \
update-alternatives --config clang++; \
update-alternatives --config clang-format; \
update-alternatives --config clangd
# clean up apt lists
RUN rm -rf /var/lig/apt/lists/*
# this is done here so that ccache sees clang++ installed, not clang-9
RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y ccache
# set default executables (update-alternatives)
RUN \
update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-10 100; \
update-alternatives --install /usr/bin/clang clang /usr/bin/clang-10 100; \
update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-10 100; \
update-alternatives --install /usr/bin/clangd clangd /usr/bin/clangd-10 100; \
update-alternatives --config clang; \
update-alternatives --config clang++; \
update-alternatives --config clang-format; \
update-alternatives --config clangd
# this is done here so that ccache sees clang++ installed, not clang-10
RUN \
apt-get update \
&& DEBIAN_FRONTEND=noninteractive apt-get install -y ccache
run pip install pybars3 paramiko mkdocs
run pip3 install paramiko cryptography
run npm install -g npx
# install python packages
RUN pip install --upgrade pip
RUN pip install pybars3 paramiko mkdocs
RUN pip3 install --upgrade pip
RUN pip3 install paramiko cryptography
# fix locale so we can run tmux
run sed -i 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen
RUN sed -i 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen
ENV LANG='en_US.utf8' LANGUAGE='en_US:en' LC_ALL='en_US.utf8' TZ='America/Chicago'
run locale-gen
RUN locale-gen
# expand libc sources
run cd /usr/src/glibc ; tar xvf glibc-*.tar.xz
RUN cd /usr/src/glibc ; tar xvf glibc-*.tar.xz
# setup ssh
RUN mkdir /var/run/sshd && echo 'root:root' | chpasswd && \
sed -i 's/#PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config && \
sed -i 's/#PasswordAuthentication yes/PasswordAuthentication yes/' /etc/ssh/sshd_config
RUN \
mkdir /var/run/sshd \
&& echo 'root:root' \
| chpasswd \
&& sed -i 's/#PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config \
&& sed -i 's/#PasswordAuthentication yes/PasswordAuthentication yes/' /etc/ssh/sshd_config
# SSH login fix. Otherwise user is kicked off after login
RUN sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd
@@ -150,13 +198,13 @@ imap jj <Esc> \n\
# for netcore3 reference https://github.com/dotnet/dotnet-docker/blob/master/3.0/sdk/bionic/amd64/Dockerfile
# taken from https://dotnet.microsoft.com/learn/dotnet/hello-world-tutorial/install
RUN wget -q https://packages.microsoft.com/config/ubuntu/18.04/packages-microsoft-prod.deb -O packages-microsoft-prod.deb \
RUN \
wget -q https://packages.microsoft.com/config/ubuntu/18.04/packages-microsoft-prod.deb -O packages-microsoft-prod.deb \
&& sudo dpkg -i packages-microsoft-prod.deb \
&& sudo apt-get update \
&& sudo apt-get install apt-transport-https -y \
&& sudo apt-get update \
&& sudo apt-get install dotnet-sdk-2.1 -y \
&& sudo apt-get install dotnet-sdk-3.1 -y
&& sudo apt-get install dotnet-sdk-6.0 -y
# Enable detection of running in a container
ENV DOTNET_RUNNING_IN_CONTAINER=true \
@@ -164,7 +212,7 @@ ENV DOTNET_RUNNING_IN_CONTAINER=true \
DOTNET_USE_POLLING_FILE_WATCHER=true \
# Skip extraction of XML docs - generally not useful within an image/container - helps performance
NUGET_XMLDOC_MODE=skip
# Trigger first run experience by running arbitrary cmd
RUN dotnet help
@@ -172,7 +220,8 @@ RUN dotnet help
ENV POWERSHELL_VERSION=7.0.0-preview.4 \
POWERSHELL_DISTRIBUTION_CHANNEL=PSDocker-DotnetCoreSDK-Ubuntu-18.04
RUN curl -SL --output PowerShell.Linux.x64.$POWERSHELL_VERSION.nupkg https://pwshtool.blob.core.windows.net/tool/$POWERSHELL_VERSION/PowerShell.Linux.x64.$POWERSHELL_VERSION.nupkg \
RUN \
curl -SL --output PowerShell.Linux.x64.$POWERSHELL_VERSION.nupkg https://pwshtool.blob.core.windows.net/tool/$POWERSHELL_VERSION/PowerShell.Linux.x64.$POWERSHELL_VERSION.nupkg \
&& powershell_sha512='0fb0167e13560371bffec38a4a87bf39377fa1a5cc39b3a078ddec8803212bede73e5821861036ba5c345bd55c74703134c9b55c49385f87dae9e2de9239f5d9' \
&& echo "$powershell_sha512 PowerShell.Linux.x64.$POWERSHELL_VERSION.nupkg" | sha512sum -c - \
&& mkdir -p /usr/share/powershell \
@@ -183,20 +232,22 @@ RUN curl -SL --output PowerShell.Linux.x64.$POWERSHELL_VERSION.nupkg https://pws
&& find /usr/share/powershell -print | grep -i '.*[.]nupkg$' | xargs rm
#vsdbg for remote .net core debugging
RUN wget https://aka.ms/getvsdbgsh -O - 2>/dev/null | /bin/sh /dev/stdin -v latest -l ~/vsdbg
RUN \
wget https://aka.ms/getvsdbgsh -O - 2>/dev/null \
| /bin/bash /dev/stdin -v latest -l ~/vsdbg
# endregion NETCORE
run DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1 dotnet tool install --tool-path /usr/local/bin paket
run chmod +X /usr/local/bin/paket
RUN DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1 dotnet tool install --tool-path /usr/local/bin paket
RUN chmod +X /usr/local/bin/paket
copy sqlite-version-3.26.0.zip extension-functions.c build_sqlite.sh /
run ./build_sqlite.sh && rm -rf /sqlite-version-3.26.0/
COPY sqlite-version-3.26.0.zip extension-functions.c build_sqlite.sh /
RUN ./build_sqlite.sh && rm -rf /sqlite-version-3.26.0/
#git-lfs
copy git-lfs /usr/bin/
run chmod +x /usr/bin/git-lfs && git lfs install
COPY git-lfs /usr/bin/
RUN chmod +x /usr/bin/git-lfs && git lfs install
##### Onload Stuff #####
@@ -216,18 +267,18 @@ ARG ONLOAD_WITHZF
RUN \
DEBIAN_FRONTEND=noninteractive apt-get update -y \
&& DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
ethtool \
gcc-multilib \
libc6-dev-i386 \
libpcap0.8-dev \
libtool-bin \
ethtool \
gcc-multilib \
libc6-dev-i386 \
libpcap0.8-dev \
libtool-bin \
&& cd /tmp \
## && curl -fSL https://www.openonload.org/download/openonload-${ONLOAD_VERSION}.tgz -o /tmp/openonload-${ONLOAD_VERSION}.tgz \
## && curl -fSL https://www.openonload.org/download/openonload-${ONLOAD_VERSION}.tgz -o /tmp/openonload-${ONLOAD_VERSION}.tgz \
&& curl -fSL https://github.com/rigtorp/openonload/archive/master.zip -o /tmp/openonload-${ONLOAD_VERSION}.zip \
## && echo "${ONLOAD_MD5SUM} openonload-${ONLOAD_VERSION}.tgz" | md5sum --check \
## && tar -zxf openonload-${ONLOAD_VERSION}.tgz \
## && echo "${ONLOAD_MD5SUM} openonload-${ONLOAD_VERSION}.tgz" | md5sum --check \
## && tar -zxf openonload-${ONLOAD_VERSION}.tgz \
&& unzip -a openonload-${ONLOAD_VERSION}.zip \
## && cd /tmp/openonload-${ONLOAD_VERSION} \
## && cd /tmp/openonload-${ONLOAD_VERSION} \
&& cd /tmp/openonload-master \
&& cd scripts \
&& ./onload_build --user \
@@ -236,28 +287,36 @@ RUN \
&& rm -rf openonload* \
&& if [ -z ${ONLOAD_WITHZF} ] ; then rm -rf /usr/include/zf /usr/bin/zf_stackdump /usr/bin/zf_debug /usr/lib/x86_64-linux-gnu/zf /usr/lib/x86_64-linux-gnu/libonload_zf* ; fi \
&& DEBIAN_FRONTEND=noninteractive apt-get remove -y --purge \
curl \
patch \
curl \
patch \
&& DEBIAN_FRONTEND=noninteractive apt-get autoremove -y
# One of the onload dependencies installed above causes apt to remove
# pkg-config (presumably due to a dependency conflict). Re-installing
# pkg-config seems to resolve the issue.
# TODO (when we have time): investigate this further
RUN \
apt-get update \
&& DEBIAN_FRONTEND=noninteractive apt-get install -y \
pkg-config
########################
# build rtags
run \
git clone --recursive https://github.com/Andersbakken/rtags.git && \
cd rtags && \
git submodule update && \
git checkout v2.38 && \
cmake . -DCMAKE_EXPORT_COMPILE_COMMANDS=1 . && \
make -j$(nproc) install && \
cd .. && \
rm -rf rtags
ARG docker_user
ARG docker_password
# build cgdb (the version shipped with ubuntu is too old)
run \
git clone https://github.com/cgdb/cgdb.git && \
cd cgdb && \
git checkout v0.7.1 && \
./autogen.sh && \
./configure && make -j$(nproc) && \
make install
RUN \
useradd -m $docker_user \
&& echo "${docker_user}:${docker_password}" | chpasswd \
&& adduser $docker_user sudo
RUN USER=$docker_user && \
GROUP=$docker_user && \
curl -SsL https://github.com/boxboat/fixuid/releases/download/v0.5.1/fixuid-0.5.1-linux-amd64.tar.gz | tar -C /usr/local/bin -xzf - && \
chown root:root /usr/local/bin/fixuid && \
chmod 4755 /usr/local/bin/fixuid && \
mkdir -p /etc/fixuid && \
printf "user: $USER\ngroup: $GROUP\n" > /etc/fixuid/config.yml
USER $docker_user:$docker_user
ENTRYPOINT [fixuid]
+8 -1
View File
@@ -1 +1,8 @@
docker build -t bluetradingsystems/linux_docker .
#!/usr/bin/env bash
docker build \
--build-arg docker_user=$USER \
--build-arg docker_user_id=$(id -u) \
--build-arg docker_password=$USER \
-t spark-crypto_18.04 \
.
+13
View File
@@ -0,0 +1,13 @@
#!/usr/bin/env bash
if [ $# -gt 1 ]; then
echo $2
exit 0
fi
declare image_name="$1"
if [ $# -eq 0 ]; then
declare image_name_cmd="$(realpath $(dirname "$0"))/docker_image_name.fish"
image_name="$(eval $image_name_cmd)"
fi
echo "${USER}s_$image_name"
+3
View File
@@ -0,0 +1,3 @@
#!/usr/bin/env bash
echo "spark-crypto_18.04"
+44
View File
@@ -0,0 +1,44 @@
#!/usr/bin/env bash
declare image_name="$1"
declare cont_name="$2"
declare this_dir="$(realpath $(dirname "$0"))"
if [ $# -lt 2 ]; then
if [ $# -lt 1 ]; then
declare image_name_cmd="$this_dir/docker_image_name.sh"
image_name="$(eval $image_name_cmd)"
fi
declare cont_name_cmd="$this_dir/docker_container_name.sh"
cont_name="$(eval $cont_name_cmd $image_name)"
fi
# The root repo path (`repo_path`) is mounted to `repo_path_out`. The latter is
# stored in a separate variable for convenience.
declare repo_path="$(cd .. && git rev-parse --show-toplevel && cd $this_dir)"
declare repo_path_out=$repo_path
declare spark_data_path="$HOME/sparkdata"
declare spark_data_path_out=$spark_data_path
# `working_dir` is the starting dir for all docker shells launched via
# `docker exec -it <container> bash`.
declare working_dir=$repo_path_out
docker run \
--rm \
--net host \
-h $(hostname) \
-e "TZ=America/New_York" \
--cap-add=SYS_PTRACE \
--security-opt seccomp=unconfined \
--name $cont_name \
-v $repo_path:$repo_path_out \
-v $spark_data_path:$spark_data_path_out \
-u $(id -u $USER) \
-w $working_dir \
--shm-size=4024m \
-t \
-i \
-d \
-e DISPLAY="$DISPLAY" \
"$image_name" \
bash