2 Commits
Author SHA1 Message Date
John Colvin 26de8f7b3b Install latest cmake
The build script, build_sparkd.sh, uses the '-S' option, which wasn't
introduced until v3.13
2022-10-01 20:46:46 -04:00
John Colvin 30707087b8 Install python3-venv 2022-10-01 20:12:07 -04:00
5 changed files with 150 additions and 276 deletions

No files matched your search

+149 -208
View File
@@ -1,182 +1,134 @@
FROM ubuntu:bionic FROM ubuntu:bionic
RUN \
apt-get update \ RUN apt-get update && \
&& DEBIAN_FRONTEND=noninteractive apt-get install -y \ DEBIAN_FRONTEND=noninteractive apt-get install -y \
gnupg \ gnupg \
software-properties-common \ software-properties-common \
wget wget
RUN \ RUN \
echo "deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-9 main" >> /etc/apt/sources.list && \ echo "deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-9 main" >> /etc/apt/sources.list && \
wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key > llvm_key && apt-key add llvm_key wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key > llvm_key && apt-key add llvm_key
# Add Kitware repo to get latest cmake. The ccapi lib relies on # Add Kitware repo to get latest cmake
# 'add_compile_definitions', which was introduced in cmake 3.12.
RUN \ RUN \
wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | apt-key add - && \ wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | apt-key add - && \
apt-add-repository 'deb https://apt.kitware.com/ubuntu/ bionic main' && \ apt-add-repository 'deb https://apt.kitware.com/ubuntu/ bionic main' && \
apt update apt update
# Add ubuntu-toolchain repo to get g++-11
RUN \ RUN \
add-apt-repository ppa:ubuntu-toolchain-r/test && \ apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
apt-get update libllvm-9-ocaml-dev libllvm9 llvm-9 llvm-9-dev llvm-9-doc llvm-9-examples llvm-9-runtime \
clang-9 clang-tools-9 clang-9-doc libclang-common-9-dev libclang-9-dev libclang1-9 clang-format-9 python-clang-9 clangd-9 \
# add the latest node repository to apt sources (npm in the standard bionic apt libfuzzer-9-dev \
# repostitory depends on libssl1.0-dev, but emumba depends on libssl-dev, so a lldb-9 \
# the latest npm must be installed to avoid this dependency conflict lld-9 \
RUN \ libc++-9-dev libc++abi-9-dev \
wget -O - https://deb.nodesource.com/setup_16.x > install_node.sh && chmod +x install_node.sh && ./install_node.sh autoconf \
automake \
RUN \ bsdmainutils \
apt-get update \ cmake \
&& DEBIAN_FRONTEND=noninteractive apt-get install -y \ curl \
libllvm-10-ocaml-dev libllvm10 llvm-10 llvm-10-dev llvm-10-doc llvm-10-examples llvm-10-runtime \ libunwind8 \
clang-10 clang-tools-10 clang-10-doc libclang-common-10-dev libclang-10-dev libclang1-10 clang-format-10 python3-clang-10 clangd-10 \ flex \
libfuzzer-9-dev \ bison \
lldb-9 \ autopoint \
lld-9 \ gettext \
libc++-9-dev libc++abi-9-dev \ g++ \
autoconf \ gcc \
automake \ gdb \
bsdmainutils \ git \
cmake \ glibc-source \
curl \ htop \
libunwind8 \ iputils-ping \
flex \ libc6-dbg \
bison \ liblttng-ust-dev \
autopoint \ libncurses5-dev \
gettext \ libreadline-dev \
g++-11 \ libtool \
gdb \ libz-dev \
git \ locales \
glibc-source \ lttng-modules-dkms \
htop \ lttng-tools \
iputils-ping \ make \
libc6-dbg \ nano \
liblttng-ust-dev \ net-tools \
libncurses5-dev \ ninja-build \
libreadline-dev \ openssh-server \
libtool \ pkg-config \
libz-dev \ python-pip \
locales \ python3-pip \
lttng-modules-dkms \ python3-venv \
lttng-tools \ screen \
make \ silversearcher-ag \
nano \ sudo \
net-tools \ tcl \
ninja-build \ tcsh \
openssh-server \ texinfo \
pkg-config \ tmux \
python-pip \ tree \
python3-pip \ unzip \
python3-venv \ uuid-dev \
rapidjson-dev \ vim \
screen \ tcpdump \
silversearcher-ag \ valgrind \
sudo \ tzdata \
tcl \ npm \
tcsh \ nodejs \
texinfo \ rsync \
tmux \ datamash \
tree \ hwloc \
unzip \ strace \
uuid-dev \ sshpass \
vim \ libssh2-1-dev \
tcpdump \ # temporary mono dependency until .net core 2.1
valgrind \ mono-runtime \
tzdata \ libfsharp-core4.3-cil \
nodejs \ libmono-system-net-http4.0-cil \
rsync \ mono-reference-assemblies-4.0 \
datamash \ bash-completion \
hwloc \ libpq-dev \
strace \ libpqxx-dev \
sshpass \ zip \
libssh2-1-dev parallel \
patchelf \
# temporary mono dependency until .net core 2.1
RUN \
apt-get update \
&& DEBIAN_FRONTEND=noninteractive apt-get install -y \
mono-runtime \
libfsharp-core4.3-cil \
libmono-system-net-http4.0-cil \
mono-reference-assemblies-4.0 \
bash-completion \
libpq-dev \
libpqxx-dev \
zip \
parallel \
patchelf
# aws tools # aws tools
RUN \ awscli \
apt-get update \ && rm -rf /var/lig/apt/lists/*
&& DEBIAN_FRONTEND=noninteractive apt-get install -y \
awscli
# emumba deps
RUN \
apt-get update \
&& DEBIAN_FRONTEND=noninteractive apt-get install -y \
tzdata \
build-essential \
cmake-curses-gui \
git-all \
doxygen \
python3-sphinx \
graphviz \
libgtest-dev \
libssl-dev \
libboost-filesystem-dev libboost-program-options-dev libboost-test-dev libboost-date-time-dev \
zlib1g-dev \
netcat \
net-tools \
gdb gdbserver \
pkg-config
# clean up apt lists run \
RUN rm -rf /var/lig/apt/lists/* update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-9 100; \
update-alternatives --install /usr/bin/clang clang /usr/bin/clang-9 100; \
update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-9 100; \
update-alternatives --install /usr/bin/clangd clangd /usr/bin/clangd-9 100; \
update-alternatives --config clang; \
update-alternatives --config clang++; \
update-alternatives --config clang-format; \
update-alternatives --config clangd
# set default executables (update-alternatives) # this is done here so that ccache sees clang++ installed, not clang-9
RUN \ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y ccache
update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-10 100; \
update-alternatives --install /usr/bin/clang clang /usr/bin/clang-10 100; \
update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-10 100; \
update-alternatives --install /usr/bin/clangd clangd /usr/bin/clangd-10 100; \
update-alternatives --config clang; \
update-alternatives --config clang++; \
update-alternatives --config clang-format; \
update-alternatives --config clangd
# this is done here so that ccache sees clang++ installed, not clang-10
RUN \
apt-get update \
&& DEBIAN_FRONTEND=noninteractive apt-get install -y ccache
# install python packages run pip install pybars3 paramiko mkdocs
RUN pip install --upgrade pip run pip3 install paramiko cryptography
RUN pip install pybars3 paramiko mkdocs run npm install -g npx
RUN pip3 install --upgrade pip
RUN pip3 install paramiko cryptography
# fix locale so we can run tmux # fix locale so we can run tmux
RUN sed -i 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen run sed -i 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen
ENV LANG='en_US.utf8' LANGUAGE='en_US:en' LC_ALL='en_US.utf8' TZ='America/Chicago' ENV LANG='en_US.utf8' LANGUAGE='en_US:en' LC_ALL='en_US.utf8' TZ='America/Chicago'
RUN locale-gen run locale-gen
# expand libc sources # expand libc sources
RUN cd /usr/src/glibc ; tar xvf glibc-*.tar.xz run cd /usr/src/glibc ; tar xvf glibc-*.tar.xz
# setup ssh # setup ssh
RUN \ RUN mkdir /var/run/sshd && echo 'root:root' | chpasswd && \
mkdir /var/run/sshd \ sed -i 's/#PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config && \
&& echo 'root:root' \ sed -i 's/#PasswordAuthentication yes/PasswordAuthentication yes/' /etc/ssh/sshd_config
| chpasswd \
&& sed -i 's/#PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config \
&& sed -i 's/#PasswordAuthentication yes/PasswordAuthentication yes/' /etc/ssh/sshd_config
# SSH login fix. Otherwise user is kicked off after login # SSH login fix. Otherwise user is kicked off after login
RUN sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd RUN sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd
@@ -198,13 +150,13 @@ imap jj <Esc> \n\
# for netcore3 reference https://github.com/dotnet/dotnet-docker/blob/master/3.0/sdk/bionic/amd64/Dockerfile # for netcore3 reference https://github.com/dotnet/dotnet-docker/blob/master/3.0/sdk/bionic/amd64/Dockerfile
# taken from https://dotnet.microsoft.com/learn/dotnet/hello-world-tutorial/install # taken from https://dotnet.microsoft.com/learn/dotnet/hello-world-tutorial/install
RUN \ RUN wget -q https://packages.microsoft.com/config/ubuntu/18.04/packages-microsoft-prod.deb -O packages-microsoft-prod.deb \
wget -q https://packages.microsoft.com/config/ubuntu/18.04/packages-microsoft-prod.deb -O packages-microsoft-prod.deb \
&& sudo dpkg -i packages-microsoft-prod.deb \ && sudo dpkg -i packages-microsoft-prod.deb \
&& sudo apt-get update \ && sudo apt-get update \
&& sudo apt-get install apt-transport-https -y \ && sudo apt-get install apt-transport-https -y \
&& sudo apt-get update \ && sudo apt-get update \
&& sudo apt-get install dotnet-sdk-6.0 -y && sudo apt-get install dotnet-sdk-2.1 -y \
&& sudo apt-get install dotnet-sdk-3.1 -y
# Enable detection of running in a container # Enable detection of running in a container
ENV DOTNET_RUNNING_IN_CONTAINER=true \ ENV DOTNET_RUNNING_IN_CONTAINER=true \
@@ -212,7 +164,7 @@ ENV DOTNET_RUNNING_IN_CONTAINER=true \
DOTNET_USE_POLLING_FILE_WATCHER=true \ DOTNET_USE_POLLING_FILE_WATCHER=true \
# Skip extraction of XML docs - generally not useful within an image/container - helps performance # Skip extraction of XML docs - generally not useful within an image/container - helps performance
NUGET_XMLDOC_MODE=skip NUGET_XMLDOC_MODE=skip
# Trigger first run experience by running arbitrary cmd # Trigger first run experience by running arbitrary cmd
RUN dotnet help RUN dotnet help
@@ -220,8 +172,7 @@ RUN dotnet help
ENV POWERSHELL_VERSION=7.0.0-preview.4 \ ENV POWERSHELL_VERSION=7.0.0-preview.4 \
POWERSHELL_DISTRIBUTION_CHANNEL=PSDocker-DotnetCoreSDK-Ubuntu-18.04 POWERSHELL_DISTRIBUTION_CHANNEL=PSDocker-DotnetCoreSDK-Ubuntu-18.04
RUN \ RUN curl -SL --output PowerShell.Linux.x64.$POWERSHELL_VERSION.nupkg https://pwshtool.blob.core.windows.net/tool/$POWERSHELL_VERSION/PowerShell.Linux.x64.$POWERSHELL_VERSION.nupkg \
curl -SL --output PowerShell.Linux.x64.$POWERSHELL_VERSION.nupkg https://pwshtool.blob.core.windows.net/tool/$POWERSHELL_VERSION/PowerShell.Linux.x64.$POWERSHELL_VERSION.nupkg \
&& powershell_sha512='0fb0167e13560371bffec38a4a87bf39377fa1a5cc39b3a078ddec8803212bede73e5821861036ba5c345bd55c74703134c9b55c49385f87dae9e2de9239f5d9' \ && powershell_sha512='0fb0167e13560371bffec38a4a87bf39377fa1a5cc39b3a078ddec8803212bede73e5821861036ba5c345bd55c74703134c9b55c49385f87dae9e2de9239f5d9' \
&& echo "$powershell_sha512 PowerShell.Linux.x64.$POWERSHELL_VERSION.nupkg" | sha512sum -c - \ && echo "$powershell_sha512 PowerShell.Linux.x64.$POWERSHELL_VERSION.nupkg" | sha512sum -c - \
&& mkdir -p /usr/share/powershell \ && mkdir -p /usr/share/powershell \
@@ -232,22 +183,20 @@ RUN \
&& find /usr/share/powershell -print | grep -i '.*[.]nupkg$' | xargs rm && find /usr/share/powershell -print | grep -i '.*[.]nupkg$' | xargs rm
#vsdbg for remote .net core debugging #vsdbg for remote .net core debugging
RUN \ RUN wget https://aka.ms/getvsdbgsh -O - 2>/dev/null | /bin/sh /dev/stdin -v latest -l ~/vsdbg
wget https://aka.ms/getvsdbgsh -O - 2>/dev/null \
| /bin/bash /dev/stdin -v latest -l ~/vsdbg
# endregion NETCORE # endregion NETCORE
RUN DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1 dotnet tool install --tool-path /usr/local/bin paket run DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1 dotnet tool install --tool-path /usr/local/bin paket
RUN chmod +X /usr/local/bin/paket run chmod +X /usr/local/bin/paket
COPY sqlite-version-3.26.0.zip extension-functions.c build_sqlite.sh / copy sqlite-version-3.26.0.zip extension-functions.c build_sqlite.sh /
RUN ./build_sqlite.sh && rm -rf /sqlite-version-3.26.0/ run ./build_sqlite.sh && rm -rf /sqlite-version-3.26.0/
#git-lfs #git-lfs
COPY git-lfs /usr/bin/ copy git-lfs /usr/bin/
RUN chmod +x /usr/bin/git-lfs && git lfs install run chmod +x /usr/bin/git-lfs && git lfs install
##### Onload Stuff ##### ##### Onload Stuff #####
@@ -267,18 +216,18 @@ ARG ONLOAD_WITHZF
RUN \ RUN \
DEBIAN_FRONTEND=noninteractive apt-get update -y \ DEBIAN_FRONTEND=noninteractive apt-get update -y \
&& DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
ethtool \ ethtool \
gcc-multilib \ gcc-multilib \
libc6-dev-i386 \ libc6-dev-i386 \
libpcap0.8-dev \ libpcap0.8-dev \
libtool-bin \ libtool-bin \
&& cd /tmp \ && cd /tmp \
## && curl -fSL https://www.openonload.org/download/openonload-${ONLOAD_VERSION}.tgz -o /tmp/openonload-${ONLOAD_VERSION}.tgz \ ## && curl -fSL https://www.openonload.org/download/openonload-${ONLOAD_VERSION}.tgz -o /tmp/openonload-${ONLOAD_VERSION}.tgz \
&& curl -fSL https://github.com/rigtorp/openonload/archive/master.zip -o /tmp/openonload-${ONLOAD_VERSION}.zip \ && curl -fSL https://github.com/rigtorp/openonload/archive/master.zip -o /tmp/openonload-${ONLOAD_VERSION}.zip \
## && echo "${ONLOAD_MD5SUM} openonload-${ONLOAD_VERSION}.tgz" | md5sum --check \ ## && echo "${ONLOAD_MD5SUM} openonload-${ONLOAD_VERSION}.tgz" | md5sum --check \
## && tar -zxf openonload-${ONLOAD_VERSION}.tgz \ ## && tar -zxf openonload-${ONLOAD_VERSION}.tgz \
&& unzip -a openonload-${ONLOAD_VERSION}.zip \ && unzip -a openonload-${ONLOAD_VERSION}.zip \
## && cd /tmp/openonload-${ONLOAD_VERSION} \ ## && cd /tmp/openonload-${ONLOAD_VERSION} \
&& cd /tmp/openonload-master \ && cd /tmp/openonload-master \
&& cd scripts \ && cd scripts \
&& ./onload_build --user \ && ./onload_build --user \
@@ -287,36 +236,28 @@ RUN \
&& rm -rf openonload* \ && rm -rf openonload* \
&& if [ -z ${ONLOAD_WITHZF} ] ; then rm -rf /usr/include/zf /usr/bin/zf_stackdump /usr/bin/zf_debug /usr/lib/x86_64-linux-gnu/zf /usr/lib/x86_64-linux-gnu/libonload_zf* ; fi \ && if [ -z ${ONLOAD_WITHZF} ] ; then rm -rf /usr/include/zf /usr/bin/zf_stackdump /usr/bin/zf_debug /usr/lib/x86_64-linux-gnu/zf /usr/lib/x86_64-linux-gnu/libonload_zf* ; fi \
&& DEBIAN_FRONTEND=noninteractive apt-get remove -y --purge \ && DEBIAN_FRONTEND=noninteractive apt-get remove -y --purge \
curl \ curl \
patch \ patch \
&& DEBIAN_FRONTEND=noninteractive apt-get autoremove -y && DEBIAN_FRONTEND=noninteractive apt-get autoremove -y
# One of the onload dependencies installed above causes apt to remove
# pkg-config (presumably due to a dependency conflict). Re-installing
# pkg-config seems to resolve the issue.
# TODO (when we have time): investigate this further
RUN \
apt-get update \
&& DEBIAN_FRONTEND=noninteractive apt-get install -y \
pkg-config
######################## ########################
ARG docker_user # build rtags
ARG docker_password run \
git clone --recursive https://github.com/Andersbakken/rtags.git && \
cd rtags && \
git submodule update && \
git checkout v2.38 && \
cmake . -DCMAKE_EXPORT_COMPILE_COMMANDS=1 . && \
make -j$(nproc) install && \
cd .. && \
rm -rf rtags
RUN \ # build cgdb (the version shipped with ubuntu is too old)
useradd -m $docker_user \ run \
&& echo "${docker_user}:${docker_password}" | chpasswd \ git clone https://github.com/cgdb/cgdb.git && \
&& adduser $docker_user sudo cd cgdb && \
git checkout v0.7.1 && \
RUN USER=$docker_user && \ ./autogen.sh && \
GROUP=$docker_user && \ ./configure && make -j$(nproc) && \
curl -SsL https://github.com/boxboat/fixuid/releases/download/v0.5.1/fixuid-0.5.1-linux-amd64.tar.gz | tar -C /usr/local/bin -xzf - && \ make install
chown root:root /usr/local/bin/fixuid && \
chmod 4755 /usr/local/bin/fixuid && \
mkdir -p /etc/fixuid && \
printf "user: $USER\ngroup: $GROUP\n" > /etc/fixuid/config.yml
USER $docker_user:$docker_user
ENTRYPOINT [fixuid]
+1 -8
View File
@@ -1,8 +1 @@
#!/usr/bin/env bash docker build -t bluetradingsystems/linux_docker .
docker build \
--build-arg docker_user=$USER \
--build-arg docker_user_id=$(id -u) \
--build-arg docker_password=$USER \
-t spark-crypto_18.04 \
.
-13
View File
@@ -1,13 +0,0 @@
#!/usr/bin/env bash
if [ $# -gt 1 ]; then
echo $2
exit 0
fi
declare image_name="$1"
if [ $# -eq 0 ]; then
declare image_name_cmd="$(realpath $(dirname "$0"))/docker_image_name.fish"
image_name="$(eval $image_name_cmd)"
fi
echo "${USER}s_$image_name"
-3
View File
@@ -1,3 +0,0 @@
#!/usr/bin/env bash
echo "spark-crypto_18.04"
-44
View File
@@ -1,44 +0,0 @@
#!/usr/bin/env bash
declare image_name="$1"
declare cont_name="$2"
declare this_dir="$(realpath $(dirname "$0"))"
if [ $# -lt 2 ]; then
if [ $# -lt 1 ]; then
declare image_name_cmd="$this_dir/docker_image_name.sh"
image_name="$(eval $image_name_cmd)"
fi
declare cont_name_cmd="$this_dir/docker_container_name.sh"
cont_name="$(eval $cont_name_cmd $image_name)"
fi
# The root repo path (`repo_path`) is mounted to `repo_path_out`. The latter is
# stored in a separate variable for convenience.
declare repo_path="$(cd .. && git rev-parse --show-toplevel && cd $this_dir)"
declare repo_path_out=$repo_path
declare spark_data_path="$HOME/sparkdata"
declare spark_data_path_out=$spark_data_path
# `working_dir` is the starting dir for all docker shells launched via
# `docker exec -it <container> bash`.
declare working_dir=$repo_path_out
docker run \
--rm \
--net host \
-h $(hostname) \
-e "TZ=America/New_York" \
--cap-add=SYS_PTRACE \
--security-opt seccomp=unconfined \
--name $cont_name \
-v $repo_path:$repo_path_out \
-v $spark_data_path:$spark_data_path_out \
-u $(id -u $USER) \
-w $working_dir \
--shm-size=4024m \
-t \
-i \
-d \
-e DISPLAY="$DISPLAY" \
"$image_name" \
bash