hlrs-software-stack/spack
7
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Update Dockerfiles and images for Spack v0.18.0 (#30216)

Browse source 
This PR updates the list of images we build nightly, deprecating 
Ubuntu 16.04 and CentOS 8 and adding Ubuntu 20.04, Ubuntu 22.04
and CentOS Stream. It also removes a lot of duplication by generating
the Dockerfiles during the CI workflow and uploading them as artifacts
for later inspection or reuse.
This commit is contained in:
Massimiliano Culpo 2022-04-22 08:51:26 +02:00 committed by GitHub
parent b00983439f
commit f961a11187
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
17 changed files with 116 additions and 437 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

46
.github/workflows/build-containers.yml vendored
View file

@ -13,6 +13,8 @@ on:
paths:
- '.github/workflows/build-containers.yml'
- 'share/spack/docker/*'
- 'share/templates/container/*'
- 'lib/spack/spack/container/*'
# Let's also build & tag Spack containers on releases.
release:
types: [published]
@ -29,11 +31,17 @@ jobs:
# A matrix of Dockerfile paths, associated tags, and which architectures
# they support.
matrix:
dockerfile: [[amazon-linux, amazonlinux-2.dockerfile, 'linux/amd64,linux/arm64'],
[centos7, centos-7.dockerfile, 'linux/amd64,linux/arm64,linux/ppc64le'],
[leap15, leap-15.dockerfile, 'linux/amd64,linux/arm64,linux/ppc64le'],
[ubuntu-xenial, ubuntu-1604.dockerfile, 'linux/amd64,linux/arm64,linux/ppc64le'],
[ubuntu-bionic, ubuntu-1804.dockerfile, 'linux/amd64,linux/arm64,linux/ppc64le']]
# Meaning of the various items in the matrix list
# 0: Container name (e.g. ubuntu-bionic)
# 1: Platforms to build for
# 2: Base image (e.g. ubuntu:18.04)
dockerfile: [[amazon-linux, 'linux/amd64,linux/arm64', 'amazonlinux:2'],
[centos7, 'linux/amd64,linux/arm64,linux/ppc64le', 'centos:7'],
[centos-stream, 'linux/amd64,linux/arm64,linux/ppc64le', 'centos:stream'],
[leap15, 'linux/amd64,linux/arm64,linux/ppc64le', 'opensuse/leap:15'],
[ubuntu-bionic, 'linux/amd64,linux/arm64,linux/ppc64le', 'ubuntu:18.04'],
[ubuntu-focal, 'linux/amd64,linux/arm64,linux/ppc64le', 'ubuntu:20.04'],
[ubuntu-jammy, 'linux/amd64,linux/arm64,linux/ppc64le', 'ubuntu:22.04']]
name: Build ${{ matrix.dockerfile[0] }}
steps:
- name: Checkout
@ -52,14 +60,26 @@ jobs:
versioned="${{matrix.dockerfile[0]}}:${GITHUB_REF##*/}"
echo "versioned=${versioned}" >> $GITHUB_ENV
- name: Check ${{ matrix.dockerfile[1] }} Exists
- name: Generate the Dockerfile
env:
SPACK_YAML_OS: "${{ matrix.dockerfile[2] }}"
run: |
printf "Preparing to build ${{ env.container }} from ${{ matrix.dockerfile[1] }}"
if [ ! -f "share/spack/docker/${{ matrix.dockerfile[1]}}" ]; then
printf "Dockerfile ${{ matrix.dockerfile[0]}} does not exist"
.github/workflows/generate_spack_yaml_containerize.sh
. share/spack/setup-env.sh
mkdir -p dockerfiles/${{ matrix.dockerfile[0] }}
spack containerize --last-stage=bootstrap | tee dockerfiles/${{ matrix.dockerfile[0] }}/Dockerfile
printf "Preparing to build ${{ env.container }} from dockerfiles/${{ matrix.dockerfile[0] }}/Dockerfile"
if [ ! -f "dockerfiles/${{ matrix.dockerfile[0] }}/Dockerfile" ]; then
printf "dockerfiles/${{ matrix.dockerfile[0] }}/Dockerfile does not exist"
exit 1;
fi
- name: Upload Dockerfile
uses: actions/upload-artifact@6673cd052c4cd6fcf4b4e6e60ea986c889389535
with:
name: dockerfiles
path: dockerfiles
- name: Set up QEMU
uses: docker/setup-qemu-action@27d0a4f181a40b142cce983c5393082c365d1480 # @v1
@ -80,12 +100,14 @@ jobs:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Build & Deploy ${{ matrix.dockerfile[1] }}
- name: Build & Deploy ${{ matrix.dockerfile[0] }}
uses: docker/build-push-action@ac9327eae2b366085ac7f6a2d02df8aa8ead720a # @v2
with:
file: share/spack/docker/${{matrix.dockerfile[1]}}
platforms: ${{ matrix.dockerfile[2] }}
context: dockerfiles/${{ matrix.dockerfile[0] }}
platforms: ${{ matrix.dockerfile[1] }}
push: ${{ github.event_name != 'pull_request' }}
cache-from: type=gha
cache-to: type=gha,mode=max
tags: |
spack/${{ env.container }}
spack/${{ env.versioned }}

9
.github/workflows/generate_spack_yaml_containerize.sh vendored Executable file
View file

@ -0,0 +1,9 @@
#!/bin/bash
(echo "spack:" \
&& echo " specs: []" \
&& echo " container:" \
&& echo " format: docker" \
&& echo " images:" \
&& echo " os: \"${SPACK_YAML_OS}\"" \
&& echo " spack:" \
&& echo " ref: ${GITHUB_REF}") > spack.yaml

38
lib/spack/spack/container/images.json
View file

@ -12,11 +12,19 @@
},
"os_package_manager": "yum_amazon"
},
"centos:8": {
"centos:stream": {
"bootstrap": {
"template": "container/centos_8.dockerfile"
"template": "container/centos_stream.dockerfile",
"image": "quay.io/centos/centos:stream"
},
"os_package_manager": "yum"
"os_package_manager": "yum",
"build": "spack/centos-stream",
"final": {
"image": "quay.io/centos/centos:stream"
},
"build_tags": {
"develop": "latest"
}
},
"centos:7": {
"bootstrap": {
@ -51,10 +59,24 @@
},
"os_package_manager": "apt"
},
"ubuntu:22.04": {
"bootstrap": {
"template": "container/ubuntu_2204.dockerfile"
},
"os_package_manager": "apt",
"build": "spack/ubuntu-jammy",
"build_tags": {
"develop": "latest"
}
},
"ubuntu:20.04": {
"bootstrap": {
"template": "container/ubuntu_2004.dockerfile"
},
"build": "spack/ubuntu-focal",
"build_tags": {
"develop": "latest"
},
"os_package_manager": "apt"
},
"ubuntu:18.04": {
@ -66,16 +88,6 @@
"build_tags": {
"develop": "latest"
}
},
"ubuntu:16.04": {
"bootstrap": {
"template": "container/ubuntu_1604.dockerfile"
},
"os_package_manager": "apt",
"build": "spack/ubuntu-xenial",
"build_tags": {
"develop": "latest"
}
}
},
"os_package_managers": {

3
lib/spack/spack/container/images.py
View file

@ -129,5 +129,6 @@ def checkout_command(url, ref, enforce_sha, verify):
if enforce_sha or verify:
ref = _verify_ref(url, ref, enforce_sha)
command = 'git clone {0} . && git checkout {1} '.format(url, ref)
command = ('git clone {0} . && git fetch origin {1}:container_branch &&'
' git checkout container_branch ').format(url, ref)
return command

69
share/spack/docker/amazonlinux-2.dockerfile
View file

@ -1,69 +0,0 @@
FROM public.ecr.aws/amazonlinux/amazonlinux:2.0.20201111.0
MAINTAINER Spack Maintainers <maintainers@spack.io>
ENV DOCKERFILE_BASE=centos \
DOCKERFILE_DISTRO=centos \
DOCKERFILE_DISTRO_VERSION=7 \
SPACK_ROOT=/opt/spack \
DEBIAN_FRONTEND=noninteractive \
CURRENTLY_BUILDING_DOCKER_IMAGE=1 \
container=docker
RUN yum update -y \
&& yum groupinstall -y "Development Tools" \
&& yum install -y \
curl \
findutils \
gcc-c++ \
gcc \
gcc-gfortran \
git \
gnupg2 \
hostname \
iproute \
make \
patch \
python3 \
python3-pip \
python3-setuptools \
tcl \
unzip \
which \
&& pip3 install boto3 \
&& rm -rf /var/cache/yum \
&& yum clean all
COPY bin $SPACK_ROOT/bin
COPY etc $SPACK_ROOT/etc
COPY lib $SPACK_ROOT/lib
COPY share $SPACK_ROOT/share
COPY var $SPACK_ROOT/var
RUN mkdir -p $SPACK_ROOT/opt/spack
RUN ln -s $SPACK_ROOT/share/spack/docker/entrypoint.bash \
/usr/local/bin/docker-shell \
&& ln -s $SPACK_ROOT/share/spack/docker/entrypoint.bash \
/usr/local/bin/interactive-shell \
&& ln -s $SPACK_ROOT/share/spack/docker/entrypoint.bash \
/usr/local/bin/spack-env
RUN mkdir -p /root/.spack \
&& cp $SPACK_ROOT/share/spack/docker/modules.yaml \
/root/.spack/modules.yaml \
&& rm -rf /root/*.* /run/nologin $SPACK_ROOT/.git
# [WORKAROUND]
# https://superuser.com/questions/1241548/
# xubuntu-16-04-ttyname-failed-inappropriate-ioctl-for-device#1253889
RUN [ -f ~/.profile ] \
&& sed -i 's/mesg n/( tty -s \\&\\& mesg n || true )/g' ~/.profile \
|| true
WORKDIR /root
SHELL ["docker-shell"]
# TODO: add a command to Spack that (re)creates the package cache
RUN spack spec hdf5+mpi
ENTRYPOINT ["/bin/bash", "/opt/spack/share/spack/docker/entrypoint.bash"]
CMD ["interactive-shell"]

73
share/spack/docker/centos-7.dockerfile
View file

@ -1,73 +0,0 @@
FROM centos:7
MAINTAINER Spack Maintainers <maintainers@spack.io>
ENV DOCKERFILE_BASE=centos \
DOCKERFILE_DISTRO=centos \
DOCKERFILE_DISTRO_VERSION=7 \
SPACK_ROOT=/opt/spack \
DEBIAN_FRONTEND=noninteractive \
CURRENTLY_BUILDING_DOCKER_IMAGE=1 \
container=docker
RUN yum update -y \
&& yum install -y epel-release \
&& yum update -y \
&& yum --enablerepo epel groupinstall -y "Development Tools" \
&& yum --enablerepo epel install -y \
curl \
findutils \
gcc-c++ \
gcc \
gcc-gfortran \
git \
gnupg2 \
hostname \
iproute \
make \
patch \
patchelf \
python3 \
python3-pip \
python3-setuptools \
tcl \
unzip \
which \
&& pip3 install boto3 \
&& rm -rf /var/cache/yum \
&& yum clean all
COPY bin $SPACK_ROOT/bin
COPY etc $SPACK_ROOT/etc
COPY lib $SPACK_ROOT/lib
COPY share $SPACK_ROOT/share
COPY var $SPACK_ROOT/var
RUN mkdir -p $SPACK_ROOT/opt/spack
RUN ln -s $SPACK_ROOT/share/spack/docker/entrypoint.bash \
/usr/local/bin/docker-shell \
&& ln -s $SPACK_ROOT/share/spack/docker/entrypoint.bash \
/usr/local/bin/interactive-shell \
&& ln -s $SPACK_ROOT/share/spack/docker/entrypoint.bash \
/usr/local/bin/spack-env
RUN mkdir -p /root/.spack \
&& cp $SPACK_ROOT/share/spack/docker/modules.yaml \
/root/.spack/modules.yaml \
&& rm -rf /root/*.* /run/nologin $SPACK_ROOT/.git
# [WORKAROUND]
# https://superuser.com/questions/1241548/
# xubuntu-16-04-ttyname-failed-inappropriate-ioctl-for-device#1253889
RUN [ -f ~/.profile ] \
&& sed -i 's/mesg n/( tty -s \\&\\& mesg n || true )/g' ~/.profile \
|| true
WORKDIR /root
SHELL ["docker-shell"]
# TODO: add a command to Spack that (re)creates the package cache
RUN spack bootstrap untrust spack-install
RUN spack spec hdf5+mpi
ENTRYPOINT ["/bin/bash", "/opt/spack/share/spack/docker/entrypoint.bash"]
CMD ["interactive-shell"]

72
share/spack/docker/leap-15.dockerfile
View file

@ -1,72 +0,0 @@
FROM opensuse/leap:15.3
MAINTAINER Christian Goll <cgoll@suse.com>
ENV DOCKERFILE_BASE=opensuse \
DOCKERFILE_DISTRO=leap \
DOCKERFILE_DISTRO_VERSION=15.3 \
SPACK_ROOT=/opt/spack \
DEBIAN_FRONTEND=noninteractive \
CURRENTLY_BUILDING_DOCKER_IMAGE=1 \
container=docker
RUN zypper ref && \
zypper up -y && \
zypper in -y \
bzip2\
curl\
file\
gcc-c++\
gcc-fortran\
make\
gzip\
patch\
patchelf\
python3-base \
python3-boto3\
tar\
xz\
&& zypper clean
# clean up manpages
RUN rm -rf /var/cache/zypp/* \
rm -rf /usr/share/doc/packages/* \
rm -rf /usr/share/doc/manual/*
# copy spack into container
COPY bin $SPACK_ROOT/bin
COPY etc $SPACK_ROOT/etc
COPY lib $SPACK_ROOT/lib
COPY share $SPACK_ROOT/share
COPY var $SPACK_ROOT/var
RUN mkdir -p $SPACK_ROOT/opt/spack
RUN ln -s $SPACK_ROOT/share/spack/docker/entrypoint.bash \
/usr/local/bin/docker-shell \
&& ln -s $SPACK_ROOT/share/spack/docker/entrypoint.bash \
/usr/local/bin/interactive-shell \
&& ln -s $SPACK_ROOT/share/spack/docker/entrypoint.bash \
/usr/local/bin/spack-env
RUN mkdir -p /root/.spack \
&& cp $SPACK_ROOT/share/spack/docker/modules.yaml \
/root/.spack/modules.yaml \
&& rm -rf /root/*.* /run/nologin $SPACK_ROOT/.git
# [WORKAROUND]
# https://superuser.com/questions/1241548/
# xubuntu-16-04-ttyname-failed-inappropriate-ioctl-for-device#1253889
RUN [ -f ~/.profile ] \
&& sed -i 's/mesg n/( tty -s \\&\\& mesg n || true )/g' ~/.profile \
|| true
WORKDIR /root
SHELL ["docker-shell"]
# Disable bootstrapping from sources
RUN ${SPACK_ROOT}/bin/spack bootstrap untrust spack-install
# TODO: add a command to Spack that (re)creates the package cache
RUN ${SPACK_ROOT}/bin/spack spec hdf5+mpi
ENTRYPOINT ["/bin/bash", "/opt/spack/share/spack/docker/entrypoint.bash"]
CMD ["interactive-shell"]

8
share/spack/docker/modules.yaml
View file

@ -1,6 +1,6 @@
modules:
enable:
- lmod
lmod:
core_compilers:
default:
enable: []
lmod:
core_compilers:
- gcc

73
share/spack/docker/ubuntu-1604.dockerfile
View file

@ -1,73 +0,0 @@
FROM ubuntu:16.04
MAINTAINER Spack Maintainers <maintainers@spack.io>
ENV DOCKERFILE_BASE=ubuntu:16.04 \
DOCKERFILE_DISTRO=ubuntu \
DOCKERFILE_DISTRO_VERSION=16.04 \
SPACK_ROOT=/opt/spack \
DEBIAN_FRONTEND=noninteractive \
CURRENTLY_BUILDING_DOCKER_IMAGE=1 \
container=docker
RUN apt-get -yqq update \
&& apt-get -yqq install --no-install-recommends \
build-essential \
ca-certificates \
curl \
file \
g++ \
gcc \
gfortran \
git \
gnupg2 \
iproute2 \
locales \
make \
python3 \
python3-pip \
python3-setuptools \
tcl \
unzip \
&& locale-gen en_US.UTF-8 \
&& pip3 install boto3 \
&& rm -rf /var/lib/apt/lists/*
COPY bin $SPACK_ROOT/bin
COPY etc $SPACK_ROOT/etc
COPY lib $SPACK_ROOT/lib
COPY share $SPACK_ROOT/share
COPY var $SPACK_ROOT/var
RUN mkdir -p $SPACK_ROOT/opt/spack
RUN ln -s $SPACK_ROOT/share/spack/docker/entrypoint.bash \
/usr/local/bin/docker-shell \
&& ln -s $SPACK_ROOT/share/spack/docker/entrypoint.bash \
/usr/local/bin/interactive-shell \
&& ln -s $SPACK_ROOT/share/spack/docker/entrypoint.bash \
/usr/local/bin/spack-env
# Add LANG default to en_US.UTF-8
ENV LANGUAGE en_US.UTF-8
ENV LANG en_US.UTF-8
ENV LC_ALL en_US.UTF-8
RUN mkdir -p /root/.spack \
&& cp $SPACK_ROOT/share/spack/docker/modules.yaml \
/root/.spack/modules.yaml \
&& rm -rf /root/*.* /run/nologin $SPACK_ROOT/.git
# [WORKAROUND]
# https://superuser.com/questions/1241548/
# xubuntu-16-04-ttyname-failed-inappropriate-ioctl-for-device#1253889
RUN [ -f ~/.profile ] \
&& sed -i 's/mesg n/( tty -s \&\& mesg n || true )/g' ~/.profile \
|| true
WORKDIR /root
SHELL ["docker-shell"]
# TODO: add a command to Spack that (re)creates the package cache
RUN spack spec hdf5+mpi
ENTRYPOINT ["/bin/bash", "/opt/spack/share/spack/docker/entrypoint.bash"]
CMD ["interactive-shell"]

73
share/spack/docker/ubuntu-1804.dockerfile
View file

@ -1,73 +0,0 @@
FROM ubuntu:18.04
MAINTAINER Spack Maintainers <maintainers@spack.io>
ENV DOCKERFILE_BASE=ubuntu \
DOCKERFILE_DISTRO=ubuntu \
DOCKERFILE_DISTRO_VERSION=18.04 \
SPACK_ROOT=/opt/spack \
DEBIAN_FRONTEND=noninteractive \
CURRENTLY_BUILDING_DOCKER_IMAGE=1 \
container=docker
RUN apt-get -yqq update \
&& apt-get -yqq install --no-install-recommends \
build-essential \
ca-certificates \
curl \
file \
g++ \
gcc \
gfortran \
git \
gnupg2 \
iproute2 \
locales \
make \
python3 \
python3-pip \
python3-setuptools \
tcl \
unzip \
&& locale-gen en_US.UTF-8 \
&& pip3 install boto3 \
&& rm -rf /var/lib/apt/lists/*
COPY bin $SPACK_ROOT/bin
COPY etc $SPACK_ROOT/etc
COPY lib $SPACK_ROOT/lib
COPY share $SPACK_ROOT/share
COPY var $SPACK_ROOT/var
RUN mkdir -p $SPACK_ROOT/opt/spack
RUN ln -s $SPACK_ROOT/share/spack/docker/entrypoint.bash \
/usr/local/bin/docker-shell \
&& ln -s $SPACK_ROOT/share/spack/docker/entrypoint.bash \
/usr/local/bin/interactive-shell \
&& ln -s $SPACK_ROOT/share/spack/docker/entrypoint.bash \
/usr/local/bin/spack-env
# Add LANG default to en_US.UTF-8
ENV LANGUAGE en_US.UTF-8
ENV LANG en_US.UTF-8
ENV LC_ALL en_US.UTF-8
RUN mkdir -p /root/.spack \
&& cp $SPACK_ROOT/share/spack/docker/modules.yaml \
/root/.spack/modules.yaml \
&& rm -rf /root/*.* /run/nologin $SPACK_ROOT/.git
# [WORKAROUND]
# https://superuser.com/questions/1241548/
# xubuntu-16-04-ttyname-failed-inappropriate-ioctl-for-device#1253889
RUN [ -f ~/.profile ] \
&& sed -i 's/mesg n/( tty -s \&\& mesg n || true )/g' ~/.profile \
|| true
WORKDIR /root
SHELL ["docker-shell"]
# TODO: add a command to Spack that (re)creates the package cache
RUN spack spec hdf5+mpi
ENTRYPOINT ["/bin/bash", "/opt/spack/share/spack/docker/entrypoint.bash"]
CMD ["interactive-shell"]

8
share/spack/templates/container/amazonlinux_2.dockerfile
View file

@ -14,11 +14,11 @@ RUN yum update -y \
iproute \
make \
patch \
python \
python-pip \
python-setuptools \
python3 \
python3-pip \
python3-setuptools \
unzip \
&& pip install boto3 \
&& pip3 install boto3 \
&& rm -rf /var/cache/yum \
&& yum clean all
{% endblock %}

8
share/spack/templates/container/centos_7.dockerfile
View file

@ -16,11 +16,11 @@ RUN yum update -y \
iproute \
make \
patch \
python \
python-pip \
python-setuptools \
python3 \
python3-pip \
python3-setuptools \
unzip \
&& pip install boto3 \
&& pip3 install boto3 \
&& rm -rf /var/cache/yum \
&& yum clean all
{% endblock %}

0
share/spack/templates/container/centos_8.dockerfile → share/spack/templates/container/centos_stream.dockerfile
View file

32
share/spack/templates/container/ubuntu_1604.dockerfile
View file

@ -1,32 +0,0 @@
{% extends "container/bootstrap-base.dockerfile" %}
{% block env_vars %}
{{ super() }}
ENV DEBIAN_FRONTEND=noninteractive \
LANGUAGE=en_US.UTF-8 \
LANG=en_US.UTF-8 \
LC_ALL=en_US.UTF-8
{% endblock %}
{% block install_os_packages %}
RUN apt-get -yqq update \
&& apt-get -yqq install --no-install-recommends \
build-essential \
ca-certificates \
curl \
file \
g++ \
gcc \
gfortran \
git \
gnupg2 \
iproute2 \
locales \
lua-posix \
make \
python3 \
python3-pip \
python3-setuptools \
unzip \
&& locale-gen en_US.UTF-8 \
&& pip3 install boto3 \
&& rm -rf /var/lib/apt/lists/*
{% endblock %}

6
share/spack/templates/container/ubuntu_1804.dockerfile
View file

@ -1,6 +0,0 @@
{% extends "container/ubuntu_1604.dockerfile" %}
{% block post_checkout %}
# [WORKAROUND]
# https://bugs.launchpad.net/ubuntu/+source/lua-posix/+bug/1752082
RUN ln -s posix_c.so /usr/lib/x86_64-linux-gnu/lua/5.2/posix.so
{% endblock %}

1
share/spack/templates/container/ubuntu_1804.dockerfile Symbolic link
View file

@ -0,0 +1 @@
ubuntu_2004.dockerfile

1
share/spack/templates/container/ubuntu_2004.dockerfile
View file

@ -1 +0,0 @@
ubuntu_1604.dockerfile

32
share/spack/templates/container/ubuntu_2004.dockerfile Normal file
View file

@ -0,0 +1,32 @@
{% extends "container/bootstrap-base.dockerfile" %}
{% block env_vars %}
{{ super() }}
ENV DEBIAN_FRONTEND=noninteractive \
LANGUAGE=en_US.UTF-8 \
LANG=en_US.UTF-8 \
LC_ALL=en_US.UTF-8
{% endblock %}
{% block install_os_packages %}
RUN apt-get -yqq update \
&& apt-get -yqq upgrade \
&& apt-get -yqq install --no-install-recommends \
build-essential \
ca-certificates \
curl \
file \
g++ \
gcc \
gfortran \
git \
gnupg2 \
iproute2 \
locales \
make \
python3 \
python3-pip \
python3-setuptools \
unzip \
&& locale-gen en_US.UTF-8 \
&& pip3 install boto3 \
&& rm -rf /var/lib/apt/lists/*
{% endblock %}

1
share/spack/templates/container/ubuntu_2204.dockerfile Symbolic link
View file

@ -0,0 +1 @@
ubuntu_2004.dockerfile