From d7297e67a5dd7ae2f902e927fafa62519c1977a6 Mon Sep 17 00:00:00 2001 From: Massimiliano Culpo Date: Tue, 7 May 2024 08:49:53 +0200 Subject: [PATCH] Maintenance for the "bootstrap" workflow in CI (#44031) Removed a lot of duplication. Fixed an issue in containers, leading to false negative --- .github/workflows/bootstrap-test.sh | 5 +- .github/workflows/bootstrap.yml | 338 ++++++---------------------- 2 files changed, 77 insertions(+), 266 deletions(-) diff --git a/.github/workflows/bootstrap-test.sh b/.github/workflows/bootstrap-test.sh index b51db3d1b7..563eb28643 100755 --- a/.github/workflows/bootstrap-test.sh +++ b/.github/workflows/bootstrap-test.sh @@ -1,7 +1,8 @@ #!/bin/bash -set -ex +set -e source share/spack/setup-env.sh +$PYTHON bin/spack bootstrap disable github-actions-v0.4 $PYTHON bin/spack bootstrap disable spack-install -$PYTHON bin/spack -d solve zlib +$PYTHON bin/spack $SPACK_FLAGS solve zlib tree $BOOTSTRAP/store exit 0 diff --git a/.github/workflows/bootstrap.yml b/.github/workflows/bootstrap.yml index d94686e1a8..1b15a87e81 100644 --- a/.github/workflows/bootstrap.yml +++ b/.github/workflows/bootstrap.yml @@ -13,118 +13,22 @@ concurrency: cancel-in-progress: true jobs: - fedora-clingo-sources: + distros-clingo-sources: runs-on: ubuntu-latest - container: "fedora:latest" + container: ${{ matrix.image }} + strategy: + matrix: + image: ["fedora:latest", "opensuse/leap:latest"] steps: - - name: Install dependencies + - name: Setup Fedora + if: ${{ matrix.image == 'fedora:latest' }} run: | dnf install -y \ - bzip2 curl file gcc-c++ gcc gcc-gfortran git gnupg2 gzip \ + bzip2 curl file gcc-c++ gcc gcc-gfortran git gzip \ make patch unzip which xz python3 python3-devel tree \ cmake bison bison-devel libstdc++-static - - name: Checkout - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b - with: - fetch-depth: 0 - - name: Setup non-root user - run: | - # See [1] below - git config --global --add safe.directory /__w/spack/spack - useradd spack-test && mkdir -p ~spack-test - chown -R spack-test . ~spack-test - - name: Setup repo - shell: runuser -u spack-test -- bash {0} - run: | - git --version - . .github/workflows/setup_git.sh - - name: Bootstrap clingo - shell: runuser -u spack-test -- bash {0} - run: | - source share/spack/setup-env.sh - spack bootstrap disable github-actions-v0.5 - spack bootstrap disable github-actions-v0.4 - spack external find cmake bison - spack -d solve zlib - tree ~/.spack/bootstrap/store/ - - ubuntu-clingo-sources: - runs-on: ubuntu-latest - container: "ubuntu:latest" - steps: - - name: Install dependencies - env: - DEBIAN_FRONTEND: noninteractive - run: | - apt-get update -y && apt-get upgrade -y - apt-get install -y \ - bzip2 curl file g++ gcc gfortran git gnupg2 gzip \ - make patch unzip xz-utils python3 python3-dev tree \ - cmake bison - - name: Checkout - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b - with: - fetch-depth: 0 - - name: Setup non-root user - run: | - # See [1] below - git config --global --add safe.directory /__w/spack/spack - useradd spack-test && mkdir -p ~spack-test - chown -R spack-test . ~spack-test - - name: Setup repo - shell: runuser -u spack-test -- bash {0} - run: | - git --version - . .github/workflows/setup_git.sh - - name: Bootstrap clingo - shell: runuser -u spack-test -- bash {0} - run: | - source share/spack/setup-env.sh - spack bootstrap disable github-actions-v0.5 - spack bootstrap disable github-actions-v0.4 - spack external find cmake bison - spack -d solve zlib - tree ~/.spack/bootstrap/store/ - - ubuntu-clingo-binaries-and-patchelf: - runs-on: ubuntu-latest - container: "ubuntu:latest" - steps: - - name: Install dependencies - env: - DEBIAN_FRONTEND: noninteractive - run: | - apt-get update -y && apt-get upgrade -y - apt-get install -y \ - bzip2 curl file g++ gcc gfortran git gnupg2 gzip \ - make patch unzip xz-utils python3 python3-dev tree - - name: Checkout - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b - with: - fetch-depth: 0 - - name: Setup non-root user - run: | - # See [1] below - git config --global --add safe.directory /__w/spack/spack - useradd spack-test && mkdir -p ~spack-test - chown -R spack-test . ~spack-test - - name: Setup repo - shell: runuser -u spack-test -- bash {0} - run: | - git --version - . .github/workflows/setup_git.sh - - name: Bootstrap clingo - shell: runuser -u spack-test -- bash {0} - run: | - source share/spack/setup-env.sh - spack -d solve zlib - tree ~/.spack/bootstrap/store/ - - opensuse-clingo-sources: - runs-on: ubuntu-latest - container: "opensuse/leap:latest" - steps: - - name: Install dependencies + - name: Setup OpenSUSE + if: ${{ matrix.image == 'opensuse/leap:latest' }} run: | # Harden CI by applying the workaround described here: https://www.suse.com/support/kb/doc/?id=000019505 zypper update -y || zypper update -y @@ -136,12 +40,6 @@ jobs: uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b with: fetch-depth: 0 - - name: Setup repo - run: | - # See [1] below - git config --global --add safe.directory /__w/spack/spack - git --version - . .github/workflows/setup_git.sh - name: Bootstrap clingo run: | source share/spack/setup-env.sh @@ -151,77 +49,102 @@ jobs: spack -d solve zlib tree ~/.spack/bootstrap/store/ - macos-clingo-sources: - runs-on: macos-latest + clingo-sources: + runs-on: ${{ matrix.runner }} + strategy: + matrix: + runner: ['macos-13', 'macos-14', "ubuntu-latest"] steps: - - name: Install dependencies + - name: Setup macOS + if: ${{ matrix.runner != 'ubuntu-latest' }} run: | - brew install cmake bison@2.7 tree + brew install cmake bison tree - name: Checkout uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b + with: + fetch-depth: 0 - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d with: python-version: "3.12" - name: Bootstrap clingo run: | source share/spack/setup-env.sh - export PATH=/usr/local/opt/bison@2.7/bin:$PATH spack bootstrap disable github-actions-v0.5 spack bootstrap disable github-actions-v0.4 spack external find --not-buildable cmake bison spack -d solve zlib tree ~/.spack/bootstrap/store/ - macos-clingo-binaries: - runs-on: ${{ matrix.macos-version }} + gnupg-sources: + runs-on: ${{ matrix.runner }} strategy: matrix: - macos-version: ['macos-11', 'macos-12'] + runner: [ 'macos-13', 'macos-14', "ubuntu-latest" ] steps: - - name: Install dependencies + - name: Setup macOS + if: ${{ matrix.runner != 'ubuntu-latest' }} run: | brew install tree - - name: Checkout - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b - - name: Bootstrap clingo + # Remove GnuPG since we want to bootstrap it + sudo rm -rf /usr/local/bin/gpg + - name: Setup Ubuntu + if: ${{ matrix.runner == 'ubuntu-latest' }} run: | - set -ex - for ver in '3.7' '3.8' '3.9' '3.10' '3.11' ; do - not_found=1 - ver_dir="$(find $RUNNER_TOOL_CACHE/Python -wholename "*/${ver}.*/*/bin" | grep . || true)" - echo "Testing $ver_dir" - if [[ -d "$ver_dir" ]] ; then - if $ver_dir/python --version ; then - export PYTHON="$ver_dir/python" - not_found=0 - old_path="$PATH" - export PATH="$ver_dir:$PATH" - ./bin/spack-tmpconfig -b ./.github/workflows/bootstrap-test.sh - export PATH="$old_path" - fi - fi - # NOTE: test all pythons that exist, not all do on 12 - done - - ubuntu-clingo-binaries: - runs-on: ubuntu-20.04 - steps: + sudo rm -rf $(which gpg) $(which gpg2) $(which patchelf) - name: Checkout uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b with: fetch-depth: 0 - - name: Setup repo + - name: Bootstrap GnuPG run: | - git --version - . .github/workflows/setup_git.sh + source share/spack/setup-env.sh + spack solve zlib + spack bootstrap disable github-actions-v0.5 + spack bootstrap disable github-actions-v0.4 + spack -d gpg list + tree ~/.spack/bootstrap/store/ + + from-binaries: + runs-on: ${{ matrix.runner }} + strategy: + matrix: + runner: ['macos-13', 'macos-14', "ubuntu-latest"] + steps: + - name: Setup macOS + if: ${{ matrix.runner != 'ubuntu-latest' }} + run: | + brew install tree + # Remove GnuPG since we want to bootstrap it + sudo rm -rf /usr/local/bin/gpg + - name: Setup Ubuntu + if: ${{ matrix.runner == 'ubuntu-latest' }} + run: | + sudo rm -rf $(which gpg) $(which gpg2) $(which patchelf) + - name: Checkout + uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b + with: + fetch-depth: 0 + - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d + with: + python-version: | + 3.8 + 3.9 + 3.10 + 3.11 + 3.12 + - name: Set bootstrap sources + run: | + source share/spack/setup-env.sh + spack bootstrap disable github-actions-v0.4 + spack bootstrap disable spack-install - name: Bootstrap clingo run: | - set -ex - for ver in '3.7' '3.8' '3.9' '3.10' '3.11' ; do + set -e + for ver in '3.8' '3.9' '3.10' '3.11' '3.12' ; do not_found=1 ver_dir="$(find $RUNNER_TOOL_CACHE/Python -wholename "*/${ver}.*/*/bin" | grep . || true)" - echo "Testing $ver_dir" if [[ -d "$ver_dir" ]] ; then + echo "Testing $ver_dir" if $ver_dir/python --version ; then export PYTHON="$ver_dir/python" not_found=0 @@ -236,122 +159,9 @@ jobs: exit 1 fi done - - ubuntu-gnupg-binaries: - runs-on: ubuntu-latest - container: "ubuntu:latest" - steps: - - name: Install dependencies - env: - DEBIAN_FRONTEND: noninteractive - run: | - apt-get update -y && apt-get upgrade -y - apt-get install -y \ - bzip2 curl file g++ gcc patchelf gfortran git gzip \ - make patch unzip xz-utils python3 python3-dev tree - - name: Checkout - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b - with: - fetch-depth: 0 - - name: Setup non-root user - run: | - # See [1] below - git config --global --add safe.directory /__w/spack/spack - useradd spack-test && mkdir -p ~spack-test - chown -R spack-test . ~spack-test - - name: Setup repo - shell: runuser -u spack-test -- bash {0} - run: | - git --version - . .github/workflows/setup_git.sh - - name: Bootstrap GnuPG - shell: runuser -u spack-test -- bash {0} - run: | - source share/spack/setup-env.sh - spack bootstrap disable github-actions-v0.4 - spack bootstrap disable spack-install - spack -d gpg list - tree ~/.spack/bootstrap/store/ - - ubuntu-gnupg-sources: - runs-on: ubuntu-latest - container: "ubuntu:latest" - steps: - - name: Install dependencies - env: - DEBIAN_FRONTEND: noninteractive - run: | - apt-get update -y && apt-get upgrade -y - apt-get install -y \ - bzip2 curl file g++ gcc patchelf gfortran git gzip \ - make patch unzip xz-utils python3 python3-dev tree \ - gawk - - name: Checkout - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b - with: - fetch-depth: 0 - - name: Setup non-root user - run: | - # See [1] below - git config --global --add safe.directory /__w/spack/spack - useradd spack-test && mkdir -p ~spack-test - chown -R spack-test . ~spack-test - - name: Setup repo - shell: runuser -u spack-test -- bash {0} - run: | - git --version - . .github/workflows/setup_git.sh - - name: Bootstrap GnuPG - shell: runuser -u spack-test -- bash {0} - run: | - source share/spack/setup-env.sh - spack solve zlib - spack bootstrap disable github-actions-v0.5 - spack bootstrap disable github-actions-v0.4 - spack -d gpg list - tree ~/.spack/bootstrap/store/ - - macos-gnupg-binaries: - runs-on: macos-latest - steps: - - name: Install dependencies - run: | - brew install tree - # Remove GnuPG since we want to bootstrap it - sudo rm -rf /usr/local/bin/gpg - - name: Checkout - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b - name: Bootstrap GnuPG run: | source share/spack/setup-env.sh - spack bootstrap disable github-actions-v0.4 - spack bootstrap disable spack-install spack -d gpg list tree ~/.spack/bootstrap/store/ - macos-gnupg-sources: - runs-on: macos-latest - steps: - - name: Install dependencies - run: | - brew install gawk tree - # Remove GnuPG since we want to bootstrap it - sudo rm -rf /usr/local/bin/gpg - - name: Checkout - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b - - name: Bootstrap GnuPG - run: | - source share/spack/setup-env.sh - spack solve zlib - spack bootstrap disable github-actions-v0.5 - spack bootstrap disable github-actions-v0.4 - spack -d gpg list - tree ~/.spack/bootstrap/store/ - - -# [1] Distros that have patched git to resolve CVE-2022-24765 (e.g. Ubuntu patching v2.25.1) -# introduce breaking behaviorso we have to set `safe.directory` in gitconfig ourselves. -# See: -# - https://github.blog/2022-04-12-git-security-vulnerability-announced/ -# - https://github.com/actions/checkout/issues/760 -# - http://changelogs.ubuntu.com/changelogs/pool/main/g/git/git_2.25.1-1ubuntu3.3/changelog