From b42b0cd45ad4dbb756b087ec610df7d29b0b1622 Mon Sep 17 00:00:00 2001 From: Massimiliano Culpo Date: Thu, 29 Jul 2021 09:08:32 +0200 Subject: [PATCH] Move build tests from GA to Gitlab (#25120) Modifications: - Remove the "build tests" workflow from GitHub Actions - Setup a similar e2e test on Gitlab In this way we'll reduce load on GitHub Actions workflows and for e2e tests will benefit from the buildcache reuse granted by pipelines. --- .github/workflows/linux_build_tests.yaml | 77 ------------------- README.md | 1 - .../gitlab/cloud_pipelines/.gitlab-ci.yml | 32 +++++++- .../stacks/build_systems/spack.yaml | 61 +++++++++++++++ 4 files changed, 92 insertions(+), 79 deletions(-) delete mode 100644 .github/workflows/linux_build_tests.yaml create mode 100644 share/spack/gitlab/cloud_pipelines/stacks/build_systems/spack.yaml diff --git a/.github/workflows/linux_build_tests.yaml b/.github/workflows/linux_build_tests.yaml deleted file mode 100644 index 256b8bc456..0000000000 --- a/.github/workflows/linux_build_tests.yaml +++ /dev/null @@ -1,77 +0,0 @@ -name: linux builds - -on: - push: - branches: - - develop - - releases/** - paths-ignore: - # Don't run if we only modified packages in the built-in repository - - 'var/spack/repos/builtin/**' - - '!var/spack/repos/builtin/packages/lz4/**' - - '!var/spack/repos/builtin/packages/mpich/**' - - '!var/spack/repos/builtin/packages/tut/**' - - '!var/spack/repos/builtin/packages/py-setuptools/**' - - '!var/spack/repos/builtin/packages/openjpeg/**' - - '!var/spack/repos/builtin/packages/r-rcpp/**' - - '!var/spack/repos/builtin/packages/ruby-rake/**' - # Don't run if we only modified documentation - - 'lib/spack/docs/**' - pull_request: - branches: - - develop - - releases/** - paths-ignore: - # Don't run if we only modified packages in the built-in repository - - 'var/spack/repos/builtin/**' - - '!var/spack/repos/builtin/packages/lz4/**' - - '!var/spack/repos/builtin/packages/mpich/**' - - '!var/spack/repos/builtin/packages/tut/**' - - '!var/spack/repos/builtin/packages/py-setuptools/**' - - '!var/spack/repos/builtin/packages/openjpeg/**' - - '!var/spack/repos/builtin/packages/r-rcpp/**' - - '!var/spack/repos/builtin/packages/ruby-rake/**' - # Don't run if we only modified documentation - - 'lib/spack/docs/**' - -jobs: - build: - runs-on: ubuntu-latest - strategy: - matrix: - package: - - lz4 # MakefilePackage - - mpich~fortran # AutotoolsPackage - - 'tut%gcc@:10.99.99' # WafPackage - - py-setuptools # PythonPackage - - openjpeg # CMakePackage - - r-rcpp # RPackage - - ruby-rake # RubyPackage - steps: - - uses: actions/checkout@v2 - - uses: actions/cache@v2.1.6 - with: - path: ~/.ccache - key: ccache-build-${{ matrix.package }} - restore-keys: | - ccache-build-${{ matrix.package }} - - uses: actions/setup-python@v2 - with: - python-version: 3.9 - - name: Install System Packages - run: | - sudo apt-get update - sudo apt-get -yqq install ccache gfortran perl perl-base r-base r-base-core r-base-dev ruby findutils openssl libssl-dev libpciaccess-dev - R --version - perl --version - ruby --version - - name: Copy Configuration - run: | - ccache -M 300M && ccache -z - # Set up external deps for build tests, b/c they take too long to compile - cp share/spack/qa/configuration/*.yaml etc/spack/ - - name: Run the build test - run: | - . share/spack/setup-env.sh - SPEC=${{ matrix.package }} share/spack/qa/run-build-tests - ccache -s diff --git a/README.md b/README.md index 964637da16..577ff81c82 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,6 @@ # Spack Spack [![Unit Tests](https://github.com/spack/spack/workflows/linux%20tests/badge.svg)](https://github.com/spack/spack/actions) -[![Linux Builds](https://github.com/spack/spack/workflows/linux%20builds/badge.svg)](https://github.com/spack/spack/actions) [![macOS Builds (nightly)](https://github.com/spack/spack/workflows/macOS%20builds%20nightly/badge.svg?branch=develop)](https://github.com/spack/spack/actions?query=workflow%3A%22macOS+builds+nightly%22) [![codecov](https://codecov.io/gh/spack/spack/branch/develop/graph/badge.svg)](https://codecov.io/gh/spack/spack) [![Read the Docs](https://readthedocs.org/projects/spack/badge/?version=latest)](https://spack.readthedocs.io) diff --git a/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml b/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml index bbcb589321..c69df706da 100644 --- a/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml +++ b/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml @@ -62,7 +62,7 @@ default: # TEMPLATE FOR ADDING ANOTHER PIPELINE ######################################## # -# First add a new spack.yml defining the pipline to run in +# First add a new spack.yml defining the pipeline to run in # share/spack/gitlab/cloud_pipelines/stacks/my-super-cool-stack/spack.yaml # # Then add the following entries at the bottom of this file: @@ -158,3 +158,33 @@ e4s-on-power-develop-build: - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml job: e4s-on-power-develop-generate strategy: depend + + +######################################### +# Build tests for different build-systems +######################################### +.build_systems: + variables: + SPACK_CI_STACK_NAME: build_systems + +build_systems-pr-generate: + extends: [ ".build_systems", ".pr-generate"] + +build_systems-develop-generate: + extends: [ ".build_systems", ".develop-generate"] + +build_systems-pr-build: + extends: [ ".build_systems", ".pr-build" ] + trigger: + include: + - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml + job: build_systems-pr-generate + strategy: depend + +build_systems-develop-build: + extends: [ ".build_systems", ".develop-build" ] + trigger: + include: + - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml + job: build_systems-develop-generate + strategy: depend diff --git a/share/spack/gitlab/cloud_pipelines/stacks/build_systems/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/build_systems/spack.yaml new file mode 100644 index 0000000000..5bbf04051d --- /dev/null +++ b/share/spack/gitlab/cloud_pipelines/stacks/build_systems/spack.yaml @@ -0,0 +1,61 @@ +spack: + view: false + concretization: separately + + config: + install_tree: + root: /home/software/spack + padded_length: 512 + projections: + all: '{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}' + + definitions: + - default_specs: + - lz4 # MakefilePackage + - mpich~fortran # AutotoolsPackage + - tut # WafPackage + - py-setuptools # PythonPackage + - openjpeg # CMakePackage + - r-rcpp # RPackage + - ruby-rake # RubyPackage + - arch: + - '%gcc@7.5.0 arch=linux-ubuntu18.04-x86_64' + + specs: + - matrix: + - - $default_specs + - - $arch + + mirrors: { "mirror": "s3://spack-binaries-develop/build_systems" } + + gitlab-ci: + script: + - . "./share/spack/setup-env.sh" + - spack --version + - cd ${SPACK_CONCRETE_ENV_DIR} + - spack env activate --without-view . + - spack config add "config:install_tree:projections:${SPACK_JOB_SPEC_PKG_NAME}:'morepadding/{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'" + - spack -d ci rebuild + + mappings: + - match: [ 'os=ubuntu18.04' ] + runner-attributes: + image: { "name": "ghcr.io/scottwittenburg/ecpe4s-ubuntu18.04-runner-x86_64:2020-09-01", "entrypoint": [ "" ] } + tags: [ "spack", "public", "large", "x86_64" ] + + temporary-storage-url-prefix: "s3://spack-binaries-prs/pipeline-storage" + broken-specs-url: "s3://spack-binaries-develop/broken-specs" + service-job-attributes: + before_script: + - . "./share/spack/setup-env.sh" + - spack --version + - cd share/spack/gitlab/cloud_pipelines/stacks/build_systems + - spack env activate --without-view . + image: { "name": "ghcr.io/scottwittenburg/ecpe4s-ubuntu18.04-runner-x86_64:2020-09-01", "entrypoint": [""] } + tags: ["spack", "public", "medium", "x86_64"] + + cdash: + build-group: Build tests for different build systems + url: https://cdash.spack.io + project: Spack Testing + site: Cloud Gitlab Infrastructure