diff --git a/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml b/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml index 643709f138..5120a5770a 100644 --- a/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml +++ b/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml @@ -4,9 +4,14 @@ default: image: { "name": "ghcr.io/spack/e4s-ubuntu-18.04:v2021-10-18", "entrypoint": [""] } # CI Platform-Arch -.cray_zen4: +.cray_rhel_zen4: variables: - SPACK_TARGET_PLATFORM: "cray" + SPACK_TARGET_PLATFORM: "cray-rhel" + SPACK_TARGET_ARCH: "zen4" + +.cray_sles_zen4: + variables: + SPACK_TARGET_PLATFORM: "cray-sles" SPACK_TARGET_ARCH: "zen4" .darwin_x86_64: @@ -797,7 +802,6 @@ deprecated-ci-build: SPACK_PRUNE_UNTOUCHED_DEPENDENT_DEPTH: "1" .generate-cray: - tags: [ "cce@15.0.1", "cray-zen4", "public" ] extends: [ ".base-cray-job" ] stage: generate script: @@ -835,28 +839,60 @@ deprecated-ci-build: when: - always +.generate-cray-rhel: + tags: [ "cray-rhel-zen4", "public" ] + extends: [ ".generate-cray" ] + +.generate-cray-sles: + tags: [ "cray-sles-zen4", "public" ] + extends: [ ".generate-cray" ] + + .build-cray: extends: [ ".base-cray-job" ] stage: build + ####################################### -# E4S - Cray +# E4S - Cray RHEL ####################################### -.e4s-cray: - extends: [ ".cray_zen4" ] +.e4s-cray-rhel: + extends: [ ".cray_rhel_zen4" ] variables: - SPACK_CI_STACK_NAME: e4s-cray + SPACK_CI_STACK_NAME: e4s-cray-rhel -e4s-cray-generate: - extends: [ ".generate-cray", ".e4s-cray" ] +e4s-cray-rhel-generate: + extends: [ ".generate-cray-rhel", ".e4s-cray-rhel" ] -e4s-cray-build: - extends: [ ".build-cray", ".e4s-cray" ] +e4s-cray-rhel-build: + extends: [ ".build-cray", ".e4s-cray-rhel" ] trigger: include: - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml - job: e4s-cray-generate + job: e4s-cray-rhel-generate strategy: depend needs: - artifacts: True - job: e4s-cray-generate \ No newline at end of file + job: e4s-cray-rhel-generate + +####################################### +# E4S - Cray SLES +####################################### +.e4s-cray-sles: + extends: [ ".cray_sles_zen4" ] + variables: + SPACK_CI_STACK_NAME: e4s-cray-sles + +e4s-cray-sles-generate: + extends: [ ".generate-cray-sles", ".e4s-cray-sles" ] + +e4s-cray-sles-build: + extends: [ ".build-cray", ".e4s-cray-sles" ] + trigger: + include: + - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml + job: e4s-cray-sles-generate + strategy: depend + needs: + - artifacts: True + job: e4s-cray-sles-generate \ No newline at end of file diff --git a/share/spack/gitlab/cloud_pipelines/configs/ci.yaml b/share/spack/gitlab/cloud_pipelines/configs/ci.yaml index 77babf10b2..53bc5f63d7 100644 --- a/share/spack/gitlab/cloud_pipelines/configs/ci.yaml +++ b/share/spack/gitlab/cloud_pipelines/configs/ci.yaml @@ -18,15 +18,16 @@ ci: - spack config add "config:install_tree:projections:${SPACK_JOB_SPEC_PKG_NAME}:'morepadding/{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'" - mkdir -p ${SPACK_ARTIFACTS_ROOT}/user_data # AWS runners mount E4S public key (verification), UO runners mount public/private (signing/verification) - - if [[ -r /mnt/key/e4s.gpg ]]; then spack gpg trust /mnt/key/e4s.gpg; fi + - k=$CI_GPG_KEY_ROOT/e4s.gpg; [[ -r $k ]] && spack gpg trust $k # UO runners mount intermediate ci public key (verification), AWS runners mount public/private (signing/verification) - - if [[ -r /mnt/key/intermediate_ci_signing_key.gpg ]]; then spack gpg trust /mnt/key/intermediate_ci_signing_key.gpg; fi - - if [[ -r /mnt/key/spack_public_key.gpg ]]; then spack gpg trust /mnt/key/spack_public_key.gpg; fi + - k=$CI_GPG_KEY_ROOT/intermediate_ci_signing_key.gpg; [[ -r $k ]] && spack gpg trust $k + - k=$CI_GPG_KEY_ROOT/spack_public_key.gpg; [[ -r $k ]] && spack gpg trust $k - spack --color=always --backtrace ci rebuild --tests > >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_out.txt) 2> >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_err.txt >&2) after_script: - - cat /proc/loadavg || true variables: CI_JOB_SIZE: "default" + CI_GPG_KEY_ROOT: /mnt/key # SPACK_VERBOSE_SCRIPT: "1" - signing-job: diff --git a/share/spack/gitlab/cloud_pipelines/configs/cray/ci.yaml b/share/spack/gitlab/cloud_pipelines/configs/cray-rhel/ci.yaml similarity index 88% rename from share/spack/gitlab/cloud_pipelines/configs/cray/ci.yaml rename to share/spack/gitlab/cloud_pipelines/configs/cray-rhel/ci.yaml index 114dcc81dc..3df3e1cc7d 100644 --- a/share/spack/gitlab/cloud_pipelines/configs/cray/ci.yaml +++ b/share/spack/gitlab/cloud_pipelines/configs/cray-rhel/ci.yaml @@ -3,12 +3,8 @@ ci: - build-job-remove: image: no-image - build-job: - script+: - # AWS runners mount E4S public key (verification), UO runners mount public/private (signing/verification) - - if [[ -r /etc/protected-runner/e4s.gpg ]]; then spack gpg trust /etc/protected-runner/e4s.gpg; fi - # UO runners mount intermediate ci public key (verification), AWS runners mount public/private (signing/verification) - - if [[ -r /etc/protected-runner/intermediate_ci_signing_key.gpg ]]; then spack gpg trust /etc/protected-runner/intermediate_ci_signing_key.gpg; fi - - if [[ -r /etc/protected-runner/spack_public_key.gpg ]]; then spack gpg trust /etc/protected-runner/spack_public_key.gpg; fi + variables: + CI_GPG_KEY_ROOT: /etc/protected-runner - match_behavior: first submapping: - match: diff --git a/share/spack/gitlab/cloud_pipelines/configs/cray-rhel/compilers.yaml b/share/spack/gitlab/cloud_pipelines/configs/cray-rhel/compilers.yaml new file mode 100644 index 0000000000..91ac37dfa5 --- /dev/null +++ b/share/spack/gitlab/cloud_pipelines/configs/cray-rhel/compilers.yaml @@ -0,0 +1,31 @@ +compilers: +- compiler: + spec: cce@15.0.1 + paths: + cc: cc + cxx: CC + f77: ftn + fc: ftn + flags: {} + operating_system: rhel8 + target: any + modules: + - PrgEnv-cray/8.3.3 + - cce/15.0.1 + environment: + set: + MACHTYPE: x86_64 +- compiler: + spec: gcc@11.2.0 + paths: + cc: gcc + cxx: g++ + f77: gfortran + fc: gfortran + flags: {} + operating_system: rhel8 + target: any + modules: + - PrgEnv-gnu + - gcc/11.2.0 + environment: {} \ No newline at end of file diff --git a/share/spack/gitlab/cloud_pipelines/configs/cray/config.yaml b/share/spack/gitlab/cloud_pipelines/configs/cray-rhel/config.yaml similarity index 100% rename from share/spack/gitlab/cloud_pipelines/configs/cray/config.yaml rename to share/spack/gitlab/cloud_pipelines/configs/cray-rhel/config.yaml diff --git a/share/spack/gitlab/cloud_pipelines/configs/cray-rhel/packages.yaml b/share/spack/gitlab/cloud_pipelines/configs/cray-rhel/packages.yaml new file mode 100644 index 0000000000..a74d19ba62 --- /dev/null +++ b/share/spack/gitlab/cloud_pipelines/configs/cray-rhel/packages.yaml @@ -0,0 +1,16 @@ +packages: + # EXTERNALS + cray-mpich: + buildable: false + externals: + - spec: cray-mpich@8.1.25 %cce@15.0.1 + prefix: /opt/cray/pe/mpich/8.1.25/ofi/cray/10.0 + modules: + - cray-mpich/8.1.25 + cray-libsci: + buildable: false + externals: + - spec: cray-libsci@23.02.1.1 %cce@15.0.1 + prefix: /opt/cray/pe/libsci/23.02.1.1/CRAY/9.0/x86_64/ + modules: + - cray-libsci/23.02.1.1 \ No newline at end of file diff --git a/share/spack/gitlab/cloud_pipelines/configs/cray-rhel/zen4/ci.yaml b/share/spack/gitlab/cloud_pipelines/configs/cray-rhel/zen4/ci.yaml new file mode 100644 index 0000000000..43bbbc9249 --- /dev/null +++ b/share/spack/gitlab/cloud_pipelines/configs/cray-rhel/zen4/ci.yaml @@ -0,0 +1,4 @@ +ci: + pipeline-gen: + - build-job: + tags: ["cray-rhel-zen4"] diff --git a/share/spack/gitlab/cloud_pipelines/configs/cray-sles/ci.yaml b/share/spack/gitlab/cloud_pipelines/configs/cray-sles/ci.yaml new file mode 100644 index 0000000000..3df3e1cc7d --- /dev/null +++ b/share/spack/gitlab/cloud_pipelines/configs/cray-sles/ci.yaml @@ -0,0 +1,291 @@ +ci: + pipeline-gen: + - build-job-remove: + image: no-image + - build-job: + variables: + CI_GPG_KEY_ROOT: /etc/protected-runner + - match_behavior: first + submapping: + - match: + - hipblas + - llvm + - llvm-amdgpu + - pango + - paraview + - py-tensorflow + - py-torch + - qt + - rocblas + - visit + build-job: + tags: [ "spack", "huge" ] + variables: + CI_JOB_SIZE: huge + SPACK_BUILD_JOBS: "12" + + - match: + - ascent + - atk + - axom + - cistem + - cmake + - ctffind + - cuda + - dealii + - dray + - dyninst + - ecp-data-vis-sdk + - gcc + - ginkgo + - hdf5 + - hpx + - kokkos-kernels + - kokkos-nvcc-wrapper + - lbann + - magma + - mesa + - mfem + - mpich + - netlib-lapack + - nvhpc + - oce + - openblas + - openfoam + - openturns + - parallelio + - plumed + - precice + #- py-tensorflow + #- qt + - raja + - relion + #- rocblas + - rocfft + - rocsolver + - rocsparse + - rust + - slate + - strumpack + - sundials + - trilinos + - umpire + #- visit + - vtk + - vtk-h + - vtk-m + - warpx + - wrf + - wxwidgets + build-job: + tags: [ "spack", "large" ] + variables: + CI_JOB_SIZE: large + SPACK_BUILD_JOBS: "8" + + - match: + - adios2 + - amrex + - archer + - ascent + - autoconf-archive + - axom + - binutils + - blaspp + - blt + - boost + - butterflypack + - cabana + - caliper + - camp + - chai + - conduit + - curl + - datatransferkit + - double-conversion + - dray + - eigen + - faodel + - ffmpeg + - fftw + - fortrilinos + - gettext + - gperftools + - gptune + - hdf5 + - heffte + - hpctoolkit + - hwloc + - hydrogen + - hypre + - kokkos + - lammps + - lapackpp + - legion + - libtool + - libxml2 + - libzmq + - llvm-openmp-ompt + - mbedtls + - mfem + - mpich + - mvapich2 + - nasm + - netlib-scalapack + - omega-h + - openblas + - openjpeg + - openmpi + - openpmd-api + - pagmo2 + - papyrus + - parsec + - pdt + - pegtl + - petsc + - pumi + - py-beniget + - py-cinemasci + - pygmo + - py-ipython-genutils + - py-packaging + - py-petsc4py + - py-scipy + - py-statsmodels + - py-warlock + - py-warpx + - raja + - samrai + - slepc + - slurm + - sqlite + - strumpack + - sundials + - superlu-dist + - tasmanian + - tau + - upcxx + - vtk + - vtk-h + - vtk-m + - zfp + build-job: + tags: [ "spack", "medium" ] + variables: + CI_JOB_SIZE: "medium" + SPACK_BUILD_JOBS: "2" + + - match: + - alsa-lib + - ant + - antlr + - argobots + - autoconf-archive + - automake + - berkeley-db + - bison + - blt + - bzip2 + - camp + - cmake + - curl + - czmq + - darshan-util + - diffutils + - docbook-xml + - exmcutils + - expat + - findutils + - flit + - freetype + - gawk + - gdbm + - gettext + - glib + - gmake + - gotcha + - hpcviewer + - hwloc + - jansson + - json-c + - libbsd + - libedit + - libevent + - libfabric + - libffi + - libgcrypt + - libiconv + - libidn2 + - libjpeg-turbo + - libmd + - libnrm + - libpciaccess + - libpng + - libsigsegv + - libsodium + - libunistring + - libunwind + - libxml2 + - libyaml + - libzmq + - lua + - lua-luaposix + - lz4 + - m4 + - meson + - metis + - mpfr + - ncurses + - ninja + - numactl + - openblas + - openjdk + - openssh + - openssl + - papi + - parallel-netcdf + - pcre + - pcre2 + - pdsh + - perl + - perl-data-dumper + - pkgconf + - py-alembic + - py-cffi + - py-cycler + - py-decorator + - py-idna + - py-jsonschema + - py-kiwisolver + - py-mistune + - py-pycparser + - py-setuptools + - py-setuptools-scm + - py-six + - py-testpath + - py-wheel + - qhull + - readline + - sed + - slurm + - snappy + - sqlite + - superlu + - swig + - tar + - tcl + - texinfo + - tut + - unzip + - util-linux-uuid + - util-macros + - xz + - yaml-cpp + - zfp + - zlib + - zstd + build-job: + tags: [ "spack", "small" ] + variables: + CI_JOB_SIZE: "small" + SPACK_BUILD_JOBS: "1" diff --git a/share/spack/gitlab/cloud_pipelines/configs/cray-sles/compilers.yaml b/share/spack/gitlab/cloud_pipelines/configs/cray-sles/compilers.yaml new file mode 100644 index 0000000000..0e5511ea05 --- /dev/null +++ b/share/spack/gitlab/cloud_pipelines/configs/cray-sles/compilers.yaml @@ -0,0 +1,14 @@ +compilers: +- compiler: + spec: gcc@=10.3.0 + paths: + cc: /opt/cray/pe/gcc/10.3.0/bin/gcc + cxx: /opt/cray/pe/gcc/10.3.0/bin/g++ + f77: /opt/cray/pe/gcc/10.3.0/bin/gfortran + fc: /opt/cray/pe/gcc/10.3.0/bin/gfortran + flags: {} + operating_system: sle_hpc15 + target: x86_64 + modules: [PrgEnv-gnu-amd] + environment: {} + extra_rpaths: [] \ No newline at end of file diff --git a/share/spack/gitlab/cloud_pipelines/configs/cray-sles/config.yaml b/share/spack/gitlab/cloud_pipelines/configs/cray-sles/config.yaml new file mode 100644 index 0000000000..81cc5c04cc --- /dev/null +++ b/share/spack/gitlab/cloud_pipelines/configs/cray-sles/config.yaml @@ -0,0 +1,4 @@ +config: + install_tree: + root: $spack/opt/spack + diff --git a/share/spack/gitlab/cloud_pipelines/configs/cray-sles/packages.yaml b/share/spack/gitlab/cloud_pipelines/configs/cray-sles/packages.yaml new file mode 100644 index 0000000000..18656517be --- /dev/null +++ b/share/spack/gitlab/cloud_pipelines/configs/cray-sles/packages.yaml @@ -0,0 +1,16 @@ +packages: + # EXTERNALS + cray-mpich: + buildable: false + externals: + - spec: cray-mpich@8.1.25 %gcc + prefix: /opt/cray/pe/mpich/8.1.25/ofi/gnu/9.1/ + modules: + - cray-mpich/8.1.25 + cray-libsci: + buildable: false + externals: + - spec: cray-libsci@23.02.1.1 %gcc + prefix: /opt/cray/pe/libsci/23.02.1.1/GNU/9.1/x86_64/ + modules: + - cray-libsci/23.02.1.1 \ No newline at end of file diff --git a/share/spack/gitlab/cloud_pipelines/configs/cray-sles/zen4/ci.yaml b/share/spack/gitlab/cloud_pipelines/configs/cray-sles/zen4/ci.yaml new file mode 100644 index 0000000000..e96413a14c --- /dev/null +++ b/share/spack/gitlab/cloud_pipelines/configs/cray-sles/zen4/ci.yaml @@ -0,0 +1,4 @@ +ci: + pipeline-gen: + - build-job: + tags: ["cray-sles-zen4"] diff --git a/share/spack/gitlab/cloud_pipelines/configs/cray/zen4/ci.yaml b/share/spack/gitlab/cloud_pipelines/configs/cray/zen4/ci.yaml deleted file mode 100644 index 3bee4c4000..0000000000 --- a/share/spack/gitlab/cloud_pipelines/configs/cray/zen4/ci.yaml +++ /dev/null @@ -1,4 +0,0 @@ -ci: - pipeline-gen: - - build-job: - tags: ["cce@15.0.1", "cray-zen4"] diff --git a/share/spack/gitlab/cloud_pipelines/stacks/e4s-cray-rhel/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/e4s-cray-rhel/spack.yaml new file mode 100644 index 0000000000..2407c17bde --- /dev/null +++ b/share/spack/gitlab/cloud_pipelines/stacks/e4s-cray-rhel/spack.yaml @@ -0,0 +1,65 @@ +spack: + view: false + + concretizer: + reuse: false + unify: false + + packages: + all: + require: '%cce' + compiler: [cce] + providers: + blas: [cray-libsci] + lapack: [cray-libsci] + mpi: [cray-mpich] + tbb: [intel-tbb] + scalapack: [netlib-scalapack] + target: [zen4] + variants: +mpi + + binutils: + variants: +ld +gold +headers +libiberty ~nls + hdf5: + variants: +fortran +hl +shared + libunwind: + variants: +pic +xz + ncurses: + require: '@6.3 +termlib' + openblas: + require: '@0.3.20' + variants: threads=openmp + xz: + variants: +pic + elfutils: + variants: +bzip2 ~nls +xz + require: '%gcc' + unzip: + require: '%gcc' + + specs: + - adios2 + - amrex + - butterflypack + - conduit + - flux-core + - h5bench + - hdf5-vol-async + - hdf5-vol-cache + - hdf5-vol-log + - hypre + - kokkos + - kokkos-kernels + - legion + - mfem + - petsc + - py-petsc4py + - raja + - slepc + - superlu-dist + - tau + + mirrors: { "mirror": "s3://spack-binaries-cray/develop/e4s-cray-rhel" } + + cdash: + build-group: E4S Cray \ No newline at end of file diff --git a/share/spack/gitlab/cloud_pipelines/stacks/e4s-cray-sles/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/e4s-cray-sles/spack.yaml new file mode 100644 index 0000000000..9fa4625f08 --- /dev/null +++ b/share/spack/gitlab/cloud_pipelines/stacks/e4s-cray-sles/spack.yaml @@ -0,0 +1,48 @@ +spack: + view: false + + concretizer: + reuse: false + unify: false + + packages: + all: + require: '%gcc' + providers: + blas: [cray-libsci] + lapack: [cray-libsci] + mpi: [cray-mpich] + tbb: [intel-tbb] + scalapack: [netlib-scalapack] + target: [zen4] + variants: +mpi + + binutils: + variants: +ld +gold +headers +libiberty ~nls + hdf5: + variants: +fortran +hl +shared + libunwind: + variants: +pic +xz + ncurses: + require: '@6.3 +termlib' + openblas: + require: '@0.3.20' + variants: threads=openmp + xz: + variants: +pic + elfutils: + variants: +bzip2 ~nls +xz + require: '%gcc' + + specs: + - butterflypack + - hypre + - kokkos + - kokkos-kernels + - raja + - superlu-dist + + mirrors: { "mirror": "s3://spack-binaries-cray/develop/e4s-cray-sles" } + + cdash: + build-group: E4S Cray SLES \ No newline at end of file diff --git a/share/spack/gitlab/cloud_pipelines/stacks/e4s-cray/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/e4s-cray/spack.yaml deleted file mode 100644 index ef644a4fb1..0000000000 --- a/share/spack/gitlab/cloud_pipelines/stacks/e4s-cray/spack.yaml +++ /dev/null @@ -1,113 +0,0 @@ -spack: - view: false - - concretizer: - reuse: false - unify: false - - compilers: - - compiler: - spec: cce@15.0.1 - paths: - cc: cc - cxx: CC - f77: ftn - fc: ftn - flags: {} - operating_system: rhel8 - target: any - modules: - - PrgEnv-cray/8.3.3 - - cce/15.0.1 - environment: - set: - MACHTYPE: x86_64 - - compiler: - spec: gcc@11.2.0 - paths: - cc: gcc - cxx: g++ - f77: gfortran - fc: gfortran - flags: {} - operating_system: rhel8 - target: any - modules: - - PrgEnv-gnu - - gcc/11.2.0 - environment: {} - - packages: - all: - require: '%cce@15.0.1' - compiler: [cce@15.0.1] - providers: - blas: [cray-libsci] - lapack: [cray-libsci] - mpi: [cray-mpich] - tbb: [intel-tbb] - scalapack: [netlib-scalapack] - target: [zen4] - variants: +mpi - - binutils: - variants: +ld +gold +headers +libiberty ~nls - hdf5: - variants: +fortran +hl +shared - libunwind: - variants: +pic +xz - ncurses: - require: '@6.3 +termlib' - openblas: - require: '@0.3.20' - variants: threads=openmp - xz: - variants: +pic - elfutils: - variants: +bzip2 ~nls +xz - require: '%gcc' - unzip: - require: '%gcc' - - # EXTERNALS - cray-mpich: - buildable: false - externals: - - spec: cray-mpich@8.1.25 %cce@15.0.1 - prefix: /opt/cray/pe/mpich/8.1.25/ofi/cray/10.0 - modules: - - cray-mpich/8.1.25 - cray-libsci: - buildable: false - externals: - - spec: cray-libsci@23.02.1.1 %cce@15.0.1 - prefix: /opt/cray/pe/libsci/23.02.1.1/CRAY/9.0/x86_64/ - modules: - - cray-libsci/23.02.1.1 - - specs: - - adios2 - - amrex - - butterflypack - - conduit - - flux-core - - h5bench - - hdf5-vol-async - - hdf5-vol-cache - - hdf5-vol-log - - hypre - - kokkos - - kokkos-kernels - - legion - - mfem - - petsc - - py-petsc4py - - raja - - slepc - - superlu-dist - - tau - - mirrors: { "mirror": "s3://spack-binaries-cray/develop/e4s-cray" } - - cdash: - build-group: E4S Cray \ No newline at end of file