diff --git a/var/spack/repos/builtin/packages/arpack-ng/package.py b/var/spack/repos/builtin/packages/arpack-ng/package.py index dab97ea656..ee3e3c218c 100644 --- a/var/spack/repos/builtin/packages/arpack-ng/package.py +++ b/var/spack/repos/builtin/packages/arpack-ng/package.py @@ -63,6 +63,11 @@ class ArpackNg(Package): # Fujitsu compiler does not support 'isnan' function. # isnan: function that determines whether it is NaN. patch('incompatible_isnan_fix.patch', when='%fj') + patch('incompatible_isnan_fix.patch', when='@3.7.0%xl') + patch('incompatible_isnan_fix.patch', when='@3.7.0%xl_r') + + patch('xlf.patch', when='@3.7.0%xl', level=0) + patch('xlf.patch', when='@3.7.0%xl_r', level=0) depends_on('blas') depends_on('lapack') diff --git a/var/spack/repos/builtin/packages/arpack-ng/xlf.patch b/var/spack/repos/builtin/packages/arpack-ng/xlf.patch new file mode 100644 index 0000000000..1746ee591d --- /dev/null +++ b/var/spack/repos/builtin/packages/arpack-ng/xlf.patch @@ -0,0 +1,50 @@ +--- debug.h.orig 2019-01-12 07:24:12.000000000 -0800 ++++ debug.h 2020-11-10 14:02:59.269942675 -0800 +@@ -5,12 +5,12 @@ + ! %---------------------------------% + ! | See debug.doc for documentation | + ! %---------------------------------% +- integer logfil, ndigit, mgetv0, & +- msaupd, msaup2, msaitr, mseigt, msapps, msgets, mseupd,& +- mnaupd, mnaup2, mnaitr, mneigh, mnapps, mngets, mneupd,& +- mcaupd, mcaup2, mcaitr, mceigh, mcapps, mcgets, mceupd +- common /debug/ & +- logfil, ndigit, mgetv0, & +- msaupd, msaup2, msaitr, mseigt, msapps, msgets, mseupd,& +- mnaupd, mnaup2, mnaitr, mneigh, mnapps, mngets, mneupd,& +- mcaupd, mcaup2, mcaitr, mceigh, mcapps, mcgets, mceupd ++ integer logfil, ndigit, mgetv0, ++ & msaupd, msaup2, msaitr, mseigt, msapps, msgets, mseupd, ++ & mnaupd, mnaup2, mnaitr, mneigh, mnapps, mngets, mneupd, ++ & mcaupd, mcaup2, mcaitr, mceigh, mcapps, mcgets, mceupd ++ common /debug/ ++ & logfil, ndigit, mgetv0, ++ & msaupd, msaup2, msaitr, mseigt, msapps, msgets, mseupd, ++ & mnaupd, mnaup2, mnaitr, mneigh, mnapps, mngets, mneupd, ++ & mcaupd, mcaup2, mcaitr, mceigh, mcapps, mcgets, mceupd +--- stat.h.orig 2019-01-12 07:24:12.000000000 -0800 ++++ stat.h 2020-11-10 14:14:01.276289946 -0800 +@@ -9,13 +9,13 @@ + save t0, t1, t2, t3, t4, t5 + ! + integer nopx, nbx, nrorth, nitref, nrstrt +- real tsaupd, tsaup2, tsaitr, tseigt, tsgets, tsapps, tsconv,& +- tnaupd, tnaup2, tnaitr, tneigh, tngets, tnapps, tnconv,& +- tcaupd, tcaup2, tcaitr, tceigh, tcgets, tcapps, tcconv,& +- tmvopx, tmvbx, tgetv0, titref, trvec +- common /timing/ & +- nopx, nbx, nrorth, nitref, nrstrt, & +- tsaupd, tsaup2, tsaitr, tseigt, tsgets, tsapps, tsconv,& +- tnaupd, tnaup2, tnaitr, tneigh, tngets, tnapps, tnconv,& +- tcaupd, tcaup2, tcaitr, tceigh, tcgets, tcapps, tcconv,& +- tmvopx, tmvbx, tgetv0, titref, trvec ++ real tsaupd, tsaup2, tsaitr, tseigt, tsgets, tsapps, tsconv, ++ & tnaupd, tnaup2, tnaitr, tneigh, tngets, tnapps, tnconv, ++ & tcaupd, tcaup2, tcaitr, tceigh, tcgets, tcapps, tcconv, ++ & tmvopx, tmvbx, tgetv0, titref, trvec ++ common /timing/ ++ & nopx, nbx, nrorth, nitref, nrstrt, ++ & tsaupd, tsaup2, tsaitr, tseigt, tsgets, tsapps, tsconv, ++ & tnaupd, tnaup2, tnaitr, tneigh, tngets, tnapps, tnconv, ++ & tcaupd, tcaup2, tcaitr, tceigh, tcgets, tcapps, tcconv, ++ & tmvopx, tmvbx, tgetv0, titref, trvec diff --git a/var/spack/repos/builtin/packages/ceed/package.py b/var/spack/repos/builtin/packages/ceed/package.py index 1090385a32..9a2c66f753 100644 --- a/var/spack/repos/builtin/packages/ceed/package.py +++ b/var/spack/repos/builtin/packages/ceed/package.py @@ -6,7 +6,7 @@ from spack import * -class Ceed(BundlePackage): +class Ceed(BundlePackage, CudaPackage, ROCmPackage): """Ceed is a collection of benchmarks, miniapps, software libraries and APIs for efficient high-order finite element and spectral element discretizations for exascale applications developed in the Department of @@ -18,12 +18,11 @@ class Ceed(BundlePackage): maintainers = ['jedbrown', 'v-dobrev', 'tzanio'] + version('4.0.0') version('3.0.0') version('2.0.0') version('1.0.0') - variant('cuda', default=False, - description='Enable CUDA support') variant('mfem', default=True, description='Build MFEM, Laghos and Remhos') variant('nek', default=True, description='Build Nek5000, GSLIB, Nekbone, and NekCEM') @@ -38,6 +37,17 @@ class Ceed(BundlePackage): # TODO: Add 'int64' variant? # LibCEED + # ceed 4.0 + depends_on('libceed@0.8~cuda', when='@4.0.0~cuda') + for arch in CudaPackage.cuda_arch_values: + depends_on('libceed@0.8+cuda+magma cuda_arch={0}'.format(arch), + when='@4.0.0+cuda cuda_arch={0}'.format(arch)) + depends_on('libceed@0.8~rocm', when='@4.0.0~rocm') + for target in ROCmPackage.amdgpu_targets: + depends_on('libceed@0.8+rocm amdgpu_target={0}'.format(target), + when='@4.0.0+rocm amdgpu_target={0}'.format(target)) + depends_on('libceed@0.8+occa', when='@4.0.0+occa') + depends_on('libceed@0.8~occa', when='@4.0.0~occa') # ceed-3.0 depends_on('libceed@0.6~cuda', when='@3.0.0~cuda') depends_on('libceed@0.6+cuda+magma', when='@3.0.0+cuda') @@ -55,6 +65,9 @@ class Ceed(BundlePackage): depends_on('libceed@0.2~occa', when='@1.0.0~occa') # OCCA + # ceed-4.0 + depends_on('occa@1.1.0~cuda', when='@4.0.0+occa~cuda') + depends_on('occa@1.1.0+cuda', when='@4.0.0+occa+cuda') # ceed-3.0 depends_on('occa@1.0.9~cuda', when='@3.0.0+occa~cuda') depends_on('occa@1.0.9+cuda', when='@3.0.0+occa+cuda') @@ -65,13 +78,23 @@ class Ceed(BundlePackage): depends_on('occa@1.0.0-alpha.5~cuda', when='@1.0.0+occa~cuda') depends_on('occa@1.0.0-alpha.5+cuda', when='@1.0.0+occa+cuda') + # NekRS + # ceed-4.0 + depends_on('nekrs@21.0', when='@4.0.0+nek') + for arch in CudaPackage.cuda_arch_values: + depends_on('nekrs@21.0+cuda cuda_arch={0}'.format(arch), + when='@4.0.0+nek+cuda cuda_arch={0}'.format(arch)) + for target in ROCmPackage.amdgpu_targets: + depends_on('nekrs@21.0+rocm amdgpu_target={0}'.format(target), + when='@4.0.0+nek+rocm amdgpu_target={0}'.format(target)) + # Nek5000, GSLIB, Nekbone, and NekCEM - # ceed-3.0 - depends_on('nek5000@19.0', when='@3.0.0+nek') - depends_on('nektools@19.0%gcc', when='@3.0.0+nek') - depends_on('gslib@1.0.6', when='@3.0.0+nek') - depends_on('nekbone@17.0', when='@3.0.0+nek') - depends_on('nekcem@c8db04b', when='@3.0.0+nek') + # ceed-3.0 and ceed-4.0 + depends_on('nek5000@19.0', when='@3.0.0:4.99+nek') + depends_on('nektools@19.0%gcc', when='@3.0.0:4.99+nek') + depends_on('gslib@1.0.6', when='@3.0.0:4.99+nek') + depends_on('nekbone@17.0', when='@3.0.0:4.99+nek') + depends_on('nekcem@c8db04b', when='@3.0.0:4.99+nek') # ceed-2.0 depends_on('nek5000@17.0', when='@2.0.0+nek') depends_on('nektools@17.0%gcc', when='@2.0.0+nek') @@ -86,6 +109,17 @@ class Ceed(BundlePackage): depends_on('nekcem@0b8bedd', when='@1.0.0+nek') # PETSc + # ceed 4.0 + depends_on('petsc@3.15.0:3.15.99', when='@4.0.0:4.99.99+petsc') + for arch in CudaPackage.cuda_arch_values: + depends_on('petsc+cuda cuda_arch={0}'.format(arch), + when='@4.0.0+petsc+cuda cuda_arch={0}'.format(arch)) + for target in ROCmPackage.amdgpu_targets: + depends_on('petsc@3.15.0:3.15.99+rocm amdgpu_target={0}'.format(target), + when='@4.0.0:4.99.99+petsc+rocm amdgpu_target={0}'.format(target)) + depends_on('petsc@3.15.0:3.15.99~hdf5~superlu-dist', + when='@4.0.0+petsc+quickbuild') + depends_on('petsc@3.15.0:3.15.99+mpi+double~int64', when='@4.0.0:4.99.99+petsc~mfem') # ceed-3.0 depends_on('petsc+cuda', when='@3.0.0+petsc+cuda') # For a +quickbuild we disable hdf5, and superlu-dist in PETSc. @@ -125,6 +159,10 @@ class Ceed(BundlePackage): depends_on('hpgmg@a0a5510df23b+fe', when='@1.0.0+petsc') # MAGMA + # ceed-4.0 + for arch in CudaPackage.cuda_arch_values: + depends_on('magma@2.5.4 cuda_arch={0}'.format(arch), + when='@4.0.0+cuda cuda_arch={0}'.format(arch)) # ceed-3.0 depends_on('magma@2.5.3', when='@3.0.0+cuda') # ceed-2.0 @@ -133,6 +171,8 @@ class Ceed(BundlePackage): depends_on('magma@2.3.0', when='@1.0.0+cuda') # PUMI + # ceed-4.0 + depends_on('pumi@2.2.5', when='@4.0.0+pumi') # ceed-3.0 depends_on('pumi@2.2.2', when='@3.0.0+pumi') # ceed-2.0 @@ -141,6 +181,22 @@ class Ceed(BundlePackage): depends_on('pumi@2.1.0', when='@1.0.0+pumi') # MFEM, Laghos, Remhos + # ceed-4.0 + depends_on('mfem@4.2.0+mpi+examples+miniapps', when='@4.0.0+mfem~petsc') + depends_on('mfem@4.2.0+mpi+petsc+examples+miniapps', + when='@4.0.0+mfem+petsc') + depends_on('mfem@4.2.0+pumi', when='@4.0.0+mfem+pumi') + depends_on('mfem@4.2.0+gslib', when='@4.0.0+mfem+nek') + depends_on('mfem@4.2.0+libceed', when='@4.0.0+mfem') + for arch in CudaPackage.cuda_arch_values: + depends_on('mfem@4.2.0+cuda cuda_arch={0}'.format(arch), + when='@4.0.0+mfem+cuda cuda_arch={0}'.format(arch)) + for target in ROCmPackage.amdgpu_targets: + depends_on('mfem@4.2.0+rocm amdgpu_target={0}'.format(target), + when='@4.0.0+mfem+rocm amdgpu_target={0}'.format(target)) + depends_on('mfem@4.2.0+occa', when='@4.0.0+mfem+occa') + depends_on('laghos@3.1', when='@4.0.0+mfem') + depends_on('remhos@1.0', when='@4.0.0+mfem') # ceed-3.0 depends_on('mfem@4.1.0+mpi+examples+miniapps', when='@3.0.0+mfem~petsc') depends_on('mfem@4.1.0+mpi+petsc+examples+miniapps', diff --git a/var/spack/repos/builtin/packages/laghos/package.py b/var/spack/repos/builtin/packages/laghos/package.py index 03dd7ca4ab..0f327ceb1c 100644 --- a/var/spack/repos/builtin/packages/laghos/package.py +++ b/var/spack/repos/builtin/packages/laghos/package.py @@ -21,6 +21,7 @@ class Laghos(MakefilePackage): maintainers = ['v-dobrev', 'tzanio', 'vladotomov'] version('develop', branch='master') + version('3.1', sha256='49b65edcbf9732c7f6c228958620e18980c43ad8381315a8ba9957ecb7534cd5') version('3.0', sha256='4db56286e15b42ecdc8d540c4888a7dec698b019df9c7ccb8319b7ea1f92d8b4') version('2.0', sha256='dd3632d5558889beec2cd3c49eb60f633f99e6d886ac868731610dd006c44c14') version('1.1', sha256='53b9bfe2af263c63eb4544ca1731dd26f40b73a0d2775a9883db51821bf23b7f') @@ -32,7 +33,8 @@ class Laghos(MakefilePackage): depends_on('mfem+mpi~metis', when='~metis') depends_on('mfem@develop', when='@develop') - depends_on('mfem@4.1.0:', when='@3.0') + depends_on('mfem@4.2.0', when='@3.1') + depends_on('mfem@4.1.0:4.1.99', when='@3.0') # Recommended mfem version for laghos v2.0 is: ^mfem@3.4.1-laghos-v2.0 depends_on('mfem@3.4.0:', when='@2.0') # Recommended mfem version for laghos v1.x is: ^mfem@3.3.1-laghos-v1.0 diff --git a/var/spack/repos/builtin/packages/libceed/libceed-v0.8-hip.patch b/var/spack/repos/builtin/packages/libceed/libceed-v0.8-hip.patch new file mode 100644 index 0000000000..c947e12efe --- /dev/null +++ b/var/spack/repos/builtin/packages/libceed/libceed-v0.8-hip.patch @@ -0,0 +1,14 @@ +diff --git a/Makefile b/Makefile +index 4f1737ee..3f1811f2 100644 +--- a/Makefile ++++ b/Makefile +@@ -403,6 +403,9 @@ ifneq ($(HIP_LIB_DIR),) + CPPFLAGS += $(subst =,,$(shell $(HIP_DIR)/bin/hipconfig -C)) + endif + $(libceeds) : CPPFLAGS += -I$(HIP_DIR)/include ++ ifneq ($(HIPBLAS_DIR),) ++ PKG_LIBS += -L$(HIPBLAS_DIR)/lib ++ endif + PKG_LIBS += -L$(abspath $(HIP_LIB_DIR)) -lamdhip64 -lhipblas + LIBCEED_CONTAINS_CXX = 1 + libceed.c += interface/ceed-hip.c diff --git a/var/spack/repos/builtin/packages/libceed/package.py b/var/spack/repos/builtin/packages/libceed/package.py index b73fa98c4d..d94d13059b 100644 --- a/var/spack/repos/builtin/packages/libceed/package.py +++ b/var/spack/repos/builtin/packages/libceed/package.py @@ -6,7 +6,7 @@ from spack import * -class Libceed(Package): +class Libceed(Package, CudaPackage, ROCmPackage): """The CEED API Library: Code for Efficient Extensible Discretizations.""" homepage = "https://github.com/CEED/libCEED" @@ -15,6 +15,7 @@ class Libceed(Package): maintainers = ['jedbrown', 'v-dobrev', 'tzanio'] version('develop', branch='main') + version('0.8', tag='v0.8') version('0.7', tag='v0.7') version('0.6', commit='c7f533e01e2f3f6720fbf37aac2af2ffed225f60') # tag v0.6 + small portability fixes version('0.5', tag='v0.5') @@ -22,9 +23,7 @@ class Libceed(Package): version('0.2', tag='v0.2') version('0.1', tag='v0.1') - variant('occa', default=True, description='Enable OCCA backends') - variant('cuda', default=False, description='Enable CUDA support') - variant('hip', default=False, description='Enable HIP support') + variant('occa', default=False, description='Enable OCCA backends') variant('debug', default=False, description='Enable debug build') variant('libxsmm', default=False, description='Enable LIBXSMM backend') variant('magma', default=False, description='Enable MAGMA backend') @@ -34,8 +33,9 @@ class Libceed(Package): conflicts('+hip', when='@:0.6') depends_on('cuda', when='+cuda') - depends_on('hip@3.8.0', when='@0.7:0.7.99+hip') - depends_on('hip@3.8.0:', when='@0.8:+hip') + depends_on('hip@3.8.0', when='@0.7:0.7.99+rocm') + depends_on('hip@3.8.0:', when='@0.8:+rocm') + depends_on('hipblas@3.8.0:', when='@0.8:+rocm') depends_on('occa@develop', when='@develop+occa') depends_on('occa@1.1.0', when='@0.7:+occa') @@ -48,6 +48,7 @@ class Libceed(Package): depends_on('magma', when='+magma') + patch('libceed-v0.8-hip.patch', when='@0.8+rocm') patch('pkgconfig-version-0.4.diff', when='@0.4') # occa: do not occaFree kernels @@ -77,6 +78,8 @@ def common_make_opts(self): opt = '-O3 -g -ffp-contract=fast' if compiler.version >= ver(4.9): opt += ' -fopenmp-simd' + if self.spec.target.family in ['x86_64', 'aarch64']: + opt += ' -march=native' elif compiler.name == 'apple-clang': opt = '-O3 -g -march=native -ffp-contract=fast' if compiler.version >= ver(10): @@ -87,11 +90,19 @@ def common_make_opts(self): opt += ' -fopenmp-simd' elif compiler.name in ['xl', 'xl_r']: opt = '-O -g -qsimd=auto' + elif compiler.name == 'intel': + opt = '-O3 -g' + makeopts += ['CC_VENDOR=icc'] else: opt = '-O -g' + # Note: spack will inject additional target-specific flags through + # the compiler wrapper. makeopts += ['OPT=%s' % opt] - if 'avx' in self.spec.target: + if spec.satisfies('@0.7') and compiler.name in ['xl', 'xl_r']: + makeopts += ['CXXFLAGS.XL=-qpic -std=c++11 -MMD'] + + if spec.satisfies('@:0.7') and 'avx' in self.spec.target: makeopts.append('AVX=1') if '+cuda' in spec: @@ -105,8 +116,10 @@ def common_make_opts(self): # Disable CUDA auto-detection: makeopts += ['CUDA_DIR=/disable-cuda'] - if '+hip' in spec: + if '+rocm' in spec: makeopts += ['HIP_DIR=%s' % spec['hip'].prefix] + if spec.satisfies('@0.8'): + makeopts += ['HIPBLAS_DIR=%s' % spec['hipblas'].prefix] if '+libxsmm' in spec: makeopts += ['XSMM_DIR=%s' % spec['libxsmm'].prefix] diff --git a/var/spack/repos/builtin/packages/mfem/mfem-4.2-petsc-3.15.0.patch b/var/spack/repos/builtin/packages/mfem/mfem-4.2-petsc-3.15.0.patch new file mode 100644 index 0000000000..cb79a2130f --- /dev/null +++ b/var/spack/repos/builtin/packages/mfem/mfem-4.2-petsc-3.15.0.patch @@ -0,0 +1,32 @@ +diff --git a/linalg/petsc.cpp b/linalg/petsc.cpp +index fcfdd325f..90eedf042 100644 +--- a/linalg/petsc.cpp ++++ b/linalg/petsc.cpp +@@ -1872,8 +1872,13 @@ void PetscSolver::SetPrintLevel(int plev) + { + ierr = PetscViewerAndFormatCreate(viewer,PETSC_VIEWER_DEFAULT,&vf); + PCHKERRQ(viewer,ierr); ++#if PETSC_VERSION_LT(3,15,0) + ierr = KSPMonitorSet(ksp,(myMonitor)KSPMonitorTrueResidualNorm,vf, + (myPetscFunc)PetscViewerAndFormatDestroy); ++#else ++ ierr = KSPMonitorSet(ksp,(myMonitor)KSPMonitorTrueResidual,vf, ++ (myPetscFunc)PetscViewerAndFormatDestroy); ++#endif + PCHKERRQ(ksp,ierr); + } + } +@@ -2887,8 +2892,13 @@ void PetscBDDCSolver::BDDCSolverConstructor(const PetscBDDCSolverParams &opts) + + ierr = VecGetArrayRead(pvec_coords,&garray); CCHKERRQ(PETSC_COMM_SELF,ierr); + ierr = VecGetArray(lvec_coords,&larray); CCHKERRQ(PETSC_COMM_SELF,ierr); ++#if PETSC_VERSION_LT(3,15,0) + ierr = PetscSFBcastBegin(sf,MPIU_SCALAR,garray,larray); CCHKERRQ(comm,ierr); + ierr = PetscSFBcastEnd(sf,MPIU_SCALAR,garray,larray); CCHKERRQ(comm,ierr); ++#else ++ ierr = PetscSFBcastBegin(sf,MPIU_SCALAR,garray,larray,MPI_REPLACE); CCHKERRQ(comm,ierr); ++ ierr = PetscSFBcastEnd(sf,MPIU_SCALAR,garray,larray,MPI_REPLACE); CCHKERRQ(comm,ierr); ++#endif + ierr = VecRestoreArrayRead(pvec_coords,&garray); CCHKERRQ(PETSC_COMM_SELF,ierr); + ierr = VecRestoreArray(lvec_coords,&larray); CCHKERRQ(PETSC_COMM_SELF,ierr); + } diff --git a/var/spack/repos/builtin/packages/mfem/mfem-4.2-slepc.patch b/var/spack/repos/builtin/packages/mfem/mfem-4.2-slepc.patch new file mode 100644 index 0000000000..8ba4338c4e --- /dev/null +++ b/var/spack/repos/builtin/packages/mfem/mfem-4.2-slepc.patch @@ -0,0 +1,40 @@ +diff --git a/linalg/slepc.hpp b/linalg/slepc.hpp +index 7f8911ba2..5b1f1c686 100644 +--- a/linalg/slepc.hpp ++++ b/linalg/slepc.hpp +@@ -19,12 +19,15 @@ + + #include "petsc.hpp" + +-// Forward declarations +-typedef struct _p_EPS *EPS; ++// Forward declaration of SLEPc's internal struct _p_EPS: ++struct _p_EPS; + + namespace mfem + { + ++// Declare an alias of SLEPc's EPS type, mfem::slepc::EPS: ++namespace slepc { typedef struct ::_p_EPS *EPS; } ++ + void MFEMInitializeSlepc(); + void MFEMInitializeSlepc(int*,char***); + void MFEMInitializeSlepc(int*,char***,const char[],const char[]); +@@ -37,7 +40,7 @@ private: + mutable bool clcustom; + + /// SLEPc linear eigensolver object +- EPS eps; ++ slepc::EPS eps; + + /// Real and imaginary part of eigenvector + mutable PetscParVector *VR, *VC; +@@ -102,7 +105,7 @@ public: + void SetSpectralTransformation(SpectralTransformation transformation); + + /// Conversion function to SLEPc's EPS type. +- operator EPS() const { return eps; } ++ operator slepc::EPS() const { return eps; } + + /// Conversion function to PetscObject + operator PetscObject() const {return (PetscObject)eps; } diff --git a/var/spack/repos/builtin/packages/mfem/mfem-4.2-umpire.patch b/var/spack/repos/builtin/packages/mfem/mfem-4.2-umpire.patch new file mode 100644 index 0000000000..9ab0214d6c --- /dev/null +++ b/var/spack/repos/builtin/packages/mfem/mfem-4.2-umpire.patch @@ -0,0 +1,24 @@ +diff --git a/general/mem_manager.cpp b/general/mem_manager.cpp +index ff4a03465..748f6e673 100644 +--- a/general/mem_manager.cpp ++++ b/general/mem_manager.cpp +@@ -482,7 +482,8 @@ public: + HostMemorySpace(), + name(mm.GetUmpireAllocatorHostName()), + rm(umpire::ResourceManager::getInstance()), +- h_allocator(rm.isAllocator(name)? rm.getAllocator(name): ++ h_allocator((!std::strcmp(name, "HOST") || rm.isAllocator(name)) ? ++ rm.getAllocator(name) : + rm.makeAllocator + (name, rm.getAllocator("HOST"))), + strat(h_allocator.getAllocationStrategy()) { } +@@ -506,7 +507,8 @@ public: + DeviceMemorySpace(), + name(mm.GetUmpireAllocatorDeviceName()), + rm(umpire::ResourceManager::getInstance()), +- d_allocator(rm.isAllocator(name)? rm.getAllocator(name): ++ d_allocator((!std::strcmp(name, "DEVICE") || rm.isAllocator(name)) ? ++ rm.getAllocator(name) : + rm.makeAllocator + (name, rm.getAllocator("DEVICE"))) { } + void Alloc(Memory &base) { base.d_ptr = d_allocator.allocate(base.bytes); } diff --git a/var/spack/repos/builtin/packages/mfem/package.py b/var/spack/repos/builtin/packages/mfem/package.py index d9e88be3d0..3b97e77190 100644 --- a/var/spack/repos/builtin/packages/mfem/package.py +++ b/var/spack/repos/builtin/packages/mfem/package.py @@ -6,9 +6,10 @@ from spack import * import os import shutil +import sys -class Mfem(Package): +class Mfem(Package, CudaPackage, ROCmPackage): """Free, lightweight, scalable C++ library for finite element methods.""" tags = ['FEM', 'finite elements', 'high-order', 'AMR', 'HPC'] @@ -100,9 +101,8 @@ class Mfem(Package): description='Enable METIS support') variant('openmp', default=False, description='Enable OpenMP parallelism') - variant('cuda', default=False, description='Enable CUDA support') - variant('cuda_arch', default='sm_60', - description='CUDA architecture to compile for') + # Note: '+cuda' and 'cuda_arch' variants are added by the CudaPackage + # Note: '+rocm' and 'amdgpu_target' variants are added by the ROCmPackage variant('occa', default=False, description='Enable OCCA backend') variant('raja', default=False, description='Enable RAJA backend') variant('libceed', default=False, description='Enable libCEED backend') @@ -121,6 +121,8 @@ class Mfem(Package): description='Enable serial, sparse direct solvers') variant('petsc', default=False, description='Enable PETSc solvers, preconditioners, etc.') + variant('slepc', default=False, + description='Enable SLEPc integration') variant('sundials', default=False, description='Enable Sundials time integrators') variant('pumi', default=False, @@ -143,6 +145,7 @@ class Mfem(Package): description='Enable secure sockets using GnuTLS') variant('libunwind', default=False, description='Enable backtrace on error support using Libunwind') + # TODO: SIMD, Ginkgo, ADIOS2, HiOp, MKL CPardiso, Axom/Sidre variant('timer', default='auto', values=('auto', 'std', 'posix', 'mac', 'mpi'), description='Timing functions to use in mfem::StopWatch') @@ -153,18 +156,21 @@ class Mfem(Package): conflicts('+shared', when='@:3.3.2') conflicts('~static~shared') - conflicts('~threadsafe', when='+openmp') + conflicts('~threadsafe', when='@:3.99.99+openmp') conflicts('+cuda', when='@:3.99.99') + conflicts('+rocm', when='@:4.1.99') + conflicts('+cuda+rocm') conflicts('+netcdf', when='@:3.1') conflicts('+superlu-dist', when='@:3.1') # STRUMPACK support was added in mfem v3.3.2, however, here we allow only - # strumpack v3 support which is available starting with mfem v4.0: + # strumpack v3+ support for which is available starting with mfem v4.0: conflicts('+strumpack', when='@:3.99.99') conflicts('+gnutls', when='@:3.1') conflicts('+zlib', when='@:3.2') conflicts('+mpfr', when='@:3.2') conflicts('+petsc', when='@:3.2') + conflicts('+slepc', when='@:4.1.99') conflicts('+sundials', when='@:3.2') conflicts('+pumi', when='@:3.3.2') conflicts('+gslib', when='@:4.0.99') @@ -183,6 +189,7 @@ class Mfem(Package): conflicts('+superlu-dist', when='~mpi') conflicts('+strumpack', when='~mpi') conflicts('+petsc', when='~mpi') + conflicts('+slepc', when='~petsc') conflicts('+pumi', when='~mpi') conflicts('timer=mpi', when='~mpi') @@ -194,27 +201,32 @@ class Mfem(Package): depends_on('blas', when='+lapack') depends_on('lapack@3.0:', when='+lapack') - depends_on('cuda', when='+cuda') - depends_on('sundials@2.7.0', when='@:3.3.0+sundials~mpi') depends_on('sundials@2.7.0+mpi+hypre', when='@:3.3.0+sundials+mpi') depends_on('sundials@2.7.0:', when='@3.3.2:+sundials~mpi') depends_on('sundials@2.7.0:+mpi+hypre', when='@3.3.2:+sundials+mpi') depends_on('sundials@5.0.0:', when='@4.0.1-xsdk:+sundials~mpi') depends_on('sundials@5.0.0:+mpi+hypre', when='@4.0.1-xsdk:+sundials+mpi') - depends_on('sundials@5.4.0:+cuda', when='@4.2.0:+sundials+cuda') - depends_on('pumi@2.2.3', when='@4.2.0:+pumi') + for sm_ in CudaPackage.cuda_arch_values: + depends_on('sundials@5.4.0:+cuda cuda_arch={0}'.format(sm_), + when='@4.2.0:+sundials+cuda cuda_arch={0}'.format(sm_)) + depends_on('pumi@2.2.3:', when='@4.2.0:+pumi') depends_on('pumi', when='+pumi~shared') depends_on('pumi+shared', when='+pumi+shared') depends_on('gslib@1.0.5:+mpi', when='+gslib+mpi') depends_on('gslib@1.0.5:~mpi~mpiio', when='+gslib~mpi') depends_on('suite-sparse', when='+suite-sparse') depends_on('superlu-dist', when='+superlu-dist') - depends_on('strumpack@3.0.0:', when='+strumpack') + depends_on('strumpack@3.0.0:', when='+strumpack~shared') + depends_on('strumpack@3.0.0:+shared', when='+strumpack+shared') + for sm_ in CudaPackage.cuda_arch_values: + depends_on('strumpack+cuda cuda_arch={0}'.format(sm_), + when='+strumpack+cuda cuda_arch={0}'.format(sm_)) # The PETSc tests in MFEM will fail if PETSc is not configured with # SuiteSparse and MUMPS. On the other hand, if we require the variants # '+suite-sparse+mumps' of PETSc, the xsdk package concretization fails. depends_on('petsc@3.8:+mpi+double+hypre', when='+petsc') + depends_on('slepc@3.8.0:', when='+slepc') # Recommended when building outside of xsdk: # depends_on('petsc@3.8:+mpi+double+hypre+suite-sparse+mumps', # when='+petsc') @@ -235,33 +247,53 @@ class Mfem(Package): # when using hypre version >= 2.16.0. # This issue is resolved in v4.2. conflicts('+strumpack', when='mfem@4.0.0:4.1.99 ^hypre@2.16.0:') + conflicts('+strumpack ^strumpack+cuda', when='~cuda') depends_on('occa@1.0.8:', when='@:4.1.99+occa') depends_on('occa@1.1.0:', when='@4.2.0:+occa') depends_on('occa+cuda', when='+occa+cuda') + # TODO: propagate '+rocm' variant to occa when it is supported depends_on('raja@0.10.0:', when='@4.0.1:+raja') depends_on('raja@0.7.0:0.9.0', when='@4.0.0+raja') - depends_on('raja+cuda', when='+raja+cuda') + for sm_ in CudaPackage.cuda_arch_values: + depends_on('raja+cuda cuda_arch={0}'.format(sm_), + when='+raja+cuda cuda_arch={0}'.format(sm_)) + for gfx in ROCmPackage.amdgpu_targets: + depends_on('raja+rocm amdgpu_target={0}'.format(gfx), + when='+raja+rocm amdgpu_target={0}'.format(gfx)) depends_on('libceed@0.6:', when='@:4.1.99+libceed') depends_on('libceed@0.7:', when='@4.2.0:+libceed') - depends_on('libceed+cuda', when='+libceed+cuda') + for sm_ in CudaPackage.cuda_arch_values: + depends_on('libceed+cuda cuda_arch={0}'.format(sm_), + when='+libceed+cuda cuda_arch={0}'.format(sm_)) + for gfx in ROCmPackage.amdgpu_targets: + depends_on('libceed+rocm amdgpu_target={0}'.format(gfx), + when='+libceed+rocm amdgpu_target={0}'.format(gfx)) depends_on('umpire@2.0.0:', when='+umpire') - depends_on('umpire+cuda', when='+umpire+cuda') - - depends_on('amgx', when='+amgx') - # MPI is enabled by default - depends_on('amgx~mpi', when='+amgx~mpi') for sm_ in CudaPackage.cuda_arch_values: - depends_on('amgx cuda_arch={0}'.format(sm_), - when='+amgx cuda_arch=sm_{0}'.format(sm_)) + depends_on('umpire+cuda cuda_arch={0}'.format(sm_), + when='+umpire+cuda cuda_arch={0}'.format(sm_)) + for gfx in ROCmPackage.amdgpu_targets: + depends_on('umpire+rocm amdgpu_target={0}'.format(gfx), + when='+umpire+rocm amdgpu_target={0}'.format(gfx)) + + # AmgX: propagate the cuda_arch and mpi settings: + for sm_ in CudaPackage.cuda_arch_values: + depends_on('amgx+mpi cuda_arch={0}'.format(sm_), + when='+amgx+mpi cuda_arch={0}'.format(sm_)) + depends_on('amgx~mpi cuda_arch={0}'.format(sm_), + when='+amgx~mpi cuda_arch={0}'.format(sm_)) patch('mfem_ppc_build.patch', when='@3.2:3.3.0 arch=ppc64le') patch('mfem-3.4.patch', when='@3.4.0') patch('mfem-3.3-3.4-petsc-3.9.patch', when='@3.3.0:3.4.0 +petsc ^petsc@3.9.0:') + patch('mfem-4.2-umpire.patch', when='@4.2.0+umpire') + patch('mfem-4.2-slepc.patch', when='@4.2.0+slepc') + patch('mfem-4.2-petsc-3.15.0.patch', when='@4.2.0+petsc ^petsc@3.15.0:') # Patch to fix MFEM makefile syntax error. See # https://github.com/mfem/mfem/issues/1042 for the bug report and @@ -339,6 +371,21 @@ def find_optional_library(name, prefix): return lib return LibraryList([]) + # Determine how to run MPI tests, e.g. when using '--test=root', when + # Spack is run inside a batch system job. + mfem_mpiexec = 'mpirun' + mfem_mpiexec_np = '-np' + if 'SLURM_JOBID' in os.environ: + mfem_mpiexec = 'srun' + mfem_mpiexec_np = '-n' + elif 'LSB_JOBID' in os.environ: + if 'LLNL_COMPUTE_NODES' in os.environ: + mfem_mpiexec = 'lrun' + mfem_mpiexec_np = '-n' + else: + mfem_mpiexec = 'jsrun' + mfem_mpiexec_np = '-p' + metis5_str = 'NO' if ('+metis' in spec) and spec['metis'].satisfies('@5:'): metis5_str = 'YES' @@ -365,6 +412,7 @@ def find_optional_library(name, prefix): 'MFEM_USE_SUITESPARSE=%s' % yes_no('+suite-sparse'), 'MFEM_USE_SUNDIALS=%s' % yes_no('+sundials'), 'MFEM_USE_PETSC=%s' % yes_no('+petsc'), + 'MFEM_USE_SLEPC=%s' % yes_no('+slepc'), 'MFEM_USE_PUMI=%s' % yes_no('+pumi'), 'MFEM_USE_GSLIB=%s' % yes_no('+gslib'), 'MFEM_USE_NETCDF=%s' % yes_no('+netcdf'), @@ -373,11 +421,14 @@ def find_optional_library(name, prefix): 'MFEM_USE_OPENMP=%s' % yes_no('+openmp'), 'MFEM_USE_CONDUIT=%s' % yes_no('+conduit'), 'MFEM_USE_CUDA=%s' % yes_no('+cuda'), + 'MFEM_USE_HIP=%s' % yes_no('+rocm'), 'MFEM_USE_OCCA=%s' % yes_no('+occa'), 'MFEM_USE_RAJA=%s' % yes_no('+raja'), 'MFEM_USE_AMGX=%s' % yes_no('+amgx'), 'MFEM_USE_CEED=%s' % yes_no('+libceed'), - 'MFEM_USE_UMPIRE=%s' % yes_no('+umpire')] + 'MFEM_USE_UMPIRE=%s' % yes_no('+umpire'), + 'MFEM_MPIEXEC=%s' % mfem_mpiexec, + 'MFEM_MPIEXEC_NP=%s' % mfem_mpiexec_np] cxxflags = spec.compiler_flags['cxxflags'] @@ -400,7 +451,7 @@ def find_optional_library(name, prefix): cxxflags = [(xcompiler + flag) for flag in cxxflags] if '+cuda' in spec: cxxflags += [ - '-x=cu --expt-extended-lambda -arch=%s' % cuda_arch, + '-x=cu --expt-extended-lambda -arch=sm_%s' % cuda_arch, '-ccbin %s' % (spec['mpi'].mpicxx if '+mpi' in spec else env['CXX'])] if self.spec.satisfies('@4.0.0:'): @@ -414,8 +465,8 @@ def find_optional_library(name, prefix): options += ['STATIC=NO'] if '+shared' in spec: options += [ - 'SHARED=YES', 'PICFLAG=%s' % self.compiler.cxx_pic_flag - ] + 'SHARED=YES', + 'PICFLAG=%s' % (xcompiler + self.compiler.cxx_pic_flag)] if '+mpi' in spec: options += ['MPICXX=%s' % spec['mpi'].mpicxx] @@ -459,24 +510,51 @@ def find_optional_library(name, prefix): # fortran library and also the MPI fortran library: if '~shared' in strumpack: if os.path.basename(env['FC']) == 'gfortran': - sp_lib += ['-lgfortran'] - if '^mpich' in strumpack: + gfortran = Executable(env['FC']) + libext = 'dylib' if sys.platform == 'darwin' else 'so' + libfile = os.path.abspath(gfortran( + '-print-file-name=libgfortran.%s' % libext, + output=str).strip()) + gfortran_lib = LibraryList(libfile) + sp_lib += [ld_flags_from_library_list(gfortran_lib)] + if ('^mpich' in strumpack) or ('^mvapich2' in strumpack): sp_lib += ['-lmpifort'] elif '^openmpi' in strumpack: sp_lib += ['-lmpi_mpifh'] + elif '^spectrum-mpi' in strumpack: + sp_lib += ['-lmpi_ibm_mpifh'] if '+openmp' in strumpack: - sp_opt += [self.compiler.openmp_flag] - if '^scalapack' in strumpack: + # The '+openmp' in the spec means strumpack will TRY to find + # OpenMP; if not found, we should not add any flags -- how do + # we figure out if strumpack found OpenMP? + if not self.spec.satisfies('%apple-clang'): + sp_opt += [xcompiler + self.compiler.openmp_flag] + if '^parmetis' in strumpack: + parmetis = strumpack['parmetis'] + sp_opt += [parmetis.headers.cpp_flags] + sp_lib += [ld_flags_from_library_list(parmetis.libs)] + if '^netlib-scalapack' in strumpack: scalapack = strumpack['scalapack'] sp_opt += ['-I%s' % scalapack.prefix.include] sp_lib += [ld_flags_from_dirs([scalapack.prefix.lib], ['scalapack'])] + elif '^scalapack' in strumpack: + scalapack = strumpack['scalapack'] + sp_opt += [scalapack.headers.cpp_flags] + sp_lib += [ld_flags_from_library_list(scalapack.libs)] if '+butterflypack' in strumpack: bp = strumpack['butterflypack'] sp_opt += ['-I%s' % bp.prefix.include] sp_lib += [ld_flags_from_dirs([bp.prefix.lib], ['dbutterflypack', 'zbutterflypack'])] + if '+zfp' in strumpack: + zfp = strumpack['zfp'] + sp_opt += ['-I%s' % zfp.prefix.include] + sp_lib += [ld_flags_from_dirs([zfp.prefix.lib], ['zfp'])] + if '+cuda' in strumpack: + # assuming also ('+cuda' in spec) + sp_lib += ['-lcusolver', '-lcublas'] options += [ 'STRUMPACK_OPT=%s' % ' '.join(sp_opt), 'STRUMPACK_LIB=%s' % ' '.join(sp_lib)] @@ -504,6 +582,12 @@ def find_optional_library(name, prefix): else: options += ['PETSC_DIR=%s' % petsc.prefix] + if '+slepc' in spec: + slepc = spec['slepc'] + options += [ + 'SLEPC_OPT=%s' % slepc.headers.cpp_flags, + 'SLEPC_LIB=%s' % ld_flags_from_library_list(slepc.libs)] + if '+pumi' in spec: pumi_libs = ['pumi', 'crv', 'ma', 'mds', 'apf', 'pcu', 'gmi', 'parma', 'lion', 'mth', 'apf_zoltan', 'spr'] @@ -563,12 +647,19 @@ def find_optional_library(name, prefix): 'LIBUNWIND_LIB=%s' % ld_flags_from_library_list(libs)] if '+openmp' in spec: - options += ['OPENMP_OPT=%s' % self.compiler.openmp_flag] + options += [ + 'OPENMP_OPT=%s' % (xcompiler + self.compiler.openmp_flag)] if '+cuda' in spec: options += [ 'CUDA_CXX=%s' % join_path(spec['cuda'].prefix, 'bin', 'nvcc'), - 'CUDA_ARCH=%s' % cuda_arch] + 'CUDA_ARCH=sm_%s' % cuda_arch] + + if '+rocm' in spec: + amdgpu_target = ','.join(spec.variants['amdgpu_target'].value) + options += [ + 'HIP_CXX=%s' % spec['hip'].hipcc, + 'HIP_ARCH=%s' % amdgpu_target] if '+occa' in spec: options += ['OCCA_OPT=-I%s' % spec['occa'].prefix.include, @@ -730,9 +821,13 @@ def test(self): @when('@4.1.0') def patch(self): # Remove the byte order mark since it messes with some compilers - filter_file(u'\uFEFF', '', 'fem/gslib.hpp') - filter_file(u'\uFEFF', '', 'fem/gslib.cpp') - filter_file(u'\uFEFF', '', 'linalg/hiop.hpp') + files_with_bom = [ + 'fem/gslib.hpp', 'fem/gslib.cpp', 'linalg/hiop.hpp', + 'miniapps/gslib/field-diff.cpp', 'miniapps/gslib/findpts.cpp', + 'miniapps/gslib/pfindpts.cpp'] + bom = '\xef\xbb\xbf' if sys.version_info < (3,) else u'\ufeff' + for f in files_with_bom: + filter_file(bom, '', f) @property def suitesparse_components(self): @@ -745,13 +840,14 @@ def suitesparse_components(self): @property def sundials_components(self): """Return the SUNDIALS components needed by MFEM.""" + spec = self.spec sun_comps = 'arkode,cvodes,nvecserial,kinsol' - if '+mpi' in self.spec: - if self.spec.satisfies('@4.2:'): + if '+mpi' in spec: + if spec.satisfies('@4.2:'): sun_comps += ',nvecparallel,nvecmpiplusx' else: sun_comps += ',nvecparhyp,nvecparallel' - if '+cuda' in self.spec: + if '+cuda' in spec and '+cuda' in spec['sundials']: sun_comps += ',nveccuda' return sun_comps diff --git a/var/spack/repos/builtin/packages/mfem/test_builds.sh b/var/spack/repos/builtin/packages/mfem/test_builds.sh index 4f3c3f787b..fd6624eaf5 100755 --- a/var/spack/repos/builtin/packages/mfem/test_builds.sh +++ b/var/spack/repos/builtin/packages/mfem/test_builds.sh @@ -2,50 +2,66 @@ # Set a compiler to test with, e.g. '%gcc', '%clang', etc. compiler='' +cuda_arch="70" mfem='mfem'${compiler} mfem_dev='mfem@develop'${compiler} backends='+occa+raja+libceed' -# Using occa@develop to help the spack concretization -backends_specs='^occa@develop~cuda ^raja~openmp' +backends_specs='^occa~cuda ^raja~openmp' -# As of 03/20/20 +mpfr breaks one of the unit tests in both @4.1.0 and @develop, -# so it is disabled here for now. -# mpfr='+mpfr' -mpfr='' +# help the concrtizer find suitable hdf5 version (conduit constraint) +hdf5_spec='^hdf5@1.8.19:1.8.999' +# petsc spec +petsc_spec='^petsc+suite-sparse+mumps' +# strumpack spec without cuda +strumpack_spec='^strumpack~slate~openmp~cuda' +strumpack_cuda_spec='^strumpack~slate~openmp' builds=( # preferred version: ${mfem} ${mfem}'~mpi~metis~zlib' - # NOTE: Skip +strumpack since mfem needs hypre < 2.16.0 in that case - ${mfem}"$backends"'+superlu-dist+suite-sparse+petsc \ - +sundials+pumi+gslib'${mpfr}'+netcdf+zlib+gnutls+libunwind+conduit \ - ^petsc+suite-sparse+mumps'" $backends_specs" + ${mfem}"$backends"'+superlu-dist+strumpack+suite-sparse+petsc+slepc \ + +sundials+pumi+gslib+mpfr+netcdf+zlib+gnutls+libunwind+conduit \ + '"$backends_specs $petsc_spec $strumpack_spec $hdf5_spec" ${mfem}'~mpi \ - '"$backends"'+suite-sparse+sundials+gslib'${mpfr}'+netcdf \ - +zlib+gnutls+libunwind+conduit'" $backends_specs" + '"$backends"'+suite-sparse+sundials+gslib+mpfr+netcdf \ + +zlib+gnutls+libunwind+conduit'" $backends_specs $hdf5_spec" # develop version: ${mfem_dev}'+shared~static' ${mfem_dev}'+shared~static~mpi~metis~zlib' - # NOTE: Skip +strumpack since mfem needs hypre < 2.16.0 in that case + # NOTE: Shared build with +gslib works on mac but not on linux + # FIXME: As of 2020/11/03 the next config fails in PETSc ex5p: + # ${mfem_dev}'+shared~static \ + # '"$backends"'+superlu-dist+strumpack+suite-sparse+petsc \ + # +sundials+pumi+mpfr+netcdf+zlib+gnutls+libunwind+conduit \ + # '"$backends_specs $petsc_spec $strumpack_spec $hdf5_spec" + # Removing just petsc works: ${mfem_dev}'+shared~static \ - '"$backends"'+superlu-dist+suite-sparse+petsc \ - +sundials+pumi'${mpfr}'+netcdf+zlib+gnutls+libunwind+conduit \ - ^petsc+suite-sparse+mumps'" $backends_specs" + '"$backends"'+superlu-dist+strumpack+suite-sparse \ + +sundials+pumi+mpfr+netcdf+zlib+gnutls+libunwind+conduit \ + '"$backends_specs $strumpack_spec $hdf5_spec" + # Removing just strumpack works on linux, fails on mac: + # ${mfem_dev}'+shared~static \ + # '"$backends"'+superlu-dist+suite-sparse+petsc \ + # +sundials+pumi+mpfr+netcdf+zlib+gnutls+libunwind+conduit \ + # '"$backends_specs $petsc_spec $hdf5_spec" + # Petsc and strumpack: fails on linux and mac in PETSc ex5p: + # ${mfem_dev}'+shared~static +strumpack+petsc \ + # '$petsc_spec $strumpack_spec" + ${mfem_dev}'+shared~static~mpi \ - '"$backends"'+suite-sparse+sundials'${mpfr}'+netcdf \ - +zlib+gnutls+libunwind+conduit'" $backends_specs" + '"$backends"'+suite-sparse+sundials+mpfr+netcdf \ + +zlib+gnutls+libunwind+conduit'" $backends_specs $hdf5_spec" ) builds2=( # preferred version ${mfem}"$backends $backends_specs" ${mfem}'+superlu-dist' - # NOTE: On mac +strumpack works only with gcc, as of 03/20/20. - ${mfem}'+strumpack ^hypre@2.15.1' + ${mfem}'+strumpack'" $strumpack_spec" ${mfem}'+suite-sparse~mpi' ${mfem}'+suite-sparse' ${mfem}'+sundials~mpi' @@ -54,16 +70,17 @@ builds2=( ${mfem}'+gslib' ${mfem}'+netcdf~mpi' ${mfem}'+netcdf' - ${mfem}${mpfr} + ${mfem}'+mpfr' ${mfem}'+gnutls' ${mfem}'+conduit~mpi' ${mfem}'+conduit' - ${mfem}'+petsc ^petsc+suite-sparse+mumps' + ${mfem}'+umpire' + ${mfem}'+petsc'" $petsc_spec" + ${mfem}'+petsc+slepc'" $petsc_spec" # develop version ${mfem_dev}"$backends $backends_specs" ${mfem_dev}'+superlu-dist' - # NOTE: On mac +strumpack works only with gcc, as of 03/20/20. - ${mfem_dev}'+strumpack ^hypre@2.15.1' + ${mfem_dev}'+strumpack'" $strumpack_spec" ${mfem_dev}'+suite-sparse~mpi' ${mfem_dev}'+suite-sparse' ${mfem_dev}'+sundials~mpi' @@ -72,19 +89,50 @@ builds2=( ${mfem_dev}'+gslib' ${mfem_dev}'+netcdf~mpi' ${mfem_dev}'+netcdf' - ${mfem_dev}${mpfr} + ${mfem_dev}'+mpfr' ${mfem_dev}'+gnutls' ${mfem_dev}'+conduit~mpi' ${mfem_dev}'+conduit' - ${mfem_dev}'+petsc ^petsc+suite-sparse+mumps' + ${mfem_dev}'+umpire' + ${mfem_dev}'+petsc'" $petsc_spec" + ${mfem_dev}'+petsc+slepc'" $petsc_spec" ) +builds_cuda=( + ${mfem}'+cuda cuda_arch='"${cuda_arch}" + + ${mfem}'+cuda+raja+occa+libceed cuda_arch='"${cuda_arch}"' \ + ^raja+cuda~openmp' + + ${mfem}'+cuda+openmp+raja+occa+libceed cuda_arch='"${cuda_arch}"' \ + +superlu-dist+strumpack+suite-sparse+petsc+slepc \ + +sundials+pumi+gslib+mpfr+netcdf+zlib+gnutls+libunwind+conduit \ + ^raja+cuda+openmp'" $strumpack_cuda_spec $petsc_spec $hdf5_spec" + + # same builds as above with ${mfem_dev} + ${mfem_dev}'+cuda cuda_arch='"${cuda_arch}" + + ${mfem_dev}'+cuda+raja+occa+libceed cuda_arch='"${cuda_arch}"' \ + ^raja+cuda~openmp' + + # add '^sundials+hypre' to help the concretizer + ${mfem_dev}'+cuda+openmp+raja+occa+libceed cuda_arch='"${cuda_arch}"' \ + +superlu-dist+strumpack+suite-sparse+petsc+slepc \ + +sundials+pumi+gslib+mpfr+netcdf+zlib+gnutls+libunwind+conduit \ + ^raja+cuda+openmp'" $strumpack_cuda_spec $petsc_spec"' \ + ^sundials+hypre'" $hdf5_spec" +) + + trap 'printf "\nScript interrupted.\n"; exit 33' INT SEP='==========================================================================' sep='--------------------------------------------------------------------------' -for bld in "${builds[@]}" "${builds2[@]}"; do +run_builds=("${builds[@]}" "${builds2[@]}") +# run_builds=("${builds_cuda[@]}") + +for bld in "${run_builds[@]}"; do printf "\n%s\n" "${SEP}" printf " %s\n" "${bld}" printf "%s\n" "${SEP}" diff --git a/var/spack/repos/builtin/packages/nekrs/package.py b/var/spack/repos/builtin/packages/nekrs/package.py index 731c34e21f..3f1fd44e69 100644 --- a/var/spack/repos/builtin/packages/nekrs/package.py +++ b/var/spack/repos/builtin/packages/nekrs/package.py @@ -22,15 +22,9 @@ class Nekrs(Package, CudaPackage, ROCmPackage): version('21.0', tag='v21.0') - variant('cuda', - default=False, - description='Activates support for CUDA') variant('opencl', default=False, description='Activates support for OpenCL') - variant('hip', - default=False, - description='Activates support for HIP') # Conflicts: # nekrs includes following packages, but in order to build as part of @@ -44,9 +38,6 @@ class Nekrs(Package, CudaPackage, ROCmPackage): depends_on('git') depends_on('cmake') - depends_on('cuda', when='+cuda') - depends_on('hip', when='+rocm') - @run_before('install') def fortran_check(self): if not self.compiler.f77: diff --git a/var/spack/repos/builtin/packages/petsc/package.py b/var/spack/repos/builtin/packages/petsc/package.py index 6617df1ba2..7c397999b8 100644 --- a/var/spack/repos/builtin/packages/petsc/package.py +++ b/var/spack/repos/builtin/packages/petsc/package.py @@ -5,7 +5,7 @@ import os -class Petsc(Package): +class Petsc(Package, CudaPackage, ROCmPackage): """PETSc is a suite of data structures and routines for the scalable (parallel) solution of scientific applications modeled by partial differential equations. @@ -76,7 +76,6 @@ class Petsc(Package): variant('shared', default=True, description='Enables the build of shared libraries') variant('mpi', default=True, description='Activates MPI support') - variant('cuda', default=False, description='Activates CUDA support') variant('double', default=True, description='Switches between single and double precision') variant('complex', default=False, description='Build with complex numbers') @@ -192,6 +191,9 @@ class Petsc(Package): depends_on('lapack') depends_on('mpi', when='+mpi') depends_on('cuda', when='+cuda') + depends_on('hip', when='+rocm') + depends_on('hipblas', when='+rocm') + depends_on('hipsparse', when='+rocm') # Build dependencies depends_on('python@2.6:2.8', type='build', when='@:3.10.99') @@ -375,40 +377,57 @@ def install(self, spec, prefix): '--with-scalapack=0' ]) - if spec.satisfies('+openmp'): - options.append('--with-openmp=1') - # Activates library support if needed (i.e. direct dependency) if '^libjpeg-turbo' in spec: - jpeg_library = 'libjpeg-turbo' + jpeg_library = ('libjpeg-turbo', 'libjpeg') else: jpeg_library = 'libjpeg' - for library in ('cuda', 'metis', 'hypre', 'parmetis', 'mumps', - 'trilinos', 'fftw', 'valgrind', 'gmp', 'libpng', - 'giflib', 'mpfr', 'netcdf-c', 'parallel-netcdf', - 'moab', 'random123', 'exodusii', 'cgns', 'memkind', - 'p4est', 'saws', 'libyaml', 'hwloc', jpeg_library): + for library in ( + 'cuda', + 'hip', + 'metis', + 'hypre', + 'parmetis', + 'mumps', + 'trilinos', + 'fftw', + 'valgrind', + 'gmp', + 'libpng', + 'giflib', + 'mpfr', + ('netcdf-c', 'netcdf'), + ('parallel-netcdf', 'pnetcdf'), + 'moab', + 'openmp', + 'random123', + 'exodusii', + 'cgns', + 'memkind', + 'p4est', + 'saws', + ('libyaml', 'yaml'), + 'hwloc', + jpeg_library, + ): # Cannot check `library in spec` because of transitive deps # Cannot check variants because parmetis keys on +metis + if isinstance(library, tuple): + library, petsclibname = library + else: + petsclibname = library + library_requested = library in spec.dependencies_dict() options.append( '--with-{library}={value}'.format( - library=('libjpeg' if library == 'libjpeg-turbo' - else 'netcdf' if library == 'netcdf-c' - else 'pnetcdf' if library == 'parallel-netcdf' - else 'yaml' if library == 'libyaml' - else library), + library=petsclibname, value=('1' if library_requested else '0')) ) if library_requested: options.append( '--with-{library}-dir={path}'.format( - library=('libjpeg' if library == 'libjpeg-turbo' - else 'netcdf' if library == 'netcdf-c' - else 'pnetcdf' if library == 'parallel-netcdf' - else 'yaml' if library == 'libyaml' - else library), path=spec[library].prefix) + library=petsclibname, path=spec[library].prefix) ) # PETSc does not pick up SuperluDist from the dir as they look for diff --git a/var/spack/repos/builtin/packages/pumi/package.py b/var/spack/repos/builtin/packages/pumi/package.py index 08930cd70d..b32f7f4f3e 100644 --- a/var/spack/repos/builtin/packages/pumi/package.py +++ b/var/spack/repos/builtin/packages/pumi/package.py @@ -78,6 +78,8 @@ def cmake_args(self): ('ON' if '~simmodsuite_version_check' in spec else 'OFF'), '-DMESHES=%s' % join_path(self.stage.source_path, 'pumi_meshes') ] + if spec.satisfies('@2.2.3'): + args += ['-DCMAKE_CXX_STANDARD=11'] if self.spec.satisfies('simmodsuite=base'): args.append('-DENABLE_SIMMETRIX=ON') if self.spec.satisfies('simmodsuite=kernels') or \ diff --git a/var/spack/repos/builtin/packages/suite-sparse/package.py b/var/spack/repos/builtin/packages/suite-sparse/package.py index 8b3c049faf..0182658afc 100644 --- a/var/spack/repos/builtin/packages/suite-sparse/package.py +++ b/var/spack/repos/builtin/packages/suite-sparse/package.py @@ -38,7 +38,7 @@ class SuiteSparse(Package): variant('cuda', default=False, description='Build with CUDA') variant('openmp', default=False, description='Build with OpenMP') - depends_on('mpfr', type=('build', 'link'), when='@5.8.0:') + depends_on('mpfr@4.0.0:', type=('build', 'link'), when='@5.8.0:') depends_on('gmp', type=('build', 'link'), when='@5.8.0:') depends_on('blas') depends_on('lapack')