New MFEM version: 4.5 (#33480)
* New MFEM version: 4.5 Add new MFEM variants: ginkgo, hiop * mfem: small tweaks * mfem: tweak testing script * mfem: try to resolve issue #30483 * mfem: fix style * mfem: tweak for Spack-built hipsparse Co-authored-by: eugeneswalker <38933153+eugeneswalker@users.noreply.github.com>
This commit is contained in:
Veselin Dobrev
GitHub
parent
e5274de7ec
commit
82470f880a
2 changed files with 124 additions and 41 deletions
|
|
@ -48,6 +48,13 @@ class Mfem(Package, CudaPackage, ROCmPackage):
|
|||
# other version.
|
||||
version("develop", branch="master")
|
||||
|
||||
version(
|
||||
"4.5.0",
|
||||
sha256="4f201bec02fc5460a902596697b6c1deb7b15ac57c71f615b2ab4a8eb65665f7",
|
||||
url="https://bit.ly/mfem-4-5",
|
||||
extension="tar.gz",
|
||||
)
|
||||
|
||||
version(
|
||||
"4.4.0",
|
||||
sha256="37250dbef6e97b16dc9ab50973e8d68bc165bb4afcdaf91b3b72c8972c87deef",
|
||||
|
|
@ -171,7 +178,9 @@ class Mfem(Package, CudaPackage, ROCmPackage):
|
|||
"libunwind", default=False, description="Enable backtrace on error support using Libunwind"
|
||||
)
|
||||
variant("fms", default=False, when="@4.3.0:", description="Enable FMS I/O support")
|
||||
# TODO: SIMD, Ginkgo, ADIOS2, HiOp, MKL CPardiso, Axom/Sidre
|
||||
variant("ginkgo", default=False, when="@4.3.0:", description="Enable Ginkgo support")
|
||||
variant("hiop", default=False, when="@4.4.0:", description="Enable HiOp support")
|
||||
# TODO: SIMD, ADIOS2, MKL CPardiso, Axom/Sidre
|
||||
variant(
|
||||
"timer",
|
||||
default="auto",
|
||||
|
|
@ -229,6 +238,7 @@ class Mfem(Package, CudaPackage, ROCmPackage):
|
|||
conflicts("^mpich@4:", when="@:4.3+mpi")
|
||||
|
||||
depends_on("mpi", when="+mpi")
|
||||
depends_on("hipsparse", when="@4.4.0:+rocm")
|
||||
depends_on("hypre@2.10.0:2.13", when="@:3.3+mpi")
|
||||
depends_on("hypre@:2.20.0", when="@3.4:4.2+mpi")
|
||||
depends_on("hypre@:2.23.0", when="@4.3.0+mpi")
|
||||
|
|
@ -256,11 +266,13 @@ class Mfem(Package, CudaPackage, ROCmPackage):
|
|||
depends_on("sundials@2.7.0+mpi+hypre", when="@:3.3.0+sundials+mpi")
|
||||
depends_on("sundials@2.7.0:", when="@3.3.2:+sundials~mpi")
|
||||
depends_on("sundials@2.7.0:+mpi+hypre", when="@3.3.2:+sundials+mpi")
|
||||
depends_on("sundials@5.0.0:5", when="@4.0.1-xsdk:+sundials~mpi")
|
||||
depends_on("sundials@5.0.0:5+mpi+hypre", when="@4.0.1-xsdk:+sundials+mpi")
|
||||
depends_on("sundials@5.0.0:5", when="@4.0.1-xsdk:4.4+sundials~mpi")
|
||||
depends_on("sundials@5.0.0:5+mpi+hypre", when="@4.0.1-xsdk:4.4+sundials+mpi")
|
||||
depends_on("sundials@5.0.0:", when="@4.5.0:+sundials~mpi")
|
||||
depends_on("sundials@5.0.0:+mpi+hypre", when="@4.5.0:+sundials+mpi")
|
||||
for sm_ in CudaPackage.cuda_arch_values:
|
||||
depends_on(
|
||||
"sundials@5.4.0:5+cuda cuda_arch={0}".format(sm_),
|
||||
"sundials@5.4.0:+cuda cuda_arch={0}".format(sm_),
|
||||
when="@4.2.0:+sundials+cuda cuda_arch={0}".format(sm_),
|
||||
)
|
||||
depends_on("pumi", when="+pumi~shared")
|
||||
|
|
@ -320,6 +332,29 @@ class Mfem(Package, CudaPackage, ROCmPackage):
|
|||
depends_on("conduit@0.3.1:,master:", when="+conduit")
|
||||
depends_on("conduit+mpi", when="+conduit+mpi")
|
||||
depends_on("libfms@0.2.0:", when="+fms")
|
||||
depends_on("ginkgo@1.4.0:", when="+ginkgo")
|
||||
for sm_ in CudaPackage.cuda_arch_values:
|
||||
depends_on(
|
||||
"ginkgo+cuda cuda_arch={0}".format(sm_),
|
||||
when="+ginkgo+cuda cuda_arch={0}".format(sm_),
|
||||
)
|
||||
for gfx in ROCmPackage.amdgpu_targets:
|
||||
depends_on(
|
||||
"ginkgo+rocm amdgpu_target={0}".format(gfx),
|
||||
when="+ginkgo+rocm amdgpu_target={0}".format(gfx),
|
||||
)
|
||||
depends_on("hiop@0.4.6:~mpi", when="+hiop~mpi")
|
||||
depends_on("hiop@0.4.6:+mpi", when="+hiop+mpi")
|
||||
for sm_ in CudaPackage.cuda_arch_values:
|
||||
depends_on(
|
||||
"hiop+cuda cuda_arch={0}".format(sm_),
|
||||
when="+hiop+cuda cuda_arch={0}".format(sm_),
|
||||
)
|
||||
for gfx in ROCmPackage.amdgpu_targets:
|
||||
depends_on(
|
||||
"hiop+rocm amdgpu_target={0}".format(gfx),
|
||||
when="+hiop+rocm amdgpu_target={0}".format(gfx),
|
||||
)
|
||||
|
||||
# The MFEM 4.0.0 SuperLU interface fails when using hypre@2.16.0 and
|
||||
# superlu-dist@6.1.1. See https://github.com/mfem/mfem/issues/983.
|
||||
|
|
@ -488,6 +523,9 @@ def find_optional_library(name, prefix):
|
|||
else:
|
||||
mfem_mpiexec = "jsrun"
|
||||
mfem_mpiexec_np = "-p"
|
||||
elif "FLUX_JOB_ID" in os.environ:
|
||||
mfem_mpiexec = "flux mini run"
|
||||
mfem_mpiexec_np = "-n"
|
||||
|
||||
metis5_str = "NO"
|
||||
if ("+metis" in spec) and spec["metis"].satisfies("@5:"):
|
||||
|
|
@ -530,11 +568,32 @@ def find_optional_library(name, prefix):
|
|||
"MFEM_USE_CEED=%s" % yes_no("+libceed"),
|
||||
"MFEM_USE_UMPIRE=%s" % yes_no("+umpire"),
|
||||
"MFEM_USE_FMS=%s" % yes_no("+fms"),
|
||||
"MFEM_USE_GINKGO=%s" % yes_no("+ginkgo"),
|
||||
"MFEM_USE_HIOP=%s" % yes_no("+hiop"),
|
||||
"MFEM_MPIEXEC=%s" % mfem_mpiexec,
|
||||
"MFEM_MPIEXEC_NP=%s" % mfem_mpiexec_np,
|
||||
"MFEM_USE_EXCEPTIONS=%s" % yes_no("+exceptions"),
|
||||
]
|
||||
|
||||
# Determine C++ standard to use:
|
||||
cxxstd = None
|
||||
if self.spec.satisfies("@4.0.0:"):
|
||||
cxxstd = "11"
|
||||
if self.spec.satisfies("^raja@2022.03.0:"):
|
||||
cxxstd = "14"
|
||||
if self.spec.satisfies("^umpire@2022.03.0:"):
|
||||
cxxstd = "14"
|
||||
if self.spec.satisfies("^sundials@6.4.0:"):
|
||||
cxxstd = "14"
|
||||
if self.spec.satisfies("^ginkgo"):
|
||||
cxxstd = "14"
|
||||
cxxstd_flag = None
|
||||
if cxxstd:
|
||||
if "+cuda" in spec:
|
||||
cxxstd_flag = "-std=c++" + cxxstd
|
||||
else:
|
||||
cxxstd_flag = getattr(self.compiler, "cxx" + cxxstd + "_flag")
|
||||
|
||||
cxxflags = spec.compiler_flags["cxxflags"]
|
||||
|
||||
if cxxflags:
|
||||
|
|
@ -557,16 +616,16 @@ def find_optional_library(name, prefix):
|
|||
"-x=cu --expt-extended-lambda -arch=sm_%s" % cuda_arch,
|
||||
"-ccbin %s" % (spec["mpi"].mpicxx if "+mpi" in spec else env["CXX"]),
|
||||
]
|
||||
if self.spec.satisfies("@4.0.0:"):
|
||||
if "+cuda" in spec:
|
||||
cxxflags.append("-std=c++11")
|
||||
else:
|
||||
cxxflags.append(self.compiler.cxx11_flag)
|
||||
if cxxstd_flag:
|
||||
cxxflags.append(cxxstd_flag)
|
||||
# The cxxflags are set by the spack c++ compiler wrapper. We also
|
||||
# set CXXFLAGS explicitly, for clarity, and to properly export the
|
||||
# cxxflags in the variable MFEM_CXXFLAGS in config.mk.
|
||||
options += ["CXXFLAGS=%s" % " ".join(cxxflags)]
|
||||
|
||||
elif cxxstd_flag:
|
||||
options += ["BASE_FLAGS=%s" % cxxstd_flag]
|
||||
|
||||
# Treat any 'CXXFLAGS' in the environment as extra c++ flags which are
|
||||
# handled through the 'CPPFLAGS' makefile variable in MFEM. Also, unset
|
||||
# 'CXXFLAGS' from the environment to prevent it from overriding the
|
||||
|
|
@ -589,9 +648,12 @@ def find_optional_library(name, prefix):
|
|||
if "+cuda" in hypre:
|
||||
hypre_gpu_libs = " -lcusparse -lcurand"
|
||||
elif "+rocm" in hypre:
|
||||
hypre_gpu_libs = " " + ld_flags_from_dirs(
|
||||
[env["ROCM_PATH"] + "/lib"], ["rocsparse", "rocrand"]
|
||||
)
|
||||
hypre_rocm_libs = LibraryList([])
|
||||
if "^rocsparse" in hypre:
|
||||
hypre_rocm_libs += hypre["rocsparse"].libs
|
||||
if "^rocrand" in hypre:
|
||||
hypre_rocm_libs += hypre["rocrand"].libs
|
||||
hypre_gpu_libs = " " + ld_flags_from_library_list(hypre_rocm_libs)
|
||||
options += [
|
||||
"HYPRE_OPT=-I%s" % hypre.prefix.include,
|
||||
"HYPRE_LIB=%s%s" % (ld_flags_from_library_list(all_hypre_libs), hypre_gpu_libs),
|
||||
|
|
@ -804,6 +866,15 @@ def find_optional_library(name, prefix):
|
|||
if "+rocm" in spec:
|
||||
amdgpu_target = ",".join(spec.variants["amdgpu_target"].value)
|
||||
options += ["HIP_CXX=%s" % spec["hip"].hipcc, "HIP_ARCH=%s" % amdgpu_target]
|
||||
if "^hipsparse" in spec: # hipsparse is needed @4.4.0:+rocm
|
||||
# Note: MFEM's defaults.mk want to find librocsparse.* in
|
||||
# $(HIP_DIR)/lib, so we set HIP_DIR to be the prefix of
|
||||
# rocsparse (which is a dependency of hipsparse).
|
||||
options += [
|
||||
"HIP_DIR=%s" % spec["rocsparse"].prefix,
|
||||
"HIP_OPT=%s" % spec["hipsparse"].headers.cpp_flags,
|
||||
"HIP_LIB=%s" % ld_flags_from_library_list(spec["hipsparse"].libs),
|
||||
]
|
||||
|
||||
if "+occa" in spec:
|
||||
options += [
|
||||
|
|
@ -895,6 +966,21 @@ def find_optional_library(name, prefix):
|
|||
"FMS_LIB=%s" % ld_flags_from_library_list(libfms.libs),
|
||||
]
|
||||
|
||||
if "+ginkgo" in spec:
|
||||
ginkgo = spec["ginkgo"]
|
||||
options += [
|
||||
"GINKGO_DIR=%s" % ginkgo.prefix,
|
||||
"GINKGO_BUILD_TYPE=%s" % ginkgo.variants["build_type"].value,
|
||||
]
|
||||
|
||||
if "+hiop" in spec:
|
||||
hiop = spec["hiop"]
|
||||
lapack_blas = spec["lapack"].libs + spec["blas"].libs
|
||||
options += [
|
||||
"HIOP_OPT=-I%s" % hiop.prefix.include,
|
||||
"HIOP_LIB=%s" % ld_flags_from_library_list(hiop.libs + lapack_blas),
|
||||
]
|
||||
|
||||
make("config", *options, parallel=False)
|
||||
make("info", parallel=False)
|
||||
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ rocm_arch="gfx908"
|
|||
spack_jobs=''
|
||||
# spack_jobs='-j 128'
|
||||
|
||||
mfem='mfem@4.4.0'${compiler}
|
||||
mfem='mfem@4.5.0'${compiler}
|
||||
mfem_dev='mfem@develop'${compiler}
|
||||
|
||||
backends='+occa+raja+libceed'
|
||||
|
|
@ -28,32 +28,29 @@ builds=(
|
|||
# preferred version:
|
||||
${mfem}
|
||||
${mfem}'~mpi~metis~zlib'
|
||||
# TODO: add back '+gslib' when the gslib test is fixed.
|
||||
# TODO: add back '+slepc' when its build is fixed.
|
||||
${mfem}"$backends"'+superlu-dist+strumpack+suite-sparse+petsc \
|
||||
+sundials+pumi+mpfr+netcdf+zlib+gnutls+libunwind+conduit \
|
||||
${mfem}"$backends"'+superlu-dist+strumpack+suite-sparse+petsc+slepc+gslib \
|
||||
+sundials+pumi+mpfr+netcdf+zlib+gnutls+libunwind+conduit+ginkgo+hiop \
|
||||
'"$backends_specs $strumpack_spec $petsc_spec $hdf5_spec"
|
||||
${mfem}'~mpi \
|
||||
'"$backends"'+suite-sparse+sundials+gslib+mpfr+netcdf \
|
||||
+zlib+gnutls+libunwind+conduit \
|
||||
+zlib+gnutls+libunwind+conduit+ginkgo+hiop \
|
||||
'"$backends_specs $hdf5_spec"' ^sundials~mpi'
|
||||
|
||||
# develop version, shared builds:
|
||||
${mfem_dev}'+shared~static'
|
||||
${mfem_dev}'+shared~static~mpi~metis~zlib'
|
||||
# NOTE: Shared build with +gslib works on mac but not on linux
|
||||
# TODO: add back '+gslib' when the gslib test is fixed and the above NOTE
|
||||
# TODO: add back '+gslib' when the above NOTE
|
||||
# is addressed.
|
||||
# TODO: add back '+slepc' when its build is fixed.
|
||||
${mfem_dev}'+shared~static \
|
||||
'"$backends"'+superlu-dist+strumpack+suite-sparse+petsc \
|
||||
+sundials+pumi+mpfr+netcdf+zlib+gnutls+libunwind+conduit \
|
||||
'"$backends"'+superlu-dist+strumpack+suite-sparse+petsc+slepc \
|
||||
+sundials+pumi+mpfr+netcdf+zlib+gnutls+libunwind+conduit+ginkgo+hiop \
|
||||
'"$backends_specs $strumpack_spec $petsc_spec $hdf5_spec"
|
||||
# NOTE: Shared build with +gslib works on mac but not on linux
|
||||
# TODO: add back '+gslib' when the above NOTE is addressed.
|
||||
${mfem_dev}'+shared~static~mpi \
|
||||
'"$backends"'+suite-sparse+sundials+mpfr+netcdf \
|
||||
+zlib+gnutls+libunwind+conduit \
|
||||
+zlib+gnutls+libunwind+conduit+ginkgo+hiop \
|
||||
'"$backends_specs $hdf5_spec"' ^sundials~mpi'
|
||||
)
|
||||
|
||||
|
|
@ -67,8 +64,7 @@ builds2=(
|
|||
${mfem}'+sundials~mpi ^sundials~mpi'
|
||||
${mfem}'+sundials'
|
||||
${mfem}'+pumi'
|
||||
# TODO: uncomment the next line when the gslib test is fixed.
|
||||
# ${mfem}'+gslib'
|
||||
${mfem}'+gslib'
|
||||
${mfem}'+netcdf~mpi'
|
||||
${mfem}'+netcdf'
|
||||
${mfem}'+mpfr'
|
||||
|
|
@ -77,10 +73,11 @@ builds2=(
|
|||
${mfem}'+conduit'
|
||||
${mfem}'+umpire'
|
||||
${mfem}'+petsc'" $petsc_spec"
|
||||
# TODO: uncomment the next line when the slepc build is fixed.
|
||||
# ${mfem}'+petsc+slepc'" $petsc_spec"
|
||||
# TODO: uncomment the next line when the threadsafe build is fixed.
|
||||
# ${mfem}'+threadsafe'
|
||||
${mfem}'+petsc+slepc'" $petsc_spec"
|
||||
${mfem}'+ginkgo'
|
||||
${mfem}'+hiop'
|
||||
${mfem}'+threadsafe'
|
||||
#
|
||||
# develop version
|
||||
${mfem_dev}"$backends $backends_specs"
|
||||
${mfem_dev}'+superlu-dist'
|
||||
|
|
@ -90,8 +87,7 @@ builds2=(
|
|||
${mfem_dev}'+sundials~mpi ^sundials~mpi'
|
||||
${mfem_dev}'+sundials'
|
||||
${mfem_dev}'+pumi'
|
||||
# TODO: uncomment the next line when the gslib test is fixed.
|
||||
# ${mfem_dev}'+gslib'
|
||||
${mfem_dev}'+gslib'
|
||||
${mfem_dev}'+netcdf~mpi'
|
||||
${mfem_dev}'+netcdf'
|
||||
${mfem_dev}'+mpfr'
|
||||
|
|
@ -101,8 +97,9 @@ builds2=(
|
|||
${mfem_dev}'+umpire'
|
||||
${mfem_dev}'+petsc'" $petsc_spec"
|
||||
${mfem_dev}'+petsc+slepc'" $petsc_spec"
|
||||
# TODO: uncomment the next line when the threadsafe build is fixed.
|
||||
# ${mfem_dev}'+threadsafe'
|
||||
${mfem_dev}'+ginkgo'
|
||||
${mfem_dev}'+hiop'
|
||||
${mfem_dev}'+threadsafe'
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -125,7 +122,7 @@ builds_cuda=(
|
|||
# TODO: add back "+petsc+slepc $petsc_spec_cuda" when it works.
|
||||
${mfem}'+cuda+openmp+raja+occa cuda_arch='"${cuda_arch}"' \
|
||||
+strumpack+suite-sparse \
|
||||
+sundials+pumi+mpfr+netcdf+zlib+gnutls+libunwind+conduit \
|
||||
+sundials+pumi+mpfr+netcdf+zlib+gnutls+libunwind+conduit+ginkgo+hiop \
|
||||
^raja+cuda+openmp'" $strumpack_cuda_spec"' \
|
||||
'"$hdf5_spec"
|
||||
|
||||
|
|
@ -138,7 +135,7 @@ builds_cuda=(
|
|||
# TODO: add back "+sundials" when it's supported with '^hypre+cuda'.
|
||||
${mfem}'+cuda+openmp+raja+occa cuda_arch='"${cuda_arch}"' \
|
||||
+suite-sparse \
|
||||
+pumi+mpfr+netcdf+zlib+gnutls+libunwind+conduit \
|
||||
+pumi+mpfr+netcdf+zlib+gnutls+libunwind+conduit+ginkgo+hiop \
|
||||
^raja+cuda+openmp ^hypre+cuda \
|
||||
'"$hdf5_spec"
|
||||
|
||||
|
|
@ -164,7 +161,7 @@ builds_cuda=(
|
|||
# TODO: add back "+petsc+slepc $petsc_spec_cuda" when it works.
|
||||
${mfem_dev}'+cuda+openmp+raja+occa cuda_arch='"${cuda_arch}"' \
|
||||
+strumpack+suite-sparse \
|
||||
+sundials+pumi+mpfr+netcdf+zlib+gnutls+libunwind+conduit \
|
||||
+sundials+pumi+mpfr+netcdf+zlib+gnutls+libunwind+conduit+ginkgo+hiop \
|
||||
^raja+cuda+openmp'" $strumpack_cuda_spec"' \
|
||||
'"$hdf5_spec"
|
||||
|
||||
|
|
@ -177,7 +174,7 @@ builds_cuda=(
|
|||
# TODO: add back "+sundials" when it's supported with '^hypre+cuda'.
|
||||
${mfem_dev}'+cuda+openmp+raja+occa cuda_arch='"${cuda_arch}"' \
|
||||
+suite-sparse \
|
||||
+pumi+mpfr+netcdf+zlib+gnutls+libunwind+conduit \
|
||||
+pumi+mpfr+netcdf+zlib+gnutls+libunwind+conduit+ginkgo+hiop \
|
||||
^raja+cuda+openmp ^hypre+cuda \
|
||||
'"$hdf5_spec"
|
||||
)
|
||||
|
|
@ -200,7 +197,7 @@ builds_rocm=(
|
|||
# TODO: add "+petsc+slepc $petsc_spec_rocm" when it is supported.
|
||||
${mfem}'+rocm+openmp+raja+occa+libceed amdgpu_target='"${rocm_arch}"' \
|
||||
+strumpack+suite-sparse \
|
||||
+sundials+pumi+mpfr+netcdf+zlib+gnutls+libunwind+conduit \
|
||||
+sundials+pumi+mpfr+netcdf+zlib+gnutls+libunwind+conduit+ginkgo+hiop \
|
||||
^raja+rocm~openmp ^occa~cuda'" $strumpack_rocm_spec"' \
|
||||
'"$hdf5_spec"
|
||||
|
||||
|
|
@ -212,7 +209,7 @@ builds_rocm=(
|
|||
# TODO: add back "+sundials" when it's supported with '^hypre+rocm'.
|
||||
${mfem}'+rocm+openmp+raja+occa+libceed amdgpu_target='"${rocm_arch}"' \
|
||||
+suite-sparse \
|
||||
+pumi+mpfr+netcdf+zlib+gnutls+libunwind+conduit \
|
||||
+pumi+mpfr+netcdf+zlib+gnutls+libunwind+conduit+ginkgo+hiop \
|
||||
^raja+rocm~openmp ^occa~cuda ^hypre+rocm \
|
||||
'"$hdf5_spec"
|
||||
|
||||
|
|
@ -238,9 +235,9 @@ for bld in "${run_builds[@]}"; do
|
|||
printf " %s\n" "${bld}"
|
||||
printf "%s\n" "${SEP}"
|
||||
eval bbb="\"${bld}\""
|
||||
spack spec -I $bbb || exit 1
|
||||
spack spec --fresh -I $bbb || exit 1
|
||||
printf "%s\n" "${sep}"
|
||||
spack install $spack_jobs --test=root $bbb || exit 2
|
||||
spack install $spack_jobs --fresh --test=root $bbb || exit 2
|
||||
done
|
||||
|
||||
# Uninstall all mfem builds:
|
||||
|
|
|
|||
Loading…
Reference in a new issue