From 9225f4f27fac3862f31da864bdb44bace375e609 Mon Sep 17 00:00:00 2001 From: Chris White Date: Mon, 3 Oct 2022 13:02:46 -0700 Subject: [PATCH] Axom: bring in changes from axom repo (#32643) * bring in changes from axom repo Co-authored-by: white238 Co-authored-by: Tamara Dahlgren <35777542+tldahlgren@users.noreply.github.com> --- .../repos/builtin/packages/axom/package.py | 200 ++++++++++++------ 1 file changed, 139 insertions(+), 61 deletions(-) diff --git a/var/spack/repos/builtin/packages/axom/package.py b/var/spack/repos/builtin/packages/axom/package.py index a63118e2c7..d1cb7f1069 100644 --- a/var/spack/repos/builtin/packages/axom/package.py +++ b/var/spack/repos/builtin/packages/axom/package.py @@ -28,7 +28,7 @@ def get_spec_path(spec, package_name, path_replacements={}, use_bin=False): return path -class Axom(CachedCMakePackage, CudaPackage): +class Axom(CachedCMakePackage, CudaPackage, ROCmPackage): """Axom provides a robust, flexible software infrastructure for the development of multi-physics applications and computational tools.""" @@ -40,6 +40,7 @@ class Axom(CachedCMakePackage, CudaPackage): version("main", branch="main") version("develop", branch="develop") + version("0.7.0", tag="v0.7.0") version("0.6.1", tag="v0.6.1") version("0.6.0", tag="v0.6.0") version("0.5.0", tag="v0.5.0") @@ -50,10 +51,6 @@ class Axom(CachedCMakePackage, CudaPackage): version("0.3.0", tag="v0.3.0") version("0.2.9", tag="v0.2.9") - @property - def submodules(self): - return True - # https://github.com/spack/spack/issues/31829 patch("examples-oneapi.patch", when="@0.6.1 +examples %oneapi") @@ -65,11 +62,11 @@ def submodules(self): # Variants # ----------------------------------------------------------------------- variant("shared", default=True, description="Enable build of shared libraries") - variant("debug", default=False, description="Build debug instead of optimized version") variant("examples", default=True, description="Build examples") variant("tools", default=True, description="Build tools") + # Hard requirement after Axom 0.6.1 variant("cpp14", default=True, description="Build with C++14 support") variant("fortran", default=True, description="Build with Fortran support") @@ -95,36 +92,48 @@ def submodules(self): # ----------------------------------------------------------------------- # Basics depends_on("cmake@3.8.2:", type="build") + depends_on("cmake@3.16.8:", type="build", when="+rocm") + + depends_on("blt", type="build") + depends_on("blt@0.5.1:", type="build", when="@0.6.2:") + depends_on("mpi", when="+mpi") # Libraries - depends_on("conduit+python", when="+python") - depends_on("conduit~python", when="~python") - depends_on("conduit+hdf5", when="+hdf5") - depends_on("conduit~hdf5", when="~hdf5") + # Forward variants to Conduit + for _var in ["fortran", "hdf5", "mpi", "python"]: + depends_on("conduit+{0}".format(_var), when="+{0}".format(_var)) + depends_on("conduit~{0}".format(_var), when="~{0}".format(_var)) + + depends_on("hdf5", when="+hdf5") depends_on("lua", when="+lua") depends_on("scr", when="+scr") - depends_on("kvtree@main", when="+scr") - depends_on("dtcmp", when="+scr") + depends_on("scr~fortran", when="+scr~fortran") with when("+umpire"): - depends_on("umpire@6.0.0:", when="@0.6.0:") + depends_on("umpire@2022.03.0:", when="@0.7.0:") + depends_on("umpire@6.0.0", when="@0.6.0") depends_on("umpire@5:5.0.1", when="@:0.5.0") depends_on("umpire +openmp", when="+openmp") depends_on("umpire +cuda", when="+cuda") with when("+raja"): - depends_on("raja@0.14.0:", when="@0.6.0:") + depends_on("raja@2022.03.0:", when="@0.7.0:") + depends_on("raja@0.14.0", when="@0.6.0") depends_on("raja@:0.13.0", when="@:0.5.0") depends_on("raja~openmp", when="~openmp") depends_on("raja+openmp", when="+openmp") depends_on("raja+cuda", when="+cuda") - for sm_ in CudaPackage.cuda_arch_values: - depends_on("raja cuda_arch={0}".format(sm_), when="+raja cuda_arch={0}".format(sm_)) - depends_on("umpire cuda_arch={0}".format(sm_), when="+umpire cuda_arch={0}".format(sm_)) + for val in CudaPackage.cuda_arch_values: + depends_on("raja cuda_arch={0}".format(val), when="+raja cuda_arch={0}".format(val)) + depends_on("umpire cuda_arch={0}".format(val), when="+umpire cuda_arch={0}".format(val)) + + for val in ROCmPackage.amdgpu_targets: + depends_on("raja amdgpu_target={0}".format(val), when="amdgpu_target={0}".format(val)) + depends_on("umpire amdgpu_target={0}".format(val), when="amdgpu_target={0}".format(val)) depends_on("mfem", when="+mfem") depends_on("mfem~mpi", when="+mfem~mpi") @@ -140,12 +149,18 @@ def submodules(self): depends_on("py-shroud", when="+devtools") depends_on("llvm+clang@10.0.0", when="+devtools", type="build") + # Hard requirement after Axom 0.6.1 + conflicts("~cpp14", when="@0.6.2:") + # Conduit's cmake config files moved and < 0.4.0 can't find it conflicts("^conduit@0.7.2:", when="@:0.4.0") # Sidre requires conduit_blueprint_mpi.hpp conflicts("^conduit@:0.6.0", when="@0.5.0:") + conflicts("+openmp", when="+rocm") + conflicts("+cuda", when="+rocm") + def flag_handler(self, name, flags): if self.spec.satisfies("%cce") and name == "fflags": flags.append("-ef") @@ -161,43 +176,51 @@ def _get_sys_type(self, spec): sys_type = env["SYS_TYPE"] return sys_type + def is_fortran_compiler(self, compiler): + if self.compiler.fc is not None and compiler in self.compiler.fc: + return True + return False + @property def cache_name(self): hostname = socket.gethostname() if "SYS_TYPE" in env: # Are we on a LLNL system then strip node number hostname = hostname.rstrip("1234567890") - return "{0}-{1}-{2}@{3}.cmake".format( + special_case = "" + if "+cuda" in self.spec: + special_case += "_cuda" + if "~fortran" in self.spec: + special_case += "_nofortran" + if "+rocm" in self.spec: + special_case += "_hip" + return "{0}-{1}-{2}@{3}{4}.cmake".format( hostname, self._get_sys_type(self.spec), self.spec.compiler.name, self.spec.compiler.version, + special_case, ) def initconfig_compiler_entries(self): spec = self.spec entries = super(Axom, self).initconfig_compiler_entries() - if "+fortran" in spec or self.compiler.fc is not None: + if "+fortran" in spec: entries.append(cmake_cache_option("ENABLE_FORTRAN", True)) + if self.is_fortran_compiler("gfortran") and "clang" in self.compiler.cxx: + libdir = pjoin(os.path.dirname(os.path.dirname(self.compiler.cxx)), "lib") + flags = "" + for _libpath in [libdir, libdir + "64"]: + if os.path.exists(_libpath): + flags += " -Wl,-rpath,{0}".format(_libpath) + description = "Adds a missing libstdc++ rpath" + if flags: + entries.append(cmake_cache_string("BLT_EXE_LINKER_FLAGS", flags, description)) else: entries.append(cmake_cache_option("ENABLE_FORTRAN", False)) - if ( - (self.compiler.fc is not None) - and ("gfortran" in self.compiler.fc) - and ("clang" in self.compiler.cxx) - ): - libdir = pjoin(os.path.dirname(os.path.dirname(self.compiler.cxx)), "lib") - flags = "" - for _libpath in [libdir, libdir + "64"]: - if os.path.exists(_libpath): - flags += " -Wl,-rpath,{0}".format(_libpath) - description = "Adds a missing libstdc++ rpath" - if flags: - entries.append(cmake_cache_string("BLT_EXE_LINKER_FLAGS", flags, description)) - - if "+cpp14" in spec: + if "+cpp14" in spec and spec.satisfies("@:0.6.1"): entries.append(cmake_cache_string("BLT_CXX_STD", "c++14", "")) return entries @@ -222,15 +245,50 @@ def initconfig_hardware_entries(self): else: entries.append("# cuda_arch could not be determined\n\n") - if "+cpp14" in spec: - cudaflags += " -std=c++14" - else: - cudaflags += " -std=c++11" + if spec.satisfies("^blt@:0.6.1"): + # This is handled internally by BLT now + if "+cpp14" in spec: + cudaflags += " -std=c++14" + else: + cudaflags += " -std=c++11" entries.append(cmake_cache_string("CMAKE_CUDA_FLAGS", cudaflags)) entries.append("# nvcc does not like gtest's 'pthreads' flag\n") entries.append(cmake_cache_option("gtest_disable_pthreads", True)) + if "+rocm" in spec: + entries.append("#------------------{0}\n".format("-" * 60)) + entries.append("# HIP\n") + entries.append("#------------------{0}\n\n".format("-" * 60)) + + entries.append(cmake_cache_option("ENABLE_HIP", True)) + + hip_root = spec["hip"].prefix + rocm_root = hip_root + "/.." + + entries.append(cmake_cache_string("HIP_ROOT_DIR", hip_root)) + entries.append(cmake_cache_string("HIP_CLANG_PATH", rocm_root + "/llvm/bin")) + + archs = self.spec.variants["amdgpu_target"].value + if archs != "none": + arch_str = ",".join(archs) + entries.append(cmake_cache_string("CMAKE_HIP_ARCHITECTURES", arch_str)) + + # Fixes for mpi for rocm until wrapper paths are fixed + # These flags are already part of the wrapped compilers on TOSS4 systems + hip_link_flags = "" + if "+fortran" in spec and self.is_fortran_compiler("amdflang"): + hip_link_flags += "-Wl,--disable-new-dtags " + hip_link_flags += "-L{0}/../llvm/lib -L{0}/lib ".format(hip_root) + hip_link_flags += "-Wl,-rpath,{0}/../llvm/lib:{0}/lib ".format(hip_root) + hip_link_flags += "-lpgmath -lflang -lflangrti -lompstub -lamdhip64 " + + # Additional libraries for TOSS4 + hip_link_flags += " -L{0}/../lib64 -Wl,-rpath,{0}/../lib64 ".format(hip_root) + hip_link_flags += "-lhsakmt -lamd_comgr " + + entries.append(cmake_cache_string("CMAKE_EXE_LINKER_FLAGS", hip_link_flags)) + entries.append("#------------------{0}".format("-" * 30)) entries.append("# Hardware Specifics") entries.append("#------------------{0}\n".format("-" * 30)) @@ -245,7 +303,7 @@ def initconfig_hardware_entries(self): ) ) - if (self.compiler.fc is not None) and ("xlf" in self.compiler.fc): + if "+fortran" in spec and self.is_fortran_compiler("xlf"): # Grab lib directory for the current fortran compiler libdir = pjoin(os.path.dirname(os.path.dirname(self.compiler.fc)), "lib") description = ( @@ -269,25 +327,25 @@ def initconfig_hardware_entries(self): ) ) - if spec.satisfies("target=ppc64le:"): - # Fix for working around CMake adding implicit link directories - # returned by the BlueOS compilers to link executables with - # non-system default stdlib - _roots = ["/usr/tce/packages/gcc/gcc-4.9.3", "/usr/tce/packages/gcc/gcc-4.9.3/gnu"] - _subdirs = ["lib64", "lib64/gcc/powerpc64le-unknown-linux-gnu/4.9.3"] - _existing_paths = [] - for root in _roots: - for subdir in _subdirs: - _curr_path = pjoin(root, subdir) - if os.path.exists(_curr_path): - _existing_paths.append(_curr_path) - if _existing_paths: - entries.append( - cmake_cache_string( - "BLT_CMAKE_IMPLICIT_LINK_DIRECTORIES_EXCLUDE", - ";".join(_existing_paths), - ) + if spec.satisfies("target=ppc64le:"): + # Fix for working around CMake adding implicit link directories + # returned by the BlueOS compilers to link executables with + # non-system default stdlib + _roots = ["/usr/tce/packages/gcc/gcc-4.9.3", "/usr/tce/packages/gcc/gcc-4.9.3/gnu"] + _subdirs = ["lib64", "lib64/gcc/powerpc64le-unknown-linux-gnu/4.9.3"] + _existing_paths = [] + for root in _roots: + for subdir in _subdirs: + _curr_path = pjoin(root, subdir) + if os.path.exists(_curr_path): + _existing_paths.append(_curr_path) + if _existing_paths: + entries.append( + cmake_cache_string( + "BLT_CMAKE_IMPLICIT_LINK_DIRECTORIES_EXCLUDE", + ";".join(_existing_paths), ) + ) return entries @@ -336,12 +394,29 @@ def initconfig_package_entries(self): else: entries.append("# %s not built\n" % dep.upper()) + if "+umpire" in spec and spec.satisfies("^camp"): + dep_dir = get_spec_path(spec, "camp", path_replacements) + entries.append(cmake_cache_path("CAMP_DIR", dep_dir)) + + # SCR does not export it's targets so we need to pull in its dependencies if "+scr" in spec: dep_dir = get_spec_path(spec, "scr", path_replacements) entries.append(cmake_cache_path("SCR_DIR", dep_dir)) # scr's dependencies - for dep in ("kvtree", "dtcmp"): + scr_deps = ( + "kvtree", + "dtcmp", + "spath", + "axl", + "lwgrp", + "er", + "rankstr", + "redset", + "shuffile", + "libyogrt", + ) + for dep in scr_deps: if spec.satisfies("^{0}".format(dep)): dep_dir = get_spec_path(spec, dep, path_replacements) entries.append(cmake_cache_path("%s_DIR" % dep.upper(), dep_dir)) @@ -366,7 +441,8 @@ def initconfig_package_entries(self): entries.append("# Root directory for generated developer tools\n") entries.append(cmake_cache_path("DEVTOOLS_ROOT", devtools_root)) - # Only turn on clangformat support if devtools is on + if "+devtools" in spec and "toss_4" not in self._get_sys_type(spec): + # Only turn on clangformat support if devtools is on and not TOSS4 clang_fmt_path = spec["llvm"].prefix.bin.join("clang-format") entries.append(cmake_cache_path("CLANGFORMAT_EXECUTABLE", clang_fmt_path)) else: @@ -383,9 +459,9 @@ def initconfig_package_entries(self): entries.append(cmake_cache_option("ENABLE_DOCS", enable_docs)) if spec.satisfies("^py-sphinx"): - python_bin_dir = get_spec_path(spec, "python", path_replacements, use_bin=True) + sphinx_bin_dir = get_spec_path(spec, "py-sphinx", path_replacements, use_bin=True) entries.append( - cmake_cache_path("SPHINX_EXECUTABLE", pjoin(python_bin_dir, "sphinx-build")) + cmake_cache_path("SPHINX_EXECUTABLE", pjoin(sphinx_bin_dir, "sphinx-build")) ) if spec.satisfies("^py-shroud"): @@ -404,6 +480,8 @@ def initconfig_package_entries(self): def cmake_args(self): options = [] + options.append("-DBLT_SOURCE_DIR:PATH={0}".format(self.spec["blt"].prefix)) + if self.run_tests is False: options.append("-DENABLE_TESTS=OFF") else: