From 68558b3dd0b8a9c172ace4c7de21c3f964d9bdb2 Mon Sep 17 00:00:00 2001 From: arezaii Date: Tue, 30 Jul 2024 19:24:56 -0600 Subject: [PATCH] Chapel package: updates post release (#45304) * Fix +rocm variant, to ensure correct dependencies on ROCm packages and use of AMD LLVM * Add a +pshm variant for comm=gasnet to enable fast shared-memory comms between co-locales * Add logic to ensure we get the native CXI libfabric network provider on Cray EX * Expand dependency type for package modules to encompass runtime dependencies * Factor logic for setting (LD_)LIBRARY_PATH and PKG_CONFIG_PATH of runtime dependencies * Workaround issue #44746 that causes a transitive dependency on lua to break SLURM * Disable nonfunctional checkChplDoc test * Annotate some variants as conditional, to improve spack info output and reduce confusion --------- Co-authored-by: Dan Bonachea --- .../repos/builtin/packages/chapel/package.py | 165 ++++++++++++------ 1 file changed, 112 insertions(+), 53 deletions(-) diff --git a/var/spack/repos/builtin/packages/chapel/package.py b/var/spack/repos/builtin/packages/chapel/package.py index 7e9bb549d3..77de22ad8f 100644 --- a/var/spack/repos/builtin/packages/chapel/package.py +++ b/var/spack/repos/builtin/packages/chapel/package.py @@ -6,6 +6,7 @@ import os import subprocess +import spack.platforms.cray from spack.package import * from spack.util.environment import is_system_path, set_env @@ -174,6 +175,14 @@ class Chapel(AutotoolsPackage, CudaPackage, ROCmPackage): values=("ibv", "ofi", "udp", "smp", "unset"), multi=False, sticky=True, # never allow the concretizer to choose this + when="comm=gasnet", + ) + + variant( + "pshm", + default=False, + description="Build Chapel with fast shared-memory comms between co-locales", + when="comm=gasnet", ) # Chapel depends on GASNet whenever comm=gasnet. @@ -186,6 +195,7 @@ class Chapel(AutotoolsPackage, CudaPackage, ROCmPackage): default="bundled", values=("bundled", "spack"), multi=False, + when="comm=gasnet", ) variant( @@ -195,6 +205,7 @@ class Chapel(AutotoolsPackage, CudaPackage, ROCmPackage): "supplied CHPL_GASNET_SEGMENT", values=("everything", "fast", "large", "unset"), multi=False, + when="comm=gasnet", ) variant( @@ -277,9 +288,25 @@ class Chapel(AutotoolsPackage, CudaPackage, ROCmPackage): variant( "libfabric", default="unset", - description="When building with ofi support, specify libfabric option", + description="Control the libfabric version used for multi-locale communication", values=("bundled", "spack", "unset"), multi=False, + when="comm=ofi", + ) + + variant( + "libfabric", + default="unset", + description="Control the libfabric version used for multi-locale communication", + values=("bundled", "spack", "unset"), + multi=False, + when="comm=gasnet comm_substrate=ofi", + ) + + requires( + "^libfabric" + (" fabrics=cxi" if spack.platforms.cray.slingshot_network() else ""), + when="libfabric=spack", + msg="libfabric requires cxi fabric provider on HPE-Cray EX machines", ) variant( @@ -357,7 +384,7 @@ class Chapel(AutotoolsPackage, CudaPackage, ROCmPackage): ), default=True, ) - depends_on(dep, when="+{0}".format(variant_name)) + depends_on(dep, when="+{0}".format(variant_name), type=("build", "link", "run", "test")) # TODO: for CHPL_X_CC and CHPL_X_CXX, can we capture an arbitrary path, possibly # with arguments? @@ -406,7 +433,14 @@ class Chapel(AutotoolsPackage, CudaPackage, ROCmPackage): conflicts("platform=windows") # Support for windows is through WSL only conflicts("+rocm", when="+cuda", msg="Chapel must be built with either CUDA or ROCm, not both") - conflicts("+rocm", when="@:2.0.0", msg="ROCm support in spack requires Chapel 2.0.0 or later") + conflicts( + "+rocm", when="@:1.99.99", msg="ROCm support in spack requires Chapel 2.0.0 or later" + ) + # Chapel restricts the allowable ROCm versions + with when("+rocm"): + depends_on("hsa-rocr-dev@4:5.4") + depends_on("hip@4:5.4") + depends_on("llvm-amdgpu@4:5.4") conflicts( "comm_substrate=unset", @@ -414,9 +448,19 @@ class Chapel(AutotoolsPackage, CudaPackage, ROCmPackage): msg="comm=gasnet requires you to also set comm_substrate= to the appropriate network", ) + conflicts( + "gasnet_segment=everything", + when="+pshm", + msg="gasnet_segment=everything does not support +pshm", + ) + + # comm_substrate=udp gasnet_segment=unset defaults to everything, + # which is incompatible with +pshm + requires("gasnet_segment=fast", when="+pshm comm_substrate=udp") + conflicts( "^python@3.12:", - when="@:2.1.0", + when="@:2.0.99", msg="Chapel versions prior to 2.1.0 may produce SyntaxWarnings with Python >= 3.12", ) @@ -443,20 +487,18 @@ class Chapel(AutotoolsPackage, CudaPackage, ROCmPackage): depends_on("doxygen@1.8.17:", when="+chpldoc") # TODO: keep up to date with util/chplenv/chpl_llvm.py - with when("llvm=spack"): + with when("llvm=spack ~rocm"): depends_on("llvm@11:17", when="@:2.0.1") depends_on("llvm@11:18", when="@2.1.0:") # Based on docs https://chapel-lang.org/docs/technotes/gpu.html#requirements - depends_on("llvm@16:", when="llvm=spack ^cuda@12:") + depends_on("llvm@16:", when="llvm=spack +cuda ^cuda@12:") requires( "^llvm targets=all", msg="llvm=spack +cuda requires LLVM support the nvptx target", when="llvm=spack +cuda", ) - depends_on("cuda@11:", when="+cuda", type=("build", "link", "run", "test")) - # This is because certain systems have binutils installed as a system package # but do not include the headers. Spack incorrectly supplies those external # packages as proper dependencies for LLVM, but then LLVM will fail to build @@ -465,11 +507,17 @@ class Chapel(AutotoolsPackage, CudaPackage, ROCmPackage): depends_on("m4") - depends_on("gmp", when="gmp=spack", type=("build", "link", "run", "test")) - depends_on("hwloc", when="hwloc=spack", type=("build", "link", "run", "test")) - depends_on("libfabric", when="libfabric=spack", type=("build", "link", "run", "test")) - depends_on("libunwind", when="unwind=spack", type=("build", "link", "run", "test")) - depends_on("jemalloc", when="host_jemalloc=spack", type=("build", "link", "run", "test")) + # Runtime dependencies: + # Note here "run" is run of the Chapel compiler built by this package, + # but many of these are ALSO run-time dependencies of the executable + # application built by that Chapel compiler from user-provided sources. + with default_args(type=("build", "link", "run", "test")): + depends_on("cuda@11:", when="+cuda") + depends_on("gmp", when="gmp=spack") + depends_on("hwloc", when="hwloc=spack") + depends_on("libfabric", when="libfabric=spack") + depends_on("libunwind", when="unwind=spack") + depends_on("jemalloc", when="host_jemalloc=spack") depends_on("gasnet conduits=none", when="gasnet=spack") depends_on("gasnet@2024.5.0: conduits=none", when="@2.1.0: gasnet=spack") @@ -519,13 +567,25 @@ def setup_chpl_compilers(self, env): # Undo spack compiler wrappers: # the C/C++ compilers must work post-install - if self.spec.satisfies("+cuda") or self.spec.satisfies("+rocm"): + if self.spec.satisfies("+rocm"): env.set("CHPL_TARGET_COMPILER", "llvm") + env.set( + "CHPL_LLVM_CONFIG", + join_path(self.spec["llvm-amdgpu"].prefix, "bin", "llvm-config"), + ) + real_cc = join_path(self.spec["llvm-amdgpu"].prefix, "bin", "clang") + real_cxx = join_path(self.spec["llvm-amdgpu"].prefix, "bin", "clang++") + + # +rocm appears to also require a matching LLVM host compiler to guarantee linkage + env.set("CHPL_HOST_COMPILER", "llvm") + env.set("CHPL_HOST_CC", real_cc) + env.set("CHPL_HOST_CXX", real_cxx) + + elif self.spec.satisfies("llvm=spack"): + env.set("CHPL_TARGET_COMPILER", "llvm") + env.set("CHPL_LLVM_CONFIG", join_path(self.spec["llvm"].prefix, "bin", "llvm-config")) real_cc = join_path(self.spec["llvm"].prefix, "bin", "clang") real_cxx = join_path(self.spec["llvm"].prefix, "bin", "clang++") - elif is_CrayEX() and os.environ.get("CRAYPE_DIR"): - real_cc = join_path(os.environ["CRAYPE_DIR"], "bin", "cc") - real_cxx = join_path(os.environ["CRAYPE_DIR"], "bin", "CC") else: real_cc = self.compiler.cc real_cxx = self.compiler.cxx @@ -535,6 +595,9 @@ def setup_chpl_compilers(self, env): def setup_chpl_comm(self, env, spec): env.set("CHPL_COMM", spec.variants["comm"].value) + if self.spec.satisfies("+pshm"): + env.set("CHPL_GASNET_MORE_CFG_OPTIONS", "--enable-pshm") + @run_before("configure", when="gasnet=spack") def setup_gasnet(self): dst = join_path(self.stage.source_path, "third-party", "gasnet", "gasnet-src") @@ -542,12 +605,6 @@ def setup_gasnet(self): os.rmdir(dst) symlink(self.spec["gasnet"].prefix.src, dst) - def setup_chpl_llvm(self, env): - if self.spec.variants["llvm"].value == "spack": - env.set( - "CHPL_LLVM_CONFIG", "{0}/{1}".format(self.spec["llvm"].prefix, "bin/llvm-config") - ) - def setup_if_not_unset(self, env, var, value): if value != "unset": if value == "spack": @@ -558,12 +615,18 @@ def prepend_cpath_include(self, env, prefix): if not is_system_path(prefix): env.prepend_path("CPATH", prefix.include) + def set_lib_path(self, env, prefix): + if not is_system_path(prefix): + env.prepend_path("LD_LIBRARY_PATH", prefix.lib) + env.prepend_path("LIBRARY_PATH", prefix.lib) + if prefix.lib.pkgconfig is not None: + env.prepend_path("PKG_CONFIG_PATH", prefix.lib.pkgconfig) + def setup_env_vars(self, env): # variants that appear unused by Spack typically correspond directly to # a CHPL_ variable which will be used by the Chapel build system for v in self.spec.variants.keys(): self.setup_if_not_unset(env, "CHPL_" + v.upper(), self.spec.variants[v].value) - self.setup_chpl_llvm(env) self.setup_chpl_compilers(env) self.setup_chpl_platform(env) @@ -574,47 +637,51 @@ def setup_env_vars(self, env): if self.spec.satisfies("+developer"): env.set("CHPL_DEVELOPER", "true") + if not self.spec.satisfies("llvm=none"): + # workaround Spack issue #44746: + # Chapel does not directly utilize lua, but many of its + # launchers depend on system installs of batch schedulers + # (notably Slurm on Cray EX) which depend on a system Lua. + # LLVM includes lua as a dependency, but a barebones lua + # install lacks many packages provided by a system Lua, + # which are often required by system services like Slurm. + # Disable the incomplete Spack lua package directory to + # allow the system one to function. + env.unset("LUA_PATH") + env.unset("LUA_CPATH") + if self.spec.variants["gmp"].value == "spack": # TODO: why must we add to CPATH to find gmp.h # TODO: why must we add to LIBRARY_PATH to find lgmp self.prepend_cpath_include(env, self.spec["gmp"].prefix) - env.prepend_path("LIBRARY_PATH", self.spec["gmp"].prefix.lib) - # Need this for the test env, where it does not appear automatic: - env.prepend_path("PKG_CONFIG_PATH", self.spec["gmp"].prefix.lib.pkgconfig) + self.set_lib_path(env, self.spec["gmp"].prefix) if self.spec.variants["hwloc"].value == "spack": - env.prepend_path("LD_LIBRARY_PATH", self.spec["hwloc"].prefix.lib) + self.set_lib_path(env, self.spec["hwloc"].prefix) # Need this for the test env, where it does not appear automatic: - env.prepend_path("PKG_CONFIG_PATH", self.spec["hwloc"].prefix.lib.pkgconfig) env.prepend_path("PKG_CONFIG_PATH", self.spec["libpciaccess"].prefix.lib.pkgconfig) + # TODO: unwind builds but resulting binaries fail to run, producing linker errors if self.spec.variants["unwind"].value == "spack": # chapel package would not build without cpath, missing libunwind.h self.prepend_cpath_include(env, self.spec["libunwind"].prefix) env.prepend_path("LD_LIBRARY_PATH", self.spec["libunwind"].prefix.lib) if self.spec.satisfies("+yaml"): - env.prepend_path("PKG_CONFIG_PATH", self.spec["libyaml"].prefix.lib.pkgconfig) self.prepend_cpath_include(env, self.spec["libyaml"].prefix) # could not compile test/library/packages/Yaml/writeAndParse.chpl without this - env.prepend_path("LIBRARY_PATH", self.spec["libyaml"].prefix.lib) + self.set_lib_path(env, self.spec["libyaml"].prefix) if self.spec.satisfies("+zmq"): self.prepend_cpath_include(env, self.spec["libzmq"].prefix) # could not compile test/library/packages/ZMQ/hello.chpl without this - env.prepend_path("LIBRARY_PATH", self.spec["libzmq"].prefix.lib) - env.prepend_path("LD_LIBRARY_PATH", self.spec["libzmq"].prefix.lib) - # could not compile test/library/packages/ZMQ/hello.chpl without this - env.prepend_path("LIBRARY_PATH", self.spec["libzmq"].prefix.lib) - env.prepend_path("PKG_CONFIG_PATH", self.spec["libzmq"].prefix.lib.pkgconfig) + self.set_lib_path(env, self.spec["libzmq"].prefix) env.prepend_path("PKG_CONFIG_PATH", self.spec["libsodium"].prefix.lib.pkgconfig) if self.spec.satisfies("+curl"): self.prepend_cpath_include(env, self.spec["curl"].prefix) # could not compile test/library/packages/Curl/check-http.chpl without this - env.prepend_path("LIBRARY_PATH", self.spec["curl"].prefix.lib) - env.prepend_path("LD_LIBRARY_PATH", self.spec["curl"].prefix.lib) - env.prepend_path("PKG_CONFIG_PATH", self.spec["curl"].prefix.lib.pkgconfig) + self.set_lib_path(env, self.spec["curl"].prefix) if self.spec.satisfies("+cuda"): # TODO: why must we add to LD_LIBRARY_PATH to find libcudart? @@ -625,18 +692,11 @@ def setup_env_vars(self, env): if self.spec.satisfies("+rocm"): env.set("CHPL_LOCALE_MODEL", "gpu") env.set("CHPL_GPU", "amd") - env.set("CHPL_HOST_COMPILER", "llvm") env.set("CHPL_GPU_ARCH", self.spec.variants["amdgpu_target"].value[0]) - env.set( - "CHPL_LLVM_CONFIG", - "{0}/{1}".format(self.spec["llvm-amdgpu"].prefix, "bin/llvm-config"), - ) self.prepend_cpath_include(env, self.spec["hip"].prefix) env.set("CHPL_ROCM_PATH", self.spec["llvm-amdgpu"].prefix) - env.prepend_path("LIBRARY_PATH", self.spec["hip"].prefix.lib) - env.prepend_path("LIBRARY_PATH", self.spec["hsa-rocr-dev"].prefix.lib) - env.prepend_path("LD_LIBRARY_PATH", self.spec["hip"].prefix.lib) - env.prepend_path("LD_LIBRARY_PATH", self.spec["hsa-rocr-dev"].prefix.lib) + self.set_lib_path(env, self.spec["hip"].prefix) + self.set_lib_path(env, self.spec["hsa-rocr-dev"].prefix) self.setup_chpl_comm(env, self.spec) def setup_build_environment(self, env): @@ -731,11 +791,9 @@ def test_chpldoc(self): if not self.spec.satisfies("+chpldoc"): print("Skipping chpldoc test as chpldoc variant is not set") return - with working_dir(self.test_suite.current_test_cache_dir): - with set_env(CHPL_HOME=self.test_suite.current_test_cache_dir): - with test_part(self, "test_chpldoc", purpose="test chpldoc"): - res = subprocess.run(["util/test/checkChplDoc"]) - assert res.returncode == 0 + else: + # TODO: Need to update checkChplDoc to work in the spack testing environment + pass # TODO: In order to run these tests, there's a lot of infrastructure to copy # from the Chapel test suite and there are conflicts with CHPL_HOME needing @@ -795,6 +853,7 @@ def copy_test_files(self): "util/test", "util/chplenv", "util/config", + "util/printchplenv", # "test/library/packages/Curl", # "test/library/packages/URL/", # "test/library/packages/ProtobufProtocolSupport/",