Chapel package: updates post release (#45304)

* Fix +rocm variant, to ensure correct dependencies on ROCm packages
  and use of AMD LLVM
* Add a +pshm variant for comm=gasnet to enable fast shared-memory
  comms between co-locales
* Add logic to ensure we get the native CXI libfabric network provider
  on Cray EX
* Expand dependency type for package modules to encompass runtime
  dependencies
* Factor logic for setting (LD_)LIBRARY_PATH and PKG_CONFIG_PATH of
  runtime dependencies
* Workaround issue #44746 that causes a transitive dependency on lua
  to break SLURM
* Disable nonfunctional checkChplDoc test
* Annotate some variants as conditional, to improve spack info output
  and reduce confusion

---------

Co-authored-by: Dan Bonachea <dobonachea@lbl.gov>

var/spack/repos/builtin/packages/chapel/package.py

@@ -6,6 +6,7 @@
 import os
 import subprocess
 
+import spack.platforms.cray
 from spack.package import *
 from spack.util.environment import is_system_path, set_env
 
@@ -174,6 +175,14 @@ class Chapel(AutotoolsPackage, CudaPackage, ROCmPackage):
         values=("ibv", "ofi", "udp", "smp", "unset"),
         multi=False,
         sticky=True,  # never allow the concretizer to choose this
+        when="comm=gasnet",
+    )
+
+    variant(
+        "pshm",
+        default=False,
+        description="Build Chapel with fast shared-memory comms between co-locales",
+        when="comm=gasnet",
     )
 
     # Chapel depends on GASNet whenever comm=gasnet.
@@ -186,6 +195,7 @@ class Chapel(AutotoolsPackage, CudaPackage, ROCmPackage):
         default="bundled",
         values=("bundled", "spack"),
         multi=False,
+        when="comm=gasnet",
     )
 
     variant(
@@ -195,6 +205,7 @@ class Chapel(AutotoolsPackage, CudaPackage, ROCmPackage):
         "supplied CHPL_GASNET_SEGMENT",
         values=("everything", "fast", "large", "unset"),
         multi=False,
+        when="comm=gasnet",
     )
 
     variant(
@@ -277,9 +288,25 @@ class Chapel(AutotoolsPackage, CudaPackage, ROCmPackage):
     variant(
         "libfabric",
         default="unset",
-        description="When building with ofi support, specify libfabric option",
+        description="Control the libfabric version used for multi-locale communication",
         values=("bundled", "spack", "unset"),
         multi=False,
+        when="comm=ofi",
+    )
+
+    variant(
+        "libfabric",
+        default="unset",
+        description="Control the libfabric version used for multi-locale communication",
+        values=("bundled", "spack", "unset"),
+        multi=False,
+        when="comm=gasnet comm_substrate=ofi",
+    )
+
+    requires(
+        "^libfabric" + (" fabrics=cxi" if spack.platforms.cray.slingshot_network() else ""),
+        when="libfabric=spack",
+        msg="libfabric requires cxi fabric provider on HPE-Cray EX machines",
     )
 
     variant(
@@ -357,7 +384,7 @@ class Chapel(AutotoolsPackage, CudaPackage, ROCmPackage):
             ),
             default=True,
         )
-        depends_on(dep, when="+{0}".format(variant_name))
+        depends_on(dep, when="+{0}".format(variant_name), type=("build", "link", "run", "test"))
 
     # TODO: for CHPL_X_CC and CHPL_X_CXX, can we capture an arbitrary path, possibly
     # with arguments?
@@ -406,7 +433,14 @@ class Chapel(AutotoolsPackage, CudaPackage, ROCmPackage):
     conflicts("platform=windows")  # Support for windows is through WSL only
 
     conflicts("+rocm", when="+cuda", msg="Chapel must be built with either CUDA or ROCm, not both")
-    conflicts("+rocm", when="@:2.0.0", msg="ROCm support in spack requires Chapel 2.0.0 or later")
+    conflicts(
+        "+rocm", when="@:1.99.99", msg="ROCm support in spack requires Chapel 2.0.0 or later"
+    )
+    # Chapel restricts the allowable ROCm versions
+    with when("+rocm"):
+        depends_on("hsa-rocr-dev@4:5.4")
+        depends_on("hip@4:5.4")
+        depends_on("llvm-amdgpu@4:5.4")
 
     conflicts(
         "comm_substrate=unset",
@@ -414,9 +448,19 @@ class Chapel(AutotoolsPackage, CudaPackage, ROCmPackage):
         msg="comm=gasnet requires you to also set comm_substrate= to the appropriate network",
     )
 
+    conflicts(
+        "gasnet_segment=everything",
+        when="+pshm",
+        msg="gasnet_segment=everything does not support +pshm",
+    )
+
+    # comm_substrate=udp gasnet_segment=unset defaults to everything,
+    # which is incompatible with +pshm
+    requires("gasnet_segment=fast", when="+pshm comm_substrate=udp")
+
     conflicts(
         "^python@3.12:",
-        when="@:2.1.0",
+        when="@:2.0.99",
         msg="Chapel versions prior to 2.1.0 may produce SyntaxWarnings with Python >= 3.12",
     )
 
@@ -443,20 +487,18 @@ class Chapel(AutotoolsPackage, CudaPackage, ROCmPackage):
     depends_on("doxygen@1.8.17:", when="+chpldoc")
 
     # TODO: keep up to date with util/chplenv/chpl_llvm.py
-    with when("llvm=spack"):
+    with when("llvm=spack ~rocm"):
         depends_on("llvm@11:17", when="@:2.0.1")
         depends_on("llvm@11:18", when="@2.1.0:")
 
     # Based on docs https://chapel-lang.org/docs/technotes/gpu.html#requirements
-    depends_on("llvm@16:", when="llvm=spack ^cuda@12:")
+    depends_on("llvm@16:", when="llvm=spack +cuda ^cuda@12:")
     requires(
         "^llvm targets=all",
         msg="llvm=spack +cuda requires LLVM support the nvptx target",
         when="llvm=spack +cuda",
     )
 
-    depends_on("cuda@11:", when="+cuda", type=("build", "link", "run", "test"))
-
     # This is because certain systems have binutils installed as a system package
     # but do not include the headers. Spack incorrectly supplies those external
     # packages as proper dependencies for LLVM, but then LLVM will fail to build
@@ -465,11 +507,17 @@ class Chapel(AutotoolsPackage, CudaPackage, ROCmPackage):
 
     depends_on("m4")
 
-    depends_on("gmp", when="gmp=spack", type=("build", "link", "run", "test"))
+    # Runtime dependencies:
-    depends_on("hwloc", when="hwloc=spack", type=("build", "link", "run", "test"))
+    # Note here "run" is run of the Chapel compiler built by this package,
-    depends_on("libfabric", when="libfabric=spack", type=("build", "link", "run", "test"))
+    # but many of these are ALSO run-time dependencies of the executable
-    depends_on("libunwind", when="unwind=spack", type=("build", "link", "run", "test"))
+    # application built by that Chapel compiler from user-provided sources.
-    depends_on("jemalloc", when="host_jemalloc=spack", type=("build", "link", "run", "test"))
+    with default_args(type=("build", "link", "run", "test")):
+        depends_on("cuda@11:", when="+cuda")
+        depends_on("gmp", when="gmp=spack")
+        depends_on("hwloc", when="hwloc=spack")
+        depends_on("libfabric", when="libfabric=spack")
+        depends_on("libunwind", when="unwind=spack")
+        depends_on("jemalloc", when="host_jemalloc=spack")
 
     depends_on("gasnet conduits=none", when="gasnet=spack")
     depends_on("gasnet@2024.5.0: conduits=none", when="@2.1.0: gasnet=spack")
@@ -519,13 +567,25 @@ def setup_chpl_compilers(self, env):
 
         # Undo spack compiler wrappers:
         # the C/C++ compilers must work post-install
-        if self.spec.satisfies("+cuda") or self.spec.satisfies("+rocm"):
+        if self.spec.satisfies("+rocm"):
             env.set("CHPL_TARGET_COMPILER", "llvm")
+            env.set(
+                "CHPL_LLVM_CONFIG",
+                join_path(self.spec["llvm-amdgpu"].prefix, "bin", "llvm-config"),
+            )
+            real_cc = join_path(self.spec["llvm-amdgpu"].prefix, "bin", "clang")
+            real_cxx = join_path(self.spec["llvm-amdgpu"].prefix, "bin", "clang++")
+
+            # +rocm appears to also require a matching LLVM host compiler to guarantee linkage
+            env.set("CHPL_HOST_COMPILER", "llvm")
+            env.set("CHPL_HOST_CC", real_cc)
+            env.set("CHPL_HOST_CXX", real_cxx)
+
+        elif self.spec.satisfies("llvm=spack"):
+            env.set("CHPL_TARGET_COMPILER", "llvm")
+            env.set("CHPL_LLVM_CONFIG", join_path(self.spec["llvm"].prefix, "bin", "llvm-config"))
             real_cc = join_path(self.spec["llvm"].prefix, "bin", "clang")
             real_cxx = join_path(self.spec["llvm"].prefix, "bin", "clang++")
-        elif is_CrayEX() and os.environ.get("CRAYPE_DIR"):
-            real_cc = join_path(os.environ["CRAYPE_DIR"], "bin", "cc")
-            real_cxx = join_path(os.environ["CRAYPE_DIR"], "bin", "CC")
         else:
             real_cc = self.compiler.cc
             real_cxx = self.compiler.cxx
@@ -535,6 +595,9 @@ def setup_chpl_compilers(self, env):
     def setup_chpl_comm(self, env, spec):
         env.set("CHPL_COMM", spec.variants["comm"].value)
 
+        if self.spec.satisfies("+pshm"):
+            env.set("CHPL_GASNET_MORE_CFG_OPTIONS", "--enable-pshm")
+
     @run_before("configure", when="gasnet=spack")
     def setup_gasnet(self):
         dst = join_path(self.stage.source_path, "third-party", "gasnet", "gasnet-src")
@@ -542,12 +605,6 @@ def setup_gasnet(self):
         os.rmdir(dst)
         symlink(self.spec["gasnet"].prefix.src, dst)
 
-    def setup_chpl_llvm(self, env):
-        if self.spec.variants["llvm"].value == "spack":
-            env.set(
-                "CHPL_LLVM_CONFIG", "{0}/{1}".format(self.spec["llvm"].prefix, "bin/llvm-config")
-            )
-
     def setup_if_not_unset(self, env, var, value):
         if value != "unset":
             if value == "spack":
@@ -558,12 +615,18 @@ def prepend_cpath_include(self, env, prefix):
         if not is_system_path(prefix):
             env.prepend_path("CPATH", prefix.include)
 
+    def set_lib_path(self, env, prefix):
+        if not is_system_path(prefix):
+            env.prepend_path("LD_LIBRARY_PATH", prefix.lib)
+            env.prepend_path("LIBRARY_PATH", prefix.lib)
+            if prefix.lib.pkgconfig is not None:
+                env.prepend_path("PKG_CONFIG_PATH", prefix.lib.pkgconfig)
+
     def setup_env_vars(self, env):
         # variants that appear unused by Spack typically correspond directly to
         # a CHPL_<variant> variable which will be used by the Chapel build system
         for v in self.spec.variants.keys():
             self.setup_if_not_unset(env, "CHPL_" + v.upper(), self.spec.variants[v].value)
-        self.setup_chpl_llvm(env)
         self.setup_chpl_compilers(env)
         self.setup_chpl_platform(env)
 
@@ -574,47 +637,51 @@ def setup_env_vars(self, env):
         if self.spec.satisfies("+developer"):
             env.set("CHPL_DEVELOPER", "true")
 
+        if not self.spec.satisfies("llvm=none"):
+            # workaround Spack issue #44746:
+            # Chapel does not directly utilize lua, but many of its
+            # launchers depend on system installs of batch schedulers
+            # (notably Slurm on Cray EX) which depend on a system Lua.
+            # LLVM includes lua as a dependency, but a barebones lua
+            # install lacks many packages provided by a system Lua,
+            # which are often required by system services like Slurm.
+            # Disable the incomplete Spack lua package directory to
+            # allow the system one to function.
+            env.unset("LUA_PATH")
+            env.unset("LUA_CPATH")
+
         if self.spec.variants["gmp"].value == "spack":
             # TODO: why must we add to CPATH to find gmp.h
             # TODO: why must we add to LIBRARY_PATH to find lgmp
             self.prepend_cpath_include(env, self.spec["gmp"].prefix)
-            env.prepend_path("LIBRARY_PATH", self.spec["gmp"].prefix.lib)
+            self.set_lib_path(env, self.spec["gmp"].prefix)
-            # Need this for the test env, where it does not appear automatic:
-            env.prepend_path("PKG_CONFIG_PATH", self.spec["gmp"].prefix.lib.pkgconfig)
 
         if self.spec.variants["hwloc"].value == "spack":
-            env.prepend_path("LD_LIBRARY_PATH", self.spec["hwloc"].prefix.lib)
+            self.set_lib_path(env, self.spec["hwloc"].prefix)
             # Need this for the test env, where it does not appear automatic:
-            env.prepend_path("PKG_CONFIG_PATH", self.spec["hwloc"].prefix.lib.pkgconfig)
             env.prepend_path("PKG_CONFIG_PATH", self.spec["libpciaccess"].prefix.lib.pkgconfig)
 
+        # TODO: unwind builds but resulting binaries fail to run, producing linker errors
         if self.spec.variants["unwind"].value == "spack":
             # chapel package would not build without cpath, missing libunwind.h
             self.prepend_cpath_include(env, self.spec["libunwind"].prefix)
             env.prepend_path("LD_LIBRARY_PATH", self.spec["libunwind"].prefix.lib)
 
         if self.spec.satisfies("+yaml"):
-            env.prepend_path("PKG_CONFIG_PATH", self.spec["libyaml"].prefix.lib.pkgconfig)
             self.prepend_cpath_include(env, self.spec["libyaml"].prefix)
             # could not compile test/library/packages/Yaml/writeAndParse.chpl without this
-            env.prepend_path("LIBRARY_PATH", self.spec["libyaml"].prefix.lib)
+            self.set_lib_path(env, self.spec["libyaml"].prefix)
 
         if self.spec.satisfies("+zmq"):
             self.prepend_cpath_include(env, self.spec["libzmq"].prefix)
             # could not compile test/library/packages/ZMQ/hello.chpl without this
-            env.prepend_path("LIBRARY_PATH", self.spec["libzmq"].prefix.lib)
+            self.set_lib_path(env, self.spec["libzmq"].prefix)
-            env.prepend_path("LD_LIBRARY_PATH", self.spec["libzmq"].prefix.lib)
-            # could not compile test/library/packages/ZMQ/hello.chpl without this
-            env.prepend_path("LIBRARY_PATH", self.spec["libzmq"].prefix.lib)
-            env.prepend_path("PKG_CONFIG_PATH", self.spec["libzmq"].prefix.lib.pkgconfig)
             env.prepend_path("PKG_CONFIG_PATH", self.spec["libsodium"].prefix.lib.pkgconfig)
 
         if self.spec.satisfies("+curl"):
             self.prepend_cpath_include(env, self.spec["curl"].prefix)
             # could not compile test/library/packages/Curl/check-http.chpl without this
-            env.prepend_path("LIBRARY_PATH", self.spec["curl"].prefix.lib)
+            self.set_lib_path(env, self.spec["curl"].prefix)
-            env.prepend_path("LD_LIBRARY_PATH", self.spec["curl"].prefix.lib)
-            env.prepend_path("PKG_CONFIG_PATH", self.spec["curl"].prefix.lib.pkgconfig)
 
         if self.spec.satisfies("+cuda"):
             # TODO: why must we add to LD_LIBRARY_PATH to find libcudart?
@@ -625,18 +692,11 @@ def setup_env_vars(self, env):
         if self.spec.satisfies("+rocm"):
             env.set("CHPL_LOCALE_MODEL", "gpu")
             env.set("CHPL_GPU", "amd")
-            env.set("CHPL_HOST_COMPILER", "llvm")
             env.set("CHPL_GPU_ARCH", self.spec.variants["amdgpu_target"].value[0])
-            env.set(
-                "CHPL_LLVM_CONFIG",
-                "{0}/{1}".format(self.spec["llvm-amdgpu"].prefix, "bin/llvm-config"),
-            )
             self.prepend_cpath_include(env, self.spec["hip"].prefix)
             env.set("CHPL_ROCM_PATH", self.spec["llvm-amdgpu"].prefix)
-            env.prepend_path("LIBRARY_PATH", self.spec["hip"].prefix.lib)
+            self.set_lib_path(env, self.spec["hip"].prefix)
-            env.prepend_path("LIBRARY_PATH", self.spec["hsa-rocr-dev"].prefix.lib)
+            self.set_lib_path(env, self.spec["hsa-rocr-dev"].prefix)
-            env.prepend_path("LD_LIBRARY_PATH", self.spec["hip"].prefix.lib)
-            env.prepend_path("LD_LIBRARY_PATH", self.spec["hsa-rocr-dev"].prefix.lib)
         self.setup_chpl_comm(env, self.spec)
 
     def setup_build_environment(self, env):
@@ -731,11 +791,9 @@ def test_chpldoc(self):
         if not self.spec.satisfies("+chpldoc"):
             print("Skipping chpldoc test as chpldoc variant is not set")
             return
-        with working_dir(self.test_suite.current_test_cache_dir):
+        else:
-            with set_env(CHPL_HOME=self.test_suite.current_test_cache_dir):
+            # TODO: Need to update checkChplDoc to work in the spack testing environment
-                with test_part(self, "test_chpldoc", purpose="test chpldoc"):
+            pass
-                    res = subprocess.run(["util/test/checkChplDoc"])
-                    assert res.returncode == 0
 
     # TODO: In order to run these tests, there's a lot of infrastructure to copy
     # from the Chapel test suite and there are conflicts with CHPL_HOME needing
@@ -795,6 +853,7 @@ def copy_test_files(self):
             "util/test",
             "util/chplenv",
             "util/config",
+            "util/printchplenv",
             #   "test/library/packages/Curl",
             #   "test/library/packages/URL/",
             #   "test/library/packages/ProtobufProtocolSupport/",