From 480b7f397e029a3de8d8a032170cd9e9862d64d3 Mon Sep 17 00:00:00 2001 From: Greg Becker Date: Mon, 17 Apr 2023 22:17:11 -0700 Subject: [PATCH] Allow users to remove items from hierarchy per-path (#31351) * lmod modules: allow users to remove items from hierarchy per-spec This allows MPI wrappers that depend on MPI to be removed from the MPI portion of the hierarchy and be made available when the appropriate compiler is loaded. module load gcc module load mpi-wrapper # implicitly loads mpi module load hdf5 This allows users to treat an mpi wrapper like an mpi program --- lib/spack/spack/modules/lmod.py | 15 +++++++++++++++ lib/spack/spack/schema/modules.py | 6 +++++- .../test/data/modules/lmod/complex_hierarchy.yaml | 3 +++ lib/spack/spack/test/modules/lmod.py | 12 +++++++++++- 4 files changed, 34 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/modules/lmod.py b/lib/spack/spack/modules/lmod.py index 81c3e74c42..812d4fcc9e 100644 --- a/lib/spack/spack/modules/lmod.py +++ b/lib/spack/spack/modules/lmod.py @@ -126,6 +126,11 @@ def core_specs(self): """Returns the list of "Core" specs""" return configuration(self.name).get("core_specs", []) + @property + def filter_hierarchy_specs(self): + """Returns the dict of specs with modified hierarchies""" + return configuration(self.name).get("filter_hierarchy_specs", {}) + @property def hierarchy_tokens(self): """Returns the list of tokens that are part of the modulefile @@ -160,11 +165,21 @@ def requires(self): if any(self.spec.satisfies(core_spec) for core_spec in self.core_specs): return {"compiler": self.core_compilers[0]} + hierarchy_filter_list = [] + for spec, filter_list in self.filter_hierarchy_specs.items(): + if self.spec.satisfies(spec): + hierarchy_filter_list = filter_list + break + # Keep track of the requirements that this package has in terms # of virtual packages that participate in the hierarchical structure requirements = {"compiler": self.spec.compiler} # For each virtual dependency in the hierarchy for x in self.hierarchy_tokens: + # Skip anything filtered for this spec + if x in hierarchy_filter_list: + continue + # If I depend on it if x in self.spec and not self.spec.package.provides(x): requirements[x] = self.spec[x] # record the actual provider diff --git a/lib/spack/spack/schema/modules.py b/lib/spack/spack/schema/modules.py index 975b512c7f..261065c8b3 100644 --- a/lib/spack/spack/schema/modules.py +++ b/lib/spack/spack/schema/modules.py @@ -17,7 +17,7 @@ #: THIS NEEDS TO BE UPDATED FOR EVERY NEW KEYWORD THAT #: IS ADDED IMMEDIATELY BELOW THE MODULE TYPE ATTRIBUTE spec_regex = ( - r"(?!hierarchy|core_specs|verbose|hash_length|defaults|" + r"(?!hierarchy|core_specs|verbose|hash_length|defaults|filter_hierarchy_specs|" r"whitelist|blacklist|" # DEPRECATED: remove in 0.20. r"include|exclude|" # use these more inclusive/consistent options r"projections|naming_scheme|core_compilers|all)(^\w[\w-]*)" @@ -127,6 +127,10 @@ "core_compilers": array_of_strings, "hierarchy": array_of_strings, "core_specs": array_of_strings, + "filter_hierarchy_specs": { + "type": "object", + "patternProperties": {spec_regex: array_of_strings}, + }, }, }, # Specific lmod extensions ] diff --git a/lib/spack/spack/test/data/modules/lmod/complex_hierarchy.yaml b/lib/spack/spack/test/data/modules/lmod/complex_hierarchy.yaml index cb39ecfa92..618163de01 100644 --- a/lib/spack/spack/test/data/modules/lmod/complex_hierarchy.yaml +++ b/lib/spack/spack/test/data/modules/lmod/complex_hierarchy.yaml @@ -14,6 +14,9 @@ lmod: - blas - mpi + filter_hierarchy_specs: + 'mpileaks@:2.1': [mpi] + verbose: false all: diff --git a/lib/spack/spack/test/modules/lmod.py b/lib/spack/spack/test/modules/lmod.py index 3ba4953844..30e1b58905 100644 --- a/lib/spack/spack/test/modules/lmod.py +++ b/lib/spack/spack/test/modules/lmod.py @@ -35,6 +35,8 @@ def compiler(request): ("mpich@3.0.1", []), ("openblas@0.2.15", ("blas",)), ("openblas-with-lapack@0.2.15", ("blas", "lapack")), + ("mpileaks@2.3", ("mpi",)), + ("mpileaks@2.1", []), ] ) def provider(request): @@ -69,12 +71,20 @@ def test_file_layout(self, compiler, provider, factory, module_configuration): path_parts = layout.available_path_parts service_part = spec_string.replace("@", "/") service_part = "-".join([service_part, layout.spec.dag_hash(length=7)]) - assert service_part in path_parts + + if "mpileaks" in spec_string: + # It's a user, not a provider, so create the provider string + service_part = layout.spec["mpi"].format("{name}/{version}-{hash:7}") + else: + # Only relevant for providers, not users, of virtuals + assert service_part in path_parts # Check that multi-providers have repetitions in path parts repetitions = len([x for x in path_parts if service_part == x]) if spec_string == "openblas-with-lapack@0.2.15": assert repetitions == 2 + elif spec_string == "mpileaks@2.1": + assert repetitions == 0 else: assert repetitions == 1