modules: hide implicit modulefiles (#36619)

Renames exclude_implicits to hide_implicits

When hide_implicits option is enabled, generate modulefile of
implicitly installed software and hide them. Even if implicit, those
modulefiles may be referred as dependency in other modulefiles thus they
should be generated to make module properly load dependent module.

A new hidden property is added to BaseConfiguration class.

To hide modulefiles, modulercs are generated along modulefiles. Such rc
files contain specific module command to indicate a module should be
hidden (for instance when using "module avail").

A modulerc property is added to TclFileLayout and LmodFileLayout classes
to get fully qualified path name of the modulerc associated to a given
modulefile.

Modulerc files will be located in each module directory, next to the
version modulefiles. This scheme is supported by both module tool
implementations.

modulerc_header and hide_cmd_format attributes are added to
TclModulefileWriter and LmodModulefileWriter. They help to know how to
generate a modulerc file with hidden commands for each module tool.

Tcl modulerc file requires an header. As we use a command introduced on
Modules 4.7 (module-hide --hidden-loaded), a version requirement is
added to header string.

For lmod, modules that open up a hierarchy are never hidden, even if
they are implicitly installed.

Modulerc is created, updated or removed when associated modulefile is
written or removed. If an implicit modulefile becomes explicit, hidden
command in modulerc for this modulefile is removed. If modulerc becomes
empty, this file is removed. Modulerc file is not rewritten when no
content change is detected.

Co-authored-by: Harmen Stoppels <me@harmenstoppels.nl>
This commit is contained in:
Xavier Delaruelle 2023-10-26 13:49:13 +02:00 committed by GitHub
parent bf88ed45da
commit 86520abb68
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
10 changed files with 407 additions and 20 deletions

View file

@ -491,10 +491,6 @@ def excluded(self):
exclude_rules = conf.get("exclude", [])
exclude_matches = [x for x in exclude_rules if spec.satisfies(x)]
# Should I exclude the module because it's implicit?
exclude_implicits = conf.get("exclude_implicits", None)
excluded_as_implicit = exclude_implicits and not self.explicit
def debug_info(line_header, match_list):
if match_list:
msg = "\t{0} : {1}".format(line_header, spec.cshort_spec)
@ -505,16 +501,28 @@ def debug_info(line_header, match_list):
debug_info("INCLUDE", include_matches)
debug_info("EXCLUDE", exclude_matches)
if excluded_as_implicit:
msg = "\tEXCLUDED_AS_IMPLICIT : {0}".format(spec.cshort_spec)
tty.debug(msg)
is_excluded = exclude_matches or excluded_as_implicit
if not include_matches and is_excluded:
if not include_matches and exclude_matches:
return True
return False
@property
def hidden(self):
"""Returns True if the module has been hidden, False otherwise."""
# A few variables for convenience of writing the method
spec = self.spec
conf = self.module.configuration(self.name)
hidden_as_implicit = not self.explicit and conf.get(
"hide_implicits", conf.get("exclude_implicits", False)
)
if hidden_as_implicit:
tty.debug(f"\tHIDDEN_AS_IMPLICIT : {spec.cshort_spec}")
return hidden_as_implicit
@property
def context(self):
return self.conf.get("context", {})
@ -849,6 +857,26 @@ def __init__(self, spec, module_set_name, explicit=None):
name = type(self).__name__
raise DefaultTemplateNotDefined(msg.format(name))
# Check if format for module hide command has been defined,
# throw if not found
try:
self.hide_cmd_format
except AttributeError:
msg = "'{0}' object has no attribute 'hide_cmd_format'\n"
msg += "Did you forget to define it in the class?"
name = type(self).__name__
raise HideCmdFormatNotDefined(msg.format(name))
# Check if modulerc header content has been defined,
# throw if not found
try:
self.modulerc_header
except AttributeError:
msg = "'{0}' object has no attribute 'modulerc_header'\n"
msg += "Did you forget to define it in the class?"
name = type(self).__name__
raise ModulercHeaderNotDefined(msg.format(name))
def _get_template(self):
"""Gets the template that will be rendered for this spec."""
# Get templates and put them in the order of importance:
@ -943,6 +971,9 @@ def write(self, overwrite=False):
# Symlink defaults if needed
self.update_module_defaults()
# record module hiddenness if implicit
self.update_module_hiddenness()
def update_module_defaults(self):
if any(self.spec.satisfies(default) for default in self.conf.defaults):
# This spec matches a default, it needs to be symlinked to default
@ -953,6 +984,60 @@ def update_module_defaults(self):
os.symlink(self.layout.filename, default_tmp)
os.rename(default_tmp, default_path)
def update_module_hiddenness(self, remove=False):
"""Update modulerc file corresponding to module to add or remove
command that hides module depending on its hidden state.
Args:
remove (bool): if True, hiddenness information for module is
removed from modulerc.
"""
modulerc_path = self.layout.modulerc
hide_module_cmd = self.hide_cmd_format % self.layout.use_name
hidden = self.conf.hidden and not remove
modulerc_exists = os.path.exists(modulerc_path)
updated = False
if modulerc_exists:
# retrieve modulerc content
with open(modulerc_path, "r") as f:
content = f.readlines()
content = "".join(content).split("\n")
# remove last empty item if any
if len(content[-1]) == 0:
del content[-1]
already_hidden = hide_module_cmd in content
# remove hide command if module not hidden
if already_hidden and not hidden:
content.remove(hide_module_cmd)
updated = True
# add hide command if module is hidden
elif not already_hidden and hidden:
if len(content) == 0:
content = self.modulerc_header.copy()
content.append(hide_module_cmd)
updated = True
else:
content = self.modulerc_header.copy()
if hidden:
content.append(hide_module_cmd)
updated = True
# no modulerc file change if no content update
if updated:
is_empty = content == self.modulerc_header or len(content) == 0
# remove existing modulerc if empty
if modulerc_exists and is_empty:
os.remove(modulerc_path)
# create or update modulerc
elif content != self.modulerc_header:
# ensure file ends with a newline character
content.append("")
with open(modulerc_path, "w") as f:
f.write("\n".join(content))
def remove(self):
"""Deletes the module file."""
mod_file = self.layout.filename
@ -960,6 +1045,7 @@ def remove(self):
try:
os.remove(mod_file) # Remove the module file
self.remove_module_defaults() # Remove default targeting module file
self.update_module_hiddenness(remove=True) # Remove hide cmd in modulerc
os.removedirs(
os.path.dirname(mod_file)
) # Remove all the empty directories from the leaf up
@ -1003,5 +1089,17 @@ class DefaultTemplateNotDefined(AttributeError, ModulesError):
"""
class HideCmdFormatNotDefined(AttributeError, ModulesError):
"""Raised if the attribute 'hide_cmd_format' has not been specified
in the derived classes.
"""
class ModulercHeaderNotDefined(AttributeError, ModulesError):
"""Raised if the attribute 'modulerc_header' has not been specified
in the derived classes.
"""
class ModulesTemplateNotFoundError(ModulesError, RuntimeError):
"""Raised if the template for a module file was not found."""

View file

@ -232,6 +232,13 @@ def missing(self):
"""Returns the list of tokens that are not available."""
return [x for x in self.hierarchy_tokens if x not in self.available]
@property
def hidden(self):
# Never hide a module that opens a hierarchy
if any(self.spec.package.provides(x) for x in self.hierarchy_tokens):
return False
return super().hidden
class LmodFileLayout(BaseFileLayout):
"""File layout for lmod module files."""
@ -274,6 +281,13 @@ def filename(self):
)
return fullname
@property
def modulerc(self):
"""Returns the modulerc file associated with current module file"""
return os.path.join(
os.path.dirname(self.filename), ".".join([".modulerc", self.extension])
)
def token_to_path(self, name, value):
"""Transforms a hierarchy token into the corresponding path part.
@ -470,6 +484,10 @@ class LmodModulefileWriter(BaseModuleFileWriter):
default_template = posixpath.join("modules", "modulefile.lua")
modulerc_header: list = []
hide_cmd_format = 'hide_version("%s")'
class CoreCompilersNotFoundError(spack.error.SpackError, KeyError):
"""Error raised if the key 'core_compilers' has not been specified

View file

@ -6,6 +6,7 @@
"""This module implements the classes necessary to generate Tcl
non-hierarchical modules.
"""
import os.path
import posixpath
from typing import Any, Dict
@ -56,6 +57,11 @@ class TclConfiguration(BaseConfiguration):
class TclFileLayout(BaseFileLayout):
"""File layout for tcl module files."""
@property
def modulerc(self):
"""Returns the modulerc file associated with current module file"""
return os.path.join(os.path.dirname(self.filename), ".modulerc")
class TclContext(BaseContext):
"""Context class for tcl module files."""
@ -73,3 +79,7 @@ class TclModulefileWriter(BaseModuleFileWriter):
# os.path.join due to spack.spec.Spec.format
# requiring forward slash path seperators at this stage
default_template = posixpath.join("modules", "modulefile.tcl")
modulerc_header = ["#%Module4.7"]
hide_cmd_format = "module-hide --soft --hidden-loaded %s"

View file

@ -17,7 +17,7 @@
#: THIS NEEDS TO BE UPDATED FOR EVERY NEW KEYWORD THAT
#: IS ADDED IMMEDIATELY BELOW THE MODULE TYPE ATTRIBUTE
spec_regex = (
r"(?!hierarchy|core_specs|verbose|hash_length|defaults|filter_hierarchy_specs|"
r"(?!hierarchy|core_specs|verbose|hash_length|defaults|filter_hierarchy_specs|hide|"
r"whitelist|blacklist|" # DEPRECATED: remove in 0.20.
r"include|exclude|" # use these more inclusive/consistent options
r"projections|naming_scheme|core_compilers|all)(^\w[\w-]*)"
@ -89,6 +89,7 @@
"exclude": array_of_strings,
"exclude_implicits": {"type": "boolean", "default": False},
"defaults": array_of_strings,
"hide_implicits": {"type": "boolean", "default": False},
"naming_scheme": {"type": "string"}, # Can we be more specific here?
"projections": projections_scheme,
"all": module_file_configuration,
@ -187,3 +188,52 @@
"additionalProperties": False,
"properties": properties,
}
# deprecated keys and their replacements
old_to_new_key = {"exclude_implicits": "hide_implicits"}
def update_keys(data, key_translations):
"""Change blacklist/whitelist to exclude/include.
Arguments:
data (dict): data from a valid modules configuration.
key_translations (dict): A dictionary of keys to translate to
their respective values.
Return:
(bool) whether anything was changed in data
"""
changed = False
if isinstance(data, dict):
keys = list(data.keys())
for key in keys:
value = data[key]
translation = key_translations.get(key)
if translation:
data[translation] = data.pop(key)
changed = True
changed |= update_keys(value, key_translations)
elif isinstance(data, list):
for elt in data:
changed |= update_keys(elt, key_translations)
return changed
def update(data):
"""Update the data in place to remove deprecated properties.
Args:
data (dict): dictionary to be updated
Returns:
True if data was changed, False otherwise
"""
# translate blacklist/whitelist to exclude/include
return update_keys(data, old_to_new_key)

View file

@ -0,0 +1,11 @@
enable:
- lmod
lmod:
hide_implicits: true
core_compilers:
- 'clang@3.3'
hierarchy:
- mpi
all:
autoload: direct

View file

@ -1,3 +1,5 @@
# DEPRECATED: remove this in ?
# See `hide_implicits.yaml` for the new syntax
enable:
- tcl
tcl:

View file

@ -0,0 +1,6 @@
enable:
- tcl
tcl:
hide_implicits: true
all:
autoload: direct

View file

@ -14,6 +14,7 @@
import spack.package_base
import spack.schema.modules
import spack.spec
import spack.util.spack_yaml as syaml
from spack.modules.common import UpstreamModuleIndex
from spack.spec import Spec
@ -190,11 +191,30 @@ def find_nothing(*args):
spack.package_base.PackageBase.uninstall_by_spec(spec)
@pytest.mark.parametrize(
"module_type, old_config,new_config",
[("tcl", "exclude_implicits.yaml", "hide_implicits.yaml")],
)
def test_exclude_include_update(module_type, old_config, new_config):
module_test_data_root = os.path.join(spack.paths.test_path, "data", "modules", module_type)
with open(os.path.join(module_test_data_root, old_config)) as f:
old_yaml = syaml.load(f)
with open(os.path.join(module_test_data_root, new_config)) as f:
new_yaml = syaml.load(f)
# ensure file that needs updating is translated to the right thing.
assert spack.schema.modules.update_keys(old_yaml, spack.schema.modules.old_to_new_key)
assert new_yaml == old_yaml
# ensure a file that doesn't need updates doesn't get updated
original_new_yaml = new_yaml.copy()
assert not spack.schema.modules.update_keys(new_yaml, spack.schema.modules.old_to_new_key)
assert original_new_yaml == new_yaml
@pytest.mark.regression("37649")
def test_check_module_set_name(mutable_config):
"""Tests that modules set name are validated correctly and an error is reported if the
name we require does not exist or is reserved by the configuration."""
# Minimal modules.yaml config.
spack.config.set(
"modules",

View file

@ -3,6 +3,7 @@
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
import os
import pytest
@ -433,3 +434,87 @@ def test_modules_no_arch(self, factory, module_configuration):
path = module.layout.filename
assert str(spec.os) not in path
def test_hide_implicits(self, module_configuration):
"""Tests the addition and removal of hide command in modulerc."""
module_configuration("hide_implicits")
spec = spack.spec.Spec("mpileaks@2.3").concretized()
# mpileaks is defined as implicit, thus hide command should appear in modulerc
writer = writer_cls(spec, "default", False)
writer.write()
assert os.path.exists(writer.layout.modulerc)
with open(writer.layout.modulerc) as f:
content = f.readlines()
content = "".join(content).split("\n")
hide_cmd = 'hide_version("%s")' % writer.layout.use_name
assert len([x for x in content if hide_cmd == x]) == 1
# mpileaks becomes explicit, thus modulerc is removed
writer = writer_cls(spec, "default", True)
writer.write(overwrite=True)
assert not os.path.exists(writer.layout.modulerc)
# mpileaks is defined as explicit, no modulerc file should exist
writer = writer_cls(spec, "default", True)
writer.write()
assert not os.path.exists(writer.layout.modulerc)
# explicit module is removed
writer.remove()
assert not os.path.exists(writer.layout.modulerc)
assert not os.path.exists(writer.layout.filename)
# implicit module is removed
writer = writer_cls(spec, "default", False)
writer.write(overwrite=True)
assert os.path.exists(writer.layout.filename)
assert os.path.exists(writer.layout.modulerc)
writer.remove()
assert not os.path.exists(writer.layout.modulerc)
assert not os.path.exists(writer.layout.filename)
# three versions of mpileaks are implicit
writer = writer_cls(spec, "default", False)
writer.write(overwrite=True)
spec_alt1 = spack.spec.Spec("mpileaks@2.2").concretized()
spec_alt2 = spack.spec.Spec("mpileaks@2.1").concretized()
writer_alt1 = writer_cls(spec_alt1, "default", False)
writer_alt1.write(overwrite=True)
writer_alt2 = writer_cls(spec_alt2, "default", False)
writer_alt2.write(overwrite=True)
assert os.path.exists(writer.layout.modulerc)
with open(writer.layout.modulerc) as f:
content = f.readlines()
content = "".join(content).split("\n")
hide_cmd = 'hide_version("%s")' % writer.layout.use_name
hide_cmd_alt1 = 'hide_version("%s")' % writer_alt1.layout.use_name
hide_cmd_alt2 = 'hide_version("%s")' % writer_alt2.layout.use_name
assert len([x for x in content if hide_cmd == x]) == 1
assert len([x for x in content if hide_cmd_alt1 == x]) == 1
assert len([x for x in content if hide_cmd_alt2 == x]) == 1
# one version is removed, a second becomes explicit
writer_alt1.remove()
writer_alt2 = writer_cls(spec_alt2, "default", True)
writer_alt2.write(overwrite=True)
assert os.path.exists(writer.layout.modulerc)
with open(writer.layout.modulerc) as f:
content = f.readlines()
content = "".join(content).split("\n")
assert len([x for x in content if hide_cmd == x]) == 1
assert len([x for x in content if hide_cmd_alt1 == x]) == 0
assert len([x for x in content if hide_cmd_alt2 == x]) == 0
# disable hide_implicits configuration option
module_configuration("autoload_direct")
writer = writer_cls(spec, "default")
writer.write(overwrite=True)
assert not os.path.exists(writer.layout.modulerc)
# reenable hide_implicits configuration option
module_configuration("hide_implicits")
writer = writer_cls(spec, "default")
writer.write(overwrite=True)
assert os.path.exists(writer.layout.modulerc)

View file

@ -3,6 +3,7 @@
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
import os
import pytest
@ -438,38 +439,40 @@ def test_extend_context(self, modulefile_content, module_configuration):
@pytest.mark.regression("4400")
@pytest.mark.db
def test_exclude_implicits(self, module_configuration, database):
module_configuration("exclude_implicits")
@pytest.mark.parametrize("config_name", ["hide_implicits", "exclude_implicits"])
def test_hide_implicits_no_arg(self, module_configuration, database, config_name):
module_configuration(config_name)
# mpileaks has been installed explicitly when setting up
# the tests database
mpileaks_specs = database.query("mpileaks")
for item in mpileaks_specs:
writer = writer_cls(item, "default")
assert not writer.conf.excluded
assert not writer.conf.hidden
# callpath is a dependency of mpileaks, and has been pulled
# in implicitly
callpath_specs = database.query("callpath")
for item in callpath_specs:
writer = writer_cls(item, "default")
assert writer.conf.excluded
assert writer.conf.hidden
@pytest.mark.regression("12105")
def test_exclude_implicits_with_arg(self, module_configuration):
module_configuration("exclude_implicits")
@pytest.mark.parametrize("config_name", ["hide_implicits", "exclude_implicits"])
def test_hide_implicits_with_arg(self, module_configuration, config_name):
module_configuration(config_name)
# mpileaks is defined as explicit with explicit argument set on writer
mpileaks_spec = spack.spec.Spec("mpileaks")
mpileaks_spec.concretize()
writer = writer_cls(mpileaks_spec, "default", True)
assert not writer.conf.excluded
assert not writer.conf.hidden
# callpath is defined as implicit with explicit argument set on writer
callpath_spec = spack.spec.Spec("callpath")
callpath_spec.concretize()
writer = writer_cls(callpath_spec, "default", False)
assert writer.conf.excluded
assert writer.conf.hidden
@pytest.mark.regression("9624")
@pytest.mark.db
@ -498,3 +501,87 @@ def test_modules_no_arch(self, factory, module_configuration):
path = module.layout.filename
assert str(spec.os) not in path
def test_hide_implicits(self, module_configuration):
"""Tests the addition and removal of hide command in modulerc."""
module_configuration("hide_implicits")
spec = spack.spec.Spec("mpileaks@2.3").concretized()
# mpileaks is defined as implicit, thus hide command should appear in modulerc
writer = writer_cls(spec, "default", False)
writer.write()
assert os.path.exists(writer.layout.modulerc)
with open(writer.layout.modulerc) as f:
content = f.readlines()
content = "".join(content).split("\n")
hide_cmd = "module-hide --soft --hidden-loaded %s" % writer.layout.use_name
assert len([x for x in content if hide_cmd == x]) == 1
# mpileaks becomes explicit, thus modulerc is removed
writer = writer_cls(spec, "default", True)
writer.write(overwrite=True)
assert not os.path.exists(writer.layout.modulerc)
# mpileaks is defined as explicit, no modulerc file should exist
writer = writer_cls(spec, "default", True)
writer.write()
assert not os.path.exists(writer.layout.modulerc)
# explicit module is removed
writer.remove()
assert not os.path.exists(writer.layout.modulerc)
assert not os.path.exists(writer.layout.filename)
# implicit module is removed
writer = writer_cls(spec, "default", False)
writer.write(overwrite=True)
assert os.path.exists(writer.layout.filename)
assert os.path.exists(writer.layout.modulerc)
writer.remove()
assert not os.path.exists(writer.layout.modulerc)
assert not os.path.exists(writer.layout.filename)
# three versions of mpileaks are implicit
writer = writer_cls(spec, "default", False)
writer.write(overwrite=True)
spec_alt1 = spack.spec.Spec("mpileaks@2.2").concretized()
spec_alt2 = spack.spec.Spec("mpileaks@2.1").concretized()
writer_alt1 = writer_cls(spec_alt1, "default", False)
writer_alt1.write(overwrite=True)
writer_alt2 = writer_cls(spec_alt2, "default", False)
writer_alt2.write(overwrite=True)
assert os.path.exists(writer.layout.modulerc)
with open(writer.layout.modulerc) as f:
content = f.readlines()
content = "".join(content).split("\n")
hide_cmd = "module-hide --soft --hidden-loaded %s" % writer.layout.use_name
hide_cmd_alt1 = "module-hide --soft --hidden-loaded %s" % writer_alt1.layout.use_name
hide_cmd_alt2 = "module-hide --soft --hidden-loaded %s" % writer_alt2.layout.use_name
assert len([x for x in content if hide_cmd == x]) == 1
assert len([x for x in content if hide_cmd_alt1 == x]) == 1
assert len([x for x in content if hide_cmd_alt2 == x]) == 1
# one version is removed, a second becomes explicit
writer_alt1.remove()
writer_alt2 = writer_cls(spec_alt2, "default", True)
writer_alt2.write(overwrite=True)
assert os.path.exists(writer.layout.modulerc)
with open(writer.layout.modulerc) as f:
content = f.readlines()
content = "".join(content).split("\n")
assert len([x for x in content if hide_cmd == x]) == 1
assert len([x for x in content if hide_cmd_alt1 == x]) == 0
assert len([x for x in content if hide_cmd_alt2 == x]) == 0
# disable hide_implicits configuration option
module_configuration("autoload_direct")
writer = writer_cls(spec, "default")
writer.write(overwrite=True)
assert not os.path.exists(writer.layout.modulerc)
# reenable hide_implicits configuration option
module_configuration("hide_implicits")
writer = writer_cls(spec, "default")
writer.write(overwrite=True)
assert os.path.exists(writer.layout.modulerc)