Environments: Add support for including definitions files (#33960)

This PR adds support for including separate definitions from `spack.yaml`.

Supporting the inclusion of files with definitions enables user to make
curated/standardized collections of packages that can re-used by others.
This commit is contained in:
Tamara Dahlgren 2023-11-05 00:47:06 -07:00 committed by GitHub
parent 5a67c578b7
commit c9dfb9b0fd
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
11 changed files with 307 additions and 160 deletions

View file

@ -69,6 +69,7 @@
SECTION_SCHEMAS = { SECTION_SCHEMAS = {
"compilers": spack.schema.compilers.schema, "compilers": spack.schema.compilers.schema,
"concretizer": spack.schema.concretizer.schema, "concretizer": spack.schema.concretizer.schema,
"definitions": spack.schema.definitions.schema,
"mirrors": spack.schema.mirrors.schema, "mirrors": spack.schema.mirrors.schema,
"repos": spack.schema.repos.schema, "repos": spack.schema.repos.schema,
"packages": spack.schema.packages.schema, "packages": spack.schema.packages.schema,
@ -994,6 +995,7 @@ def read_config_file(filename, schema=None):
key = next(iter(data)) key = next(iter(data))
schema = _ALL_SCHEMAS[key] schema = _ALL_SCHEMAS[key]
validate(data, schema) validate(data, schema)
return data return data
except StopIteration: except StopIteration:

View file

@ -781,10 +781,18 @@ def _re_read(self):
"""Reinitialize the environment object.""" """Reinitialize the environment object."""
self.clear(re_read=True) self.clear(re_read=True)
self.manifest = EnvironmentManifestFile(self.path) self.manifest = EnvironmentManifestFile(self.path)
self._read() self._read(re_read=True)
def _read(self): def _read(self, re_read=False):
self._construct_state_from_manifest() # If the manifest has included files, then some of the information
# (e.g., definitions) MAY be in those files. So we need to ensure
# the config is populated with any associated spec lists in order
# to fully construct the manifest state.
includes = self.manifest[TOP_LEVEL_KEY].get("include", [])
if includes and not re_read:
prepare_config_scope(self)
self._construct_state_from_manifest(re_read)
if os.path.exists(self.lock_path): if os.path.exists(self.lock_path):
with open(self.lock_path) as f: with open(self.lock_path) as f:
@ -798,11 +806,8 @@ def write_transaction(self):
"""Get a write lock context manager for use in a `with` block.""" """Get a write lock context manager for use in a `with` block."""
return lk.WriteTransaction(self.txlock, acquire=self._re_read) return lk.WriteTransaction(self.txlock, acquire=self._re_read)
def _construct_state_from_manifest(self): def _process_definition(self, item):
"""Read manifest file and set up user specs.""" """Process a single spec definition item."""
self.spec_lists = collections.OrderedDict()
env_configuration = self.manifest[TOP_LEVEL_KEY]
for item in env_configuration.get("definitions", []):
entry = copy.deepcopy(item) entry = copy.deepcopy(item)
when = _eval_conditional(entry.pop("when", "True")) when = _eval_conditional(entry.pop("when", "True"))
assert len(entry) == 1 assert len(entry) == 1
@ -814,6 +819,18 @@ def _construct_state_from_manifest(self):
else: else:
self.spec_lists[name] = user_specs self.spec_lists[name] = user_specs
def _construct_state_from_manifest(self, re_read=False):
"""Read manifest file and set up user specs."""
self.spec_lists = collections.OrderedDict()
if not re_read:
for item in spack.config.get("definitions", []):
self._process_definition(item)
env_configuration = self.manifest[TOP_LEVEL_KEY]
for item in env_configuration.get("definitions", []):
self._process_definition(item)
spec_list = env_configuration.get(user_speclist_name, []) spec_list = env_configuration.get(user_speclist_name, [])
user_specs = SpecList( user_specs = SpecList(
user_speclist_name, [s for s in spec_list if s], self.spec_lists.copy() user_speclist_name, [s for s in spec_list if s], self.spec_lists.copy()
@ -857,7 +874,9 @@ def clear(self, re_read=False):
yaml, and need to be maintained when re-reading an existing yaml, and need to be maintained when re-reading an existing
environment. environment.
""" """
self.spec_lists = {user_speclist_name: SpecList()} # specs from yaml self.spec_lists = collections.OrderedDict()
self.spec_lists[user_speclist_name] = SpecList()
self.dev_specs = {} # dev-build specs from yaml self.dev_specs = {} # dev-build specs from yaml
self.concretized_user_specs = [] # user specs from last concretize self.concretized_user_specs = [] # user specs from last concretize
self.concretized_order = [] # roots of last concretize, in order self.concretized_order = [] # roots of last concretize, in order
@ -1006,7 +1025,8 @@ def included_config_scopes(self):
elif include_url.scheme: elif include_url.scheme:
raise ValueError( raise ValueError(
"Unsupported URL scheme for environment include: {}".format(config_path) f"Unsupported URL scheme ({include_url.scheme}) for "
f"environment include: {config_path}"
) )
# treat relative paths as relative to the environment # treat relative paths as relative to the environment
@ -1068,8 +1088,10 @@ def update_stale_references(self, from_list=None):
from_list = next(iter(self.spec_lists.keys())) from_list = next(iter(self.spec_lists.keys()))
index = list(self.spec_lists.keys()).index(from_list) index = list(self.spec_lists.keys()).index(from_list)
# spec_lists is an OrderedDict, all list entries after the modified # spec_lists is an OrderedDict to ensure lists read from the manifest
# list may refer to the modified list. Update stale references # are maintainted in order, hence, all list entries after the modified
# list may refer to the modified list requiring stale references to be
# updated.
for i, (name, speclist) in enumerate( for i, (name, speclist) in enumerate(
list(self.spec_lists.items())[index + 1 :], index + 1 list(self.spec_lists.items())[index + 1 :], index + 1
): ):
@ -1167,7 +1189,7 @@ def change_existing_spec(
def remove(self, query_spec, list_name=user_speclist_name, force=False): def remove(self, query_spec, list_name=user_speclist_name, force=False):
"""Remove specs from an environment that match a query_spec""" """Remove specs from an environment that match a query_spec"""
err_msg_header = ( err_msg_header = (
f"cannot remove {query_spec} from '{list_name}' definition " f"Cannot remove '{query_spec}' from '{list_name}' definition "
f"in {self.manifest.manifest_file}" f"in {self.manifest.manifest_file}"
) )
query_spec = Spec(query_spec) query_spec = Spec(query_spec)
@ -1198,11 +1220,10 @@ def remove(self, query_spec, list_name=user_speclist_name, force=False):
list_to_change.remove(spec) list_to_change.remove(spec)
self.update_stale_references(list_name) self.update_stale_references(list_name)
new_specs = set(self.user_specs) new_specs = set(self.user_specs)
except spack.spec_list.SpecListError: except spack.spec_list.SpecListError as e:
# define new specs list # define new specs list
new_specs = set(self.user_specs) new_specs = set(self.user_specs)
msg = f"Spec '{spec}' is part of a spec matrix and " msg = str(e)
msg += f"cannot be removed from list '{list_to_change}'."
if force: if force:
msg += " It will be removed from the concrete specs." msg += " It will be removed from the concrete specs."
# Mock new specs, so we can remove this spec from concrete spec lists # Mock new specs, so we can remove this spec from concrete spec lists
@ -2067,7 +2088,7 @@ def matching_spec(self, spec):
def removed_specs(self): def removed_specs(self):
"""Tuples of (user spec, concrete spec) for all specs that will be """Tuples of (user spec, concrete spec) for all specs that will be
removed on nexg concretize.""" removed on next concretize."""
needed = set() needed = set()
for s, c in self.concretized_specs(): for s, c in self.concretized_specs():
if s in self.user_specs: if s in self.user_specs:
@ -2726,7 +2747,7 @@ def override_user_spec(self, user_spec: str, idx: int) -> None:
self.changed = True self.changed = True
def add_definition(self, user_spec: str, list_name: str) -> None: def add_definition(self, user_spec: str, list_name: str) -> None:
"""Appends a user spec to the first active definition mathing the name passed as argument. """Appends a user spec to the first active definition matching the name passed as argument.
Args: Args:
user_spec: user spec to be appended user_spec: user spec to be appended

View file

@ -62,3 +62,25 @@ def _deprecated_properties(validator, deprecated, instance, schema):
Validator = llnl.util.lang.Singleton(_make_validator) Validator = llnl.util.lang.Singleton(_make_validator)
spec_list_schema = {
"type": "array",
"default": [],
"items": {
"anyOf": [
{
"type": "object",
"additionalProperties": False,
"properties": {
"matrix": {
"type": "array",
"items": {"type": "array", "items": {"type": "string"}},
},
"exclude": {"type": "array", "items": {"type": "string"}},
},
},
{"type": "string"},
{"type": "null"},
]
},
}

View file

@ -0,0 +1,34 @@
# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
"""Schema for definitions
.. literalinclude:: _spack_root/lib/spack/spack/schema/definitions.py
:lines: 13-
"""
import spack.schema
#: Properties for inclusion in other schemas
properties = {
"definitions": {
"type": "array",
"default": [],
"items": {
"type": "object",
"properties": {"when": {"type": "string"}},
"patternProperties": {r"^(?!when$)\w*": spack.schema.spec_list_schema},
},
}
}
#: Full schema with metadata
schema = {
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "Spack definitions configuration file schema",
"type": "object",
"additionalProperties": False,
"properties": properties,
}

View file

@ -12,34 +12,11 @@
import spack.schema.gitlab_ci # DEPRECATED import spack.schema.gitlab_ci # DEPRECATED
import spack.schema.merged import spack.schema.merged
import spack.schema.packages
import spack.schema.projections import spack.schema.projections
#: Top level key in a manifest file #: Top level key in a manifest file
TOP_LEVEL_KEY = "spack" TOP_LEVEL_KEY = "spack"
spec_list_schema = {
"type": "array",
"default": [],
"items": {
"anyOf": [
{
"type": "object",
"additionalProperties": False,
"properties": {
"matrix": {
"type": "array",
"items": {"type": "array", "items": {"type": "string"}},
},
"exclude": {"type": "array", "items": {"type": "string"}},
},
},
{"type": "string"},
{"type": "null"},
]
},
}
projections_scheme = spack.schema.projections.properties["projections"] projections_scheme = spack.schema.projections.properties["projections"]
schema = { schema = {
@ -75,16 +52,7 @@
} }
}, },
}, },
"definitions": { "specs": spack.schema.spec_list_schema,
"type": "array",
"default": [],
"items": {
"type": "object",
"properties": {"when": {"type": "string"}},
"patternProperties": {r"^(?!when$)\w*": spec_list_schema},
},
},
"specs": spec_list_schema,
"view": { "view": {
"anyOf": [ "anyOf": [
{"type": "boolean"}, {"type": "boolean"},

View file

@ -17,6 +17,7 @@
import spack.schema.concretizer import spack.schema.concretizer
import spack.schema.config import spack.schema.config
import spack.schema.container import spack.schema.container
import spack.schema.definitions
import spack.schema.mirrors import spack.schema.mirrors
import spack.schema.modules import spack.schema.modules
import spack.schema.packages import spack.schema.packages
@ -32,6 +33,7 @@
spack.schema.config.properties, spack.schema.config.properties,
spack.schema.container.properties, spack.schema.container.properties,
spack.schema.ci.properties, spack.schema.ci.properties,
spack.schema.definitions.properties,
spack.schema.mirrors.properties, spack.schema.mirrors.properties,
spack.schema.modules.properties, spack.schema.modules.properties,
spack.schema.packages.properties, spack.schema.packages.properties,

View file

@ -93,8 +93,8 @@ def remove(self, spec):
if (isinstance(s, str) and not s.startswith("$")) and Spec(s) == Spec(spec) if (isinstance(s, str) and not s.startswith("$")) and Spec(s) == Spec(spec)
] ]
if not remove: if not remove:
msg = "Cannot remove %s from SpecList %s\n" % (spec, self.name) msg = f"Cannot remove {spec} from SpecList {self.name}.\n"
msg += "Either %s is not in %s or %s is " % (spec, self.name, spec) msg += f"Either {spec} is not in {self.name} or {spec} is "
msg += "expanded from a matrix and cannot be removed directly." msg += "expanded from a matrix and cannot be removed directly."
raise SpecListError(msg) raise SpecListError(msg)
@ -133,9 +133,8 @@ def _parse_reference(self, name):
# Make sure the reference is valid # Make sure the reference is valid
if name not in self._reference: if name not in self._reference:
msg = "SpecList %s refers to " % self.name msg = f"SpecList '{self.name}' refers to named list '{name}'"
msg += "named list %s " % name msg += " which does not appear in its reference dict."
msg += "which does not appear in its reference dict"
raise UndefinedReferenceError(msg) raise UndefinedReferenceError(msg)
return (name, sigil) return (name, sigil)

View file

@ -632,7 +632,7 @@ def test_env_view_external_prefix(tmp_path, mutable_database, mock_packages):
manifest_dir.mkdir(parents=True, exist_ok=False) manifest_dir.mkdir(parents=True, exist_ok=False)
manifest_file = manifest_dir / ev.manifest_name manifest_file = manifest_dir / ev.manifest_name
manifest_file.write_text( manifest_file.write_text(
""" """\
spack: spack:
specs: specs:
- a - a
@ -720,6 +720,7 @@ def test_env_with_config(environment_from_manifest):
def test_with_config_bad_include(environment_from_manifest): def test_with_config_bad_include(environment_from_manifest):
"""Confirm missing include paths raise expected exception and error.""" """Confirm missing include paths raise expected exception and error."""
with pytest.raises(spack.config.ConfigFileError, match="2 missing include path"):
e = environment_from_manifest( e = environment_from_manifest(
""" """
spack: spack:
@ -728,30 +729,16 @@ def test_with_config_bad_include(environment_from_manifest):
- no/such/file.yaml - no/such/file.yaml
""" """
) )
with pytest.raises(spack.config.ConfigFileError, match="2 missing include path"):
with e: with e:
e.concretize() e.concretize()
assert ev.active_environment() is None assert ev.active_environment() is None
def test_env_with_include_config_files_same_basename(environment_from_manifest): def test_env_with_include_config_files_same_basename(tmp_path, environment_from_manifest):
e = environment_from_manifest( file1 = fs.join_path(tmp_path, "path", "to", "included-config.yaml")
""" fs.mkdirp(os.path.dirname(file1))
spack: with open(file1, "w") as f:
include:
- ./path/to/included-config.yaml
- ./second/path/to/include-config.yaml
specs:
- libelf
- mpileaks
"""
)
e = ev.read("test")
fs.mkdirp(os.path.join(e.path, "path", "to"))
with open(os.path.join(e.path, "./path/to/included-config.yaml"), "w") as f:
f.write( f.write(
"""\ """\
packages: packages:
@ -760,8 +747,9 @@ def test_env_with_include_config_files_same_basename(environment_from_manifest):
""" """
) )
fs.mkdirp(os.path.join(e.path, "second", "path", "to")) file2 = fs.join_path(tmp_path, "second", "path", "included-config.yaml")
with open(os.path.join(e.path, "./second/path/to/include-config.yaml"), "w") as f: fs.mkdirp(os.path.dirname(file2))
with open(file2, "w") as f:
f.write( f.write(
"""\ """\
packages: packages:
@ -770,6 +758,18 @@ def test_env_with_include_config_files_same_basename(environment_from_manifest):
""" """
) )
e = environment_from_manifest(
f"""
spack:
include:
- {file1}
- {file2}
specs:
- libelf
- mpileaks
"""
)
with e: with e:
e.concretize() e.concretize()
@ -806,12 +806,18 @@ def mpileaks_env_config(include_path):
) )
def test_env_with_included_config_file(environment_from_manifest, packages_file): def test_env_with_included_config_file(mutable_mock_env_path, packages_file):
"""Test inclusion of a relative packages configuration file added to an """Test inclusion of a relative packages configuration file added to an
existing environment. existing environment.
""" """
env_root = mutable_mock_env_path
fs.mkdirp(env_root)
include_filename = "included-config.yaml" include_filename = "included-config.yaml"
e = environment_from_manifest( included_path = env_root / include_filename
shutil.move(packages_file.strpath, included_path)
spack_yaml = env_root / ev.manifest_name
spack_yaml.write_text(
f"""\ f"""\
spack: spack:
include: include:
@ -821,9 +827,7 @@ def test_env_with_included_config_file(environment_from_manifest, packages_file)
""" """
) )
included_path = os.path.join(e.path, include_filename) e = ev.Environment(env_root)
shutil.move(packages_file.strpath, included_path)
with e: with e:
e.concretize() e.concretize()
@ -856,68 +860,67 @@ def test_env_with_included_config_missing_file(tmpdir, mutable_empty_config):
with spack_yaml.open("w") as f: with spack_yaml.open("w") as f:
f.write("spack:\n include:\n - {0}\n".format(missing_file.strpath)) f.write("spack:\n include:\n - {0}\n".format(missing_file.strpath))
env = ev.Environment(tmpdir.strpath)
with pytest.raises(spack.config.ConfigError, match="missing include path"): with pytest.raises(spack.config.ConfigError, match="missing include path"):
ev.activate(env) ev.Environment(tmpdir.strpath)
def test_env_with_included_config_scope(environment_from_manifest, packages_file): def test_env_with_included_config_scope(mutable_mock_env_path, packages_file):
"""Test inclusion of a package file from the environment's configuration """Test inclusion of a package file from the environment's configuration
stage directory. This test is intended to represent a case where a remote stage directory. This test is intended to represent a case where a remote
file has already been staged.""" file has already been staged."""
config_scope_path = os.path.join(ev.root("test"), "config") env_root = mutable_mock_env_path
config_scope_path = env_root / "config"
# Configure the environment to include file(s) from the environment's
# remote configuration stage directory.
e = environment_from_manifest(mpileaks_env_config(config_scope_path))
# Copy the packages.yaml file to the environment configuration # Copy the packages.yaml file to the environment configuration
# directory, so it is picked up during concretization. (Using # directory, so it is picked up during concretization. (Using
# copy instead of rename in case the fixture scope changes.) # copy instead of rename in case the fixture scope changes.)
fs.mkdirp(config_scope_path) fs.mkdirp(config_scope_path)
include_filename = os.path.basename(packages_file.strpath) include_filename = os.path.basename(packages_file.strpath)
included_path = os.path.join(config_scope_path, include_filename) included_path = config_scope_path / include_filename
fs.copy(packages_file.strpath, included_path) fs.copy(packages_file.strpath, included_path)
# Configure the environment to include file(s) from the environment's
# remote configuration stage directory.
spack_yaml = env_root / ev.manifest_name
spack_yaml.write_text(mpileaks_env_config(config_scope_path))
# Ensure the concretized environment reflects contents of the # Ensure the concretized environment reflects contents of the
# packages.yaml file. # packages.yaml file.
e = ev.Environment(env_root)
with e: with e:
e.concretize() e.concretize()
assert any(x.satisfies("mpileaks@2.2") for x in e._get_environment_specs()) assert any(x.satisfies("mpileaks@2.2") for x in e._get_environment_specs())
def test_env_with_included_config_var_path(environment_from_manifest, packages_file): def test_env_with_included_config_var_path(tmpdir, packages_file):
"""Test inclusion of a package configuration file with path variables """Test inclusion of a package configuration file with path variables
"staged" in the environment's configuration stage directory.""" "staged" in the environment's configuration stage directory."""
config_var_path = os.path.join("$tempdir", "included-config.yaml") included_file = packages_file.strpath
e = environment_from_manifest(mpileaks_env_config(config_var_path)) env_path = pathlib.PosixPath(tmpdir)
config_var_path = os.path.join("$tempdir", "included-packages.yaml")
spack_yaml = env_path / ev.manifest_name
spack_yaml.write_text(mpileaks_env_config(config_var_path))
config_real_path = substitute_path_variables(config_var_path) config_real_path = substitute_path_variables(config_var_path)
fs.mkdirp(os.path.dirname(config_real_path)) shutil.move(included_file, config_real_path)
shutil.move(packages_file.strpath, config_real_path)
assert os.path.exists(config_real_path) assert os.path.exists(config_real_path)
e = ev.Environment(env_path)
with e: with e:
e.concretize() e.concretize()
assert any(x.satisfies("mpileaks@2.2") for x in e._get_environment_specs()) assert any(x.satisfies("mpileaks@2.2") for x in e._get_environment_specs())
def test_env_config_precedence(environment_from_manifest): def test_env_with_included_config_precedence(tmp_path):
e = environment_from_manifest( """Test included scope and manifest precedence when including a package
""" configuration file."""
spack:
packages: included_file = "included-packages.yaml"
libelf: included_path = tmp_path / included_file
version: ["0.8.12"] with open(included_path, "w") as f:
include:
- ./included-config.yaml
specs:
- mpileaks
"""
)
with open(os.path.join(e.path, "included-config.yaml"), "w") as f:
f.write( f.write(
"""\ """\
packages: packages:
@ -928,29 +931,50 @@ def test_env_config_precedence(environment_from_manifest):
""" """
) )
with e: spack_yaml = tmp_path / ev.manifest_name
e.concretize() spack_yaml.write_text(
f"""\
# ensure included scope took effect
assert any(x.satisfies("mpileaks@2.2") for x in e._get_environment_specs())
# ensure env file takes precedence
assert any(x.satisfies("libelf@0.8.12") for x in e._get_environment_specs())
def test_included_config_precedence(environment_from_manifest):
e = environment_from_manifest(
"""
spack: spack:
packages:
libelf:
version: ["0.8.12"]
include: include:
- ./high-config.yaml # this one should take precedence - {os.path.join(".", included_file)}
- ./low-config.yaml
specs: specs:
- mpileaks - mpileaks
""" """
) )
with open(os.path.join(e.path, "high-config.yaml"), "w") as f: e = ev.Environment(tmp_path)
with e:
e.concretize()
specs = e._get_environment_specs()
# ensure included scope took effect
assert any(x.satisfies("mpileaks@2.2") for x in specs)
# ensure env file takes precedence
assert any(x.satisfies("libelf@0.8.12") for x in specs)
def test_env_with_included_configs_precedence(tmp_path):
"""Test precendence of multiple included configuration files."""
file1 = "high-config.yaml"
file2 = "low-config.yaml"
spack_yaml = tmp_path / ev.manifest_name
spack_yaml.write_text(
f"""\
spack:
include:
- {os.path.join(".", file1)} # this one should take precedence
- {os.path.join(".", file2)}
specs:
- mpileaks
"""
)
with open(tmp_path / file1, "w") as f:
f.write( f.write(
"""\ """\
packages: packages:
@ -959,7 +983,7 @@ def test_included_config_precedence(environment_from_manifest):
""" """
) )
with open(os.path.join(e.path, "low-config.yaml"), "w") as f: with open(tmp_path / file2, "w") as f:
f.write( f.write(
"""\ """\
packages: packages:
@ -970,12 +994,16 @@ def test_included_config_precedence(environment_from_manifest):
""" """
) )
e = ev.Environment(tmp_path)
with e: with e:
e.concretize() e.concretize()
specs = e._get_environment_specs()
assert any(x.satisfies("mpileaks@2.2") for x in e._get_environment_specs()) # ensure included package spec took precedence over manifest spec
assert any(x.satisfies("mpileaks@2.2") for x in specs)
assert any([x.satisfies("libelf@0.8.10") for x in e._get_environment_specs()]) # ensure first included package spec took precedence over one from second
assert any(x.satisfies("libelf@0.8.10") for x in specs)
def test_bad_env_yaml_format(environment_from_manifest): def test_bad_env_yaml_format(environment_from_manifest):
@ -1578,10 +1606,9 @@ def test_stack_yaml_remove_from_list(tmpdir):
assert Spec("callpath") in test.user_specs assert Spec("callpath") in test.user_specs
def test_stack_yaml_remove_from_list_force(tmpdir): def test_stack_yaml_remove_from_list_force(tmp_path):
filename = str(tmpdir.join("spack.yaml")) spack_yaml = tmp_path / ev.manifest_name
with open(filename, "w") as f: spack_yaml.write_text(
f.write(
"""\ """\
spack: spack:
definitions: definitions:
@ -1592,8 +1619,8 @@ def test_stack_yaml_remove_from_list_force(tmpdir):
- [^mpich, ^zmpi] - [^mpich, ^zmpi]
""" """
) )
with tmpdir.as_cwd():
env("create", "test", "./spack.yaml") env("create", "test", str(spack_yaml))
with ev.read("test"): with ev.read("test"):
concretize() concretize()
remove("-f", "-l", "packages", "mpileaks") remove("-f", "-l", "packages", "mpileaks")
@ -1650,7 +1677,7 @@ def test_stack_yaml_force_remove_from_matrix(tmpdir):
with tmpdir.as_cwd(): with tmpdir.as_cwd():
env("create", "test", "./spack.yaml") env("create", "test", "./spack.yaml")
with ev.read("test") as e: with ev.read("test") as e:
concretize() e.concretize()
before_user = e.user_specs.specs before_user = e.user_specs.specs
before_conc = e.concretized_user_specs before_conc = e.concretized_user_specs

View file

@ -18,6 +18,7 @@
SpackEnvironmentViewError, SpackEnvironmentViewError,
_error_on_nonempty_view_dir, _error_on_nonempty_view_dir,
) )
from spack.spec_list import UndefinedReferenceError
pytestmark = pytest.mark.not_on_windows("Envs are not supported on windows") pytestmark = pytest.mark.not_on_windows("Envs are not supported on windows")
@ -716,3 +717,64 @@ def test_variant_propagation_with_unify_false(tmp_path, mock_packages):
root = env.matching_spec("parent-foo") root = env.matching_spec("parent-foo")
for node in root.traverse(): for node in root.traverse():
assert node.satisfies("+foo") assert node.satisfies("+foo")
def test_env_with_include_defs(mutable_mock_env_path, mock_packages):
"""Test environment with included definitions file."""
env_path = mutable_mock_env_path
env_path.mkdir()
defs_file = env_path / "definitions.yaml"
defs_file.write_text(
"""definitions:
- core_specs: [libdwarf, libelf]
- compilers: ['%gcc']
"""
)
spack_yaml = env_path / ev.manifest_name
spack_yaml.write_text(
f"""spack:
include:
- file://{defs_file}
definitions:
- my_packages: [zlib]
specs:
- matrix:
- [$core_specs]
- [$compilers]
- $my_packages
"""
)
e = ev.Environment(env_path)
with e:
e.concretize()
def test_env_with_include_def_missing(mutable_mock_env_path, mock_packages):
"""Test environment with included definitions file that is missing a definition."""
env_path = mutable_mock_env_path
env_path.mkdir()
filename = "missing-def.yaml"
defs_file = env_path / filename
defs_file.write_text("definitions:\n- my_compilers: ['%gcc']\n")
spack_yaml = env_path / ev.manifest_name
spack_yaml.write_text(
f"""spack:
include:
- file://{defs_file}
specs:
- matrix:
- [$core_specs]
- [$my_compilers]
"""
)
e = ev.Environment(env_path)
with e:
with pytest.raises(UndefinedReferenceError, match=r"which does not appear"):
e.concretize()

View file

@ -80,7 +80,17 @@ def test_module_suffixes(module_suffixes_schema):
@pytest.mark.regression("10246") @pytest.mark.regression("10246")
@pytest.mark.parametrize( @pytest.mark.parametrize(
"config_name", "config_name",
["compilers", "config", "env", "merged", "mirrors", "modules", "packages", "repos"], [
"compilers",
"config",
"definitions",
"env",
"merged",
"mirrors",
"modules",
"packages",
"repos",
],
) )
def test_schema_validation(meta_schema, config_name): def test_schema_validation(meta_schema, config_name):
import importlib import importlib

View file

@ -1159,19 +1159,19 @@ complete -c spack -n '__fish_spack_using_command config' -l scope -r -d 'configu
# spack config get # spack config get
set -g __fish_spack_optspecs_spack_config_get h/help set -g __fish_spack_optspecs_spack_config_get h/help
complete -c spack -n '__fish_spack_using_command_pos 0 config get' -f -a 'bootstrap cdash ci compilers concretizer config mirrors modules packages repos upstreams' complete -c spack -n '__fish_spack_using_command_pos 0 config get' -f -a 'bootstrap cdash ci compilers concretizer config definitions mirrors modules packages repos upstreams'
complete -c spack -n '__fish_spack_using_command config get' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command config get' -s h -l help -f -a help
complete -c spack -n '__fish_spack_using_command config get' -s h -l help -d 'show this help message and exit' complete -c spack -n '__fish_spack_using_command config get' -s h -l help -d 'show this help message and exit'
# spack config blame # spack config blame
set -g __fish_spack_optspecs_spack_config_blame h/help set -g __fish_spack_optspecs_spack_config_blame h/help
complete -c spack -n '__fish_spack_using_command_pos 0 config blame' -f -a 'bootstrap cdash ci compilers concretizer config mirrors modules packages repos upstreams' complete -c spack -n '__fish_spack_using_command_pos 0 config blame' -f -a 'bootstrap cdash ci compilers concretizer config definitions mirrors modules packages repos upstreams'
complete -c spack -n '__fish_spack_using_command config blame' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command config blame' -s h -l help -f -a help
complete -c spack -n '__fish_spack_using_command config blame' -s h -l help -d 'show this help message and exit' complete -c spack -n '__fish_spack_using_command config blame' -s h -l help -d 'show this help message and exit'
# spack config edit # spack config edit
set -g __fish_spack_optspecs_spack_config_edit h/help print-file set -g __fish_spack_optspecs_spack_config_edit h/help print-file
complete -c spack -n '__fish_spack_using_command_pos 0 config edit' -f -a 'bootstrap cdash ci compilers concretizer config mirrors modules packages repos upstreams' complete -c spack -n '__fish_spack_using_command_pos 0 config edit' -f -a 'bootstrap cdash ci compilers concretizer config definitions mirrors modules packages repos upstreams'
complete -c spack -n '__fish_spack_using_command config edit' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command config edit' -s h -l help -f -a help
complete -c spack -n '__fish_spack_using_command config edit' -s h -l help -d 'show this help message and exit' complete -c spack -n '__fish_spack_using_command config edit' -s h -l help -d 'show this help message and exit'
complete -c spack -n '__fish_spack_using_command config edit' -l print-file -f -a print_file complete -c spack -n '__fish_spack_using_command config edit' -l print-file -f -a print_file