Environments: Add support for including definitions files (#33960)
This PR adds support for including separate definitions from `spack.yaml`. Supporting the inclusion of files with definitions enables user to make curated/standardized collections of packages that can re-used by others.
This commit is contained in:
parent
5a67c578b7
commit
c9dfb9b0fd
11 changed files with 307 additions and 160 deletions
|
@ -69,6 +69,7 @@
|
||||||
SECTION_SCHEMAS = {
|
SECTION_SCHEMAS = {
|
||||||
"compilers": spack.schema.compilers.schema,
|
"compilers": spack.schema.compilers.schema,
|
||||||
"concretizer": spack.schema.concretizer.schema,
|
"concretizer": spack.schema.concretizer.schema,
|
||||||
|
"definitions": spack.schema.definitions.schema,
|
||||||
"mirrors": spack.schema.mirrors.schema,
|
"mirrors": spack.schema.mirrors.schema,
|
||||||
"repos": spack.schema.repos.schema,
|
"repos": spack.schema.repos.schema,
|
||||||
"packages": spack.schema.packages.schema,
|
"packages": spack.schema.packages.schema,
|
||||||
|
@ -994,6 +995,7 @@ def read_config_file(filename, schema=None):
|
||||||
key = next(iter(data))
|
key = next(iter(data))
|
||||||
schema = _ALL_SCHEMAS[key]
|
schema = _ALL_SCHEMAS[key]
|
||||||
validate(data, schema)
|
validate(data, schema)
|
||||||
|
|
||||||
return data
|
return data
|
||||||
|
|
||||||
except StopIteration:
|
except StopIteration:
|
||||||
|
|
|
@ -781,10 +781,18 @@ def _re_read(self):
|
||||||
"""Reinitialize the environment object."""
|
"""Reinitialize the environment object."""
|
||||||
self.clear(re_read=True)
|
self.clear(re_read=True)
|
||||||
self.manifest = EnvironmentManifestFile(self.path)
|
self.manifest = EnvironmentManifestFile(self.path)
|
||||||
self._read()
|
self._read(re_read=True)
|
||||||
|
|
||||||
def _read(self):
|
def _read(self, re_read=False):
|
||||||
self._construct_state_from_manifest()
|
# If the manifest has included files, then some of the information
|
||||||
|
# (e.g., definitions) MAY be in those files. So we need to ensure
|
||||||
|
# the config is populated with any associated spec lists in order
|
||||||
|
# to fully construct the manifest state.
|
||||||
|
includes = self.manifest[TOP_LEVEL_KEY].get("include", [])
|
||||||
|
if includes and not re_read:
|
||||||
|
prepare_config_scope(self)
|
||||||
|
|
||||||
|
self._construct_state_from_manifest(re_read)
|
||||||
|
|
||||||
if os.path.exists(self.lock_path):
|
if os.path.exists(self.lock_path):
|
||||||
with open(self.lock_path) as f:
|
with open(self.lock_path) as f:
|
||||||
|
@ -798,11 +806,8 @@ def write_transaction(self):
|
||||||
"""Get a write lock context manager for use in a `with` block."""
|
"""Get a write lock context manager for use in a `with` block."""
|
||||||
return lk.WriteTransaction(self.txlock, acquire=self._re_read)
|
return lk.WriteTransaction(self.txlock, acquire=self._re_read)
|
||||||
|
|
||||||
def _construct_state_from_manifest(self):
|
def _process_definition(self, item):
|
||||||
"""Read manifest file and set up user specs."""
|
"""Process a single spec definition item."""
|
||||||
self.spec_lists = collections.OrderedDict()
|
|
||||||
env_configuration = self.manifest[TOP_LEVEL_KEY]
|
|
||||||
for item in env_configuration.get("definitions", []):
|
|
||||||
entry = copy.deepcopy(item)
|
entry = copy.deepcopy(item)
|
||||||
when = _eval_conditional(entry.pop("when", "True"))
|
when = _eval_conditional(entry.pop("when", "True"))
|
||||||
assert len(entry) == 1
|
assert len(entry) == 1
|
||||||
|
@ -814,6 +819,18 @@ def _construct_state_from_manifest(self):
|
||||||
else:
|
else:
|
||||||
self.spec_lists[name] = user_specs
|
self.spec_lists[name] = user_specs
|
||||||
|
|
||||||
|
def _construct_state_from_manifest(self, re_read=False):
|
||||||
|
"""Read manifest file and set up user specs."""
|
||||||
|
self.spec_lists = collections.OrderedDict()
|
||||||
|
|
||||||
|
if not re_read:
|
||||||
|
for item in spack.config.get("definitions", []):
|
||||||
|
self._process_definition(item)
|
||||||
|
|
||||||
|
env_configuration = self.manifest[TOP_LEVEL_KEY]
|
||||||
|
for item in env_configuration.get("definitions", []):
|
||||||
|
self._process_definition(item)
|
||||||
|
|
||||||
spec_list = env_configuration.get(user_speclist_name, [])
|
spec_list = env_configuration.get(user_speclist_name, [])
|
||||||
user_specs = SpecList(
|
user_specs = SpecList(
|
||||||
user_speclist_name, [s for s in spec_list if s], self.spec_lists.copy()
|
user_speclist_name, [s for s in spec_list if s], self.spec_lists.copy()
|
||||||
|
@ -857,7 +874,9 @@ def clear(self, re_read=False):
|
||||||
yaml, and need to be maintained when re-reading an existing
|
yaml, and need to be maintained when re-reading an existing
|
||||||
environment.
|
environment.
|
||||||
"""
|
"""
|
||||||
self.spec_lists = {user_speclist_name: SpecList()} # specs from yaml
|
self.spec_lists = collections.OrderedDict()
|
||||||
|
self.spec_lists[user_speclist_name] = SpecList()
|
||||||
|
|
||||||
self.dev_specs = {} # dev-build specs from yaml
|
self.dev_specs = {} # dev-build specs from yaml
|
||||||
self.concretized_user_specs = [] # user specs from last concretize
|
self.concretized_user_specs = [] # user specs from last concretize
|
||||||
self.concretized_order = [] # roots of last concretize, in order
|
self.concretized_order = [] # roots of last concretize, in order
|
||||||
|
@ -1006,7 +1025,8 @@ def included_config_scopes(self):
|
||||||
|
|
||||||
elif include_url.scheme:
|
elif include_url.scheme:
|
||||||
raise ValueError(
|
raise ValueError(
|
||||||
"Unsupported URL scheme for environment include: {}".format(config_path)
|
f"Unsupported URL scheme ({include_url.scheme}) for "
|
||||||
|
f"environment include: {config_path}"
|
||||||
)
|
)
|
||||||
|
|
||||||
# treat relative paths as relative to the environment
|
# treat relative paths as relative to the environment
|
||||||
|
@ -1068,8 +1088,10 @@ def update_stale_references(self, from_list=None):
|
||||||
from_list = next(iter(self.spec_lists.keys()))
|
from_list = next(iter(self.spec_lists.keys()))
|
||||||
index = list(self.spec_lists.keys()).index(from_list)
|
index = list(self.spec_lists.keys()).index(from_list)
|
||||||
|
|
||||||
# spec_lists is an OrderedDict, all list entries after the modified
|
# spec_lists is an OrderedDict to ensure lists read from the manifest
|
||||||
# list may refer to the modified list. Update stale references
|
# are maintainted in order, hence, all list entries after the modified
|
||||||
|
# list may refer to the modified list requiring stale references to be
|
||||||
|
# updated.
|
||||||
for i, (name, speclist) in enumerate(
|
for i, (name, speclist) in enumerate(
|
||||||
list(self.spec_lists.items())[index + 1 :], index + 1
|
list(self.spec_lists.items())[index + 1 :], index + 1
|
||||||
):
|
):
|
||||||
|
@ -1167,7 +1189,7 @@ def change_existing_spec(
|
||||||
def remove(self, query_spec, list_name=user_speclist_name, force=False):
|
def remove(self, query_spec, list_name=user_speclist_name, force=False):
|
||||||
"""Remove specs from an environment that match a query_spec"""
|
"""Remove specs from an environment that match a query_spec"""
|
||||||
err_msg_header = (
|
err_msg_header = (
|
||||||
f"cannot remove {query_spec} from '{list_name}' definition "
|
f"Cannot remove '{query_spec}' from '{list_name}' definition "
|
||||||
f"in {self.manifest.manifest_file}"
|
f"in {self.manifest.manifest_file}"
|
||||||
)
|
)
|
||||||
query_spec = Spec(query_spec)
|
query_spec = Spec(query_spec)
|
||||||
|
@ -1198,11 +1220,10 @@ def remove(self, query_spec, list_name=user_speclist_name, force=False):
|
||||||
list_to_change.remove(spec)
|
list_to_change.remove(spec)
|
||||||
self.update_stale_references(list_name)
|
self.update_stale_references(list_name)
|
||||||
new_specs = set(self.user_specs)
|
new_specs = set(self.user_specs)
|
||||||
except spack.spec_list.SpecListError:
|
except spack.spec_list.SpecListError as e:
|
||||||
# define new specs list
|
# define new specs list
|
||||||
new_specs = set(self.user_specs)
|
new_specs = set(self.user_specs)
|
||||||
msg = f"Spec '{spec}' is part of a spec matrix and "
|
msg = str(e)
|
||||||
msg += f"cannot be removed from list '{list_to_change}'."
|
|
||||||
if force:
|
if force:
|
||||||
msg += " It will be removed from the concrete specs."
|
msg += " It will be removed from the concrete specs."
|
||||||
# Mock new specs, so we can remove this spec from concrete spec lists
|
# Mock new specs, so we can remove this spec from concrete spec lists
|
||||||
|
@ -2067,7 +2088,7 @@ def matching_spec(self, spec):
|
||||||
|
|
||||||
def removed_specs(self):
|
def removed_specs(self):
|
||||||
"""Tuples of (user spec, concrete spec) for all specs that will be
|
"""Tuples of (user spec, concrete spec) for all specs that will be
|
||||||
removed on nexg concretize."""
|
removed on next concretize."""
|
||||||
needed = set()
|
needed = set()
|
||||||
for s, c in self.concretized_specs():
|
for s, c in self.concretized_specs():
|
||||||
if s in self.user_specs:
|
if s in self.user_specs:
|
||||||
|
@ -2726,7 +2747,7 @@ def override_user_spec(self, user_spec: str, idx: int) -> None:
|
||||||
self.changed = True
|
self.changed = True
|
||||||
|
|
||||||
def add_definition(self, user_spec: str, list_name: str) -> None:
|
def add_definition(self, user_spec: str, list_name: str) -> None:
|
||||||
"""Appends a user spec to the first active definition mathing the name passed as argument.
|
"""Appends a user spec to the first active definition matching the name passed as argument.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
user_spec: user spec to be appended
|
user_spec: user spec to be appended
|
||||||
|
|
|
@ -62,3 +62,25 @@ def _deprecated_properties(validator, deprecated, instance, schema):
|
||||||
|
|
||||||
|
|
||||||
Validator = llnl.util.lang.Singleton(_make_validator)
|
Validator = llnl.util.lang.Singleton(_make_validator)
|
||||||
|
|
||||||
|
spec_list_schema = {
|
||||||
|
"type": "array",
|
||||||
|
"default": [],
|
||||||
|
"items": {
|
||||||
|
"anyOf": [
|
||||||
|
{
|
||||||
|
"type": "object",
|
||||||
|
"additionalProperties": False,
|
||||||
|
"properties": {
|
||||||
|
"matrix": {
|
||||||
|
"type": "array",
|
||||||
|
"items": {"type": "array", "items": {"type": "string"}},
|
||||||
|
},
|
||||||
|
"exclude": {"type": "array", "items": {"type": "string"}},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{"type": "string"},
|
||||||
|
{"type": "null"},
|
||||||
|
]
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
34
lib/spack/spack/schema/definitions.py
Normal file
34
lib/spack/spack/schema/definitions.py
Normal file
|
@ -0,0 +1,34 @@
|
||||||
|
# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
|
||||||
|
# Spack Project Developers. See the top-level COPYRIGHT file for details.
|
||||||
|
#
|
||||||
|
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
|
||||||
|
|
||||||
|
"""Schema for definitions
|
||||||
|
|
||||||
|
.. literalinclude:: _spack_root/lib/spack/spack/schema/definitions.py
|
||||||
|
:lines: 13-
|
||||||
|
"""
|
||||||
|
|
||||||
|
import spack.schema
|
||||||
|
|
||||||
|
#: Properties for inclusion in other schemas
|
||||||
|
properties = {
|
||||||
|
"definitions": {
|
||||||
|
"type": "array",
|
||||||
|
"default": [],
|
||||||
|
"items": {
|
||||||
|
"type": "object",
|
||||||
|
"properties": {"when": {"type": "string"}},
|
||||||
|
"patternProperties": {r"^(?!when$)\w*": spack.schema.spec_list_schema},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#: Full schema with metadata
|
||||||
|
schema = {
|
||||||
|
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||||
|
"title": "Spack definitions configuration file schema",
|
||||||
|
"type": "object",
|
||||||
|
"additionalProperties": False,
|
||||||
|
"properties": properties,
|
||||||
|
}
|
|
@ -12,34 +12,11 @@
|
||||||
|
|
||||||
import spack.schema.gitlab_ci # DEPRECATED
|
import spack.schema.gitlab_ci # DEPRECATED
|
||||||
import spack.schema.merged
|
import spack.schema.merged
|
||||||
import spack.schema.packages
|
|
||||||
import spack.schema.projections
|
import spack.schema.projections
|
||||||
|
|
||||||
#: Top level key in a manifest file
|
#: Top level key in a manifest file
|
||||||
TOP_LEVEL_KEY = "spack"
|
TOP_LEVEL_KEY = "spack"
|
||||||
|
|
||||||
spec_list_schema = {
|
|
||||||
"type": "array",
|
|
||||||
"default": [],
|
|
||||||
"items": {
|
|
||||||
"anyOf": [
|
|
||||||
{
|
|
||||||
"type": "object",
|
|
||||||
"additionalProperties": False,
|
|
||||||
"properties": {
|
|
||||||
"matrix": {
|
|
||||||
"type": "array",
|
|
||||||
"items": {"type": "array", "items": {"type": "string"}},
|
|
||||||
},
|
|
||||||
"exclude": {"type": "array", "items": {"type": "string"}},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{"type": "string"},
|
|
||||||
{"type": "null"},
|
|
||||||
]
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
projections_scheme = spack.schema.projections.properties["projections"]
|
projections_scheme = spack.schema.projections.properties["projections"]
|
||||||
|
|
||||||
schema = {
|
schema = {
|
||||||
|
@ -75,16 +52,7 @@
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
"definitions": {
|
"specs": spack.schema.spec_list_schema,
|
||||||
"type": "array",
|
|
||||||
"default": [],
|
|
||||||
"items": {
|
|
||||||
"type": "object",
|
|
||||||
"properties": {"when": {"type": "string"}},
|
|
||||||
"patternProperties": {r"^(?!when$)\w*": spec_list_schema},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
"specs": spec_list_schema,
|
|
||||||
"view": {
|
"view": {
|
||||||
"anyOf": [
|
"anyOf": [
|
||||||
{"type": "boolean"},
|
{"type": "boolean"},
|
||||||
|
|
|
@ -17,6 +17,7 @@
|
||||||
import spack.schema.concretizer
|
import spack.schema.concretizer
|
||||||
import spack.schema.config
|
import spack.schema.config
|
||||||
import spack.schema.container
|
import spack.schema.container
|
||||||
|
import spack.schema.definitions
|
||||||
import spack.schema.mirrors
|
import spack.schema.mirrors
|
||||||
import spack.schema.modules
|
import spack.schema.modules
|
||||||
import spack.schema.packages
|
import spack.schema.packages
|
||||||
|
@ -32,6 +33,7 @@
|
||||||
spack.schema.config.properties,
|
spack.schema.config.properties,
|
||||||
spack.schema.container.properties,
|
spack.schema.container.properties,
|
||||||
spack.schema.ci.properties,
|
spack.schema.ci.properties,
|
||||||
|
spack.schema.definitions.properties,
|
||||||
spack.schema.mirrors.properties,
|
spack.schema.mirrors.properties,
|
||||||
spack.schema.modules.properties,
|
spack.schema.modules.properties,
|
||||||
spack.schema.packages.properties,
|
spack.schema.packages.properties,
|
||||||
|
|
|
@ -93,8 +93,8 @@ def remove(self, spec):
|
||||||
if (isinstance(s, str) and not s.startswith("$")) and Spec(s) == Spec(spec)
|
if (isinstance(s, str) and not s.startswith("$")) and Spec(s) == Spec(spec)
|
||||||
]
|
]
|
||||||
if not remove:
|
if not remove:
|
||||||
msg = "Cannot remove %s from SpecList %s\n" % (spec, self.name)
|
msg = f"Cannot remove {spec} from SpecList {self.name}.\n"
|
||||||
msg += "Either %s is not in %s or %s is " % (spec, self.name, spec)
|
msg += f"Either {spec} is not in {self.name} or {spec} is "
|
||||||
msg += "expanded from a matrix and cannot be removed directly."
|
msg += "expanded from a matrix and cannot be removed directly."
|
||||||
raise SpecListError(msg)
|
raise SpecListError(msg)
|
||||||
|
|
||||||
|
@ -133,9 +133,8 @@ def _parse_reference(self, name):
|
||||||
|
|
||||||
# Make sure the reference is valid
|
# Make sure the reference is valid
|
||||||
if name not in self._reference:
|
if name not in self._reference:
|
||||||
msg = "SpecList %s refers to " % self.name
|
msg = f"SpecList '{self.name}' refers to named list '{name}'"
|
||||||
msg += "named list %s " % name
|
msg += " which does not appear in its reference dict."
|
||||||
msg += "which does not appear in its reference dict"
|
|
||||||
raise UndefinedReferenceError(msg)
|
raise UndefinedReferenceError(msg)
|
||||||
|
|
||||||
return (name, sigil)
|
return (name, sigil)
|
||||||
|
|
|
@ -632,7 +632,7 @@ def test_env_view_external_prefix(tmp_path, mutable_database, mock_packages):
|
||||||
manifest_dir.mkdir(parents=True, exist_ok=False)
|
manifest_dir.mkdir(parents=True, exist_ok=False)
|
||||||
manifest_file = manifest_dir / ev.manifest_name
|
manifest_file = manifest_dir / ev.manifest_name
|
||||||
manifest_file.write_text(
|
manifest_file.write_text(
|
||||||
"""
|
"""\
|
||||||
spack:
|
spack:
|
||||||
specs:
|
specs:
|
||||||
- a
|
- a
|
||||||
|
@ -720,6 +720,7 @@ def test_env_with_config(environment_from_manifest):
|
||||||
|
|
||||||
def test_with_config_bad_include(environment_from_manifest):
|
def test_with_config_bad_include(environment_from_manifest):
|
||||||
"""Confirm missing include paths raise expected exception and error."""
|
"""Confirm missing include paths raise expected exception and error."""
|
||||||
|
with pytest.raises(spack.config.ConfigFileError, match="2 missing include path"):
|
||||||
e = environment_from_manifest(
|
e = environment_from_manifest(
|
||||||
"""
|
"""
|
||||||
spack:
|
spack:
|
||||||
|
@ -728,30 +729,16 @@ def test_with_config_bad_include(environment_from_manifest):
|
||||||
- no/such/file.yaml
|
- no/such/file.yaml
|
||||||
"""
|
"""
|
||||||
)
|
)
|
||||||
with pytest.raises(spack.config.ConfigFileError, match="2 missing include path"):
|
|
||||||
with e:
|
with e:
|
||||||
e.concretize()
|
e.concretize()
|
||||||
|
|
||||||
assert ev.active_environment() is None
|
assert ev.active_environment() is None
|
||||||
|
|
||||||
|
|
||||||
def test_env_with_include_config_files_same_basename(environment_from_manifest):
|
def test_env_with_include_config_files_same_basename(tmp_path, environment_from_manifest):
|
||||||
e = environment_from_manifest(
|
file1 = fs.join_path(tmp_path, "path", "to", "included-config.yaml")
|
||||||
"""
|
fs.mkdirp(os.path.dirname(file1))
|
||||||
spack:
|
with open(file1, "w") as f:
|
||||||
include:
|
|
||||||
- ./path/to/included-config.yaml
|
|
||||||
- ./second/path/to/include-config.yaml
|
|
||||||
specs:
|
|
||||||
- libelf
|
|
||||||
- mpileaks
|
|
||||||
"""
|
|
||||||
)
|
|
||||||
|
|
||||||
e = ev.read("test")
|
|
||||||
|
|
||||||
fs.mkdirp(os.path.join(e.path, "path", "to"))
|
|
||||||
with open(os.path.join(e.path, "./path/to/included-config.yaml"), "w") as f:
|
|
||||||
f.write(
|
f.write(
|
||||||
"""\
|
"""\
|
||||||
packages:
|
packages:
|
||||||
|
@ -760,8 +747,9 @@ def test_env_with_include_config_files_same_basename(environment_from_manifest):
|
||||||
"""
|
"""
|
||||||
)
|
)
|
||||||
|
|
||||||
fs.mkdirp(os.path.join(e.path, "second", "path", "to"))
|
file2 = fs.join_path(tmp_path, "second", "path", "included-config.yaml")
|
||||||
with open(os.path.join(e.path, "./second/path/to/include-config.yaml"), "w") as f:
|
fs.mkdirp(os.path.dirname(file2))
|
||||||
|
with open(file2, "w") as f:
|
||||||
f.write(
|
f.write(
|
||||||
"""\
|
"""\
|
||||||
packages:
|
packages:
|
||||||
|
@ -770,6 +758,18 @@ def test_env_with_include_config_files_same_basename(environment_from_manifest):
|
||||||
"""
|
"""
|
||||||
)
|
)
|
||||||
|
|
||||||
|
e = environment_from_manifest(
|
||||||
|
f"""
|
||||||
|
spack:
|
||||||
|
include:
|
||||||
|
- {file1}
|
||||||
|
- {file2}
|
||||||
|
specs:
|
||||||
|
- libelf
|
||||||
|
- mpileaks
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
|
||||||
with e:
|
with e:
|
||||||
e.concretize()
|
e.concretize()
|
||||||
|
|
||||||
|
@ -806,12 +806,18 @@ def mpileaks_env_config(include_path):
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def test_env_with_included_config_file(environment_from_manifest, packages_file):
|
def test_env_with_included_config_file(mutable_mock_env_path, packages_file):
|
||||||
"""Test inclusion of a relative packages configuration file added to an
|
"""Test inclusion of a relative packages configuration file added to an
|
||||||
existing environment.
|
existing environment.
|
||||||
"""
|
"""
|
||||||
|
env_root = mutable_mock_env_path
|
||||||
|
fs.mkdirp(env_root)
|
||||||
include_filename = "included-config.yaml"
|
include_filename = "included-config.yaml"
|
||||||
e = environment_from_manifest(
|
included_path = env_root / include_filename
|
||||||
|
shutil.move(packages_file.strpath, included_path)
|
||||||
|
|
||||||
|
spack_yaml = env_root / ev.manifest_name
|
||||||
|
spack_yaml.write_text(
|
||||||
f"""\
|
f"""\
|
||||||
spack:
|
spack:
|
||||||
include:
|
include:
|
||||||
|
@ -821,9 +827,7 @@ def test_env_with_included_config_file(environment_from_manifest, packages_file)
|
||||||
"""
|
"""
|
||||||
)
|
)
|
||||||
|
|
||||||
included_path = os.path.join(e.path, include_filename)
|
e = ev.Environment(env_root)
|
||||||
shutil.move(packages_file.strpath, included_path)
|
|
||||||
|
|
||||||
with e:
|
with e:
|
||||||
e.concretize()
|
e.concretize()
|
||||||
|
|
||||||
|
@ -856,68 +860,67 @@ def test_env_with_included_config_missing_file(tmpdir, mutable_empty_config):
|
||||||
with spack_yaml.open("w") as f:
|
with spack_yaml.open("w") as f:
|
||||||
f.write("spack:\n include:\n - {0}\n".format(missing_file.strpath))
|
f.write("spack:\n include:\n - {0}\n".format(missing_file.strpath))
|
||||||
|
|
||||||
env = ev.Environment(tmpdir.strpath)
|
|
||||||
with pytest.raises(spack.config.ConfigError, match="missing include path"):
|
with pytest.raises(spack.config.ConfigError, match="missing include path"):
|
||||||
ev.activate(env)
|
ev.Environment(tmpdir.strpath)
|
||||||
|
|
||||||
|
|
||||||
def test_env_with_included_config_scope(environment_from_manifest, packages_file):
|
def test_env_with_included_config_scope(mutable_mock_env_path, packages_file):
|
||||||
"""Test inclusion of a package file from the environment's configuration
|
"""Test inclusion of a package file from the environment's configuration
|
||||||
stage directory. This test is intended to represent a case where a remote
|
stage directory. This test is intended to represent a case where a remote
|
||||||
file has already been staged."""
|
file has already been staged."""
|
||||||
config_scope_path = os.path.join(ev.root("test"), "config")
|
env_root = mutable_mock_env_path
|
||||||
|
config_scope_path = env_root / "config"
|
||||||
# Configure the environment to include file(s) from the environment's
|
|
||||||
# remote configuration stage directory.
|
|
||||||
e = environment_from_manifest(mpileaks_env_config(config_scope_path))
|
|
||||||
|
|
||||||
# Copy the packages.yaml file to the environment configuration
|
# Copy the packages.yaml file to the environment configuration
|
||||||
# directory, so it is picked up during concretization. (Using
|
# directory, so it is picked up during concretization. (Using
|
||||||
# copy instead of rename in case the fixture scope changes.)
|
# copy instead of rename in case the fixture scope changes.)
|
||||||
fs.mkdirp(config_scope_path)
|
fs.mkdirp(config_scope_path)
|
||||||
include_filename = os.path.basename(packages_file.strpath)
|
include_filename = os.path.basename(packages_file.strpath)
|
||||||
included_path = os.path.join(config_scope_path, include_filename)
|
included_path = config_scope_path / include_filename
|
||||||
fs.copy(packages_file.strpath, included_path)
|
fs.copy(packages_file.strpath, included_path)
|
||||||
|
|
||||||
|
# Configure the environment to include file(s) from the environment's
|
||||||
|
# remote configuration stage directory.
|
||||||
|
spack_yaml = env_root / ev.manifest_name
|
||||||
|
spack_yaml.write_text(mpileaks_env_config(config_scope_path))
|
||||||
|
|
||||||
# Ensure the concretized environment reflects contents of the
|
# Ensure the concretized environment reflects contents of the
|
||||||
# packages.yaml file.
|
# packages.yaml file.
|
||||||
|
e = ev.Environment(env_root)
|
||||||
with e:
|
with e:
|
||||||
e.concretize()
|
e.concretize()
|
||||||
|
|
||||||
assert any(x.satisfies("mpileaks@2.2") for x in e._get_environment_specs())
|
assert any(x.satisfies("mpileaks@2.2") for x in e._get_environment_specs())
|
||||||
|
|
||||||
|
|
||||||
def test_env_with_included_config_var_path(environment_from_manifest, packages_file):
|
def test_env_with_included_config_var_path(tmpdir, packages_file):
|
||||||
"""Test inclusion of a package configuration file with path variables
|
"""Test inclusion of a package configuration file with path variables
|
||||||
"staged" in the environment's configuration stage directory."""
|
"staged" in the environment's configuration stage directory."""
|
||||||
config_var_path = os.path.join("$tempdir", "included-config.yaml")
|
included_file = packages_file.strpath
|
||||||
e = environment_from_manifest(mpileaks_env_config(config_var_path))
|
env_path = pathlib.PosixPath(tmpdir)
|
||||||
|
config_var_path = os.path.join("$tempdir", "included-packages.yaml")
|
||||||
|
|
||||||
|
spack_yaml = env_path / ev.manifest_name
|
||||||
|
spack_yaml.write_text(mpileaks_env_config(config_var_path))
|
||||||
|
|
||||||
config_real_path = substitute_path_variables(config_var_path)
|
config_real_path = substitute_path_variables(config_var_path)
|
||||||
fs.mkdirp(os.path.dirname(config_real_path))
|
shutil.move(included_file, config_real_path)
|
||||||
shutil.move(packages_file.strpath, config_real_path)
|
|
||||||
assert os.path.exists(config_real_path)
|
assert os.path.exists(config_real_path)
|
||||||
|
|
||||||
|
e = ev.Environment(env_path)
|
||||||
with e:
|
with e:
|
||||||
e.concretize()
|
e.concretize()
|
||||||
|
|
||||||
assert any(x.satisfies("mpileaks@2.2") for x in e._get_environment_specs())
|
assert any(x.satisfies("mpileaks@2.2") for x in e._get_environment_specs())
|
||||||
|
|
||||||
|
|
||||||
def test_env_config_precedence(environment_from_manifest):
|
def test_env_with_included_config_precedence(tmp_path):
|
||||||
e = environment_from_manifest(
|
"""Test included scope and manifest precedence when including a package
|
||||||
"""
|
configuration file."""
|
||||||
spack:
|
|
||||||
packages:
|
included_file = "included-packages.yaml"
|
||||||
libelf:
|
included_path = tmp_path / included_file
|
||||||
version: ["0.8.12"]
|
with open(included_path, "w") as f:
|
||||||
include:
|
|
||||||
- ./included-config.yaml
|
|
||||||
specs:
|
|
||||||
- mpileaks
|
|
||||||
"""
|
|
||||||
)
|
|
||||||
with open(os.path.join(e.path, "included-config.yaml"), "w") as f:
|
|
||||||
f.write(
|
f.write(
|
||||||
"""\
|
"""\
|
||||||
packages:
|
packages:
|
||||||
|
@ -928,29 +931,50 @@ def test_env_config_precedence(environment_from_manifest):
|
||||||
"""
|
"""
|
||||||
)
|
)
|
||||||
|
|
||||||
with e:
|
spack_yaml = tmp_path / ev.manifest_name
|
||||||
e.concretize()
|
spack_yaml.write_text(
|
||||||
|
f"""\
|
||||||
# ensure included scope took effect
|
|
||||||
assert any(x.satisfies("mpileaks@2.2") for x in e._get_environment_specs())
|
|
||||||
|
|
||||||
# ensure env file takes precedence
|
|
||||||
assert any(x.satisfies("libelf@0.8.12") for x in e._get_environment_specs())
|
|
||||||
|
|
||||||
|
|
||||||
def test_included_config_precedence(environment_from_manifest):
|
|
||||||
e = environment_from_manifest(
|
|
||||||
"""
|
|
||||||
spack:
|
spack:
|
||||||
|
packages:
|
||||||
|
libelf:
|
||||||
|
version: ["0.8.12"]
|
||||||
include:
|
include:
|
||||||
- ./high-config.yaml # this one should take precedence
|
- {os.path.join(".", included_file)}
|
||||||
- ./low-config.yaml
|
|
||||||
specs:
|
specs:
|
||||||
- mpileaks
|
- mpileaks
|
||||||
"""
|
"""
|
||||||
)
|
)
|
||||||
|
|
||||||
with open(os.path.join(e.path, "high-config.yaml"), "w") as f:
|
e = ev.Environment(tmp_path)
|
||||||
|
with e:
|
||||||
|
e.concretize()
|
||||||
|
specs = e._get_environment_specs()
|
||||||
|
|
||||||
|
# ensure included scope took effect
|
||||||
|
assert any(x.satisfies("mpileaks@2.2") for x in specs)
|
||||||
|
|
||||||
|
# ensure env file takes precedence
|
||||||
|
assert any(x.satisfies("libelf@0.8.12") for x in specs)
|
||||||
|
|
||||||
|
|
||||||
|
def test_env_with_included_configs_precedence(tmp_path):
|
||||||
|
"""Test precendence of multiple included configuration files."""
|
||||||
|
file1 = "high-config.yaml"
|
||||||
|
file2 = "low-config.yaml"
|
||||||
|
|
||||||
|
spack_yaml = tmp_path / ev.manifest_name
|
||||||
|
spack_yaml.write_text(
|
||||||
|
f"""\
|
||||||
|
spack:
|
||||||
|
include:
|
||||||
|
- {os.path.join(".", file1)} # this one should take precedence
|
||||||
|
- {os.path.join(".", file2)}
|
||||||
|
specs:
|
||||||
|
- mpileaks
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
|
||||||
|
with open(tmp_path / file1, "w") as f:
|
||||||
f.write(
|
f.write(
|
||||||
"""\
|
"""\
|
||||||
packages:
|
packages:
|
||||||
|
@ -959,7 +983,7 @@ def test_included_config_precedence(environment_from_manifest):
|
||||||
"""
|
"""
|
||||||
)
|
)
|
||||||
|
|
||||||
with open(os.path.join(e.path, "low-config.yaml"), "w") as f:
|
with open(tmp_path / file2, "w") as f:
|
||||||
f.write(
|
f.write(
|
||||||
"""\
|
"""\
|
||||||
packages:
|
packages:
|
||||||
|
@ -970,12 +994,16 @@ def test_included_config_precedence(environment_from_manifest):
|
||||||
"""
|
"""
|
||||||
)
|
)
|
||||||
|
|
||||||
|
e = ev.Environment(tmp_path)
|
||||||
with e:
|
with e:
|
||||||
e.concretize()
|
e.concretize()
|
||||||
|
specs = e._get_environment_specs()
|
||||||
|
|
||||||
assert any(x.satisfies("mpileaks@2.2") for x in e._get_environment_specs())
|
# ensure included package spec took precedence over manifest spec
|
||||||
|
assert any(x.satisfies("mpileaks@2.2") for x in specs)
|
||||||
|
|
||||||
assert any([x.satisfies("libelf@0.8.10") for x in e._get_environment_specs()])
|
# ensure first included package spec took precedence over one from second
|
||||||
|
assert any(x.satisfies("libelf@0.8.10") for x in specs)
|
||||||
|
|
||||||
|
|
||||||
def test_bad_env_yaml_format(environment_from_manifest):
|
def test_bad_env_yaml_format(environment_from_manifest):
|
||||||
|
@ -1578,10 +1606,9 @@ def test_stack_yaml_remove_from_list(tmpdir):
|
||||||
assert Spec("callpath") in test.user_specs
|
assert Spec("callpath") in test.user_specs
|
||||||
|
|
||||||
|
|
||||||
def test_stack_yaml_remove_from_list_force(tmpdir):
|
def test_stack_yaml_remove_from_list_force(tmp_path):
|
||||||
filename = str(tmpdir.join("spack.yaml"))
|
spack_yaml = tmp_path / ev.manifest_name
|
||||||
with open(filename, "w") as f:
|
spack_yaml.write_text(
|
||||||
f.write(
|
|
||||||
"""\
|
"""\
|
||||||
spack:
|
spack:
|
||||||
definitions:
|
definitions:
|
||||||
|
@ -1592,8 +1619,8 @@ def test_stack_yaml_remove_from_list_force(tmpdir):
|
||||||
- [^mpich, ^zmpi]
|
- [^mpich, ^zmpi]
|
||||||
"""
|
"""
|
||||||
)
|
)
|
||||||
with tmpdir.as_cwd():
|
|
||||||
env("create", "test", "./spack.yaml")
|
env("create", "test", str(spack_yaml))
|
||||||
with ev.read("test"):
|
with ev.read("test"):
|
||||||
concretize()
|
concretize()
|
||||||
remove("-f", "-l", "packages", "mpileaks")
|
remove("-f", "-l", "packages", "mpileaks")
|
||||||
|
@ -1650,7 +1677,7 @@ def test_stack_yaml_force_remove_from_matrix(tmpdir):
|
||||||
with tmpdir.as_cwd():
|
with tmpdir.as_cwd():
|
||||||
env("create", "test", "./spack.yaml")
|
env("create", "test", "./spack.yaml")
|
||||||
with ev.read("test") as e:
|
with ev.read("test") as e:
|
||||||
concretize()
|
e.concretize()
|
||||||
|
|
||||||
before_user = e.user_specs.specs
|
before_user = e.user_specs.specs
|
||||||
before_conc = e.concretized_user_specs
|
before_conc = e.concretized_user_specs
|
||||||
|
|
|
@ -18,6 +18,7 @@
|
||||||
SpackEnvironmentViewError,
|
SpackEnvironmentViewError,
|
||||||
_error_on_nonempty_view_dir,
|
_error_on_nonempty_view_dir,
|
||||||
)
|
)
|
||||||
|
from spack.spec_list import UndefinedReferenceError
|
||||||
|
|
||||||
pytestmark = pytest.mark.not_on_windows("Envs are not supported on windows")
|
pytestmark = pytest.mark.not_on_windows("Envs are not supported on windows")
|
||||||
|
|
||||||
|
@ -716,3 +717,64 @@ def test_variant_propagation_with_unify_false(tmp_path, mock_packages):
|
||||||
root = env.matching_spec("parent-foo")
|
root = env.matching_spec("parent-foo")
|
||||||
for node in root.traverse():
|
for node in root.traverse():
|
||||||
assert node.satisfies("+foo")
|
assert node.satisfies("+foo")
|
||||||
|
|
||||||
|
|
||||||
|
def test_env_with_include_defs(mutable_mock_env_path, mock_packages):
|
||||||
|
"""Test environment with included definitions file."""
|
||||||
|
env_path = mutable_mock_env_path
|
||||||
|
env_path.mkdir()
|
||||||
|
defs_file = env_path / "definitions.yaml"
|
||||||
|
defs_file.write_text(
|
||||||
|
"""definitions:
|
||||||
|
- core_specs: [libdwarf, libelf]
|
||||||
|
- compilers: ['%gcc']
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
|
||||||
|
spack_yaml = env_path / ev.manifest_name
|
||||||
|
spack_yaml.write_text(
|
||||||
|
f"""spack:
|
||||||
|
include:
|
||||||
|
- file://{defs_file}
|
||||||
|
|
||||||
|
definitions:
|
||||||
|
- my_packages: [zlib]
|
||||||
|
|
||||||
|
specs:
|
||||||
|
- matrix:
|
||||||
|
- [$core_specs]
|
||||||
|
- [$compilers]
|
||||||
|
- $my_packages
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
|
||||||
|
e = ev.Environment(env_path)
|
||||||
|
with e:
|
||||||
|
e.concretize()
|
||||||
|
|
||||||
|
|
||||||
|
def test_env_with_include_def_missing(mutable_mock_env_path, mock_packages):
|
||||||
|
"""Test environment with included definitions file that is missing a definition."""
|
||||||
|
env_path = mutable_mock_env_path
|
||||||
|
env_path.mkdir()
|
||||||
|
filename = "missing-def.yaml"
|
||||||
|
defs_file = env_path / filename
|
||||||
|
defs_file.write_text("definitions:\n- my_compilers: ['%gcc']\n")
|
||||||
|
|
||||||
|
spack_yaml = env_path / ev.manifest_name
|
||||||
|
spack_yaml.write_text(
|
||||||
|
f"""spack:
|
||||||
|
include:
|
||||||
|
- file://{defs_file}
|
||||||
|
|
||||||
|
specs:
|
||||||
|
- matrix:
|
||||||
|
- [$core_specs]
|
||||||
|
- [$my_compilers]
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
|
||||||
|
e = ev.Environment(env_path)
|
||||||
|
with e:
|
||||||
|
with pytest.raises(UndefinedReferenceError, match=r"which does not appear"):
|
||||||
|
e.concretize()
|
||||||
|
|
|
@ -80,7 +80,17 @@ def test_module_suffixes(module_suffixes_schema):
|
||||||
@pytest.mark.regression("10246")
|
@pytest.mark.regression("10246")
|
||||||
@pytest.mark.parametrize(
|
@pytest.mark.parametrize(
|
||||||
"config_name",
|
"config_name",
|
||||||
["compilers", "config", "env", "merged", "mirrors", "modules", "packages", "repos"],
|
[
|
||||||
|
"compilers",
|
||||||
|
"config",
|
||||||
|
"definitions",
|
||||||
|
"env",
|
||||||
|
"merged",
|
||||||
|
"mirrors",
|
||||||
|
"modules",
|
||||||
|
"packages",
|
||||||
|
"repos",
|
||||||
|
],
|
||||||
)
|
)
|
||||||
def test_schema_validation(meta_schema, config_name):
|
def test_schema_validation(meta_schema, config_name):
|
||||||
import importlib
|
import importlib
|
||||||
|
|
|
@ -1159,19 +1159,19 @@ complete -c spack -n '__fish_spack_using_command config' -l scope -r -d 'configu
|
||||||
|
|
||||||
# spack config get
|
# spack config get
|
||||||
set -g __fish_spack_optspecs_spack_config_get h/help
|
set -g __fish_spack_optspecs_spack_config_get h/help
|
||||||
complete -c spack -n '__fish_spack_using_command_pos 0 config get' -f -a 'bootstrap cdash ci compilers concretizer config mirrors modules packages repos upstreams'
|
complete -c spack -n '__fish_spack_using_command_pos 0 config get' -f -a 'bootstrap cdash ci compilers concretizer config definitions mirrors modules packages repos upstreams'
|
||||||
complete -c spack -n '__fish_spack_using_command config get' -s h -l help -f -a help
|
complete -c spack -n '__fish_spack_using_command config get' -s h -l help -f -a help
|
||||||
complete -c spack -n '__fish_spack_using_command config get' -s h -l help -d 'show this help message and exit'
|
complete -c spack -n '__fish_spack_using_command config get' -s h -l help -d 'show this help message and exit'
|
||||||
|
|
||||||
# spack config blame
|
# spack config blame
|
||||||
set -g __fish_spack_optspecs_spack_config_blame h/help
|
set -g __fish_spack_optspecs_spack_config_blame h/help
|
||||||
complete -c spack -n '__fish_spack_using_command_pos 0 config blame' -f -a 'bootstrap cdash ci compilers concretizer config mirrors modules packages repos upstreams'
|
complete -c spack -n '__fish_spack_using_command_pos 0 config blame' -f -a 'bootstrap cdash ci compilers concretizer config definitions mirrors modules packages repos upstreams'
|
||||||
complete -c spack -n '__fish_spack_using_command config blame' -s h -l help -f -a help
|
complete -c spack -n '__fish_spack_using_command config blame' -s h -l help -f -a help
|
||||||
complete -c spack -n '__fish_spack_using_command config blame' -s h -l help -d 'show this help message and exit'
|
complete -c spack -n '__fish_spack_using_command config blame' -s h -l help -d 'show this help message and exit'
|
||||||
|
|
||||||
# spack config edit
|
# spack config edit
|
||||||
set -g __fish_spack_optspecs_spack_config_edit h/help print-file
|
set -g __fish_spack_optspecs_spack_config_edit h/help print-file
|
||||||
complete -c spack -n '__fish_spack_using_command_pos 0 config edit' -f -a 'bootstrap cdash ci compilers concretizer config mirrors modules packages repos upstreams'
|
complete -c spack -n '__fish_spack_using_command_pos 0 config edit' -f -a 'bootstrap cdash ci compilers concretizer config definitions mirrors modules packages repos upstreams'
|
||||||
complete -c spack -n '__fish_spack_using_command config edit' -s h -l help -f -a help
|
complete -c spack -n '__fish_spack_using_command config edit' -s h -l help -f -a help
|
||||||
complete -c spack -n '__fish_spack_using_command config edit' -s h -l help -d 'show this help message and exit'
|
complete -c spack -n '__fish_spack_using_command config edit' -s h -l help -d 'show this help message and exit'
|
||||||
complete -c spack -n '__fish_spack_using_command config edit' -l print-file -f -a print_file
|
complete -c spack -n '__fish_spack_using_command config edit' -l print-file -f -a print_file
|
||||||
|
|
Loading…
Reference in a new issue