Integrate namespace attribute into spec, spec DAG, spec YAML.
This commit is contained in:
Todd Gamblin
parent
a338e0efd5
commit
73ef06018e
6 changed files with 125 additions and 35 deletions
|
|
@ -115,7 +115,7 @@ def __init__(self, name, filename, merge, strip):
|
|||
self.result_dict = {}
|
||||
_config_sections[name] = self
|
||||
|
||||
_ConfigCategory('config', 'config.yaml', True, False)
|
||||
_ConfigCategory('repos', 'repos.yaml', True, True)
|
||||
_ConfigCategory('compilers', 'compilers.yaml', True, True)
|
||||
_ConfigCategory('mirrors', 'mirrors.yaml', True, True)
|
||||
_ConfigCategory('view', 'views.yaml', True, True)
|
||||
|
|
@ -212,7 +212,7 @@ def substitute_spack_prefix(path):
|
|||
return path.replace('$spack', spack.prefix)
|
||||
|
||||
|
||||
def get_config(category='config'):
|
||||
def get_config(category):
|
||||
"""Get the confguration tree for a category.
|
||||
|
||||
Strips off the top-level category entry from the dict
|
||||
|
|
@ -233,6 +233,10 @@ def get_config(category='config'):
|
|||
continue
|
||||
result = result[category.name]
|
||||
|
||||
# ignore empty sections for easy commenting of single-line configs.
|
||||
if result is None:
|
||||
continue
|
||||
|
||||
category.files_read_from.insert(0, path)
|
||||
if category.merge:
|
||||
category.result_dict = _merge_yaml(category.result_dict, result)
|
||||
|
|
@ -266,12 +270,18 @@ def get_compilers_config(arch=None):
|
|||
|
||||
|
||||
def get_repos_config():
|
||||
config = get_config()
|
||||
if 'repos' not in config:
|
||||
repo_list = get_config('repos')
|
||||
if repo_list is None:
|
||||
return []
|
||||
|
||||
repo_list = config['repos']
|
||||
return [substitute_spack_prefix(repo) for repo in repo_list]
|
||||
if not isinstance(repo_list, list):
|
||||
tty.die("Bad repository configuration. 'repos' element does not contain a list.")
|
||||
|
||||
def expand_repo_path(path):
|
||||
path = substitute_spack_prefix(path)
|
||||
path = os.path.expanduser(path)
|
||||
return path
|
||||
return [expand_repo_path(repo) for repo in repo_list]
|
||||
|
||||
|
||||
def get_mirror_config():
|
||||
|
|
|
|||
|
|
@ -211,6 +211,8 @@ def _read_spec_from_yaml(self, hash_key, installs, parent_key=None):
|
|||
child = self._read_spec_from_yaml(dep_hash, installs, hash_key)
|
||||
spec._add_dependency(child)
|
||||
|
||||
spec._normal = True
|
||||
spec._concrete = True
|
||||
return spec
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -211,9 +211,12 @@ def read_spec(self, path):
|
|||
with open(path) as f:
|
||||
spec = Spec.from_yaml(f)
|
||||
|
||||
# Specs read from actual installations are always concrete
|
||||
spec._normal = True
|
||||
spec._concrete = True
|
||||
# Specs read from actual installs are always concrete, so mark
|
||||
# all parts of the spec.
|
||||
for s in spec.traverse():
|
||||
s._normal = True
|
||||
s._concrete = True
|
||||
|
||||
return spec
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -238,7 +238,16 @@ def get(self, spec, new=False):
|
|||
|
||||
Raises UnknownPackageError if not found.
|
||||
"""
|
||||
return self.repo_for_pkg(spec.name).get(spec)
|
||||
# if the spec has a fully qualified namespace, we grab it
|
||||
# directly and ignore overlay precedence.
|
||||
if spec.namespace:
|
||||
fullspace = '%s.%s' % (self.super_namespace, spec.namespace)
|
||||
if not fullspace in self.by_namespace:
|
||||
raise UnknownPackageError(
|
||||
"No configured repository contains package %s." % spec.fullname)
|
||||
return self.by_namespace[fullspace].get(spec)
|
||||
else:
|
||||
return self.repo_for_pkg(spec.name).get(spec)
|
||||
|
||||
|
||||
def dirname_for_package_name(self, pkg_name):
|
||||
|
|
@ -454,20 +463,24 @@ def get(self, spec, new=False):
|
|||
if spec.virtual:
|
||||
raise UnknownPackageError(spec.name)
|
||||
|
||||
if new and spec in self._instances:
|
||||
del self._instances[spec]
|
||||
if spec.namespace and spec.namespace != self.namespace:
|
||||
raise UnknownPackageError("Repository %s does not contain package %s."
|
||||
% (self.namespace, spec.fullname))
|
||||
|
||||
if not spec in self._instances:
|
||||
if new or spec not in self._instances:
|
||||
PackageClass = self._get_pkg_class(spec.name)
|
||||
try:
|
||||
copy = spec.copy()
|
||||
self._instances[copy] = PackageClass(copy)
|
||||
package = PackageClass(spec.copy())
|
||||
self._instances[spec] = package
|
||||
return package
|
||||
|
||||
except Exception, e:
|
||||
if spack.debug:
|
||||
sys.excepthook(*sys.exc_info())
|
||||
raise FailedConstructorError(spec.name, *sys.exc_info())
|
||||
raise FailedConstructorError(spec.fullname, *sys.exc_info())
|
||||
|
||||
return self._instances[spec]
|
||||
else:
|
||||
return self._instances[spec]
|
||||
|
||||
|
||||
def purge(self):
|
||||
|
|
|
|||
|
|
@ -465,6 +465,13 @@ def _add_dependency(self, spec):
|
|||
self.dependencies[spec.name] = spec
|
||||
spec.dependents[self.name] = self
|
||||
|
||||
#
|
||||
# Public interface
|
||||
#
|
||||
@property
|
||||
def fullname(self):
|
||||
return '%s.%s' % (self.namespace, self.name) if self.namespace else self.name
|
||||
|
||||
|
||||
@property
|
||||
def root(self):
|
||||
|
|
@ -518,6 +525,7 @@ def concrete(self):
|
|||
return True
|
||||
|
||||
self._concrete = bool(not self.virtual
|
||||
and self.namespace is not None
|
||||
and self.versions.concrete
|
||||
and self.variants.concrete
|
||||
and self.architecture
|
||||
|
|
@ -658,6 +666,12 @@ def to_node_dict(self):
|
|||
'dependencies' : dict((d, self.dependencies[d].dag_hash())
|
||||
for d in sorted(self.dependencies))
|
||||
}
|
||||
|
||||
# Older concrete specs do not have a namespace. Omit for
|
||||
# consistent hashing.
|
||||
if not self.concrete or self.namespace:
|
||||
d['namespace'] = self.namespace
|
||||
|
||||
if self.compiler:
|
||||
d.update(self.compiler.to_dict())
|
||||
else:
|
||||
|
|
@ -682,6 +696,7 @@ def from_node_dict(node):
|
|||
node = node[name]
|
||||
|
||||
spec = Spec(name)
|
||||
spec.namespace = node.get('namespace', None)
|
||||
spec.versions = VersionList.from_dict(node)
|
||||
spec.architecture = node['arch']
|
||||
|
||||
|
|
@ -834,7 +849,20 @@ def concretize(self):
|
|||
changed = any(changes)
|
||||
force=True
|
||||
|
||||
self._concrete = True
|
||||
for s in self.traverse():
|
||||
# After concretizing, assign namespaces to anything left.
|
||||
# Note that this doesn't count as a "change". The repository
|
||||
# configuration is constant throughout a spack run, and
|
||||
# normalize and concretize evaluate Packages using Repo.get(),
|
||||
# which respects precedence. So, a namespace assignment isn't
|
||||
# changing how a package name would have been interpreted and
|
||||
# we can do it as late as possible to allow as much
|
||||
# compatibility across repositories as possible.
|
||||
if s.namespace is None:
|
||||
s.namespace = spack.repo.repo_for_pkg(s.name).namespace
|
||||
|
||||
# Mark everything in the spec as concrete, as well.
|
||||
s._concrete = True
|
||||
|
||||
|
||||
def concretized(self):
|
||||
|
|
@ -909,7 +937,7 @@ def _evaluate_dependency_conditions(self, name):
|
|||
the dependency. If no conditions are True (and we don't
|
||||
depend on it), return None.
|
||||
"""
|
||||
pkg = spack.repo.get(self.name)
|
||||
pkg = spack.repo.get(self.fullname)
|
||||
conditions = pkg.dependencies[name]
|
||||
|
||||
# evaluate when specs to figure out constraints on the dependency.
|
||||
|
|
@ -1037,7 +1065,7 @@ def _normalize_helper(self, visited, spec_deps, provider_index):
|
|||
any_change = False
|
||||
changed = True
|
||||
|
||||
pkg = spack.repo.get(self.name)
|
||||
pkg = spack.repo.get(self.fullname)
|
||||
while changed:
|
||||
changed = False
|
||||
for dep_name in pkg.dependencies:
|
||||
|
|
@ -1058,18 +1086,17 @@ def normalize(self, force=False):
|
|||
the root, and ONLY the ones that were explicitly provided are there.
|
||||
Normalization turns a partial flat spec into a DAG, where:
|
||||
|
||||
1. ALL dependencies of the root package are in the DAG.
|
||||
2. Each node's dependencies dict only contains its direct deps.
|
||||
1. Known dependencies of the root package are in the DAG.
|
||||
2. Each node's dependencies dict only contains its known direct deps.
|
||||
3. There is only ONE unique spec for each package in the DAG.
|
||||
|
||||
* This includes virtual packages. If there a non-virtual
|
||||
package that provides a virtual package that is in the spec,
|
||||
then we replace the virtual package with the non-virtual one.
|
||||
|
||||
4. The spec DAG matches package DAG, including default variant values.
|
||||
|
||||
TODO: normalize should probably implement some form of cycle detection,
|
||||
to ensure that the spec is actually a DAG.
|
||||
|
||||
"""
|
||||
if self._normal and not force:
|
||||
return False
|
||||
|
|
@ -1115,7 +1142,7 @@ def validate_names(self):
|
|||
for spec in self.traverse():
|
||||
# Don't get a package for a virtual name.
|
||||
if not spec.virtual:
|
||||
spack.repo.get(spec.name)
|
||||
spack.repo.get(spec.fullname)
|
||||
|
||||
# validate compiler in addition to the package name.
|
||||
if spec.compiler:
|
||||
|
|
@ -1138,6 +1165,10 @@ def constrain(self, other, deps=True):
|
|||
if not self.name == other.name:
|
||||
raise UnsatisfiableSpecNameError(self.name, other.name)
|
||||
|
||||
if other.namespace is not None:
|
||||
if self.namespace is not None and other.namespace != self.namespace:
|
||||
raise UnsatisfiableSpecNameError(self.fullname, other.fullname)
|
||||
|
||||
if not self.versions.overlaps(other.versions):
|
||||
raise UnsatisfiableVersionSpecError(self.versions, other.versions)
|
||||
|
||||
|
|
@ -1181,7 +1212,7 @@ def _constrain_dependencies(self, other):
|
|||
|
||||
# TODO: might want more detail than this, e.g. specific deps
|
||||
# in violation. if this becomes a priority get rid of this
|
||||
# check and be more specici about what's wrong.
|
||||
# check and be more specific about what's wrong.
|
||||
if not other.satisfies_dependencies(self):
|
||||
raise UnsatisfiableDependencySpecError(other, self)
|
||||
|
||||
|
|
@ -1247,7 +1278,7 @@ def satisfies(self, other, deps=True, strict=False):
|
|||
|
||||
# A concrete provider can satisfy a virtual dependency.
|
||||
if not self.virtual and other.virtual:
|
||||
pkg = spack.repo.get(self.name)
|
||||
pkg = spack.repo.get(self.fullname)
|
||||
if pkg.provides(other.name):
|
||||
for provided, when_spec in pkg.provided.items():
|
||||
if self.satisfies(when_spec, deps=False, strict=strict):
|
||||
|
|
@ -1259,6 +1290,11 @@ def satisfies(self, other, deps=True, strict=False):
|
|||
if self.name != other.name:
|
||||
return False
|
||||
|
||||
# namespaces either match, or other doesn't require one.
|
||||
if other.namespace is not None:
|
||||
if self.namespace is not None and self.namespace != other.namespace:
|
||||
return False
|
||||
|
||||
if self.versions and other.versions:
|
||||
if not self.versions.satisfies(other.versions, strict=strict):
|
||||
return False
|
||||
|
|
@ -1476,8 +1512,8 @@ def ne_dag(self, other):
|
|||
|
||||
def _cmp_node(self):
|
||||
"""Comparison key for just *this node* and not its deps."""
|
||||
return (self.name, self.versions, self.variants,
|
||||
self.architecture, self.compiler)
|
||||
return (self.name, self.namespace, self.versions,
|
||||
self.variants, self.architecture, self.compiler)
|
||||
|
||||
|
||||
def eq_node(self, other):
|
||||
|
|
@ -1507,7 +1543,7 @@ def format(self, format_string='$_$@$%@$+$=', **kwargs):
|
|||
in the format string. The format strings you can provide are::
|
||||
|
||||
$_ Package name
|
||||
$. Long package name
|
||||
$. Full package name (with namespace)
|
||||
$@ Version
|
||||
$% Compiler
|
||||
$%@ Compiler & compiler version
|
||||
|
|
@ -1556,8 +1592,7 @@ def write(s, c):
|
|||
if c == '_':
|
||||
out.write(fmt % self.name)
|
||||
elif c == '.':
|
||||
longname = '%s.%s.%s' % (self.namespace, self.name) if self.namespace else self.name
|
||||
out.write(fmt % longname)
|
||||
out.write(fmt % self.fullname)
|
||||
elif c == '@':
|
||||
if self.versions and self.versions != _any_version:
|
||||
write(fmt % (c + str(self.versions)), c)
|
||||
|
|
|
|||
|
|
@ -35,7 +35,10 @@ class SpecSematicsTest(MockPackagesTest):
|
|||
# ================================================================================
|
||||
def check_satisfies(self, spec, anon_spec, concrete=False):
|
||||
left = Spec(spec, concrete=concrete)
|
||||
right = parse_anonymous_spec(anon_spec, left.name)
|
||||
try:
|
||||
right = Spec(anon_spec) # if it's not anonymous, allow it.
|
||||
except:
|
||||
right = parse_anonymous_spec(anon_spec, left.name)
|
||||
|
||||
# Satisfies is one-directional.
|
||||
self.assertTrue(left.satisfies(right))
|
||||
|
|
@ -48,7 +51,10 @@ def check_satisfies(self, spec, anon_spec, concrete=False):
|
|||
|
||||
def check_unsatisfiable(self, spec, anon_spec, concrete=False):
|
||||
left = Spec(spec, concrete=concrete)
|
||||
right = parse_anonymous_spec(anon_spec, left.name)
|
||||
try:
|
||||
right = Spec(anon_spec) # if it's not anonymous, allow it.
|
||||
except:
|
||||
right = parse_anonymous_spec(anon_spec, left.name)
|
||||
|
||||
self.assertFalse(left.satisfies(right))
|
||||
self.assertFalse(left.satisfies(anon_spec))
|
||||
|
|
@ -88,6 +94,28 @@ def test_satisfies(self):
|
|||
self.check_satisfies('libdwarf^libelf@0.8.13', '^libelf@0:1')
|
||||
|
||||
|
||||
def test_satisfies_namespace(self):
|
||||
self.check_satisfies('builtin.mpich', 'mpich')
|
||||
self.check_satisfies('builtin.mock.mpich', 'mpich')
|
||||
|
||||
# TODO: only works for deps now, but shouldn't we allow this for root spec?
|
||||
# self.check_satisfies('builtin.mock.mpich', 'mpi')
|
||||
|
||||
self.check_satisfies('builtin.mock.mpich', 'builtin.mock.mpich')
|
||||
|
||||
self.check_unsatisfiable('builtin.mock.mpich', 'builtin.mpich')
|
||||
|
||||
|
||||
def test_satisfies_namespaced_dep(self):
|
||||
"""Ensure spec from same or unspecified namespace satisfies namespace constraint."""
|
||||
self.check_satisfies('mpileaks ^builtin.mock.mpich', '^mpich')
|
||||
|
||||
self.check_satisfies('mpileaks ^builtin.mock.mpich', '^mpi')
|
||||
self.check_satisfies('mpileaks ^builtin.mock.mpich', '^builtin.mock.mpich')
|
||||
|
||||
self.check_unsatisfiable('mpileaks ^builtin.mock.mpich', '^builtin.mpich')
|
||||
|
||||
|
||||
def test_satisfies_compiler(self):
|
||||
self.check_satisfies('foo%gcc', '%gcc')
|
||||
self.check_satisfies('foo%intel', '%intel')
|
||||
|
|
@ -327,4 +355,3 @@ def test_constrain_dependency_not_changed(self):
|
|||
self.check_constrain_not_changed('libelf^foo+debug', 'libelf^foo+debug')
|
||||
self.check_constrain_not_changed('libelf^foo~debug', 'libelf^foo~debug')
|
||||
self.check_constrain_not_changed('libelf^foo=bgqos_0', 'libelf^foo=bgqos_0')
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue