hlrs-software-stack/spack
7
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

hashing: fix caching of dependency hashes in to_node_dict

Browse source
This commit is contained in:
Gregory Becker 2019-07-23 09:37:48 -05:00 committed by Todd Gamblin
parent 4ff2773704
commit a086dda82f
2 changed files with 17 additions and 11 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
lib/spack/spack/hash_types.py
View file

@ -18,19 +18,22 @@ class SpecHashDescriptor(object):
We currently use different hashes for different use cases. We currently use different hashes for different use cases.
""" """
def __init__(self, deptype=('link', 'run'), package_hash=False): def __init__(self, deptype=('link', 'run'), package_hash=False, attr=None):
self.deptype = dp.canonical_deptype(deptype) self.deptype = dp.canonical_deptype(deptype)
self.package_hash = package_hash self.package_hash = package_hash
self.attr = attr
#: Default Hash descriptor, used by Spec.dag_hash() and stored in the DB. #: Default Hash descriptor, used by Spec.dag_hash() and stored in the DB.
dag_hash = SpecHashDescriptor(deptype=('link', 'run'), package_hash=False) dag_hash = SpecHashDescriptor(deptype=('link', 'run'), package_hash=False,
attr='_hash')
#: Hash descriptor that includes build dependencies. #: Hash descriptor that includes build dependencies.
build_hash = SpecHashDescriptor( build_hash = SpecHashDescriptor(
deptype=('build', 'link', 'run'), package_hash=False) deptype=('build', 'link', 'run'), package_hash=False, attr='_build_hash')
#: Full hash used in build pipelines to determine when to rebuild packages. #: Full hash used in build pipelines to determine when to rebuild packages.
full_hash = SpecHashDescriptor(deptype=('link', 'run'), package_hash=True) full_hash = SpecHashDescriptor(deptype=('link', 'run'), package_hash=True,
attr='_full_hash')

17
lib/spack/spack/spec.py
View file

@ -1305,7 +1305,7 @@ def _spec_hash(self, hash):
return b32_hash return b32_hash
def _cached_hash(self, length, attr, hash): def _cached_hash(self, hash, length=None):
"""Helper function for storing a cached hash on the spec. """Helper function for storing a cached hash on the spec.
This will run _spec_hash() with the deptype and package_hash This will run _spec_hash() with the deptype and package_hash
@ -1315,13 +1315,16 @@ def _cached_hash(self, length, attr, hash):
Arguments: Arguments:
hash (SpecHashDescriptor): type of hash to generate. hash (SpecHashDescriptor): type of hash to generate.
""" """
hash_string = getattr(self, attr, None) if not hash.attr:
return self._spec_hash(hash)[:length]
hash_string = getattr(self, hash.attr, None)
if hash_string: if hash_string:
return hash_string[:length] return hash_string[:length]
else: else:
hash_string = self._spec_hash(hash) hash_string = self._spec_hash(hash)
if self.concrete: if self.concrete:
setattr(self, attr, hash_string) setattr(self, hash.attr, hash_string)
return hash_string[:length] return hash_string[:length]
@ -1333,7 +1336,7 @@ def dag_hash(self, length=None):
revise this to include more detailed provenance when the revise this to include more detailed provenance when the
concretizer can more aggressievly reuse installed dependencies. concretizer can more aggressievly reuse installed dependencies.
""" """
return self._cached_hash(length, '_hash', ht.dag_hash) return self._cached_hash(ht.dag_hash, length)
def build_hash(self, length=None): def build_hash(self, length=None):
"""Hash used to store specs in environments. """Hash used to store specs in environments.
@ -1341,7 +1344,7 @@ def build_hash(self, length=None):
This hash includes build dependencies, and we need to preserve This hash includes build dependencies, and we need to preserve
them to be able to rebuild an entire environment for a user. them to be able to rebuild an entire environment for a user.
""" """
return self._cached_hash(length, '_build_hash', ht.build_hash) return self._cached_hash(ht.build_hash, length)
def full_hash(self, length=None): def full_hash(self, length=None):
"""Hash to determine when to rebuild packages in the build pipeline. """Hash to determine when to rebuild packages in the build pipeline.
@ -1352,7 +1355,7 @@ def full_hash(self, length=None):
TODO: investigate whether to include build deps here. TODO: investigate whether to include build deps here.
""" """
return self._cached_hash(length, '_full_hash', ht.full_hash) return self._cached_hash(ht.full_hash, length)
def dag_hash_bit_prefix(self, bits): def dag_hash_bit_prefix(self, bits):
"""Get the first <bits> bits of the DAG hash as an integer type.""" """Get the first <bits> bits of the DAG hash as an integer type."""
@ -1456,7 +1459,7 @@ def to_node_dict(self, hash=ht.dag_hash):
d['dependencies'] = syaml_dict([ d['dependencies'] = syaml_dict([
(name, (name,
syaml_dict([ syaml_dict([
('hash', dspec.spec._spec_hash(hash)), ('hash', dspec.spec._cached_hash(hash)),
('type', sorted(str(s) for s in dspec.deptypes))]) ('type', sorted(str(s) for s in dspec.deptypes))])
) for name, dspec in sorted(deps.items()) ) for name, dspec in sorted(deps.items())
]) ])