hlrs-software-stack/spack
7
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

hashes: remove full_hash and build_hash from spack

Browse source
This commit is contained in:
Scott Wittenburg 2022-01-28 09:49:15 -07:00 committed by Todd Gamblin
parent 512645ff2e
commit f6e7c0b740
28 changed files with 335 additions and 711 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

244
lib/spack/spack/binary_distribution.py
View file

@ -182,7 +182,6 @@ def _associate_built_specs_with_mirror(self, cache_key, mirror_url):
for indexed_spec in spec_list:
dag_hash = indexed_spec.dag_hash()
full_hash = indexed_spec._full_hash
if dag_hash not in self._mirrors_for_spec:
self._mirrors_for_spec[dag_hash] = []
@ -190,11 +189,8 @@ def _associate_built_specs_with_mirror(self, cache_key, mirror_url):
for entry in self._mirrors_for_spec[dag_hash]:
# A binary mirror can only have one spec per DAG hash, so
# if we already have an entry under this DAG hash for this
# mirror url, we may need to replace the spec associated
# with it (but only if it has a different full_hash).
# mirror url, we're done.
if entry['mirror_url'] == mirror_url:
if full_hash and full_hash != entry['spec']._full_hash:
entry['spec'] = indexed_spec
break
else:
self._mirrors_for_spec[dag_hash].append({
@ -403,6 +399,11 @@ def _fetch_and_cache_index(self, mirror_url, expect_hash=None):
hash_fetch_url = url_util.join(
mirror_url, _build_cache_relative_path, 'index.json.hash')
if not web_util.url_exists(index_fetch_url):
# A binary mirror is not required to have an index, so avoid
# raising FetchCacheError in that case.
return False
old_cache_key = None
fetched_hash = None
@ -790,8 +791,13 @@ def generate_package_index(cache_prefix):
tty.debug('Retrieving spec descriptor files from {0} to build index'.format(
cache_prefix))
all_mirror_specs = {}
tmpdir = tempfile.mkdtemp()
db_root_dir = os.path.join(tmpdir, 'db_root')
db = spack_db.Database(None, db_dir=db_root_dir,
enable_transaction_locking=False,
record_fields=['spec', 'ref_count', 'in_buildcache'])
try:
for file_path in file_list:
try:
spec_url = url_util.join(cache_prefix, file_path)
@ -805,84 +811,20 @@ def generate_package_index(cache_prefix):
elif spec_url.endswith('.yaml'):
spec_dict = syaml.load(spec_file_contents)
s = Spec.from_yaml(spec_file_contents)
all_mirror_specs[s.dag_hash()] = {
'spec_url': spec_url,
'spec': s,
'num_deps': len(list(s.traverse(root=False))),
'binary_cache_checksum': spec_dict['binary_cache_checksum'],
'buildinfo': spec_dict['buildinfo'],
}
if s:
db.add(s, None)
db.mark(s, 'in_buildcache', True)
except (URLError, web_util.SpackWebError) as url_err:
tty.error('Error reading specfile: {0}'.format(file_path))
tty.error(url_err)
sorted_specs = sorted(all_mirror_specs.keys(),
key=lambda k: all_mirror_specs[k]['num_deps'])
tmpdir = tempfile.mkdtemp()
db_root_dir = os.path.join(tmpdir, 'db_root')
db = spack_db.Database(None, db_dir=db_root_dir,
enable_transaction_locking=False,
record_fields=['spec', 'ref_count', 'in_buildcache'])
try:
tty.debug('Specs sorted by number of dependencies:')
for dag_hash in sorted_specs:
spec_record = all_mirror_specs[dag_hash]
s = spec_record['spec']
num_deps = spec_record['num_deps']
tty.debug(' {0}/{1} -> {2}'.format(
s.name, dag_hash[:7], num_deps))
if num_deps > 0:
# Check each of this spec's dependencies (which we have already
# processed), as they are the source of truth for their own
# full hash. If the full hash we have for any deps does not
# match what those deps have themselves, then we need to splice
# this spec with those deps, and push this spliced spec
# (spec.json file) back to the mirror, as well as update the
# all_mirror_specs dictionary with this spliced spec.
to_splice = []
for dep in s.dependencies():
dep_dag_hash = dep.dag_hash()
if dep_dag_hash in all_mirror_specs:
true_dep = all_mirror_specs[dep_dag_hash]['spec']
if true_dep.full_hash() != dep.full_hash():
to_splice.append(true_dep)
if to_splice:
tty.debug(' needs the following deps spliced:')
for true_dep in to_splice:
tty.debug(' {0}/{1}'.format(
true_dep.name, true_dep.dag_hash()[:7]))
s = s.splice(true_dep, True)
# Push this spliced spec back to the mirror
spliced_spec_dict = s.to_dict(hash=ht.full_hash)
for key in ['binary_cache_checksum', 'buildinfo']:
spliced_spec_dict[key] = spec_record[key]
temp_json_path = os.path.join(tmpdir, 'spliced.spec.json')
with open(temp_json_path, 'w') as fd:
fd.write(sjson.dump(spliced_spec_dict))
spliced_spec_url = spec_record['spec_url']
web_util.push_to_url(
temp_json_path, spliced_spec_url, keep_original=False)
tty.debug(' spliced and wrote {0}'.format(
spliced_spec_url))
spec_record['spec'] = s
db.add(s, None)
db.mark(s, 'in_buildcache', True)
# Now that we have fixed any old specfiles that might have had the wrong
# full hash for their dependencies, we can generate the index, compute
# the hash, and push those files to the mirror.
# Now generate the index, compute its hash, and push the two files to
# the mirror.
index_json_path = os.path.join(db_root_dir, 'index.json')
with open(index_json_path, 'w') as f:
db._write_to_file(f)
# Read the index back in and compute it's hash
# Read the index back in and compute its hash
with open(index_json_path) as f:
index_string = f.read()
index_hash = compute_hash(index_string)
@ -1610,16 +1552,14 @@ def install_single_spec(spec, allow_root=False, unsigned=False, force=False):
install_root_node(node, allow_root=allow_root, unsigned=unsigned, force=force)
def try_direct_fetch(spec, full_hash_match=False, mirrors=None):
def try_direct_fetch(spec, mirrors=None):
"""
Try to find the spec directly on the configured mirrors
"""
deprecated_specfile_name = tarball_name(spec, '.spec.yaml')
specfile_name = tarball_name(spec, '.spec.json')
specfile_is_json = True
lenient = not full_hash_match
found_specs = []
spec_full_hash = spec.full_hash()
for mirror in spack.mirror.MirrorCollection(mirrors=mirrors).values():
buildcache_fetch_url_yaml = url_util.join(
@ -1649,9 +1589,6 @@ def try_direct_fetch(spec, full_hash_match=False, mirrors=None):
fetched_spec = Spec.from_yaml(specfile_contents)
fetched_spec._mark_concrete()
# Do not recompute the full hash for the fetched spec, instead just
# read the property.
if lenient or fetched_spec._full_hash == spec_full_hash:
found_specs.append({
'mirror_url': mirror.fetch_url,
'spec': fetched_spec,
@ -1660,18 +1597,13 @@ def try_direct_fetch(spec, full_hash_match=False, mirrors=None):
return found_specs
def get_mirrors_for_spec(spec=None, full_hash_match=False,
mirrors_to_check=None, index_only=False):
def get_mirrors_for_spec(spec=None, mirrors_to_check=None, index_only=False):
"""
Check if concrete spec exists on mirrors and return a list
indicating the mirrors on which it can be found
Args:
spec (spack.spec.Spec): The spec to look for in binary mirrors
full_hash_match (bool): If True, only includes mirrors where the spec
full hash matches the locally computed full hash of the ``spec``
argument. If False, any mirror which has a matching DAG hash
is included in the results.
mirrors_to_check (dict): Optionally override the configured mirrors
with the mirrors in this dictionary.
index_only (bool): Do not attempt direct fetching of ``spec.json``
@ -1688,29 +1620,14 @@ def get_mirrors_for_spec(spec=None, full_hash_match=False,
tty.debug("No Spack mirrors are currently configured")
return {}
results = []
lenient = not full_hash_match
spec_full_hash = spec.full_hash()
def filter_candidates(candidate_list):
filtered_candidates = []
for candidate in candidate_list:
candidate_full_hash = candidate['spec']._full_hash
if lenient or spec_full_hash == candidate_full_hash:
filtered_candidates.append(candidate)
return filtered_candidates
candidates = binary_index.find_built_spec(spec)
if candidates:
results = filter_candidates(candidates)
results = binary_index.find_built_spec(spec)
# Maybe we just didn't have the latest information from the mirror, so
# try to fetch directly, unless we are only considering the indices.
if not results and not index_only:
results = try_direct_fetch(spec,
full_hash_match=full_hash_match,
mirrors=mirrors_to_check)
results = try_direct_fetch(spec, mirrors=mirrors_to_check)
# We found a spec by the direct fetch approach, we might as well
# add it to our mapping.
if results:
binary_index.update_spec(spec, results)
@ -1860,124 +1777,35 @@ def push_keys(*mirrors, **kwargs):
shutil.rmtree(tmpdir)
def needs_rebuild(spec, mirror_url, rebuild_on_errors=False):
def needs_rebuild(spec, mirror_url):
if not spec.concrete:
raise ValueError('spec must be concrete to check against mirror')
pkg_name = spec.name
pkg_version = spec.version
pkg_hash = spec.dag_hash()
pkg_full_hash = spec.full_hash()
tty.debug('Checking {0}-{1}, dag_hash = {2}, full_hash = {3}'.format(
pkg_name, pkg_version, pkg_hash, pkg_full_hash))
tty.debug('Checking {0}-{1}, dag_hash = {2}'.format(
pkg_name, pkg_version, pkg_hash))
tty.debug(spec.tree())
# Try to retrieve the specfile directly, based on the known
# format of the name, in order to determine if the package
# needs to be rebuilt.
cache_prefix = build_cache_prefix(mirror_url)
specfile_is_json = True
specfile_name = tarball_name(spec, '.spec.json')
deprecated_specfile_name = tarball_name(spec, '.spec.yaml')
specfile_path = os.path.join(cache_prefix, specfile_name)
deprecated_specfile_path = os.path.join(cache_prefix,
deprecated_specfile_name)
result_of_error = 'Package ({0}) will {1}be rebuilt'.format(
spec.short_spec, '' if rebuild_on_errors else 'not ')
try:
_, _, spec_file = web_util.read_from_url(specfile_path)
except (URLError, web_util.SpackWebError) as url_err:
try:
_, _, spec_file = web_util.read_from_url(deprecated_specfile_path)
specfile_is_json = False
except (URLError, web_util.SpackWebError) as url_err_y:
err_msg = [
'Unable to determine whether {0} needs rebuilding,',
' caught exception attempting to read from {1} or {2}.',
]
tty.error(''.join(err_msg).format(
spec.short_spec,
specfile_path,
deprecated_specfile_path))
tty.debug(url_err)
tty.debug(url_err_y)
tty.warn(result_of_error)
return rebuild_on_errors
spec_file_contents = codecs.getreader('utf-8')(spec_file).read()
if not spec_file_contents:
tty.error('Reading {0} returned nothing'.format(
specfile_path if specfile_is_json else deprecated_specfile_path))
tty.warn(result_of_error)
return rebuild_on_errors
spec_dict = (sjson.load(spec_file_contents)
if specfile_is_json else syaml.load(spec_file_contents))
try:
nodes = spec_dict['spec']['nodes']
except KeyError:
# Prior node dict format omitted 'nodes' key
nodes = spec_dict['spec']
name = spec.name
# In the old format:
# The "spec" key represents a list of objects, each with a single
# key that is the package name. While the list usually just contains
# a single object, we iterate over the list looking for the object
# with the name of this concrete spec as a key, out of an abundance
# of caution.
# In format version 2:
# ['spec']['nodes'] is still a list of objects, but with a
# multitude of keys. The list will commonly contain many objects, and in the
# case of build specs, it is highly likely that the same name will occur
# once as the actual package, and then again as the build provenance of that
# same package. Hence format version 2 matches on the dag hash, not name.
if nodes and 'name' not in nodes[0]:
# old style
cached_pkg_specs = [item[name] for item in nodes if name in item]
elif nodes and spec_dict['spec']['_meta']['version'] == 2:
cached_pkg_specs = [item for item in nodes
if item[ht.dag_hash.name] == spec.dag_hash()]
cached_target = cached_pkg_specs[0] if cached_pkg_specs else None
# If either the full_hash didn't exist in the specfile, or it
# did, but didn't match the one we computed locally, then we should
# just rebuild. This can be simplified once the dag_hash and the
# full_hash become the same thing.
rebuild = False
if not cached_target:
reason = 'did not find spec in specfile contents'
rebuild = True
elif ht.full_hash.name not in cached_target:
reason = 'full_hash was missing from remote specfile'
rebuild = True
else:
full_hash = cached_target[ht.full_hash.name]
if full_hash != pkg_full_hash:
reason = 'hash mismatch, remote = {0}, local = {1}'.format(
full_hash, pkg_full_hash)
rebuild = True
if rebuild:
tty.msg('Rebuilding {0}, reason: {1}'.format(
spec.short_spec, reason))
tty.msg(spec.tree())
return rebuild
# Only check for the presence of the json version of the spec. If the
# mirror only has the yaml version, or doesn't have the spec at all, we
# need to rebuild.
return not web_util.url_exists(specfile_path)
def check_specs_against_mirrors(mirrors, specs, output_file=None,
rebuild_on_errors=False):
def check_specs_against_mirrors(mirrors, specs, output_file=None):
"""Check all the given specs against buildcaches on the given mirrors and
determine if any of the specs need to be rebuilt. Reasons for needing to
rebuild include binary cache for spec isn't present on a mirror, or it is
present but the full_hash has changed since last time spec was built.
determine if any of the specs need to be rebuilt. Specs need to be rebuilt
when their hash doesn't exist in the mirror.
Arguments:
mirrors (dict): Mirrors to check against
@ -1985,8 +1813,6 @@ def check_specs_against_mirrors(mirrors, specs, output_file=None,
output_file (str): Path to output file to be written. If provided,
mirrors with missing or out-of-date specs will be formatted as a
JSON object and written to this file.
rebuild_on_errors (bool): Treat any errors encountered while
checking specs as a signal to rebuild package.
Returns: 1 if any spec was out-of-date on any mirror, 0 otherwise.
@ -1998,7 +1824,7 @@ def check_specs_against_mirrors(mirrors, specs, output_file=None,
rebuild_list = []
for spec in specs:
if needs_rebuild(spec, mirror.fetch_url, rebuild_on_errors):
if needs_rebuild(spec, mirror.fetch_url):
rebuild_list.append({
'short_spec': spec.short_spec,
'hash': spec.dag_hash()

14
lib/spack/spack/ci.py
View file

@ -399,7 +399,7 @@ def append_dep(s, d):
continue
up_to_date_mirrors = bindist.get_mirrors_for_spec(
spec=s, full_hash_match=True, index_only=check_index_only)
spec=s, index_only=check_index_only)
skey = spec_deps_key(s)
spec_labels[skey] = {
@ -801,7 +801,7 @@ def generate_gitlab_ci_yaml(env, print_summary, output_file,
max_needs_job = ''
# If this is configured, spack will fail "spack ci generate" if it
# generates any full hash which exists under the broken specs url.
# generates any hash which exists under the broken specs url.
broken_spec_urls = None
if broken_specs_url:
if broken_specs_url.startswith('http'):
@ -829,9 +829,8 @@ def generate_gitlab_ci_yaml(env, print_summary, output_file,
root_spec = spec_record['rootSpec']
pkg_name = pkg_name_from_spec_label(spec_label)
release_spec = root_spec[pkg_name]
release_spec_full_hash = release_spec.full_hash()
release_spec_dag_hash = release_spec.dag_hash()
release_spec_build_hash = release_spec.build_hash()
release_spec_runtime_hash = release_spec.runtime_hash()
if prune_untouched_packages:
if release_spec not in affected_specs:
@ -901,8 +900,7 @@ def generate_gitlab_ci_yaml(env, print_summary, output_file,
'SPACK_ROOT_SPEC': format_root_spec(
root_spec, main_phase, strip_compilers),
'SPACK_JOB_SPEC_DAG_HASH': release_spec_dag_hash,
'SPACK_JOB_SPEC_BUILD_HASH': release_spec_build_hash,
'SPACK_JOB_SPEC_FULL_HASH': release_spec_full_hash,
'SPACK_JOB_SPEC_RUNTIME_HASH': release_spec_runtime_hash,
'SPACK_JOB_SPEC_PKG_NAME': release_spec.name,
'SPACK_COMPILER_ACTION': compiler_action
}
@ -1006,9 +1004,9 @@ def generate_gitlab_ci_yaml(env, print_summary, output_file,
continue
if (broken_spec_urls is not None and
release_spec_full_hash in broken_spec_urls):
release_spec_dag_hash in broken_spec_urls):
known_broken_specs_encountered.append('{0} ({1})'.format(
release_spec, release_spec_full_hash))
release_spec, release_spec_dag_hash))
if artifacts_root:
job_dependencies.append({

11
lib/spack/spack/cmd/buildcache.py
View file

@ -161,11 +161,6 @@ def setup_parser(subparser):
help=('Check single spec from json or yaml file instead of release ' +
'specs file'))
check.add_argument(
'--rebuild-on-error', default=False, action='store_true',
help="Default to rebuilding packages if errors are encountered " +
"during the process of checking whether rebuilding is needed")
check.set_defaults(func=check_fn)
# Download tarball and specfile
@ -361,7 +356,7 @@ def list_fn(args):
try:
specs = bindist.update_cache_and_get_specs()
except bindist.FetchCacheError as e:
tty.error(e)
tty.die(e)
if not args.allarch:
arch = spack.spec.Spec.default_arch()
@ -430,7 +425,7 @@ def check_fn(args):
sys.exit(0)
sys.exit(bindist.check_specs_against_mirrors(
configured_mirrors, specs, args.output_file, args.rebuild_on_error))
configured_mirrors, specs, args.output_file))
def download_fn(args):
@ -486,7 +481,7 @@ def save_specfile_fn(args):
else:
root_spec = Spec(args.root_spec)
root_spec.concretize()
root_spec_as_json = root_spec.to_json(hash=ht.build_hash)
root_spec_as_json = root_spec.to_json(hash=ht.dag_hash)
spec_format = 'yaml' if args.root_specfile.endswith('yaml') else 'json'
save_dependency_specfiles(
root_spec_as_json, args.specfile_dir, args.specs.split(), spec_format)

57
lib/spack/spack/cmd/ci.py
View file

@ -167,8 +167,7 @@ def ci_reindex(args):
def ci_rebuild(args):
"""Check a single spec against the remote mirror, and rebuild it from
source if the mirror does not contain the full hash match of the spec
as computed locally. """
source if the mirror does not contain the hash. """
env = spack.cmd.require_active_env(cmd_name='ci rebuild')
# Make sure the environment is "gitlab-enabled", or else there's nothing
@ -280,8 +279,8 @@ def ci_rebuild(args):
env, root_spec, job_spec_pkg_name, compiler_action)
job_spec = spec_map[job_spec_pkg_name]
job_spec_yaml_file = '{0}.yaml'.format(job_spec_pkg_name)
job_spec_yaml_path = os.path.join(repro_dir, job_spec_yaml_file)
job_spec_json_file = '{0}.json'.format(job_spec_pkg_name)
job_spec_json_path = os.path.join(repro_dir, job_spec_json_file)
# To provide logs, cdash reports, etc for developer download/perusal,
# these things have to be put into artifacts. This means downstream
@ -335,23 +334,23 @@ def ci_rebuild(args):
# using a compiler already installed on the target system).
spack_ci.configure_compilers(compiler_action)
# Write this job's spec yaml into the reproduction directory, and it will
# Write this job's spec json into the reproduction directory, and it will
# also be used in the generated "spack install" command to install the spec
tty.debug('job concrete spec path: {0}'.format(job_spec_yaml_path))
with open(job_spec_yaml_path, 'w') as fd:
fd.write(job_spec.to_yaml(hash=ht.build_hash))
tty.debug('job concrete spec path: {0}'.format(job_spec_json_path))
with open(job_spec_json_path, 'w') as fd:
fd.write(job_spec.to_json(hash=ht.dag_hash))
# Write the concrete root spec yaml into the reproduction directory
root_spec_yaml_path = os.path.join(repro_dir, 'root.yaml')
with open(root_spec_yaml_path, 'w') as fd:
fd.write(spec_map['root'].to_yaml(hash=ht.build_hash))
# Write the concrete root spec json into the reproduction directory
root_spec_json_path = os.path.join(repro_dir, 'root.json')
with open(root_spec_json_path, 'w') as fd:
fd.write(spec_map['root'].to_json(hash=ht.dag_hash))
# Write some other details to aid in reproduction into an artifact
repro_file = os.path.join(repro_dir, 'repro.json')
repro_details = {
'job_name': ci_job_name,
'job_spec_yaml': job_spec_yaml_file,
'root_spec_yaml': 'root.yaml',
'job_spec_json': job_spec_json_file,
'root_spec_json': 'root.json',
'ci_project_dir': ci_project_dir
}
with open(repro_file, 'w') as fd:
@ -366,25 +365,24 @@ def ci_rebuild(args):
fd.write(b'\n')
# If we decided there should be a temporary storage mechanism, add that
# mirror now so it's used when we check for a full hash match already
# mirror now so it's used when we check for a hash match already
# built for this spec.
if pipeline_mirror_url:
spack.mirror.add(spack_ci.TEMP_STORAGE_MIRROR_NAME,
pipeline_mirror_url,
cfg.default_modify_scope())
# Check configured mirrors for a built spec with a matching full hash
matches = bindist.get_mirrors_for_spec(
job_spec, full_hash_match=True, index_only=False)
# Check configured mirrors for a built spec with a matching hash
matches = bindist.get_mirrors_for_spec(job_spec, index_only=False)
if matches:
# Got a full hash match on at least one configured mirror. All
# Got a hash match on at least one configured mirror. All
# matches represent the fully up-to-date spec, so should all be
# equivalent. If artifacts mirror is enabled, we just pick one
# of the matches and download the buildcache files from there to
# the artifacts, so they're available to be used by dependent
# jobs in subsequent stages.
tty.msg('No need to rebuild {0}, found full hash match at: '.format(
tty.msg('No need to rebuild {0}, found hash match at: '.format(
job_spec_pkg_name))
for match in matches:
tty.msg(' {0}'.format(match['mirror_url']))
@ -403,7 +401,7 @@ def ci_rebuild(args):
# Now we are done and successful
sys.exit(0)
# No full hash match anywhere means we need to rebuild spec
# No hash match anywhere means we need to rebuild spec
# Start with spack arguments
install_args = [base_arg for base_arg in CI_REBUILD_INSTALL_BASE_ARGS]
@ -415,7 +413,6 @@ def ci_rebuild(args):
install_args.extend([
'install',
'--keep-stage',
'--require-full-hash-match',
])
can_verify = spack_ci.can_verify_binaries()
@ -443,8 +440,8 @@ def ci_rebuild(args):
# TODO: once we have the concrete spec registry, use the DAG hash
# to identify the spec to install, rather than the concrete spec
# yaml file.
install_args.extend(['-f', job_spec_yaml_path])
# json file.
install_args.extend(['-f', job_spec_json_path])
tty.debug('Installing {0} from source'.format(job_spec.name))
tty.debug('spack install arguments: {0}'.format(
@ -477,13 +474,13 @@ def ci_rebuild(args):
tty.debug('spack install exited {0}'.format(install_exit_code))
# If a spec fails to build in a spack develop pipeline, we add it to a
# list of known broken full hashes. This allows spack PR pipelines to
# list of known broken hashes. This allows spack PR pipelines to
# avoid wasting compute cycles attempting to build those hashes.
if install_exit_code == INSTALL_FAIL_CODE and spack_is_develop_pipeline:
tty.debug('Install failed on develop')
if 'broken-specs-url' in gitlab_ci:
broken_specs_url = gitlab_ci['broken-specs-url']
dev_fail_hash = job_spec.full_hash()
dev_fail_hash = job_spec.dag_hash()
broken_spec_path = url_util.join(broken_specs_url, dev_fail_hash)
tty.msg('Reporting broken develop build as: {0}'.format(
broken_spec_path))
@ -494,7 +491,7 @@ def ci_rebuild(args):
'broken-spec': {
'job-url': get_env_var('CI_JOB_URL'),
'pipeline-url': get_env_var('CI_PIPELINE_URL'),
'concrete-spec-yaml': job_spec.to_dict(hash=ht.full_hash)
'concrete-spec-dict': job_spec.to_dict(hash=ht.dag_hash)
}
}
@ -539,7 +536,7 @@ def ci_rebuild(args):
# per-PR mirror, if this is a PR pipeline
if buildcache_mirror_url:
spack_ci.push_mirror_contents(
env, job_spec_yaml_path, buildcache_mirror_url, sign_binaries
env, job_spec_json_path, buildcache_mirror_url, sign_binaries
)
# Create another copy of that buildcache in the per-pipeline
@ -548,14 +545,14 @@ def ci_rebuild(args):
# prefix is set)
if pipeline_mirror_url:
spack_ci.push_mirror_contents(
env, job_spec_yaml_path, pipeline_mirror_url, sign_binaries
env, job_spec_json_path, pipeline_mirror_url, sign_binaries
)
# If this is a develop pipeline, check if the spec that we just built is
# on the broken-specs list. If so, remove it.
if spack_is_develop_pipeline and 'broken-specs-url' in gitlab_ci:
broken_specs_url = gitlab_ci['broken-specs-url']
just_built_hash = job_spec.full_hash()
just_built_hash = job_spec.dag_hash()
broken_spec_path = url_util.join(broken_specs_url, just_built_hash)
if web_util.url_exists(broken_spec_path):
tty.msg('Removing {0} from the list of broken specs'.format(

8
lib/spack/spack/cmd/install.py
View file

@ -47,7 +47,6 @@ def update_kwargs_from_args(args, kwargs):
'explicit': True, # Always true for install command
'stop_at': args.until,
'unsigned': args.unsigned,
'full_hash_match': args.full_hash_match,
})
kwargs.update({
@ -117,11 +116,6 @@ def setup_parser(subparser):
'--no-check-signature', action='store_true',
dest='unsigned', default=False,
help="do not check signatures of binary packages")
subparser.add_argument(
'--require-full-hash-match', action='store_true',
dest='full_hash_match', default=False, help="""when installing from
binary mirrors, do not install binary package unless the full hash of the
remote spec matches that of the local spec""")
subparser.add_argument(
'--show-log-on-error', action='store_true',
help="print full build log to stderr if build fails")
@ -470,7 +464,7 @@ def get_tests(specs):
})
# If we are using the monitor, we send configs. and create build
# The full_hash is the main package id, the build_hash for others
# The dag_hash is the main package id
if args.use_monitor and specs:
monitor.new_configuration(specs)
install_specs(args, kwargs, zip(abstract_specs, specs))

4
lib/spack/spack/cmd/solve.py
View file

@ -151,9 +151,9 @@ def solve(parser, args):
# With -y, just print YAML to output.
if args.format == 'yaml':
# use write because to_yaml already has a newline.
sys.stdout.write(spec.to_yaml(hash=ht.build_hash))
sys.stdout.write(spec.to_yaml(hash=ht.dag_hash))
elif args.format == 'json':
sys.stdout.write(spec.to_json(hash=ht.build_hash))
sys.stdout.write(spec.to_json(hash=ht.dag_hash))
else:
sys.stdout.write(
spec.tree(color=sys.stdout.isatty(), **kwargs))

4
lib/spack/spack/cmd/spec.py
View file

@ -52,8 +52,8 @@ def setup_parser(subparser):
'-N', '--namespaces', action='store_true', default=False,
help='show fully qualified package names')
subparser.add_argument(
'--hash-type', default="build_hash",
choices=['build_hash', 'full_hash', 'dag_hash'],
'--hash-type', default="dag_hash",
choices=['runtime_hash', 'dag_hash'],
help='generate spec with a particular hash type.')
subparser.add_argument(
'-t', '--types', action='store_true', default=False,

5
lib/spack/spack/cmd/test.py
View file

@ -332,13 +332,18 @@ def _report_suite_results(test_suite, args, constraints):
.format(results_desc, test_suite.name, matching))
results = {}
tty.msg('test results')
with open(test_suite.results_file, 'r') as f:
for line in f:
pkg_id, status = line.split()
results[pkg_id] = status
tty.msg(' {0}'.format(pkg_id))
tty.msg('test specs:')
failed, skipped, untested = 0, 0, 0
for pkg_id in test_specs:
tty.msg(' {0}'.format(pkg_id))
if pkg_id in results:
status = results[pkg_id]
if status == 'FAILED':

15
lib/spack/spack/database.py
View file

@ -433,7 +433,7 @@ def _failed_spec_path(self, spec):
.format(spec.name))
return os.path.join(self._failure_dir,
'{0}-{1}'.format(spec.name, spec.full_hash()))
'{0}-{1}'.format(spec.name, spec.dag_hash()))
def clear_all_failures(self):
"""Force remove install failure tracking files."""
@ -645,8 +645,12 @@ def _write_to_file(self, stream):
# TODO: fix this before we support multiple install locations.
database = {
'database': {
# TODO: move this to a top-level _meta section if we ever
# TODO: bump the DB version to 7
'version': str(_db_version),
# dictionary of installation records, keyed by DAG hash
'installs': installs,
'version': str(_db_version)
}
}
@ -1092,6 +1096,8 @@ def _add(
"Specs added to DB must be concrete.")
key = spec.dag_hash()
spec_run_hash = spec._runtime_hash
spec_pkg_hash = spec._package_hash
upstream, record = self.query_by_spec_hash(key)
if upstream:
return
@ -1153,10 +1159,11 @@ def _add(
record.ref_count += 1
# Mark concrete once everything is built, and preserve
# the original hash of concrete specs.
# the original hashes of concrete specs.
new_spec._mark_concrete()
new_spec._hash = key
new_spec._full_hash = spec._full_hash
new_spec._runtime_hash = spec_run_hash
new_spec._package_hash = spec_pkg_hash
else:
# It is already in the database

10
lib/spack/spack/directory_layout.py
View file

@ -110,13 +110,9 @@ def write_spec(self, spec, path):
"""Write a spec out to a file."""
_check_concrete(spec)
with open(path, 'w') as f:
# The hash the the projection is the DAG hash but we write out the
# full provenance by full hash so it's availabe if we want it later
# extension = os.path.splitext(path)[-1].lower()
# if 'json' in extension:
spec.to_json(f, hash=ht.full_hash)
# elif 'yaml' in extension:
# spec.to_yaml(f, hash=ht.full_hash)
# The hash of the projection is the DAG hash which contains
# the full provenance, so it's availabe if we want it later
spec.to_json(f, hash=ht.dag_hash)
def write_host_environment(self, spec):
"""The host environment is a json file with os, kernel, and spack

17
lib/spack/spack/environment/environment.py
View file

@ -1819,6 +1819,9 @@ def _read_lockfile_dict(self, d):
json_specs_by_hash = d['concrete_specs']
root_hashes = set(self.concretized_order)
import pdb
pdb.set_trace()
specs_by_hash = {}
for dag_hash, node_dict in json_specs_by_hash.items():
spec = Spec.from_node_dict(node_dict)
@ -1856,12 +1859,20 @@ def _read_lockfile_dict(self, d):
# trees and binary mirrors, and as such, must be considered the
# permanent id of the spec.
dag_hash = spec.dag_hash() # == full_hash()
build_hash = spec.build_hash()
runtime_hash = spec.runtime_hash() # == old dag_hash()
if runtime_hash in root_hashes:
if dag_hash in root_hashes:
# This spec's dag hash (now computed with build deps and pkg
# hash) is in the keys found in the file, so we're looking at
# the current format
pass
elif runtime_hash in root_hashes:
# This spec's runtime hash (the old dag hash w/out build deps,
# etc) is a key in this lockfile, so this is the oldest format
old_hash_to_new[runtime_hash] = dag_hash
elif build_hash in root_hashes:
else:
# Neither of this spec's hashes appeared as a key in the lock
# file, so
old_hash_to_new[build_hash] = dag_hash
if (runtime_hash in root_hashes or

23
lib/spack/spack/hash_types.py
View file

@ -39,16 +39,6 @@ def attr(self):
deptype=('build', 'link', 'run'), package_hash=True, name='hash')
#: Same as dag_hash; old name.
full_hash = SpecHashDescriptor(
deptype=('build', 'link', 'run'), package_hash=True, name='full_hash')
#: Hash descriptor that includes build dependencies.
build_hash = SpecHashDescriptor(
deptype=('build', 'link', 'run'), package_hash=False, name='build_hash')
#: Hash descriptor used only to transfer a DAG, as is, across processes
process_hash = SpecHashDescriptor(
deptype=('build', 'link', 'run', 'test'),
@ -57,7 +47,18 @@ def attr(self):
)
#: Package hash used as part of full hash
#: Package hash used as part of dag hash
package_hash = SpecHashDescriptor(
deptype=(), package_hash=True, name='package_hash',
override=lambda s: s.package.content_hash())
# Deprecated hash types, no longer used, but needed to understand old serialized
# spec formats
full_hash = SpecHashDescriptor(
deptype=('build', 'link', 'run'), package_hash=True, name='full_hash')
build_hash = SpecHashDescriptor(
deptype=('build', 'link', 'run'), package_hash=False, name='build_hash')

16
lib/spack/spack/installer.py
View file

@ -260,8 +260,7 @@ def _hms(seconds):
return ' '.join(parts)
def _install_from_cache(pkg, cache_only, explicit, unsigned=False,
full_hash_match=False):
def _install_from_cache(pkg, cache_only, explicit, unsigned=False):
"""
Extract the package from binary cache
@ -278,7 +277,7 @@ def _install_from_cache(pkg, cache_only, explicit, unsigned=False,
``False`` otherwise
"""
installed_from_cache = _try_install_from_binary_cache(
pkg, explicit, unsigned=unsigned, full_hash_match=full_hash_match)
pkg, explicit, unsigned=unsigned)
pkg_id = package_id(pkg)
if not installed_from_cache:
pre = 'No binary for {0} found'.format(pkg_id)
@ -390,8 +389,7 @@ def _process_binary_cache_tarball(pkg, binary_spec, explicit, unsigned,
return True
def _try_install_from_binary_cache(pkg, explicit, unsigned=False,
full_hash_match=False):
def _try_install_from_binary_cache(pkg, explicit, unsigned=False):
"""
Try to extract the package from binary cache.
@ -403,8 +401,7 @@ def _try_install_from_binary_cache(pkg, explicit, unsigned=False,
"""
pkg_id = package_id(pkg)
tty.debug('Searching for binary cache of {0}'.format(pkg_id))
matches = binary_distribution.get_mirrors_for_spec(
pkg.spec, full_hash_match=full_hash_match)
matches = binary_distribution.get_mirrors_for_spec(pkg.spec)
if not matches:
return False
@ -1204,7 +1201,6 @@ def _install_task(self, task):
install_args = task.request.install_args
cache_only = install_args.get('cache_only')
explicit = task.explicit
full_hash_match = install_args.get('full_hash_match')
tests = install_args.get('tests')
unsigned = install_args.get('unsigned')
use_cache = install_args.get('use_cache')
@ -1217,8 +1213,7 @@ def _install_task(self, task):
# Use the binary cache if requested
if use_cache and \
_install_from_cache(pkg, cache_only, explicit, unsigned,
full_hash_match):
_install_from_cache(pkg, cache_only, explicit, unsigned):
self._update_installed(task)
if task.compiler:
spack.compilers.add_compilers_to_config(
@ -2306,7 +2301,6 @@ def _add_default_args(self):
('dirty', False),
('fail_fast', False),
('fake', False),
('full_hash_match', False),
('install_deps', True),
('install_package', True),
('install_source', False),

26
lib/spack/spack/monitor.py
View file

@ -132,7 +132,7 @@ def __init__(self, host=None, prefix="ms1", allow_fail=False, tags=None,
self.tags = tags
self.save_local = save_local
# We keey lookup of build_id by full_hash
# We keey lookup of build_id by dag_hash
self.build_ids = {}
self.setup_save()
@ -412,7 +412,7 @@ def new_configuration(self, specs):
spec.concretize()
# Remove extra level of nesting
as_dict = {"spec": spec.to_dict(hash=ht.full_hash)['spec'],
as_dict = {"spec": spec.to_dict(hash=ht.dag_hash)['spec'],
"spack_version": self.spack_version}
if self.save_local:
@ -437,7 +437,7 @@ def failed_concretization(self, specs):
meta = spec.to_dict()['spec']
nodes = []
for node in meta.get("nodes", []):
for hashtype in ["build_hash", "full_hash"]:
for hashtype in ["hash", "runtime_hash"]:
node[hashtype] = "FAILED_CONCRETIZATION"
nodes.append(node)
meta['nodes'] = nodes
@ -470,13 +470,13 @@ def get_build_id(self, spec, return_response=False, spec_exists=True):
"""
Retrieve a build id, either in the local cache, or query the server.
"""
full_hash = spec.full_hash()
if full_hash in self.build_ids:
return self.build_ids[full_hash]
dag_hash = spec.dag_hash()
if dag_hash in self.build_ids:
return self.build_ids[dag_hash]
# Prepare build environment data (including spack version)
data = self.build_environment.copy()
data['full_hash'] = full_hash
data['hash'] = dag_hash
# If the build should be tagged, add it
if self.tags:
@ -494,10 +494,10 @@ def get_build_id(self, spec, return_response=False, spec_exists=True):
data['spec'] = syaml.load(read_file(spec_file))
if self.save_local:
return self.get_local_build_id(data, full_hash, return_response)
return self.get_server_build_id(data, full_hash, return_response)
return self.get_local_build_id(data, dag_hash, return_response)
return self.get_server_build_id(data, dag_hash, return_response)
def get_local_build_id(self, data, full_hash, return_response):
def get_local_build_id(self, data, dag_hash, return_response):
"""
Generate a local build id based on hashing the expected data
"""
@ -510,15 +510,15 @@ def get_local_build_id(self, data, full_hash, return_response):
return response
return bid
def get_server_build_id(self, data, full_hash, return_response=False):
def get_server_build_id(self, data, dag_hash, return_response=False):
"""
Retrieve a build id from the spack monitor server
"""
response = self.do_request("builds/new/", data=sjson.dump(data))
# Add the build id to the lookup
bid = self.build_ids[full_hash] = response['data']['build']['build_id']
self.build_ids[full_hash] = bid
bid = self.build_ids[dag_hash] = response['data']['build']['build_id']
self.build_ids[dag_hash] = bid
# If the function is called directly, the user might want output
if return_response:

3
lib/spack/spack/schema/spec.py
View file

@ -110,8 +110,7 @@
'properties': {
'name': {'type': 'string'},
'hash': {'type': 'string'},
'full_hash': {'type': 'string'},
'build_hash': {'type': 'string'},
'runtime_hash': {'type': 'string'},
'package_hash': {'type': 'string'},
'version': {
'oneOf': [

108
lib/spack/spack/spec.py
View file

@ -1192,8 +1192,7 @@ def __init__(self, spec_like=None, normal=False,
self.namespace = None
self._hash = None
self._build_hash = None
self._full_hash = None
self._runtime_hash = None
self._package_hash = None
self._dunder_hash = None
self._package = None
@ -1210,12 +1209,12 @@ def __init__(self, spec_like=None, normal=False,
self.external_path = external_path
self.external_modules = Spec._format_module_list(external_modules)
# Older spack versions did not compute full_hash or build_hash,
# and we may not have the necessary information to recompute them
# if we read in old specs. Old concrete specs are marked "final"
# when read in to indicate that we shouldn't recompute full_hash
# or build_hash. New specs are not final; we can lazily compute
# their hashes.
# Older spack versions may have either computed different hashes or
# computed them differently, and we may not have the necessary
# information to recompute them if we read in old specs.
# Old concrete specs are marked "final" when read in to indicate
# that we shouldn't recompute the current dag_hash. New specs are
# not final; we can lazily compute their hashes.
self._hashes_final = False
# This attribute is used to store custom information for
@ -1804,27 +1803,13 @@ def dag_hash(self, length=None):
"""This is Spack's default hash, used to identify installations.
Same as the full hash (includes package hash and build/link/run deps).
Tells us when package files and any dependencies have changes.
NOTE: Versions of Spack prior to 0.18 only included link and run deps.
"""
return self._cached_hash(ht.dag_hash, length)
def full_hash(self, length=None):
"""Hash that includes all build and run inputs for a spec.
Inputs are: the package hash (to identify the package.py),
build, link, and run dependencies.
"""
return self._cached_hash(ht.full_hash, length)
def build_hash(self, length=None):
"""Hash used to store specs in environments.
This hash includes build dependencies, and we need to preserve
them to be able to rebuild an entire environment for a user.
"""
return self._cached_hash(ht.build_hash, length)
def process_hash(self, length=None):
"""Hash used to transfer specs among processes.
@ -1833,6 +1818,7 @@ def process_hash(self, length=None):
"""
return self._cached_hash(ht.process_hash, length)
def dag_hash_bit_prefix(self, bits):
"""Get the first <bits> bits of the DAG hash as an integer type."""
return spack.util.hash.base32_prefix_bits(self.dag_hash(), bits)
@ -2002,7 +1988,7 @@ def to_dict(self, hash=ht.dag_hash):
"dependencies": [
{
"name": "readline",
"build_hash": "4f47cggum7p4qmp3xna4hi547o66unva",
"hash": "4f47cggum7p4qmp3xna4hi547o66unva",
"type": [
"build",
"link"
@ -2010,16 +1996,15 @@ def to_dict(self, hash=ht.dag_hash):
},
{
"name": "zlib",
"build_hash": "uvgh6p7rhll4kexqnr47bvqxb3t33jtq",
"hash": "uvgh6p7rhll4kexqnr47bvqxb3t33jtq",
"type": [
"build",
"link"
]
}
],
"hash": "d2yzqp2highd7sn4nr5ndkw3ydcrlhtk",
"full_hash": "tve45xfqkfgmzwcyfetze2z6syrg7eaf",
"build_hash": "tsjnz7lgob7bu2wd4sqzzjenxewc2zha"
"runtime_hash": "d2yzqp2highd7sn4nr5ndkw3ydcrlhtk",
"hash": "tve45xfqkfgmzwcyfetze2z6syrg7eaf",
},
# ... more node dicts for readline and its dependencies ...
]
@ -2071,37 +2056,29 @@ def node_dict_with_hashes(self, hash=ht.dag_hash):
node = self.to_node_dict(hash)
node[ht.dag_hash.name] = self.dag_hash()
# full_hash and build_hash are lazily computed -- but if we write
# a spec out, we want them to be included. This is effectively
# the last chance we get to compute them accurately.
# dag_hash is lazily computed -- but if we write a spec out, we want it
# to be included. This is effectively the last chance we get to compute
# it accurately.
if self.concrete:
# build and full hashes can be written out if:
# 1. they're precomputed (i.e. we read them from somewhere
# and they were already on the spec
# 2. we can still compute them lazily (i.e. we just made them and
# dag_hash can be written out if:
# 1. it's precomputed (i.e. we read it from somewhere
# and it was already on the spec)
# 2. we can still compute it lazily (i.e. we just made the spec and
# have the full dependency graph on-hand)
#
# we want to avoid recomputing either hash for specs we read
# we want to avoid recomputing the dag_hash for specs we read
# in from the DB or elsewhere, as we may not have the info
# (like patches, package versions, etc.) that we need to
# compute them. Unknown hashes are better than wrong hashes.
write_full_hash = (
self._hashes_final and self._full_hash or # cached and final
# compute it. Unknown hashes are better than wrong hashes.
write_dag_hash = (
self._hashes_final and self._hash or # cached and final
not self._hashes_final) # lazily compute
if write_full_hash:
node[ht.full_hash.name] = self.full_hash()
write_build_hash = 'build' in hash.deptype and (
self._hashes_final and self._build_hash or # cached and final
not self._hashes_final) # lazily compute
if write_build_hash:
node[ht.build_hash.name] = self.build_hash()
if write_dag_hash:
node[ht.dag_hash.name] = self.dag_hash()
else:
node['concrete'] = False
if hash.name == 'build_hash':
node[hash.name] = self.build_hash()
elif hash.name == 'process_hash':
if hash.name == 'process_hash':
node[hash.name] = self.process_hash()
elif hash.name == 'runtime_hash':
node[hash.name] = self.runtime_hash()
@ -2187,7 +2164,7 @@ def from_node_dict(node):
# this spec may have been built with older packages than we have
# on-hand, and we may not have the build dependencies, so mark it
# so we don't recompute full_hash and build_hash.
# so we don't recompute dag_hash.
spec._hashes_final = spec._concrete
if 'patches' in node:
@ -2200,7 +2177,7 @@ def from_node_dict(node):
# FIXME: Monkey patches mvar to store patches order
mvar._patches_in_order_of_appearance = patches
# Don't read dependencies here; from_node_dict() is used by
# Don't read dependencies here; from_dict() is used by
# from_yaml() and from_json() to read the root *and* each dependency
# spec.
@ -2227,7 +2204,6 @@ def dependencies_from_node_dict(node):
@staticmethod
def read_yaml_dep_specs(deps, hash_type=ht.dag_hash.name):
"""Read the DependencySpec portion of a YAML-formatted Spec.
This needs to be backward-compatible with older spack spec
formats so that reindex will work on old specs/databases.
"""
@ -2247,16 +2223,11 @@ def read_yaml_dep_specs(deps, hash_type=ht.dag_hash.name):
elif isinstance(elt, dict):
# new format: elements of dependency spec are keyed.
for key in (ht.dag_hash.name,
ht.full_hash.name,
ht.build_hash.name,
ht.full_hash.name,
ht.runtime_hash.name,
ht.process_hash.name):
if key in elt:
# FIXME: if the key is 'hash' it could mean the old
# dag hash without build deps, or the new dag hash which
# is equivalent to the full hash. If this was the old
# dag hash, we need to keep the hash value but set the
# key hash type to "runtime_hash".
dep_hash, deptypes = elt[key], elt['type']
hash_type = key
break
@ -3797,20 +3768,17 @@ def _dup(self, other, deps=True, cleardeps=True):
if self._concrete:
self._hash = other._hash
self._build_hash = other._build_hash
self._dunder_hash = other._dunder_hash
self._normal = True
self._normal = other._normal
self._full_hash = other._full_hash
self._runtime_hash = other._runtime_hash
self._package_hash = other._package_hash
else:
self._hash = None
self._build_hash = None
self._dunder_hash = None
# Note, we could use other._normal if we are copying all deps, but
# always set it False here to avoid the complexity of checking
self._normal = False
self._full_hash = None
self._runtime_hash = None
self._package_hash = None
@ -4751,12 +4719,9 @@ def from_self(name, transitive):
for dep in ret.traverse(root=True, order='post'):
opposite = other_nodes if dep.name in self_nodes else self_nodes
if any(name in dep for name in opposite.keys()):
# package hash cannot be affected by splice
dep.clear_cached_hashes(ignore=['package_hash'])
dep.build_hash()
dep.full_hash()
dep.dag_hash()
return nodes[self.name]
@ -4844,7 +4809,7 @@ def _spec_from_old_dict(data):
if 'dependencies' not in node[name]:
continue
for dname, dhash, dtypes, _ in Spec.dependencies_from_node_dict(node):
for dname, _, dtypes, _ in Spec.dependencies_from_node_dict(node):
deps[name]._add_dependency(deps[dname], dtypes)
return spec
@ -4878,7 +4843,7 @@ def _spec_from_dict(data):
break
if not any_deps: # If we never see a dependency...
hash_type = ht.dag_hash.name # use the full_hash provenance
hash_type = ht.dag_hash.name
elif not hash_type: # Seen a dependency, still don't know hash_type
raise spack.error.SpecError("Spec dictionary contains malformed "
"dependencies. Old format?")
@ -4888,9 +4853,6 @@ def _spec_from_dict(data):
# Pass 1: Create a single lookup dictionary by hash
for i, node in enumerate(nodes):
if 'build_spec' in node.keys():
node_hash = node[hash_type]
else:
node_hash = node[hash_type]
node_spec = Spec.from_node_dict(node)
hash_dict[node_hash] = node
@ -5350,7 +5312,7 @@ def save_dependency_specfiles(
json_path = os.path.join(output_directory, '{0}.json'.format(dep_name))
with open(json_path, 'w') as fd:
fd.write(dep_spec.to_json(hash=ht.build_hash))
fd.write(dep_spec.to_json(hash=ht.dag_hash))
class SpecParseError(spack.error.SpecError):

82
lib/spack/spack/test/bindist.py
View file

@ -394,31 +394,12 @@ def test_built_spec_cache(mirror_dir):
gspec, cspec = Spec('garply').concretized(), Spec('corge').concretized()
full_hash_map = {
'garply': gspec.full_hash(),
'corge': cspec.full_hash(),
}
gspec_results = bindist.get_mirrors_for_spec(gspec)
gspec_mirrors = {}
for result in gspec_results:
s = result['spec']
assert(s._full_hash == full_hash_map[s.name])
assert(result['mirror_url'] not in gspec_mirrors)
gspec_mirrors[result['mirror_url']] = True
cspec_results = bindist.get_mirrors_for_spec(cspec, full_hash_match=True)
cspec_mirrors = {}
for result in cspec_results:
s = result['spec']
assert(s._full_hash == full_hash_map[s.name])
assert(result['mirror_url'] not in cspec_mirrors)
cspec_mirrors[result['mirror_url']] = True
for s in [gspec, cspec]:
results = bindist.get_mirrors_for_spec(s)
assert(any([r['spec'] == s for r in results]))
def fake_full_hash(spec):
def fake_dag_hash(spec):
# Generate an arbitrary hash that is intended to be different than
# whatever a Spec reported before (to test actions that trigger when
# the hash changes)
@ -430,7 +411,7 @@ def fake_full_hash(spec):
'test_mirror'
)
def test_spec_needs_rebuild(monkeypatch, tmpdir):
"""Make sure needs_rebuild properly compares remote full_hash
"""Make sure needs_rebuild properly compares remote hash
against locally computed one, avoiding unnecessary rebuilds"""
# Create a temp mirror directory for buildcache usage
@ -450,7 +431,7 @@ def test_spec_needs_rebuild(monkeypatch, tmpdir):
assert not rebuild
# Now monkey patch Spec to change the full hash on the package
monkeypatch.setattr(spack.spec.Spec, 'full_hash', fake_full_hash)
monkeypatch.setattr(spack.spec.Spec, 'dag_hash', fake_dag_hash)
rebuild = bindist.needs_rebuild(s, mirror_url, rebuild_on_errors=True)
@ -624,57 +605,6 @@ def test_install_legacy_yaml(test_legacy_mirror, install_mockery_mutable_config,
uninstall_cmd('-y', '/t5mczux3tfqpxwmg7egp7axy2jvyulqk')
@pytest.mark.usefixtures(
'install_mockery_mutable_config', 'mock_packages', 'mock_fetch',
)
def test_update_index_fix_deps(monkeypatch, tmpdir, mutable_config):
"""Ensure spack buildcache update-index properly fixes up spec descriptor
files on the mirror when updating the buildcache index."""
# Create a temp mirror directory for buildcache usage
mirror_dir = tmpdir.join('mirror_dir')
mirror_url = 'file://{0}'.format(mirror_dir.strpath)
spack.config.set('mirrors', {'test': mirror_url})
a = Spec('a').concretized()
b = Spec('b').concretized()
new_b_full_hash = 'abcdef'
# Install package a with dep b
install_cmd('--no-cache', a.name)
# Create a buildcache for a and its dep b, and update index
buildcache_cmd('create', '-uad', mirror_dir.strpath, a.name)
buildcache_cmd('update-index', '-d', mirror_dir.strpath)
# Simulate an update to b that only affects full hash by simply overwriting
# the full hash in the spec.json file on the mirror
b_spec_json_name = bindist.tarball_name(b, '.spec.json')
b_spec_json_path = os.path.join(mirror_dir.strpath,
bindist.build_cache_relative_path(),
b_spec_json_name)
fs.filter_file(r'"full_hash":\s"\S+"',
'"full_hash": "{0}"'.format(new_b_full_hash),
b_spec_json_path)
# When we update the index, spack should notice that a's notion of the
# full hash of b doesn't match b's notion of it's own full hash, and as
# a result, spack should fix the spec.json for a
buildcache_cmd('update-index', '-d', mirror_dir.strpath)
# Read in the concrete spec json of a
a_spec_json_name = bindist.tarball_name(a, '.spec.json')
a_spec_json_path = os.path.join(mirror_dir.strpath,
bindist.build_cache_relative_path(),
a_spec_json_name)
# Turn concrete spec json into a concrete spec (a)
with open(a_spec_json_path) as fd:
a_prime = spec.Spec.from_json(fd.read())
# Make sure the full hash of b in a's spec json matches the new value
assert(a_prime[b.name].full_hash() == new_b_full_hash)
def test_FetchCacheError_only_accepts_lists_of_errors():
with pytest.raises(TypeError, match="list"):
bindist.FetchCacheError("error")

81
lib/spack/spack/test/cmd/ci.py
View file

@ -767,19 +767,13 @@ def test_ci_rebuild(tmpdir, mutable_mock_env_path,
shutil.copyfile(env.lock_path,
os.path.join(env_dir.strpath, 'spack.lock'))
root_spec_build_hash = None
job_spec_dag_hash = None
job_spec_full_hash = None
root_spec_dag_hash = None
for h, s in env.specs_by_hash.items():
if s.name == 'archive-files':
root_spec_build_hash = h
job_spec_dag_hash = s.dag_hash()
job_spec_full_hash = s.full_hash()
root_spec_dag_hash = h
assert root_spec_build_hash
assert job_spec_dag_hash
assert job_spec_full_hash
assert root_spec_dag_hash
def fake_cdash_register(build_name, base_url, project, site, track):
return ('fakebuildid', 'fakestamp')
@ -801,8 +795,8 @@ def fake_cdash_register(build_name, base_url, project, site, track):
'SPACK_CONCRETE_ENV_DIR': env_dir.strpath,
'CI_PIPELINE_ID': '7192',
'SPACK_SIGNING_KEY': signing_key,
'SPACK_ROOT_SPEC': root_spec_build_hash,
'SPACK_JOB_SPEC_DAG_HASH': job_spec_dag_hash,
'SPACK_ROOT_SPEC': root_spec_dag_hash,
'SPACK_JOB_SPEC_DAG_HASH': root_spec_dag_hash,
'SPACK_JOB_SPEC_PKG_NAME': 'archive-files',
'SPACK_COMPILER_ACTION': 'NONE',
'SPACK_CDASH_BUILD_NAME': '(specs) archive-files',
@ -816,8 +810,8 @@ def fake_cdash_register(build_name, base_url, project, site, track):
expected_repro_files = [
'install.sh',
'root.yaml',
'archive-files.yaml',
'root.json',
'archive-files.json',
'spack.yaml',
'spack.lock'
]
@ -839,14 +833,13 @@ def mystrip(s):
install_parts = [mystrip(s) for s in install_line.split(' ')]
assert('--keep-stage' in install_parts)
assert('--require-full-hash-match' in install_parts)
assert('--no-check-signature' not in install_parts)
assert('--no-add' in install_parts)
assert('-f' in install_parts)
flag_index = install_parts.index('-f')
assert('archive-files.yaml' in install_parts[flag_index + 1])
assert('archive-files.json' in install_parts[flag_index + 1])
broken_spec_file = os.path.join(broken_specs_path, job_spec_full_hash)
broken_spec_file = os.path.join(broken_specs_path, root_spec_dag_hash)
with open(broken_spec_file) as fd:
broken_spec_content = fd.read()
assert(ci_job_url in broken_spec_content)
@ -894,13 +887,11 @@ def test_ci_nothing_to_rebuild(tmpdir, mutable_mock_env_path,
env_cmd('create', 'test', './spack.yaml')
with ev.read('test') as env:
env.concretize()
root_spec_build_hash = None
job_spec_dag_hash = None
root_spec_dag_hash = None
for h, s in env.specs_by_hash.items():
if s.name == 'archive-files':
root_spec_build_hash = h
job_spec_dag_hash = s.dag_hash()
root_spec_dag_hash = h
# Create environment variables as gitlab would do it
os.environ.update({
@ -909,8 +900,8 @@ def test_ci_nothing_to_rebuild(tmpdir, mutable_mock_env_path,
'SPACK_JOB_REPRO_DIR': 'repro_dir',
'SPACK_LOCAL_MIRROR_DIR': mirror_dir.strpath,
'SPACK_CONCRETE_ENV_DIR': tmpdir.strpath,
'SPACK_ROOT_SPEC': root_spec_build_hash,
'SPACK_JOB_SPEC_DAG_HASH': job_spec_dag_hash,
'SPACK_ROOT_SPEC': root_spec_dag_hash,
'SPACK_JOB_SPEC_DAG_HASH': root_spec_dag_hash,
'SPACK_JOB_SPEC_PKG_NAME': 'archive-files',
'SPACK_COMPILER_ACTION': 'NONE',
'SPACK_REMOTE_MIRROR_URL': mirror_url,
@ -980,7 +971,7 @@ def test_push_mirror_contents(tmpdir, mutable_mock_env_path,
spec_map = ci.get_concrete_specs(
env, 'patchelf', 'patchelf', 'FIND_ANY')
concrete_spec = spec_map['patchelf']
spec_json = concrete_spec.to_json(hash=ht.build_hash)
spec_json = concrete_spec.to_json(hash=ht.dag_hash)
json_path = str(tmpdir.join('spec.json'))
with open(json_path, 'w') as ypfd:
ypfd.write(spec_json)
@ -1323,12 +1314,12 @@ def test_ci_rebuild_index(tmpdir, mutable_mock_env_path,
spec_map = ci.get_concrete_specs(
env, 'callpath', 'callpath', 'FIND_ANY')
concrete_spec = spec_map['callpath']
spec_yaml = concrete_spec.to_yaml(hash=ht.build_hash)
yaml_path = str(tmpdir.join('spec.yaml'))
with open(yaml_path, 'w') as ypfd:
ypfd.write(spec_yaml)
spec_json = concrete_spec.to_json(hash=ht.dag_hash)
json_path = str(tmpdir.join('spec.json'))
with open(json_path, 'w') as ypfd:
ypfd.write(spec_json)
install_cmd('--keep-stage', '-f', yaml_path)
install_cmd('--keep-stage', '-f', json_path)
buildcache_cmd('create', '-u', '-a', '-f', '--mirror-url',
mirror_url, 'callpath')
ci_cmd('rebuild-index')
@ -1412,8 +1403,8 @@ def test_ci_generate_bootstrap_prune_dag(
# nothing in the environment needs rebuilding. With the monkeypatch, the
# process sees the compiler as needing a rebuild, which should then result
# in the specs built with that compiler needing a rebuild too.
def fake_get_mirrors_for_spec(spec=None, full_hash_match=False,
mirrors_to_check=None, index_only=False):
def fake_get_mirrors_for_spec(spec=None, mirrors_to_check=None,
index_only=False):
if spec.name == 'gcc':
return []
else:
@ -1674,14 +1665,14 @@ def test_ci_generate_read_broken_specs_url(tmpdir, mutable_mock_env_path,
"""Verify that `broken-specs-url` works as intended"""
spec_a = Spec('a')
spec_a.concretize()
a_full_hash = spec_a.full_hash()
a_dag_hash = spec_a.dag_hash()
spec_flattendeps = Spec('flatten-deps')
spec_flattendeps.concretize()
flattendeps_full_hash = spec_flattendeps.full_hash()
flattendeps_dag_hash = spec_flattendeps.dag_hash()
# Mark 'a' as broken (but not 'flatten-deps')
broken_spec_a_path = str(tmpdir.join(a_full_hash))
broken_spec_a_path = str(tmpdir.join(a_dag_hash))
with open(broken_spec_a_path, 'w') as bsf:
bsf.write('')
@ -1718,10 +1709,10 @@ def test_ci_generate_read_broken_specs_url(tmpdir, mutable_mock_env_path,
output = ci_cmd('generate', output=str, fail_on_error=False)
assert('known to be broken' in output)
ex = '({0})'.format(a_full_hash)
ex = '({0})'.format(a_dag_hash)
assert(ex in output)
ex = '({0})'.format(flattendeps_full_hash)
ex = '({0})'.format(flattendeps_dag_hash)
assert(ex not in output)
@ -1776,15 +1767,15 @@ def test_ci_reproduce(tmpdir, mutable_mock_env_path,
root_spec = s
job_spec = s
job_spec_yaml_path = os.path.join(
working_dir.strpath, 'archivefiles.yaml')
with open(job_spec_yaml_path, 'w') as fd:
fd.write(job_spec.to_yaml(hash=ht.build_hash))
job_spec_json_path = os.path.join(
working_dir.strpath, 'archivefiles.json')
with open(job_spec_json_path, 'w') as fd:
fd.write(job_spec.to_json(hash=ht.dag_hash))
root_spec_yaml_path = os.path.join(
working_dir.strpath, 'root.yaml')
with open(root_spec_yaml_path, 'w') as fd:
fd.write(root_spec.to_yaml(hash=ht.build_hash))
root_spec_json_path = os.path.join(
working_dir.strpath, 'root.json')
with open(root_spec_json_path, 'w') as fd:
fd.write(root_spec.to_json(hash=ht.dag_hash))
artifacts_root = os.path.join(working_dir.strpath, 'scratch_dir')
pipeline_path = os.path.join(artifacts_root, 'pipeline.yml')
@ -1798,8 +1789,8 @@ def test_ci_reproduce(tmpdir, mutable_mock_env_path,
repro_file = os.path.join(working_dir.strpath, 'repro.json')
repro_details = {
'job_name': job_name,
'job_spec_yaml': 'archivefiles.yaml',
'root_spec_yaml': 'root.yaml',
'job_spec_json': 'archivefiles.json',
'root_spec_json': 'root.json',
'ci_project_dir': working_dir.strpath
}
with open(repro_file, 'w') as fd:

10
lib/spack/spack/test/cmd/env.py
View file

@ -982,7 +982,7 @@ def create_v1_lockfile_dict(roots, all_specs):
# Version one lockfiles use the dag hash without build deps as keys,
# but they write out the full node dict (including build deps)
"concrete_specs": dict(
(s.runtime_hash(), s.to_node_dict(hash=ht.build_hash))
(s.runtime_hash(), s.to_node_dict(hash=ht.dag_hash))
for s in all_specs
)
}
@ -2469,19 +2469,19 @@ def test_newline_in_commented_sequence_is_not_an_issue(tmpdir):
abspath = tmpdir.join('spack.yaml')
abspath.write(spack_yaml)
def extract_build_hash(environment):
def extract_dag_hash(environment):
_, dyninst = next(iter(environment.specs_by_hash.items()))
return dyninst['libelf'].build_hash()
return dyninst['libelf'].dag_hash()
# Concretize a first time and create a lockfile
with ev.Environment(str(tmpdir)) as e:
concretize()
libelf_first_hash = extract_build_hash(e)
libelf_first_hash = extract_dag_hash(e)
# Check that a second run won't error
with ev.Environment(str(tmpdir)) as e:
concretize()
libelf_second_hash = extract_build_hash(e)
libelf_second_hash = extract_dag_hash(e)
assert libelf_first_hash == libelf_second_hash

91
lib/spack/spack/test/cmd/install.py
View file

@ -7,7 +7,6 @@
import filecmp
import os
import re
import shutil
import sys
import time
@ -621,20 +620,20 @@ def test_cdash_buildstamp_param(tmpdir, mock_fetch, install_mockery, capfd):
@pytest.mark.disable_clean_stage_check
def test_cdash_install_from_spec_yaml(tmpdir, mock_fetch, install_mockery,
def test_cdash_install_from_spec_json(tmpdir, mock_fetch, install_mockery,
capfd, mock_packages, mock_archive,
config):
# capfd interferes with Spack's capturing
with capfd.disabled():
with tmpdir.as_cwd():
spec_yaml_path = str(tmpdir.join('spec.yaml'))
spec_json_path = str(tmpdir.join('spec.json'))
pkg_spec = Spec('a')
pkg_spec.concretize()
with open(spec_yaml_path, 'w') as fd:
fd.write(pkg_spec.to_yaml(hash=ht.build_hash))
with open(spec_json_path, 'w') as fd:
fd.write(pkg_spec.to_json(hash=ht.dag_hash))
install(
'--log-format=cdash',
@ -642,7 +641,7 @@ def test_cdash_install_from_spec_yaml(tmpdir, mock_fetch, install_mockery,
'--cdash-build=my_custom_build',
'--cdash-site=my_custom_site',
'--cdash-track=my_custom_track',
'-f', spec_yaml_path)
'-f', spec_json_path)
report_dir = tmpdir.join('cdash_reports')
assert report_dir in tmpdir.listdir()
@ -846,14 +845,14 @@ def test_install_no_add_in_env(tmpdir, mock_fetch, install_mockery,
post_install_specs = e.all_specs()
assert all([s in env_specs for s in post_install_specs])
# Make sure we can install a concrete dependency spec from a spec.yaml
# Make sure we can install a concrete dependency spec from a spec.json
# file on disk, using the ``--no-add` option, and the spec is installed
# but not added as a root
mpi_spec_yaml_path = tmpdir.join('{0}.yaml'.format(mpi_spec.name))
with open(mpi_spec_yaml_path.strpath, 'w') as fd:
fd.write(mpi_spec.to_yaml(hash=ht.build_hash))
mpi_spec_json_path = tmpdir.join('{0}.json'.format(mpi_spec.name))
with open(mpi_spec_json_path.strpath, 'w') as fd:
fd.write(mpi_spec.to_json(hash=ht.dag_hash))
install('--no-add', '-f', mpi_spec_yaml_path.strpath)
install('--no-add', '-f', mpi_spec_json_path.strpath)
assert(mpi_spec not in e.roots())
find_output = find('-l', output=str)
@ -1016,76 +1015,6 @@ def test_install_fails_no_args_suggests_env_activation(tmpdir):
assert 'using the `spack.yaml` in this directory' in output
default_full_hash = spack.spec.Spec.full_hash
def fake_full_hash(spec):
# Generate an arbitrary hash that is intended to be different than
# whatever a Spec reported before (to test actions that trigger when
# the hash changes)
if spec.name == 'libdwarf':
return 'tal4c7h4z0gqmixb1eqa92mjoybxn5l6'
return default_full_hash(spec)
def test_cache_install_full_hash_match(
install_mockery_mutable_config, mock_packages, mock_fetch,
mock_archive, mutable_config, monkeypatch, tmpdir):
"""Make sure installing from cache respects full hash argument"""
# Create a temp mirror directory for buildcache usage
mirror_dir = tmpdir.join('mirror_dir')
mirror_url = 'file://{0}'.format(mirror_dir.strpath)
s = Spec('libdwarf').concretized()
package_id = spack.installer.package_id(s.package)
# Install a package
install(s.name)
# Put installed package in the buildcache
buildcache('create', '-u', '-a', '-f', '-d', mirror_dir.strpath, s.name)
# Now uninstall the package
uninstall('-y', s.name)
# Configure the mirror with the binary package in it
mirror('add', 'test-mirror', mirror_url)
# Make sure we get the binary version by default
install_output = install('--no-check-signature', s.name, output=str)
expect_extract_msg = 'Extracting {0} from binary cache'.format(package_id)
assert expect_extract_msg in install_output
uninstall('-y', s.name)
# Now monkey patch Spec to change the full hash on the package
monkeypatch.setattr(spack.spec.Spec, 'full_hash', fake_full_hash)
# Check that even if the full hash changes, we install from binary when
# we don't explicitly require the full hash to match
install_output = install('--no-check-signature', s.name, output=str)
assert expect_extract_msg in install_output
uninstall('-y', s.name)
# Finally, make sure that if we insist on the full hash match, spack
# installs from source.
install_output = install('--require-full-hash-match', s.name, output=str)
expect_msg = 'No binary for {0} found: installing from source'.format(
package_id)
assert expect_msg in install_output
uninstall('-y', s.name)
mirror('rm', 'test-mirror')
# Get rid of that libdwarf binary in the mirror so other tests don't try to
# use it and fail because of NoVerifyException
shutil.rmtree(mirror_dir.strpath)
def test_install_env_with_tests_all(tmpdir, mock_packages, mock_fetch,
install_mockery, mutable_mock_env_path):
env('create', 'test')

8
lib/spack/spack/test/cmd/test.py
View file

@ -256,11 +256,11 @@ def test_hash_change(mock_test_stage, mock_packages, mock_archive, mock_fetch,
outfile = os.path.join(testdir, 'test_suite.lock')
with open(outfile, 'r') as f:
output = f.read()
val_replace = '"full_hash": "{0}"'.format(
spack.store.db.query('printing-package')[0].full_hash())
val_replace = '"hash": "{0}"'.format(
spack.store.db.query('printing-package')[0].dag_hash())
changed_hash = output.replace(
val_replace,
'"full_hash": "fakehash492ucwhwvzhxfbmcc45x49ha"')
'"hash": "fakehash492ucwhwvzhxfbmcc45x49ha"')
with open(outfile, 'w') as f:
f.write(changed_hash)
@ -271,6 +271,8 @@ def test_hash_change(mock_test_stage, mock_packages, mock_archive, mock_fetch,
results_output = spack_test('results')
assert 'PASSED' in results_output
assert(False)
def test_test_results_none(mock_packages, mock_test_stage):
name = 'trivial'

3
lib/spack/spack/test/concretize.py
View file

@ -1080,8 +1080,7 @@ def test_external_packages_have_consistent_hash(self):
s._old_concretize(), t._new_concretize()
assert s.dag_hash() == t.dag_hash()
assert s.build_hash() == t.build_hash()
assert s.full_hash() == t.full_hash()
assert s.runtime_hash() == t.runtime_hash()
def test_external_that_would_require_a_virtual_dependency(self):
s = Spec('requires-virtual').concretized()

8
lib/spack/spack/test/directory_layout.py
View file

@ -118,13 +118,7 @@ def test_read_and_write_spec(temporary_store, config, mock_packages):
# Make sure spec file can be read back in to get the original spec
spec_from_file = layout.read_spec(spec_path)
# currently we don't store build dependency information when
# we write out specs to the filesystem.
# TODO: fix this when we can concretize more loosely based on
# TODO: what is installed. We currently omit these to
# TODO: increase reuse of build dependencies.
stored_deptypes = spack.hash_types.full_hash
stored_deptypes = spack.hash_types.dag_hash
expected = spec.copy(deps=stored_deptypes)
expected._mark_concrete()

8
lib/spack/spack/test/monitor.py
View file

@ -49,7 +49,7 @@ def mock_monitor_request(monkeypatch):
def mock_do_request(self, endpoint, *args, **kwargs):
build = {"build_id": 1,
"spec_full_hash": "bpfvysmqndtmods4rmy6d6cfquwblngp",
"spec_hash": "bpfvysmqndtmods4rmy6d6cfquwblngp",
"spec_name": "dttop"}
# Service Info
@ -111,7 +111,7 @@ def mock_do_request(self, endpoint, *args, **kwargs):
elif endpoint == "specs/new/":
return {"message": "success",
"data": {
"full_hash": "bpfvysmqndtmods4rmy6d6cfquwblngp",
"hash": "bpfvysmqndtmods4rmy6d6cfquwblngp",
"name": "dttop",
"version": "1.0",
"spack_version": "0.16.0-1379-7a5351d495",
@ -264,12 +264,12 @@ def test_install_monitor_save_local(install_mockery_mutable_config,
# Get the spec name
spec = spack.spec.Spec("dttop")
spec.concretize()
full_hash = spec.full_hash()
dag_hash = spec.dag_hash()
# Ensure we have monitor results saved
for dirname in os.listdir(str(reports_dir)):
dated_dir = os.path.join(str(reports_dir), dirname)
build_metadata = "build-metadata-%s.json" % full_hash
build_metadata = "build-metadata-%s.json" % dag_hash
assert build_metadata in os.listdir(dated_dir)
spec_file = "spec-dttop-%s-config.json" % spec.version
assert spec_file in os.listdir(dated_dir)

102
lib/spack/spack/test/spec_semantics.py
View file

@ -1001,7 +1001,7 @@ def test_splice(self, transitive):
dep.concretize()
# Sanity checking that these are not the same thing.
assert dep.dag_hash() != spec['splice-h'].dag_hash()
assert dep.build_hash() != spec['splice-h'].build_hash()
assert dep.runtime_hash() != spec['splice-h'].runtime_hash()
# Do the splice.
out = spec.splice(dep, transitive)
# Returned spec should still be concrete.
@ -1009,18 +1009,18 @@ def test_splice(self, transitive):
# Traverse the spec and assert that all dependencies are accounted for.
for node in spec.traverse():
assert node.name in out
# If the splice worked, then the full hash of the spliced dep should
# now match the full hash of the build spec of the dependency from the
# If the splice worked, then the dag hash of the spliced dep should
# now match the dag hash of the build spec of the dependency from the
# returned spec.
out_h_build = out['splice-h'].build_spec
assert out_h_build.full_hash() == dep.full_hash()
assert out_h_build.dag_hash() == dep.dag_hash()
# Transitivity should determine whether the transitive dependency was
# changed.
expected_z = dep['splice-z'] if transitive else spec['splice-z']
assert out['splice-z'].full_hash() == expected_z.full_hash()
assert out['splice-z'].dag_hash() == expected_z.dag_hash()
# Sanity check build spec of out should be the original spec.
assert (out['splice-t'].build_spec.full_hash() ==
spec['splice-t'].full_hash())
assert (out['splice-t'].build_spec.dag_hash() ==
spec['splice-t'].dag_hash())
# Finally, the spec should know it's been spliced:
assert out.spliced
@ -1032,39 +1032,39 @@ def test_splice_with_cached_hashes(self, transitive):
dep.concretize()
# monkeypatch hashes so we can test that they are cached
spec._full_hash = 'aaaaaa'
spec._build_hash = 'aaaaaa'
dep._full_hash = 'bbbbbb'
dep._build_hash = 'bbbbbb'
spec['splice-h']._full_hash = 'cccccc'
spec['splice-h']._build_hash = 'cccccc'
spec['splice-z']._full_hash = 'dddddd'
spec['splice-z']._build_hash = 'dddddd'
dep['splice-z']._full_hash = 'eeeeee'
dep['splice-z']._build_hash = 'eeeeee'
spec._hash = 'aaaaaa'
spec._runtime_hash = 'aaaaaa'
dep._hash = 'bbbbbb'
dep._runtime_hash = 'bbbbbb'
spec['splice-h']._hash = 'cccccc'
spec['splice-h']._runtime_hash = 'cccccc'
spec['splice-z']._hash = 'dddddd'
spec['splice-z']._runtime_hash = 'dddddd'
dep['splice-z']._hash = 'eeeeee'
dep['splice-z']._runtime_hash = 'eeeeee'
out = spec.splice(dep, transitive=transitive)
out_z_expected = (dep if transitive else spec)['splice-z']
assert out.full_hash() != spec.full_hash()
assert (out['splice-h'].full_hash() == dep.full_hash()) == transitive
assert out['splice-z'].full_hash() == out_z_expected.full_hash()
assert out.dag_hash() != spec.dag_hash()
assert (out['splice-h'].dag_hash() == dep.dag_hash()) == transitive
assert out['splice-z'].dag_hash() == out_z_expected.dag_hash()
assert out.build_hash() != spec.build_hash()
assert (out['splice-h'].build_hash() == dep.build_hash()) == transitive
assert out['splice-z'].build_hash() == out_z_expected.build_hash()
assert out.runtime_hash() != spec.runtime_hash()
assert (out['splice-h'].runtime_hash() == dep.runtime_hash()) == transitive
assert out['splice-z'].runtime_hash() == out_z_expected.runtime_hash()
@pytest.mark.parametrize('transitive', [True, False])
def test_splice_input_unchanged(self, transitive):
spec = Spec('splice-t').concretized()
dep = Spec('splice-h+foo').concretized()
orig_spec_hash = spec.full_hash()
orig_dep_hash = dep.full_hash()
orig_spec_hash = spec.dag_hash()
orig_dep_hash = dep.dag_hash()
spec.splice(dep, transitive)
# Post-splice, dag hash should still be different; no changes should be
# made to these specs.
assert spec.full_hash() == orig_spec_hash
assert dep.full_hash() == orig_dep_hash
assert spec.dag_hash() == orig_spec_hash
assert dep.dag_hash() == orig_dep_hash
@pytest.mark.parametrize('transitive', [True, False])
def test_splice_subsequent(self, transitive):
@ -1079,12 +1079,12 @@ def test_splice_subsequent(self, transitive):
# Transitivity shouldn't matter since Splice Z has no dependencies.
out2 = out.splice(dep, transitive)
assert out2.concrete
assert out2['splice-z'].build_hash() != spec['splice-z'].build_hash()
assert out2['splice-z'].build_hash() != out['splice-z'].build_hash()
assert out2['splice-z'].full_hash() != spec['splice-z'].full_hash()
assert out2['splice-z'].full_hash() != out['splice-z'].full_hash()
assert (out2['splice-t'].build_spec.full_hash() ==
spec['splice-t'].full_hash())
assert out2['splice-z'].runtime_hash() != spec['splice-z'].runtime_hash()
assert out2['splice-z'].runtime_hash() != out['splice-z'].runtime_hash()
assert out2['splice-z'].dag_hash() != spec['splice-z'].dag_hash()
assert out2['splice-z'].dag_hash() != out['splice-z'].dag_hash()
assert (out2['splice-t'].build_spec.dag_hash() ==
spec['splice-t'].dag_hash())
assert out2.spliced
@pytest.mark.parametrize('transitive', [True, False])
@ -1096,13 +1096,13 @@ def test_splice_dict(self, transitive):
out = spec.splice(dep, transitive)
# Sanity check all hashes are unique...
assert spec.full_hash() != dep.full_hash()
assert out.full_hash() != dep.full_hash()
assert out.full_hash() != spec.full_hash()
assert spec.dag_hash() != dep.dag_hash()
assert out.dag_hash() != dep.dag_hash()
assert out.dag_hash() != spec.dag_hash()
node_list = out.to_dict()['spec']['nodes']
root_nodes = [n for n in node_list if n['full_hash'] == out.full_hash()]
build_spec_nodes = [n for n in node_list if n['full_hash'] == spec.full_hash()]
assert spec.full_hash() == out.build_spec.full_hash()
root_nodes = [n for n in node_list if n['hash'] == out.dag_hash()]
build_spec_nodes = [n for n in node_list if n['hash'] == spec.dag_hash()]
assert spec.dag_hash() == out.build_spec.dag_hash()
assert len(root_nodes) == 1
assert len(build_spec_nodes) == 1
@ -1115,28 +1115,28 @@ def test_splice_dict_roundtrip(self, transitive):
out = spec.splice(dep, transitive)
# Sanity check all hashes are unique...
assert spec.full_hash() != dep.full_hash()
assert out.full_hash() != dep.full_hash()
assert out.full_hash() != spec.full_hash()
assert spec.dag_hash() != dep.dag_hash()
assert out.dag_hash() != dep.dag_hash()
assert out.dag_hash() != spec.dag_hash()
out_rt_spec = Spec.from_dict(out.to_dict()) # rt is "round trip"
assert out_rt_spec.full_hash() == out.full_hash()
out_rt_spec_bld_hash = out_rt_spec.build_spec.full_hash()
out_rt_spec_h_bld_hash = out_rt_spec['splice-h'].build_spec.full_hash()
out_rt_spec_z_bld_hash = out_rt_spec['splice-z'].build_spec.full_hash()
assert out_rt_spec.dag_hash() == out.dag_hash()
out_rt_spec_bld_hash = out_rt_spec.build_spec.dag_hash()
out_rt_spec_h_bld_hash = out_rt_spec['splice-h'].build_spec.dag_hash()
out_rt_spec_z_bld_hash = out_rt_spec['splice-z'].build_spec.dag_hash()
# In any case, the build spec for splice-t (root) should point to the
# original spec, preserving build provenance.
assert spec.full_hash() == out_rt_spec_bld_hash
assert out_rt_spec.full_hash() != out_rt_spec_bld_hash
assert spec.dag_hash() == out_rt_spec_bld_hash
assert out_rt_spec.dag_hash() != out_rt_spec_bld_hash
# The build spec for splice-h should always point to the introduced
# spec, since that is the spec spliced in.
assert dep['splice-h'].full_hash() == out_rt_spec_h_bld_hash
assert dep['splice-h'].dag_hash() == out_rt_spec_h_bld_hash
# The build spec for splice-z will depend on whether or not the splice
# was transitive.
expected_z_bld_hash = (dep['splice-z'].full_hash() if transitive else
spec['splice-z'].full_hash())
expected_z_bld_hash = (dep['splice-z'].dag_hash() if transitive else
spec['splice-z'].dag_hash())
assert expected_z_bld_hash == out_rt_spec_z_bld_hash
@pytest.mark.parametrize('spec,constraint,expected_result', [

14
lib/spack/spack/test/spec_syntax.py
View file

@ -499,7 +499,7 @@ def test_parse_yaml_simple(self, mock_packages, tmpdir):
specfile = tmpdir.join('libdwarf.yaml')
with specfile.open('w') as f:
f.write(s.to_yaml(hash=ht.build_hash))
f.write(s.to_yaml(hash=ht.dag_hash))
# Check an absolute path to spec.yaml by itself:
# "spack spec /path/to/libdwarf.yaml"
@ -522,7 +522,7 @@ def test_parse_filename_missing_slash_as_spec(self, mock_packages, tmpdir):
# write the file to the current directory to make sure it exists,
# and that we still do not parse the spec as a file.
with specfile.open('w') as f:
f.write(s.to_yaml(hash=ht.build_hash))
f.write(s.to_yaml(hash=ht.dag_hash))
# Check the spec `libelf.yaml` in the working directory, which
# should evaluate to a spec called `yaml` in the `libelf`
@ -562,7 +562,7 @@ def test_parse_yaml_dependency(self, mock_packages, tmpdir):
specfile = tmpdir.join('libelf.yaml')
with specfile.open('w') as f:
f.write(s['libelf'].to_yaml(hash=ht.build_hash))
f.write(s['libelf'].to_yaml(hash=ht.dag_hash))
# Make sure we can use yaml path as dependency, e.g.:
# "spack spec libdwarf ^ /path/to/libelf.yaml"
@ -577,7 +577,7 @@ def test_parse_yaml_relative_paths(self, mock_packages, tmpdir):
specfile = tmpdir.join('libdwarf.yaml')
with specfile.open('w') as f:
f.write(s.to_yaml(hash=ht.build_hash))
f.write(s.to_yaml(hash=ht.dag_hash))
file_name = specfile.basename
parent_dir = os.path.basename(specfile.dirname)
@ -612,7 +612,7 @@ def test_parse_yaml_relative_subdir_path(self, mock_packages, tmpdir):
specfile = tmpdir.mkdir('subdir').join('libdwarf.yaml')
with specfile.open('w') as f:
f.write(s.to_yaml(hash=ht.build_hash))
f.write(s.to_yaml(hash=ht.dag_hash))
file_name = specfile.basename
@ -632,7 +632,7 @@ def test_parse_yaml_dependency_relative_paths(self, mock_packages, tmpdir):
specfile = tmpdir.join('libelf.yaml')
with specfile.open('w') as f:
f.write(s['libelf'].to_yaml(hash=ht.build_hash))
f.write(s['libelf'].to_yaml(hash=ht.dag_hash))
file_name = specfile.basename
parent_dir = os.path.basename(specfile.dirname)
@ -691,7 +691,7 @@ def test_parse_yaml_variant_error(self, mock_packages, tmpdir):
specfile = tmpdir.join('a.yaml')
with specfile.open('w') as f:
f.write(s.to_yaml(hash=ht.build_hash))
f.write(s.to_yaml(hash=ht.dag_hash))
with pytest.raises(RedundantSpecError):
# Trying to change a variant on a concrete spec is an error

30
lib/spack/spack/test/spec_yaml.py
View file

@ -152,13 +152,8 @@ def descend_and_check(iterable, level=0):
assert level >= 5
@pytest.mark.parametrize("hash_type", [
ht.dag_hash,
ht.build_hash,
ht.full_hash
])
def test_ordered_read_not_required_for_consistent_dag_hash(
hash_type, config, mock_packages
config, mock_packages
):
"""Make sure ordered serialization isn't required to preserve hashes.
@ -175,15 +170,15 @@ def test_ordered_read_not_required_for_consistent_dag_hash(
#
# Dict & corresponding YAML & JSON from the original spec.
#
spec_dict = spec.to_dict(hash=hash_type)
spec_yaml = spec.to_yaml(hash=hash_type)
spec_json = spec.to_json(hash=hash_type)
spec_dict = spec.to_dict()
spec_yaml = spec.to_yaml()
spec_json = spec.to_json()
#
# Make a spec with reversed OrderedDicts for every
# OrderedDict in the original.
#
reversed_spec_dict = reverse_all_dicts(spec.to_dict(hash=hash_type))
reversed_spec_dict = reverse_all_dicts(spec.to_dict())
#
# Dump to YAML and JSON
@ -218,7 +213,7 @@ def test_ordered_read_not_required_for_consistent_dag_hash(
)
# Strip spec if we stripped the yaml
spec = spec.copy(deps=hash_type.deptype)
spec = spec.copy(deps=ht.dag_hash.deptype)
# specs are equal to the original
assert spec == round_trip_yaml_spec
@ -234,17 +229,16 @@ def test_ordered_read_not_required_for_consistent_dag_hash(
assert spec.dag_hash() == round_trip_reversed_yaml_spec.dag_hash()
assert spec.dag_hash() == round_trip_reversed_json_spec.dag_hash()
# full_hashes are equal if we round-tripped by build_hash or full_hash
if hash_type in (ht.build_hash, ht.full_hash):
# dag_hash is equal after round-trip by dag_hash
spec.concretize()
round_trip_yaml_spec.concretize()
round_trip_json_spec.concretize()
round_trip_reversed_yaml_spec.concretize()
round_trip_reversed_json_spec.concretize()
assert spec.full_hash() == round_trip_yaml_spec.full_hash()
assert spec.full_hash() == round_trip_json_spec.full_hash()
assert spec.full_hash() == round_trip_reversed_yaml_spec.full_hash()
assert spec.full_hash() == round_trip_reversed_json_spec.full_hash()
assert spec.dag_hash() == round_trip_yaml_spec.dag_hash()
assert spec.dag_hash() == round_trip_json_spec.dag_hash()
assert spec.dag_hash() == round_trip_reversed_yaml_spec.dag_hash()
assert spec.dag_hash() == round_trip_reversed_json_spec.dag_hash()
@pytest.mark.parametrize("module", [
@ -350,7 +344,7 @@ def test_save_dependency_spec_jsons_subset(tmpdir, config):
spec_a.concretize()
b_spec = spec_a['b']
c_spec = spec_a['c']
spec_a_json = spec_a.to_json(hash=ht.build_hash)
spec_a_json = spec_a.to_json()
save_dependency_specfiles(spec_a_json, output_path, ['b', 'c'])

4
share/spack/spack-completion.bash
View file

@ -544,7 +544,7 @@ _spack_buildcache_preview() {
}
_spack_buildcache_check() {
SPACK_COMPREPLY="-h --help -m --mirror-url -o --output-file --scope -s --spec --spec-file --rebuild-on-error"
SPACK_COMPREPLY="-h --help -m --mirror-url -o --output-file --scope -s --spec --spec-file"
}
_spack_buildcache_download() {
@ -1174,7 +1174,7 @@ _spack_info() {
_spack_install() {
if $list_options
then
SPACK_COMPREPLY="-h --help --only -u --until -j --jobs --overwrite --fail-fast --keep-prefix --keep-stage --dont-restage --use-cache --no-cache --cache-only --monitor --monitor-save-local --monitor-tags --monitor-keep-going --monitor-host --monitor-prefix --include-build-deps --no-check-signature --require-full-hash-match --show-log-on-error --source -n --no-checksum --deprecated -v --verbose --fake --only-concrete --no-add -f --file --clean --dirty --test --log-format --log-file --help-cdash --cdash-upload-url --cdash-build --cdash-site --cdash-track --cdash-buildstamp -y --yes-to-all -U --fresh --reuse"
SPACK_COMPREPLY="-h --help --only -u --until -j --jobs --overwrite --fail-fast --keep-prefix --keep-stage --dont-restage --use-cache --no-cache --cache-only --monitor --monitor-save-local --monitor-tags --monitor-keep-going --monitor-host --monitor-prefix --include-build-deps --no-check-signature --show-log-on-error --source -n --no-checksum --deprecated -v --verbose --fake --only-concrete --no-add -f --file --clean --dirty --test --log-format --log-file --help-cdash --cdash-upload-url --cdash-build --cdash-site --cdash-track --cdash-buildstamp -y --yes-to-all -U --fresh --reuse"
else
_all_packages
fi