hlrs-software-stack/spack
7
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Speed up pipeline generation (#26622)

Browse source 
- [x] Stage already concretized specs instead of abstract ones
- [x] Reduce number of network calls by reading naughty list up front
This commit is contained in:
Scott Wittenburg 2021-10-18 21:58:02 -06:00 committed by GitHub
parent d41ddb8a9c
commit 95538de731
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 38 additions and 16 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

40
lib/spack/spack/ci.py
View file

@ -396,9 +396,6 @@ def append_dep(s, d):
})
for spec in spec_list:
spec.concretize()
# root_spec = get_spec_string(spec)
root_spec = spec
for s in spec.traverse(deptype=all):
@ -677,9 +674,25 @@ def generate_gitlab_ci_yaml(env, print_summary, output_file,
try:
for phase in phases:
phase_name = phase['name']
if phase_name == 'specs':
# Anything in the "specs" of the environment are already
# concretized by the block at the top of this method, so we
# only need to find the concrete versions, and then avoid
# re-concretizing them needlessly later on.
concrete_phase_specs = [
concrete for abstract, concrete in env.concretized_specs()
if abstract in env.spec_lists[phase_name]
]
else:
# Any specs lists in other definitions (but not in the
# "specs") of the environment are not yet concretized so we
# have to concretize them explicitly here.
concrete_phase_specs = env.spec_lists[phase_name]
with spack.concretize.disable_compiler_existence_check():
for phase_spec in concrete_phase_specs:
phase_spec.concretize()
staged_phases[phase_name] = stage_spec_jobs(
env.spec_lists[phase_name],
concrete_phase_specs,
check_index_only=check_index_only)
finally:
# Clean up PR mirror if enabled
@ -696,6 +709,17 @@ def generate_gitlab_ci_yaml(env, print_summary, output_file,
max_length_needs = 0
max_needs_job = ''
# If this is configured, spack will fail "spack ci generate" if it
# generates any full hash which exists under the broken specs url.
broken_spec_urls = None
if broken_specs_url:
if broken_specs_url.startswith('http'):
# To make checking each spec against the list faster, we require
# a url protocol that allows us to iterate the url in advance.
tty.msg('Cannot use an http(s) url for broken specs, ignoring')
else:
broken_spec_urls = web_util.list_url(broken_specs_url)
before_script, after_script = None, None
for phase in phases:
phase_name = phase['name']
@ -882,12 +906,8 @@ def generate_gitlab_ci_yaml(env, print_summary, output_file,
tty.debug('Pruning spec that does not need to be rebuilt.')
continue
# Check if this spec is in our list of known failures, now that
# we know this spec needs a rebuild
if broken_specs_url:
broken_spec_path = url_util.join(
broken_specs_url, release_spec_full_hash)
if web_util.url_exists(broken_spec_path):
if (broken_spec_urls is not None and
release_spec_full_hash in broken_spec_urls):
known_broken_specs_encountered.append('{0} ({1})'.format(
release_spec, release_spec_full_hash))

4
lib/spack/spack/test/cmd/ci.py
View file

@ -1617,6 +1617,8 @@ def test_ci_generate_read_broken_specs_url(tmpdir, mutable_mock_env_path,
with open(broken_spec_a_path, 'w') as bsf:
bsf.write('')
broken_specs_url = 'file://{0}'.format(tmpdir.strpath)
# Test that `spack ci generate` notices this broken spec and fails.
filename = str(tmpdir.join('spack.yaml'))
with open(filename, 'w') as f:
@ -1639,7 +1641,7 @@ def test_ci_generate_read_broken_specs_url(tmpdir, mutable_mock_env_path,
tags:
- donotcare
image: donotcare
""".format(tmpdir.strpath))
""".format(broken_specs_url))
with tmpdir.as_cwd():
env_cmd('create', 'test', './spack.yaml')