Speed up pipeline generation (#26622)

- [x] Stage already concretized specs instead of abstract ones
- [x] Reduce number of network calls by reading naughty list up front

lib/spack/spack/ci.py

@@ -396,9 +396,6 @@ def append_dep(s, d):
         })
 
     for spec in spec_list:
-        spec.concretize()
-
-        # root_spec = get_spec_string(spec)
         root_spec = spec
 
         for s in spec.traverse(deptype=all):
@@ -677,10 +674,26 @@ def generate_gitlab_ci_yaml(env, print_summary, output_file,
     try:
         for phase in phases:
             phase_name = phase['name']
-            with spack.concretize.disable_compiler_existence_check():
-                staged_phases[phase_name] = stage_spec_jobs(
-                    env.spec_lists[phase_name],
-                    check_index_only=check_index_only)
+            if phase_name == 'specs':
+                # Anything in the "specs" of the environment are already
+                # concretized by the block at the top of this method, so we
+                # only need to find the concrete versions, and then avoid
+                # re-concretizing them needlessly later on.
+                concrete_phase_specs = [
+                    concrete for abstract, concrete in env.concretized_specs()
+                    if abstract in env.spec_lists[phase_name]
+                ]
+            else:
+                # Any specs lists in other definitions (but not in the
+                # "specs") of the environment are not yet concretized so we
+                # have to concretize them explicitly here.
+                concrete_phase_specs = env.spec_lists[phase_name]
+                with spack.concretize.disable_compiler_existence_check():
+                    for phase_spec in concrete_phase_specs:
+                        phase_spec.concretize()
+            staged_phases[phase_name] = stage_spec_jobs(
+                concrete_phase_specs,
+                check_index_only=check_index_only)
     finally:
         # Clean up PR mirror if enabled
         if pr_mirror_url:
@@ -696,6 +709,17 @@ def generate_gitlab_ci_yaml(env, print_summary, output_file,
     max_length_needs = 0
     max_needs_job = ''
 
+    # If this is configured, spack will fail "spack ci generate" if it
+    # generates any full hash which exists under the broken specs url.
+    broken_spec_urls = None
+    if broken_specs_url:
+        if broken_specs_url.startswith('http'):
+            # To make checking each spec against the list faster, we require
+            # a url protocol that allows us to iterate the url in advance.
+            tty.msg('Cannot use an http(s) url for broken specs, ignoring')
+        else:
+            broken_spec_urls = web_util.list_url(broken_specs_url)
+
     before_script, after_script = None, None
     for phase in phases:
         phase_name = phase['name']
@@ -882,14 +906,10 @@ def generate_gitlab_ci_yaml(env, print_summary, output_file,
                     tty.debug('Pruning spec that does not need to be rebuilt.')
                     continue
 
-                # Check if this spec is in our list of known failures, now that
-                # we know this spec needs a rebuild
-                if broken_specs_url:
-                    broken_spec_path = url_util.join(
-                        broken_specs_url, release_spec_full_hash)
-                    if web_util.url_exists(broken_spec_path):
-                        known_broken_specs_encountered.append('{0} ({1})'.format(
-                            release_spec, release_spec_full_hash))
+                if (broken_spec_urls is not None and
+                        release_spec_full_hash in broken_spec_urls):
+                    known_broken_specs_encountered.append('{0} ({1})'.format(
+                        release_spec, release_spec_full_hash))
 
                 if artifacts_root:
                     job_dependencies.append({

lib/spack/spack/test/cmd/ci.py

@@ -1617,6 +1617,8 @@ def test_ci_generate_read_broken_specs_url(tmpdir, mutable_mock_env_path,
     with open(broken_spec_a_path, 'w') as bsf:
         bsf.write('')
 
+    broken_specs_url = 'file://{0}'.format(tmpdir.strpath)
+
     # Test that `spack ci generate` notices this broken spec and fails.
     filename = str(tmpdir.join('spack.yaml'))
     with open(filename, 'w') as f:
@@ -1639,7 +1641,7 @@ def test_ci_generate_read_broken_specs_url(tmpdir, mutable_mock_env_path,
           tags:
             - donotcare
           image: donotcare
-""".format(tmpdir.strpath))
+""".format(broken_specs_url))
 
     with tmpdir.as_cwd():
         env_cmd('create', 'test', './spack.yaml')