Update buildcache key index when we update the package index (#19117)

This changes makes sure that when we run the pipeline job that updates
the buildcache package index on the remote mirror, we also update the
key index.  The public keys corresponding to the signing keys used to
sign the package was pushed to the mirror as a part of creating the
buildcache index, so this is just ensuring those keys are reflected
in the key index.

Also, this change makes sure the "spack buildcache update-index"
job runs even when there may have been pipeline failures, since we
would like the index always to reflect the true state of the mirror.

lib/spack/spack/binary_distribution.py

@@ -400,6 +400,10 @@ def build_cache_relative_path():
     return _build_cache_relative_path
 
 
+def build_cache_keys_relative_path():
+    return _build_cache_keys_relative_path
+
+
 def build_cache_prefix(prefix):
     return os.path.join(prefix, build_cache_relative_path())
 

lib/spack/spack/ci.py

@@ -806,9 +806,10 @@ def generate_gitlab_ci_yaml(env, print_summary, output_file,
         final_stage = 'stage-rebuild-index'
         final_job = {
             'stage': final_stage,
-            'script': 'spack buildcache update-index -d {0}'.format(
+            'script': 'spack buildcache update-index --keys -d {0}'.format(
                 mirror_urls[0]),
-            'tags': final_job_config['tags']
+            'tags': final_job_config['tags'],
+            'when': 'always'
         }
         if 'image' in final_job_config:
             final_job['image'] = final_job_config['image']

lib/spack/spack/cmd/buildcache.py

@@ -231,6 +231,9 @@ def setup_parser(subparser):
         'update-index', help=buildcache_update_index.__doc__)
     update_index.add_argument(
         '-d', '--mirror-url', default=None, help='Destination mirror url')
+    update_index.add_argument(
+        '-k', '--keys', default=False, action='store_true',
+        help='If provided, key index will be updated as well as package index')
     update_index.set_defaults(func=buildcache_update_index)
 
 
@@ -777,6 +780,13 @@ def buildcache_update_index(args):
     bindist.generate_package_index(
         url_util.join(outdir, bindist.build_cache_relative_path()))
 
+    if args.keys:
+        keys_url = url_util.join(outdir,
+                                 bindist.build_cache_relative_path(),
+                                 bindist.build_cache_keys_relative_path())
+
+        bindist.generate_key_index(keys_url)
+
 
 def buildcache(parser, args):
     if args.func:

lib/spack/spack/test/cmd/buildcache.py

@@ -19,6 +19,9 @@
 install = spack.main.SpackCommand('install')
 env = spack.main.SpackCommand('env')
 add = spack.main.SpackCommand('add')
+gpg = spack.main.SpackCommand('gpg')
+mirror = spack.main.SpackCommand('mirror')
+uninstall = spack.main.SpackCommand('uninstall')
 
 
 @pytest.fixture()
@@ -133,3 +136,41 @@ def test_buildcache_create_fail_on_perm_denied(
                    '--unsigned', 'trivial-install-test-package')
     assert error.value.errno == errno.EACCES
     tmpdir.chmod(0o700)
+
+
+@pytest.mark.skipif(not spack.util.gpg.has_gpg(),
+                    reason='This test requires gpg')
+def test_update_key_index(tmpdir, mutable_mock_env_path,
+                          install_mockery, mock_packages, mock_fetch,
+                          mock_stage, mock_gnupghome):
+    """Test the update-index command with the --keys option"""
+    working_dir = tmpdir.join('working_dir')
+
+    mirror_dir = working_dir.join('mirror')
+    mirror_url = 'file://{0}'.format(mirror_dir.strpath)
+
+    mirror('add', 'test-mirror', mirror_url)
+
+    gpg('create', 'Test Signing Key', 'nobody@nowhere.com')
+
+    s = Spec('libdwarf').concretized()
+
+    # Install a package
+    install(s.name)
+
+    # Put installed package in the buildcache, which, because we're signing
+    # it, should result in the public key getting pushed to the buildcache
+    # as well.
+    buildcache('create', '-a', '-d', mirror_dir.strpath, s.name)
+
+    # Now make sure that when we pass the "--keys" argument to update-index
+    # it causes the index to get update.
+    buildcache('update-index', '--keys', '-d', mirror_dir.strpath)
+
+    key_dir_list = os.listdir(os.path.join(
+        mirror_dir.strpath, 'build_cache', '_pgp'))
+
+    uninstall('-y', s.name)
+    mirror('rm', 'test-mirror')
+
+    assert 'index.json' in key_dir_list

share/spack/spack-completion.bash

@@ -434,7 +434,7 @@ _spack_buildcache_copy() {
 }
 
 _spack_buildcache_update_index() {
-    SPACK_COMPREPLY="-h --help -d --mirror-url"
+    SPACK_COMPREPLY="-h --help -d --mirror-url -k --keys"
 }
 
 _spack_cd() {