hlrs-software-stack/spack
7
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Streamline key management for build caches (#17792)

Browse source 
* Rework spack.util.web.list_url()

list_url() now accepts an optional recursive argument (default: False)
for controlling whether to only return files within the prefix url or to
return all files whose path starts with the prefix url.  Allows for the
most effecient implementation for the given prefix url scheme.  For
example, only recursive queries are supported for S3 prefixes, so the
returned list is trimmed down if recursive == False, but the native
search is returned as-is when recursive == True.  Suitable
implementations for each case are also used for file system URLs.

* Switch to using an explicit index for public keys

Switches to maintaining a build cache's keys under build_cache/_pgp.
Within this directory is an index.json file listing all the available
keys and a <fingerprint>.pub file for each such key.

 - Adds spack.binary_distribution.generate_key_index()
   - (re)generates a build cache's key index

 - Modifies spack.binary_distribution.build_tarball()
   - if tarball is signed, automatically pushes the key used for signing
     along with the tarball
   - if regenerate_index == True, automatically (re)generates the build
     cache's key index along with the build cache's package index; as in
     spack.binary_distribution.generate_key_index()

 - Modifies spack.binary_distribution.get_keys()
   - a build cache's key index is now used instead of programmatic
     listing

 - Adds spack.binary_distribution.push_keys()
   - publishes keys from Spack's keyring to a given list of mirrors

 - Adds new spack subcommand: spack gpg publish
   - publishes keys from Spack's keyring to a given list of mirrors

 - Modifies spack.util.gpg.Gpg.signing_keys()
   - Accepts optional positional arguments for filtering the set of keys
     returned

 - Adds spack.util.gpg.Gpg.public_keys()
   - As spack.util.gpg.Gpg.signing_keys(), except public keys are
     returned

 - Modifies spack.util.gpg.Gpg.export_keys()
   - Fixes an issue where GnuPG would prompt for user input if trying to
     overwrite an existing file

 - Modifies spack.util.gpg.Gpg.untrust()
   - Fixes an issue where GnuPG would fail for input that were not key
     fingerprints

 - Modifies spack.util.web.url_exists()
   - Fixes an issue where url_exists() would throw instead of returning
     False

* rework gpg module/fix error with very long GNUPGHOME dir

* add a shim for functools.cached_property

* handle permission denied error in gpg util

* fix tests/make gpgconf optional if no socket dir is available
This commit is contained in:
Omar Padron 2020-09-25 12:54:24 -04:00 committed by GitHub
parent 421f4e12a7
commit 2d93154119
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
14 changed files with 734 additions and 214 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

214
lib/spack/spack/binary_distribution.py
View file

@ -26,17 +26,18 @@
import spack.config as config
import spack.database as spack_db
import spack.fetch_strategy as fs
import spack.util.gpg
import spack.relocate as relocate
import spack.util.gpg
import spack.util.spack_json as sjson
import spack.util.spack_yaml as syaml
import spack.mirror
import spack.util.url as url_util
import spack.util.web as web_util
from spack.spec import Spec
from spack.stage import Stage
from spack.util.gpg import Gpg
_build_cache_relative_path = 'build_cache'
_build_cache_keys_relative_path = '_pgp'
BUILD_CACHE_INDEX_TEMPLATE = '''
<html>
@ -247,15 +248,9 @@ def checksum_tarball(file):
return hasher.hexdigest()
def sign_tarball(key, force, specfile_path):
# Sign the packages if keys available
if spack.util.gpg.Gpg.gpg() is None:
raise NoGpgException(
"gpg2 is not available in $PATH .\n"
"Use spack install gnupg and spack load gnupg.")
def select_signing_key(key=None):
if key is None:
keys = Gpg.signing_keys()
keys = spack.util.gpg.signing_keys()
if len(keys) == 1:
key = keys[0]
@ -263,26 +258,30 @@ def sign_tarball(key, force, specfile_path):
raise PickKeyException(str(keys))
if len(keys) == 0:
msg = "No default key available for signing.\n"
msg += "Use spack gpg init and spack gpg create"
msg += " to create a default key."
raise NoKeyException(msg)
raise NoKeyException(
"No default key available for signing.\n"
"Use spack gpg init and spack gpg create"
" to create a default key.")
return key
def sign_tarball(key, force, specfile_path):
if os.path.exists('%s.asc' % specfile_path):
if force:
os.remove('%s.asc' % specfile_path)
else:
raise NoOverwriteException('%s.asc' % specfile_path)
Gpg.sign(key, specfile_path, '%s.asc' % specfile_path)
key = select_signing_key(key)
spack.util.gpg.sign(key, specfile_path, '%s.asc' % specfile_path)
def generate_package_index(cache_prefix):
"""Create the build cache index page.
Creates (or replaces) the "index.json" page at the location given in
cache_prefix. This page contains a link for each binary package (*.yaml)
and public key (*.key) under cache_prefix.
cache_prefix. This page contains a link for each binary package (.yaml)
under cache_prefix.
"""
tmpdir = tempfile.mkdtemp()
db_root_dir = os.path.join(tmpdir, 'db_root')
@ -325,6 +324,45 @@ def generate_package_index(cache_prefix):
shutil.rmtree(tmpdir)
def generate_key_index(key_prefix, tmpdir=None):
"""Create the key index page.
Creates (or replaces) the "index.json" page at the location given in
key_prefix. This page contains an entry for each key (.pub) under
key_prefix.
"""
tty.debug(' '.join(('Retrieving key.pub files from',
url_util.format(key_prefix),
'to build key index')))
fingerprints = (
entry[:-4]
for entry in web_util.list_url(key_prefix, recursive=False)
if entry.endswith('.pub'))
keys_local = url_util.local_file_path(key_prefix)
if keys_local:
target = os.path.join(keys_local, 'index.json')
else:
target = os.path.join(tmpdir, 'index.json')
index = {
'keys': dict(
(fingerprint, {}) for fingerprint
in sorted(set(fingerprints)))
}
with open(target, 'w') as f:
sjson.dump(index, f)
if not keys_local:
web_util.push_to_url(
target,
url_util.join(key_prefix, 'index.json'),
keep_original=False,
extra_args={'ContentType': 'application/json'})
def build_tarball(spec, outdir, force=False, rel=False, unsigned=False,
allow_root=False, key=None, regenerate_index=False):
"""
@ -445,7 +483,9 @@ def build_tarball(spec, outdir, force=False, rel=False, unsigned=False,
# sign the tarball and spec file with gpg
if not unsigned:
key = select_signing_key(key)
sign_tarball(key, force, specfile_path)
# put tarball, spec and signature files in .spack archive
with closing(tarfile.open(spackfile_path, 'w')) as tar:
tar.add(name=tarfile_path, arcname='%s' % tarfile_name)
@ -468,7 +508,15 @@ def build_tarball(spec, outdir, force=False, rel=False, unsigned=False,
.format(spec, remote_spackfile_path))
try:
# create an index.html for the build_cache directory so specs can be
# push the key to the build cache's _pgp directory so it can be
# imported
if not unsigned:
push_keys(outdir,
keys=[key],
regenerate_index=regenerate_index,
tmpdir=tmpdir)
# create an index.json for the build_cache directory so specs can be
# found
if regenerate_index:
generate_package_index(url_util.join(
@ -695,7 +743,8 @@ def extract_tarball(spec, filename, allow_root=False, unsigned=False,
if os.path.exists('%s.asc' % specfile_path):
try:
suppress = config.get('config:suppress_gpg_warnings', False)
Gpg.verify('%s.asc' % specfile_path, specfile_path, suppress)
spack.util.gpg.verify(
'%s.asc' % specfile_path, specfile_path, suppress)
except Exception as e:
shutil.rmtree(tmpdir)
raise e
@ -898,41 +947,46 @@ def get_specs():
return _cached_specs
def get_keys(install=False, trust=False, force=False):
def get_keys(install=False, trust=False, force=False, mirrors=None):
"""Get pgp public keys available on mirror with suffix .pub
"""
Get pgp public keys available on mirror
with suffix .key or .pub
"""
if not spack.mirror.MirrorCollection():
mirror_collection = (mirrors or spack.mirror.MirrorCollection())
if not mirror_collection:
tty.die("Please add a spack mirror to allow " +
"download of build caches.")
keys = set()
for mirror in mirror_collection.values():
fetch_url = mirror.fetch_url
keys_url = url_util.join(fetch_url,
_build_cache_relative_path,
_build_cache_keys_relative_path)
keys_index = url_util.join(keys_url, 'index.json')
for mirror in spack.mirror.MirrorCollection().values():
fetch_url_build_cache = url_util.join(
mirror.fetch_url, _build_cache_relative_path)
tty.debug('Finding public keys in {0}'.format(
url_util.format(fetch_url)))
mirror_dir = url_util.local_file_path(fetch_url_build_cache)
if mirror_dir:
tty.debug('Finding public keys in {0}'.format(mirror_dir))
files = os.listdir(str(mirror_dir))
for file in files:
if re.search(r'\.key', file) or re.search(r'\.pub', file):
link = url_util.join(fetch_url_build_cache, file)
keys.add(link)
else:
tty.debug('Finding public keys at {0}'
.format(url_util.format(fetch_url_build_cache)))
# For s3 mirror need to request index.html directly
p, links = web_util.spider(
url_util.join(fetch_url_build_cache, 'index.html'))
try:
_, _, json_file = web_util.read_from_url(keys_index)
json_index = sjson.load(codecs.getreader('utf-8')(json_file))
except (URLError, web_util.SpackWebError) as url_err:
if web_util.url_exists(keys_index):
err_msg = [
'Unable to find public keys in {0},',
' caught exception attempting to read from {1}.',
]
for link in links:
if re.search(r'\.key', link) or re.search(r'\.pub', link):
keys.add(link)
tty.error(''.join(err_msg).format(
url_util.format(fetch_url),
url_util.format(keys_index)))
tty.debug(url_err)
continue
for fingerprint, key_attributes in json_index['keys'].items():
link = os.path.join(keys_url, fingerprint + '.pub')
for link in keys:
with Stage(link, name="build_cache", keep=True) as stage:
if os.path.exists(stage.save_filename) and force:
os.remove(stage.save_filename)
@ -941,16 +995,80 @@ def get_keys(install=False, trust=False, force=False):
stage.fetch()
except fs.FetchError:
continue
tty.debug('Found key {0}'.format(link))
tty.debug('Found key {0}'.format(fingerprint))
if install:
if trust:
Gpg.trust(stage.save_filename)
spack.util.gpg.trust(stage.save_filename)
tty.debug('Added this key to trusted keys.')
else:
tty.debug('Will not add this key to trusted keys.'
'Use -t to install all downloaded keys')
def push_keys(*mirrors, **kwargs):
"""
Upload pgp public keys to the given mirrors
"""
keys = kwargs.get('keys')
regenerate_index = kwargs.get('regenerate_index', False)
tmpdir = kwargs.get('tmpdir')
remove_tmpdir = False
keys = spack.util.gpg.public_keys(*(keys or []))
try:
for mirror in mirrors:
push_url = getattr(mirror, 'push_url', mirror)
keys_url = url_util.join(push_url,
_build_cache_relative_path,
_build_cache_keys_relative_path)
keys_local = url_util.local_file_path(keys_url)
verb = 'Writing' if keys_local else 'Uploading'
tty.debug('{0} public keys to {1}'.format(
verb, url_util.format(push_url)))
if keys_local: # mirror is local, don't bother with the tmpdir
prefix = keys_local
mkdirp(keys_local)
else:
# A tmp dir is created for the first mirror that is non-local.
# On the off-hand chance that all the mirrors are local, then
# we can avoid the need to create a tmp dir.
if tmpdir is None:
tmpdir = tempfile.mkdtemp()
remove_tmpdir = True
prefix = tmpdir
for fingerprint in keys:
tty.debug(' ' + fingerprint)
filename = fingerprint + '.pub'
export_target = os.path.join(prefix, filename)
spack.util.gpg.export_keys(export_target, fingerprint)
# If mirror is local, the above export writes directly to the
# mirror (export_target points directly to the mirror).
#
# If not, then export_target is a tmpfile that needs to be
# uploaded to the mirror.
if not keys_local:
spack.util.web.push_to_url(
export_target,
url_util.join(keys_url, filename),
keep_original=False)
if regenerate_index:
if keys_local:
generate_key_index(keys_url)
else:
generate_key_index(keys_url, tmpdir)
finally:
if remove_tmpdir:
shutil.rmtree(tmpdir)
def needs_rebuild(spec, mirror_url, rebuild_on_errors=False):
if not spec.concrete:
raise ValueError('spec must be concrete to check against mirror')

68
lib/spack/spack/cmd/gpg.py
View file

@ -6,9 +6,10 @@
import os
import argparse
import spack.binary_distribution
import spack.cmd.common.arguments as arguments
import spack.paths
from spack.util.gpg import Gpg
import spack.util.gpg
description = "handle GPG actions for spack"
section = "packaging"
@ -81,37 +82,64 @@ def setup_parser(subparser):
'all secret keys if unspecified')
export.set_defaults(func=gpg_export)
publish = subparsers.add_parser('publish', help=gpg_publish.__doc__)
output = publish.add_mutually_exclusive_group(required=True)
output.add_argument('-d', '--directory',
metavar='directory',
type=str,
help="local directory where " +
"keys will be published.")
output.add_argument('-m', '--mirror-name',
metavar='mirror-name',
type=str,
help="name of the mirror where " +
"keys will be published.")
output.add_argument('--mirror-url',
metavar='mirror-url',
type=str,
help="URL of the mirror where " +
"keys will be published.")
publish.add_argument('--rebuild-index', action='store_true',
default=False, help=(
"Regenerate buildcache key index "
"after publishing key(s)"))
publish.add_argument('keys', nargs='*',
help='the keys to publish; '
'all public keys if unspecified')
publish.set_defaults(func=gpg_publish)
def gpg_create(args):
"""create a new key"""
if args.export:
old_sec_keys = Gpg.signing_keys()
Gpg.create(name=args.name, email=args.email,
comment=args.comment, expires=args.expires)
old_sec_keys = spack.util.gpg.signing_keys()
spack.util.gpg.create(name=args.name, email=args.email,
comment=args.comment, expires=args.expires)
if args.export:
new_sec_keys = set(Gpg.signing_keys())
new_sec_keys = set(spack.util.gpg.signing_keys())
new_keys = new_sec_keys.difference(old_sec_keys)
Gpg.export_keys(args.export, *new_keys)
spack.util.gpg.export_keys(args.export, *new_keys)
def gpg_export(args):
"""export a secret key"""
keys = args.keys
if not keys:
keys = Gpg.signing_keys()
Gpg.export_keys(args.location, *keys)
keys = spack.util.gpg.signing_keys()
spack.util.gpg.export_keys(args.location, *keys)
def gpg_list(args):
"""list keys available in the keyring"""
Gpg.list(args.trusted, args.signing)
spack.util.gpg.list(args.trusted, args.signing)
def gpg_sign(args):
"""sign a package"""
key = args.key
if key is None:
keys = Gpg.signing_keys()
keys = spack.util.gpg.signing_keys()
if len(keys) == 1:
key = keys[0]
elif not keys:
@ -123,12 +151,12 @@ def gpg_sign(args):
if not output:
output = args.spec[0] + '.asc'
# TODO: Support the package format Spack creates.
Gpg.sign(key, ' '.join(args.spec), output, args.clearsign)
spack.util.gpg.sign(key, ' '.join(args.spec), output, args.clearsign)
def gpg_trust(args):
"""add a key to the keyring"""
Gpg.trust(args.keyfile)
spack.util.gpg.trust(args.keyfile)
def gpg_init(args):
@ -141,12 +169,12 @@ def gpg_init(args):
for filename in filenames:
if not filename.endswith('.key'):
continue
Gpg.trust(os.path.join(root, filename))
spack.util.gpg.trust(os.path.join(root, filename))
def gpg_untrust(args):
"""remove a key from the keyring"""
Gpg.untrust(args.signing, *args.keys)
spack.util.gpg.untrust(args.signing, *args.keys)
def gpg_verify(args):
@ -155,7 +183,17 @@ def gpg_verify(args):
signature = args.signature
if signature is None:
signature = args.spec[0] + '.asc'
Gpg.verify(signature, ' '.join(args.spec))
spack.util.gpg.verify(signature, ' '.join(args.spec))
def gpg_publish(args):
"""publish public keys to a build cache"""
# TODO(opadron): switch to using the mirror args once #17547 is merged
mirror = args.directory
spack.binary_distribution.push_keys(
mirror, keys=args.keys, regenerate_index=args.rebuild_index)
def gpg(parser, args):

41
lib/spack/spack/test/bindist.py
View file

@ -19,8 +19,10 @@
import spack.cmd.install as install
import spack.cmd.uninstall as uninstall
import spack.cmd.mirror as mirror
from spack.spec import Spec
import spack.mirror
import spack.util.gpg
from spack.directory_layout import YamlDirectoryLayout
from spack.spec import Spec
def_install_path_scheme = '${ARCHITECTURE}/${COMPILERNAME}-${COMPILERVER}/${PACKAGE}-${VERSION}-${HASH}' # noqa: E501
@ -469,3 +471,40 @@ def test_relative_rpaths_install_nondefault(tmpdir,
margs = mparser.parse_args(
['rm', '--scope', 'site', 'test-mirror-rel'])
mirror.mirror(mparser, margs)
def test_push_and_fetch_keys(mock_gnupghome):
testpath = str(mock_gnupghome)
mirror = os.path.join(testpath, 'mirror')
mirrors = {'test-mirror': mirror}
mirrors = spack.mirror.MirrorCollection(mirrors)
mirror = spack.mirror.Mirror('file://' + mirror)
gpg_dir1 = os.path.join(testpath, 'gpg1')
gpg_dir2 = os.path.join(testpath, 'gpg2')
# dir 1: create a new key, record its fingerprint, and push it to a new
# mirror
with spack.util.gpg.gnupg_home_override(gpg_dir1):
spack.util.gpg.create(name='test-key',
email='fake@test.key',
expires='0',
comment=None)
keys = spack.util.gpg.public_keys()
assert len(keys) == 1
fpr = keys[0]
bindist.push_keys(mirror, keys=[fpr], regenerate_index=True)
# dir 2: import the key from the mirror, and confirm that its fingerprint
# matches the one created above
with spack.util.gpg.gnupg_home_override(gpg_dir2):
assert len(spack.util.gpg.public_keys()) == 0
bindist.get_keys(mirrors=mirrors, install=True, trust=True, force=True)
new_keys = spack.util.gpg.public_keys()
assert len(new_keys) == 1
assert new_keys[0] == fpr

11
lib/spack/spack/test/ci.py
View file

@ -53,15 +53,8 @@ def test_urlencode_string():
assert(s_enc == 'Spack+Test+Project')
def has_gpg():
try:
gpg = spack.util.gpg.Gpg.gpg()
except spack.util.gpg.SpackGPGError:
gpg = None
return bool(gpg)
@pytest.mark.skipif(not has_gpg(), reason='This test requires gpg')
@pytest.mark.skipif(not spack.util.gpg.has_gpg(),
reason='This test requires gpg')
def test_import_signing_key(mock_gnupghome):
signing_key_dir = spack_paths.mock_gpg_keys_path
signing_key_path = os.path.join(signing_key_dir, 'package-signing-key')

11
lib/spack/spack/test/cmd/ci.py
View file

@ -35,14 +35,6 @@
git = exe.which('git', required=True)
def has_gpg():
try:
gpg = spack.util.gpg.Gpg.gpg()
except spack.util.gpg.SpackGPGError:
gpg = None
return bool(gpg)
@pytest.fixture()
def env_deactivate():
yield
@ -690,7 +682,8 @@ def test_ci_rebuild_basic(tmpdir, mutable_mock_env_path, env_deactivate,
@pytest.mark.disable_clean_stage_check
@pytest.mark.skipif(not has_gpg(), reason='This test requires gpg')
@pytest.mark.skipif(not spack.util.gpg.has_gpg(),
reason='This test requires gpg')
def test_push_mirror_contents(tmpdir, mutable_mock_env_path, env_deactivate,
install_mockery, mock_packages, mock_fetch,
mock_stage, mock_gnupghome):

36
lib/spack/spack/test/cmd/gpg.py
View file

@ -29,39 +29,35 @@
('gpg2', 'gpg (GnuPG) 2.2.19'), # gpg2 command
])
def test_find_gpg(cmd_name, version, tmpdir, mock_gnupghome, monkeypatch):
TEMPLATE = ('#!/bin/sh\n'
'echo "{version}"\n')
with tmpdir.as_cwd():
with open(cmd_name, 'w') as f:
f.write("""\
#!/bin/sh
echo "{version}"
""".format(version=version))
fs.set_executable(cmd_name)
for fname in (cmd_name, 'gpgconf'):
with open(fname, 'w') as f:
f.write(TEMPLATE.format(version=version))
fs.set_executable(fname)
monkeypatch.setitem(os.environ, "PATH", str(tmpdir))
if version == 'undetectable' or version.endswith('1.3.4'):
with pytest.raises(spack.util.gpg.SpackGPGError):
exe = spack.util.gpg.Gpg.gpg()
spack.util.gpg.ensure_gpg(reevaluate=True)
else:
exe = spack.util.gpg.Gpg.gpg()
assert isinstance(exe, spack.util.executable.Executable)
spack.util.gpg.ensure_gpg(reevaluate=True)
gpg_exe = spack.util.gpg.get_global_gpg_instance().gpg_exe
assert isinstance(gpg_exe, spack.util.executable.Executable)
gpgconf_exe = spack.util.gpg.get_global_gpg_instance().gpgconf_exe
assert isinstance(gpgconf_exe, spack.util.executable.Executable)
def test_no_gpg_in_path(tmpdir, mock_gnupghome, monkeypatch):
monkeypatch.setitem(os.environ, "PATH", str(tmpdir))
with pytest.raises(spack.util.gpg.SpackGPGError):
spack.util.gpg.Gpg.gpg()
def has_gpg():
try:
gpg = spack.util.gpg.Gpg.gpg()
except spack.util.gpg.SpackGPGError:
gpg = None
return bool(gpg)
spack.util.gpg.ensure_gpg(reevaluate=True)
@pytest.mark.maybeslow
@pytest.mark.skipif(not has_gpg(),
@pytest.mark.skipif(not spack.util.gpg.has_gpg(),
reason='These tests require gnupg2')
def test_gpg(tmpdir, mock_gnupghome):
# Verify a file with an empty keyring.
@ -103,7 +99,7 @@ def test_gpg(tmpdir, mock_gnupghome):
'--export', str(keypath),
'Spack testing 1',
'spack@googlegroups.com')
keyfp = spack.util.gpg.Gpg.signing_keys()[0]
keyfp = spack.util.gpg.signing_keys()[0]
# List the keys.
# TODO: Test the output here.

6
lib/spack/spack/test/conftest.py
View file

@ -781,10 +781,8 @@ def mock_gnupghome(monkeypatch):
# This comes up because tmp paths on macOS are already long-ish, and
# pytest makes them longer.
short_name_tmpdir = tempfile.mkdtemp()
monkeypatch.setattr(spack.util.gpg, 'GNUPGHOME', short_name_tmpdir)
monkeypatch.setattr(spack.util.gpg.Gpg, '_gpg', None)
yield
with spack.util.gpg.gnupg_home_override(short_name_tmpdir):
yield short_name_tmpdir
# clean up, since we are doing this manually
shutil.rmtree(short_name_tmpdir)

25
lib/spack/spack/test/packaging.py
View file

@ -20,6 +20,7 @@
import spack.store
import spack.binary_distribution as bindist
import spack.cmd.buildcache as buildcache
import spack.util.gpg
from spack.spec import Spec
from spack.paths import mock_gpg_keys_path
from spack.fetch_strategy import URLFetchStrategy, FetchStrategyComposite
@ -31,14 +32,6 @@
from spack.relocate import file_is_relocatable
def has_gpg():
try:
gpg = spack.util.gpg.Gpg.gpg()
except spack.util.gpg.SpackGPGError:
gpg = None
return bool(gpg)
def fake_fetchify(url, pkg):
"""Fake the URL for a package so it downloads from a file."""
fetcher = FetchStrategyComposite()
@ -46,7 +39,8 @@ def fake_fetchify(url, pkg):
pkg.fetcher = fetcher
@pytest.mark.skipif(not has_gpg(), reason='This test requires gpg')
@pytest.mark.skipif(not spack.util.gpg.has_gpg(),
reason='This test requires gpg')
@pytest.mark.usefixtures('install_mockery', 'mock_gnupghome')
def test_buildcache(mock_archive, tmpdir):
# tweak patchelf to only do a download
@ -101,12 +95,9 @@ def test_buildcache(mock_archive, tmpdir):
create_args = ['create', '-a', '-f', '-d', mirror_path, pkghash]
# Create a private key to sign package with if gpg2 available
if spack.util.gpg.Gpg.gpg():
spack.util.gpg.Gpg.create(name='test key 1', expires='0',
email='spack@googlegroups.com',
comment='Spack test key')
else:
create_args.insert(create_args.index('-a'), '-u')
spack.util.gpg.create(name='test key 1', expires='0',
email='spack@googlegroups.com',
comment='Spack test key')
create_args.insert(create_args.index('-a'), '--rebuild-index')
@ -119,8 +110,6 @@ def test_buildcache(mock_archive, tmpdir):
pkg.do_uninstall(force=True)
install_args = ['install', '-a', '-f', pkghash]
if not spack.util.gpg.Gpg.gpg():
install_args.insert(install_args.index('-a'), '-u')
args = parser.parse_args(install_args)
# Test install
buildcache.buildcache(parser, args)
@ -144,8 +133,6 @@ def test_buildcache(mock_archive, tmpdir):
# Uninstall the package
pkg.do_uninstall(force=True)
if not spack.util.gpg.Gpg.gpg():
install_args.insert(install_args.index('-a'), '-u')
args = parser.parse_args(install_args)
buildcache.buildcache(parser, args)

3
lib/spack/spack/test/stage.py
View file

@ -839,7 +839,8 @@ def test_get_stage_root_in_spack(self, clear_stage_root):
assert 'spack' in path.split(os.path.sep)
# Make sure cached stage path value was changed appropriately
assert spack.stage._stage_root == test_path
assert spack.stage._stage_root in (
test_path, os.path.join(test_path, getpass.getuser()))
# Make sure the directory exists
assert os.path.isdir(spack.stage._stage_root)

32
lib/spack/spack/test/util/util_gpg.py
View file

@ -3,7 +3,10 @@
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
import spack.util.gpg as gpg
import os
import pytest
import spack.util.gpg
def test_parse_gpg_output_case_one():
@ -17,7 +20,7 @@ def test_parse_gpg_output_case_one():
uid:::::::AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA::Joe (Test) <j.s@s.com>:
ssb::2048:1:AAAAAAAAAAAAAAAA:AAAAAAAAAA::::::::::
"""
keys = gpg.parse_keys_output(output)
keys = spack.util.gpg.parse_secret_keys_output(output)
assert len(keys) == 2
assert keys[0] == 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'
@ -34,7 +37,7 @@ def test_parse_gpg_output_case_two():
fpr:::::::::YYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYY:
grp:::::::::AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA:
"""
keys = gpg.parse_keys_output(output)
keys = spack.util.gpg.parse_secret_keys_output(output)
assert len(keys) == 1
assert keys[0] == 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'
@ -53,8 +56,29 @@ def test_parse_gpg_output_case_three():
ssb::2048:1:AAAAAAAAAAAAAAAA:AAAAAAAAAA::::::::::
fpr:::::::::ZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZ:"""
keys = gpg.parse_keys_output(output)
keys = spack.util.gpg.parse_secret_keys_output(output)
assert len(keys) == 2
assert keys[0] == 'WWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWW'
assert keys[1] == 'YYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYY'
@pytest.mark.skipif(not spack.util.gpg.GpgConstants.user_run_dir,
reason='This test requires /var/run/user/$(id -u)')
def test_really_long_gnupg_home_dir(tmpdir):
N = 960
tdir = str(tmpdir)
while len(tdir) < N:
tdir = os.path.join(tdir, 'filler')
tdir = tdir[:N].rstrip(os.sep)
tdir += '0' * (N - len(tdir))
with spack.util.gpg.gnupg_home_override(tdir):
spack.util.gpg.create(name='Spack testing 1',
email='test@spack.io',
comment='Spack testing key',
expires='0')
spack.util.gpg.list(True, True)

28
lib/spack/spack/test/web.py
View file

@ -167,3 +167,31 @@ def test_get_header():
# If there isn't even a fuzzy match, raise KeyError
with pytest.raises(KeyError):
spack.util.web.get_header(headers, 'ContentLength')
def test_list_url(tmpdir):
testpath = str(tmpdir)
os.mkdir(os.path.join(testpath, 'dir'))
with open(os.path.join(testpath, 'file-0.txt'), 'w'):
pass
with open(os.path.join(testpath, 'file-1.txt'), 'w'):
pass
with open(os.path.join(testpath, 'file-2.txt'), 'w'):
pass
with open(os.path.join(testpath, 'dir', 'another-file.txt'), 'w'):
pass
list_url = lambda recursive: list(sorted(
spack.util.web.list_url(testpath, recursive=recursive)))
assert list_url(False) == ['file-0.txt',
'file-1.txt',
'file-2.txt']
assert list_url(True) == ['dir/another-file.txt',
'file-0.txt',
'file-1.txt',
'file-2.txt']

444
lib/spack/spack/util/gpg.py
View file

@ -3,20 +3,73 @@
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
import contextlib
import errno
import functools
import os
import re
import llnl.util.lang
import spack.error
import spack.paths
import spack.util.executable
import spack.version
from spack.util.executable import which
_gnupg_version_re = r"^gpg \(GnuPG\) (.*)$"
GNUPGHOME = os.getenv('SPACK_GNUPGHOME', spack.paths.gpg_path)
def parse_keys_output(output):
_gnupg_version_re = r"^gpg(conf)? \(GnuPG\) (.*)$"
_gnupg_home_override = None
_global_gpg_instance = None
def get_gnupg_home(gnupg_home=None):
"""Returns the directory that should be used as the GNUPGHOME environment
variable when calling gpg.
If a [gnupg_home] is passed directly (and not None), that value will be
used.
Otherwise, if there is an override set (and it is not None), then that
value will be used.
Otherwise, if the environment variable "SPACK_GNUPGHOME" is set, then that
value will be used.
Otherwise, the default gpg path for Spack will be used.
See also: gnupg_home_override()
"""
return (gnupg_home or
_gnupg_home_override or
os.getenv('SPACK_GNUPGHOME') or
spack.paths.gpg_path)
@contextlib.contextmanager
def gnupg_home_override(new_gnupg_home):
global _gnupg_home_override
global _global_gpg_instance
old_gnupg_home_override = _gnupg_home_override
old_global_gpg_instance = _global_gpg_instance
_gnupg_home_override = new_gnupg_home
_global_gpg_instance = None
yield
_gnupg_home_override = old_gnupg_home_override
_global_gpg_instance = old_global_gpg_instance
def get_global_gpg_instance():
global _global_gpg_instance
if _global_gpg_instance is None:
_global_gpg_instance = Gpg()
return _global_gpg_instance
def parse_secret_keys_output(output):
keys = []
found_sec = False
for line in output.split('\n'):
@ -31,43 +84,230 @@ def parse_keys_output(output):
return keys
def parse_public_keys_output(output):
keys = []
found_pub = False
for line in output.split('\n'):
if found_pub:
if line.startswith('fpr'):
keys.append(line.split(':')[9])
found_pub = False
elif line.startswith('ssb'):
found_pub = False
elif line.startswith('pub'):
found_pub = True
return keys
cached_property = getattr(functools, 'cached_property', None)
# If older python version has no cached_property, emulate it here.
# TODO(opadron): maybe this shim should be moved to llnl.util.lang?
if not cached_property:
def cached_property(*args, **kwargs):
result = property(llnl.util.lang.memoized(*args, **kwargs))
attr = result.fget.__name__
@result.deleter
def result(self):
getattr(type(self), attr).fget.cache.pop((self,), None)
return result
class GpgConstants(object):
@cached_property
def target_version(self):
return spack.version.Version('2')
@cached_property
def gpgconf_string(self):
exe_str = spack.util.executable.which_string(
'gpgconf', 'gpg2conf', 'gpgconf2')
no_gpgconf_msg = (
'Spack requires gpgconf version >= 2\n'
' To install a suitable version using Spack, run\n'
' spack install gnupg@2:\n'
' and load it by running\n'
' spack load gnupg@2:')
if not exe_str:
raise SpackGPGError(no_gpgconf_msg)
exe = spack.util.executable.Executable(exe_str)
output = exe('--version', output=str)
match = re.search(_gnupg_version_re, output, re.M)
if not match:
raise SpackGPGError('Could not determine gpgconf version')
if spack.version.Version(match.group(2)) < self.target_version:
raise SpackGPGError(no_gpgconf_msg)
# ensure that the gpgconf we found can run "gpgconf --create-socketdir"
try:
exe('--dry-run', '--create-socketdir')
except spack.util.executable.ProcessError:
# no dice
exe_str = None
return exe_str
@cached_property
def gpg_string(self):
exe_str = spack.util.executable.which_string('gpg2', 'gpg')
no_gpg_msg = (
'Spack requires gpg version >= 2\n'
' To install a suitable version using Spack, run\n'
' spack install gnupg@2:\n'
' and load it by running\n'
' spack load gnupg@2:')
if not exe_str:
raise SpackGPGError(no_gpg_msg)
exe = spack.util.executable.Executable(exe_str)
output = exe('--version', output=str)
match = re.search(_gnupg_version_re, output, re.M)
if not match:
raise SpackGPGError('Could not determine gpg version')
if spack.version.Version(match.group(2)) < self.target_version:
raise SpackGPGError(no_gpg_msg)
return exe_str
@cached_property
def user_run_dir(self):
# Try to ensure that (/var)/run/user/$(id -u) exists so that
# `gpgconf --create-socketdir` can be run later.
#
# NOTE(opadron): This action helps prevent a large class of
# "file-name-too-long" errors in gpg.
try:
has_suitable_gpgconf = bool(GpgConstants.gpgconf_string)
except SpackGPGError:
has_suitable_gpgconf = False
# If there is no suitable gpgconf, don't even bother trying to
# precreate a user run dir.
if not has_suitable_gpgconf:
return None
result = None
for var_run in ('/run', '/var/run'):
if not os.path.exists(var_run):
continue
var_run_user = os.path.join(var_run, 'user')
try:
if not os.path.exists(var_run_user):
os.mkdir(var_run_user)
os.chmod(var_run_user, 0o777)
user_dir = os.path.join(var_run_user, str(os.getuid()))
if not os.path.exists(user_dir):
os.mkdir(user_dir)
os.chmod(user_dir, 0o700)
# If the above operation fails due to lack of permissions, then
# just carry on without running gpgconf and hope for the best.
#
# NOTE(opadron): Without a dir in which to create a socket for IPC,
# gnupg may fail if GNUPGHOME is set to a path that
# is too long, where "too long" in this context is
# actually quite short; somewhere in the
# neighborhood of more than 100 characters.
#
# TODO(opadron): Maybe a warning should be printed in this case?
except OSError as exc:
if exc.errno not in (errno.EPERM, errno.EACCES):
raise
user_dir = None
# return the last iteration that provides a usable user run dir
if user_dir is not None:
result = user_dir
return result
def clear(self):
for attr in ('gpgconf_string', 'gpg_string', 'user_run_dir'):
try:
delattr(self, attr)
except AttributeError:
pass
GpgConstants = GpgConstants()
def ensure_gpg(reevaluate=False):
if reevaluate:
GpgConstants.clear()
if GpgConstants.user_run_dir is not None:
GpgConstants.gpgconf_string
GpgConstants.gpg_string
return True
def has_gpg(*args, **kwargs):
try:
return ensure_gpg(*args, **kwargs)
except SpackGPGError:
return False
# NOTE(opadron): When adding methods to this class, consider adding convenience
# wrapper functions further down in this file.
class Gpg(object):
_gpg = None
def __init__(self, gnupg_home=None):
self.gnupg_home = get_gnupg_home(gnupg_home)
@staticmethod
def gpg():
# TODO: Support loading up a GPG environment from a built gpg.
if Gpg._gpg is None:
gpg = which('gpg2', 'gpg')
@cached_property
def prep(self):
# Make sure that suitable versions of gpgconf and gpg are available
ensure_gpg()
if not gpg:
raise SpackGPGError("Spack requires gpg version 2 or higher.")
# Make sure that the GNUPGHOME exists
if not os.path.exists(self.gnupg_home):
os.makedirs(self.gnupg_home)
os.chmod(self.gnupg_home, 0o700)
# ensure that the version is actually >= 2 if we find 'gpg'
if gpg.name == 'gpg':
output = gpg('--version', output=str)
match = re.search(_gnupg_version_re, output, re.M)
if not os.path.isdir(self.gnupg_home):
raise SpackGPGError(
'GNUPGHOME "{0}" exists and is not a directory'.format(
self.gnupg_home))
if not match:
raise SpackGPGError("Couldn't determine version of gpg")
if GpgConstants.user_run_dir is not None:
self.gpgconf_exe('--create-socketdir')
v = spack.version.Version(match.group(1))
if v < spack.version.Version('2'):
raise SpackGPGError("Spack requires GPG version >= 2")
return True
# make the GNU PG path if we need to
# TODO: does this need to be in the spack directory?
# we should probably just use GPG's regular conventions
if not os.path.exists(GNUPGHOME):
os.makedirs(GNUPGHOME)
os.chmod(GNUPGHOME, 0o700)
gpg.add_default_env('GNUPGHOME', GNUPGHOME)
@cached_property
def gpgconf_exe(self):
exe = spack.util.executable.Executable(GpgConstants.gpgconf_string)
exe.add_default_env('GNUPGHOME', self.gnupg_home)
return exe
Gpg._gpg = gpg
return Gpg._gpg
@cached_property
def gpg_exe(self):
exe = spack.util.executable.Executable(GpgConstants.gpg_string)
exe.add_default_env('GNUPGHOME', self.gnupg_home)
return exe
@classmethod
def create(cls, **kwargs):
def __call__(self, *args, **kwargs):
if self.prep:
return self.gpg_exe(*args, **kwargs)
def create(self, **kwargs):
r, w = os.pipe()
r = os.fdopen(r, 'r')
w = os.fdopen(w, 'w')
@ -83,64 +323,108 @@ def create(cls, **kwargs):
%%commit
''' % kwargs)
w.close()
cls.gpg()('--gen-key', '--batch', input=r)
self('--gen-key', '--batch', input=r)
r.close()
@classmethod
def signing_keys(cls):
output = cls.gpg()('--list-secret-keys', '--with-colons',
'--fingerprint', '--fingerprint', output=str)
return parse_keys_output(output)
def signing_keys(self, *args):
output = self('--list-secret-keys', '--with-colons', '--fingerprint',
*args, output=str)
return parse_secret_keys_output(output)
@classmethod
def export_keys(cls, location, *keys):
cls.gpg()('--armor', '--export', '--output', location, *keys)
def public_keys(self, *args):
output = self('--list-public-keys', '--with-colons', '--fingerprint',
*args, output=str)
return parse_public_keys_output(output)
@classmethod
def trust(cls, keyfile):
cls.gpg()('--import', keyfile)
def export_keys(self, location, *keys):
self('--batch', '--yes',
'--armor', '--export',
'--output', location, *keys)
@classmethod
def untrust(cls, signing, *keys):
args = [
'--yes',
'--batch',
]
def trust(self, keyfile):
self('--import', keyfile)
def untrust(self, signing, *keys):
if signing:
signing_args = args + ['--delete-secret-keys'] + list(keys)
cls.gpg()(*signing_args)
args.append('--delete-keys')
args.extend(keys)
cls.gpg()(*args)
skeys = self.signing_keys(*keys)
self('--batch', '--yes', '--delete-secret-keys', *skeys)
@classmethod
def sign(cls, key, file, output, clearsign=False):
args = [
'--armor',
'--default-key', key,
'--output', output,
file,
]
if clearsign:
args.insert(0, '--clearsign')
else:
args.insert(0, '--detach-sign')
cls.gpg()(*args)
pkeys = self.public_keys(*keys)
self('--batch', '--yes', '--delete-keys', *pkeys)
@classmethod
def verify(cls, signature, file, suppress_warnings=False):
if suppress_warnings:
cls.gpg()('--verify', signature, file, error=str)
else:
cls.gpg()('--verify', signature, file)
def sign(self, key, file, output, clearsign=False):
self(('--clearsign' if clearsign else '--detach-sign'),
'--armor', '--default-key', key,
'--output', output, file)
@classmethod
def list(cls, trusted, signing):
def verify(self, signature, file, suppress_warnings=False):
self('--verify', signature, file,
**({'error': str} if suppress_warnings else {}))
def list(self, trusted, signing):
if trusted:
cls.gpg()('--list-public-keys')
self('--list-public-keys')
if signing:
cls.gpg()('--list-secret-keys')
self('--list-secret-keys')
class SpackGPGError(spack.error.SpackError):
"""Class raised when GPG errors are detected."""
# Convenience wrappers for methods of the Gpg class
# __call__ is a bit of a special case, since the Gpg instance is, itself, the
# "thing" that is being called.
@functools.wraps(Gpg.__call__)
def gpg(*args, **kwargs):
return get_global_gpg_instance()(*args, **kwargs)
gpg.name = 'gpg'
@functools.wraps(Gpg.create)
def create(*args, **kwargs):
return get_global_gpg_instance().create(*args, **kwargs)
@functools.wraps(Gpg.signing_keys)
def signing_keys(*args, **kwargs):
return get_global_gpg_instance().signing_keys(*args, **kwargs)
@functools.wraps(Gpg.public_keys)
def public_keys(*args, **kwargs):
return get_global_gpg_instance().public_keys(*args, **kwargs)
@functools.wraps(Gpg.export_keys)
def export_keys(*args, **kwargs):
return get_global_gpg_instance().export_keys(*args, **kwargs)
@functools.wraps(Gpg.trust)
def trust(*args, **kwargs):
return get_global_gpg_instance().trust(*args, **kwargs)
@functools.wraps(Gpg.untrust)
def untrust(*args, **kwargs):
return get_global_gpg_instance().untrust(*args, **kwargs)
@functools.wraps(Gpg.sign)
def sign(*args, **kwargs):
return get_global_gpg_instance().sign(*args, **kwargs)
@functools.wraps(Gpg.verify)
def verify(*args, **kwargs):
return get_global_gpg_instance().verify(*args, **kwargs)
@functools.wraps(Gpg.list)
def list(*args, **kwargs):
return get_global_gpg_instance().list(*args, **kwargs)

18
lib/spack/spack/util/web.py
View file

@ -224,7 +224,7 @@ def url_exists(url):
try:
read_from_url(url)
return True
except URLError:
except (SpackWebError, URLError):
return False
@ -295,15 +295,27 @@ def _iter_s3_prefix(client, url, num_entries=1024):
break
def list_url(url):
def _iter_local_prefix(path):
for root, _, files in os.walk(path):
for f in files:
yield os.path.relpath(os.path.join(root, f), path)
def list_url(url, recursive=False):
url = url_util.parse(url)
local_path = url_util.local_file_path(url)
if local_path:
return os.listdir(local_path)
if recursive:
return list(_iter_local_prefix(local_path))
return [subpath for subpath in os.listdir(local_path)
if os.path.isfile(os.path.join(local_path, subpath))]
if url.scheme == 's3':
s3 = s3_util.create_s3_session(url)
if recursive:
return list(_iter_s3_prefix(s3, url))
return list(set(
key.split('/', 1)[0]
for key in _iter_s3_prefix(s3, url)))

11
share/spack/spack-completion.bash
View file

@ -906,7 +906,7 @@ _spack_gpg() {
then
SPACK_COMPREPLY="-h --help"
else
SPACK_COMPREPLY="verify trust untrust sign create list init export"
SPACK_COMPREPLY="verify trust untrust sign create list init export publish"
fi
}
@ -972,6 +972,15 @@ _spack_gpg_export() {
fi
}
_spack_gpg_publish() {
if $list_options
then
SPACK_COMPREPLY="-h --help -d --directory -m --mirror-name --mirror-url --rebuild-index"
else
_keys
fi
}
_spack_graph() {
if $list_options
then