hlrs-software-stack/spack
7
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

package: Add fetch_options variable (#15317)

Browse source 
PR #15212 added a new connect_timeout option that can be overridden
using fetch_options but had to specified per-version. This adds a new
per-package variable that can be used to override fetch_options for
all versions in the package. This includes connect_timeout as well
as 'cookie' (e.g. for the jdk package).

Packages can combine package-level fetch_options with per-version
fetch_options, in which case the version fetch_options completely
override the package-level fetch_options.

This commit includes tests for the added behavior.
This commit is contained in:
Michael Kuhn 2020-03-13 19:41:19 +01:00 committed by GitHub
parent 7b00712fff
commit 14441e00dc
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
14 changed files with 121 additions and 41 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
lib/spack/spack/fetch_strategy.py
View file

@ -256,7 +256,7 @@ def __init__(self, url=None, checksum=None, **kwargs):
self.digest = kwargs[h]
self.expand_archive = kwargs.get('expand', True)
self.extra_options = kwargs.get('fetch_options', [])
self.extra_options = kwargs.get('fetch_options', {})
self._curl = None
self.extension = kwargs.get('extension', None)
@ -1247,7 +1247,8 @@ def _check_version_attributes(fetcher, pkg, version):
def _extrapolate(pkg, version):
"""Create a fetcher from an extrapolated URL for this version."""
try:
return URLFetchStrategy(pkg.url_for_version(version))
return URLFetchStrategy(pkg.url_for_version(version),
fetch_options=pkg.fetch_options)
except spack.package.NoURLError:
msg = ("Can't extrapolate a URL for version %s "
"because package %s defines no URLs")
@ -1267,6 +1268,7 @@ def _from_merged_attrs(fetcher, pkg, version):
url = getattr(pkg, fetcher.url_attr)
attrs = {fetcher.url_attr: url}
attrs['fetch_options'] = pkg.fetch_options
attrs.update(pkg.versions[version])
return fetcher(**attrs)
@ -1289,8 +1291,10 @@ def for_package_version(pkg, version):
if version not in pkg.versions:
return _extrapolate(pkg, version)
# Set package args first so version args can override them
args = {'fetch_options': pkg.fetch_options}
# Grab a dict of args out of the package version dict
args = pkg.versions[version]
args.update(pkg.versions[version])
# If the version specifies a `url_attr` directly, use that.
for fetcher in all_strategies:
@ -1370,7 +1374,8 @@ def from_list_url(pkg):
args.get('checksum'))
# construct a fetcher
return URLFetchStrategy(url_from_list, checksum)
return URLFetchStrategy(url_from_list, checksum,
fetch_options=pkg.fetch_options)
except KeyError as e:
tty.debug(e)
tty.msg("Cannot find version %s in url_list" % pkg.version)

3
lib/spack/spack/package.py
View file

@ -477,6 +477,9 @@ class PackageBase(with_metaclass(PackageMeta, PackageViewMixin, object)):
#: This is currently only used by package sanity tests.
manual_download = False
#: Set of additional options used when fetching package versions.
fetch_options = {}
#
# Set default licensing information
#

21
lib/spack/spack/test/packages.py
View file

@ -402,3 +402,24 @@ def test_bundle_patch_directive(mock_directive_bundle,
match="Patches are not allowed"):
patch = spack.directives.patch('mock/patch.txt')
patch(mock_directive_bundle)
def test_fetch_options(mock_packages, config):
"""Test fetch options inference."""
pkg = spack.repo.get('fetch-options')
fetcher = spack.fetch_strategy.for_package_version(pkg, '1.0')
assert isinstance(fetcher, spack.fetch_strategy.URLFetchStrategy)
assert fetcher.digest == 'abc10'
assert fetcher.extra_options == {'timeout': 42, 'cookie': 'foobar'}
fetcher = spack.fetch_strategy.for_package_version(pkg, '1.1')
assert isinstance(fetcher, spack.fetch_strategy.URLFetchStrategy)
assert fetcher.digest == 'abc11'
assert fetcher.extra_options == {'timeout': 65}
fetcher = spack.fetch_strategy.for_package_version(pkg, '1.2')
assert isinstance(fetcher, spack.fetch_strategy.URLFetchStrategy)
assert fetcher.digest == 'abc12'
assert fetcher.extra_options == {'cookie': 'baz'}

19
lib/spack/spack/test/url_fetch.py
View file

@ -26,10 +26,10 @@ def checksum_type(request):
@pytest.fixture
def pkg_factory():
Pkg = collections.namedtuple(
'Pkg', ['url_for_version', 'urls', 'url', 'versions']
'Pkg', ['url_for_version', 'urls', 'url', 'versions', 'fetch_options']
)
def factory(url, urls):
def factory(url, urls, fetch_options={}):
def fn(v):
main_url = url or urls[0]
@ -37,7 +37,8 @@ def fn(v):
return Pkg(
url_for_version=fn, url=url, urls=urls,
versions=collections.defaultdict(dict)
versions=collections.defaultdict(dict),
fetch_options=fetch_options
)
return factory
@ -130,6 +131,10 @@ def test_from_list_url(mock_packages, config, spec, url, digest):
assert isinstance(fetch_strategy, fs.URLFetchStrategy)
assert os.path.basename(fetch_strategy.url) == url
assert fetch_strategy.digest == digest
assert fetch_strategy.extra_options == {}
pkg.fetch_options = {'timeout': 60}
fetch_strategy = fs.from_list_url(pkg)
assert fetch_strategy.extra_options == {'timeout': 60}
def test_from_list_url_unspecified(mock_packages, config):
@ -142,6 +147,10 @@ def test_from_list_url_unspecified(mock_packages, config):
assert isinstance(fetch_strategy, fs.URLFetchStrategy)
assert os.path.basename(fetch_strategy.url) == 'foo-2.0.0.tar.gz'
assert fetch_strategy.digest is None
assert fetch_strategy.extra_options == {}
pkg.fetch_options = {'timeout': 60}
fetch_strategy = fs.from_list_url(pkg)
assert fetch_strategy.extra_options == {'timeout': 60}
def test_nosource_from_list_url(mock_packages, config):
@ -191,3 +200,7 @@ def test_candidate_urls(pkg_factory, url, urls, version, expected):
pkg = pkg_factory(url, urls)
f = fs._from_merged_attrs(fs.URLFetchStrategy, pkg, version)
assert f.candidate_urls == expected
assert f.extra_options == {}
pkg = pkg_factory(url, urls, fetch_options={'timeout': 60})
f = fs._from_merged_attrs(fs.URLFetchStrategy, pkg, version)
assert f.extra_options == {'timeout': 60}

22
var/spack/repos/builtin.mock/packages/fetch-options/package.py Normal file
View file

@ -0,0 +1,22 @@
# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
from spack import *
class FetchOptions(Package):
"""Mock package with fetch_options."""
homepage = "http://www.fetch-options-example.com"
url = 'https://example.com/some/tarball-1.0.tar.gz'
fetch_options = {'timeout': 42, 'cookie': 'foobar'}
timeout = {'timeout': 65}
cookie = {'cookie': 'baz'}
version('1.2', 'abc12', fetch_options=cookie)
version('1.1', 'abc11', fetch_options=timeout)
version('1.0', 'abc10')

6
var/spack/repos/builtin/packages/bzip2/package.py
View file

@ -19,9 +19,9 @@ class Bzip2(Package):
# The server is sometimes a bit slow to respond
fetch_options = {'timeout': 60}
version('1.0.8', sha256='ab5a03176ee106d3f0fa90e381da478ddae405918153cca248e682cd0c4a2269', fetch_options=fetch_options)
version('1.0.7', sha256='e768a87c5b1a79511499beb41500bcc4caf203726fff46a6f5f9ad27fe08ab2b', fetch_options=fetch_options)
version('1.0.6', sha256='a2848f34fcd5d6cf47def00461fcb528a0484d8edef8208d6d2e2909dc61d9cd', fetch_options=fetch_options)
version('1.0.8', sha256='ab5a03176ee106d3f0fa90e381da478ddae405918153cca248e682cd0c4a2269')
version('1.0.7', sha256='e768a87c5b1a79511499beb41500bcc4caf203726fff46a6f5f9ad27fe08ab2b')
version('1.0.6', sha256='a2848f34fcd5d6cf47def00461fcb528a0484d8edef8208d6d2e2909dc61d9cd')
variant('shared', default=True, description='Enables the build of shared libraries.')

3
var/spack/repos/builtin/packages/eclipse-gcj-parser/package.py
View file

@ -19,6 +19,9 @@ class EclipseGcjParser(Package):
maintainers = ['citibeth']
# The server is sometimes a bit slow to respond
fetch_options = {'timeout': 60}
version('4.8', sha256='98fd128f1d374d9e42fd9d4836bdd249c6d511ebc6c0df17fbc1b9df96c3d781', expand=False)
phases = ('build', 'install')

20
var/spack/repos/builtin/packages/elfutils/package.py
View file

@ -22,17 +22,17 @@ class Elfutils(AutotoolsPackage):
list_depth = 1
# Sourceware is often slow to respond.
timeout = {'timeout': 60}
fetch_options = {'timeout': 60}
version('0.178', sha256='31e7a00e96d4e9c4bda452e1f2cdac4daf8abd24f5e154dee232131899f3a0f2', fetch_options=timeout)
version('0.177', sha256='fa489deccbcae7d8c920f60d85906124c1989c591196d90e0fd668e3dc05042e', fetch_options=timeout)
version('0.176', sha256='eb5747c371b0af0f71e86215a5ebb88728533c3a104a43d4231963f308cd1023', fetch_options=timeout)
version('0.175', sha256='f7ef925541ee32c6d15ae5cb27da5f119e01a5ccdbe9fe57bf836730d7b7a65b', fetch_options=timeout)
version('0.174', sha256='cdf27e70076e10a29539d89e367101d516bc4aa11b0d7777fe52139e3fcad08a', fetch_options=timeout)
version('0.173', sha256='b76d8c133f68dad46250f5c223482c8299d454a69430d9aa5c19123345a000ff', fetch_options=timeout)
version('0.170', sha256='1f844775576b79bdc9f9c717a50058d08620323c1e935458223a12f249c9e066', fetch_options=timeout)
version('0.168', sha256='b88d07893ba1373c7dd69a7855974706d05377766568a7d9002706d5de72c276', fetch_options=timeout)
version('0.163', sha256='7c774f1eef329309f3b05e730bdac50013155d437518a2ec0e24871d312f2e23', fetch_options=timeout)
version('0.178', sha256='31e7a00e96d4e9c4bda452e1f2cdac4daf8abd24f5e154dee232131899f3a0f2')
version('0.177', sha256='fa489deccbcae7d8c920f60d85906124c1989c591196d90e0fd668e3dc05042e')
version('0.176', sha256='eb5747c371b0af0f71e86215a5ebb88728533c3a104a43d4231963f308cd1023')
version('0.175', sha256='f7ef925541ee32c6d15ae5cb27da5f119e01a5ccdbe9fe57bf836730d7b7a65b')
version('0.174', sha256='cdf27e70076e10a29539d89e367101d516bc4aa11b0d7777fe52139e3fcad08a')
version('0.173', sha256='b76d8c133f68dad46250f5c223482c8299d454a69430d9aa5c19123345a000ff')
version('0.170', sha256='1f844775576b79bdc9f9c717a50058d08620323c1e935458223a12f249c9e066')
version('0.168', sha256='b88d07893ba1373c7dd69a7855974706d05377766568a7d9002706d5de72c276')
version('0.163', sha256='7c774f1eef329309f3b05e730bdac50013155d437518a2ec0e24871d312f2e23')
# Libraries for reading compressed DWARF sections.
variant('bzip2', default=False,

6
var/spack/repos/builtin/packages/gcc/package.py
View file

@ -109,12 +109,16 @@ class Gcc(AutotoolsPackage, GNUMirrorPackage):
depends_on('zip', type='build', when='languages=java')
depends_on('cuda', when='+nvptx')
# The server is sometimes a bit slow to respond
timeout = {'timeout': 60}
resource(
name='newlib',
url='ftp://sourceware.org/pub/newlib/newlib-3.0.0.20180831.tar.gz',
sha256='3ad3664f227357df15ff34e954bfd9f501009a647667cd307bf0658aefd6eb5b',
destination='newlibsource',
when='+nvptx'
when='+nvptx',
fetch_options=timeout
)
# nvptx-tools does not seem to work as a dependency,

28
var/spack/repos/builtin/packages/jdk/package.py
View file

@ -24,9 +24,7 @@ class Jdk(Package):
# automate this process, we need to utilize these additional curl
# command-line options. See:
# http://stackoverflow.com/questions/10268583/how-to-automate-download-and-installation-of-java-jdk-on-linux
fetch_options = {
'cookie': 'oraclelicense=accept-securebackup-cookie'
}
fetch_options = {'cookie': 'oraclelicense=accept-securebackup-cookie'}
# To add the latest version, go to the homepage listed above,
# click "JDK Download", click "Accept License Agreement", right-click the
@ -34,29 +32,29 @@ class Jdk(Package):
# found in a link above. The build number can be deciphered from the URL.
# Alternatively, run `bin/java -version` after extracting. Replace '+'
# symbol in version with '_', otherwise it will be interpreted as a variant
version('12.0.2_10', sha256='2dde6fda89a4ec6e6560ed464e917861c9e40bf576e7a64856dafc55abaaff51', fetch_options=fetch_options,
version('12.0.2_10', sha256='2dde6fda89a4ec6e6560ed464e917861c9e40bf576e7a64856dafc55abaaff51',
url='https://download.oracle.com/otn-pub/java/jdk/12.0.2+10/e482c34c86bd4bf8b56c0b35558996b9/jdk-12.0.2_linux-x64_bin.tar.gz')
version('12.0.1_12', sha256='9fd6dcdaf2cfca7da59e39b009a0f5bcd53bec2fb16105f7ca8d689cdab68d75', fetch_options=fetch_options,
version('12.0.1_12', sha256='9fd6dcdaf2cfca7da59e39b009a0f5bcd53bec2fb16105f7ca8d689cdab68d75',
url='https://download.oracle.com/otn-pub/java/jdk/12.0.1+12/69cfe15208a647278a19ef0990eea691/jdk-12.0.1_linux-x64_bin.tar.gz')
version('11.0.2_9', sha256='7b4fd8ffcf53e9ff699d964a80e4abf9706b5bdb5644a765c2b96f99e3a2cdc8', fetch_options=fetch_options,
version('11.0.2_9', sha256='7b4fd8ffcf53e9ff699d964a80e4abf9706b5bdb5644a765c2b96f99e3a2cdc8',
url='https://download.oracle.com/otn-pub/java/jdk/11.0.2+9/f51449fcd52f4d52b93a989c5c56ed3c/jdk-11.0.2_linux-x64_bin.tar.gz')
version('11.0.1_13', sha256='e7fd856bacad04b6dbf3606094b6a81fa9930d6dbb044bbd787be7ea93abc885', fetch_options=fetch_options,
version('11.0.1_13', sha256='e7fd856bacad04b6dbf3606094b6a81fa9930d6dbb044bbd787be7ea93abc885',
url='https://download.oracle.com/otn-pub/java/jdk/11.0.1+13/90cf5d8f270a4347a95050320eef3fb7/jdk-11.0.1_linux-x64_bin.tar.gz')
version('10.0.2_13', sha256='6633c20d53c50c20835364d0f3e172e0cbbce78fff81867488f22a6298fa372b', fetch_options=fetch_options,
version('10.0.2_13', sha256='6633c20d53c50c20835364d0f3e172e0cbbce78fff81867488f22a6298fa372b',
url='https://download.oracle.com/otn-pub/java/jdk/10.0.2+13/19aef61b38124481863b1413dce1855f/jdk-10.0.2_linux-x64_bin.tar.gz')
version('10.0.1_10', sha256='ae8ed645e6af38432a56a847597ac61d4283b7536688dbab44ab536199d1e5a4', fetch_options=fetch_options,
version('10.0.1_10', sha256='ae8ed645e6af38432a56a847597ac61d4283b7536688dbab44ab536199d1e5a4',
url='https://download.oracle.com/otn-pub/java/jdk/10.0.1+10/fb4372174a714e6b8c52526dc134031e/jdk-10.0.1_linux-x64_bin.tar.gz')
version('1.8.0_241-b07', sha256='419d32677855f676076a25aed58e79432969142bbd778ff8eb57cb618c69e8cb', fetch_options=fetch_options,
version('1.8.0_241-b07', sha256='419d32677855f676076a25aed58e79432969142bbd778ff8eb57cb618c69e8cb',
url='https://download.oracle.com/otn-pub/java/jdk/8u241-b07/1f5b5a70bf22433b84d0e960903adac8/jdk-8u241-linux-x64.tar.gz')
version('1.8.0_231-b11', sha256='a011584a2c9378bf70c6903ef5fbf101b30b08937441dc2ec67932fb3620b2cf', fetch_options=fetch_options,
version('1.8.0_231-b11', sha256='a011584a2c9378bf70c6903ef5fbf101b30b08937441dc2ec67932fb3620b2cf',
url='https://download.oracle.com/otn-pub/java/jdk/8u231-b11/5b13a193868b4bf28bcb45c792fce896/jdk-8u231-linux-x64.tar.gz')
version('1.8.0_212-b10', sha256='3160c50aa8d8e081c8c7fe0f859ea452922eca5d2ae8f8ef22011ae87e6fedfb', fetch_options=fetch_options,
version('1.8.0_212-b10', sha256='3160c50aa8d8e081c8c7fe0f859ea452922eca5d2ae8f8ef22011ae87e6fedfb',
url='https://download.oracle.com/otn-pub/java/jdk/8u212-b10/59066701cf1a433da9770636fbc4c9aa/jdk-8u212-linux-x64.tar.gz')
version('1.8.0_202-b08', sha256='9a5c32411a6a06e22b69c495b7975034409fa1652d03aeb8eb5b6f59fd4594e0', fetch_options=fetch_options,
version('1.8.0_202-b08', sha256='9a5c32411a6a06e22b69c495b7975034409fa1652d03aeb8eb5b6f59fd4594e0',
url='https://download.oracle.com/otn-pub/java/jdk/8u202-b08/1961070e4c9b4e26a04e7f5a083f551e/jdk-8u202-linux-x64.tar.gz')
version('1.8.0_141-b15', sha256='041d5218fbea6cd7e81c8c15e51d0d32911573af2ed69e066787a8dc8a39ba4f', fetch_options=fetch_options,
version('1.8.0_141-b15', sha256='041d5218fbea6cd7e81c8c15e51d0d32911573af2ed69e066787a8dc8a39ba4f',
url='https://download.oracle.com/otn-pub/java/jdk/8u141-b15/336fa29ff2bb4ef291e347e091f7f4a7/jdk-8u141-linux-x64.tar.gz')
version('1.8.0_131-b11', sha256='62b215bdfb48bace523723cdbb2157c665e6a25429c73828a32f00e587301236', fetch_options=fetch_options,
version('1.8.0_131-b11', sha256='62b215bdfb48bace523723cdbb2157c665e6a25429c73828a32f00e587301236',
url='https://download.oracle.com/otn-pub/java/jdk/8u131-b11/d54c1d3a095b4ff2b6607d096fa80163/jdk-8u131-linux-x64.tar.gz')
provides('java@12', when='@12.0:12.999')

3
var/spack/repos/builtin/packages/libffi/package.py
View file

@ -17,8 +17,7 @@ class Libffi(AutotoolsPackage):
fetch_options = {'timeout': 60}
version('3.2.1', sha256='d06ebb8e1d9a22d19e38d63fdb83954253f39bedc5d46232a05645685722ca37',
url="https://sourceware.org/pub/libffi/libffi-3.2.1.tar.gz",
fetch_options=fetch_options)
url="https://sourceware.org/pub/libffi/libffi-3.2.1.tar.gz")
@property
def headers(self):

3
var/spack/repos/builtin/packages/lvm2/package.py
View file

@ -20,6 +20,9 @@ class Lvm2(AutotoolsPackage):
homepage = "https://www.sourceware.org/lvm2"
url = "https://sourceware.org/pub/lvm2/releases/LVM2.2.03.05.tgz"
# The server is sometimes a bit slow to respond
fetch_options = {'timeout': 60}
version('2.03.05', sha256='ca52815c999b20c6d25e3192f142f081b93d01f07b9d787e99664b169dba2700')
version('2.03.04', sha256='f151f36fc0039997d2d9369b607b9262568b1a268afe19fd1535807355402142')
version('2.03.03', sha256='cedefa63ec5ae1b62fedbfddfc30706c095be0fc7c6aaed6fd1c50bc8c840dde')

3
var/spack/repos/builtin/packages/valgrind/package.py
View file

@ -22,6 +22,9 @@ class Valgrind(AutotoolsPackage):
url = "https://sourceware.org/pub/valgrind/valgrind-3.13.0.tar.bz2"
git = "git://sourceware.org/git/valgrind.git"
# The server is sometimes a bit slow to respond
fetch_options = {'timeout': 60}
version('develop', branch='master')
version('3.15.0', sha256='417c7a9da8f60dd05698b3a7bc6002e4ef996f14c13f0ff96679a16873e78ab1')
version('3.14.0', sha256='037c11bfefd477cc6e9ebe8f193bb237fe397f7ce791b4a4ce3fa1c6a520baa5')

12
var/spack/repos/builtin/packages/verrou/package.py
View file

@ -32,18 +32,24 @@ class Verrou(AutotoolsPackage):
version('2.0.0', sha256='798df6e426ec57646a2a626d756b72f0171647ae5b07c982952dae2d71e26045')
version('1.1.0', sha256='b5105f61c65680f31551199cd143b2e15f412c34c821537998a7165e315dde2d')
# The server is sometimes a bit slow to respond
timeout = {'timeout': 60}
resource(name='valgrind-3.15.0',
url='https://sourceware.org/pub/valgrind/valgrind-3.15.0.tar.bz2',
sha256='417c7a9da8f60dd05698b3a7bc6002e4ef996f14c13f0ff96679a16873e78ab1',
when='@2.2.0:')
when='@2.2.0:',
fetch_options=timeout)
resource(name='valgrind-3.14.0',
url='https://sourceware.org/pub/valgrind/valgrind-3.14.0.tar.bz2',
sha256='037c11bfefd477cc6e9ebe8f193bb237fe397f7ce791b4a4ce3fa1c6a520baa5',
when='@2.1.0:2.1.99')
when='@2.1.0:2.1.99',
fetch_options=timeout)
resource(name='valgrind-3.13.0',
url='https://sourceware.org/pub/valgrind/valgrind-3.13.0.tar.bz2',
sha256='d76680ef03f00cd5e970bbdcd4e57fb1f6df7d2e2c071635ef2be74790190c3b',
when='@1.1.0:2.0.99')
when='@1.1.0:2.0.99',
fetch_options=timeout)
variant('fma', default=True,
description='Activates fused multiply-add support for Verrou')