package: Add fetch_options variable (#15317)

PR #15212 added a new connect_timeout option that can be overridden
using fetch_options but had to specified per-version. This adds a new
per-package variable that can be used to override fetch_options for
all versions in the package. This includes connect_timeout as well
as 'cookie' (e.g. for the jdk package).

Packages can combine package-level fetch_options with per-version
fetch_options, in which case the version fetch_options completely
override the package-level fetch_options.

This commit includes tests for the added behavior.

lib/spack/spack/fetch_strategy.py

@@ -256,7 +256,7 @@ def __init__(self, url=None, checksum=None, **kwargs):
                 self.digest = kwargs[h]
 
         self.expand_archive = kwargs.get('expand', True)
-        self.extra_options = kwargs.get('fetch_options', [])
+        self.extra_options = kwargs.get('fetch_options', {})
         self._curl = None
 
         self.extension = kwargs.get('extension', None)
@@ -1247,7 +1247,8 @@ def _check_version_attributes(fetcher, pkg, version):
 def _extrapolate(pkg, version):
     """Create a fetcher from an extrapolated URL for this version."""
     try:
-        return URLFetchStrategy(pkg.url_for_version(version))
+        return URLFetchStrategy(pkg.url_for_version(version),
+                                fetch_options=pkg.fetch_options)
     except spack.package.NoURLError:
         msg = ("Can't extrapolate a URL for version %s "
                "because package %s defines no URLs")
@@ -1267,6 +1268,7 @@ def _from_merged_attrs(fetcher, pkg, version):
         url = getattr(pkg, fetcher.url_attr)
         attrs = {fetcher.url_attr: url}
 
+    attrs['fetch_options'] = pkg.fetch_options
     attrs.update(pkg.versions[version])
     return fetcher(**attrs)
 
@@ -1289,8 +1291,10 @@ def for_package_version(pkg, version):
     if version not in pkg.versions:
         return _extrapolate(pkg, version)
 
+    # Set package args first so version args can override them
+    args = {'fetch_options': pkg.fetch_options}
     # Grab a dict of args out of the package version dict
-    args = pkg.versions[version]
+    args.update(pkg.versions[version])
 
     # If the version specifies a `url_attr` directly, use that.
     for fetcher in all_strategies:
@@ -1370,7 +1374,8 @@ def from_list_url(pkg):
                         args.get('checksum'))
 
                 # construct a fetcher
-                return URLFetchStrategy(url_from_list, checksum)
+                return URLFetchStrategy(url_from_list, checksum,
+                                        fetch_options=pkg.fetch_options)
             except KeyError as e:
                 tty.debug(e)
                 tty.msg("Cannot find version %s in url_list" % pkg.version)

lib/spack/spack/package.py

@@ -477,6 +477,9 @@ class PackageBase(with_metaclass(PackageMeta, PackageViewMixin, object)):
     #: This is currently only used by package sanity tests.
     manual_download = False
 
+    #: Set of additional options used when fetching package versions.
+    fetch_options = {}
+
     #
     # Set default licensing information
     #

lib/spack/spack/test/packages.py

@@ -402,3 +402,24 @@ def test_bundle_patch_directive(mock_directive_bundle,
                        match="Patches are not allowed"):
         patch = spack.directives.patch('mock/patch.txt')
         patch(mock_directive_bundle)
+
+
+def test_fetch_options(mock_packages, config):
+    """Test fetch options inference."""
+
+    pkg = spack.repo.get('fetch-options')
+
+    fetcher = spack.fetch_strategy.for_package_version(pkg, '1.0')
+    assert isinstance(fetcher, spack.fetch_strategy.URLFetchStrategy)
+    assert fetcher.digest == 'abc10'
+    assert fetcher.extra_options == {'timeout': 42, 'cookie': 'foobar'}
+
+    fetcher = spack.fetch_strategy.for_package_version(pkg, '1.1')
+    assert isinstance(fetcher, spack.fetch_strategy.URLFetchStrategy)
+    assert fetcher.digest == 'abc11'
+    assert fetcher.extra_options == {'timeout': 65}
+
+    fetcher = spack.fetch_strategy.for_package_version(pkg, '1.2')
+    assert isinstance(fetcher, spack.fetch_strategy.URLFetchStrategy)
+    assert fetcher.digest == 'abc12'
+    assert fetcher.extra_options == {'cookie': 'baz'}

lib/spack/spack/test/url_fetch.py

@@ -26,10 +26,10 @@ def checksum_type(request):
 @pytest.fixture
 def pkg_factory():
     Pkg = collections.namedtuple(
-        'Pkg', ['url_for_version', 'urls', 'url', 'versions']
+        'Pkg', ['url_for_version', 'urls', 'url', 'versions', 'fetch_options']
     )
 
-    def factory(url, urls):
+    def factory(url, urls, fetch_options={}):
 
         def fn(v):
             main_url = url or urls[0]
@@ -37,7 +37,8 @@ def fn(v):
 
         return Pkg(
             url_for_version=fn, url=url, urls=urls,
-            versions=collections.defaultdict(dict)
+            versions=collections.defaultdict(dict),
+            fetch_options=fetch_options
         )
 
     return factory
@@ -130,6 +131,10 @@ def test_from_list_url(mock_packages, config, spec, url, digest):
     assert isinstance(fetch_strategy, fs.URLFetchStrategy)
     assert os.path.basename(fetch_strategy.url) == url
     assert fetch_strategy.digest == digest
+    assert fetch_strategy.extra_options == {}
+    pkg.fetch_options = {'timeout': 60}
+    fetch_strategy = fs.from_list_url(pkg)
+    assert fetch_strategy.extra_options == {'timeout': 60}
 
 
 def test_from_list_url_unspecified(mock_packages, config):
@@ -142,6 +147,10 @@ def test_from_list_url_unspecified(mock_packages, config):
     assert isinstance(fetch_strategy, fs.URLFetchStrategy)
     assert os.path.basename(fetch_strategy.url) == 'foo-2.0.0.tar.gz'
     assert fetch_strategy.digest is None
+    assert fetch_strategy.extra_options == {}
+    pkg.fetch_options = {'timeout': 60}
+    fetch_strategy = fs.from_list_url(pkg)
+    assert fetch_strategy.extra_options == {'timeout': 60}
 
 
 def test_nosource_from_list_url(mock_packages, config):
@@ -191,3 +200,7 @@ def test_candidate_urls(pkg_factory, url, urls, version, expected):
     pkg = pkg_factory(url, urls)
     f = fs._from_merged_attrs(fs.URLFetchStrategy, pkg, version)
     assert f.candidate_urls == expected
+    assert f.extra_options == {}
+    pkg = pkg_factory(url, urls, fetch_options={'timeout': 60})
+    f = fs._from_merged_attrs(fs.URLFetchStrategy, pkg, version)
+    assert f.extra_options == {'timeout': 60}

var/spack/repos/builtin.mock/packages/fetch-options/package.py

@@ -0,0 +1,22 @@
+# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack import *
+
+
+class FetchOptions(Package):
+    """Mock package with fetch_options."""
+
+    homepage = "http://www.fetch-options-example.com"
+
+    url = 'https://example.com/some/tarball-1.0.tar.gz'
+
+    fetch_options = {'timeout': 42, 'cookie': 'foobar'}
+    timeout = {'timeout': 65}
+    cookie = {'cookie': 'baz'}
+
+    version('1.2', 'abc12', fetch_options=cookie)
+    version('1.1', 'abc11', fetch_options=timeout)
+    version('1.0', 'abc10')

var/spack/repos/builtin/packages/bzip2/package.py

@@ -19,9 +19,9 @@ class Bzip2(Package):
     # The server is sometimes a bit slow to respond
     fetch_options = {'timeout': 60}
 
-    version('1.0.8', sha256='ab5a03176ee106d3f0fa90e381da478ddae405918153cca248e682cd0c4a2269', fetch_options=fetch_options)
+    version('1.0.8', sha256='ab5a03176ee106d3f0fa90e381da478ddae405918153cca248e682cd0c4a2269')
-    version('1.0.7', sha256='e768a87c5b1a79511499beb41500bcc4caf203726fff46a6f5f9ad27fe08ab2b', fetch_options=fetch_options)
+    version('1.0.7', sha256='e768a87c5b1a79511499beb41500bcc4caf203726fff46a6f5f9ad27fe08ab2b')
-    version('1.0.6', sha256='a2848f34fcd5d6cf47def00461fcb528a0484d8edef8208d6d2e2909dc61d9cd', fetch_options=fetch_options)
+    version('1.0.6', sha256='a2848f34fcd5d6cf47def00461fcb528a0484d8edef8208d6d2e2909dc61d9cd')
 
     variant('shared', default=True, description='Enables the build of shared libraries.')
 

var/spack/repos/builtin/packages/eclipse-gcj-parser/package.py

@@ -19,6 +19,9 @@ class EclipseGcjParser(Package):
 
     maintainers = ['citibeth']
 
+    # The server is sometimes a bit slow to respond
+    fetch_options = {'timeout': 60}
+
     version('4.8', sha256='98fd128f1d374d9e42fd9d4836bdd249c6d511ebc6c0df17fbc1b9df96c3d781', expand=False)
 
     phases = ('build', 'install')

var/spack/repos/builtin/packages/elfutils/package.py

@@ -22,17 +22,17 @@ class Elfutils(AutotoolsPackage):
     list_depth = 1
 
     # Sourceware is often slow to respond.
-    timeout = {'timeout': 60}
+    fetch_options = {'timeout': 60}
 
-    version('0.178', sha256='31e7a00e96d4e9c4bda452e1f2cdac4daf8abd24f5e154dee232131899f3a0f2', fetch_options=timeout)
+    version('0.178', sha256='31e7a00e96d4e9c4bda452e1f2cdac4daf8abd24f5e154dee232131899f3a0f2')
-    version('0.177', sha256='fa489deccbcae7d8c920f60d85906124c1989c591196d90e0fd668e3dc05042e', fetch_options=timeout)
+    version('0.177', sha256='fa489deccbcae7d8c920f60d85906124c1989c591196d90e0fd668e3dc05042e')
-    version('0.176', sha256='eb5747c371b0af0f71e86215a5ebb88728533c3a104a43d4231963f308cd1023', fetch_options=timeout)
+    version('0.176', sha256='eb5747c371b0af0f71e86215a5ebb88728533c3a104a43d4231963f308cd1023')
-    version('0.175', sha256='f7ef925541ee32c6d15ae5cb27da5f119e01a5ccdbe9fe57bf836730d7b7a65b', fetch_options=timeout)
+    version('0.175', sha256='f7ef925541ee32c6d15ae5cb27da5f119e01a5ccdbe9fe57bf836730d7b7a65b')
-    version('0.174', sha256='cdf27e70076e10a29539d89e367101d516bc4aa11b0d7777fe52139e3fcad08a', fetch_options=timeout)
+    version('0.174', sha256='cdf27e70076e10a29539d89e367101d516bc4aa11b0d7777fe52139e3fcad08a')
-    version('0.173', sha256='b76d8c133f68dad46250f5c223482c8299d454a69430d9aa5c19123345a000ff', fetch_options=timeout)
+    version('0.173', sha256='b76d8c133f68dad46250f5c223482c8299d454a69430d9aa5c19123345a000ff')
-    version('0.170', sha256='1f844775576b79bdc9f9c717a50058d08620323c1e935458223a12f249c9e066', fetch_options=timeout)
+    version('0.170', sha256='1f844775576b79bdc9f9c717a50058d08620323c1e935458223a12f249c9e066')
-    version('0.168', sha256='b88d07893ba1373c7dd69a7855974706d05377766568a7d9002706d5de72c276', fetch_options=timeout)
+    version('0.168', sha256='b88d07893ba1373c7dd69a7855974706d05377766568a7d9002706d5de72c276')
-    version('0.163', sha256='7c774f1eef329309f3b05e730bdac50013155d437518a2ec0e24871d312f2e23', fetch_options=timeout)
+    version('0.163', sha256='7c774f1eef329309f3b05e730bdac50013155d437518a2ec0e24871d312f2e23')
 
     # Libraries for reading compressed DWARF sections.
     variant('bzip2', default=False,

var/spack/repos/builtin/packages/gcc/package.py

@@ -109,12 +109,16 @@ class Gcc(AutotoolsPackage, GNUMirrorPackage):
     depends_on('zip', type='build', when='languages=java')
     depends_on('cuda', when='+nvptx')
 
+    # The server is sometimes a bit slow to respond
+    timeout = {'timeout': 60}
+
     resource(
              name='newlib',
              url='ftp://sourceware.org/pub/newlib/newlib-3.0.0.20180831.tar.gz',
              sha256='3ad3664f227357df15ff34e954bfd9f501009a647667cd307bf0658aefd6eb5b',
              destination='newlibsource',
-             when='+nvptx'
+             when='+nvptx',
+             fetch_options=timeout
             )
 
     # nvptx-tools does not seem to work as a dependency,

var/spack/repos/builtin/packages/jdk/package.py

@@ -24,9 +24,7 @@ class Jdk(Package):
     # automate this process, we need to utilize these additional curl
     # command-line options. See:
     # http://stackoverflow.com/questions/10268583/how-to-automate-download-and-installation-of-java-jdk-on-linux
-    fetch_options = {
+    fetch_options = {'cookie': 'oraclelicense=accept-securebackup-cookie'}
-        'cookie': 'oraclelicense=accept-securebackup-cookie'
-    }
 
     # To add the latest version, go to the homepage listed above,
     # click "JDK Download", click "Accept License Agreement", right-click the
@@ -34,29 +32,29 @@ class Jdk(Package):
     # found in a link above. The build number can be deciphered from the URL.
     # Alternatively, run `bin/java -version` after extracting. Replace '+'
     # symbol in version with '_', otherwise it will be interpreted as a variant
-    version('12.0.2_10', sha256='2dde6fda89a4ec6e6560ed464e917861c9e40bf576e7a64856dafc55abaaff51', fetch_options=fetch_options,
+    version('12.0.2_10', sha256='2dde6fda89a4ec6e6560ed464e917861c9e40bf576e7a64856dafc55abaaff51',
             url='https://download.oracle.com/otn-pub/java/jdk/12.0.2+10/e482c34c86bd4bf8b56c0b35558996b9/jdk-12.0.2_linux-x64_bin.tar.gz')
-    version('12.0.1_12', sha256='9fd6dcdaf2cfca7da59e39b009a0f5bcd53bec2fb16105f7ca8d689cdab68d75', fetch_options=fetch_options,
+    version('12.0.1_12', sha256='9fd6dcdaf2cfca7da59e39b009a0f5bcd53bec2fb16105f7ca8d689cdab68d75',
             url='https://download.oracle.com/otn-pub/java/jdk/12.0.1+12/69cfe15208a647278a19ef0990eea691/jdk-12.0.1_linux-x64_bin.tar.gz')
-    version('11.0.2_9', sha256='7b4fd8ffcf53e9ff699d964a80e4abf9706b5bdb5644a765c2b96f99e3a2cdc8', fetch_options=fetch_options,
+    version('11.0.2_9', sha256='7b4fd8ffcf53e9ff699d964a80e4abf9706b5bdb5644a765c2b96f99e3a2cdc8',
             url='https://download.oracle.com/otn-pub/java/jdk/11.0.2+9/f51449fcd52f4d52b93a989c5c56ed3c/jdk-11.0.2_linux-x64_bin.tar.gz')
-    version('11.0.1_13', sha256='e7fd856bacad04b6dbf3606094b6a81fa9930d6dbb044bbd787be7ea93abc885', fetch_options=fetch_options,
+    version('11.0.1_13', sha256='e7fd856bacad04b6dbf3606094b6a81fa9930d6dbb044bbd787be7ea93abc885',
             url='https://download.oracle.com/otn-pub/java/jdk/11.0.1+13/90cf5d8f270a4347a95050320eef3fb7/jdk-11.0.1_linux-x64_bin.tar.gz')
-    version('10.0.2_13', sha256='6633c20d53c50c20835364d0f3e172e0cbbce78fff81867488f22a6298fa372b', fetch_options=fetch_options,
+    version('10.0.2_13', sha256='6633c20d53c50c20835364d0f3e172e0cbbce78fff81867488f22a6298fa372b',
             url='https://download.oracle.com/otn-pub/java/jdk/10.0.2+13/19aef61b38124481863b1413dce1855f/jdk-10.0.2_linux-x64_bin.tar.gz')
-    version('10.0.1_10', sha256='ae8ed645e6af38432a56a847597ac61d4283b7536688dbab44ab536199d1e5a4', fetch_options=fetch_options,
+    version('10.0.1_10', sha256='ae8ed645e6af38432a56a847597ac61d4283b7536688dbab44ab536199d1e5a4',
             url='https://download.oracle.com/otn-pub/java/jdk/10.0.1+10/fb4372174a714e6b8c52526dc134031e/jdk-10.0.1_linux-x64_bin.tar.gz')
-    version('1.8.0_241-b07', sha256='419d32677855f676076a25aed58e79432969142bbd778ff8eb57cb618c69e8cb', fetch_options=fetch_options,
+    version('1.8.0_241-b07', sha256='419d32677855f676076a25aed58e79432969142bbd778ff8eb57cb618c69e8cb',
             url='https://download.oracle.com/otn-pub/java/jdk/8u241-b07/1f5b5a70bf22433b84d0e960903adac8/jdk-8u241-linux-x64.tar.gz')
-    version('1.8.0_231-b11', sha256='a011584a2c9378bf70c6903ef5fbf101b30b08937441dc2ec67932fb3620b2cf', fetch_options=fetch_options,
+    version('1.8.0_231-b11', sha256='a011584a2c9378bf70c6903ef5fbf101b30b08937441dc2ec67932fb3620b2cf',
             url='https://download.oracle.com/otn-pub/java/jdk/8u231-b11/5b13a193868b4bf28bcb45c792fce896/jdk-8u231-linux-x64.tar.gz')
-    version('1.8.0_212-b10', sha256='3160c50aa8d8e081c8c7fe0f859ea452922eca5d2ae8f8ef22011ae87e6fedfb', fetch_options=fetch_options,
+    version('1.8.0_212-b10', sha256='3160c50aa8d8e081c8c7fe0f859ea452922eca5d2ae8f8ef22011ae87e6fedfb',
             url='https://download.oracle.com/otn-pub/java/jdk/8u212-b10/59066701cf1a433da9770636fbc4c9aa/jdk-8u212-linux-x64.tar.gz')
-    version('1.8.0_202-b08', sha256='9a5c32411a6a06e22b69c495b7975034409fa1652d03aeb8eb5b6f59fd4594e0', fetch_options=fetch_options,
+    version('1.8.0_202-b08', sha256='9a5c32411a6a06e22b69c495b7975034409fa1652d03aeb8eb5b6f59fd4594e0',
             url='https://download.oracle.com/otn-pub/java/jdk/8u202-b08/1961070e4c9b4e26a04e7f5a083f551e/jdk-8u202-linux-x64.tar.gz')
-    version('1.8.0_141-b15', sha256='041d5218fbea6cd7e81c8c15e51d0d32911573af2ed69e066787a8dc8a39ba4f', fetch_options=fetch_options,
+    version('1.8.0_141-b15', sha256='041d5218fbea6cd7e81c8c15e51d0d32911573af2ed69e066787a8dc8a39ba4f',
             url='https://download.oracle.com/otn-pub/java/jdk/8u141-b15/336fa29ff2bb4ef291e347e091f7f4a7/jdk-8u141-linux-x64.tar.gz')
-    version('1.8.0_131-b11', sha256='62b215bdfb48bace523723cdbb2157c665e6a25429c73828a32f00e587301236', fetch_options=fetch_options,
+    version('1.8.0_131-b11', sha256='62b215bdfb48bace523723cdbb2157c665e6a25429c73828a32f00e587301236',
             url='https://download.oracle.com/otn-pub/java/jdk/8u131-b11/d54c1d3a095b4ff2b6607d096fa80163/jdk-8u131-linux-x64.tar.gz')
 
     provides('java@12', when='@12.0:12.999')

var/spack/repos/builtin/packages/libffi/package.py

@@ -17,8 +17,7 @@ class Libffi(AutotoolsPackage):
     fetch_options = {'timeout': 60}
 
     version('3.2.1', sha256='d06ebb8e1d9a22d19e38d63fdb83954253f39bedc5d46232a05645685722ca37',
-            url="https://sourceware.org/pub/libffi/libffi-3.2.1.tar.gz",
+            url="https://sourceware.org/pub/libffi/libffi-3.2.1.tar.gz")
-            fetch_options=fetch_options)
 
     @property
     def headers(self):

var/spack/repos/builtin/packages/lvm2/package.py

@@ -20,6 +20,9 @@ class Lvm2(AutotoolsPackage):
     homepage = "https://www.sourceware.org/lvm2"
     url = "https://sourceware.org/pub/lvm2/releases/LVM2.2.03.05.tgz"
 
+    # The server is sometimes a bit slow to respond
+    fetch_options = {'timeout': 60}
+
     version('2.03.05', sha256='ca52815c999b20c6d25e3192f142f081b93d01f07b9d787e99664b169dba2700')
     version('2.03.04', sha256='f151f36fc0039997d2d9369b607b9262568b1a268afe19fd1535807355402142')
     version('2.03.03', sha256='cedefa63ec5ae1b62fedbfddfc30706c095be0fc7c6aaed6fd1c50bc8c840dde')

var/spack/repos/builtin/packages/valgrind/package.py

@@ -22,6 +22,9 @@ class Valgrind(AutotoolsPackage):
     url      = "https://sourceware.org/pub/valgrind/valgrind-3.13.0.tar.bz2"
     git      = "git://sourceware.org/git/valgrind.git"
 
+    # The server is sometimes a bit slow to respond
+    fetch_options = {'timeout': 60}
+
     version('develop', branch='master')
     version('3.15.0', sha256='417c7a9da8f60dd05698b3a7bc6002e4ef996f14c13f0ff96679a16873e78ab1')
     version('3.14.0', sha256='037c11bfefd477cc6e9ebe8f193bb237fe397f7ce791b4a4ce3fa1c6a520baa5')

var/spack/repos/builtin/packages/verrou/package.py

@@ -32,18 +32,24 @@ class Verrou(AutotoolsPackage):
     version('2.0.0', sha256='798df6e426ec57646a2a626d756b72f0171647ae5b07c982952dae2d71e26045')
     version('1.1.0', sha256='b5105f61c65680f31551199cd143b2e15f412c34c821537998a7165e315dde2d')
 
+    # The server is sometimes a bit slow to respond
+    timeout = {'timeout': 60}
+
     resource(name='valgrind-3.15.0',
              url='https://sourceware.org/pub/valgrind/valgrind-3.15.0.tar.bz2',
              sha256='417c7a9da8f60dd05698b3a7bc6002e4ef996f14c13f0ff96679a16873e78ab1',
-             when='@2.2.0:')
+             when='@2.2.0:',
+             fetch_options=timeout)
     resource(name='valgrind-3.14.0',
              url='https://sourceware.org/pub/valgrind/valgrind-3.14.0.tar.bz2',
              sha256='037c11bfefd477cc6e9ebe8f193bb237fe397f7ce791b4a4ce3fa1c6a520baa5',
-             when='@2.1.0:2.1.99')
+             when='@2.1.0:2.1.99',
+             fetch_options=timeout)
     resource(name='valgrind-3.13.0',
              url='https://sourceware.org/pub/valgrind/valgrind-3.13.0.tar.bz2',
              sha256='d76680ef03f00cd5e970bbdcd4e57fb1f6df7d2e2c071635ef2be74790190c3b',
-             when='@1.1.0:2.0.99')
+             when='@1.1.0:2.0.99',
+             fetch_options=timeout)
 
     variant('fma', default=True,
             description='Activates fused multiply-add support for Verrou')