hlrs-software-stack/spack
7
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

document config option "url_fetch_method" (#24638)

Browse source 
- Change config from the undocumented `use_curl: true/false` to `url_fetch_method: urllib/curl`.
- Documentation of `url_fetch_method` in `defaults/config.yaml`
- Default fetch option explicitly set to `urllib` for users who may not have curl on their system

To upgrade from `use_curl` to `url_fetch_method`, run `spack config update config`
This commit is contained in:
loulawrence 2021-08-02 04:30:25 -04:00 committed by GitHub
parent cd8f7d844d
commit be3e6a0e9b
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 58 additions and 44 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
etc/spack/defaults/config.yaml
View file

@ -134,6 +134,10 @@ config:
# enabling locks.
locks: true
# The default url fetch method to use.
# If set to 'curl', Spack will require curl on the user's system
# If set to 'urllib', Spack will use python built-in libs to fetch
url_fetch_method: urllib
# The maximum number of jobs to use for the build system (e.g. `make`), when
# the -j flag is not given on the command line. Defaults to 16 when not set.

4
lib/spack/spack/fetch_strategy.py
View file

@ -338,7 +338,7 @@ def fetch(self):
def _existing_url(self, url):
tty.debug('Checking existence of {0}'.format(url))
if spack.config.get('config:use_curl'):
if spack.config.get('config:url_fetch_method') == 'curl':
curl = self.curl
# Telling curl to fetch the first byte (-r 0-0) is supposed to be
# portable.
@ -357,7 +357,7 @@ def _existing_url(self, url):
return (response.getcode() is None or response.getcode() == 200)
def _fetch_from_url(self, url):
if spack.config.get('config:use_curl'):
if spack.config.get('config:url_fetch_method') == 'curl':
return self._fetch_curl(url)
else:
return self._fetch_urllib(url)

10
lib/spack/spack/schema/config.py
View file

@ -99,6 +99,10 @@
},
'allow_sgid': {'type': 'boolean'},
'binary_index_root': {'type': 'string'},
'url_fetch_method': {
'type': 'string',
'enum': ['urllib', 'curl']
},
},
},
}
@ -153,4 +157,10 @@ def update(data):
update_data = spack.config.merge_yaml(update_data, projections_data)
data['install_tree'] = update_data
changed = True
use_curl = data.pop('use_curl', None)
if use_curl is not None:
data['url_fetch_method'] = 'curl' if use_curl else 'urllib'
changed = True
return changed

12
lib/spack/spack/test/cache_fetch.py
View file

@ -14,25 +14,25 @@
from spack.stage import Stage
@pytest.mark.parametrize('use_curl', [True, False])
def test_fetch_missing_cache(tmpdir, use_curl):
@pytest.mark.parametrize('_fetch_method', ['curl', 'urllib'])
def test_fetch_missing_cache(tmpdir, _fetch_method):
"""Ensure raise a missing cache file."""
testpath = str(tmpdir)
with spack.config.override('config:use_curl', use_curl):
with spack.config.override('config:url_fetch_method', _fetch_method):
fetcher = CacheURLFetchStrategy(url='file:///not-a-real-cache-file')
with Stage(fetcher, path=testpath):
with pytest.raises(NoCacheError, match=r'No cache'):
fetcher.fetch()
@pytest.mark.parametrize('use_curl', [True, False])
def test_fetch(tmpdir, use_curl):
@pytest.mark.parametrize('_fetch_method', ['curl', 'urllib'])
def test_fetch(tmpdir, _fetch_method):
"""Ensure a fetch after expanding is effectively a no-op."""
testpath = str(tmpdir)
cache = os.path.join(testpath, 'cache.tar.gz')
touch(cache)
url = 'file:///{0}'.format(cache)
with spack.config.override('config:use_curl', use_curl):
with spack.config.override('config:url_fetch_method', _fetch_method):
fetcher = CacheURLFetchStrategy(url=url)
with Stage(fetcher, path=testpath) as stage:
source_path = stage.source_path

18
lib/spack/spack/test/s3_fetch.py
View file

@ -12,20 +12,20 @@
import spack.stage as spack_stage
@pytest.mark.parametrize('use_curl', [True, False])
def test_s3fetchstrategy_sans_url(use_curl):
@pytest.mark.parametrize('_fetch_method', ['curl', 'urllib'])
def test_s3fetchstrategy_sans_url(_fetch_method):
"""Ensure constructor with no URL fails."""
with spack_config.override('config:use_curl', use_curl):
with spack_config.override('config:url_fetch_method', _fetch_method):
with pytest.raises(ValueError):
spack_fs.S3FetchStrategy(None)
@pytest.mark.parametrize('use_curl', [True, False])
def test_s3fetchstrategy_bad_url(tmpdir, use_curl):
@pytest.mark.parametrize('_fetch_method', ['curl', 'urllib'])
def test_s3fetchstrategy_bad_url(tmpdir, _fetch_method):
"""Ensure fetch with bad URL fails as expected."""
testpath = str(tmpdir)
with spack_config.override('config:use_curl', use_curl):
with spack_config.override('config:url_fetch_method', _fetch_method):
fetcher = spack_fs.S3FetchStrategy(url='file:///does-not-exist')
assert fetcher is not None
@ -36,13 +36,13 @@ def test_s3fetchstrategy_bad_url(tmpdir, use_curl):
fetcher.fetch()
@pytest.mark.parametrize('use_curl', [True, False])
def test_s3fetchstrategy_downloaded(tmpdir, use_curl):
@pytest.mark.parametrize('_fetch_method', ['curl', 'urllib'])
def test_s3fetchstrategy_downloaded(tmpdir, _fetch_method):
"""Ensure fetch with archive file already downloaded is a noop."""
testpath = str(tmpdir)
archive = os.path.join(testpath, 's3.tar.gz')
with spack_config.override('config:use_curl', use_curl):
with spack_config.override('config:url_fetch_method', _fetch_method):
class Archived_S3FS(spack_fs.S3FetchStrategy):
@property
def archive_file(self):

54
lib/spack/spack/test/url_fetch.py
View file

@ -49,20 +49,20 @@ def fn(v):
return factory
@pytest.mark.parametrize('use_curl', [True, False])
def test_urlfetchstrategy_sans_url(use_curl):
@pytest.mark.parametrize('_fetch_method', ['curl', 'urllib'])
def test_urlfetchstrategy_sans_url(_fetch_method):
"""Ensure constructor with no URL fails."""
with spack.config.override('config:use_curl', use_curl):
with spack.config.override('config:url_fetch_method', _fetch_method):
with pytest.raises(ValueError):
with fs.URLFetchStrategy(None):
pass
@pytest.mark.parametrize('use_curl', [True, False])
def test_urlfetchstrategy_bad_url(tmpdir, use_curl):
@pytest.mark.parametrize('_fetch_method', ['curl', 'urllib'])
def test_urlfetchstrategy_bad_url(tmpdir, _fetch_method):
"""Ensure fetch with bad URL fails as expected."""
testpath = str(tmpdir)
with spack.config.override('config:use_curl', use_curl):
with spack.config.override('config:url_fetch_method', _fetch_method):
with pytest.raises(fs.FailedDownloadError):
fetcher = fs.URLFetchStrategy(url='file:///does-not-exist')
assert fetcher is not None
@ -75,7 +75,7 @@ def test_urlfetchstrategy_bad_url(tmpdir, use_curl):
def test_fetch_options(tmpdir, mock_archive):
testpath = str(tmpdir)
with spack.config.override('config:use_curl', True):
with spack.config.override('config:url_fetch_method', 'curl'):
fetcher = fs.URLFetchStrategy(url=mock_archive.url,
fetch_options={'cookie': 'True',
'timeout': 10})
@ -87,11 +87,11 @@ def test_fetch_options(tmpdir, mock_archive):
fetcher.fetch()
@pytest.mark.parametrize('use_curl', [True, False])
def test_archive_file_errors(tmpdir, mock_archive, use_curl):
@pytest.mark.parametrize('_fetch_method', ['curl', 'urllib'])
def test_archive_file_errors(tmpdir, mock_archive, _fetch_method):
"""Ensure FetchStrategy commands may only be used as intended"""
testpath = str(tmpdir)
with spack.config.override('config:use_curl', use_curl):
with spack.config.override('config:url_fetch_method', _fetch_method):
fetcher = fs.URLFetchStrategy(url=mock_archive.url)
assert fetcher is not None
with pytest.raises(fs.FailedDownloadError):
@ -112,7 +112,7 @@ def test_archive_file_errors(tmpdir, mock_archive, use_curl):
@pytest.mark.parametrize('secure', [True, False])
@pytest.mark.parametrize('use_curl', [True, False])
@pytest.mark.parametrize('_fetch_method', ['curl', 'urllib'])
@pytest.mark.parametrize('mock_archive',
[('.tar.gz', 'z'), ('.tgz', 'z'),
('.tar.bz2', 'j'), ('.tbz2', 'j'),
@ -121,7 +121,7 @@ def test_archive_file_errors(tmpdir, mock_archive, use_curl):
def test_fetch(
mock_archive,
secure,
use_curl,
_fetch_method,
checksum_type,
config,
mutable_mock_repo
@ -147,7 +147,7 @@ def test_fetch(
# Enter the stage directory and check some properties
with pkg.stage:
with spack.config.override('config:verify_ssl', secure):
with spack.config.override('config:use_curl', use_curl):
with spack.config.override('config:url_fetch_method', _fetch_method):
pkg.do_stage()
with working_dir(pkg.stage.source_path):
assert os.path.exists('configure')
@ -168,13 +168,13 @@ def test_fetch(
('url-list-test @3.0a1', 'foo-3.0a1.tar.gz', 'abc30a1'),
('url-list-test @4.5-rc5', 'foo-4.5-rc5.tar.gz', 'abc45rc5'),
])
@pytest.mark.parametrize('use_curl', [True, False])
def test_from_list_url(mock_packages, config, spec, url, digest, use_curl):
@pytest.mark.parametrize('_fetch_method', ['curl', 'urllib'])
def test_from_list_url(mock_packages, config, spec, url, digest, _fetch_method):
"""
Test URLs in the url-list-test package, which means they should
have checksums in the package.
"""
with spack.config.override('config:use_curl', use_curl):
with spack.config.override('config:url_fetch_method', _fetch_method):
specification = Spec(spec).concretized()
pkg = spack.repo.get(specification)
fetch_strategy = fs.from_list_url(pkg)
@ -187,10 +187,10 @@ def test_from_list_url(mock_packages, config, spec, url, digest, use_curl):
assert fetch_strategy.extra_options == {'timeout': 60}
@pytest.mark.parametrize('use_curl', [True, False])
def test_from_list_url_unspecified(mock_packages, config, use_curl):
@pytest.mark.parametrize('_fetch_method', ['curl', 'urllib'])
def test_from_list_url_unspecified(mock_packages, config, _fetch_method):
"""Test non-specific URLs from the url-list-test package."""
with spack.config.override('config:use_curl', use_curl):
with spack.config.override('config:url_fetch_method', _fetch_method):
pkg = spack.repo.get('url-list-test')
spec = Spec('url-list-test @2.0.0').concretized()
@ -236,7 +236,7 @@ def is_true():
monkeypatch.setattr(sys.stdout, 'isatty', is_true)
monkeypatch.setattr(tty, 'msg_enabled', is_true)
with spack.config.override('config:use_curl', True):
with spack.config.override('config:url_fetch_method', 'curl'):
fetcher = fs.URLFetchStrategy(mock_archive.url)
with Stage(fetcher, path=testpath) as stage:
assert fetcher.archive_file is None
@ -246,10 +246,10 @@ def is_true():
assert '##### 100' in status
@pytest.mark.parametrize('use_curl', [True, False])
def test_url_extra_fetch(tmpdir, mock_archive, use_curl):
@pytest.mark.parametrize('_fetch_method', ['curl', 'urllib'])
def test_url_extra_fetch(tmpdir, mock_archive, _fetch_method):
"""Ensure a fetch after downloading is effectively a no-op."""
with spack.config.override('config:use_curl', use_curl):
with spack.config.override('config:url_fetch_method', _fetch_method):
testpath = str(tmpdir)
fetcher = fs.URLFetchStrategy(mock_archive.url)
with Stage(fetcher, path=testpath) as stage:
@ -267,12 +267,12 @@ def test_url_extra_fetch(tmpdir, mock_archive, use_curl):
['https://ftpmirror.gnu.org/autoconf/autoconf-2.62.tar.gz',
'https://ftp.gnu.org/gnu/autoconf/autoconf-2.62.tar.gz'])
])
@pytest.mark.parametrize('use_curl', [True, False])
def test_candidate_urls(pkg_factory, url, urls, version, expected, use_curl):
@pytest.mark.parametrize('_fetch_method', ['curl', 'urllib'])
def test_candidate_urls(pkg_factory, url, urls, version, expected, _fetch_method):
"""Tests that candidate urls include mirrors and that they go through
pattern matching and substitution for versions.
"""
with spack.config.override('config:use_curl', use_curl):
with spack.config.override('config:url_fetch_method', _fetch_method):
pkg = pkg_factory(url, urls)
f = fs._from_merged_attrs(fs.URLFetchStrategy, pkg, version)
assert f.candidate_urls == expected
@ -298,7 +298,7 @@ def _which(*args, **kwargs):
testpath = str(tmpdir)
url = 'http://github.com/spack/spack'
with spack.config.override('config:use_curl', True):
with spack.config.override('config:url_fetch_method', 'curl'):
fetcher = fs.URLFetchStrategy(url=url)
assert fetcher is not None
with pytest.raises(TypeError, match='object is not callable'):