hlrs-software-stack/spack
7
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

tests: add checks and tests for consistent version() arguments

Browse source 
- Previously, Spack didn't check the arguments you put in version()
  directives.

- So, you could do something like this, where there are arguments for a
  URL fetcher AND for a git fetcher:

  version('1.0', md5='abc123', git='https://foo.bar', commit='feda2343')

- Now, we check the arguments before constructing a fetcher, to ensure
  that each package has *only* arguments for a single type of fetcher.

- Also added `test_package_version_consistency()` to the `package_sanity`
  test, so that all builtin packages are required to have valid
  `version()` directives.
This commit is contained in:
Todd Gamblin 2018-07-22 16:15:37 -07:00
parent 6f7eaecfa0
commit 020c34e136
10 changed files with 91 additions and 32 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

24
lib/spack/spack/fetch_strategy.py
View file

@ -993,6 +993,27 @@ def check_pkg_attributes(pkg):
% (pkg.name, comma_and(quote(conflicts))))
def _check_version_attributes(fetcher, pkg, version):
"""Ensure that the fetcher for a version is not ambiguous.
This assumes that we have already determined the fetcher for the
specific version using ``for_package_version()``
"""
all_optionals = set(a for s in all_strategies for a in s.optional_attrs)
args = pkg.versions[version]
extra = set(args) - set(fetcher.optional_attrs) - set([fetcher.url_attr])
extra.intersection_update(all_optionals)
if extra:
legal_attrs = [fetcher.url_attr] + list(fetcher.optional_attrs)
raise FetcherConflict(
"%s version '%s' has extra arguments: %s"
% (pkg.name, version, comma_and(quote(extra))),
"Valid arguments for a %s fetcher are: \n %s"
% (fetcher.url_attr, comma_and(quote(legal_attrs))))
def _extrapolate(pkg, version):
"""Create a fetcher from an extrapolated URL for this version."""
try:
@ -1033,6 +1054,7 @@ def for_package_version(pkg, version):
# If the version specifies a `url_attr` directly, use that.
for fetcher in all_strategies:
if fetcher.url_attr in args:
_check_version_attributes(fetcher, pkg, version)
return fetcher(**args)
# if a version's optional attributes imply a particular fetch
@ -1041,6 +1063,7 @@ def for_package_version(pkg, version):
if hasattr(pkg, fetcher.url_attr) or fetcher.url_attr == 'url':
optionals = fetcher.optional_attrs
if optionals and any(a in args for a in optionals):
_check_version_attributes(fetcher, pkg, version)
return _from_merged_attrs(fetcher, pkg, version)
# if the optional attributes tell us nothing, then use any `url_attr`
@ -1048,6 +1071,7 @@ def for_package_version(pkg, version):
# defined first in this file.
for fetcher in all_strategies:
if hasattr(pkg, fetcher.url_attr):
_check_version_attributes(fetcher, pkg, version)
return _from_merged_attrs(fetcher, pkg, version)
raise InvalidArgsError(pkg, version, **args)

10
lib/spack/spack/test/package_sanity.py
View file

@ -29,6 +29,7 @@
import spack.paths
import spack.repo
import spack.fetch_strategy
def check_repo():
@ -68,3 +69,12 @@ def test_all_virtual_packages_have_default_providers():
for provider in providers:
assert provider in default_providers
def test_package_version_consistency():
"""Make sure all versions on builtin packages can produce a fetcher."""
for name in spack.repo.all_package_names():
pkg = spack.repo.get(name)
spack.fetch_strategy.check_pkg_attributes(pkg)
for version in pkg.versions:
assert spack.fetch_strategy.for_package_version(pkg, version)

28
lib/spack/spack/test/packages.py
View file

@ -275,8 +275,8 @@ def test_two_vcs_fetchers_top_level(mock_packages, config):
spack.fetch_strategy.for_package_version(pkg, '1.0')
def test_git_url_top_level(mock_packages, config):
"""Test fetch strategy inference when url is specified with a VCS."""
def test_git_url_top_level_url_versions(mock_packages, config):
"""Test URL fetch strategy inference when url is specified with git."""
pkg = spack.repo.get('git-url-top-level')
@ -300,6 +300,12 @@ def test_git_url_top_level(mock_packages, config):
assert fetcher.url == 'https://www.example.com/foo2.3.tar.gz'
assert fetcher.digest == 'abc23'
def test_git_url_top_level_git_versions(mock_packages, config):
"""Test git fetch strategy inference when url is specified with git."""
pkg = spack.repo.get('git-url-top-level')
fetcher = spack.fetch_strategy.for_package_version(pkg, '3.0')
assert isinstance(fetcher, spack.fetch_strategy.GitFetchStrategy)
assert fetcher.url == 'https://example.com/some/git/repo'
@ -341,3 +347,21 @@ def test_git_url_top_level(mock_packages, config):
assert fetcher.tag is None
assert fetcher.commit is None
assert fetcher.branch == 'develop'
def test_git_url_top_level_conflicts(mock_packages, config):
"""Test git fetch strategy inference when url is specified with git."""
pkg = spack.repo.get('git-url-top-level')
with pytest.raises(spack.fetch_strategy.FetcherConflict):
spack.fetch_strategy.for_package_version(pkg, '1.0')
with pytest.raises(spack.fetch_strategy.FetcherConflict):
spack.fetch_strategy.for_package_version(pkg, '1.1')
with pytest.raises(spack.fetch_strategy.FetcherConflict):
spack.fetch_strategy.for_package_version(pkg, '1.2')
with pytest.raises(spack.fetch_strategy.FetcherConflict):
spack.fetch_strategy.for_package_version(pkg, '1.3')

8
var/spack/repos/builtin.mock/packages/git-url-top-level/package.py
View file

@ -37,6 +37,7 @@ class GitUrlTopLevel(Package):
git = 'https://example.com/some/git/repo'
url = 'https://example.com/some/tarball-1.0.tar.gz'
# These resolve to git fetchers
version('develop', branch='develop')
version('3.4', commit='abc34')
version('3.3', branch='releases/v3.3', commit='abc33')
@ -44,10 +45,17 @@ class GitUrlTopLevel(Package):
version('3.1', tag='v3.1', commit='abc31')
version('3.0', tag='v3.0')
# These resolve to URL fetchers
version('2.3', 'abc23', url='https://www.example.com/foo2.3.tar.gz')
version('2.2', sha256='abc22', url='https://www.example.com/foo2.2.tar.gz')
version('2.1', sha256='abc21')
version('2.0', 'abc20')
# These result in a FetcherConflict b/c we can't tell what to use
version('1.3', sha256='abc13', commit='abc13')
version('1.2', sha512='abc12', branch='releases/v1.2')
version('1.1', md5='abc11', tag='v1.1')
version('1.0', 'abc11', tag='abc123')
def install(self, spec, prefix):
pass

2
var/spack/repos/builtin.mock/packages/svn-test/package.py
View file

@ -29,7 +29,7 @@ class SvnTest(Package):
"""Mock package that uses svn for fetching."""
url = "http://www.example.com/svn-test-1.0.tar.gz"
version('svn', 'to-be-filled-in-by-test')
version('svn', svn='to-be-filled-in-by-test')
def install(self, spec, prefix):
pass

5
var/spack/repos/builtin/packages/damselfly/package.py
View file

@ -28,9 +28,8 @@
class Damselfly(CMakePackage):
"""Damselfly is a model-based parallel network simulator."""
homepage = "https://github.com/llnl/damselfly"
url = "https://github.com/llnl/damselfly"
url = "https://github.com/LLNL/damselfly/archive/v1.0.tar.gz"
version('1.0', '05cf7e2d8ece4408c0f2abb7ab63fd74c0d62895',
git='https://github.com/llnl/damselfly.git', tag='v1.0')
version('1.0', sha256='560e1b800c9036766396a1033c00914bd8d181b911e87140c3ac8879baf6545a')
depends_on('cmake@2.6:', type='build')

4
var/spack/repos/builtin/packages/nekbone/package.py
View file

@ -32,11 +32,11 @@ class Nekbone(Package):
the spectral element method."""
homepage = "https://github.com/Nek5000/Nekbone"
url = "https://github.com/Nek5000/Nekbone/tarball/v17.0"
url = "https://github.com/Nek5000/Nekbone/archive/v17.0.tar.gz"
tags = ['proxy-app', 'ecp-proxy-app']
version('17.0', 'cc339684547614a0725959e41839fec1', git='https://github.com/Nek5000/Nekbone.git')
version('17.0', sha256='ae361cc61368a924398a28a296f675b7f0c4a9516788a7f8fa3c09d787cdf69b')
version('develop', git='https://github.com/Nek5000/Nekbone.git')
# Variants

33
var/spack/repos/builtin/packages/py-yt/package.py
View file

@ -35,26 +35,21 @@ class PyYt(PythonPackage):
interdisciplinary community.
"""
homepage = "http://yt-project.org"
url = "https://github.com/yt-project/yt.git"
url = "https://github.com/yt-project/yt/archive/yt-3.4.0.tar.gz"
git = "https://github.com/yt-project/yt.git"
version("3.4.0", "413b835f1b0e2a0bd26f1044ff7dbc94",
url="https://github.com/yt-project/yt/archive/yt-3.4.0.tar.gz")
version("3.3.5", "2ad314ff3d3261e41825d15db027b0e7",
url="https://bitbucket.org/yt_analysis/yt/get/yt-3.3.5.tar.gz")
version("3.3.4", "3a84e56dfd82f9dd923f3fb8490e679c",
url="https://bitbucket.org/yt_analysis/yt/get/yt-3.3.4.tar.gz")
version("3.3.1", hg="https://bitbucket.org/yt_analysis/yt",
tag="yt-3.3.1", commit="9bc3d0e9b750c923d44d73c447df64fc431f5838")
version("3.2.3", hg="https://bitbucket.org/yt_analysis/yt",
tag="yt-3.2.3", commit="83d2c1e9313e7d83eb5b96888451ff2646fd8ff3")
version("3.1.0", hg="https://bitbucket.org/yt_analysis/yt",
tag="yt-3.1.0", commit="fd7cdc4836188a3badf81adb477bcc1b9632e485")
version("3.0.2", hg="https://bitbucket.org/yt_analysis/yt",
tag="yt-3.0.2", commit="511887af4c995a78fe606e58ce8162c88380ecdc")
version("2.6.3", hg="https://bitbucket.org/yt_analysis/yt",
tag="yt-2.6.3", commit="816186f16396a16853810ac9ebcde5057d8d5b1a")
version("develop", git="https://github.com/yt-project/yt.git",
branch="master")
version("develop", branch="master")
version('3.4.1', sha256='b9a73ade3726a8163fc992999c8c1010ca89473131901fe4d48b820ab2ced486')
version('3.4.0', sha256='2120793a76864cf3165b2b7290ef719e358fa57501ee8721941e7cfc434cfb2b')
version('3.3.5', sha256='2ebe4bbefd9f5367563ce4d7eb87d3f6ef0de1f97ed1c03106d9541e71b7e1ca')
version('3.3.4', sha256='2842bab891cfbf3269a3c4bd8f22fef23c9a15a790ba48c6490730cb51ce9b0e')
version('3.3.3', sha256='7b9244089e92b1d32cef791cd72760bb8c80b391eaec29672a5377c33f932d88')
version('3.3.2', sha256='d323419ad3919e86d2af1738c846021fd7f5b5dc5c06059cdf3a2bc63226466a')
version('3.3.1', sha256='7ac68d5e05e2b57fb3635f1027f3201094f3547d584e72ab55fedbfd3bc09a36')
version('3.3.0', sha256='e6be799c0d9a83a06649f0d77a61ad9c23b94b34f94e16724e2b18f5c7513c33')
version('3.2.3', sha256='96476d17e9ce35f0d4380b2ddb398fe729e39f1f3894602ff07e49844541e5ca')
version('3.2.2', sha256='498ed77b3dae8c54929602d4931f3c3e0a3420a9b500cbd870f50b1e0efea8c3')
variant("astropy", default=True, description="enable astropy support")
variant("h5py", default=True, description="enable h5py support")

4
var/spack/repos/builtin/packages/r-affycompatible/package.py
View file

@ -34,9 +34,9 @@ class RAffycompatible(RPackage):
(AGCC)-compatible sample annotation files."""
homepage = "https://www.bioconductor.org/packages/AffyCompatible/"
url = "https://git.bioconductor.org/packages/AffyCompatible"
git = "https://git.bioconductor.org/packages/AffyCompatible"
version('1.36.0', 'https://git.bioconductor.org/packages/AffyCompatible', commit='dbbfd43a54ae1de6173336683a9461084ebf38c3')
version('1.36.0', commit='dbbfd43a54ae1de6173336683a9461084ebf38c3')
depends_on('r@3.4.0:3.4.9', when=('@1.36.0'))
depends_on('r-xml', type=('build', 'run'))

5
var/spack/repos/builtin/packages/ruby-narray/package.py
View file

@ -31,10 +31,9 @@ class RubyNarray(Package):
similar to numpy.ndaray."""
homepage = "https://rubygems.org/gems/narray"
url = "https://github.com/ruby-numo/narray"
git = "https://github.com/ruby-numo/narray"
version('0.9.0.9', '9cadbbccf1e01b6d1bc143c19d598cad1c420869',
git='https://github.com/ruby-numo/narray.git')
version('0.9.0.9', commit='9cadbbccf1e01b6d1bc143c19d598cad1c420869')
extends('ruby')