tests: add checks and tests for consistent version() arguments

- Previously, Spack didn't check the arguments you put in version()
  directives.

- So, you could do something like this, where there are arguments for a
  URL fetcher AND for a git fetcher:

  version('1.0', md5='abc123', git='https://foo.bar', commit='feda2343')

- Now, we check the arguments before constructing a fetcher, to ensure
  that each package has *only* arguments for a single type of fetcher.

- Also added `test_package_version_consistency()` to the `package_sanity`
  test, so that all builtin packages are required to have valid
  `version()` directives.

lib/spack/spack/fetch_strategy.py

@@ -993,6 +993,27 @@ def check_pkg_attributes(pkg):
             % (pkg.name, comma_and(quote(conflicts))))
 
 
+def _check_version_attributes(fetcher, pkg, version):
+    """Ensure that the fetcher for a version is not ambiguous.
+
+    This assumes that we have already determined the fetcher for the
+    specific version using ``for_package_version()``
+    """
+    all_optionals = set(a for s in all_strategies for a in s.optional_attrs)
+
+    args = pkg.versions[version]
+    extra = set(args) - set(fetcher.optional_attrs) - set([fetcher.url_attr])
+    extra.intersection_update(all_optionals)
+
+    if extra:
+        legal_attrs = [fetcher.url_attr] + list(fetcher.optional_attrs)
+        raise FetcherConflict(
+            "%s version '%s' has extra arguments: %s"
+            % (pkg.name, version, comma_and(quote(extra))),
+            "Valid arguments for a %s fetcher are: \n    %s"
+            % (fetcher.url_attr, comma_and(quote(legal_attrs))))
+
+
 def _extrapolate(pkg, version):
     """Create a fetcher from an extrapolated URL for this version."""
     try:
@@ -1033,6 +1054,7 @@ def for_package_version(pkg, version):
     # If the version specifies a `url_attr` directly, use that.
     for fetcher in all_strategies:
         if fetcher.url_attr in args:
+            _check_version_attributes(fetcher, pkg, version)
             return fetcher(**args)
 
     # if a version's optional attributes imply a particular fetch
@@ -1041,6 +1063,7 @@ def for_package_version(pkg, version):
         if hasattr(pkg, fetcher.url_attr) or fetcher.url_attr == 'url':
             optionals = fetcher.optional_attrs
             if optionals and any(a in args for a in optionals):
+                _check_version_attributes(fetcher, pkg, version)
                 return _from_merged_attrs(fetcher, pkg, version)
 
     # if the optional attributes tell us nothing, then use any `url_attr`
@@ -1048,6 +1071,7 @@ def for_package_version(pkg, version):
     # defined first in this file.
     for fetcher in all_strategies:
         if hasattr(pkg, fetcher.url_attr):
+            _check_version_attributes(fetcher, pkg, version)
             return _from_merged_attrs(fetcher, pkg, version)
 
     raise InvalidArgsError(pkg, version, **args)

lib/spack/spack/test/package_sanity.py

@@ -29,6 +29,7 @@
 
 import spack.paths
 import spack.repo
+import spack.fetch_strategy
 
 
 def check_repo():
@@ -68,3 +69,12 @@ def test_all_virtual_packages_have_default_providers():
 
     for provider in providers:
         assert provider in default_providers
+
+
+def test_package_version_consistency():
+    """Make sure all versions on builtin packages can produce a fetcher."""
+    for name in spack.repo.all_package_names():
+        pkg = spack.repo.get(name)
+        spack.fetch_strategy.check_pkg_attributes(pkg)
+        for version in pkg.versions:
+            assert spack.fetch_strategy.for_package_version(pkg, version)

lib/spack/spack/test/packages.py

@@ -275,8 +275,8 @@ def test_two_vcs_fetchers_top_level(mock_packages, config):
         spack.fetch_strategy.for_package_version(pkg, '1.0')
 
 
-def test_git_url_top_level(mock_packages, config):
+def test_git_url_top_level_url_versions(mock_packages, config):
-    """Test fetch strategy inference when url is specified with a VCS."""
+    """Test URL fetch strategy inference when url is specified with git."""
 
     pkg = spack.repo.get('git-url-top-level')
 
@@ -300,6 +300,12 @@ def test_git_url_top_level(mock_packages, config):
     assert fetcher.url == 'https://www.example.com/foo2.3.tar.gz'
     assert fetcher.digest == 'abc23'
 
+
+def test_git_url_top_level_git_versions(mock_packages, config):
+    """Test git fetch strategy inference when url is specified with git."""
+
+    pkg = spack.repo.get('git-url-top-level')
+
     fetcher = spack.fetch_strategy.for_package_version(pkg, '3.0')
     assert isinstance(fetcher, spack.fetch_strategy.GitFetchStrategy)
     assert fetcher.url == 'https://example.com/some/git/repo'
@@ -341,3 +347,21 @@ def test_git_url_top_level(mock_packages, config):
     assert fetcher.tag is None
     assert fetcher.commit is None
     assert fetcher.branch == 'develop'
+
+
+def test_git_url_top_level_conflicts(mock_packages, config):
+    """Test git fetch strategy inference when url is specified with git."""
+
+    pkg = spack.repo.get('git-url-top-level')
+
+    with pytest.raises(spack.fetch_strategy.FetcherConflict):
+        spack.fetch_strategy.for_package_version(pkg, '1.0')
+
+    with pytest.raises(spack.fetch_strategy.FetcherConflict):
+        spack.fetch_strategy.for_package_version(pkg, '1.1')
+
+    with pytest.raises(spack.fetch_strategy.FetcherConflict):
+        spack.fetch_strategy.for_package_version(pkg, '1.2')
+
+    with pytest.raises(spack.fetch_strategy.FetcherConflict):
+        spack.fetch_strategy.for_package_version(pkg, '1.3')

var/spack/repos/builtin.mock/packages/git-url-top-level/package.py

@@ -37,6 +37,7 @@ class GitUrlTopLevel(Package):
     git = 'https://example.com/some/git/repo'
     url = 'https://example.com/some/tarball-1.0.tar.gz'
 
+    # These resolve to git fetchers
     version('develop', branch='develop')
     version('3.4', commit='abc34')
     version('3.3', branch='releases/v3.3', commit='abc33')
@@ -44,10 +45,17 @@ class GitUrlTopLevel(Package):
     version('3.1', tag='v3.1', commit='abc31')
     version('3.0', tag='v3.0')
 
+    # These resolve to URL fetchers
     version('2.3', 'abc23', url='https://www.example.com/foo2.3.tar.gz')
     version('2.2', sha256='abc22', url='https://www.example.com/foo2.2.tar.gz')
     version('2.1', sha256='abc21')
     version('2.0', 'abc20')
 
+    # These result in a FetcherConflict b/c we can't tell what to use
+    version('1.3', sha256='abc13', commit='abc13')
+    version('1.2', sha512='abc12', branch='releases/v1.2')
+    version('1.1', md5='abc11', tag='v1.1')
+    version('1.0', 'abc11', tag='abc123')
+
     def install(self, spec, prefix):
         pass

var/spack/repos/builtin.mock/packages/svn-test/package.py

@@ -29,7 +29,7 @@ class SvnTest(Package):
     """Mock package that uses svn for fetching."""
     url      = "http://www.example.com/svn-test-1.0.tar.gz"
 
-    version('svn', 'to-be-filled-in-by-test')
+    version('svn', svn='to-be-filled-in-by-test')
 
     def install(self, spec, prefix):
         pass

var/spack/repos/builtin/packages/damselfly/package.py

@@ -28,9 +28,8 @@
 class Damselfly(CMakePackage):
     """Damselfly is a model-based parallel network simulator."""
     homepage = "https://github.com/llnl/damselfly"
-    url      = "https://github.com/llnl/damselfly"
+    url      = "https://github.com/LLNL/damselfly/archive/v1.0.tar.gz"
 
-    version('1.0', '05cf7e2d8ece4408c0f2abb7ab63fd74c0d62895',
+    version('1.0', sha256='560e1b800c9036766396a1033c00914bd8d181b911e87140c3ac8879baf6545a')
-            git='https://github.com/llnl/damselfly.git', tag='v1.0')
 
     depends_on('cmake@2.6:', type='build')

var/spack/repos/builtin/packages/nekbone/package.py

@@ -32,11 +32,11 @@ class Nekbone(Package):
        the spectral element method."""
 
     homepage = "https://github.com/Nek5000/Nekbone"
-    url = "https://github.com/Nek5000/Nekbone/tarball/v17.0"
+    url = "https://github.com/Nek5000/Nekbone/archive/v17.0.tar.gz"
 
     tags = ['proxy-app', 'ecp-proxy-app']
 
-    version('17.0', 'cc339684547614a0725959e41839fec1', git='https://github.com/Nek5000/Nekbone.git')
+    version('17.0', sha256='ae361cc61368a924398a28a296f675b7f0c4a9516788a7f8fa3c09d787cdf69b')
     version('develop', git='https://github.com/Nek5000/Nekbone.git')
 
     # Variants

var/spack/repos/builtin/packages/py-yt/package.py

@@ -35,26 +35,21 @@ class PyYt(PythonPackage):
        interdisciplinary community.
     """
     homepage = "http://yt-project.org"
-    url = "https://github.com/yt-project/yt.git"
+    url = "https://github.com/yt-project/yt/archive/yt-3.4.0.tar.gz"
+    git = "https://github.com/yt-project/yt.git"
 
-    version("3.4.0", "413b835f1b0e2a0bd26f1044ff7dbc94",
+    version("develop", branch="master")
-            url="https://github.com/yt-project/yt/archive/yt-3.4.0.tar.gz")
+
-    version("3.3.5", "2ad314ff3d3261e41825d15db027b0e7",
+    version('3.4.1', sha256='b9a73ade3726a8163fc992999c8c1010ca89473131901fe4d48b820ab2ced486')
-            url="https://bitbucket.org/yt_analysis/yt/get/yt-3.3.5.tar.gz")
+    version('3.4.0', sha256='2120793a76864cf3165b2b7290ef719e358fa57501ee8721941e7cfc434cfb2b')
-    version("3.3.4", "3a84e56dfd82f9dd923f3fb8490e679c",
+    version('3.3.5', sha256='2ebe4bbefd9f5367563ce4d7eb87d3f6ef0de1f97ed1c03106d9541e71b7e1ca')
-            url="https://bitbucket.org/yt_analysis/yt/get/yt-3.3.4.tar.gz")
+    version('3.3.4', sha256='2842bab891cfbf3269a3c4bd8f22fef23c9a15a790ba48c6490730cb51ce9b0e')
-    version("3.3.1", hg="https://bitbucket.org/yt_analysis/yt",
+    version('3.3.3', sha256='7b9244089e92b1d32cef791cd72760bb8c80b391eaec29672a5377c33f932d88')
-            tag="yt-3.3.1", commit="9bc3d0e9b750c923d44d73c447df64fc431f5838")
+    version('3.3.2', sha256='d323419ad3919e86d2af1738c846021fd7f5b5dc5c06059cdf3a2bc63226466a')
-    version("3.2.3", hg="https://bitbucket.org/yt_analysis/yt",
+    version('3.3.1', sha256='7ac68d5e05e2b57fb3635f1027f3201094f3547d584e72ab55fedbfd3bc09a36')
-            tag="yt-3.2.3", commit="83d2c1e9313e7d83eb5b96888451ff2646fd8ff3")
+    version('3.3.0', sha256='e6be799c0d9a83a06649f0d77a61ad9c23b94b34f94e16724e2b18f5c7513c33')
-    version("3.1.0", hg="https://bitbucket.org/yt_analysis/yt",
+    version('3.2.3', sha256='96476d17e9ce35f0d4380b2ddb398fe729e39f1f3894602ff07e49844541e5ca')
-            tag="yt-3.1.0", commit="fd7cdc4836188a3badf81adb477bcc1b9632e485")
+    version('3.2.2', sha256='498ed77b3dae8c54929602d4931f3c3e0a3420a9b500cbd870f50b1e0efea8c3')
-    version("3.0.2", hg="https://bitbucket.org/yt_analysis/yt",
-            tag="yt-3.0.2", commit="511887af4c995a78fe606e58ce8162c88380ecdc")
-    version("2.6.3", hg="https://bitbucket.org/yt_analysis/yt",
-            tag="yt-2.6.3", commit="816186f16396a16853810ac9ebcde5057d8d5b1a")
-    version("develop", git="https://github.com/yt-project/yt.git",
-            branch="master")
 
     variant("astropy", default=True, description="enable astropy support")
     variant("h5py", default=True, description="enable h5py support")

var/spack/repos/builtin/packages/r-affycompatible/package.py

@@ -34,9 +34,9 @@ class RAffycompatible(RPackage):
     (AGCC)-compatible sample annotation files."""
 
     homepage = "https://www.bioconductor.org/packages/AffyCompatible/"
-    url      = "https://git.bioconductor.org/packages/AffyCompatible"
+    git      = "https://git.bioconductor.org/packages/AffyCompatible"
 
-    version('1.36.0', 'https://git.bioconductor.org/packages/AffyCompatible', commit='dbbfd43a54ae1de6173336683a9461084ebf38c3')
+    version('1.36.0', commit='dbbfd43a54ae1de6173336683a9461084ebf38c3')
 
     depends_on('r@3.4.0:3.4.9', when=('@1.36.0'))
     depends_on('r-xml', type=('build', 'run'))

var/spack/repos/builtin/packages/ruby-narray/package.py

@@ -31,10 +31,9 @@ class RubyNarray(Package):
        similar to numpy.ndaray."""
 
     homepage = "https://rubygems.org/gems/narray"
-    url      = "https://github.com/ruby-numo/narray"
+    git      = "https://github.com/ruby-numo/narray"
 
-    version('0.9.0.9', '9cadbbccf1e01b6d1bc143c19d598cad1c420869',
+    version('0.9.0.9', commit='9cadbbccf1e01b6d1bc143c19d598cad1c420869')
-            git='https://github.com/ruby-numo/narray.git')
 
     extends('ruby')