hlrs-software-stack/spack
7
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

core: differentiate package-level fetch URLs by args to version()

Browse source 
- packagers can specify two top-level fetch URLs if one is `url`
  - e.g., `url` and `git` or `url` and `svn`

- allow only one VCS fetcher so we can differentiate between URL and VCS.

- also clean up fetcher logic and class structure
This commit is contained in:
Todd Gamblin 2018-07-22 13:49:01 -07:00
parent 04aec9d6f8
commit 773cfe088f
5 changed files with 282 additions and 86 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

192
lib/spack/spack/fetch_strategy.py
View file

@ -56,7 +56,7 @@
import spack.util.crypto as crypto
import spack.util.pattern as pattern
from spack.util.executable import which
from spack.util.string import comma_or, comma_and, quote
from spack.util.string import comma_and, quote
from spack.version import Version, ver
from spack.util.compression import decompressor_for, extension
@ -89,7 +89,15 @@ def __init__(cls, name, bases, dict):
class FetchStrategy(with_metaclass(FSMeta, object)):
"""Superclass of all fetch strategies."""
enabled = False # Non-abstract subclasses should be enabled.
required_attributes = None # Attributes required in version() args.
#: The URL attribute must be specified either at the package class
#: level, or as a keyword argument to ``version()``. It is used to
#: distinguish fetchers for different versions in the package DSL.
url_attr = None
#: Optional attributes can be used to distinguish fetchers when :
#: classes have multiple ``url_attrs`` at the top-level.
optional_attrs = [] # optional attributes in version() args.
def __init__(self):
# The stage is initialized late, so that fetch strategies can be
@ -156,7 +164,7 @@ def __str__(self): # Should be human readable URL.
# arguments in packages.
@classmethod
def matches(cls, args):
return any(k in args for k in cls.required_attributes)
return cls.url_attr in args
@pattern.composite(interface=FetchStrategy)
@ -179,7 +187,11 @@ class URLFetchStrategy(FetchStrategy):
checks the archive against a checksum,and decompresses the archive.
"""
enabled = True
required_attributes = ['url']
url_attr = 'url'
# these are checksum types. The generic 'checksum' is deprecated for
# specific hash names, but we need it for backward compatibility
optional_attrs = list(crypto.hashes.keys()) + ['checksum']
def __init__(self, url=None, checksum=None, **kwargs):
super(URLFetchStrategy, self).__init__()
@ -190,7 +202,7 @@ def __init__(self, url=None, checksum=None, **kwargs):
# digest can be set as the first argument, or from an explicit
# kwarg by the hash name.
self.digest = kwargs.get('checksum', checksum)
for h in crypto.hashes:
for h in self.optional_attrs:
if h in kwargs:
self.digest = kwargs[h]
@ -419,9 +431,6 @@ def __str__(self):
class CacheURLFetchStrategy(URLFetchStrategy):
"""The resource associated with a cache URL may be out of date."""
def __init__(self, *args, **kwargs):
super(CacheURLFetchStrategy, self).__init__(*args, **kwargs)
@_needs_stage
def fetch(self):
path = re.sub('^file://', '', self.url)
@ -452,35 +461,38 @@ def fetch(self):
class VCSFetchStrategy(FetchStrategy):
"""Superclass for version control system fetch strategies.
def __init__(self, name, *rev_types, **kwargs):
Like all fetchers, VCS fetchers are identified by the attributes
passed to the ``version`` directive. The optional_attrs for a VCS
fetch strategy represent types of revisions, e.g. tags, branches,
commits, etc.
The required attributes (git, svn, etc.) are used to specify the URL
and to distinguish a VCS fetch strategy from a URL fetch strategy.
"""
def __init__(self, **kwargs):
super(VCSFetchStrategy, self).__init__()
self.name = name
# Set a URL based on the type of fetch strategy.
self.url = kwargs.get(name, None)
self.url = kwargs.get(self.url_attr, None)
if not self.url:
raise ValueError(
"%s requires %s argument." % (self.__class__, name))
"%s requires %s argument." % (self.__class__, self.url_attr))
# Ensure that there's only one of the rev_types
if sum(k in kwargs for k in rev_types) > 1:
raise ValueError(
"Supply only one of %s to fetch with %s" % (
comma_or(rev_types), name
))
# Set attributes for each rev type.
for rt in rev_types:
setattr(self, rt, kwargs.get(rt, None))
for attr in self.optional_attrs:
setattr(self, attr, kwargs.get(attr, None))
@_needs_stage
def check(self):
tty.msg("No checksum needed when fetching with %s" % self.name)
tty.msg("No checksum needed when fetching with %s" % self.url_attr)
@_needs_stage
def expand(self):
tty.debug("Source fetched with %s is already expanded." % self.name)
tty.debug(
"Source fetched with %s is already expanded." % self.url_attr)
@_needs_stage
def archive(self, destination, **kwargs):
@ -517,15 +529,15 @@ class GoFetchStrategy(VCSFetchStrategy):
Go get does not natively support versions, they can be faked with git
"""
enabled = True
required_attributes = ('go', )
url_attr = 'go'
def __init__(self, **kwargs):
# Discards the keywords in kwargs that may conflict with the next
# call to __init__
forwarded_args = copy.copy(kwargs)
forwarded_args.pop('name', None)
super(GoFetchStrategy, self).__init__(**forwarded_args)
super(GoFetchStrategy, self).__init__('go', **forwarded_args)
self._go = None
@property
@ -583,16 +595,16 @@ class GitFetchStrategy(VCSFetchStrategy):
* ``commit``: Particular commit hash in the repo
"""
enabled = True
required_attributes = ('git', )
url_attr = 'git'
optional_attrs = ['tag', 'branch', 'commit']
def __init__(self, **kwargs):
# Discards the keywords in kwargs that may conflict with the next call
# to __init__
forwarded_args = copy.copy(kwargs)
forwarded_args.pop('name', None)
super(GitFetchStrategy, self).__init__(**forwarded_args)
super(GitFetchStrategy, self).__init__(
'git', 'tag', 'branch', 'commit', **forwarded_args)
self._git = None
self.submodules = kwargs.get('submodules', False)
@ -745,16 +757,16 @@ class SvnFetchStrategy(VCSFetchStrategy):
revision='1641')
"""
enabled = True
required_attributes = ['svn']
url_attr = 'svn'
optional_attrs = ['revision']
def __init__(self, **kwargs):
# Discards the keywords in kwargs that may conflict with the next call
# to __init__
forwarded_args = copy.copy(kwargs)
forwarded_args.pop('name', None)
super(SvnFetchStrategy, self).__init__(**forwarded_args)
super(SvnFetchStrategy, self).__init__(
'svn', 'revision', **forwarded_args)
self._svn = None
if self.revision is not None:
self.revision = str(self.revision)
@ -845,16 +857,16 @@ class HgFetchStrategy(VCSFetchStrategy):
* ``revision``: Particular revision, branch, or tag.
"""
enabled = True
required_attributes = ['hg']
url_attr = 'hg'
optional_attrs = ['revision']
def __init__(self, **kwargs):
# Discards the keywords in kwargs that may conflict with the next call
# to __init__
forwarded_args = copy.copy(kwargs)
forwarded_args.pop('name', None)
super(HgFetchStrategy, self).__init__(**forwarded_args)
super(HgFetchStrategy, self).__init__(
'hg', 'revision', **forwarded_args)
self._hg = None
@property
@ -957,74 +969,88 @@ def from_kwargs(**kwargs):
for fetcher in all_strategies:
if fetcher.matches(kwargs):
return fetcher(**kwargs)
# Raise an error in case we can't instantiate any known strategy
message = "Cannot instantiate any FetchStrategy"
long_message = message + " from the given arguments : {arguments}".format(
arguments=kwargs)
raise FetchError(message, long_message)
raise InvalidArgsError(**kwargs)
def args_are_for(args, fetcher):
fetcher.matches(args)
def check_pkg_attributes(pkg):
"""Find ambiguous top-level fetch attributes in a package.
Currently this only ensures that two or more VCS fetch strategies are
not specified at once.
"""
# a single package cannot have URL attributes for multiple VCS fetch
# strategies *unless* they are the same attribute.
conflicts = set([s.url_attr for s in all_strategies
if hasattr(pkg, s.url_attr)])
def check_attributes(pkg):
"""Find ambiguous top-level fetch attributes in a package."""
# a single package cannot have required attributes from multiple
# fetch strategies *unless* they are the same attribute.
conflicts = set(
sum([[a for a in s.required_attributes if hasattr(pkg, a)]
for s in all_strategies],
[]))
# URL isn't a VCS fetch method. We can use it with a VCS method.
conflicts -= set(['url'])
if len(conflicts) > 1:
raise FetcherConflict(
'Package %s cannot specify %s together. Must pick only one.'
'Package %s cannot specify %s together. Pick at most one.'
% (pkg.name, comma_and(quote(conflicts))))
def _extrapolate(pkg, version):
"""Create a fetcher from an extrapolated URL for this version."""
try:
return URLFetchStrategy(pkg.url_for_version(version))
except spack.package.NoURLError:
msg = ("Can't extrapolate a URL for version %s "
"because package %s defines no URLs")
raise ExtrapolationError(msg % (version, pkg.name))
def _from_merged_attrs(fetcher, pkg, version):
"""Create a fetcher from merged package and version attributes."""
if fetcher.url_attr == 'url':
url = pkg.url_for_version(version)
else:
url = getattr(pkg, fetcher.url_attr)
attrs = {fetcher.url_attr: url}
attrs.update(pkg.versions[version])
return fetcher(**attrs)
def for_package_version(pkg, version):
"""Determine a fetch strategy based on the arguments supplied to
version() in the package description."""
check_attributes(pkg)
check_pkg_attributes(pkg)
if not isinstance(version, Version):
version = Version(version)
# If it's not a known version, extrapolate one by URL.
# If it's not a known version, try to extrapolate one by URL
if version not in pkg.versions:
try:
url = pkg.url_for_version(version)
except spack.package.NoURLError:
msg = ("Can't extrapolate a URL for version %s "
"because package %s defines no URLs")
raise ExtrapolationError(msg % (version, pkg.name))
if not url:
raise InvalidArgsError(pkg, version)
return URLFetchStrategy(url)
return _extrapolate(pkg, version)
# Grab a dict of args out of the package version dict
args = pkg.versions[version]
# Test all strategies against per-version arguments.
# If the version specifies a `url_attr` directly, use that.
for fetcher in all_strategies:
if fetcher.matches(args):
if fetcher.url_attr in args:
return fetcher(**args)
# If nothing matched for a *specific* version, test all strategies
# against attributes in the version directives and on the package
# if a version's optional attributes imply a particular fetch
# strategy, and we have the `url_attr`, then use that strategy.
for fetcher in all_strategies:
attrs = dict((attr, getattr(pkg, attr))
for attr in fetcher.required_attributes
if hasattr(pkg, attr))
if 'url' in attrs:
attrs['url'] = pkg.url_for_version(version)
attrs.update(args)
if fetcher.matches(attrs):
return fetcher(**attrs)
if hasattr(pkg, fetcher.url_attr) or fetcher.url_attr == 'url':
optionals = fetcher.optional_attrs
if optionals and any(a in args for a in optionals):
return _from_merged_attrs(fetcher, pkg, version)
raise InvalidArgsError(pkg, version)
# if the optional attributes tell us nothing, then use any `url_attr`
# on the package. This prefers URL vs. VCS, b/c URLFetchStrategy is
# defined first in this file.
for fetcher in all_strategies:
if hasattr(pkg, fetcher.url_attr):
return _from_merged_attrs(fetcher, pkg, version)
raise InvalidArgsError(pkg, version, **args)
def from_list_url(pkg):
@ -1116,11 +1142,15 @@ class FetcherConflict(FetchError):
class InvalidArgsError(FetchError):
def __init__(self, pkg, version):
msg = ("Could not construct a fetch strategy for package %s at "
"version %s")
msg %= (pkg.name, version)
super(InvalidArgsError, self).__init__(msg)
"""Raised when a version can't be deduced from a set of arguments."""
def __init__(self, pkg=None, version=None, **args):
msg = "Could not guess a fetch strategy"
if pkg:
msg += ' for {pkg}'.format(pkg=pkg)
if version:
msg += '@{version}'.format(version=version)
long_msg = 'with arguments: {args}'.format(args=args)
super(InvalidArgsError, self).__init__(msg, long_msg)
class ChecksumError(FetchError):

78
lib/spack/spack/test/packages.py
View file

@ -263,9 +263,81 @@ def test_no_extrapolate_without_url(mock_packages, config):
spack.fetch_strategy.for_package_version(pkg, '1.1')
def test_git_and_url_top_level(mock_packages, config):
"""Verify conflict when url and git are specified together."""
def test_two_vcs_fetchers_top_level(mock_packages, config):
"""Verify conflict when two VCS strategies are specified together."""
pkg = spack.repo.get('git-and-url-top-level')
pkg = spack.repo.get('git-url-svn-top-level')
with pytest.raises(spack.fetch_strategy.FetcherConflict):
spack.fetch_strategy.for_package_version(pkg, '1.0')
pkg = spack.repo.get('git-svn-top-level')
with pytest.raises(spack.fetch_strategy.FetcherConflict):
spack.fetch_strategy.for_package_version(pkg, '1.0')
def test_git_url_top_level(mock_packages, config):
"""Test fetch strategy inference when url is specified with a VCS."""
pkg = spack.repo.get('git-url-top-level')
fetcher = spack.fetch_strategy.for_package_version(pkg, '2.0')
assert isinstance(fetcher, spack.fetch_strategy.URLFetchStrategy)
assert fetcher.url == 'https://example.com/some/tarball-2.0.tar.gz'
assert fetcher.digest == 'abc20'
fetcher = spack.fetch_strategy.for_package_version(pkg, '2.1')
assert isinstance(fetcher, spack.fetch_strategy.URLFetchStrategy)
assert fetcher.url == 'https://example.com/some/tarball-2.1.tar.gz'
assert fetcher.digest == 'abc21'
fetcher = spack.fetch_strategy.for_package_version(pkg, '2.2')
assert isinstance(fetcher, spack.fetch_strategy.URLFetchStrategy)
assert fetcher.url == 'https://www.example.com/foo2.2.tar.gz'
assert fetcher.digest == 'abc22'
fetcher = spack.fetch_strategy.for_package_version(pkg, '2.3')
assert isinstance(fetcher, spack.fetch_strategy.URLFetchStrategy)
assert fetcher.url == 'https://www.example.com/foo2.3.tar.gz'
assert fetcher.digest == 'abc23'
fetcher = spack.fetch_strategy.for_package_version(pkg, '3.0')
assert isinstance(fetcher, spack.fetch_strategy.GitFetchStrategy)
assert fetcher.url == 'https://example.com/some/git/repo'
assert fetcher.tag == 'v3.0'
assert fetcher.commit is None
assert fetcher.branch is None
fetcher = spack.fetch_strategy.for_package_version(pkg, '3.1')
assert isinstance(fetcher, spack.fetch_strategy.GitFetchStrategy)
assert fetcher.url == 'https://example.com/some/git/repo'
assert fetcher.tag == 'v3.1'
assert fetcher.commit == 'abc31'
assert fetcher.branch is None
fetcher = spack.fetch_strategy.for_package_version(pkg, '3.2')
assert isinstance(fetcher, spack.fetch_strategy.GitFetchStrategy)
assert fetcher.url == 'https://example.com/some/git/repo'
assert fetcher.tag is None
assert fetcher.commit is None
assert fetcher.branch == 'releases/v3.2'
fetcher = spack.fetch_strategy.for_package_version(pkg, '3.3')
assert isinstance(fetcher, spack.fetch_strategy.GitFetchStrategy)
assert fetcher.url == 'https://example.com/some/git/repo'
assert fetcher.tag is None
assert fetcher.commit == 'abc33'
assert fetcher.branch == 'releases/v3.3'
fetcher = spack.fetch_strategy.for_package_version(pkg, '3.4')
assert isinstance(fetcher, spack.fetch_strategy.GitFetchStrategy)
assert fetcher.url == 'https://example.com/some/git/repo'
assert fetcher.tag is None
assert fetcher.commit == 'abc34'
assert fetcher.branch is None
fetcher = spack.fetch_strategy.for_package_version(pkg, 'develop')
assert isinstance(fetcher, spack.fetch_strategy.GitFetchStrategy)
assert fetcher.url == 'https://example.com/some/git/repo'
assert fetcher.tag is None
assert fetcher.commit is None
assert fetcher.branch == 'develop'

39
var/spack/repos/builtin.mock/packages/git-svn-top-level/package.py Normal file
View file

@ -0,0 +1,39 @@
##############################################################################
# Copyright (c) 2013-2018, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# This file is part of Spack.
# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
# For details, see https://github.com/spack/spack
# Please also see the NOTICE and LICENSE files for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License (as
# published by the Free Software Foundation) version 2.1, February 1999.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
# conditions of the GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
from spack import *
class GitSvnTopLevel(Package):
"""Mock package that uses git for fetching."""
homepage = "http://www.git-fetch-example.com"
# can't have two VCS fetchers.
git = 'https://example.com/some/git/repo'
svn = 'https://example.com/some/svn/repo'
version('2.0')
def install(self, spec, prefix):
pass

6
var/spack/repos/builtin.mock/packages/git-and-url-top-level/package.py → var/spack/repos/builtin.mock/packages/git-url-svn-top-level/package.py
View file

@ -25,12 +25,14 @@
from spack import *
class GitAndUrlTopLevel(Package):
class GitUrlSvnTopLevel(Package):
"""Mock package that uses git for fetching."""
homepage = "http://www.git-fetch-example.com"
git = 'https://example.com/some/git/repo'
# can't have two VCS fetchers.
url = 'https://example.com/some/tarball-1.0.tar.gz'
git = 'https://example.com/some/git/repo'
svn = 'https://example.com/some/svn/repo'
version('2.0')

53
var/spack/repos/builtin.mock/packages/git-url-top-level/package.py Normal file
View file

@ -0,0 +1,53 @@
##############################################################################
# Copyright (c) 2013-2018, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# This file is part of Spack.
# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
# For details, see https://github.com/spack/spack
# Please also see the NOTICE and LICENSE files for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License (as
# published by the Free Software Foundation) version 2.1, February 1999.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
# conditions of the GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
from spack import *
class GitUrlTopLevel(Package):
"""Mock package that top-level git and url attributes.
This demonstrates how Spack infers fetch mechanisms from parameters
to the ``version`` directive.
"""
homepage = "http://www.git-fetch-example.com"
git = 'https://example.com/some/git/repo'
url = 'https://example.com/some/tarball-1.0.tar.gz'
version('develop', branch='develop')
version('3.4', commit='abc34')
version('3.3', branch='releases/v3.3', commit='abc33')
version('3.2', branch='releases/v3.2')
version('3.1', tag='v3.1', commit='abc31')
version('3.0', tag='v3.0')
version('2.3', 'abc23', url='https://www.example.com/foo2.3.tar.gz')
version('2.2', sha256='abc22', url='https://www.example.com/foo2.2.tar.gz')
version('2.1', sha256='abc21')
version('2.0', 'abc20')
def install(self, spec, prefix):
pass