diff --git a/bin/spack b/bin/spack index 2f74761a78..75874ca39e 100755 --- a/bin/spack +++ b/bin/spack @@ -96,7 +96,7 @@ if args.mock: # If the user asked for it, don't check ssl certs. if args.insecure: - tty.warn("You asked for --insecure, which does not check SSL certificates.") + tty.warn("You asked for --insecure, which does not check SSL certificates or checksums.") spack.curl.add_default_arg('-k') # Try to load the particular command asked for and run it diff --git a/lib/spack/spack/cmd/mirror.py b/lib/spack/spack/cmd/mirror.py index 6a6c2b60c7..6e2e4dfb24 100644 --- a/lib/spack/spack/cmd/mirror.py +++ b/lib/spack/spack/cmd/mirror.py @@ -54,6 +54,9 @@ def setup_parser(subparser): 'specs', nargs=argparse.REMAINDER, help="Specs of packages to put in mirror") create_parser.add_argument( '-f', '--file', help="File with specs of packages to put in mirror.") + create_parser.add_argument( + '-o', '--one-version-per-spec', action='store_const', const=1, default=0, + help="Only fetch one 'preferred' version per spec, not all known versions.") add_parser = sp.add_parser('add', help=mirror_add.__doc__) add_parser.add_argument('name', help="Mnemonic name for mirror.") @@ -128,26 +131,29 @@ def mirror_create(args): # If nothing is passed, use all packages. if not specs: specs = [Spec(n) for n in spack.db.all_package_names()] + specs.sort(key=lambda s: s.format("$_$@").lower()) # Default name for directory is spack-mirror- - if not args.directory: + directory = args.directory + if not directory: timestamp = datetime.now().strftime("%Y-%m-%d") - args.directory = 'spack-mirror-' + timestamp + directory = 'spack-mirror-' + timestamp # Make sure nothing is in the way. existed = False - if os.path.isfile(args.directory): - tty.error("%s already exists and is a file." % args.directory) - elif os.path.isdir(args.directory): + if os.path.isfile(directory): + tty.error("%s already exists and is a file." % directory) + elif os.path.isdir(directory): existed = True # Actually do the work to create the mirror - present, mirrored, error = spack.mirror.create(args.directory, specs) + present, mirrored, error = spack.mirror.create( + directory, specs, num_versions=args.one_version_per_spec) p, m, e = len(present), len(mirrored), len(error) verb = "updated" if existed else "created" tty.msg( - "Successfully %s mirror in %s." % (verb, args.directory), + "Successfully %s mirror in %s." % (verb, directory), "Archive stats:", " %-4d already present" % p, " %-4d added" % m, diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 9f9cd1789d..e603806af9 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -73,7 +73,6 @@ def concretize_version(self, spec): if valid_versions: spec.versions = ver([valid_versions[-1]]) else: - print spec raise NoValidVersionError(spec) diff --git a/lib/spack/spack/fetch_strategy.py b/lib/spack/spack/fetch_strategy.py index 2cff15845b..2b574eaba7 100644 --- a/lib/spack/spack/fetch_strategy.py +++ b/lib/spack/spack/fetch_strategy.py @@ -93,9 +93,6 @@ def archive(self, destination): pass # Used to create tarball for mirror. def __str__(self): # Should be human readable URL. return "FetchStrategy.__str___" - @property - def unique_name(self): pass - # This method is used to match fetch strategies to version() # arguments in packages. @classmethod @@ -197,7 +194,10 @@ def archive(self, destination): """Just moves this archive to the destination.""" if not self.archive_file: raise NoArchiveFileError("Cannot call archive() before fetching.") - assert(extension(destination) == extension(self.archive_file)) + + if not extension(destination) == extension(self.archive_file): + raise ValueError("Cannot archive without matching extensions.") + shutil.move(self.archive_file, destination) @@ -236,10 +236,6 @@ def __str__(self): else: return "URLFetchStrategy" - @property - def unique_name(self): - return "spack-fetch-url:%s" % self - class VCSFetchStrategy(FetchStrategy): def __init__(self, name, *rev_types, **kwargs): @@ -393,17 +389,6 @@ def reset(self): self.git('clean', '-f') - @property - def unique_name(self): - name = "spack-fetch-git:%s" % self.url - if self.commit: - name += "@" + self.commit - elif self.branch: - name += "@" + self.branch - elif self.tag: - name += "@" + self.tag - - class SvnFetchStrategy(VCSFetchStrategy): """Fetch strategy that gets source code from a subversion repository. Use like this in a package: @@ -477,14 +462,6 @@ def reset(self): self.svn('revert', '.', '-R') - @property - def unique_name(self): - name = "spack-fetch-svn:%s" % self.url - if self.revision: - name += "@" + self.revision - - - class HgFetchStrategy(VCSFetchStrategy): """Fetch strategy that gets source code from a Mercurial repository. Use like this in a package: @@ -560,14 +537,6 @@ def reset(self): self.stage.chdir_to_source() - @property - def unique_name(self): - name = "spack-fetch-hg:%s" % self.url - if self.revision: - name += "@" + self.revision - - - def from_url(url): """Given a URL, find an appropriate fetch strategy for it. Currently just gives you a URLFetchStrategy that uses curl. diff --git a/lib/spack/spack/mirror.py b/lib/spack/spack/mirror.py index e116ce83b2..f7bbb3f840 100644 --- a/lib/spack/spack/mirror.py +++ b/lib/spack/spack/mirror.py @@ -49,16 +49,18 @@ def mirror_archive_filename(spec): if not spec.version.concrete: raise ValueError("mirror.path requires spec with concrete version.") - url = spec.package.default_url - if url is None: - ext = 'tar.gz' + fetcher = spec.package.fetcher + if isinstance(fetcher, fs.URLFetchStrategy): + # If we fetch this version with a URLFetchStrategy, use URL's archive type + ext = extension(fetcher.url) else: - ext = extension(url) + # Otherwise we'll make a .tar.gz ourselves + ext = 'tar.gz' return "%s-%s.%s" % (spec.package.name, spec.version, ext) -def get_matching_versions(specs): +def get_matching_versions(specs, **kwargs): """Get a spec for EACH known version matching any spec in the list.""" matching = [] for spec in specs: @@ -69,11 +71,18 @@ def get_matching_versions(specs): tty.msg("No safe (checksummed) versions for package %s." % pkg.name) continue - for v in reversed(sorted(pkg.versions)): + num_versions = kwargs.get('num_versions', 0) + for i, v in enumerate(reversed(sorted(pkg.versions))): + # Generate no more than num_versions versions for each spec. + if num_versions and i >= num_versions: + break + + # Generate only versions that satisfy the spec. if v.satisfies(spec.versions): s = Spec(pkg.name) s.versions = VersionList([v]) matching.append(s) + return matching @@ -86,6 +95,11 @@ def create(path, specs, **kwargs): specs Any package versions matching these specs will be added to the mirror. + Keyword args: + no_checksum: If True, do not checkpoint when fetching (default False) + num_versions: Max number of versions to fetch per spec, + if spec is ambiguous (default is 0 for all of them) + Return Value: Returns a tuple of lists: (present, mirrored, error) * present: Package specs that were already prsent. @@ -104,13 +118,15 @@ def create(path, specs, **kwargs): specs = [s if isinstance(s, Spec) else Spec(s) for s in specs] # Get concrete specs for each matching version of these specs. - version_specs = get_matching_versions(specs) + version_specs = get_matching_versions( + specs, num_versions=kwargs.get('num_versions', 0)) for s in version_specs: s.concretize() - # Create a directory if none exists - if not os.path.isdir(path): - mkdirp(path) + # Get the absolute path of the root before we start jumping around. + mirror_root = os.path.abspath(path) + if not os.path.isdir(mirror_root): + mkdirp(mirror_root) # Things to keep track of while parsing specs. present = [] @@ -124,19 +140,21 @@ def create(path, specs, **kwargs): stage = None try: # create a subdirectory for the current package@version - realpath = os.path.realpath(path) - subdir = join_path(realpath, pkg.name) + subdir = join_path(mirror_root, pkg.name) mkdirp(subdir) archive_file = mirror_archive_filename(spec) archive_path = join_path(subdir, archive_file) - if os.path.exists(archive_path): + + if os.path.exists(archive_file): + tty.msg("%s is already present. Skipping." % spec.format("$_$@")) present.append(spec) continue # Set up a stage and a fetcher for the download + unique_fetch_name = spec.format("$_$@") fetcher = fs.for_package_version(pkg, pkg.version) - stage = Stage(fetcher, name=fetcher.unique_name) + stage = Stage(fetcher, name=unique_fetch_name) fetcher.set_stage(stage) # Do the fetch and checksum if necessary @@ -148,7 +166,7 @@ def create(path, specs, **kwargs): # Fetchers have to know how to archive their files. Use # that to move/copy/create an archive in the mirror. fetcher.archive(archive_path) - tty.msg("Added %s to mirror" % archive_path) + tty.msg("Added %s." % spec.format("$_$@")) mirrored.append(spec) except Exception, e: diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 34efe1bec9..e462562e85 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -598,18 +598,6 @@ def url_version(self, version): return str(version) - @property - def default_url(self): - if self.spec.versions.concrete: - return self.url_for_version(self.version) - else: - url = getattr(self, 'url', None) - if url: - return url - - return None - - def remove_prefix(self): """Removes the prefix for a package along with any empty parent directories.""" spack.install_layout.remove_path_for_spec(self.spec) diff --git a/lib/spack/spack/test/__init__.py b/lib/spack/spack/test/__init__.py index ca4c869e42..be9ac5a560 100644 --- a/lib/spack/spack/test/__init__.py +++ b/lib/spack/spack/test/__init__.py @@ -50,7 +50,8 @@ 'python_version', 'git_fetch', 'svn_fetch', - 'hg_fetch'] + 'hg_fetch', + 'mirror'] def list_tests(): diff --git a/lib/spack/spack/test/install.py b/lib/spack/spack/test/install.py index 0d53bb45c7..e052f53e77 100644 --- a/lib/spack/spack/test/install.py +++ b/lib/spack/spack/test/install.py @@ -36,39 +36,17 @@ from spack.directory_layout import SpecHashDirectoryLayout from spack.util.executable import which from spack.test.mock_packages_test import * +from spack.test.mock_repo import MockArchive -dir_name = 'trivial-1.0' -archive_name = 'trivial-1.0.tar.gz' -install_test_package = 'trivial_install_test_package' - class InstallTest(MockPackagesTest): """Tests install and uninstall on a trivial package.""" def setUp(self): super(InstallTest, self).setUp() - self.stage = Stage('not_a_real_url') - archive_dir = join_path(self.stage.path, dir_name) - dummy_configure = join_path(archive_dir, 'configure') - - mkdirp(archive_dir) - with closing(open(dummy_configure, 'w')) as configure: - configure.write( - "#!/bin/sh\n" - "prefix=$(echo $1 | sed 's/--prefix=//')\n" - "cat > Makefile < Makefile <