Add test cases for mirroring.

This commit is contained in:
Todd Gamblin 2014-10-15 21:07:41 -04:00
parent 8e3c2d8a26
commit 6fdfd83e6b
10 changed files with 281 additions and 121 deletions

View file

@ -96,7 +96,7 @@ if args.mock:
# If the user asked for it, don't check ssl certs.
if args.insecure:
tty.warn("You asked for --insecure, which does not check SSL certificates.")
tty.warn("You asked for --insecure, which does not check SSL certificates or checksums.")
spack.curl.add_default_arg('-k')
# Try to load the particular command asked for and run it

View file

@ -54,6 +54,9 @@ def setup_parser(subparser):
'specs', nargs=argparse.REMAINDER, help="Specs of packages to put in mirror")
create_parser.add_argument(
'-f', '--file', help="File with specs of packages to put in mirror.")
create_parser.add_argument(
'-o', '--one-version-per-spec', action='store_const', const=1, default=0,
help="Only fetch one 'preferred' version per spec, not all known versions.")
add_parser = sp.add_parser('add', help=mirror_add.__doc__)
add_parser.add_argument('name', help="Mnemonic name for mirror.")
@ -128,26 +131,29 @@ def mirror_create(args):
# If nothing is passed, use all packages.
if not specs:
specs = [Spec(n) for n in spack.db.all_package_names()]
specs.sort(key=lambda s: s.format("$_$@").lower())
# Default name for directory is spack-mirror-<DATESTAMP>
if not args.directory:
directory = args.directory
if not directory:
timestamp = datetime.now().strftime("%Y-%m-%d")
args.directory = 'spack-mirror-' + timestamp
directory = 'spack-mirror-' + timestamp
# Make sure nothing is in the way.
existed = False
if os.path.isfile(args.directory):
tty.error("%s already exists and is a file." % args.directory)
elif os.path.isdir(args.directory):
if os.path.isfile(directory):
tty.error("%s already exists and is a file." % directory)
elif os.path.isdir(directory):
existed = True
# Actually do the work to create the mirror
present, mirrored, error = spack.mirror.create(args.directory, specs)
present, mirrored, error = spack.mirror.create(
directory, specs, num_versions=args.one_version_per_spec)
p, m, e = len(present), len(mirrored), len(error)
verb = "updated" if existed else "created"
tty.msg(
"Successfully %s mirror in %s." % (verb, args.directory),
"Successfully %s mirror in %s." % (verb, directory),
"Archive stats:",
" %-4d already present" % p,
" %-4d added" % m,

View file

@ -73,7 +73,6 @@ def concretize_version(self, spec):
if valid_versions:
spec.versions = ver([valid_versions[-1]])
else:
print spec
raise NoValidVersionError(spec)

View file

@ -93,9 +93,6 @@ def archive(self, destination): pass # Used to create tarball for mirror.
def __str__(self): # Should be human readable URL.
return "FetchStrategy.__str___"
@property
def unique_name(self): pass
# This method is used to match fetch strategies to version()
# arguments in packages.
@classmethod
@ -197,7 +194,10 @@ def archive(self, destination):
"""Just moves this archive to the destination."""
if not self.archive_file:
raise NoArchiveFileError("Cannot call archive() before fetching.")
assert(extension(destination) == extension(self.archive_file))
if not extension(destination) == extension(self.archive_file):
raise ValueError("Cannot archive without matching extensions.")
shutil.move(self.archive_file, destination)
@ -236,10 +236,6 @@ def __str__(self):
else:
return "URLFetchStrategy<no url>"
@property
def unique_name(self):
return "spack-fetch-url:%s" % self
class VCSFetchStrategy(FetchStrategy):
def __init__(self, name, *rev_types, **kwargs):
@ -393,17 +389,6 @@ def reset(self):
self.git('clean', '-f')
@property
def unique_name(self):
name = "spack-fetch-git:%s" % self.url
if self.commit:
name += "@" + self.commit
elif self.branch:
name += "@" + self.branch
elif self.tag:
name += "@" + self.tag
class SvnFetchStrategy(VCSFetchStrategy):
"""Fetch strategy that gets source code from a subversion repository.
Use like this in a package:
@ -477,14 +462,6 @@ def reset(self):
self.svn('revert', '.', '-R')
@property
def unique_name(self):
name = "spack-fetch-svn:%s" % self.url
if self.revision:
name += "@" + self.revision
class HgFetchStrategy(VCSFetchStrategy):
"""Fetch strategy that gets source code from a Mercurial repository.
Use like this in a package:
@ -560,14 +537,6 @@ def reset(self):
self.stage.chdir_to_source()
@property
def unique_name(self):
name = "spack-fetch-hg:%s" % self.url
if self.revision:
name += "@" + self.revision
def from_url(url):
"""Given a URL, find an appropriate fetch strategy for it.
Currently just gives you a URLFetchStrategy that uses curl.

View file

@ -49,16 +49,18 @@ def mirror_archive_filename(spec):
if not spec.version.concrete:
raise ValueError("mirror.path requires spec with concrete version.")
url = spec.package.default_url
if url is None:
ext = 'tar.gz'
fetcher = spec.package.fetcher
if isinstance(fetcher, fs.URLFetchStrategy):
# If we fetch this version with a URLFetchStrategy, use URL's archive type
ext = extension(fetcher.url)
else:
ext = extension(url)
# Otherwise we'll make a .tar.gz ourselves
ext = 'tar.gz'
return "%s-%s.%s" % (spec.package.name, spec.version, ext)
def get_matching_versions(specs):
def get_matching_versions(specs, **kwargs):
"""Get a spec for EACH known version matching any spec in the list."""
matching = []
for spec in specs:
@ -69,11 +71,18 @@ def get_matching_versions(specs):
tty.msg("No safe (checksummed) versions for package %s." % pkg.name)
continue
for v in reversed(sorted(pkg.versions)):
num_versions = kwargs.get('num_versions', 0)
for i, v in enumerate(reversed(sorted(pkg.versions))):
# Generate no more than num_versions versions for each spec.
if num_versions and i >= num_versions:
break
# Generate only versions that satisfy the spec.
if v.satisfies(spec.versions):
s = Spec(pkg.name)
s.versions = VersionList([v])
matching.append(s)
return matching
@ -86,6 +95,11 @@ def create(path, specs, **kwargs):
specs Any package versions matching these specs will be added
to the mirror.
Keyword args:
no_checksum: If True, do not checkpoint when fetching (default False)
num_versions: Max number of versions to fetch per spec,
if spec is ambiguous (default is 0 for all of them)
Return Value:
Returns a tuple of lists: (present, mirrored, error)
* present: Package specs that were already prsent.
@ -104,13 +118,15 @@ def create(path, specs, **kwargs):
specs = [s if isinstance(s, Spec) else Spec(s) for s in specs]
# Get concrete specs for each matching version of these specs.
version_specs = get_matching_versions(specs)
version_specs = get_matching_versions(
specs, num_versions=kwargs.get('num_versions', 0))
for s in version_specs:
s.concretize()
# Create a directory if none exists
if not os.path.isdir(path):
mkdirp(path)
# Get the absolute path of the root before we start jumping around.
mirror_root = os.path.abspath(path)
if not os.path.isdir(mirror_root):
mkdirp(mirror_root)
# Things to keep track of while parsing specs.
present = []
@ -124,19 +140,21 @@ def create(path, specs, **kwargs):
stage = None
try:
# create a subdirectory for the current package@version
realpath = os.path.realpath(path)
subdir = join_path(realpath, pkg.name)
subdir = join_path(mirror_root, pkg.name)
mkdirp(subdir)
archive_file = mirror_archive_filename(spec)
archive_path = join_path(subdir, archive_file)
if os.path.exists(archive_path):
if os.path.exists(archive_file):
tty.msg("%s is already present. Skipping." % spec.format("$_$@"))
present.append(spec)
continue
# Set up a stage and a fetcher for the download
unique_fetch_name = spec.format("$_$@")
fetcher = fs.for_package_version(pkg, pkg.version)
stage = Stage(fetcher, name=fetcher.unique_name)
stage = Stage(fetcher, name=unique_fetch_name)
fetcher.set_stage(stage)
# Do the fetch and checksum if necessary
@ -148,7 +166,7 @@ def create(path, specs, **kwargs):
# Fetchers have to know how to archive their files. Use
# that to move/copy/create an archive in the mirror.
fetcher.archive(archive_path)
tty.msg("Added %s to mirror" % archive_path)
tty.msg("Added %s." % spec.format("$_$@"))
mirrored.append(spec)
except Exception, e:

View file

@ -598,18 +598,6 @@ def url_version(self, version):
return str(version)
@property
def default_url(self):
if self.spec.versions.concrete:
return self.url_for_version(self.version)
else:
url = getattr(self, 'url', None)
if url:
return url
return None
def remove_prefix(self):
"""Removes the prefix for a package along with any empty parent directories."""
spack.install_layout.remove_path_for_spec(self.spec)

View file

@ -50,7 +50,8 @@
'python_version',
'git_fetch',
'svn_fetch',
'hg_fetch']
'hg_fetch',
'mirror']
def list_tests():

View file

@ -36,39 +36,17 @@
from spack.directory_layout import SpecHashDirectoryLayout
from spack.util.executable import which
from spack.test.mock_packages_test import *
from spack.test.mock_repo import MockArchive
dir_name = 'trivial-1.0'
archive_name = 'trivial-1.0.tar.gz'
install_test_package = 'trivial_install_test_package'
class InstallTest(MockPackagesTest):
"""Tests install and uninstall on a trivial package."""
def setUp(self):
super(InstallTest, self).setUp()
self.stage = Stage('not_a_real_url')
archive_dir = join_path(self.stage.path, dir_name)
dummy_configure = join_path(archive_dir, 'configure')
mkdirp(archive_dir)
with closing(open(dummy_configure, 'w')) as configure:
configure.write(
"#!/bin/sh\n"
"prefix=$(echo $1 | sed 's/--prefix=//')\n"
"cat > Makefile <<EOF\n"
"all:\n"
"\techo Building...\n\n"
"install:\n"
"\tmkdir -p $prefix\n"
"\ttouch $prefix/dummy_file\n"
"EOF\n")
os.chmod(dummy_configure, 0755)
with working_dir(self.stage.path):
tar = which('tar')
tar('-czf', archive_name, dir_name)
# create a simple installable package directory and tarball
self.repo = MockArchive()
# We use a fake package, so skip the checksum.
spack.do_checksum = False
@ -83,8 +61,8 @@ def setUp(self):
def tearDown(self):
super(InstallTest, self).tearDown()
if self.stage is not None:
self.stage.destroy()
if self.repo.stage is not None:
self.repo.stage.destroy()
# Turn checksumming back on
spack.do_checksum = True
@ -96,7 +74,7 @@ def tearDown(self):
def test_install_and_uninstall(self):
# Get a basic concrete spec for the trivial install package.
spec = Spec(install_test_package)
spec = Spec('trivial_install_test_package')
spec.concretize()
self.assertTrue(spec.concrete)
@ -104,8 +82,7 @@ def test_install_and_uninstall(self):
pkg = spack.db.get(spec)
# Fake the URL for the package so it downloads from a file.
archive_path = join_path(self.stage.path, archive_name)
pkg.fetcher = URLFetchStrategy('file://' + archive_path)
pkg.fetcher = URLFetchStrategy(self.repo.url)
try:
pkg.do_install()

View file

@ -0,0 +1,156 @@
##############################################################################
# Copyright (c) 2013, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# This file is part of Spack.
# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
# For details, see https://scalability-llnl.github.io/spack
# Please also see the LICENSE file for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License (as published by
# the Free Software Foundation) version 2.1 dated February 1999.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
# conditions of the GNU General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program; if not, write to the Free Software Foundation,
# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
import os
from filecmp import dircmp
import spack
import spack.mirror
from spack.util.compression import decompressor_for
from spack.test.mock_packages_test import *
from spack.test.mock_repo import *
# paths in repos that shouldn't be in the mirror tarballs.
exclude = ['.hg', '.git', '.svn']
class MirrorTest(MockPackagesTest):
def setUp(self):
"""Sets up a mock package and a mock repo for each fetch strategy, to
ensure that the mirror can create archives for each of them.
"""
super(MirrorTest, self).setUp()
self.repos = {}
def set_up_package(self, name, mock_repo_class, url_attr):
"""Use this to set up a mock package to be mirrored.
Each package needs us to:
1. Set up a mock repo/archive to fetch from.
2. Point the package's version args at that repo.
"""
# Set up packages to point at mock repos.
spec = Spec(name)
spec.concretize()
# Get the package and fix its fetch args to point to a mock repo
pkg = spack.db.get(spec)
repo = mock_repo_class()
self.repos[name] = repo
# change the fetch args of the first (only) version.
assert(len(pkg.versions) == 1)
v = next(iter(pkg.versions))
pkg.versions[v][url_attr] = repo.url
def tearDown(self):
"""Destroy all the stages created by the repos in setup."""
super(MirrorTest, self).tearDown()
for name, repo in self.repos.items():
if repo.stage:
repo.stage.destroy()
self.repos.clear()
def check_mirror(self):
stage = Stage('spack-mirror-test')
mirror_root = join_path(stage.path, 'test-mirror')
try:
os.chdir(stage.path)
spack.mirror.create(
mirror_root, self.repos, no_checksum=True)
# Stage directory exists
self.assertTrue(os.path.isdir(mirror_root))
# subdirs for each package
for name in self.repos:
subdir = join_path(mirror_root, name)
self.assertTrue(os.path.isdir(subdir))
files = os.listdir(subdir)
self.assertEqual(len(files), 1)
# Decompress archive in the mirror
archive = files[0]
archive_path = join_path(subdir, archive)
decomp = decompressor_for(archive_path)
with working_dir(subdir):
decomp(archive_path)
# Find the untarred archive directory.
files = os.listdir(subdir)
self.assertEqual(len(files), 2)
self.assertTrue(archive in files)
files.remove(archive)
expanded_archive = join_path(subdir, files[0])
self.assertTrue(os.path.isdir(expanded_archive))
# Compare the original repo with the expanded archive
repo = self.repos[name]
if not 'svn' in name:
original_path = repo.path
else:
co = 'checked_out'
svn('checkout', repo.url, co)
original_path = join_path(subdir, co)
dcmp = dircmp(original_path, expanded_archive)
# make sure there are no new files in the expanded tarball
self.assertFalse(dcmp.right_only)
self.assertTrue(all(l in exclude for l in dcmp.left_only))
finally:
stage.destroy()
def test_git_mirror(self):
self.set_up_package('git-test', MockGitRepo, 'git')
self.check_mirror()
def test_svn_mirror(self):
self.set_up_package('svn-test', MockSvnRepo, 'svn')
self.check_mirror()
def test_hg_mirror(self):
self.set_up_package('hg-test', MockHgRepo, 'hg')
self.check_mirror()
def test_url_mirror(self):
self.set_up_package('trivial_install_test_package', MockArchive, 'url')
self.check_mirror()
def test_all_mirror(self):
self.set_up_package('git-test', MockGitRepo, 'git')
self.set_up_package('svn-test', MockSvnRepo, 'svn')
self.set_up_package('hg-test', MockHgRepo, 'hg')
self.set_up_package('trivial_install_test_package', MockArchive, 'url')
self.check_mirror()

View file

@ -22,7 +22,9 @@
# along with this program; if not, write to the Free Software Foundation,
# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
import os
import shutil
from contextlib import closing
from llnl.util.filesystem import *
@ -32,21 +34,6 @@
from spack.util.executable import which
class MockRepo(object):
def __init__(self, stage_name, repo_name):
"""This creates a stage and a repo directory within the stage."""
# Stage where this repo has been created
self.stage = Stage(stage_name)
# Full path to the repo within the stage.
self.path = join_path(self.stage.path, 'mock-git-repo')
mkdirp(self.path)
# Name for rev0 & rev1 files in the repo to be
self.r0_file = 'r0_file'
self.r1_file = 'r1_file'
#
# VCS Systems used by mock repo code.
#
@ -54,9 +41,64 @@ def __init__(self, stage_name, repo_name):
svn = which('svn', required=True)
svnadmin = which('svnadmin', required=True)
hg = which('hg', required=True)
tar = which('tar', required=True)
class MockGitRepo(MockRepo):
class MockRepo(object):
def __init__(self, stage_name, repo_name):
"""This creates a stage where some archive/repo files can be staged
for testing spack's fetch strategies."""
# Stage where this repo has been created
self.stage = Stage(stage_name)
# Full path to the repo within the stage.
self.path = join_path(self.stage.path, repo_name)
mkdirp(self.path)
class MockArchive(MockRepo):
"""Creates a very simple archive directory with a configure script and a
makefile that installs to a prefix. Tars it up into an archive."""
def __init__(self):
repo_name = 'mock-archive-repo'
super(MockArchive, self).__init__('mock-archive-stage', repo_name)
with working_dir(self.path):
configure = join_path(self.path, 'configure')
with closing(open(configure, 'w')) as cfg_file:
cfg_file.write(
"#!/bin/sh\n"
"prefix=$(echo $1 | sed 's/--prefix=//')\n"
"cat > Makefile <<EOF\n"
"all:\n"
"\techo Building...\n\n"
"install:\n"
"\tmkdir -p $prefix\n"
"\ttouch $prefix/dummy_file\n"
"EOF\n")
os.chmod(configure, 0755)
with working_dir(self.stage.path):
archive_name = "%s.tar.gz" % repo_name
tar('-czf', archive_name, repo_name)
self.archive_path = join_path(self.stage.path, archive_name)
self.url = 'file://' + self.archive_path
class MockVCSRepo(MockRepo):
def __init__(self, stage_name, repo_name):
"""This creates a stage and a repo directory within the stage."""
super(MockVCSRepo, self).__init__(stage_name, repo_name)
# Name for rev0 & rev1 files in the repo to be
self.r0_file = 'r0_file'
self.r1_file = 'r1_file'
class MockGitRepo(MockVCSRepo):
def __init__(self):
super(MockGitRepo, self).__init__('mock-git-stage', 'mock-git-repo')
@ -97,17 +139,20 @@ def __init__(self):
self.r1 = self.rev_hash(self.branch)
self.r1_file = self.branch_file
self.url = self.path
def rev_hash(self, rev):
return git('rev-parse', rev, return_output=True).strip()
class MockSvnRepo(MockRepo):
class MockSvnRepo(MockVCSRepo):
def __init__(self):
super(MockSvnRepo, self).__init__('mock-svn-stage', 'mock-svn-repo')
self.url = 'file://' + self.path
with working_dir(self.stage.path):
svnadmin('create', self.path)
self.url = 'file://' + self.path
tmp_path = join_path(self.stage.path, 'tmp-path')
mkdirp(tmp_path)
@ -129,9 +174,10 @@ def __init__(self):
self.r1 = '2'
class MockHgRepo(MockRepo):
class MockHgRepo(MockVCSRepo):
def __init__(self):
super(MockHgRepo, self).__init__('mock-hg-stage', 'mock-hg-repo')
self.url = 'file://' + self.path
with working_dir(self.path):
hg('init')