hlrs-software-stack/spack
7
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

make create and checksum consistent.

Browse source 
- create now searches and prompts for checksums.
- makes package creation easier
This commit is contained in:
Todd Gamblin 2013-12-24 00:57:56 -08:00
parent 2f1eae8c0d
commit 354c8a281b
6 changed files with 182 additions and 96 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

77
lib/spack/spack/cmd/checksum.py
View file

@ -12,9 +12,7 @@
from spack.colify import colify
from spack.version import *
default_number_to_fetch = 10
description ="Checksum available versions of a package, print out checksums for addition to a package file."
description ="Checksum available versions of a package to update a package file."
def setup_parser(subparser):
subparser.add_argument(
@ -23,6 +21,32 @@ def setup_parser(subparser):
'versions', nargs=argparse.REMAINDER, help='Versions to generate checksums for')
def get_checksums(versions, urls, **kwargs):
# Allow commands like create() to do some analysis on the first
# archive after it is downloaded.
first_stage_function = kwargs.get('first_stage_function', None)
tty.msg("Downloading...")
hashes = []
for i, (url, version) in enumerate(zip(urls, versions)):
stage = Stage(url)
try:
stage.fetch()
if i == 0 and first_stage_function:
first_stage_function(stage)
hashes.append(
spack.util.crypto.checksum(hashlib.md5, stage.archive_file))
except FailedDownloadError, e:
tty.msg("Failed to fetch %s" % url)
continue
finally:
stage.destroy()
return zip(versions, hashes)
def checksum(parser, args):
# get the package we're going to generate checksums for
pkg = packages.get(args.package)
@ -42,47 +66,24 @@ def checksum(parser, args):
versions = list(reversed(versions))
urls = [pkg.url_for_version(v) for v in versions]
version_listings = ["%-10s%s" % (v,u) for v, u in zip(versions, urls)]
tty.msg("Found %s versions to checksum." % len(urls),
*version_listings)
tty.msg("Found %s versions of %s." % (len(urls), pkg.name),
*["%-10s%s" % (v,u) for v, u in zip(versions, urls)])
print
while True:
ans = raw_input("How many would you like to checksum? (default 10, 0 to abort) ")
try:
if not ans:
to_download = default_number_to_fetch
else:
to_download = int(ans)
break
except ValueError:
tty.msg("Please enter a valid number.")
pass
archives_to_fetch = tty.get_number(
"How many would you like to checksum?", default=5, abort='q')
if not to_download:
if not archives_to_fetch:
tty.msg("Aborted.")
return
else:
urls = urls[:to_download]
tty.msg("Downloading...")
hashes = []
for url, version in zip(urls, versions):
stage = Stage(url)
try:
stage.fetch()
hashes.append(spack.util.crypto.checksum(
hashlib.md5, stage.archive_file))
except FailedDownloadError, e:
tty.msg("Failed to fetch %s" % url)
continue
version_hashes = get_checksums(
versions[:archives_to_fetch], urls[:archives_to_fetch])
finally:
stage.destroy()
if not version_hashes:
tty.die("Could not fetch any available versions for %s." % pkg.name)
dict_string = [" '%s' : '%s'," % (v, h) for v, h in version_hashes]
dict_string = ['{'] + dict_string + ["}"]
dict_string = ["{"]
for i, (v, h) in enumerate(zip(versions, hashes)):
comma = "" if i == len(hashes) - 1 else ","
dict_string.append(" '%s' : '%s'%s" % (str(v), str(h), comma))
dict_string.append("}")
tty.msg("Checksummed new versions of %s:" % pkg.name, *dict_string)

116
lib/spack/spack/cmd/create.py
View file

@ -2,32 +2,37 @@
import os
import hashlib
import re
from contextlib import closing
import spack
import spack.package
import spack.packages as packages
import spack.tty as tty
import spack.url
import spack.util.crypto as crypto
import spack.cmd.checksum
from spack.util.executable import which
from spack.stage import Stage
from contextlib import closing
description = "Create a new package file from an archive URL"
package_template = string.Template("""\
# FIXME:
# This is a template package file for Spack. We've conveniently
# put giant "FIXME" labels next to all the things you'll probably
# want to change.
# put "FIXME" labels next to all the things you'll want to change.
#
# Once you've edited all the FIXME's, delete this whole message,
# save this file, and test out your package like this:
#
# spack install ${name}
#
# You can always get back here with 'spack edit ${name}'. See
# the spack documentation for more information on building
# You can always get back here to change things with:
#
# spack edit ${name}
#
# See the spack documentation for more information on building
# packages.
#
from spack import *
@ -52,28 +57,31 @@ def install(self, prefix):
def setup_parser(subparser):
subparser.add_argument('url', nargs='?', help="url of package archive")
subparser.add_argument('-f', '--force', action='store_true', dest='force',
help="Remove existing package file.")
subparser.add_argument(
'-f', '--force', action='store_true', dest='force',
help="Overwrite any existing package file with the same name.")
def guess_configure(archive_file):
"""Try to guess the type of build system used by the project, and return
an appropriate configure line.
"""
tar = which('tar')
output = tar("--exclude=*/*/*", "-tf", archive_file, return_output=True)
class ConfigureGuesser(object):
def __call__(self, stage):
"""Try to guess the type of build system used by the project, and return
an appropriate configure line.
"""
tar = which('tar')
output = tar(
"--exclude=*/*/*", "-tf", stage.archive_file, return_output=True)
autotools = 'configure("--prefix=%s" % prefix)'
cmake = 'cmake(".", *std_cmake_args)'
lines = output.split('\n')
autotools = 'configure("--prefix=%s" % prefix)'
cmake = 'cmake(".", *std_cmake_args)'
lines = output.split('\n')
if any(re.search(r'/configure$', l) for l in lines):
return autotools
elif any(re.search(r'/CMakeLists.txt$', l) for l in lines):
return cmake
else:
# Both, with cmake commented out
return '%s\n # %s' % (autotools, cmake)
if any(re.search(r'/configure$', l) for l in lines):
self.configure = autotools
elif any(re.search(r'/CMakeLists.txt$', l) for l in lines):
self.configure = cmake
else:
# Both, with cmake commented out
self.configure = '%s\n # %s' % (autotools, cmake)
def create(parser, args):
@ -82,43 +90,67 @@ def create(parser, args):
# Try to deduce name and version of the new package from the URL
name, version = spack.url.parse_name_and_version(url)
if not name:
print "Couldn't guess a name for this package."
tty.msg("Couldn't guess a name for this package.")
while not name:
new_name = raw_input("Name: ")
if packages.valid_name(name):
name = new_name
else:
print "Package names must contain letters, numbers, and '_' or '-'"
print "Package name can only contain A-Z, a-z, 0-9, '_' and '-'"
if not version:
tty.die("Couldn't guess a version string from %s." % url)
path = packages.filename_for_package_name(name)
if not args.force and os.path.exists(path):
tty.die("%s already exists." % path)
tty.msg("Creating template for package %s" % name)
# make a stage and fetch the archive.
try:
stage = Stage(url)
archive_file = stage.fetch()
except spack.FailedDownloadException, e:
tty.die(e.message)
pkg_path = packages.filename_for_package_name(name)
if os.path.exists(pkg_path) and not args.force:
tty.die("%s already exists." % pkg_path)
md5 = crypto.checksum(hashlib.md5, archive_file)
versions = '{ "%s" : "%s" }' % (version, md5)
class_name = packages.class_name_for_package_name(name)
configure = guess_configure(archive_file)
versions = list(reversed(spack.package.find_versions_of_archive(url)))
archives_to_fetch = 1
if not versions:
# If the fetch failed for some reason, revert to what the user provided
versions = [version]
urls = [url]
else:
urls = [spack.url.substitute_version(url, v) for v in versions]
if len(urls) > 1:
tty.msg("Found %s versions of %s to checksum." % (len(urls), name),
*["%-10s%s" % (v,u) for v, u in zip(versions, urls)])
print
archives_to_fetch = tty.get_number(
"Include how many checksums in the package file?",
default=5, abort='q')
if not archives_to_fetch:
tty.msg("Aborted.")
return
guesser = ConfigureGuesser()
version_hashes = spack.cmd.checksum.get_checksums(
versions[:archives_to_fetch], urls[:archives_to_fetch],
first_stage_function=guesser)
if not version_hashes:
tty.die("Could not fetch any tarballs for %s." % name)
sep = '\n '
versions_string = '{ ' + sep.join(
"'%s' : '%s'," % (v, h) for v, h in version_hashes) + ' }'
# Write out a template for the file
tty.msg("Editing %s." % path)
with closing(open(path, "w")) as pkg_file:
with closing(open(pkg_path, "w")) as pkg_file:
pkg_file.write(
package_template.substitute(
name=name,
configure=configure,
configure=guesser.configure,
class_name=class_name,
url=url,
versions=versions))
versions=versions_string))
# If everything checks out, go ahead and edit.
spack.editor(path)
spack.editor(pkg_path)
tty.msg("Created package %s." % pkg_path)

48
lib/spack/spack/package.py
View file

@ -338,7 +338,7 @@ def __init__(self, spec):
# Set a default list URL (place to find available versions)
if not hasattr(self, 'list_url'):
self.list_url = os.path.dirname(self.url)
self.list_url = None
if not hasattr(self, 'list_depth'):
self.list_depth = 1
@ -733,21 +733,12 @@ def do_clean_dist(self):
def fetch_available_versions(self):
# If not, then try to fetch using list_url
if not self._available_versions:
self._available_versions = VersionList()
url_regex = os.path.basename(url.wildcard_version(self.url))
wildcard = self.default_version.wildcard()
try:
page_map = get_pages(self.list_url, depth=self.list_depth)
for site, page in page_map.iteritems():
strings = re.findall(url_regex, page)
for s in strings:
match = re.search(wildcard, s)
if match:
v = match.group(0)
self._available_versions.add(Version(v))
self._available_versions = find_versions_of_archive(
self.url,
list_url=self.list_url,
list_depth=self.list_depth,
wildcard=self.default_version.wildcard())
if not self._available_versions:
tty.warn("Found no versions for %s" % self.name,
@ -774,6 +765,33 @@ def available_versions(self):
return vlist
def find_versions_of_archive(archive_url, **kwargs):
list_url = kwargs.get('list_url', None)
list_depth = kwargs.get('list_depth', 1)
wildcard = kwargs.get('wildcard', None)
if not list_url:
list_url = os.path.dirname(archive_url)
if not wildcard:
wildcard = url.parse_version(archive_url).wildcard()
versions = VersionList()
url_regex = os.path.basename(url.wildcard_version(archive_url))
page_map = get_pages(list_url, depth=list_depth)
for site, page in page_map.iteritems():
strings = re.findall(url_regex, page)
for s in strings:
match = re.search(wildcard, s)
if match:
v = match.group(0)
versions.add(Version(v))
return versions
class MakeExecutable(Executable):
"""Special Executable for make so the user can specify parallel or
not on a per-invocation basis. Using 'parallel' as a kwarg will

5
lib/spack/spack/stage.py
View file

@ -267,6 +267,11 @@ def destroy(self):
"""Remove this stage directory."""
remove_linked_tree(self.path)
# Make sure we don't end up in a removed directory
try:
os.getcwd()
except OSError:
os.chdir(os.path.dirname(self.path))
def ensure_access(file=spack.stage_path):

30
lib/spack/spack/tty.py
View file

@ -51,3 +51,33 @@ def pkg(message):
else:
cwrite('@*g{[+]} ')
print message
def get_number(prompt, **kwargs):
default = kwargs.get('default', None)
abort = kwargs.get('abort', None)
if default is not None and abort is not None:
prompt += ' (default is %s, %s to abort) ' % (default, abort)
elif default is not None:
prompt += ' (default is %s) ' % default
elif abort is not None:
prompt += ' (%s to abort) ' % abort
number = None
while number is None:
ans = raw_input(prompt)
if ans == str(abort):
return None
if ans:
try:
number = int(ans)
if number < 1:
msg("Please enter a valid number.")
number = None
except ValueError:
msg("Please enter a valid number.")
elif default is not None:
number = default
return number

2
lib/spack/spack/url.py
View file

@ -168,7 +168,7 @@ def substitute_version(path, new_version):
the new version for it.
"""
ver, start, end = parse_version_string_with_indices(path)
return path[:start] + new_version + path[end:]
return path[:start] + str(new_version) + path[end:]
def wildcard_version(path):