From 354c8a281bb40338403add7eb3ec9245662365d8 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 24 Dec 2013 00:57:56 -0800 Subject: [PATCH] make create and checksum consistent. - create now searches and prompts for checksums. - makes package creation easier --- lib/spack/spack/cmd/checksum.py | 77 ++++++++++----------- lib/spack/spack/cmd/create.py | 116 ++++++++++++++++++++------------ lib/spack/spack/package.py | 48 ++++++++----- lib/spack/spack/stage.py | 5 ++ lib/spack/spack/tty.py | 30 +++++++++ lib/spack/spack/url.py | 2 +- 6 files changed, 182 insertions(+), 96 deletions(-) diff --git a/lib/spack/spack/cmd/checksum.py b/lib/spack/spack/cmd/checksum.py index 70670390a8..81d6fbf2b9 100644 --- a/lib/spack/spack/cmd/checksum.py +++ b/lib/spack/spack/cmd/checksum.py @@ -12,9 +12,7 @@ from spack.colify import colify from spack.version import * -default_number_to_fetch = 10 - -description ="Checksum available versions of a package, print out checksums for addition to a package file." +description ="Checksum available versions of a package to update a package file." def setup_parser(subparser): subparser.add_argument( @@ -23,6 +21,32 @@ def setup_parser(subparser): 'versions', nargs=argparse.REMAINDER, help='Versions to generate checksums for') +def get_checksums(versions, urls, **kwargs): + # Allow commands like create() to do some analysis on the first + # archive after it is downloaded. + first_stage_function = kwargs.get('first_stage_function', None) + + tty.msg("Downloading...") + hashes = [] + for i, (url, version) in enumerate(zip(urls, versions)): + stage = Stage(url) + try: + stage.fetch() + if i == 0 and first_stage_function: + first_stage_function(stage) + + hashes.append( + spack.util.crypto.checksum(hashlib.md5, stage.archive_file)) + except FailedDownloadError, e: + tty.msg("Failed to fetch %s" % url) + continue + + finally: + stage.destroy() + + return zip(versions, hashes) + + def checksum(parser, args): # get the package we're going to generate checksums for pkg = packages.get(args.package) @@ -42,47 +66,24 @@ def checksum(parser, args): versions = list(reversed(versions)) urls = [pkg.url_for_version(v) for v in versions] - version_listings = ["%-10s%s" % (v,u) for v, u in zip(versions, urls)] - tty.msg("Found %s versions to checksum." % len(urls), - *version_listings) + tty.msg("Found %s versions of %s." % (len(urls), pkg.name), + *["%-10s%s" % (v,u) for v, u in zip(versions, urls)]) print - while True: - ans = raw_input("How many would you like to checksum? (default 10, 0 to abort) ") - try: - if not ans: - to_download = default_number_to_fetch - else: - to_download = int(ans) - break - except ValueError: - tty.msg("Please enter a valid number.") - pass + archives_to_fetch = tty.get_number( + "How many would you like to checksum?", default=5, abort='q') - if not to_download: + if not archives_to_fetch: tty.msg("Aborted.") return - else: - urls = urls[:to_download] - tty.msg("Downloading...") - hashes = [] - for url, version in zip(urls, versions): - stage = Stage(url) - try: - stage.fetch() - hashes.append(spack.util.crypto.checksum( - hashlib.md5, stage.archive_file)) - except FailedDownloadError, e: - tty.msg("Failed to fetch %s" % url) - continue + version_hashes = get_checksums( + versions[:archives_to_fetch], urls[:archives_to_fetch]) - finally: - stage.destroy() + if not version_hashes: + tty.die("Could not fetch any available versions for %s." % pkg.name) + + dict_string = [" '%s' : '%s'," % (v, h) for v, h in version_hashes] + dict_string = ['{'] + dict_string + ["}"] - dict_string = ["{"] - for i, (v, h) in enumerate(zip(versions, hashes)): - comma = "" if i == len(hashes) - 1 else "," - dict_string.append(" '%s' : '%s'%s" % (str(v), str(h), comma)) - dict_string.append("}") tty.msg("Checksummed new versions of %s:" % pkg.name, *dict_string) diff --git a/lib/spack/spack/cmd/create.py b/lib/spack/spack/cmd/create.py index af75f40d73..0001c8556e 100644 --- a/lib/spack/spack/cmd/create.py +++ b/lib/spack/spack/cmd/create.py @@ -2,32 +2,37 @@ import os import hashlib import re +from contextlib import closing import spack +import spack.package import spack.packages as packages import spack.tty as tty import spack.url import spack.util.crypto as crypto +import spack.cmd.checksum from spack.util.executable import which from spack.stage import Stage -from contextlib import closing + description = "Create a new package file from an archive URL" package_template = string.Template("""\ # FIXME: # This is a template package file for Spack. We've conveniently -# put giant "FIXME" labels next to all the things you'll probably -# want to change. +# put "FIXME" labels next to all the things you'll want to change. # # Once you've edited all the FIXME's, delete this whole message, # save this file, and test out your package like this: # # spack install ${name} # -# You can always get back here with 'spack edit ${name}'. See -# the spack documentation for more information on building +# You can always get back here to change things with: +# +# spack edit ${name} +# +# See the spack documentation for more information on building # packages. # from spack import * @@ -52,28 +57,31 @@ def install(self, prefix): def setup_parser(subparser): subparser.add_argument('url', nargs='?', help="url of package archive") - subparser.add_argument('-f', '--force', action='store_true', dest='force', - help="Remove existing package file.") + subparser.add_argument( + '-f', '--force', action='store_true', dest='force', + help="Overwrite any existing package file with the same name.") -def guess_configure(archive_file): - """Try to guess the type of build system used by the project, and return - an appropriate configure line. - """ - tar = which('tar') - output = tar("--exclude=*/*/*", "-tf", archive_file, return_output=True) +class ConfigureGuesser(object): + def __call__(self, stage): + """Try to guess the type of build system used by the project, and return + an appropriate configure line. + """ + tar = which('tar') + output = tar( + "--exclude=*/*/*", "-tf", stage.archive_file, return_output=True) - autotools = 'configure("--prefix=%s" % prefix)' - cmake = 'cmake(".", *std_cmake_args)' - lines = output.split('\n') + autotools = 'configure("--prefix=%s" % prefix)' + cmake = 'cmake(".", *std_cmake_args)' + lines = output.split('\n') - if any(re.search(r'/configure$', l) for l in lines): - return autotools - elif any(re.search(r'/CMakeLists.txt$', l) for l in lines): - return cmake - else: - # Both, with cmake commented out - return '%s\n # %s' % (autotools, cmake) + if any(re.search(r'/configure$', l) for l in lines): + self.configure = autotools + elif any(re.search(r'/CMakeLists.txt$', l) for l in lines): + self.configure = cmake + else: + # Both, with cmake commented out + self.configure = '%s\n # %s' % (autotools, cmake) def create(parser, args): @@ -82,43 +90,67 @@ def create(parser, args): # Try to deduce name and version of the new package from the URL name, version = spack.url.parse_name_and_version(url) if not name: - print "Couldn't guess a name for this package." + tty.msg("Couldn't guess a name for this package.") while not name: new_name = raw_input("Name: ") if packages.valid_name(name): name = new_name else: - print "Package names must contain letters, numbers, and '_' or '-'" + print "Package name can only contain A-Z, a-z, 0-9, '_' and '-'" if not version: tty.die("Couldn't guess a version string from %s." % url) - path = packages.filename_for_package_name(name) - if not args.force and os.path.exists(path): - tty.die("%s already exists." % path) + tty.msg("Creating template for package %s" % name) - # make a stage and fetch the archive. - try: - stage = Stage(url) - archive_file = stage.fetch() - except spack.FailedDownloadException, e: - tty.die(e.message) + pkg_path = packages.filename_for_package_name(name) + if os.path.exists(pkg_path) and not args.force: + tty.die("%s already exists." % pkg_path) - md5 = crypto.checksum(hashlib.md5, archive_file) - versions = '{ "%s" : "%s" }' % (version, md5) class_name = packages.class_name_for_package_name(name) - configure = guess_configure(archive_file) + versions = list(reversed(spack.package.find_versions_of_archive(url))) + + archives_to_fetch = 1 + if not versions: + # If the fetch failed for some reason, revert to what the user provided + versions = [version] + urls = [url] + else: + urls = [spack.url.substitute_version(url, v) for v in versions] + if len(urls) > 1: + tty.msg("Found %s versions of %s to checksum." % (len(urls), name), + *["%-10s%s" % (v,u) for v, u in zip(versions, urls)]) + print + archives_to_fetch = tty.get_number( + "Include how many checksums in the package file?", + default=5, abort='q') + + if not archives_to_fetch: + tty.msg("Aborted.") + return + + guesser = ConfigureGuesser() + version_hashes = spack.cmd.checksum.get_checksums( + versions[:archives_to_fetch], urls[:archives_to_fetch], + first_stage_function=guesser) + + if not version_hashes: + tty.die("Could not fetch any tarballs for %s." % name) + + sep = '\n ' + versions_string = '{ ' + sep.join( + "'%s' : '%s'," % (v, h) for v, h in version_hashes) + ' }' # Write out a template for the file - tty.msg("Editing %s." % path) - with closing(open(path, "w")) as pkg_file: + with closing(open(pkg_path, "w")) as pkg_file: pkg_file.write( package_template.substitute( name=name, - configure=configure, + configure=guesser.configure, class_name=class_name, url=url, - versions=versions)) + versions=versions_string)) # If everything checks out, go ahead and edit. - spack.editor(path) + spack.editor(pkg_path) + tty.msg("Created package %s." % pkg_path) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index dee102e171..cb779c4047 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -338,7 +338,7 @@ def __init__(self, spec): # Set a default list URL (place to find available versions) if not hasattr(self, 'list_url'): - self.list_url = os.path.dirname(self.url) + self.list_url = None if not hasattr(self, 'list_depth'): self.list_depth = 1 @@ -733,21 +733,12 @@ def do_clean_dist(self): def fetch_available_versions(self): # If not, then try to fetch using list_url if not self._available_versions: - self._available_versions = VersionList() - url_regex = os.path.basename(url.wildcard_version(self.url)) - wildcard = self.default_version.wildcard() - try: - page_map = get_pages(self.list_url, depth=self.list_depth) - - for site, page in page_map.iteritems(): - strings = re.findall(url_regex, page) - - for s in strings: - match = re.search(wildcard, s) - if match: - v = match.group(0) - self._available_versions.add(Version(v)) + self._available_versions = find_versions_of_archive( + self.url, + list_url=self.list_url, + list_depth=self.list_depth, + wildcard=self.default_version.wildcard()) if not self._available_versions: tty.warn("Found no versions for %s" % self.name, @@ -774,6 +765,33 @@ def available_versions(self): return vlist +def find_versions_of_archive(archive_url, **kwargs): + list_url = kwargs.get('list_url', None) + list_depth = kwargs.get('list_depth', 1) + wildcard = kwargs.get('wildcard', None) + + if not list_url: + list_url = os.path.dirname(archive_url) + if not wildcard: + wildcard = url.parse_version(archive_url).wildcard() + + versions = VersionList() + url_regex = os.path.basename(url.wildcard_version(archive_url)) + + page_map = get_pages(list_url, depth=list_depth) + + for site, page in page_map.iteritems(): + strings = re.findall(url_regex, page) + + for s in strings: + match = re.search(wildcard, s) + if match: + v = match.group(0) + versions.add(Version(v)) + + return versions + + class MakeExecutable(Executable): """Special Executable for make so the user can specify parallel or not on a per-invocation basis. Using 'parallel' as a kwarg will diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index 7f01fca12b..5d7f0b72f9 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -267,6 +267,11 @@ def destroy(self): """Remove this stage directory.""" remove_linked_tree(self.path) + # Make sure we don't end up in a removed directory + try: + os.getcwd() + except OSError: + os.chdir(os.path.dirname(self.path)) def ensure_access(file=spack.stage_path): diff --git a/lib/spack/spack/tty.py b/lib/spack/spack/tty.py index 4584b3de3e..f73ebbc4cb 100644 --- a/lib/spack/spack/tty.py +++ b/lib/spack/spack/tty.py @@ -51,3 +51,33 @@ def pkg(message): else: cwrite('@*g{[+]} ') print message + + +def get_number(prompt, **kwargs): + default = kwargs.get('default', None) + abort = kwargs.get('abort', None) + + if default is not None and abort is not None: + prompt += ' (default is %s, %s to abort) ' % (default, abort) + elif default is not None: + prompt += ' (default is %s) ' % default + elif abort is not None: + prompt += ' (%s to abort) ' % abort + + number = None + while number is None: + ans = raw_input(prompt) + if ans == str(abort): + return None + + if ans: + try: + number = int(ans) + if number < 1: + msg("Please enter a valid number.") + number = None + except ValueError: + msg("Please enter a valid number.") + elif default is not None: + number = default + return number diff --git a/lib/spack/spack/url.py b/lib/spack/spack/url.py index 02872527c4..c70c1107a9 100644 --- a/lib/spack/spack/url.py +++ b/lib/spack/spack/url.py @@ -168,7 +168,7 @@ def substitute_version(path, new_version): the new version for it. """ ver, start, end = parse_version_string_with_indices(path) - return path[:start] + new_version + path[end:] + return path[:start] + str(new_version) + path[end:] def wildcard_version(path):