From fb172bc702726959fa0269e24700d9e95489e32b Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Wed, 20 Feb 2013 15:22:00 -0800 Subject: [PATCH] Added libunwind and fixed link issues in cc. --- lib/spack/env/cc | 100 +++++++++++++++----------- lib/spack/spack/Package.py | 31 ++++---- lib/spack/spack/cmd/clean.py | 12 ++++ lib/spack/spack/cmd/create.py | 6 +- lib/spack/spack/compilation.py | 51 +++++++------ lib/spack/spack/globals.py | 11 +++ lib/spack/spack/packages/libdwarf.py | 2 +- lib/spack/spack/packages/libunwind.py | 11 +++ lib/spack/spack/stage.py | 31 +++++--- 9 files changed, 170 insertions(+), 85 deletions(-) create mode 100644 lib/spack/spack/packages/libunwind.py diff --git a/lib/spack/env/cc b/lib/spack/env/cc index 8f7a7a35fd..9fa1a18573 100755 --- a/lib/spack/env/cc +++ b/lib/spack/env/cc @@ -1,12 +1,10 @@ #!/usr/bin/env python import sys import os +import re import subprocess import argparse - -# reimplement some tty stuff to minimize imports -blue, green, yellow, reset = [ - '\033[1;39m', '\033[1;92m', '\033[4;33m', '\033[0m'] +from contextlib import closing # Import spack parameters through the build environment. spack_lib = os.environ.get("SPACK_LIB") @@ -19,14 +17,27 @@ sys.path.append(spack_lib) from spack.compilation import * import spack.tty as tty -spack_prefix = get_env_var("SPACK_PREFIX") -spack_debug = get_env_flag("SPACK_DEBUG") -spack_deps = get_path("SPACK_DEPENDENCIES") -spack_env_path = get_path("SPACK_ENV_PATH") +spack_prefix = get_env_var("SPACK_PREFIX") +spack_build_root = get_env_var("SPACK_BUILD_ROOT") +spack_debug = get_env_flag("SPACK_DEBUG") +spack_deps = get_path("SPACK_DEPENDENCIES") +spack_env_path = get_path("SPACK_ENV_PATH") # Figure out what type of operation we're doing command = os.path.basename(sys.argv[0]) +cpp, cc, ccld, ld = range(4) +if command == 'cpp': + mode = cpp +elif command == 'ld': + mode = ld +elif '-E' in sys.argv: + mode = cpp +elif '-c' in sys.argv: + mode = cc +else: + mode = ccld +# Parse out the includes, libs, etc. so we can adjust them if need be. parser = argparse.ArgumentParser(add_help=False) parser.add_argument("-I", action='append', default=[], dest='include_path') parser.add_argument("-L", action='append', default=[], dest='lib_path') @@ -35,9 +46,38 @@ parser.add_argument("-l", action='append', default=[], dest='libs') options, other_args = parser.parse_known_args() rpaths, other_args = parse_rpaths(other_args) -if rpaths: - print "{}Warning{}: Spack stripping non-spack rpaths: ".format(yellow, reset) - for rp in rpaths: print " %s" % rp +# Add dependencies' include and lib paths to our compiler flags. +def append_if_dir(path_list, *dirs): + full_path = os.path.join(*dirs) + if os.path.isdir(full_path): + path_list.append(full_path) + +for dep_dir in spack_deps: + append_if_dir(options.include_path, dep_dir, "include") + append_if_dir(options.lib_path, dep_dir, "lib") + append_if_dir(options.lib_path, dep_dir, "lib64") + +# Add our modified arguments to it. +arguments = ['-I%s' % path for path in options.include_path] +arguments += other_args +arguments += ['-L%s' % path for path in options.lib_path] +arguments += ['-l%s' % path for path in options.libs] + +# Add rpaths to install dir and its dependencies. We add both lib and lib64 +# here because we don't know which will be created. +rpaths.extend(options.lib_path) +rpaths.append('%s/lib' % spack_prefix) +rpaths.append('%s/lib64' % spack_prefix) +if mode == ccld: + arguments += ['-Wl,-rpath,%s' % p for p in rpaths] +elif mode == ld: + pairs = [('-rpath', '%s' % p) for p in rpaths] + arguments += [item for sublist in pairs for item in sublist] + +# Unset some pesky environment variables +for var in ["LD_LIBRARY_PATH", "LD_RUN_PATH", "DYLD_LIBRARY_PATH"]: + if var in os.environ: + os.environ.pop(var) # Ensure that the delegated command doesn't just call this script again. clean_path = get_path("PATH") @@ -46,35 +86,15 @@ for item in ['.'] + spack_env_path: clean_path.remove(item) os.environ["PATH"] = ":".join(clean_path) -# Add dependence's paths to our compiler flags. -def append_if_dir(path_list, prefix, *dirs): - full_path = os.path.join(prefix, *dirs) - if os.path.isdir(full_path): - path_list.append(full_path) - -for prefix in spack_deps: - append_if_dir(options.include_path, prefix, "include") - append_if_dir(options.lib_path, prefix, "lib") - append_if_dir(options.lib_path, prefix, "lib64") - -# Add our modified arguments to it. -arguments = ['-I%s' % path for path in options.include_path] -arguments += other_args -arguments += ['-L%s' % path for path in options.lib_path] -arguments += ['-l%s' % path for path in options.libs] - -spack_rpaths = [spack_prefix] + spack_deps -arguments += ['-Wl,-rpath,%s/lib64' % path for path in spack_rpaths] -arguments += ['-Wl,-rpath,%s/lib' % path for path in spack_rpaths] - -# Unset some pesky environment variables -for var in ["LD_LIBRARY_PATH", "LD_RUN_PATH", "DYLD_LIBRARY_PATH"]: - if var in os.environ: - os.environ.pop(var) - +full_command = [command] + arguments if spack_debug: - sys.stderr.write("{}==>{} {} {}\n".format( - green, reset, command, " ".join(arguments))) + input_log = os.path.join(spack_build_root, 'spack_cc_in.log') + output_log = os.path.join(spack_build_root, 'spack_cc_out.log') + with closing(open(input_log, 'a')) as log: + args = [os.path.basename(sys.argv[0])] + sys.argv[1:] + log.write("%s\n" % " ".join(arg.replace(' ', r'\ ') for arg in args)) + with closing(open(output_log, 'a')) as log: + log.write("%s\n" % " ".join(full_command)) -rcode = subprocess.call([command] + arguments) +rcode = subprocess.call(full_command) sys.exit(rcode) diff --git a/lib/spack/spack/Package.py b/lib/spack/spack/Package.py index 0aeb3cbd52..7d1ef169ff 100644 --- a/lib/spack/spack/Package.py +++ b/lib/spack/spack/Package.py @@ -65,7 +65,7 @@ def __init__(self, name, parallel): def __call__(self, *args, **kwargs): parallel = kwargs.get('parallel', self.parallel) - env_parallel = not env_flag("SPACK_NO_PARALLEL_MAKE") + env_parallel = not env_flag(SPACK_NO_PARALLEL_MAKE) if parallel and env_parallel: args += ("-j%d" % multiprocessing.cpu_count(),) @@ -88,13 +88,17 @@ def __init__(self, arch=arch.sys_type()): # Name of package is the name of its module (the file that contains it) self.name = inspect.getmodulename(self.module.__file__) + # Don't allow the default homepage. + if re.search(r'example.com', self.homepage): + tty.die("Bad homepage in %s: %s" % (self.name, self.homepage)) + # Make sure URL is an allowed type validate.url(self.url) # Set up version attr.setdefault(self, 'version', version.parse_version(self.url)) if not self.version: - tty.die("Couldn't extract version from '%s'. " + + tty.die("Couldn't extract version from %s. " + "You must specify it explicitly for this URL." % self.url) # This adds a bunch of convenient commands to the package's module scope. @@ -109,6 +113,9 @@ def __init__(self, arch=arch.sys_type()): # Whether to remove intermediate build/install when things go wrong. self.dirty = False + # stage used to build this package. + self.stage = Stage(self.stage_name, self.url) + def make_make(self): """Create a make command set up with the proper default arguments.""" @@ -201,11 +208,6 @@ def all_dependents(self): return tuple(all_deps) - @property - def stage(self): - return Stage(self.stage_name, self.url) - - @property def stage_name(self): return "%s-%s" % (self.name, self.version) @@ -270,7 +272,7 @@ def do_stage(self): archive_dir = stage.expanded_archive_path if not archive_dir: - tty.msg("Staging archive: '%s'" % stage.archive_file) + tty.msg("Staging archive: %s" % stage.archive_file) stage.expand_archive() else: tty.msg("Already staged %s" % self.name) @@ -316,7 +318,7 @@ def setup_install_environment(self): # Add spack environment at front of path and pass the # lib location along so the compiler script can find spack - os.environ["SPACK_LIB"] = lib_path + os.environ[SPACK_LIB] = lib_path # Fix for case-insensitive file systems. Conflicting links are # in directories called "case*" within the env directory. @@ -326,14 +328,17 @@ def setup_install_environment(self): if file.startswith("case") and os.path.isdir(path): env_paths.append(path) path_put_first("PATH", env_paths) - path_set("SPACK_ENV_PATH", env_paths) + path_set(SPACK_ENV_PATH, env_paths) # Pass along prefixes of dependencies here - path_set("SPACK_DEPENDENCIES", + path_set(SPACK_DEPENDENCIES, [dep.package.prefix for dep in self.dependencies]) # Install location - os.environ["SPACK_PREFIX"] = self.prefix + os.environ[SPACK_PREFIX] = self.prefix + + # Build root for logging. + os.environ[SPACK_BUILD_ROOT] = self.stage.expanded_archive_path def do_install_dependencies(self): @@ -379,7 +384,7 @@ def do_clean(self): def clean(self): """By default just runs make clean. Override if this isn't good.""" try: - make = MakeExecutable('make') + make = MakeExecutable('make', self.parallel) make('clean') tty.msg("Successfully cleaned %s" % self.name) except subprocess.CalledProcessError, e: diff --git a/lib/spack/spack/cmd/clean.py b/lib/spack/spack/cmd/clean.py index f3243f57cc..ac3147fe12 100644 --- a/lib/spack/spack/cmd/clean.py +++ b/lib/spack/spack/cmd/clean.py @@ -1,7 +1,10 @@ import spack.packages as packages +import spack.tty as tty +import spack.stage as stage def setup_parser(subparser): subparser.add_argument('names', nargs='+', help="name(s) of package(s) to clean") + subparser.add_mutually_exclusive_group() subparser.add_argument('-c', "--clean", action="store_true", dest='clean', help="run make clean in the stage directory (default)") @@ -9,8 +12,17 @@ def setup_parser(subparser): help="delete and re-expand the entire stage directory") subparser.add_argument('-d', "--dist", action="store_true", dest='dist', help="delete the downloaded archive.") + subparser.add_argument('-a', "--all", action="store_true", dest='purge', + help="delete the entire build staging area") def clean(args): + if args.purge: + stage.purge() + return + + if not args.names: + tty.die("spack clean requires at least one package name.") + for name in args.names: package = packages.get(name) if args.dist: diff --git a/lib/spack/spack/cmd/create.py b/lib/spack/spack/cmd/create.py index 7407ae7ee3..13fc713acf 100644 --- a/lib/spack/spack/cmd/create.py +++ b/lib/spack/spack/cmd/create.py @@ -28,6 +28,8 @@ def install(self, prefix): def setup_parser(subparser): subparser.add_argument('url', nargs='?', help="url of package archive") + subparser.add_argument('-f', '--force', action='store_true', dest='force', + help="Remove existing package file.") def create(args): @@ -48,12 +50,12 @@ def create(args): tty.die("Couldn't guess a version string from %s." % url) path = packages.filename_for(name) - if os.path.exists(path): + if not args.force and os.path.exists(path): tty.die("%s already exists." % path) # make a stage and fetch the archive. try: - stage = Stage(name, url) + stage = Stage("%s-%s" % (name, version), url) archive_file = stage.fetch() except spack.FailedDownloadException, e: tty.die(e.message) diff --git a/lib/spack/spack/compilation.py b/lib/spack/spack/compilation.py index 10d67db6e9..ecd3c63571 100644 --- a/lib/spack/spack/compilation.py +++ b/lib/spack/spack/compilation.py @@ -1,4 +1,6 @@ import os +import sys + def get_env_var(name, required=True): value = os.environ.get(name) @@ -16,16 +18,16 @@ def get_env_flag(name, required=False): def get_path(name): - path = os.environ.get(name, "") - return path.split(":") + path = os.environ.get(name, "").strip() + if path: + return path.split(":") + else: + return [] def parse_rpaths(arguments): """argparse, for all its features, cannot understand most compilers' rpath arguments. This handles '-Wl,', '-Xlinker', and '-R'""" - linker_args = [] - other_args = [] - def get_next(arg, args): """Get an expected next value of an iterator, or die if it's not there""" try: @@ -34,23 +36,32 @@ def get_next(arg, args): # quietly ignore -rpath and -Xlinker without args. return None - # Separate linker args from non-linker args - args = iter(arguments) - for arg in args: - if arg.startswith('-Wl,'): - sub_args = [sub for sub in arg.replace('-Wl,', '', 1).split(',')] - linker_args.extend(sub_args) - elif arg == '-Xlinker': - target = get_next(arg, args) - if target != None: - linker_args.append(target) - else: - other_args.append(arg) + other_args = [] + def linker_args(): + """This generator function allows us to parse the linker args separately + from the compiler args, so that we can handle them more naturally. + """ + args = iter(arguments) + for arg in args: + if arg.startswith('-Wl,'): + sub_args = [sub for sub in arg.replace('-Wl,', '', 1).split(',')] + for arg in sub_args: + yield arg + elif arg == '-Xlinker': + target = get_next(arg, args) + if target != None: + yield target + else: + other_args.append(arg) - # Extract all the possible ways rpath can appear in linker args - # and append non-rpaths to other_args + # Extract all the possible ways rpath can appear in linker args, then + # append non-rpaths to other_args. This happens in-line as the linker + # args are extracted, so we preserve the original order of arguments. + # This is important for args like --whole-archive, --no-whole-archive, + # and others that tell the linker how to handle the next few libraries + # it encounters on the command line. rpaths = [] - largs = iter(linker_args) + largs = linker_args() for arg in largs: if arg == '-rpath': target = get_next(arg, largs) diff --git a/lib/spack/spack/globals.py b/lib/spack/spack/globals.py index e36305826b..3ef7e53bb8 100644 --- a/lib/spack/spack/globals.py +++ b/lib/spack/spack/globals.py @@ -35,3 +35,14 @@ verbose = False debug = False + +# Whether stage should use tmp filesystem or build in the spack prefix +use_tmp_stage = True + +# Important environment variables +SPACK_NO_PARALLEL_MAKE = 'SPACK_NO_PARALLEL_MAKE' +SPACK_LIB = 'SPACK_LIB' +SPACK_ENV_PATH = 'SPACK_ENV_PATH' +SPACK_DEPENDENCIES = 'SPACK_DEPENDENCIES' +SPACK_PREFIX = 'SPACK_PREFIX' +SPACK_BUILD_ROOT = 'SPACK_BUILD_ROOT' diff --git a/lib/spack/spack/packages/libdwarf.py b/lib/spack/spack/packages/libdwarf.py index 5dbe8f93ad..3b1cdc5891 100644 --- a/lib/spack/spack/packages/libdwarf.py +++ b/lib/spack/spack/packages/libdwarf.py @@ -5,7 +5,7 @@ dwarf_dirs = ['libdwarf', 'dwarfdump2'] class Libdwarf(Package): - homepage = "http://www.example.com" + homepage = "http://reality.sgiweb.org/davea/dwarf.html" url = "http://reality.sgiweb.org/davea/libdwarf-20130207.tar.gz" md5 = "64b42692e947d5180e162e46c689dfbf" diff --git a/lib/spack/spack/packages/libunwind.py b/lib/spack/spack/packages/libunwind.py new file mode 100644 index 0000000000..57b0624805 --- /dev/null +++ b/lib/spack/spack/packages/libunwind.py @@ -0,0 +1,11 @@ +from spack import * + +class Libunwind(Package): + homepage = "http://www.nongnu.org/libunwind/" + url = "http://download.savannah.gnu.org/releases/libunwind/libunwind-1.1.tar.gz" + md5 = "fb4ea2f6fbbe45bf032cd36e586883ce" + + def install(self, prefix): + configure("--prefix=%s" % prefix) + make() + make("install") diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index b476070c2b..9fd273d26f 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -1,6 +1,7 @@ import os import re import shutil +import tempfile import spack import packages @@ -12,6 +13,12 @@ def ensure_access(dir=spack.stage_path): tty.die("Insufficient permissions on directory %s" % dir) +def purge(): + """Remove the entire stage path.""" + if os.path.isdir(spack.stage_path): + shutil.rmtree(spack.stage_path, True) + + class Stage(object): def __init__(self, stage_name, url): self.stage_name = stage_name @@ -42,8 +49,8 @@ def archive_file(self): @property def expanded_archive_path(self): - """"Returns the path to the expanded archive directory if it's expanded; - None if the archive hasn't been expanded. + """Returns the path to the expanded archive directory if it's expanded; + None if the archive hasn't been expanded. """ for file in os.listdir(self.path): archive_path = spack.new_path(self.path, file) @@ -72,14 +79,20 @@ def fetch(self): tty.msg("Fetching %s" % self.url) # Run curl but grab the mime type from the http headers - headers = spack.curl('-#', '-O', '-D', '-', self.url, return_output=True) + headers = spack.curl('-#', # status bar + '-O', # save file to disk + '-D', '-', # print out HTML headers + '-L', self.url, return_output=True) - # output this if we somehow got an HTML file rather than the archive we - # asked for. - if re.search(r'Content-Type: text/html', headers): - tty.warn("The contents of %s look like HTML. The checksum will "+ - "likely fail. Use 'spack clean %s' to delete this file. " - "The fix the gateway issue and install again." % (self.archive_file, self.name)) + # Check if we somehow got an HTML file rather than the archive we + # asked for. We only look at the last content type, to handle + # redirects properly. + content_types = re.findall(r'Content-Type:[^\r\n]+', headers) + if content_types and 'text/html' in content_types[-1]: + tty.warn("The contents of " + self.archive_file + " look like HTML.", + "The checksum will likely be bad. If it is, you can use", + "'spack clean --all' to remove the bad archive, then fix", + "your internet gateway issue and install again.") if not self.archive_file: raise FailedDownloadException(url)