From d534c539d44fa036510f4afd455d7b0521c092e7 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 19 Feb 2013 17:08:38 -0800 Subject: [PATCH] Fixes, remove parallel build for libdwarf due to race case. --- lib/spack/env/cc | 52 +++++++++++++++------------- lib/spack/spack/Package.py | 27 ++++++++++++--- lib/spack/spack/cmd/install.py | 3 ++ lib/spack/spack/colify.py | 13 +++---- lib/spack/spack/compilation.py | 20 +++++++++++ lib/spack/spack/packages/libdwarf.py | 3 ++ lib/spack/spack/stage.py | 8 ++--- lib/spack/spack/utils.py | 8 ----- 8 files changed, 86 insertions(+), 48 deletions(-) diff --git a/lib/spack/env/cc b/lib/spack/env/cc index 1f706a7ab6..fd072ef6f1 100755 --- a/lib/spack/env/cc +++ b/lib/spack/env/cc @@ -4,28 +4,29 @@ import os import subprocess import argparse -def get_path(name): - path = os.environ.get(name, "") - return path.split(":") +# reimplement some tty stuff to minimize imports +blue, green, yellow, reset = [ + '\033[1;39m', '\033[1;92m', '\033[4;33m', '\033[0m'] # Import spack parameters through the build environment. spack_lib = os.environ.get("SPACK_LIB") -spack_prefix = os.environ.get("SPACK_PREFIX") -spack_deps = get_path("SPACK_DEPENDENCIES") -spack_env_path = get_path("SPACK_ENV_PATH") -if not spack_lib or spack_deps == None: - print "%s must be run from spack." % os.path.abspath(sys.argv[0]) +if not spack_lib: + print "Spack compiler must be run from spack!" sys.exit(1) -# Figure out what type of operation we're doing -command = os.path.basename(sys.argv[0]) - # Grab a minimal set of spack packages sys.path.append(spack_lib) -from spack.utils import * -from spack.compilation import parse_rpaths +from spack.compilation import * import spack.tty as tty +spack_prefix = get_env_var("SPACK_PREFIX") +spack_debug = get_env_flag("SPACK_DEBUG") +spack_deps = get_path("SPACK_DEPENDENCIES") +spack_env_path = get_path("SPACK_ENV_PATH") + +# Figure out what type of operation we're doing +command = os.path.basename(sys.argv[0]) + parser = argparse.ArgumentParser(add_help=False) parser.add_argument("-I", action='append', default=[], dest='include_path') parser.add_argument("-L", action='append', default=[], dest='lib_path') @@ -35,15 +36,15 @@ options, other_args = parser.parse_known_args() rpaths, other_args = parse_rpaths(other_args) if rpaths: - tty.warn("Spack stripping non-spack rpaths: ", *rpaths) + print "{}Warning{}: Spack stripping non-spack rpaths: ".format(yellow, reset) + for rp in rpaths: print " %s" % rp -# Find the actual command the build is trying to run by removing -# Spack's env paths from the path. We use this later for which() -script_dir = os.path.dirname(os.path.expanduser(__file__)) +# Ensure that the delegated command doesn't just call this script again. clean_path = get_path("PATH") -remove_items(clean_path, '.') -for path in spack_env_path: - remove_items(clean_path, path) +for item in ['.'] + spack_env_path: + if item in clean_path: + clean_path.remove(item) +os.environ["PATH"] = ":".join(clean_path) # Add dependence's paths to our compiler flags. def append_if_dir(path_list, prefix, *dirs): @@ -57,7 +58,6 @@ for prefix in spack_deps: append_if_dir(options.lib_path, prefix, "lib64") # Add our modified arguments to it. -cmd = which(command, path=clean_path) arguments = ['-I%s' % path for path in options.include_path] arguments += other_args arguments += ['-L%s' % path for path in options.lib_path] @@ -68,10 +68,12 @@ arguments += ['-Wl,-rpath,%s/lib64' % path for path in spack_rpaths] arguments += ['-Wl,-rpath,%s/lib' % path for path in spack_rpaths] # Unset some pesky environment variables -pop_keys(os.environ, "LD_LIBRARY_PATH", "LD_RUN_PATH", "DYLD_LIBRARY_PATH") +for var in ["LD_LIBRARY_PATH", "LD_RUN_PATH", "DYLD_LIBRARY_PATH"]: + if var in os.environ: + os.environ.pop(var) +if spack_debug: + print "{}==>{}: {} {}".format(green, reset, cmd, " ".join(arguments)) -sys.stderr.write(" ".join(arguments)) - -rcode = cmd(*arguments, fail_on_error=False) +rcode = subprocess.call([command] + arguments) sys.exit(rcode) diff --git a/lib/spack/spack/Package.py b/lib/spack/spack/Package.py index 5fc1a0be8d..836cd9a3c0 100644 --- a/lib/spack/spack/Package.py +++ b/lib/spack/spack/Package.py @@ -55,7 +55,9 @@ def __init__(self, arch=arch.sys_type()): attr.required(self, 'homepage') attr.required(self, 'url') attr.required(self, 'md5') - attr.setdefault(self, "dependencies", []) + + attr.setdefault(self, 'dependencies', []) + attr.setdefault(self, 'parallel', True) # Architecture for this package. self.arch = arch @@ -81,13 +83,23 @@ def __init__(self, arch=arch.sys_type()): # Empty at first; only compute dependents if necessary self._dependents = None + # Whether to remove intermediate build/install when things go wrong. + self.dirty = False + + + def make_make(self): + """Create a make command set up with the proper default arguments.""" + make = which('make', required=True) + if self.parallel and not env_flag("SPACK_NO_PARALLEL_MAKE"): + make.add_default_arg("-j%d" % multiprocessing.cpu_count()) + return make def add_commands_to_module(self): """Populate the module scope of install() with some useful functions. This makes things easier for package writers. """ - self.module.make = make_make() + self.module.make = self.make_make() # Find the configure script in the archive path # Don't use which for this; we want to find it in the current dir. @@ -203,8 +215,12 @@ def prefix(self): def remove_prefix(self): """Removes the prefix for a package along with any empty parent directories.""" - shutil.rmtree(self.prefix, True) + if os.path.exists(self.prefix): + shutil.rmtree(self.prefix, True) + for dir in (self.package_path, self.platform_path): + if not os.path.isdir(dir): + continue if not os.listdir(dir): os.rmdir(dir) else: @@ -260,7 +276,8 @@ def do_install(self): tty.die("Install failed for %s. No install dir created." % self.name) except Exception, e: # Blow away the install tree if anything goes wrong. - self.remove_prefix() + if not self.dirty: + self.remove_prefix() tty.die("Install failed for %s" % self.name, e.message) @@ -337,7 +354,7 @@ def do_clean(self): def clean(self): """By default just runs make clean. Override if this isn't good.""" try: - make = make_make() + make = self.make_make() make('clean') tty.msg("Successfully cleaned %s" % self.name) except subprocess.CalledProcessError, e: diff --git a/lib/spack/spack/cmd/install.py b/lib/spack/spack/cmd/install.py index eab9d3969b..2b1c032fea 100644 --- a/lib/spack/spack/cmd/install.py +++ b/lib/spack/spack/cmd/install.py @@ -6,9 +6,12 @@ def setup_parser(subparser): subparser.add_argument('-i', '--ignore-dependencies', action='store_true', dest='ignore_dependencies', help="Do not try to install dependencies of requested packages.") + subparser.add_argument('-d', '--dirty', action='store_true', dest='dirty', + help="Don't clean up partially completed build/installation on error.") def install(args): spack.ignore_dependencies = args.ignore_dependencies for name in args.names: package = packages.get(name) + package.dirty = args.dirty package.do_install() diff --git a/lib/spack/spack/colify.py b/lib/spack/spack/colify.py index 886928eefb..46a905eb77 100644 --- a/lib/spack/spack/colify.py +++ b/lib/spack/spack/colify.py @@ -10,14 +10,16 @@ # # Run colify -h for more information. # +import os +import sys +import fcntl +import termios +import struct def get_terminal_size(): - import os - """Get the dimensions of the console.""" def ioctl_GWINSZ(fd): try: - import fcntl, termios, struct cr = struct.unpack('hh', fcntl.ioctl(fd, termios.TIOCGWINSZ, '1234')) except: return @@ -89,8 +91,6 @@ def config_uniform_cols(elts, console_cols, padding): def colify(elts, **options): - import sys - # Get keyword arguments or set defaults output = options.get("output", sys.stdout) indent = options.get("indent", 0) @@ -133,8 +133,9 @@ def colify(elts, **options): if row == rows_last_col: cols -= 1 + if __name__ == "__main__": - import optparse, sys + import optparse cols, rows = get_terminal_size() parser = optparse.OptionParser() diff --git a/lib/spack/spack/compilation.py b/lib/spack/spack/compilation.py index 82adfcb917..10d67db6e9 100644 --- a/lib/spack/spack/compilation.py +++ b/lib/spack/spack/compilation.py @@ -1,5 +1,25 @@ import os +def get_env_var(name, required=True): + value = os.environ.get(name) + if required and value == None: + print "%s must be run from spack." % os.path.abspath(sys.argv[0]) + sys.exit(1) + return value + + +def get_env_flag(name, required=False): + value = get_env_var(name, required) + if value: + return value.lower() == "true" + return False + + +def get_path(name): + path = os.environ.get(name, "") + return path.split(":") + + def parse_rpaths(arguments): """argparse, for all its features, cannot understand most compilers' rpath arguments. This handles '-Wl,', '-Xlinker', and '-R'""" diff --git a/lib/spack/spack/packages/libdwarf.py b/lib/spack/spack/packages/libdwarf.py index 480e3280c0..5dbe8f93ad 100644 --- a/lib/spack/spack/packages/libdwarf.py +++ b/lib/spack/spack/packages/libdwarf.py @@ -9,6 +9,9 @@ class Libdwarf(Package): url = "http://reality.sgiweb.org/davea/libdwarf-20130207.tar.gz" md5 = "64b42692e947d5180e162e46c689dfbf" + # There's some kind of race in the makefile + parallel = False + depends_on("libelf") def clean(self): diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index 93708a1efe..b476070c2b 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -9,7 +9,7 @@ def ensure_access(dir=spack.stage_path): if not os.access(dir, os.R_OK|os.W_OK): - tty.die("Insufficient permissions on directory '%s'" % dir) + tty.die("Insufficient permissions on directory %s" % dir) class Stage(object): @@ -25,7 +25,7 @@ def path(self): def setup(self): if os.path.exists(self.path): if not os.path.isdir(self.path): - tty.die("Stage path '%s' is not a directory!" % self.path) + tty.die("Stage path %s is not a directory!" % self.path) else: os.makedirs(self.path) @@ -77,7 +77,7 @@ def fetch(self): # output this if we somehow got an HTML file rather than the archive we # asked for. if re.search(r'Content-Type: text/html', headers): - tty.warn("The contents of '%s' look like HTML. The checksum will "+ + tty.warn("The contents of %s look like HTML. The checksum will "+ "likely fail. Use 'spack clean %s' to delete this file. " "The fix the gateway issue and install again." % (self.archive_file, self.name)) @@ -107,7 +107,7 @@ def chdir_to_archive(self): else: os.chdir(path) if not os.listdir(path): - tty.die("Archive was empty for '%s'" % self.name) + tty.die("Archive was empty for %s" % self.name) def restage(self): diff --git a/lib/spack/spack/utils.py b/lib/spack/spack/utils.py index 5e135aa799..7222228b39 100644 --- a/lib/spack/spack/utils.py +++ b/lib/spack/spack/utils.py @@ -30,14 +30,6 @@ def memoizer(*args, **kwargs): return memoizer -def make_make(): - """Gets a make set up with the proper default arguments.""" - make = which('make', required=True) - if not env_flag("SPACK_NO_PARALLEL_MAKE"): - make.add_default_arg("-j%d" % multiprocessing.cpu_count()) - return make - - def install(src, dest): tty.info("Installing %s to %s" % (src, dest)) shutil.copy(src, dest)