Fixes, remove parallel build for libdwarf due to race case.

lib/spack/env/cc

@@ -4,28 +4,29 @@ import os
 import subprocess
 import argparse
 
-def get_path(name):
-    path = os.environ.get(name, "")
-    return path.split(":")
+# reimplement some tty stuff to minimize imports
+blue, green, yellow, reset = [
+    '\033[1;39m', '\033[1;92m', '\033[4;33m', '\033[0m']
 
 # Import spack parameters through the build environment.
 spack_lib      = os.environ.get("SPACK_LIB")
-spack_prefix   = os.environ.get("SPACK_PREFIX")
-spack_deps     = get_path("SPACK_DEPENDENCIES")
-spack_env_path = get_path("SPACK_ENV_PATH")
-if not spack_lib or spack_deps == None:
-    print "%s must be run from spack." % os.path.abspath(sys.argv[0])
+if not spack_lib:
+    print "Spack compiler must be run from spack!"
     sys.exit(1)
 
-# Figure out what type of operation we're doing
-command = os.path.basename(sys.argv[0])
-
 # Grab a minimal set of spack packages
 sys.path.append(spack_lib)
-from spack.utils import *
-from spack.compilation import parse_rpaths
+from spack.compilation import *
 import spack.tty as tty
 
+spack_prefix   = get_env_var("SPACK_PREFIX")
+spack_debug    = get_env_flag("SPACK_DEBUG")
+spack_deps     = get_path("SPACK_DEPENDENCIES")
+spack_env_path = get_path("SPACK_ENV_PATH")
+
+# Figure out what type of operation we're doing
+command = os.path.basename(sys.argv[0])
+
 parser = argparse.ArgumentParser(add_help=False)
 parser.add_argument("-I", action='append', default=[], dest='include_path')
 parser.add_argument("-L", action='append', default=[], dest='lib_path')
@@ -35,15 +36,15 @@ options, other_args = parser.parse_known_args()
 rpaths, other_args = parse_rpaths(other_args)
 
 if rpaths:
-    tty.warn("Spack stripping non-spack rpaths: ", *rpaths)
+    print "{}Warning{}: Spack stripping non-spack rpaths: ".format(yellow, reset)
+    for rp in rpaths: print "  %s" % rp
 
-# Find the actual command the build is trying to run by removing
-# Spack's env paths from the path.  We use this later for which()
-script_dir = os.path.dirname(os.path.expanduser(__file__))
+# Ensure that the delegated command doesn't just call this script again.
 clean_path = get_path("PATH")
-remove_items(clean_path, '.')
-for path in spack_env_path:
-    remove_items(clean_path, path)
+for item in ['.'] + spack_env_path:
+    if item in clean_path:
+        clean_path.remove(item)
+os.environ["PATH"] = ":".join(clean_path)
 
 # Add dependence's paths to our compiler flags.
 def append_if_dir(path_list, prefix, *dirs):
@@ -57,7 +58,6 @@ for prefix in spack_deps:
     append_if_dir(options.lib_path, prefix, "lib64")
 
 # Add our modified arguments to it.
-cmd = which(command, path=clean_path)
 arguments  = ['-I%s' % path for path in options.include_path]
 arguments += other_args
 arguments += ['-L%s' % path for path in options.lib_path]
@@ -68,10 +68,12 @@ arguments += ['-Wl,-rpath,%s/lib64' % path for path in spack_rpaths]
 arguments += ['-Wl,-rpath,%s/lib' % path for path in spack_rpaths]
 
 # Unset some pesky environment variables
-pop_keys(os.environ, "LD_LIBRARY_PATH", "LD_RUN_PATH", "DYLD_LIBRARY_PATH")
+for var in ["LD_LIBRARY_PATH", "LD_RUN_PATH", "DYLD_LIBRARY_PATH"]:
+    if var in os.environ:
+        os.environ.pop(var)
 
+if spack_debug:
+    print "{}==>{}: {} {}".format(green, reset, cmd, " ".join(arguments))
 
-sys.stderr.write(" ".join(arguments))
-
-rcode = cmd(*arguments, fail_on_error=False)
+rcode = subprocess.call([command] + arguments)
 sys.exit(rcode)

lib/spack/spack/Package.py

@@ -55,7 +55,9 @@ def __init__(self, arch=arch.sys_type()):
         attr.required(self, 'homepage')
         attr.required(self, 'url')
         attr.required(self, 'md5')
-        attr.setdefault(self, "dependencies", [])
+
+        attr.setdefault(self, 'dependencies', [])
+        attr.setdefault(self, 'parallel', True)
 
         # Architecture for this package.
         self.arch = arch
@@ -81,13 +83,23 @@ def __init__(self, arch=arch.sys_type()):
         # Empty at first; only compute dependents if necessary
         self._dependents = None
 
+        # Whether to remove intermediate build/install when things go wrong.
+        self.dirty = False
+
+
+    def make_make(self):
+        """Create a make command set up with the proper default arguments."""
+        make = which('make', required=True)
+        if self.parallel and not env_flag("SPACK_NO_PARALLEL_MAKE"):
+            make.add_default_arg("-j%d" % multiprocessing.cpu_count())
+        return make
 
 
     def add_commands_to_module(self):
         """Populate the module scope of install() with some useful functions.
            This makes things easier for package writers.
         """
-        self.module.make  = make_make()
+        self.module.make  = self.make_make()
 
         # Find the configure script in the archive path
         # Don't use which for this; we want to find it in the current dir.
@@ -203,8 +215,12 @@ def prefix(self):
 
     def remove_prefix(self):
         """Removes the prefix for a package along with any empty parent directories."""
-        shutil.rmtree(self.prefix, True)
+        if os.path.exists(self.prefix):
+            shutil.rmtree(self.prefix, True)
+
         for dir in (self.package_path, self.platform_path):
+            if not os.path.isdir(dir):
+                continue
             if not os.listdir(dir):
                 os.rmdir(dir)
             else:
@@ -260,7 +276,8 @@ def do_install(self):
                 tty.die("Install failed for %s.  No install dir created." % self.name)
         except Exception, e:
             # Blow away the install tree if anything goes wrong.
-            self.remove_prefix()
+            if not self.dirty:
+                self.remove_prefix()
             tty.die("Install failed for %s" % self.name, e.message)
 
 
@@ -337,7 +354,7 @@ def do_clean(self):
     def clean(self):
         """By default just runs make clean.  Override if this isn't good."""
         try:
-            make = make_make()
+            make = self.make_make()
             make('clean')
             tty.msg("Successfully cleaned %s" % self.name)
         except subprocess.CalledProcessError, e:

lib/spack/spack/cmd/install.py

@@ -6,9 +6,12 @@ def setup_parser(subparser):
     subparser.add_argument('-i', '--ignore-dependencies',
                            action='store_true', dest='ignore_dependencies',
                            help="Do not try to install dependencies of requested packages.")
+    subparser.add_argument('-d', '--dirty', action='store_true', dest='dirty',
+                           help="Don't clean up partially completed build/installation on error.")
 
 def install(args):
     spack.ignore_dependencies = args.ignore_dependencies
     for name in args.names:
         package = packages.get(name)
+        package.dirty = args.dirty
         package.do_install()

lib/spack/spack/colify.py

@@ -10,14 +10,16 @@
 #
 # Run colify -h for more information.
 #
+import os
+import sys
+import fcntl
+import termios
+import struct
 
 def get_terminal_size():
-    import os
-
     """Get the dimensions of the console."""
     def ioctl_GWINSZ(fd):
         try:
-            import fcntl, termios, struct
             cr = struct.unpack('hh', fcntl.ioctl(fd, termios.TIOCGWINSZ, '1234'))
         except:
             return
@@ -89,8 +91,6 @@ def config_uniform_cols(elts, console_cols, padding):
 
 
 def colify(elts, **options):
-    import sys
-
     # Get keyword arguments or set defaults
     output       = options.get("output", sys.stdout)
     indent       = options.get("indent", 0)
@@ -133,8 +133,9 @@ def colify(elts, **options):
         if row == rows_last_col:
             cols -= 1
 
+
 if __name__ == "__main__":
-    import optparse, sys
+    import optparse
 
     cols, rows = get_terminal_size()
     parser = optparse.OptionParser()

lib/spack/spack/compilation.py

@@ -1,5 +1,25 @@
 import os
 
+def get_env_var(name, required=True):
+    value = os.environ.get(name)
+    if required and value == None:
+        print "%s must be run from spack." % os.path.abspath(sys.argv[0])
+        sys.exit(1)
+    return value
+
+
+def get_env_flag(name, required=False):
+    value = get_env_var(name, required)
+    if value:
+        return value.lower() == "true"
+    return False
+
+
+def get_path(name):
+    path = os.environ.get(name, "")
+    return path.split(":")
+
+
 def parse_rpaths(arguments):
     """argparse, for all its features, cannot understand most compilers'
        rpath arguments.  This handles '-Wl,', '-Xlinker', and '-R'"""

lib/spack/spack/packages/libdwarf.py

@@ -9,6 +9,9 @@ class Libdwarf(Package):
     url      = "http://reality.sgiweb.org/davea/libdwarf-20130207.tar.gz"
     md5      = "64b42692e947d5180e162e46c689dfbf"
 
+    # There's some kind of race in the makefile
+    parallel = False
+
     depends_on("libelf")
 
     def clean(self):

lib/spack/spack/stage.py

@@ -9,7 +9,7 @@
 
 def ensure_access(dir=spack.stage_path):
     if not os.access(dir, os.R_OK|os.W_OK):
-        tty.die("Insufficient permissions on directory '%s'" % dir)
+        tty.die("Insufficient permissions on directory %s" % dir)
 
 
 class Stage(object):
@@ -25,7 +25,7 @@ def path(self):
     def setup(self):
         if os.path.exists(self.path):
             if not os.path.isdir(self.path):
-                tty.die("Stage path '%s' is not a directory!" % self.path)
+                tty.die("Stage path %s is not a directory!" % self.path)
         else:
             os.makedirs(self.path)
 
@@ -77,7 +77,7 @@ def fetch(self):
             # output this if we somehow got an HTML file rather than the archive we
             # asked for.
             if re.search(r'Content-Type: text/html', headers):
-                tty.warn("The contents of '%s' look like HTML.  The checksum will "+
+                tty.warn("The contents of %s look like HTML.  The checksum will "+
                          "likely fail.  Use 'spack clean %s' to delete this file. "
                          "The fix the gateway issue and install again." % (self.archive_file, self.name))
 
@@ -107,7 +107,7 @@ def chdir_to_archive(self):
         else:
             os.chdir(path)
             if not os.listdir(path):
-                tty.die("Archive was empty for '%s'" % self.name)
+                tty.die("Archive was empty for %s" % self.name)
 
 
     def restage(self):

lib/spack/spack/utils.py

@@ -30,14 +30,6 @@ def memoizer(*args, **kwargs):
     return memoizer
 
 
-def make_make():
-    """Gets a make set up with the proper default arguments."""
-    make = which('make', required=True)
-    if not env_flag("SPACK_NO_PARALLEL_MAKE"):
-        make.add_default_arg("-j%d" % multiprocessing.cpu_count())
-    return make
-
-
 def install(src, dest):
     tty.info("Installing %s to %s" % (src, dest))
     shutil.copy(src, dest)