Merge branch 'develop' into correct-cc

# Conflicts:
#	lib/spack/env/cc

.travis.yml

@@ -16,7 +16,10 @@ before_install:
 
 script:
   - . share/spack/setup-env.sh
+  - spack compilers
+  - spack config get compilers
   - spack test
+  - spack install -v libdwarf
 
 notifications:
   email:

README.md

@@ -19,7 +19,7 @@ written in pure Python, and specs allow package authors to write a
 single build script for many different builds of the same package.
 
 See the
-[Feature Overview](http://llnl.github.io/spack/features.html)
+[Feature Overview](http://software.llnl.gov/spack/features.html)
 for examples and highlights.
 
 To install spack and install your first package:
@@ -31,7 +31,7 @@ To install spack and install your first package:
 Documentation
 ----------------
 
-[**Full documentation**](http://llnl.github.io/spack) for Spack is
+[**Full documentation**](http://software.llnl.gov/spack) for Spack is
 the first place to look.
 
 See also:

bin/sbang

@@ -0,0 +1,84 @@
+#!/bin/bash
+#
+# `sbang`: Run scripts with long shebang lines.
+#
+# Many operating systems limit the length of shebang lines, making it
+# hard to use interpreters that are deep in the directory hierarchy.
+# `sbang` can run such scripts, either as a shebang interpreter, or
+# directly on the command line.
+#
+# Usage
+# -----------------------------
+# Suppose you have a script, long-shebang.sh, like this:
+#
+#     1    #!/very/long/path/to/some/interpreter
+#     2
+#     3    echo "success!"
+#
+# Invoking this script will result in an error on some OS's.  On
+# Linux, you get this:
+#
+#     $ ./long-shebang.sh
+#     -bash: ./long: /very/long/path/to/some/interp: bad interpreter:
+#            No such file or directory
+#
+# On Mac OS X, the system simply assumes the interpreter is the shell
+# and tries to run with it, which is likely not what you want.
+#
+#
+# `sbang` on the command line
+# -----------------------------
+# You can use `sbang` in two ways.  The first is to use it directly,
+# from the command line, like this:
+#
+#     $ sbang ./long-shebang.sh
+#     success!
+#
+#
+# `sbang` as the interpreter
+# -----------------------------
+# You can also use `sbang` *as* the interpreter for your script. Put
+# `#!/bin/bash /path/to/sbang` on line 1, and move the original
+# shebang to line 2 of the script:
+#
+#     1    #!/bin/bash /path/to/sbang
+#     2    #!/long/path/to/real/interpreter with arguments
+#     3
+#     4    echo "success!"
+#
+#     $ ./long-shebang.sh
+#     success!
+#
+# On Linux, you could shorten line 1 to `#!/path/to/sbang`, but other
+# operating systems like Mac OS X require the interpreter to be a
+# binary, so it's best to use `sbang` as a `bash` argument.
+# Obviously, for this to work, `sbang` needs to have a short enough
+# path that *it* will run without hitting OS limits.
+#
+#
+# How it works
+# -----------------------------
+# `sbang` is a very simple bash script. It looks at the first two
+# lines of a script argument and runs the last line starting with
+# `#!`, with the script as an argument. It also forwards arguments.
+#
+
+# First argument is the script we want to actually run.
+script="$1"
+
+# Search the first two lines of script for interpreters.
+lines=0
+while read line && ((lines < 2)) ; do
+    if [[ "$line" = '#!'* ]]; then
+        interpreter="${line#\#!}"
+    fi
+    lines=$((lines+1))
+done < "$script"
+
+# Invoke any interpreter found, or raise an error if none was found.
+if [ -n "$interpreter" ]; then
+    exec $interpreter "$@"
+else
+    echo "error: sbang found no interpreter in $script"
+    exit 1
+fi

lib/spack/docs/basic_usage.rst

@@ -357,7 +357,7 @@ Spack, you can simply run ``spack compiler add`` with the path to
 where the compiler is installed.  For example::
 
     $ spack compiler add /usr/local/tools/ic-13.0.079
-    ==> Added 1 new compiler to /Users/gamblin2/.spackconfig
+    ==> Added 1 new compiler to /Users/gamblin2/.spack/compilers.yaml
         intel@13.0.079
 
 Or you can run ``spack compiler add`` with no arguments to force
@@ -367,7 +367,7 @@ installed, but you know that new compilers have been added to your
 
     $ module load gcc-4.9.0
     $ spack compiler add
-    ==> Added 1 new compiler to /Users/gamblin2/.spackconfig
+    ==> Added 1 new compiler to /Users/gamblin2/.spack/compilers.yaml
         gcc@4.9.0
 
 This loads the environment module for gcc-4.9.0 to get it into the
@@ -398,27 +398,34 @@ Manual compiler configuration
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 If auto-detection fails, you can manually configure a compiler by
-editing your ``~/.spackconfig`` file.  You can do this by running
+editing your ``~/.spack/compilers.yaml`` file.  You can do this by running
-``spack config edit``, which will open the file in your ``$EDITOR``.
+``spack config edit compilers``, which will open the file in your ``$EDITOR``.
 
 Each compiler configuration in the file looks like this::
 
     ...
-    [compiler "intel@15.0.0"]
+    chaos_5_x86_64_ib:
-        cc = /usr/local/bin/icc-15.0.024-beta
-        cxx = /usr/local/bin/icpc-15.0.024-beta
-        f77 = /usr/local/bin/ifort-15.0.024-beta
-        fc = /usr/local/bin/ifort-15.0.024-beta
       ...
+      intel@15.0.0:
+          cc: /usr/local/bin/icc-15.0.024-beta
+          cxx: /usr/local/bin/icpc-15.0.024-beta
+          f77: /usr/local/bin/ifort-15.0.024-beta
+          fc: /usr/local/bin/ifort-15.0.024-beta
+      ...
+
+The chaos_5_x86_64_ib string is an architecture string, and multiple
+compilers can be listed underneath an architecture.  The architecture
+string may be replaced with the string 'all' to signify compilers that
+work on all architectures.
 
 For compilers, like ``clang``, that do not support Fortran, put
 ``None`` for ``f77`` and ``fc``::
 
-    [compiler "clang@3.3svn"]
+    clang@3.3svn:
-        cc = /usr/bin/clang
+        cc: /usr/bin/clang
-        cxx = /usr/bin/clang++
+        cxx: /usr/bin/clang++
-        f77 = None
+        f77: None
-        fc = None
+        fc: None
 
 Once you save the file, the configured compilers will show up in the
 list displayed by ``spack compilers``.
@@ -896,7 +903,7 @@ Or, similarly with modules, you could type:
    $ spack load mpich %gcc@4.4.7
 
 These commands will add appropriate directories to your ``PATH``,
-``MANPATH``, and ``LD_LIBRARY_PATH``.  When you no longer want to use
+``MANPATH``, ``CPATH``, and ``LD_LIBRARY_PATH``.  When you no longer want to use
 a package, you can type unload or unuse similarly:
 
 .. code-block:: sh

lib/spack/docs/developer_guide.rst

@@ -73,19 +73,32 @@ with a high level view of Spack's directory structure::
   spack/                  <- installation root
      bin/
         spack             <- main spack executable
+
+     etc/
+        spack/            <- Spack config files.
+                             Can be overridden by files in ~/.spack.
+
      var/
         spack/            <- build & stage directories
+        repos/            <- contains package repositories
+           builtin/       <- pkg repository that comes with Spack
+              repo.yaml   <- descriptor for the builtin repository
+              packages/   <- directories under here contain packages
+
      opt/
         spack/            <- packages are installed here
+
      lib/
         spack/
            docs/          <- source for this documentation
            env/           <- compiler wrappers for build environment
 
+           external/      <- external libs included in Spack distro
+           llnl/          <- some general-use libraries
+
            spack/         <- spack module; contains Python code
               cmd/        <- each file in here is a spack subcommand
               compilers/  <- compiler description files
-              packages/   <- each file in here is a spack package
               test/       <- unit test modules
               util/       <- common code
 

lib/spack/docs/features.rst

@@ -103,7 +103,7 @@ creates a simple python file:
 It doesn't take much python coding to get from there to a working
 package:
 
-.. literalinclude:: ../../../var/spack/packages/libelf/package.py
+.. literalinclude:: ../../../var/spack/repos/builtin/packages/libelf/package.py
    :lines: 25-
 
 Spack also provides wrapper functions around common commands like

lib/spack/docs/getting_started.rst

@@ -22,7 +22,7 @@ go:
    $ spack install libelf
 
 For a richer experience, use Spack's `shell support
-<http://llnl.github.io/spack/basic_usage.html#environment-modules>`_:
+<http://software.llnl.gov/spack/basic_usage.html#environment-modules>`_:
 
 .. code-block:: sh
 

lib/spack/docs/index.rst

@@ -18,7 +18,7 @@ configurations can coexist on the same system.
 Most importantly, Spack is *simple*.  It offers a simple *spec* syntax
 so that users can specify versions and configuration options
 concisely.  Spack is also simple for package authors: package files
-are writtin in pure Python, and specs allow package authors to
+are written in pure Python, and specs allow package authors to
 maintain a single file for many different builds of the same package.
 
 See the :doc:`features` for examples and highlights.

lib/spack/docs/mirrors.rst

@@ -38,7 +38,7 @@ contains tarballs for each package, named after each package.
 
 .. note::
 
-   Archives are **not** named exactly they were in the package's fetch
+   Archives are **not** named exactly the way they were in the package's fetch
    URL.  They have the form ``<name>-<version>.<extension>``, where
    ``<name>`` is Spack's name for the package, ``<version>`` is the
    version of the tarball, and ``<extension>`` is whatever format the
@@ -186,7 +186,7 @@ Each mirror has a name so that you can refer to it again later.
 ``spack mirror list``
 ----------------------------
 
-If you want to see all the mirrors Spack knows about you can run ``spack mirror list``::
+To see all the mirrors Spack knows about, run ``spack mirror list``::
 
    $ spack mirror list
    local_filesystem    file:///Users/gamblin2/spack-mirror-2014-06-24
@@ -196,7 +196,7 @@ If you want to see all the mirrors Spack knows about you can run ``spack mirror
 ``spack mirror remove``
 ----------------------------
 
-And, if you want to remove a mirror, just remove it by name::
+To remove a mirror by name::
 
    $ spack mirror remove local_filesystem
    $ spack mirror list
@@ -205,12 +205,11 @@ And, if you want to remove a mirror, just remove it by name::
 Mirror precedence
 ----------------------------
 
-Adding a mirror really just adds a section in ``~/.spackconfig``::
+Adding a mirror really adds a line in ``~/.spack/mirrors.yaml``::
 
-   [mirror "local_filesystem"]
+   mirrors:
-       url = file:///Users/gamblin2/spack-mirror-2014-06-24
+     local_filesystem: file:///Users/gamblin2/spack-mirror-2014-06-24
-   [mirror "remote_server"]
+     remote_server: https://example.com/some/web-hosted/directory/spack-mirror-2014-06-24
-       url = https://example.com/some/web-hosted/directory/spack-mirror-2014-06-24
 
 If you want to change the order in which mirrors are searched for
 packages, you can edit this file and reorder the sections.  Spack will

lib/spack/docs/packaging_guide.rst

@@ -84,7 +84,7 @@ always choose to download just one tarball initially, and run
 
    If it fails entirely, you can get minimal boilerplate by using
    :ref:`spack-edit-f`, or you can manually create a directory and
-   ``package.py`` file for the package in ``var/spack/packages``.
+   ``package.py`` file for the package in ``var/spack/repos/builtin/packages``.
 
 .. note::
 
@@ -203,7 +203,7 @@ edit`` command:
 So, if you used ``spack create`` to create a package, then saved and
 closed the resulting file, you can get back to it with ``spack edit``.
 The ``cmake`` package actually lives in
-``$SPACK_ROOT/var/spack/packages/cmake/package.py``, but this provides
+``$SPACK_ROOT/var/spack/repos/builtin/packages/cmake/package.py``, but this provides
 a much simpler shortcut and saves you the trouble of typing the full
 path.
 
@@ -269,18 +269,18 @@ live in Spack's directory structure.  In general, `spack-create`_ and
 `spack-edit`_ handle creating package files for you, so you can skip
 most of the details here.
 
-``var/spack/packages``
+``var/spack/repos/builtin/packages``
 ~~~~~~~~~~~~~~~~~~~~~~~
 
 A Spack installation directory is structured like a standard UNIX
 install prefix (``bin``, ``lib``, ``include``, ``var``, ``opt``,
 etc.).  Most of the code for Spack lives in ``$SPACK_ROOT/lib/spack``.
-Packages themselves live in ``$SPACK_ROOT/var/spack/packages``.
+Packages themselves live in ``$SPACK_ROOT/var/spack/repos/builtin/packages``.
 
 If you ``cd`` to that directory, you will see directories for each
 package:
 
-.. command-output::  cd $SPACK_ROOT/var/spack/packages;  ls -CF
+.. command-output::  cd $SPACK_ROOT/var/spack/repos/builtin/packages;  ls -CF
    :shell:
    :ellipsis: 10
 
@@ -288,7 +288,7 @@ Each directory contains a file called ``package.py``, which is where
 all the python code for the package goes.  For example, the ``libelf``
 package lives in::
 
-   $SPACK_ROOT/var/spack/packages/libelf/package.py
+   $SPACK_ROOT/var/spack/repos/builtin/packages/libelf/package.py
 
 Alongside the ``package.py`` file, a package may contain extra
 directories or files (like patches) that it needs to build.
@@ -301,7 +301,7 @@ Packages are named after the directory containing ``package.py``.  So,
 ``libelf``'s ``package.py`` lives in a directory called ``libelf``.
 The ``package.py`` file defines a class called ``Libelf``, which
 extends Spack's ``Package`` class.  for example, here is
-``$SPACK_ROOT/var/spack/packages/libelf/package.py``:
+``$SPACK_ROOT/var/spack/repos/builtin/packages/libelf/package.py``:
 
 .. code-block:: python
    :linenos:
@@ -328,7 +328,7 @@ these:
    $ spack install libelf@0.8.13
 
 Spack sees the package name in the spec and looks for
-``libelf/package.py`` in ``var/spack/packages``.  Likewise, if you say
+``libelf/package.py`` in ``var/spack/repos/builtin/packages``.  Likewise, if you say
 ``spack install py-numpy``, then Spack looks for
 ``py-numpy/package.py``.
 
@@ -401,6 +401,35 @@ construct the new one for ``8.2.1``.
 When you supply a custom URL for a version, Spack uses that URL
 *verbatim* and does not perform extrapolation.
 
+Skipping the expand step
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Spack normally expands archives automatically after downloading
+them. If you want to skip this step (e.g., for self-extracting
+executables and other custom archive types), you can add
+``expand=False`` to a ``version`` directive.
+
+.. code-block:: python
+
+   version('8.2.1', '4136d7b4c04df68b686570afa26988ac',
+           url='http://example.com/foo-8.2.1-special-version.tar.gz', 'expand=False')
+
+When ``expand`` is set to ``False``, Spack sets the current working
+directory to the directory containing the downloaded archive before it
+calls your ``install`` method.  Within ``install``, the path to the
+downloaded archive is available as ``self.stage.archive_file``.
+
+Here is an example snippet for packages distributed as self-extracting
+archives.  The example sets permissions on the downloaded file to make
+it executable, then runs it with some arguments.
+
+.. code-block:: python
+
+   def install(self, spec, prefix):
+       set_executable(self.stage.archive_file)
+       installer = Executable(self.stage.archive_file)
+       installer('--prefix=%s' % prefix, 'arg1', 'arg2', 'etc.')
+
 Checksums
 ~~~~~~~~~~~~~~~~~
 
@@ -632,7 +661,7 @@ Default
   revision instead.
 
 Revisions
-  Add ``hg`` and ``revision``parameters:
+  Add ``hg`` and ``revision`` parameters:
 
   .. code-block:: python
 
@@ -703,7 +732,7 @@ supply is a filename, then the patch needs to live within the spack
 source tree.  For example, the patch above lives in a directory
 structure like this::
 
-   $SPACK_ROOT/var/spack/packages/
+   $SPACK_ROOT/var/spack/repos/builtin/packages/
        mvapich2/
            package.py
            ad_lustre_rwcontig_open_source.patch
@@ -1524,6 +1553,69 @@ This is useful when you want to know exactly what Spack will do when
 you ask for a particular spec.
 
 
+``Concretization Policies``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+A user may have certain preferences for how packages should
+be concretized on their system.  For example, one user may prefer packages
+built with OpenMPI and the Intel compiler.  Another user may prefer
+packages be built with MVAPICH and GCC.  
+
+Spack can be configured to prefer certain compilers, package
+versions, depends_on, and variants during concretization.
+The preferred configuration can be controlled via the 
+``~/.spack/packages.yaml`` file for user configuations, or the
+``etc/spack/packages.yaml`` site configuration.
+
+
+Here's an example packages.yaml file that sets preferred packages:
+
+.. code-block:: sh
+
+    packages:
+      dyninst:
+        compiler: [gcc@4.9]
+        variants: +debug
+      gperftools:
+        version: [2.2, 2.4, 2.3]
+      all:
+        compiler: [gcc@4.4.7, gcc@4.6:, intel, clang, pgi]
+        providers:
+          mpi: [mvapich, mpich, openmpi]
+          
+
+At a high level, this example is specifying how packages should be
+concretized.  The dyninst package should prefer using gcc 4.9 and 
+be built with debug options.  The gperftools package should prefer version
+2.2 over 2.4.  Every package on the system should prefer mvapich for
+its MPI and gcc 4.4.7 (except for Dyninst, which overrides this by preferring gcc 4.9).  
+These options are used to fill in implicit defaults.  Any of them can be overwritten 
+on the command line if explicitly requested.
+
+Each packages.yaml file begins with the string ``packages:`` and 
+package names are specified on the next level. The special string ``all``
+applies settings to each package. Underneath each package name is 
+one or more components: ``compiler``, ``variants``, ``version``, 
+or ``providers``.  Each component has an ordered list of spec 
+``constraints``, with earlier entries in the list being preferred over
+later entries.
+
+Sometimes a package installation may have constraints that forbid 
+the first concretization rule, in which case Spack will use the first
+legal concretization rule.  Going back to the example, if a user
+requests gperftools 2.3 or later, then Spack will install version 2.4 
+as the 2.4 version of gperftools is preferred over 2.3.
+
+An explicit concretization rule in the preferred section will always 
+take preference over unlisted concretizations.  In the above example, 
+xlc isn't listed in the compiler list.  Every listed compiler from
+gcc to pgi will thus be preferred over the xlc compiler.
+
+The syntax for the ``provider`` section differs slightly from other
+concretization rules.  A provider lists a value that packages may
+``depend_on`` (e.g, mpi) and a list of rules for fulfilling that
+dependency.
+
 .. _install-method:
 
 Implementing the ``install`` method
@@ -1533,7 +1625,7 @@ The last element of a package is its ``install()`` method.  This is
 where the real work of installation happens, and it's the main part of
 the package you'll need to customize for each piece of software.
 
-.. literalinclude::  ../../../var/spack/packages/libelf/package.py
+.. literalinclude::  ../../../var/spack/repos/builtin/packages/libelf/package.py
    :start-after: 0.8.12
    :linenos:
 
@@ -1711,15 +1803,15 @@ Compile-time library search paths
   * ``-L$dep_prefix/lib``
   * ``-L$dep_prefix/lib64``
 Runtime library search paths (RPATHs)
-  * ``-Wl,-rpath=$dep_prefix/lib``
+  * ``-Wl,-rpath,$dep_prefix/lib``
-  * ``-Wl,-rpath=$dep_prefix/lib64``
+  * ``-Wl,-rpath,$dep_prefix/lib64``
 Include search paths
   * ``-I$dep_prefix/include``
 
 An example of this would be the ``libdwarf`` build, which has one
 dependency: ``libelf``.  Every call to ``cc`` in the ``libdwarf``
 build will have ``-I$LIBELF_PREFIX/include``,
-``-L$LIBELF_PREFIX/lib``, and ``-Wl,-rpath=$LIBELF_PREFIX/lib``
+``-L$LIBELF_PREFIX/lib``, and ``-Wl,-rpath,$LIBELF_PREFIX/lib``
 inserted on the command line.  This is done transparently to the
 project's build system, which will just think it's using a system
 where ``libelf`` is readily available.  Because of this, you **do
@@ -2108,6 +2200,15 @@ Filtering functions
 
   Examples:
 
+  #. Filtering a Makefile to force it to use Spack's compiler wrappers:
+
+     .. code-block:: python
+
+        filter_file(r'^CC\s*=.*',  spack_cc,  'Makefile')
+        filter_file(r'^CXX\s*=.*', spack_cxx, 'Makefile')
+        filter_file(r'^F77\s*=.*', spack_f77, 'Makefile')
+        filter_file(r'^FC\s*=.*',  spack_fc,  'Makefile')
+
   #. Replacing ``#!/usr/bin/perl`` with ``#!/usr/bin/env perl`` in ``bib2xhtml``:
 
      .. code-block:: python

lib/spack/docs/site_configuration.rst

@@ -54,87 +54,73 @@ more elements to the list to indicate where your own site's temporary
 directory is.
 
 
-.. _concretization-policies:
+External Packages
+~~~~~~~~~~~~~~~~~~~~~
+Spack can be configured to use externally-installed
+packages rather than building its own packages. This may be desirable
+if machines ship with system packages, such as a customized MPI
+that should be used instead of Spack building its own MPI.
 
-Concretization policies
+External packages are configured through the ``packages.yaml`` file found
-----------------------------
+in a Spack installation's ``etc/spack/`` or a user's ``~/.spack/``
+directory. Here's an example of an external configuration:
 
-When a user asks for a package like ``mpileaks`` to be installed,
+.. code-block:: yaml
-Spack has to make decisions like what version should be installed,
-what compiler to use, and how its dependencies should be configured.
-This process is called *concretization*, and it's covered in detail in
-:ref:`its own section <abstract-and-concrete>`.
 
-The default concretization policies are in the
+   packages:
-:py:mod:`spack.concretize` module, specifically in the
+      openmpi:
-:py:class:`spack.concretize.DefaultConcretizer` class.  These are the
+         paths:
-important methods used in the concretization process:
+            openmpi@1.4.3%gcc@4.4.7=chaos_5_x86_64_ib: /opt/openmpi-1.4.3
+            openmpi@1.4.3%gcc@4.4.7=chaos_5_x86_64_ib+debug: /opt/openmpi-1.4.3-debug
+            openmpi@1.6.5%intel@10.1=chaos_5_x86_64_ib: /opt/openmpi-1.6.5-intel
 
-* :py:meth:`concretize_version(self, spec) <spack.concretize.DefaultConcretizer.concretize_version>`
+This example lists three installations of OpenMPI, one built with gcc,
-* :py:meth:`concretize_architecture(self, spec) <spack.concretize.DefaultConcretizer.concretize_architecture>`
+one built with gcc and debug information, and another built with Intel.
-* :py:meth:`concretize_compiler(self, spec) <spack.concretize.DefaultConcretizer.concretize_compiler>`
+If Spack is asked to build a package that uses one of these MPIs as a
-* :py:meth:`choose_provider(self, spec, providers) <spack.concretize.DefaultConcretizer.choose_provider>`
+dependency, it will use the the pre-installed OpenMPI in
+the given directory.  
 
-The first three take a :py:class:`Spec <spack.spec.Spec>` object and
+Each ``packages.yaml`` begins with a ``packages:`` token, followed
-modify it by adding constraints for the version.  For example, if the
+by a list of package names.  To specify externals, add a ``paths``
-input spec had a version range like `1.0:5.0.3`, then the
+token under the package name, which lists externals in a
-``concretize_version`` method should set the spec's version to a
+``spec : /path`` format.  Each spec should be as
-*single* version in that range.  Likewise, ``concretize_architecture``
+well-defined as reasonably possible.  If a
-selects an architecture when the input spec does not have one, and
+package lacks a spec component, such as missing a compiler or
-``concretize_compiler`` needs to set both a concrete compiler and a
+package version, then Spack will guess the missing component based
-concrete compiler version.
+on its most-favored packages, and it may guess incorrectly.
 
-``choose_provider()`` affects how concrete implementations are chosen
+Each package version and compilers listed in an external should
-based on a virtual dependency spec.  The input spec is some virtual
+have entries in Spack's packages and compiler configuration, even
-dependency and the ``providers`` index is a :py:class:`ProviderIndex
+though the package and compiler may not every be built.
-<spack.packages.ProviderIndex>` object.  The ``ProviderIndex`` maps
-the virtual spec to specs for possible implementations, and
-``choose_provider()`` should simply choose one of these.  The
-``concretize_*`` methods will be called on the chosen implementation
-later, so there is no need to fully concretize the spec when returning
-it.
 
-The ``DefaultConcretizer`` is intended to provide sensible defaults
+The packages configuration can tell Spack to use an external location
-for each policy, but there are certain choices that it can't know
+for certain package versions, but it does not restrict Spack to using
-about.  For example, one site might prefer ``OpenMPI`` over ``MPICH``,
+external packages.  In the above example, if an OpenMPI 1.8.4 became
-or another might prefer an old version of some packages.  These types
+available Spack may choose to start building and linking with that version
-of special cases can be integrated with custom concretizers.
+rather than continue using the pre-installed OpenMPI versions.
 
-Writing a custom concretizer
+To prevent this, the ``packages.yaml`` configuration also allows packages
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+to be flagged as non-buildable.  The previous example could be modified to
+be:
 
-To write your own concretizer, you need only subclass
+.. code-block:: yaml
-``DefaultConcretizer`` and override the methods you want to change.
-For example, you might write a class like this to change *only* the
-``concretize_version()`` behavior:
 
-.. code-block:: python
+  packages:
+    openmpi:
+      paths:
+        openmpi@1.4.3%gcc@4.4.7=chaos_5_x86_64_ib: /opt/openmpi-1.4.3
+        openmpi@1.4.3%gcc@4.4.7=chaos_5_x86_64_ib+debug: /opt/openmpi-1.4.3-debug
+        openmpi@1.6.5%intel@10.1=chaos_5_x86_64_ib: /opt/openmpi-1.6.5-intel                
+      buildable: False
 
-   from spack.concretize import DefaultConcretizer
+The addition of the ``buildable`` flag tells Spack that it should never build
+its own version of OpenMPI, and it will instead always rely on a pre-built
+OpenMPI.  Similar to ``paths``, ``buildable`` is specified as a property under
+a package name.
 
-   class MyConcretizer(DefaultConcretizer):
+The ``buildable`` does not need to be paired with external packages.  
-       def concretize_version(self, spec):
+It could also be used alone to forbid packages that may be 
-           # implement custom logic here.
+buggy or otherwise undesirable.
-
-Once you have written your custom concretizer, you can make Spack use
-it by editing ``globals.py``.  Find this part of the file:
-
-.. code-block:: python
-
-   #
-   # This controls how things are concretized in spack.
-   # Replace it with a subclass if you want different
-   # policies.
-   #
-   concretizer = DefaultConcretizer()
-
-Set concretizer to *your own* class instead of the default:
-
-.. code-block:: python
-
-   concretizer = MyConcretizer()
-
-The next time you run Spack, your changes should take effect.
 
 
 Profiling

lib/spack/env/cc

@@ -90,15 +90,15 @@ case "$command" in
         command="$SPACK_CC"
         language="C"
         ;;
-    c++|CC|g++|clang++|icpc|pgCC|xlc++)
+    c++|CC|g++|clang++|icpc|pgc++|xlc++)
         command="$SPACK_CXX"
         language="C++"
         ;;
-    f90|fc|f95|gfortran|ifort|pgf90|xlf90|nagfor)
+    f90|fc|f95|gfortran|ifort|pgfortran|xlf90|nagfor)
         command="$SPACK_FC"
         language="Fortran 90"
         ;;
-    f77|gfortran|ifort|pgf77|xlf|nagfor)
+    f77|gfortran|ifort|pgfortran|xlf|nagfor)
         command="$SPACK_F77"
         language="Fortran 77"
         ;;
@@ -113,14 +113,22 @@ case "$command" in
         ;;
 esac
 
-# Finish setting up the mode.
+# If any of the arguments below is present then the mode is vcheck. In vcheck mode nothing is added in terms of extra search paths or libraries
 if [ -z "$mode" ]; then
-    mode=ccld
     for arg in "$@"; do
         if [ "$arg" = -v -o "$arg" = -V -o "$arg" = --version -o "$arg" = -dumpversion ]; then
             mode=vcheck
             break
-        elif [ "$arg" = -E ]; then
+    fi
+    done
+fi
+
+# Finish setting up the mode.
+
+if [ -z "$mode" ]; then
+    mode=ccld
+    for arg in "$@"; do
+        if [ "$arg" = -E ]; then
             mode=cpp
             break
         elif [ "$arg" = -c ]; then
@@ -130,7 +138,7 @@ if [ -z "$mode" ]; then
     done
 fi
 
-# Dump the version and exist if we're in testing mode.
+# Dump the version and exit if we're in testing mode.
 if [ "$SPACK_TEST_COMMAND" = "dump-mode" ]; then
     echo "$mode"
     exit
@@ -146,6 +154,7 @@ fi
 input_command="$@"
 args=("$@")
 
+<<<<<<< HEAD
 # Dump parsed values for unit testing if asked for
 if [[ -n $SPACK_TEST_COMMAND ]]; then
 
@@ -213,6 +222,88 @@ if [[ -n $SPACK_TEST_COMMAND ]]; then
         esac
         shift
     done
+=======
+if [ "$mode" == vcheck ] ; then
+    exec ${command} "$@"
+fi
+
+#
+# Now do real parsing of the command line args, trying hard to keep
+# non-rpath linker arguments in the proper order w.r.t. other command
+# line arguments.  This is important for things like groups.
+#
+includes=()
+libraries=()
+libs=()
+rpaths=()
+other_args=()
+
+while [ -n "$1" ]; do
+    case "$1" in
+        -I*)
+            arg="${1#-I}"
+            if [ -z "$arg" ]; then shift; arg="$1"; fi
+            includes+=("$arg")
+            ;;
+        -L*)
+            arg="${1#-L}"
+            if [ -z "$arg" ]; then shift; arg="$1"; fi
+            libraries+=("$arg")
+            ;;
+        -l*)
+            arg="${1#-l}"
+            if [ -z "$arg" ]; then shift; arg="$1"; fi
+            libs+=("$arg")
+            ;;
+        -Wl,*)
+            arg="${1#-Wl,}"
+            # TODO: Handle multiple -Wl, continuations of -Wl,-rpath
+            if [[ $arg == -rpath=* ]]; then
+                arg="${arg#-rpath=}"
+                for rpath in ${arg//,/ }; do
+                    rpaths+=("$rpath")
+                done
+            elif [[ $arg == -rpath,* ]]; then
+                arg="${arg#-rpath,}"
+                for rpath in ${arg//,/ }; do
+                    rpaths+=("$rpath")
+                done
+            elif [[ $arg == -rpath ]]; then
+                shift; arg="$1"
+                if [[ $arg != '-Wl,'* ]]; then
+                    die "-Wl,-rpath was not followed by -Wl,*"
+                fi
+                arg="${arg#-Wl,}"
+                for rpath in ${arg//,/ }; do
+                    rpaths+=("$rpath")
+                done
+            else
+                other_args+=("-Wl,$arg")
+            fi
+            ;;
+        -Xlinker)
+            shift; arg="$1";
+            if [[ $arg = -rpath=* ]]; then
+                rpaths+=("${arg#-rpath=}")
+            elif [[ $arg = -rpath ]]; then
+                shift; arg="$1"
+                if [[ $arg != -Xlinker ]]; then
+                    die "-Xlinker -rpath was not followed by -Xlinker <arg>"
+                fi
+                shift; arg="$1"
+                rpaths+=("$arg")
+            else
+                other_args+=("-Xlinker")
+                other_args+=("$arg")
+            fi
+            ;;
+        *)
+            other_args+=("$1")
+            ;;
+    esac
+    shift
+done
+>>>>>>> develop
 
     IFS=$'\n'
     case "$SPACK_TEST_COMMAND" in

lib/spack/env/pgi/case-insensitive/pgCC

@@ -1 +0,0 @@
-../../cc

lib/spack/llnl/util/filesystem.py

@@ -25,7 +25,9 @@
 __all__ = ['set_install_permissions', 'install', 'install_tree', 'traverse_tree',
            'expand_user', 'working_dir', 'touch', 'touchp', 'mkdirp',
            'force_remove', 'join_path', 'ancestor', 'can_access', 'filter_file',
-           'FileFilter', 'change_sed_delimiter', 'is_exe', 'force_symlink']
+           'FileFilter', 'change_sed_delimiter', 'is_exe', 'force_symlink',
+           'set_executable', 'copy_mode', 'unset_executable_mode',
+           'remove_dead_links', 'remove_linked_tree']
 
 import os
 import sys
@@ -152,15 +154,28 @@ def set_install_permissions(path):
 def copy_mode(src, dest):
     src_mode = os.stat(src).st_mode
     dest_mode = os.stat(dest).st_mode
-    if src_mode | stat.S_IXUSR: dest_mode |= stat.S_IXUSR
+    if src_mode & stat.S_IXUSR: dest_mode |= stat.S_IXUSR
-    if src_mode | stat.S_IXGRP: dest_mode |= stat.S_IXGRP
+    if src_mode & stat.S_IXGRP: dest_mode |= stat.S_IXGRP
-    if src_mode | stat.S_IXOTH: dest_mode |= stat.S_IXOTH
+    if src_mode & stat.S_IXOTH: dest_mode |= stat.S_IXOTH
     os.chmod(dest, dest_mode)
 
 
+def unset_executable_mode(path):
+    mode = os.stat(path).st_mode
+    mode &= ~stat.S_IXUSR
+    mode &= ~stat.S_IXGRP
+    mode &= ~stat.S_IXOTH
+    os.chmod(path, mode)
+
+
 def install(src, dest):
     """Manually install a file to a particular location."""
     tty.debug("Installing %s to %s" % (src, dest))
+
+    # Expand dsst to its eventual full path if it is a directory.
+    if os.path.isdir(dest):
+        dest = join_path(dest, os.path.basename(src))
+
     shutil.copy(src, dest)
     set_install_permissions(dest)
     copy_mode(src, dest)
@@ -235,7 +250,7 @@ def touchp(path):
 def force_symlink(src, dest):
     try:
         os.symlink(src, dest)
-    except OSError, e:
+    except OSError as e:
         os.remove(dest)
         os.symlink(src, dest)
 
@@ -339,3 +354,41 @@ def traverse_tree(source_root, dest_root, rel_path='', **kwargs):
 
     if order == 'post':
         yield (source_path, dest_path)
+
+
+def set_executable(path):
+    st = os.stat(path)
+    os.chmod(path, st.st_mode | stat.S_IEXEC)
+
+
+def remove_dead_links(root):
+    """
+    Removes any dead link that is present in root
+
+    Args:
+        root: path where to search for dead links
+
+    """
+    for file in os.listdir(root):
+        path = join_path(root, file)
+        if os.path.islink(path):
+            real_path = os.path.realpath(path)
+            if not os.path.exists(real_path):
+                os.unlink(path)
+
+def remove_linked_tree(path):
+    """
+    Removes a directory and its contents.  If the directory is a
+    symlink, follows the link and removes the real directory before
+    removing the link.
+
+    Args:
+        path: directory to be removed
+
+    """
+    if os.path.exists(path):
+        if os.path.islink(path):
+            shutil.rmtree(os.path.realpath(path), True)
+            os.unlink(path)
+        else:
+            shutil.rmtree(path, True)

lib/spack/llnl/util/lang.py

@@ -235,11 +235,11 @@ def setter(name, value):
     if not has_method(cls, '_cmp_key'):
         raise TypeError("'%s' doesn't define _cmp_key()." % cls.__name__)
 
-    setter('__eq__', lambda s,o: o is not None and s._cmp_key() == o._cmp_key())
+    setter('__eq__', lambda s,o: (s is o) or (o is not None and s._cmp_key() == o._cmp_key()))
     setter('__lt__', lambda s,o: o is not None and s._cmp_key() <  o._cmp_key())
     setter('__le__', lambda s,o: o is not None and s._cmp_key() <= o._cmp_key())
 
-    setter('__ne__', lambda s,o: o is None or s._cmp_key() != o._cmp_key())
+    setter('__ne__', lambda s,o: (s is not o) and (o is None or s._cmp_key() != o._cmp_key()))
     setter('__gt__', lambda s,o: o is None or s._cmp_key() >  o._cmp_key())
     setter('__ge__', lambda s,o: o is None or s._cmp_key() >= o._cmp_key())
 

lib/spack/spack/__init__.py

@@ -81,6 +81,20 @@
 from spack.directory_layout import YamlDirectoryLayout
 install_layout = YamlDirectoryLayout(install_path)
 
+#
+# This controls how packages are sorted when trying to choose
+# the most preferred package.  More preferred packages are sorted
+# first.
+#
+from spack.preferred_packages import PreferredPackages
+pkgsort = PreferredPackages()
+
+#
+# This tests ABI compatibility between packages
+#
+from spack.abi import ABI
+abi = ABI()
+
 #
 # This controls how things are concretized in spack.
 # Replace it with a subclass if you want different

lib/spack/spack/abi.py

@@ -0,0 +1,128 @@
+##############################################################################
+# Copyright (c) 2015, Lawrence Livermore National Security, LLC.
+# Produced at the Lawrence Livermore National Laboratory.
+#
+# This file is part of Spack.
+# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
+# LLNL-CODE-647188
+#
+# For details, see https://scalability-llnl.github.io/spack
+# Please also see the LICENSE file for our notice and the LGPL.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License (as published by
+# the Free Software Foundation) version 2.1 dated February 1999.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
+# conditions of the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+##############################################################################
+
+import os
+import spack
+import spack.spec
+from spack.spec import CompilerSpec
+from spack.util.executable import Executable, ProcessError
+from llnl.util.lang import memoized
+
+class ABI(object):
+    """This class provides methods to test ABI compatibility between specs.
+       The current implementation is rather rough and could be improved."""
+
+    def architecture_compatible(self, parent, child):
+        """Returns true iff the parent and child specs have ABI compatible architectures."""
+        return not parent.architecture or not child.architecture or parent.architecture == child.architecture
+
+
+    @memoized
+    def _gcc_get_libstdcxx_version(self, version):
+        """Returns gcc ABI compatibility info by getting the library version of
+           a compiler's libstdc++.so or libgcc_s.so"""
+        spec = CompilerSpec("gcc", version)
+        compilers = spack.compilers.compilers_for_spec(spec)
+        if not compilers:
+            return None
+        compiler = compilers[0]
+        rungcc = None
+        libname = None
+        output = None
+        if compiler.cxx:
+            rungcc = Executable(compiler.cxx)
+            libname = "libstdc++.so"
+        elif compiler.cc:
+            rungcc = Executable(compiler.cc)
+            libname = "libgcc_s.so"
+        else:
+            return None
+        try:
+            output = rungcc("--print-file-name=%s" % libname, return_output=True)
+        except ProcessError, e:
+            return None
+        if not output:
+            return None
+        libpath = os.readlink(output.strip())
+        if not libpath:
+            return None
+        return os.path.basename(libpath)
+
+
+    @memoized
+    def _gcc_compiler_compare(self, pversion, cversion):
+        """Returns true iff the gcc version pversion and cversion
+          are ABI compatible."""
+        plib = self._gcc_get_libstdcxx_version(pversion)
+        clib = self._gcc_get_libstdcxx_version(cversion)
+        if not plib or not clib:
+            return False
+        return plib == clib
+
+
+    def _intel_compiler_compare(self, pversion, cversion):
+        """Returns true iff the intel version pversion and cversion
+           are ABI compatible"""
+
+        # Test major and minor versions.  Ignore build version.
+        if (len(pversion.version) < 2 or len(cversion.version) < 2):
+            return False
+        return pversion.version[:2] == cversion.version[:2]
+
+
+    def compiler_compatible(self, parent, child, **kwargs):
+        """Returns true iff the compilers for parent and child specs are ABI compatible"""
+        if not parent.compiler or not child.compiler:
+            return True
+
+        if parent.compiler.name != child.compiler.name:
+            # Different compiler families are assumed ABI incompatible
+            return False
+
+        if kwargs.get('loose', False):
+            return True
+
+        # TODO: Can we move the specialized ABI matching stuff
+        # TODO: into compiler classes?
+        for pversion in parent.compiler.versions:
+            for cversion in child.compiler.versions:
+                # For a few compilers use specialized comparisons.  Otherwise
+                # match on version match.
+                if pversion.satisfies(cversion):
+                    return True
+                elif (parent.compiler.name == "gcc" and
+                      self._gcc_compiler_compare(pversion, cversion)):
+                    return True
+                elif (parent.compiler.name == "intel" and
+                      self._intel_compiler_compare(pversion, cversion)):
+                    return True
+        return False
+
+
+    def compatible(self, parent, child, **kwargs):
+        """Returns true iff a parent and child spec are ABI compatible"""
+        loosematch = kwargs.get('loose', False)
+        return self.architecture_compatible(parent, child) and \
+               self.compiler_compatible(parent, child, loose=loosematch)

lib/spack/spack/build_environment.py

@@ -177,8 +177,6 @@ def set_module_variables_for_package(pkg, m):
     """Populate the module scope of install() with some useful functions.
        This makes things easier for package writers.
     """
-    m = pkg.module
-
     # number of jobs spack will to build with.
     jobs = multiprocessing.cpu_count()
     if not pkg.parallel:
@@ -214,6 +212,13 @@ def set_module_variables_for_package(pkg, m):
     m.std_cmake_args.append('-DCMAKE_INSTALL_RPATH_USE_LINK_PATH=FALSE')
     m.std_cmake_args.append('-DCMAKE_INSTALL_RPATH=%s' % ":".join(get_rpaths(pkg)))
 
+    # Put spack compiler paths in module scope.
+    link_dir = spack.build_env_path
+    m.spack_cc  = join_path(link_dir, pkg.compiler.link_paths['cc'])
+    m.spack_cxx = join_path(link_dir, pkg.compiler.link_paths['cxx'])
+    m.spack_f77 = join_path(link_dir, pkg.compiler.link_paths['f77'])
+    m.spack_f90 = join_path(link_dir, pkg.compiler.link_paths['fc'])
+
     # Emulate some shell commands for convenience
     m.pwd          = os.getcwd
     m.cd           = os.chdir

lib/spack/spack/cmd/checksum.py

@@ -22,23 +22,18 @@
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 ##############################################################################
-import os
-import re
 import argparse
 import hashlib
-from pprint import pprint
-from subprocess import CalledProcessError
 
 import llnl.util.tty as tty
-from llnl.util.tty.colify import colify
-
 import spack
 import spack.cmd
 import spack.util.crypto
 from spack.stage import Stage, FailedDownloadError
 from spack.version import *
 
-description ="Checksum available versions of a package."
+description = "Checksum available versions of a package."
+
 
 def setup_parser(subparser):
     subparser.add_argument(
@@ -58,25 +53,23 @@ def get_checksums(versions, urls, **kwargs):
 
     tty.msg("Downloading...")
     hashes = []
-    for i, (url, version) in enumerate(zip(urls, versions)):
+    i = 0
-        stage = Stage(url)
+    for url, version in zip(urls, versions):
         try:
+            with Stage(url, keep=keep_stage) as stage:
                 stage.fetch()
                 if i == 0 and first_stage_function:
                     first_stage_function(stage)
 
-            hashes.append(
+                hashes.append((version,
-                spack.util.crypto.checksum(hashlib.md5, stage.archive_file))
+                               spack.util.crypto.checksum(hashlib.md5, stage.archive_file)))
-        except FailedDownloadError, e:
+                i += 1
+        except FailedDownloadError as e:
             tty.msg("Failed to fetch %s" % url)
-            continue
+        except Exception as e:
-
+            tty.msg('Something failed on %s, skipping.\n    (%s)' % (url, e))
-        finally:
-            if not keep_stage:
-                stage.destroy()
-
-    return zip(versions, hashes)
 
+    return hashes
 
 
 def checksum(parser, args):
@@ -95,11 +88,11 @@ def checksum(parser, args):
     else:
         versions = pkg.fetch_remote_versions()
         if not versions:
-            tty.die("Could not fetch any versions for %s." % pkg.name)
+            tty.die("Could not fetch any versions for %s" % pkg.name)
 
     sorted_versions = sorted(versions, reverse=True)
 
-    tty.msg("Found %s versions of %s." % (len(versions), pkg.name),
+    tty.msg("Found %s versions of %s" % (len(versions), pkg.name),
             *spack.cmd.elide_list(
                 ["%-10s%s" % (v, versions[v]) for v in sorted_versions]))
     print
@@ -116,7 +109,7 @@ def checksum(parser, args):
         keep_stage=args.keep_stage)
 
     if not version_hashes:
-        tty.die("Could not fetch any versions for %s." % pkg.name)
+        tty.die("Could not fetch any versions for %s" % pkg.name)
 
     version_lines = ["    version('%s', '%s')" % (v, h) for v, h in version_hashes]
     tty.msg("Checksummed new versions of %s:" % pkg.name, *version_lines)

lib/spack/spack/cmd/compiler.py

@@ -96,7 +96,7 @@ def compiler_remove(args):
     compilers = spack.compilers.compilers_for_spec(cspec, scope=args.scope)
 
     if not compilers:
-        tty.die("No compilers match spec %s." % cspec)
+        tty.die("No compilers match spec %s" % cspec)
     elif not args.all and len(compilers) > 1:
         tty.error("Multiple compilers match spec %s. Choose one:" % cspec)
         colify(reversed(sorted([c.spec for c in compilers])), indent=4)
@@ -105,7 +105,7 @@ def compiler_remove(args):
 
     for compiler in compilers:
         spack.compilers.remove_compiler_from_config(compiler.spec, scope=args.scope)
-        tty.msg("Removed compiler %s." % compiler.spec)
+        tty.msg("Removed compiler %s" % compiler.spec)
 
 
 def compiler_info(args):
@@ -114,7 +114,7 @@ def compiler_info(args):
     compilers = spack.compilers.compilers_for_spec(cspec, scope=args.scope)
 
     if not compilers:
-        tty.error("No compilers match spec %s." % cspec)
+        tty.error("No compilers match spec %s" % cspec)
     else:
         for c in compilers:
             print str(c.spec) + ":"

lib/spack/spack/cmd/create.py

@@ -156,7 +156,7 @@ def guess_name_and_version(url, args):
     # Try to deduce name and version of the new package from the URL
     version = spack.url.parse_version(url)
     if not version:
-        tty.die("Couldn't guess a version string from %s." % url)
+        tty.die("Couldn't guess a version string from %s" % url)
 
     # Try to guess a name.  If it doesn't work, allow the user to override.
     if args.alternate_name:
@@ -189,7 +189,7 @@ def find_repository(spec, args):
         try:
             repo = Repo(repo_path)
             if spec.namespace and spec.namespace != repo.namespace:
-                tty.die("Can't create package with namespace %s in repo with namespace %s."
+                tty.die("Can't create package with namespace %s in repo with namespace %s"
                         % (spec.namespace, repo.namespace))
         except RepoError as e:
             tty.die(str(e))
@@ -208,7 +208,7 @@ def find_repository(spec, args):
     return repo
 
 
-def fetch_tarballs(url, name, args):
+def fetch_tarballs(url, name, version):
     """Try to find versions of the supplied archive by scraping the web.
 
     Prompts the user to select how many to download if many are found.
@@ -252,11 +252,11 @@ def create(parser, args):
     name = spec.name  # factors out namespace, if any
     repo = find_repository(spec, args)
 
-    tty.msg("This looks like a URL for %s version %s." % (name, version))
+    tty.msg("This looks like a URL for %s version %s" % (name, version))
     tty.msg("Creating template for package %s" % name)
 
     # Fetch tarballs (prompting user if necessary)
-    versions, urls = fetch_tarballs(url, name, args)
+    versions, urls = fetch_tarballs(url, name, version)
 
     # Try to guess what configure system is used.
     guesser = ConfigureGuesser()
@@ -266,7 +266,7 @@ def create(parser, args):
         keep_stage=args.keep_stage)
 
     if not ver_hash_tuples:
-        tty.die("Could not fetch any tarballs for %s." % name)
+        tty.die("Could not fetch any tarballs for %s" % name)
 
     # Prepend 'py-' to python package names, by convention.
     if guesser.build_system == 'python':
@@ -291,4 +291,4 @@ def create(parser, args):
 
     # If everything checks out, go ahead and edit.
     spack.editor(pkg_path)
-    tty.msg("Created package %s." % pkg_path)
+    tty.msg("Created package %s" % pkg_path)

lib/spack/spack/cmd/diy.py

@@ -45,6 +45,9 @@ def setup_parser(subparser):
     subparser.add_argument(
         '--skip-patch', action='store_true',
         help="Skip patching for the DIY build.")
+    subparser.add_argument(
+        '-q', '--quiet', action='store_true', dest='quiet',
+        help="Do not display verbose build output while installing.")
     subparser.add_argument(
         'spec', nargs=argparse.REMAINDER,
         help="specs to use for install.  Must contain package AND verison.")
@@ -72,8 +75,8 @@ def diy(self, args):
                 edit_package(spec.name, spack.repo.first_repo(), None, True)
                 return
 
-        if not spec.version.concrete:
+        if not spec.versions.concrete:
-            tty.die("spack diy spec must have a single, concrete version.")
+            tty.die("spack diy spec must have a single, concrete version.  Did you forget a package version number?")
 
         spec.concretize()
         package = spack.repo.get(spec)
@@ -92,4 +95,5 @@ def diy(self, args):
         package.do_install(
             keep_prefix=args.keep_prefix,
             ignore_deps=args.ignore_deps,
+            verbose=not args.quiet,
             keep_stage=True)   # don't remove source dir for DIY.

lib/spack/spack/cmd/md5.py

@@ -22,51 +22,51 @@
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 ##############################################################################
-import os
 import argparse
 import hashlib
-
+import os
-from contextlib import contextmanager
 
 import llnl.util.tty as tty
-from llnl.util.filesystem import *
-
 import spack.util.crypto
 from spack.stage import Stage, FailedDownloadError
 
 description = "Calculate md5 checksums for files/urls."
 
-@contextmanager
-def stager(url):
-    _cwd = os.getcwd()
-    _stager = Stage(url)
-    try:
-        _stager.fetch()
-        yield _stager
-    except FailedDownloadError:
-        tty.msg("Failed to fetch %s" % url)
-    finally:
-        _stager.destroy()
-        os.chdir(_cwd) # the Stage class changes the current working dir so it has to be restored
 
 def setup_parser(subparser):
     setup_parser.parser = subparser
     subparser.add_argument('files', nargs=argparse.REMAINDER,
                            help="Files to checksum.")
 
+
+def compute_md5_checksum(url):
+    if not os.path.isfile(url):
+        with Stage(url) as stage:
+            stage.fetch()
+            value = spack.util.crypto.checksum(hashlib.md5, stage.archive_file)
+    else:
+        value = spack.util.crypto.checksum(hashlib.md5, url)
+    return value
+
+
 def md5(parser, args):
     if not args.files:
         setup_parser.parser.print_help()
         return 1
 
-    for f in args.files:
+    results = []
-        if not os.path.isfile(f):
+    for url in args.files:
-            with stager(f) as stage:
+        try:
-                checksum = spack.util.crypto.checksum(hashlib.md5, stage.archive_file)
+            checksum = compute_md5_checksum(url)
-                print "%s  %s" % (checksum, f)
+            results.append((checksum, url))
-        else:
+        except FailedDownloadError as e:
-            if not can_access(f):
+            tty.warn("Failed to fetch %s" % url)
-                tty.die("Cannot read file: %s" % f)
+            tty.warn("%s" % e)
+        except IOError as e:
+            tty.warn("Error when reading %s" % url)
+            tty.warn("%s" % e)
 
-            checksum = spack.util.crypto.checksum(hashlib.md5, f)
+    # Dump the MD5s at last without interleaving them with downloads
-            print "%s  %s" % (checksum, f)
+    tty.msg("%d MD5 checksums:" % len(results))
+    for checksum, url in results:
+        print "%s  %s" % (checksum, url)

lib/spack/spack/cmd/mirror.py

@@ -126,7 +126,7 @@ def mirror_remove(args):
 
     old_value = mirrors.pop(name)
     spack.config.update_config('mirrors', mirrors, scope=args.scope)
-    tty.msg("Removed mirror %s with url %s." % (name, old_value))
+    tty.msg("Removed mirror %s with url %s" % (name, old_value))
 
 
 def mirror_list(args):
@@ -203,7 +203,7 @@ def mirror_create(args):
 
     verb = "updated" if existed else "created"
     tty.msg(
-        "Successfully %s mirror in %s." % (verb, directory),
+        "Successfully %s mirror in %s" % (verb, directory),
         "Archive stats:",
         "  %-4d already present"  % p,
         "  %-4d added"            % m,

lib/spack/spack/cmd/module.py

@@ -58,7 +58,7 @@ def module_find(mtype, spec_array):
        should type to use that package's module.
     """
     if mtype not in module_types:
-        tty.die("Invalid module type: '%s'.  Options are %s." % (mtype, comma_or(module_types)))
+        tty.die("Invalid module type: '%s'.  Options are %s" % (mtype, comma_or(module_types)))
 
     specs = spack.cmd.parse_specs(spec_array)
     if len(specs) > 1:
@@ -78,7 +78,7 @@ def module_find(mtype, spec_array):
     mt = module_types[mtype]
     mod = mt(specs[0])
     if not os.path.isfile(mod.file_name):
-        tty.die("No %s module is installed for %s." % (mtype, spec))
+        tty.die("No %s module is installed for %s" % (mtype, spec))
 
     print mod.use_name
 
@@ -94,7 +94,7 @@ def module_refresh():
             shutil.rmtree(cls.path, ignore_errors=False)
         mkdirp(cls.path)
         for spec in specs:
-            tty.debug("   Writing file for %s." % spec)
+            tty.debug("   Writing file for %s" % spec)
             cls(spec).write()
 
 

lib/spack/spack/cmd/patch.py

@@ -24,6 +24,7 @@
 ##############################################################################
 import argparse
 
+import llnl.util.tty as tty
 import spack.cmd
 import spack
 

lib/spack/spack/cmd/repo.py

@@ -6,7 +6,7 @@
 # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
 # LLNL-CODE-647188
 #
-# For details, see https://llnl.github.io/spack
+# For details, see https://software.llnl.gov/spack
 # Please also see the LICENSE file for our notice and the LGPL.
 #
 # This program is free software; you can redistribute it and/or modify
@@ -74,51 +74,7 @@ def setup_parser(subparser):
 
 def repo_create(args):
     """Create a new package repository."""
-    root = canonicalize_path(args.directory)
+    full_path, namespace = create_repo(args.directory, args.namespace)
-    namespace = args.namespace
-
-    if not args.namespace:
-        namespace = os.path.basename(root)
-
-    if not re.match(r'\w[\.\w-]*', namespace):
-        tty.die("'%s' is not a valid namespace." % namespace)
-
-    existed = False
-    if os.path.exists(root):
-        if os.path.isfile(root):
-            tty.die('File %s already exists and is not a directory' % root)
-        elif os.path.isdir(root):
-            if not os.access(root, os.R_OK | os.W_OK):
-                tty.die('Cannot create new repo in %s: cannot access directory.' % root)
-            if os.listdir(root):
-                tty.die('Cannot create new repo in %s: directory is not empty.' % root)
-        existed = True
-
-    full_path = os.path.realpath(root)
-    parent = os.path.dirname(full_path)
-    if not os.access(parent, os.R_OK | os.W_OK):
-        tty.die("Cannot create repository in %s: can't access parent!" % root)
-
-    try:
-        config_path = os.path.join(root, repo_config_name)
-        packages_path = os.path.join(root, packages_dir_name)
-
-        mkdirp(packages_path)
-        with open(config_path, 'w') as config:
-            config.write("repo:\n")
-            config.write("  namespace: '%s'\n" % namespace)
-
-    except (IOError, OSError) as e:
-        tty.die('Failed to create new repository in %s.' % root,
-                "Caused by %s: %s" % (type(e), e))
-
-        # try to clean up.
-        if existed:
-            shutil.rmtree(config_path, ignore_errors=True)
-            shutil.rmtree(packages_path, ignore_errors=True)
-        else:
-            shutil.rmtree(root, ignore_errors=True)
-
     tty.msg("Created repo with namespace '%s'." % namespace)
     tty.msg("To register it with spack, run this command:",
             'spack repo add %s' % full_path)
@@ -133,11 +89,11 @@ def repo_add(args):
 
     # check if the path exists
     if not os.path.exists(canon_path):
-        tty.die("No such file or directory: '%s'." % path)
+        tty.die("No such file or directory: %s" % path)
 
     # Make sure the path is a directory.
     if not os.path.isdir(canon_path):
-        tty.die("Not a Spack repository: '%s'." % path)
+        tty.die("Not a Spack repository: %s" % path)
 
     # Make sure it's actually a spack repository by constructing it.
     repo = Repo(canon_path)
@@ -147,7 +103,7 @@ def repo_add(args):
     if not repos: repos = []
 
     if repo.root in repos or path in repos:
-        tty.die("Repository is already registered with Spack: '%s'" % path)
+        tty.die("Repository is already registered with Spack: %s" % path)
 
     repos.insert(0, canon_path)
     spack.config.update_config('repos', repos, args.scope)
@@ -166,7 +122,7 @@ def repo_remove(args):
         if canon_path == repo_canon_path:
             repos.remove(repo_path)
             spack.config.update_config('repos', repos, args.scope)
-            tty.msg("Removed repository '%s'." % repo_path)
+            tty.msg("Removed repository %s" % repo_path)
             return
 
     # If it is a namespace, remove corresponding repo
@@ -176,13 +132,13 @@ def repo_remove(args):
             if repo.namespace == path_or_namespace:
                 repos.remove(path)
                 spack.config.update_config('repos', repos, args.scope)
-                tty.msg("Removed repository '%s' with namespace %s."
+                tty.msg("Removed repository %s with namespace '%s'."
                         % (repo.root, repo.namespace))
                 return
         except RepoError as e:
             continue
 
-    tty.die("No repository with path or namespace: '%s'"
+    tty.die("No repository with path or namespace: %s"
             % path_or_namespace)
 
 

lib/spack/spack/compiler.py

@@ -256,12 +256,12 @@ def find(cls, *path):
 
 
     def __repr__(self):
-        """Return a string represntation of the compiler toolchain."""
+        """Return a string representation of the compiler toolchain."""
         return self.__str__()
 
 
     def __str__(self):
-        """Return a string represntation of the compiler toolchain."""
+        """Return a string representation of the compiler toolchain."""
         return "%s(%s)" % (
             self.name, '\n     '.join((str(s) for s in (self.cc, self.cxx, self.f77, self.fc))))
 

lib/spack/spack/compilers/__init__.py

@@ -74,28 +74,36 @@ def _to_dict(compiler):
 def get_compiler_config(arch=None, scope=None):
     """Return the compiler configuration for the specified architecture.
     """
-    # If any configuration file has compilers, just stick with the
+    # Check whether we're on a front-end (native) architecture.
-    # ones already configured.
-    config = spack.config.get_config('compilers', scope=scope)
-
     my_arch = spack.architecture.sys_type()
     if arch is None:
         arch = my_arch
 
-    if arch in config:
+    def init_compiler_config():
-        return config[arch]
+        """Compiler search used when Spack has no compilers."""
-
-    # Only for the current arch in *highest* scope: automatically try to
-    # find compilers if none are configured yet.
-    if arch == my_arch and scope == 'user':
         config[arch] = {}
         compilers = find_compilers(*get_path('PATH'))
         for compiler in compilers:
             config[arch].update(_to_dict(compiler))
         spack.config.update_config('compilers', config, scope=scope)
-        return config[arch]
 
-    return {}
+    config = spack.config.get_config('compilers', scope=scope)
+
+    # Update the configuration if there are currently no compilers
+    # configured.  Avoid updating automatically if there ARE site
+    # compilers configured but no user ones.
+    if arch == my_arch and arch not in config:
+        if scope is None:
+            # We know no compilers were configured in any scope.
+            init_compiler_config()
+        elif scope == 'user':
+            # Check the site config and update the user config if
+            # nothing is configured at the site level.
+            site_config = spack.config.get_config('compilers', scope='site')
+            if not site_config:
+                init_compiler_config()
+
+    return config[arch] if arch in config else {}
 
 
 def add_compilers_to_config(compilers, arch=None, scope=None):

lib/spack/spack/compilers/pgi.py

@@ -29,28 +29,28 @@ class Pgi(Compiler):
     cc_names = ['pgcc']
 
     # Subclasses use possible names of C++ compiler
-    cxx_names = ['pgCC']
+    cxx_names = ['pgc++', 'pgCC']
 
     # Subclasses use possible names of Fortran 77 compiler
-    f77_names = ['pgf77']
+    f77_names = ['pgfortran', 'pgf77']
 
     # Subclasses use possible names of Fortran 90 compiler
-    fc_names = ['pgf95', 'pgf90']
+    fc_names = ['pgfortran', 'pgf95', 'pgf90']
 
     # Named wrapper links within spack.build_env_path
     link_paths = { 'cc'  : 'pgi/pgcc',
-                   'cxx' : 'pgi/case-insensitive/pgCC',
+                   'cxx' : 'pgi/pgc++',
-                   'f77' : 'pgi/pgf77',
+                   'f77' : 'pgi/pgfortran',
-                   'fc'  : 'pgi/pgf90' }
+                   'fc'  : 'pgi/pgfortran' }
 
     @classmethod
     def default_version(cls, comp):
         """The '-V' option works for all the PGI compilers.
            Output looks like this::
 
-               pgf95 10.2-0 64-bit target on x86-64 Linux -tp nehalem-64
+               pgcc 15.10-0 64-bit target on x86-64 Linux -tp sandybridge
-               Copyright 1989-2000, The Portland Group, Inc.  All Rights Reserved.
+               The Portland Group - PGI Compilers and Tools
-               Copyright 2000-2010, STMicroelectronics, Inc.  All Rights Reserved.
+               Copyright (c) 2015, NVIDIA CORPORATION.  All rights reserved.
         """
         return get_compiler_version(
             comp, '-V', r'pg[^ ]* ([^ ]+) \d\d\d?-bit target')

lib/spack/spack/concretize.py

@@ -33,12 +33,16 @@
 TODO: make this customizable and allow users to configure
       concretization  policies.
 """
+import spack
 import spack.spec
 import spack.compilers
 import spack.architecture
 import spack.error
 from spack.version import *
-
+from functools import partial
+from spec import DependencyMap
+from itertools import chain
+from spack.config import *
 
 class DefaultConcretizer(object):
     """This class doesn't have any state, it just provides some methods for
@@ -46,10 +50,92 @@ class DefaultConcretizer(object):
        default concretization strategies, or you can override all of them.
     """
 
+    def _valid_virtuals_and_externals(self, spec):
+        """Returns a list of candidate virtual dep providers and external
+           packages that coiuld be used to concretize a spec."""
+        # First construct a list of concrete candidates to replace spec with.
+        candidates = [spec]
+        if spec.virtual:
+            providers = spack.repo.providers_for(spec)
+            if not providers:
+                raise UnsatisfiableProviderSpecError(providers[0], spec)
+            spec_w_preferred_providers = find_spec(
+                spec, lambda(x): spack.pkgsort.spec_has_preferred_provider(x.name, spec.name))
+            if not spec_w_preferred_providers:
+                spec_w_preferred_providers = spec
+            provider_cmp = partial(spack.pkgsort.provider_compare, spec_w_preferred_providers.name, spec.name)
+            candidates = sorted(providers, cmp=provider_cmp)
+
+        # For each candidate package, if it has externals, add those to the usable list.
+        # if it's not buildable, then *only* add the externals.
+        usable = []
+        for cspec in candidates:
+            if is_spec_buildable(cspec):
+                usable.append(cspec)
+            externals = spec_externals(cspec)
+            for ext in externals:
+                if ext.satisfies(spec):
+                    usable.append(ext)
+
+        # If nothing is in the usable list now, it's because we aren't
+        # allowed to build anything.
+        if not usable:
+            raise NoBuildError(spec)
+
+        def cmp_externals(a, b):
+            if a.name != b.name:
+                # We're choosing between different providers, so
+                # maintain order from provider sort
+                return candidates.index(a) - candidates.index(b)
+
+            result = cmp_specs(a, b)
+            if result != 0:
+                return result
+
+            # prefer external packages to internal packages.
+            if a.external is None or b.external is None:
+                return -cmp(a.external, b.external)
+            else:
+                return cmp(a.external, b.external)
+
+        usable.sort(cmp=cmp_externals)
+        return usable
+
+
+    def choose_virtual_or_external(self, spec):
+        """Given a list of candidate virtual and external packages, try to
+           find one that is most ABI compatible.
+        """
+        candidates = self._valid_virtuals_and_externals(spec)
+        if not candidates:
+            return candidates
+
+        # Find the nearest spec in the dag that has a compiler.  We'll
+        # use that spec to calibrate compiler compatibility.
+        abi_exemplar = find_spec(spec, lambda(x): x.compiler)
+        if not abi_exemplar:
+            abi_exemplar = spec.root
+
+        # Make a list including ABI compatibility of specs with the exemplar.
+        strict = [spack.abi.compatible(c, abi_exemplar) for c in candidates]
+        loose  = [spack.abi.compatible(c, abi_exemplar, loose=True) for c in candidates]
+        keys = zip(strict, loose, candidates)
+
+        # Sort candidates from most to least compatibility.
+        # Note:
+        #   1. We reverse because True > False.
+        #   2. Sort is stable, so c's keep their order.
+        keys.sort(key=lambda k:k[:2], reverse=True)
+
+        # Pull the candidates back out and return them in order
+        candidates = [c for s,l,c in keys]
+        return candidates
+
+
     def concretize_version(self, spec):
         """If the spec is already concrete, return.  Otherwise take
-           the most recent available version, and default to the package's
+           the preferred version from spackconfig, and default to the package's
-           version if there are no avaialble versions.
+           version if there are no available versions.
 
            TODO: In many cases we probably want to look for installed
                  versions of each package and use an installed version
@@ -67,20 +153,14 @@ def concretize_version(self, spec):
         # If there are known available versions, return the most recent
         # version that satisfies the spec
         pkg = spec.package
-
+        cmp_versions = partial(spack.pkgsort.version_compare, spec.name)
-        # Key function to sort versions first by whether they were
-        # marked `preferred=True`, then by most recent.
-        def preferred_key(v):
-            prefer = pkg.versions[v].get('preferred', False)
-            return (prefer, v)
-
         valid_versions = sorted(
             [v for v in pkg.versions
              if any(v.satisfies(sv) for sv in spec.versions)],
-            key=preferred_key)
+            cmp=cmp_versions)
 
         if valid_versions:
-            spec.versions = ver([valid_versions[-1]])
+            spec.versions = ver([valid_versions[0]])
         else:
             # We don't know of any SAFE versions that match the given
             # spec.  Grab the spec's versions and grab the highest
@@ -134,7 +214,7 @@ def concretize_variants(self, spec):
            the default variants from the package specification.
         """
         changed = False
-        for name, variant in spec.package.variants.items():
+        for name, variant in spec.package_class.variants.items():
             if name not in spec.variants:
                 spec.variants[name] = spack.spec.VariantSpec(name, variant.default)
                 changed = True
@@ -145,10 +225,10 @@ def concretize_compiler(self, spec):
         """If the spec already has a compiler, we're done.  If not, then take
            the compiler used for the nearest ancestor with a compiler
            spec and use that.  If the ancestor's compiler is not
-           concrete, then give it a valid version.  If there is no
+           concrete, then used the preferred compiler as specified in
-           ancestor with a compiler, use the system default compiler.
+           spackconfig.
 
-           Intuition: Use the system default if no package that depends on
+           Intuition: Use the spackconfig default if no package that depends on
            this one has a strict compiler requirement.  Otherwise, try to
            build with the compiler that will be used by libraries that
            link to this one, to maximize compatibility.
@@ -160,40 +240,91 @@ def concretize_compiler(self, spec):
             spec.compiler in all_compilers):
             return False
 
-        try:
+        #Find the another spec that has a compiler, or the root if none do
-            nearest = next(p for p in spec.traverse(direction='parents')
+        other_spec = find_spec(spec, lambda(x) : x.compiler)
-                           if p.compiler is not None).compiler
+        if not other_spec:
+            other_spec = spec.root
+        other_compiler = other_spec.compiler
+        assert(other_spec)
 
-            if not nearest in all_compilers:
+        # Check if the compiler is already fully specified
-                # Take the newest compiler that saisfies the spec
+        if other_compiler in all_compilers:
-                matches = sorted(spack.compilers.find(nearest))
+            spec.compiler = other_compiler.copy()
+            return True
+
+        # Filter the compilers into a sorted list based on the compiler_order from spackconfig
+        compiler_list = all_compilers if not other_compiler else spack.compilers.find(other_compiler)
+        cmp_compilers = partial(spack.pkgsort.compiler_compare, other_spec.name)
+        matches = sorted(compiler_list, cmp=cmp_compilers)
         if not matches:
-                    raise UnavailableCompilerVersionError(nearest)
+            raise UnavailableCompilerVersionError(other_compiler)
-
-                # copy concrete version into nearest spec
-                nearest.versions = matches[-1].versions.copy()
-                assert(nearest.concrete)
-
-            spec.compiler = nearest.copy()
-
-        except StopIteration:
-            spec.compiler = spack.compilers.default_compiler().copy()
 
+        # copy concrete version into other_compiler
+        spec.compiler = matches[0].copy()
+        assert(spec.compiler.concrete)
         return True  # things changed.
 
 
-    def choose_provider(self, spec, providers):
+def find_spec(spec, condition):
-        """This is invoked for virtual specs.  Given a spec with a virtual name,
+    """Searches the dag from spec in an intelligent order and looks
-           say "mpi", and a list of specs of possible providers of that spec,
+       for a spec that matches a condition"""
-           select a provider and return it.
+    # First search parents, then search children
-        """
+    dagiter = chain(spec.traverse(direction='parents',  root=False),
-        assert(spec.virtual)
+                    spec.traverse(direction='children', root=False))
-        assert(providers)
+    visited = set()
+    for relative in dagiter:
+        if condition(relative):
+            return relative
+        visited.add(id(relative))
+
+    # Then search all other relatives in the DAG *except* spec
+    for relative in spec.root.traverse():
+        if relative is spec: continue
+        if id(relative) in visited: continue
+        if condition(relative):
+            return relative
+
+    # Finally search spec itself.
+    if condition(spec):
+        return spec
+
+    return None   # Nohting matched the condition.
+
+
+def cmp_specs(lhs, rhs):
+    # Package name sort order is not configurable, always goes alphabetical
+    if lhs.name != rhs.name:
+        return cmp(lhs.name, rhs.name)
+
+    # Package version is second in compare order
+    pkgname = lhs.name
+    if lhs.versions != rhs.versions:
+        return spack.pkgsort.version_compare(
+            pkgname, lhs.versions, rhs.versions)
+
+    # Compiler is third
+    if lhs.compiler != rhs.compiler:
+        return spack.pkgsort.compiler_compare(
+            pkgname, lhs.compiler, rhs.compiler)
+
+    # Variants
+    if lhs.variants != rhs.variants:
+        return spack.pkgsort.variant_compare(
+            pkgname, lhs.variants, rhs.variants)
+
+    # Architecture
+    if lhs.architecture != rhs.architecture:
+        return spack.pkgsort.architecture_compare(
+            pkgname, lhs.architecture, rhs.architecture)
+
+    # Dependency is not configurable
+    lhash, rhash = hash(lhs), hash(rhs)
+    if lhash != rhash:
+        return -1 if lhash < rhash else 1
+
+    # Equal specs
+    return 0
 
-        index = spack.spec.index_specs(providers)
-        first_key = sorted(index.keys())[0]
-        latest_version = sorted(index[first_key])[-1]
-        return latest_version
 
 
 class UnavailableCompilerVersionError(spack.error.SpackError):
@@ -211,3 +342,11 @@ class NoValidVersionError(spack.error.SpackError):
     def __init__(self, spec):
         super(NoValidVersionError, self).__init__(
             "There are no valid versions for %s that match '%s'" % (spec.name, spec.versions))
+
+
+class NoBuildError(spack.error.SpackError):
+    """Raised when a package is configured with the buildable option False, but
+       no satisfactory external versions can be found"""
+    def __init__(self, spec):
+        super(NoBuildError, self).__init__(
+            "The spec '%s' is configured as not buildable, and no matching external installs were found" % spec.name)

lib/spack/spack/config.py

@@ -129,6 +129,7 @@
 
 import llnl.util.tty as tty
 from llnl.util.filesystem import mkdirp
+import copy
 
 import spack
 from spack.error import SpackError
@@ -194,6 +195,49 @@
                 'default': [],
                 'items': {
                     'type': 'string'},},},},
+    'packages': {
+        '$schema': 'http://json-schema.org/schema#',
+        'title': 'Spack package configuration file schema',
+        'type': 'object',
+        'additionalProperties': False,
+        'patternProperties': {
+            r'packages:?': {
+                'type': 'object',
+                'default': {},
+                'additionalProperties': False,
+                'patternProperties': {
+                    r'\w[\w-]*': { # package name
+                        'type': 'object',
+                        'default': {},
+                        'additionalProperties': False,
+                        'properties': {
+                            'version': {
+                                'type' : 'array',
+                                'default' : [],
+                                'items' : { 'anyOf' : [ { 'type' : 'string' },
+                                                        { 'type' : 'number'}]}}, #version strings
+                            'compiler': {
+                                'type' : 'array',
+                                'default' : [],
+                                'items' : { 'type' : 'string' } }, #compiler specs
+                            'buildable': {
+                                'type':  'boolean',
+                                'default': True,
+                             },
+                            'providers': {
+                                'type':  'object',
+                                'default': {},
+                                'additionalProperties': False,
+                                'patternProperties': {
+                                    r'\w[\w-]*': {
+                                        'type' : 'array',
+                                        'default' : [],
+                                        'items' : { 'type' : 'string' },},},},
+                            'paths': {
+                                'type' : 'object',
+                                'default' : {},
+                            }
+                        },},},},},}
 }
 
 """OrderedDict of config scopes keyed by name.
@@ -205,7 +249,7 @@
 def validate_section_name(section):
     """Raise a ValueError if the section is not a valid section."""
     if section not in section_schemas:
-        raise ValueError("Invalid config section: '%s'.  Options are %s."
+        raise ValueError("Invalid config section: '%s'.  Options are %s"
                          % (section, section_schemas))
 
 
@@ -335,7 +379,7 @@ def validate_scope(scope):
         return config_scopes[scope]
 
     else:
-        raise ValueError("Invalid config scope: '%s'.  Must be one of %s."
+        raise ValueError("Invalid config scope: '%s'.  Must be one of %s"
                          % (scope, config_scopes.keys()))
 
 
@@ -350,7 +394,7 @@ def _read_config_file(filename, schema):
             "Invlaid configuration. %s exists but is not a file." % filename)
 
     elif not os.access(filename, os.R_OK):
-        raise ConfigFileError("Config file is not readable: %s." % filename)
+        raise ConfigFileError("Config file is not readable: %s" % filename)
 
     try:
         tty.debug("Reading config file %s" % filename)
@@ -494,6 +538,39 @@ def print_section(section):
         raise ConfigError("Error reading configuration: %s" % section)
 
 
+def spec_externals(spec):
+    """Return a list of external specs (with external directory path filled in),
+       one for each known external installation."""
+    allpkgs = get_config('packages')
+    name = spec.name
+
+    external_specs = []
+    pkg_paths = allpkgs.get(name, {}).get('paths', None)
+    if not pkg_paths:
+        return []
+
+    for external_spec, path in pkg_paths.iteritems():
+        if not path:
+            # skip entries without paths (avoid creating extra Specs)
+            continue
+
+        external_spec = spack.spec.Spec(external_spec, external=path)
+        if external_spec.satisfies(spec):
+            external_specs.append(external_spec)
+    return external_specs
+
+
+def is_spec_buildable(spec):
+    """Return true if the spec pkgspec is configured as buildable"""
+    allpkgs = get_config('packages')
+    name = spec.name
+    if not spec.name in allpkgs:
+        return True
+    if not 'buildable' in allpkgs[spec.name]:
+        return True
+    return allpkgs[spec.name]['buildable']
+
+
 class ConfigError(SpackError): pass
 class ConfigFileError(ConfigError): pass
 
@@ -509,7 +586,7 @@ def __init__(self, validation_error, data):
         # Try to get line number from erroneous instance and its parent
         instance_mark = getattr(validation_error.instance, '_start_mark', None)
         parent_mark = getattr(validation_error.parent, '_start_mark', None)
-        path = getattr(validation_error, 'path', None)
+        path = [str(s) for s in getattr(validation_error, 'path', None)]
 
         # Try really hard to get the parent (which sometimes is not
         # set) This digs it out of the validated structure if it's not

lib/spack/spack/database.py

@@ -330,7 +330,7 @@ def _check_ref_counts(self):
             found = rec.ref_count
             if not expected == found:
                 raise AssertionError(
-                    "Invalid ref_count: %s: %d (expected %d), in DB %s."
+                    "Invalid ref_count: %s: %d (expected %d), in DB %s"
                     % (key, found, expected, self._index_path))
 
 

lib/spack/spack/directives.py

@@ -125,7 +125,7 @@ def __init__(self, dicts=None):
             dicts = (dicts,)
         elif type(dicts) not in (list, tuple):
             raise TypeError(
-                "dicts arg must be list, tuple, or string. Found %s."
+                "dicts arg must be list, tuple, or string. Found %s"
                 % type(dicts))
 
         self.dicts = dicts
@@ -174,7 +174,11 @@ def version(pkg, ver, checksum=None, **kwargs):
 
 
 def _depends_on(pkg, spec, when=None):
-    if when is None:
+    # If when is False do nothing
+    if when is False:
+        return
+    # If when is None or True make sure the condition is always satisfied
+    if when is None or when is True:
         when = pkg.name
     when_spec = parse_anonymous_spec(when, pkg.name)
 
@@ -296,8 +300,8 @@ def resource(pkg, **kwargs):
         raise RuntimeError(message)
     when_spec = parse_anonymous_spec(when, pkg.name)
     resources = pkg.resources.setdefault(when_spec, [])
-    fetcher = from_kwargs(**kwargs)
     name = kwargs.get('name')
+    fetcher = from_kwargs(**kwargs)
     resources.append(Resource(name, fetcher, destination, placement))
 
 
@@ -313,5 +317,5 @@ class CircularReferenceError(DirectiveError):
     def __init__(self, directive, package):
         super(CircularReferenceError, self).__init__(
             directive,
-            "Package '%s' cannot pass itself to %s." % (package, directive))
+            "Package '%s' cannot pass itself to %s" % (package, directive))
         self.package = package

lib/spack/spack/directory_layout.py

@@ -85,6 +85,16 @@ def create_install_directory(self, spec):
         raise NotImplementedError()
 
 
+    def check_installed(self, spec):
+        """Checks whether a spec is installed.
+
+        Return the spec's prefix, if it is installed, None otherwise.
+
+        Raise an exception if the install is inconsistent or corrupt.
+        """
+        raise NotImplementedError()
+
+
     def extension_map(self, spec):
         """Get a dict of currently installed extension packages for a spec.
 
@@ -173,7 +183,9 @@ def __init__(self, root, **kwargs):
 
         self.spec_file_name      = 'spec.yaml'
         self.extension_file_name = 'extensions.yaml'
-        self.build_log_name      = 'build.out' # TODO: use config file.
+        self.build_log_name      = 'build.out'  # build log.
+        self.build_env_name      = 'build.env'  # build environment
+        self.packages_dir        = 'repos'      # archive of package.py files
 
         # Cache of already written/read extension maps.
         self._extension_maps = {}
@@ -186,6 +198,10 @@ def hidden_file_paths(self):
 
     def relative_path_for_spec(self, spec):
         _check_concrete(spec)
+
+        if spec.external:
+            return spec.external
+
         dir_name = "%s-%s-%s" % (
             spec.name,
             spec.version,
@@ -231,20 +247,43 @@ def build_log_path(self, spec):
                          self.build_log_name)
 
 
+    def build_env_path(self, spec):
+        return join_path(self.path_for_spec(spec), self.metadata_dir,
+                         self.build_env_name)
+
+
+    def build_packages_path(self, spec):
+        return join_path(self.path_for_spec(spec), self.metadata_dir,
+                         self.packages_dir)
+
+
     def create_install_directory(self, spec):
         _check_concrete(spec)
 
+        prefix = self.check_installed(spec)
+        if prefix:
+            raise InstallDirectoryAlreadyExistsError(prefix)
+
+        mkdirp(self.metadata_path(spec))
+        self.write_spec(spec, self.spec_file_path(spec))
+
+
+    def check_installed(self, spec):
+        _check_concrete(spec)
         path = self.path_for_spec(spec)
         spec_file_path = self.spec_file_path(spec)
 
-        if os.path.isdir(path):
+        if not os.path.isdir(path):
+            return None
+
         if not os.path.isfile(spec_file_path):
             raise InconsistentInstallDirectoryError(
-                    'No spec file found at path %s' % spec_file_path)
+                'Inconsistent state: install prefix exists but contains no spec.yaml:',
+                "  " + path)
 
         installed_spec = self.read_spec(spec_file_path)
-            if installed_spec == self.spec:
+        if installed_spec == spec:
-                raise InstallDirectoryAlreadyExistsError(path)
+            return path
 
         if spec.dag_hash() == installed_spec.dag_hash():
             raise SpecHashCollisionError(installed_hash, spec_hash)
@@ -252,9 +291,6 @@ def create_install_directory(self, spec):
             raise InconsistentInstallDirectoryError(
                 'Spec file in %s does not match hash!' % spec_file_path)
 
-        mkdirp(self.metadata_path(spec))
-        self.write_spec(spec, spec_file_path)
-
 
     def all_specs(self):
         if not os.path.isdir(self.root):
@@ -323,7 +359,7 @@ def _extension_map(self, spec):
 
                         if not dag_hash in by_hash:
                             raise InvalidExtensionSpecError(
-                                "Spec %s not found in %s." % (dag_hash, prefix))
+                                "Spec %s not found in %s" % (dag_hash, prefix))
 
                         ext_spec = by_hash[dag_hash]
                         if not prefix == ext_spec.prefix:
@@ -387,8 +423,8 @@ def remove_extension(self, spec, ext_spec):
 
 class DirectoryLayoutError(SpackError):
     """Superclass for directory layout errors."""
-    def __init__(self, message):
+    def __init__(self, message, long_msg=None):
-        super(DirectoryLayoutError, self).__init__(message)
+        super(DirectoryLayoutError, self).__init__(message, long_msg)
 
 
 class SpecHashCollisionError(DirectoryLayoutError):
@@ -410,8 +446,8 @@ def __init__(self, installed_spec, prefix, error):
 
 class InconsistentInstallDirectoryError(DirectoryLayoutError):
     """Raised when a package seems to be installed to the wrong place."""
-    def __init__(self, message):
+    def __init__(self, message, long_msg=None):
-        super(InconsistentInstallDirectoryError, self).__init__(message)
+        super(InconsistentInstallDirectoryError, self).__init__(message, long_msg)
 
 
 class InstallDirectoryAlreadyExistsError(DirectoryLayoutError):
@@ -438,7 +474,7 @@ class ExtensionConflictError(DirectoryLayoutError):
     """Raised when an extension is added to a package that already has it."""
     def __init__(self, spec, ext_spec, conflict):
         super(ExtensionConflictError, self).__init__(
-            "%s cannot be installed in %s because it conflicts with %s."% (
+            "%s cannot be installed in %s because it conflicts with %s"% (
                 ext_spec.short_spec, spec.short_spec, conflict.short_spec))
 
 

lib/spack/spack/fetch_strategy.py

@@ -44,6 +44,7 @@
 import sys
 import re
 import shutil
+import copy
 from functools import wraps
 import llnl.util.tty as tty
 from llnl.util.filesystem import *
@@ -55,17 +56,22 @@
 from spack.version import Version, ver
 from spack.util.compression import decompressor_for, extension
 
+import spack.util.pattern as pattern
+
 """List of all fetch strategies, created by FetchStrategy metaclass."""
 all_strategies = []
 
+
 def _needs_stage(fun):
     """Many methods on fetch strategies require a stage to be set
        using set_stage().  This decorator adds a check for self.stage."""
+
     @wraps(fun)
     def wrapper(self, *args, **kwargs):
         if not self.stage:
             raise NoStageError(fun)
         return fun(self, *args, **kwargs)
+
     return wrapper
 
 
@@ -80,23 +86,23 @@ def __init__(cls, name, bases, dict):
             type.__init__(cls, name, bases, dict)
             if cls.enabled: all_strategies.append(cls)
 
-
     def __init__(self):
         # The stage is initialized late, so that fetch strategies can be constructed
         # at package construction time.  This is where things will be fetched.
         self.stage = None
 
-
     def set_stage(self, stage):
         """This is called by Stage before any of the fetching
            methods are called on the stage."""
         self.stage = stage
 
-
     # Subclasses need to implement these methods
     def fetch(self):   pass  # Return True on success, False on fail.
+
     def check(self):   pass  # Do checksum.
+
     def expand(self):  pass  # Expand archive.
+
     def reset(self):   pass  # Revert to freshly downloaded state.
 
     def archive(self, destination): pass  # Used to create tarball for mirror.
@@ -111,6 +117,15 @@ def matches(cls, args):
         return any(k in args for k in cls.required_attributes)
 
 
+@pattern.composite(interface=FetchStrategy)
+class FetchStrategyComposite(object):
+    """
+    Composite for a FetchStrategy object. Implements the GoF composite pattern.
+    """
+    matches = FetchStrategy.matches
+    set_stage = FetchStrategy.set_stage
+
+
 class URLFetchStrategy(FetchStrategy):
     """FetchStrategy that pulls source code from a URL for an archive,
        checks the archive against a checksum,and decompresses the archive.
@@ -129,6 +144,8 @@ def __init__(self, url=None, digest=None, **kwargs):
         self.digest = kwargs.get('md5', None)
         if not self.digest: self.digest = digest
 
+        self.expand_archive = kwargs.get('expand', True)
+
         if not self.url:
             raise ValueError("URLFetchStrategy requires a url for fetching.")
 
@@ -137,7 +154,7 @@ def fetch(self):
         self.stage.chdir()
 
         if self.archive_file:
-            tty.msg("Already downloaded %s." % self.archive_file)
+            tty.msg("Already downloaded %s" % self.archive_file)
             return
 
         tty.msg("Trying to fetch from %s" % self.url)
@@ -145,7 +162,7 @@ def fetch(self):
         curl_args = ['-O',  # save file to disk
                      '-f',  # fail on >400 errors
                      '-D', '-',  # print out HTML headers
-                     '-L', self.url,]
+                     '-L', self.url, ]
 
         if sys.stdout.isatty():
             curl_args.append('-#')  # status bar when using a tty
@@ -182,7 +199,6 @@ def fetch(self):
                 raise FailedDownloadError(
                         self.url, "Curl failed with error %d" % spack.curl.returncode)
 
-
         # Check if we somehow got an HTML file rather than the archive we
         # asked for.  We only look at the last content type, to handle
         # redirects properly.
@@ -196,7 +212,6 @@ def fetch(self):
         if not self.archive_file:
             raise FailedDownloadError(self.url)
 
-
     @property
     def archive_file(self):
         """Path to the source archive within this stage directory."""
@@ -204,6 +219,10 @@ def archive_file(self):
 
     @_needs_stage
     def expand(self):
+        if not self.expand_archive:
+            tty.msg("Skipping expand step for %s" % self.archive_file)
+            return
+
         tty.msg("Staging archive: %s" % self.archive_file)
 
         self.stage.chdir()
@@ -241,7 +260,6 @@ def expand(self):
         # Set the wd back to the stage when done.
         self.stage.chdir()
 
-
     def archive(self, destination):
         """Just moves this archive to the destination."""
         if not self.archive_file:
@@ -252,7 +270,6 @@ def archive(self, destination):
 
         shutil.move(self.archive_file, destination)
 
-
     @_needs_stage
     def check(self):
         """Check the downloaded archive against a checksum digest.
@@ -263,9 +280,8 @@ def check(self):
         checker = crypto.Checker(self.digest)
         if not checker.check(self.archive_file):
             raise ChecksumError(
-                "%s checksum failed for %s." % (checker.hash_name, self.archive_file),
+                    "%s checksum failed for %s" % (checker.hash_name, self.archive_file),
-                "Expected %s but got %s." % (self.digest, checker.sum))
+                    "Expected %s but got %s" % (self.digest, checker.sum))
-
 
     @_needs_stage
     def reset(self):
@@ -277,12 +293,10 @@ def reset(self):
             shutil.rmtree(self.stage.source_path, ignore_errors=True)
         self.expand()
 
-
     def __repr__(self):
         url = self.url if self.url else "no url"
         return "URLFetchStrategy<%s>" % url
 
-
     def __str__(self):
         if self.url:
             return self.url
@@ -303,28 +317,25 @@ def __init__(self, name, *rev_types, **kwargs):
         # Ensure that there's only one of the rev_types
         if sum(k in kwargs for k in rev_types) > 1:
             raise FetchStrategyError(
-                "Supply only one of %s to fetch with %s." % (
+                    "Supply only one of %s to fetch with %s" % (
                         comma_or(rev_types), name))
 
         # Set attributes for each rev type.
         for rt in rev_types:
             setattr(self, rt, kwargs.get(rt, None))
 
-
     @_needs_stage
     def check(self):
-        tty.msg("No checksum needed when fetching with %s." % self.name)
+        tty.msg("No checksum needed when fetching with %s" % self.name)
-
 
     @_needs_stage
     def expand(self):
         tty.debug("Source fetched with %s is already expanded." % self.name)
 
-
     @_needs_stage
     def archive(self, destination, **kwargs):
-        assert(extension(destination) == 'tar.gz')
+        assert (extension(destination) == 'tar.gz')
-        assert(self.stage.source_path.startswith(self.stage.path))
+        assert (self.stage.source_path.startswith(self.stage.path))
 
         tar = which('tar', required=True)
 
@@ -338,16 +349,13 @@ def archive(self, destination, **kwargs):
         self.stage.chdir()
         tar('-czf', destination, os.path.basename(self.stage.source_path))
 
-
     def __str__(self):
         return "VCS: %s" % self.url
 
-
     def __repr__(self):
         return "%s<%s>" % (self.__class__, self.url)
 
 
-
 class GitFetchStrategy(VCSFetchStrategy):
     """Fetch strategy that gets source code from a git repository.
        Use like this in a package:
@@ -368,30 +376,31 @@ class GitFetchStrategy(VCSFetchStrategy):
     required_attributes = ('git',)
 
     def __init__(self, **kwargs):
-        super(GitFetchStrategy, self).__init__(
+        # Discards the keywords in kwargs that may conflict with the next call to __init__
-            'git', 'tag', 'branch', 'commit', **kwargs)
+        forwarded_args = copy.copy(kwargs)
-        self._git = None
+        forwarded_args.pop('name', None)
 
+        super(GitFetchStrategy, self).__init__(
+                'git', 'tag', 'branch', 'commit', **forwarded_args)
+        self._git = None
 
     @property
     def git_version(self):
         vstring = self.git('--version', output=str).lstrip('git version ')
         return Version(vstring)
 
-
     @property
     def git(self):
         if not self._git:
             self._git = which('git', required=True)
         return self._git
 
-
     @_needs_stage
     def fetch(self):
         self.stage.chdir()
 
         if self.stage.source_path:
-            tty.msg("Already fetched %s." % self.stage.source_path)
+            tty.msg("Already fetched %s" % self.stage.source_path)
             return
 
         args = []
@@ -429,7 +438,7 @@ def fetch(self):
             # Yet more efficiency, only download a 1-commit deep tree
             if self.git_version >= ver('1.7.1'):
                 try:
-                    self.git(*(args + ['--depth','1', self.url]))
+                    self.git(*(args + ['--depth', '1', self.url]))
                     cloned = True
                 except spack.error.SpackError:
                     # This will fail with the dumb HTTP transport
@@ -452,18 +461,15 @@ def fetch(self):
                 self.git('pull', '--tags', ignore_errors=1)
                 self.git('checkout', self.tag)
 
-
     def archive(self, destination):
         super(GitFetchStrategy, self).archive(destination, exclude='.git')
 
-
     @_needs_stage
     def reset(self):
         self.stage.chdir_to_source()
         self.git('checkout', '.')
         self.git('clean', '-f')
 
-
     def __str__(self):
         return "[git] %s" % self.url
 
@@ -483,26 +489,28 @@ class SvnFetchStrategy(VCSFetchStrategy):
     required_attributes = ['svn']
 
     def __init__(self, **kwargs):
+        # Discards the keywords in kwargs that may conflict with the next call to __init__
+        forwarded_args = copy.copy(kwargs)
+        forwarded_args.pop('name', None)
+
         super(SvnFetchStrategy, self).__init__(
-            'svn', 'revision', **kwargs)
+                'svn', 'revision', **forwarded_args)
         self._svn = None
         if self.revision is not None:
             self.revision = str(self.revision)
 
-
     @property
     def svn(self):
         if not self._svn:
             self._svn = which('svn', required=True)
         return self._svn
 
-
     @_needs_stage
     def fetch(self):
         self.stage.chdir()
 
         if self.stage.source_path:
-            tty.msg("Already fetched %s." % self.stage.source_path)
+            tty.msg("Already fetched %s" % self.stage.source_path)
             return
 
         tty.msg("Trying to check out svn repository: %s" % self.url)
@@ -515,7 +523,6 @@ def fetch(self):
         self.svn(*args)
         self.stage.chdir_to_source()
 
-
     def _remove_untracked_files(self):
         """Removes untracked files in an svn repository."""
         status = self.svn('status', '--no-ignore', output=str)
@@ -529,23 +536,19 @@ def _remove_untracked_files(self):
             elif os.path.isdir(path):
                 shutil.rmtree(path, ignore_errors=True)
 
-
     def archive(self, destination):
         super(SvnFetchStrategy, self).archive(destination, exclude='.svn')
 
-
     @_needs_stage
     def reset(self):
         self.stage.chdir_to_source()
         self._remove_untracked_files()
         self.svn('revert', '.', '-R')
 
-
     def __str__(self):
         return "[svn] %s" % self.url
 
 
-
 class HgFetchStrategy(VCSFetchStrategy):
     """Fetch strategy that gets source code from a Mercurial repository.
        Use like this in a package:
@@ -567,10 +570,13 @@ class HgFetchStrategy(VCSFetchStrategy):
     required_attributes = ['hg']
 
     def __init__(self, **kwargs):
-        super(HgFetchStrategy, self).__init__(
+        # Discards the keywords in kwargs that may conflict with the next call to __init__
-            'hg', 'revision', **kwargs)
+        forwarded_args = copy.copy(kwargs)
-        self._hg = None
+        forwarded_args.pop('name', None)
 
+        super(HgFetchStrategy, self).__init__(
+                'hg', 'revision', **forwarded_args)
+        self._hg = None
 
     @property
     def hg(self):
@@ -583,7 +589,7 @@ def fetch(self):
         self.stage.chdir()
 
         if self.stage.source_path:
-            tty.msg("Already fetched %s." % self.stage.source_path)
+            tty.msg("Already fetched %s" % self.stage.source_path)
             return
 
         args = []
@@ -597,11 +603,9 @@ def fetch(self):
 
         self.hg(*args)
 
-
     def archive(self, destination):
         super(HgFetchStrategy, self).archive(destination, exclude='.hg')
 
-
     @_needs_stage
     def reset(self):
         self.stage.chdir()
@@ -619,7 +623,6 @@ def reset(self):
         shutil.move(scrubbed, source_path)
         self.stage.chdir_to_source()
 
-
     def __str__(self):
         return "[hg] %s" % self.url
 
@@ -693,6 +696,7 @@ def __init__(self, msg, long_msg=None):
 
 class FailedDownloadError(FetchError):
     """Raised wen a download fails."""
+
     def __init__(self, url, msg=""):
         super(FailedDownloadError, self).__init__(
                 "Failed to fetch file from URL: %s" % url, msg)
@@ -718,12 +722,14 @@ def __init__(self, pkg, version):
 
 class ChecksumError(FetchError):
     """Raised when archive fails to checksum."""
+
     def __init__(self, message, long_msg=None):
         super(ChecksumError, self).__init__(message, long_msg)
 
 
 class NoStageError(FetchError):
     """Raised when fetch operations are called before set_stage()."""
+
     def __init__(self, method):
         super(NoStageError, self).__init__(
                 "Must call FetchStrategy.set_stage() before calling %s" % method.__name__)

lib/spack/spack/hooks/sbang.py

@@ -0,0 +1,98 @@
+##############################################################################
+# Copyright (c) 2013-2015, Lawrence Livermore National Security, LLC.
+# Produced at the Lawrence Livermore National Laboratory.
+#
+# This file is part of Spack.
+# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
+# LLNL-CODE-647188
+#
+# For details, see https://github.com/llnl/spack
+# Please also see the LICENSE file for our notice and the LGPL.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License (as published by
+# the Free Software Foundation) version 2.1 dated February 1999.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
+# conditions of the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+##############################################################################
+import os
+
+from llnl.util.filesystem import *
+import llnl.util.tty as tty
+
+import spack
+import spack.modules
+
+# Character limit for shebang line.  Using Linux's 127 characters
+# here, as it is the shortest I could find on a modern OS.
+shebang_limit = 127
+
+def shebang_too_long(path):
+    """Detects whether a file has a shebang line that is too long."""
+    with open(path, 'r') as script:
+        bytes = script.read(2)
+        if bytes != '#!':
+            return False
+
+        line = bytes + script.readline()
+        return len(line) > shebang_limit
+
+
+def filter_shebang(path):
+    """Adds a second shebang line, using sbang, at the beginning of a file."""
+    with open(path, 'r') as original_file:
+        original = original_file.read()
+
+    # This line will be prepended to file
+    new_sbang_line = '#!/bin/bash %s/bin/sbang\n' % spack.spack_root
+
+    # Skip files that are already using sbang.
+    if original.startswith(new_sbang_line):
+        return
+
+    backup = path + ".shebang.bak"
+    os.rename(path, backup)
+
+    with open(path, 'w') as new_file:
+        new_file.write(new_sbang_line)
+        new_file.write(original)
+
+    copy_mode(backup, path)
+    unset_executable_mode(backup)
+
+    tty.warn("Patched overly long shebang in %s" % path)
+
+
+def filter_shebangs_in_directory(directory):
+    for file in os.listdir(directory):
+        path = os.path.join(directory, file)
+
+        # only handle files
+        if not os.path.isfile(path):
+            continue
+
+        # only handle links that resolve within THIS package's prefix.
+        if os.path.islink(path):
+            real_path = os.path.realpath(path)
+            if not real_path.startswith(directory + os.sep):
+                continue
+
+        # test the file for a long shebang, and filter
+        if shebang_too_long(path):
+            filter_shebang(path)
+
+
+def post_install(pkg):
+    """This hook edits scripts so that they call /bin/bash
+       $spack_prefix/bin/sbang instead of something longer than the
+       shebang limit."""
+    if not os.path.isdir(pkg.prefix.bin):
+        return
+    filter_shebangs_in_directory(pkg.prefix.bin)

lib/spack/spack/mirror.py

@@ -45,25 +45,31 @@
 from spack.util.compression import extension, allowed_archive
 
 
-def mirror_archive_filename(spec):
+def mirror_archive_filename(spec, fetcher):
     """Get the name of the spec's archive in the mirror."""
     if not spec.version.concrete:
         raise ValueError("mirror.path requires spec with concrete version.")
 
-    fetcher = spec.package.fetcher
     if isinstance(fetcher, fs.URLFetchStrategy):
+        if fetcher.expand_archive:
             # If we fetch this version with a URLFetchStrategy, use URL's archive type
             ext = url.downloaded_file_extension(fetcher.url)
+        else:
+            # If the archive shouldn't be expanded, don't check for its extension.
+            ext = None
     else:
         # Otherwise we'll make a .tar.gz ourselves
         ext = 'tar.gz'
 
-    return "%s-%s.%s" % (spec.package.name, spec.version, ext)
+    filename = "%s-%s" % (spec.package.name, spec.version)
+    if ext:
+        filename += ".%s" % ext
+    return filename
 
 
-def mirror_archive_path(spec):
+def mirror_archive_path(spec, fetcher):
     """Get the relative path to the spec's archive within a mirror."""
-    return join_path(spec.name, mirror_archive_filename(spec))
+    return join_path(spec.name, mirror_archive_filename(spec, fetcher))
 
 
 def get_matching_versions(specs, **kwargs):
@@ -74,7 +80,7 @@ def get_matching_versions(specs, **kwargs):
 
         # Skip any package that has no known versions.
         if not pkg.versions:
-            tty.msg("No safe (checksummed) versions for package %s." % pkg.name)
+            tty.msg("No safe (checksummed) versions for package %s" % pkg.name)
             continue
 
         num_versions = kwargs.get('num_versions', 0)
@@ -111,7 +117,6 @@ def suggest_archive_basename(resource):
     return basename
 
 
-
 def create(path, specs, **kwargs):
     """Create a directory to be used as a spack mirror, and fill it with
        package archives.
@@ -159,82 +164,65 @@ def create(path, specs, **kwargs):
                 "Cannot create directory '%s':" % mirror_root, str(e))
 
     # Things to keep track of while parsing specs.
-    present  = []
+    categories = {
-    mirrored = []
+        'present': [],
-    error    = []
+        'mirrored': [],
+        'error': []
+    }
 
     # Iterate through packages and download all the safe tarballs for each of them
-    everything_already_exists = True
     for spec in version_specs:
-        pkg = spec.package
+        add_single_spec(spec, mirror_root, categories, **kwargs)
 
-        stage = None
+    return categories['present'], categories['mirrored'], categories['error']
+
+
+def add_single_spec(spec, mirror_root, categories, **kwargs):
+    tty.msg("Adding package {pkg} to mirror".format(pkg=spec.format("$_$@")))
+    spec_exists_in_mirror = True
     try:
+        with spec.package.stage:
+            # fetcher = stage.fetcher
+            # fetcher.fetch()
+            # ...
+            # fetcher.archive(archive_path)
+            for ii, stage in enumerate(spec.package.stage):
+                fetcher = stage.fetcher
+                if ii == 0:
                     # create a subdirectory for the current package@version
-            archive_path = os.path.abspath(join_path(mirror_root, mirror_archive_path(spec)))
+                    archive_path = os.path.abspath(join_path(mirror_root, mirror_archive_path(spec, fetcher)))
+                    name = spec.format("$_$@")
+                else:
+                    resource = stage.resource
+                    archive_path = join_path(subdir, suggest_archive_basename(resource))
+                    name = "{resource} ({pkg}).".format(resource=resource.name, pkg=spec.format("$_$@"))
                 subdir = os.path.dirname(archive_path)
-            try:
                 mkdirp(subdir)
-            except OSError as e:
-                raise MirrorError(
-                    "Cannot create directory '%s':" % subdir, str(e))
 
                 if os.path.exists(archive_path):
-                tty.msg("Already added %s" % spec.format("$_$@"))
+                    tty.msg("{name} : already added".format(name=name))
                 else:
-                everything_already_exists = False
+                    spec_exists_in_mirror = False
-                # Set up a stage and a fetcher for the download
-                unique_fetch_name = spec.format("$_$@")
-                fetcher = fs.for_package_version(pkg, pkg.version)
-                stage = Stage(fetcher, name=unique_fetch_name)
-                fetcher.set_stage(stage)
-
-                # Do the fetch and checksum if necessary
                     fetcher.fetch()
                     if not kwargs.get('no_checksum', False):
                         fetcher.check()
-                    tty.msg("Checksum passed for %s@%s" % (pkg.name, pkg.version))
+                        tty.msg("{name} : checksum passed".format(name=name))
 
                     # Fetchers have to know how to archive their files.  Use
                     # that to move/copy/create an archive in the mirror.
                     fetcher.archive(archive_path)
-                tty.msg("Added %s." % spec.format("$_$@"))
+                    tty.msg("{name} : added".format(name=name))
 
-            # Fetch resources if they are associated with the spec
+        if spec_exists_in_mirror:
-            resources = pkg._get_resources()
+            categories['present'].append(spec)
-            for resource in resources:
-                resource_archive_path = join_path(subdir, suggest_archive_basename(resource))
-                if os.path.exists(resource_archive_path):
-                    tty.msg("Already added resource %s (%s@%s)." % (resource.name, pkg.name, pkg.version))
-                    continue
-                everything_already_exists = False
-                resource_stage_folder = pkg._resource_stage(resource)
-                resource_stage = Stage(resource.fetcher, name=resource_stage_folder)
-                resource.fetcher.set_stage(resource_stage)
-                resource.fetcher.fetch()
-                if not kwargs.get('no_checksum', False):
-                    resource.fetcher.check()
-                    tty.msg("Checksum passed for the resource %s (%s@%s)" % (resource.name, pkg.name, pkg.version))
-                resource.fetcher.archive(resource_archive_path)
-                tty.msg("Added resource %s (%s@%s)." % (resource.name, pkg.name, pkg.version))
-
-            if everything_already_exists:
-                present.append(spec)
         else:
-                mirrored.append(spec)
+            categories['mirrored'].append(spec)
-
+    except Exception as e:
-        except Exception, e:
         if spack.debug:
             sys.excepthook(*sys.exc_info())
         else:
-                tty.warn("Error while fetching %s." % spec.format('$_$@'), e.message)
+            tty.warn("Error while fetching %s" % spec.format('$_$@'), e.message)
-            error.append(spec)
+        categories['error'].append(spec)
-
-        finally:
-            if stage:
-                stage.destroy()
-
-    return (present, mirrored, error)
 
 
 class MirrorError(spack.error.SpackError):

lib/spack/spack/modules.py

@@ -33,6 +33,7 @@
 
   * /bin directories to be appended to PATH
   * /lib* directories for LD_LIBRARY_PATH
+  * /include directories for CPATH
   * /man* and /share/man* directories for MANPATH
   * the package prefix for CMAKE_PREFIX_PATH
 
@@ -121,6 +122,7 @@ def add_path(path_name, directory):
                     ('LIBRARY_PATH', self.spec.prefix.lib64),
                     ('LD_LIBRARY_PATH', self.spec.prefix.lib),
                     ('LD_LIBRARY_PATH', self.spec.prefix.lib64),
+                    ('CPATH', self.spec.prefix.include),
                     ('PKG_CONFIG_PATH', join_path(self.spec.prefix.lib, 'pkgconfig')),
                     ('PKG_CONFIG_PATH', join_path(self.spec.prefix.lib64, 'pkgconfig'))]:
 
@@ -194,12 +196,14 @@ class Dotkit(EnvModule):
     @property
     def file_name(self):
         return join_path(Dotkit.path, self.spec.architecture,
-                         self.spec.format('$_$@$%@$+$#.dk'))
+                         '%s.dk' % self.use_name)
 
     @property
     def use_name(self):
-        return self.spec.format('$_$@$%@$+$#')
+      return "%s-%s-%s-%s-%s" % (self.spec.name, self.spec.version,
-
+                                 self.spec.compiler.name,
+                                 self.spec.compiler.version, 
+                                 self.spec.dag_hash())
 
     def _write(self, dk_file):
         # Category
@@ -235,7 +239,10 @@ def file_name(self):
 
     @property
     def use_name(self):
-        return self.spec.format('$_$@$%@$+$#')
+      return "%s-%s-%s-%s-%s" % (self.spec.name, self.spec.version,
+                                 self.spec.compiler.name,
+                                 self.spec.compiler.version, 
+                                 self.spec.dag_hash())
 
 
     def _write(self, m_file):

lib/spack/spack/multimethod.py

@@ -138,7 +138,7 @@ class when(object):
        methods like install() that depend on the package's spec.
        For example:
 
-       .. code-block::
+       .. code-block:: python
 
           class SomePackage(Package):
               ...
@@ -163,6 +163,8 @@ def install(self, prefix):
        if you only have part of the install that is platform specific, you
        could do this:
 
+       .. code-block:: python
+
           class SomePackage(Package):
               ...
               # virtual dependence on MPI.
@@ -193,10 +195,11 @@ def install(self, prefix):
        platform-specific versions.  There's not much we can do to get
        around this because of the way decorators work.
     """
-class when(object):
     def __init__(self, spec):
         pkg = get_calling_module_name()
-        self.spec = parse_anonymous_spec(spec, pkg)
+        if spec is True:
+            spec = pkg
+        self.spec = parse_anonymous_spec(spec, pkg) if spec is not False else None
 
     def __call__(self, method):
         # Get the first definition of the method in the calling scope
@@ -207,7 +210,9 @@ def __call__(self, method):
         if not type(original_method) == SpecMultiMethod:
             original_method = SpecMultiMethod(original_method)
 
+        if self.spec is not None:
             original_method.register(self.spec, method)
+
         return original_method
 
 

lib/spack/spack/package.py

@@ -58,14 +58,16 @@
 import spack.mirror
 import spack.hooks
 import spack.directives
+import spack.repository
 import spack.build_environment
 import spack.url
 import spack.util.web
 import spack.fetch_strategy as fs
 from spack.version import *
-from spack.stage import Stage
+from spack.stage import Stage, ResourceStage, StageComposite
 from spack.util.compression import allowed_archive, extension
 from spack.util.executable import ProcessError
+from spack.util.environment import dump_environment
 
 """Allowed URL schemes for spack packages."""
 _ALLOWED_URL_SCHEMES = ["http", "https", "ftp", "file", "git"]
@@ -316,6 +318,17 @@ class SomePackage(Package):
     """Most packages are NOT extendable.  Set to True if you want extensions."""
     extendable = False
 
+    """List of prefix-relative file paths. If these do not exist after
+       install, or if they exist but are not files, sanity checks fail.
+    """
+    sanity_check_files = []
+
+    """List of prefix-relative directory paths. If these do not exist
+       after install, or if they exist but are not directories, sanity
+       checks will fail.
+    """
+    sanity_check_dirs = []
+
 
     def __init__(self, spec):
         # this determines how the package should be built.
@@ -433,23 +446,51 @@ def url_for_version(self, version):
         return spack.url.substitute_version(self.nearest_url(version),
                                             self.url_version(version))
 
+    def _make_resource_stage(self, root_stage, fetcher, resource):
+        resource_stage_folder = self._resource_stage(resource)
+        resource_mirror = join_path(self.name, os.path.basename(fetcher.url))
+        stage = ResourceStage(resource.fetcher, root=root_stage, resource=resource,
+                              name=resource_stage_folder, mirror_path=resource_mirror)
+        return stage
+
+    def _make_root_stage(self, fetcher):
+        # Construct a mirror path (TODO: get this out of package.py)
+        mp = spack.mirror.mirror_archive_path(self.spec, fetcher)
+        # Construct a path where the stage should build..
+        s = self.spec
+        stage_name = "%s-%s-%s" % (s.name, s.version, s.dag_hash())
+        # Build the composite stage
+        stage = Stage(fetcher, mirror_path=mp, name=stage_name)
+        return stage
+
+    def _make_stage(self):
+        # Construct a composite stage on top of the composite FetchStrategy
+        composite_fetcher = self.fetcher
+        composite_stage = StageComposite()
+        resources = self._get_needed_resources()
+        for ii, fetcher in enumerate(composite_fetcher):
+            if ii == 0:
+                # Construct root stage first
+                stage = self._make_root_stage(fetcher)
+            else:
+                # Construct resource stage
+                resource = resources[ii - 1]  # ii == 0 is root!
+                stage = self._make_resource_stage(composite_stage[0], fetcher, resource)
+            # Append the item to the composite
+            composite_stage.append(stage)
+
+        # Create stage on first access.  Needed because fetch, stage,
+        # patch, and install can be called independently of each
+        # other, so `with self.stage:` in do_install isn't sufficient.
+        composite_stage.create()
+        return composite_stage
 
     @property
     def stage(self):
         if not self.spec.concrete:
             raise ValueError("Can only get a stage for a concrete package.")
-
         if self._stage is None:
-            # Construct a mirror path (TODO: get this out of package.py)
+            self._stage = self._make_stage()
-            mp = spack.mirror.mirror_archive_path(self.spec)
-
-            # Construct a path where the stage should build..
-            s = self.spec
-            stage_name = "%s-%s-%s" % (s.name, s.version, s.dag_hash())
-
-            # Build the stage
-            self._stage = Stage(self.fetcher, mirror_path=mp, name=stage_name)
-
         return self._stage
 
 
@@ -459,17 +500,27 @@ def stage(self, stage):
         self._stage = stage
 
 
+    def _make_fetcher(self):
+        # Construct a composite fetcher that always contains at least
+        # one element (the root package). In case there are resources
+        # associated with the package, append their fetcher to the
+        # composite.
+        root_fetcher = fs.for_package_version(self, self.version)
+        fetcher = fs.FetchStrategyComposite()  # Composite fetcher
+        fetcher.append(root_fetcher)  # Root fetcher is always present
+        resources = self._get_needed_resources()
+        for resource in resources:
+            fetcher.append(resource.fetcher)
+        return fetcher
+
     @property
     def fetcher(self):
         if not self.spec.versions.concrete:
-            raise ValueError(
+            raise ValueError("Can only get a fetcher for a package with concrete versions.")
-                "Can only get a fetcher for a package with concrete versions.")
-
         if not self._fetcher:
-            self._fetcher = fs.for_package_version(self, self.version)
+            self._fetcher = self._make_fetcher()
         return self._fetcher
 
-
     @fetcher.setter
     def fetcher(self, f):
         self._fetcher = f
@@ -632,7 +683,7 @@ def remove_prefix(self):
 
 
     def do_fetch(self, mirror_only=False):
-        """Creates a stage directory and downloads the taball for this package.
+        """Creates a stage directory and downloads the tarball for this package.
            Working directory will be set to the stage directory.
         """
         if not self.spec.concrete:
@@ -654,24 +705,10 @@ def do_fetch(self, mirror_only=False):
 
             if not ignore_checksum:
                 raise FetchError(
-                    "Will not fetch %s." % self.spec.format('$_$@'), checksum_msg)
+                    "Will not fetch %s" % self.spec.format('$_$@'), checksum_msg)
 
         self.stage.fetch(mirror_only)
 
-        ##########
-        # Fetch resources
-        resources = self._get_resources()
-        for resource in resources:
-            resource_stage_folder = self._resource_stage(resource)
-            # FIXME : works only for URLFetchStrategy
-            resource_mirror = join_path(self.name, os.path.basename(resource.fetcher.url))
-            resource_stage = Stage(resource.fetcher, name=resource_stage_folder, mirror_path=resource_mirror)
-            resource.fetcher.set_stage(resource_stage)
-            # Delegate to stage object to trigger mirror logic
-            resource_stage.fetch()
-            resource_stage.check()
-        ##########
-
         self._fetch_time = time.time() - start_time
 
         if spack.do_checksum and self.version in self.versions:
@@ -684,49 +721,8 @@ def do_stage(self, mirror_only=False):
         if not self.spec.concrete:
             raise ValueError("Can only stage concrete packages.")
 
-        def _expand_archive(stage, name=self.name):
-            archive_dir = stage.source_path
-            if not archive_dir:
-                stage.expand_archive()
-                tty.msg("Created stage in %s." % stage.path)
-            else:
-                tty.msg("Already staged %s in %s." % (name, stage.path))
-
         self.do_fetch(mirror_only)
-        _expand_archive(self.stage)
+        self.stage.expand_archive()
-
-        ##########
-        # Stage resources in appropriate path
-        resources = self._get_resources()
-        # TODO: this is to allow nested resources, a better solution would be
-        # good
-        for resource in sorted(resources, key=lambda res: len(res.destination)):
-            stage = resource.fetcher.stage
-            _expand_archive(stage, resource.name)
-            # Turn placement into a dict with relative paths
-            placement = os.path.basename(stage.source_path) if resource.placement is None else resource.placement
-            if not isinstance(placement, dict):
-                placement = {'': placement}
-            # Make the paths in the dictionary absolute and link
-            for key, value in placement.iteritems():
-                target_path = join_path(self.stage.source_path, resource.destination)
-                link_path = join_path(target_path, value)
-                source_path = join_path(stage.source_path, key)
-
-                try:
-                    os.makedirs(target_path)
-                except OSError as err:
-                    if err.errno == errno.EEXIST and os.path.isdir(target_path):
-                        pass
-                    else: raise
-
-                # NOTE: a reasonable fix for the TODO above might be to have
-                # these expand in place, but expand_archive does not offer
-                # this
-
-                if not os.path.exists(link_path):
-                    shutil.move(source_path, link_path)
-        ##########
         self.stage.chdir_to_source()
 
 
@@ -744,7 +740,7 @@ def do_patch(self):
 
         # If there are no patches, note it.
         if not self.patches and not has_patch_fun:
-            tty.msg("No patches needed for %s." % self.name)
+            tty.msg("No patches needed for %s" % self.name)
             return
 
         # Construct paths to special files in the archive dir used to
@@ -767,7 +763,7 @@ def do_patch(self):
             tty.msg("Already patched %s" % self.name)
             return
         elif os.path.isfile(no_patches_file):
-            tty.msg("No patches needed for %s." % self.name)
+            tty.msg("No patches needed for %s" % self.name)
             return
 
         # Apply all the patches for specs that match this one
@@ -788,10 +784,10 @@ def do_patch(self):
         if has_patch_fun:
             try:
                 self.patch()
-                tty.msg("Ran patch() for %s." % self.name)
+                tty.msg("Ran patch() for %s" % self.name)
                 patched = True
             except:
-                tty.msg("patch() function failed for %s." % self.name)
+                tty.msg("patch() function failed for %s" % self.name)
                 touch(bad_file)
                 raise
 
@@ -822,12 +818,15 @@ def do_fake_install(self):
         mkdirp(self.prefix.man1)
 
 
-    def _get_resources(self):
+    def _get_needed_resources(self):
         resources = []
         # Select the resources that are needed for this build
         for when_spec, resource_list in self.resources.items():
             if when_spec in self.spec:
                 resources.extend(resource_list)
+        # Sorts the resources by the length of the string representing their destination. Since any nested resource
+        # must contain another resource's name in its path, it seems that should work
+        resources = sorted(resources, key=lambda res: len(res.destination))
         return resources
 
     def _resource_stage(self, resource):
@@ -846,7 +845,9 @@ def do_install(self,
 
         Args:
         keep_prefix -- Keep install prefix on failure. By default, destroys it.
-        keep_stage  -- Keep stage on successful build. By default, destroys it.
+        keep_stage  -- By default, stage is destroyed only if there are no
+                       exceptions during build. Set to True to keep the stage
+                       even with exceptions.
         ignore_deps -- Do not install dependencies before installing this package.
         fake        -- Don't really build -- install fake stub files instead.
         skip_patch  -- Skip patch stage of build if True.
@@ -856,18 +857,31 @@ def do_install(self,
         if not self.spec.concrete:
             raise ValueError("Can only install concrete packages.")
 
-        if os.path.exists(self.prefix):
+        # No installation needed if package is external
-            tty.msg("%s is already installed in %s." % (self.name, self.prefix))
+        if self.spec.external:
+            tty.msg("%s is externally installed in %s" % (self.name, self.spec.external))
+            return
+
+        # Ensure package is not already installed
+        if spack.install_layout.check_installed(self.spec):
+            tty.msg("%s is already installed in %s" % (self.name, self.prefix))
             return
 
         tty.msg("Installing %s" % self.name)
 
+        # First, install dependencies recursively.
         if not ignore_deps:
             self.do_install_dependencies(
                 keep_prefix=keep_prefix, keep_stage=keep_stage, ignore_deps=ignore_deps,
-                fake=fake, skip_patch=skip_patch, verbose=verbose,
+                fake=fake, skip_patch=skip_patch, verbose=verbose, make_jobs=make_jobs)
-                make_jobs=make_jobs)
 
+        # Set parallelism before starting build.
+        self.make_jobs = make_jobs
+
+        # Then install the package itself.
+        def build_process():
+            """Forked for each build. Has its own process and python
+               module space set up by build_environment.fork()."""
             start_time = time.time()
             if not fake:
                 if not skip_patch:
@@ -875,81 +889,72 @@ def do_install(self,
                 else:
                     self.do_stage()
 
-        # create the install directory.  The install layout
+            tty.msg("Building %s" % self.name)
-        # handles this in case so that it can use whatever
-        # package naming scheme it likes.
-        spack.install_layout.create_install_directory(self.spec)
-
-        def cleanup():
-            if not keep_prefix:
-                # If anything goes wrong, remove the install prefix
-                self.remove_prefix()
-            else:
-                tty.warn("Keeping install prefix in place despite error.",
-                         "Spack will think this package is installed." +
-                         "Manually remove this directory to fix:",
-                         self.prefix, wrap=True)
-
-
-        def real_work():
-            try:
-                tty.msg("Building %s." % self.name)
 
+            self.stage.keep = keep_stage
+            with self.stage:
                 # Run the pre-install hook in the child process after
                 # the directory is created.
                 spack.hooks.pre_install(self)
 
-                # Set up process's build environment before running install.
                 if fake:
                     self.do_fake_install()
                 else:
                     # Do the real install in the source directory.
                      self.stage.chdir_to_source()
 
-                    # This redirects I/O to a build log (and optionally to the terminal)
+                     # Save the build environment in a file before building.
+                     env_path = join_path(os.getcwd(), 'spack-build.env')
+
+                     try:
+                        # Redirect I/O to a build log (and optionally to the terminal)
                         log_path = join_path(os.getcwd(), 'spack-build.out')
                         log_file = open(log_path, 'w')
                         with log_output(log_file, verbose, sys.stdout.isatty(), True):
+                            dump_environment(env_path)
                             self.install(self.spec, self.prefix)
 
+                     except ProcessError as e:
+                         # Annotate ProcessErrors with the location of the build log.
+                         e.build_log = log_path
+                         raise e
+
                      # Ensure that something was actually installed.
-                self._sanity_check_install()
+                     self.sanity_check_prefix()
 
-                # Move build log into install directory on success
+                     # Copy provenance into the install directory on success
-                if not fake:
                      log_install_path = spack.install_layout.build_log_path(self.spec)
-                    install(log_path, log_install_path)
+                     env_install_path = spack.install_layout.build_env_path(self.spec)
+                     packages_dir = spack.install_layout.build_packages_path(self.spec)
 
-                # On successful install, remove the stage.
+                     install(log_path, log_install_path)
-                if not keep_stage:
+                     install(env_path, env_install_path)
-                    self.stage.destroy()
+                     dump_packages(self.spec, packages_dir)
 
             # Stop timer.
             self._total_time = time.time() - start_time
             build_time = self._total_time - self._fetch_time
 
-                tty.msg("Successfully installed %s." % self.name,
+            tty.msg("Successfully installed %s" % self.name,
                     "Fetch: %s.  Build: %s.  Total: %s."
                     % (_hms(self._fetch_time), _hms(build_time), _hms(self._total_time)))
             print_pkg(self.prefix)
 
-            except ProcessError, e:
+        try:
-                # Annotate with location of build log.
+            # Create the install prefix and fork the build process.
-                e.build_log = log_path
+            spack.install_layout.create_install_directory(self.spec)
-                cleanup()
+            spack.build_environment.fork(self, build_process)
-                raise e
-
         except:
-                # other exceptions just clean up and raise.
+            # remove the install prefix if anything went wrong during install.
-                cleanup()
+            if not keep_prefix:
+                self.remove_prefix()
+            else:
+                tty.warn("Keeping install prefix in place despite error.",
+                         "Spack will think this package is installed. " +
+                         "Manually remove this directory to fix:",
+                         self.prefix, wrap=True)
             raise
 
-        # Set parallelism before starting build.
-        self.make_jobs = make_jobs
-
-        # Do the build.
-        spack.build_environment.fork(self, real_work)
-
         # note: PARENT of the build process adds the new package to
         # the database, so that we don't need to re-read from file.
         spack.installed_db.add(self.spec, self.prefix)
@@ -958,7 +963,18 @@ def real_work():
         spack.hooks.post_install(self)
 
 
-    def _sanity_check_install(self):
+    def sanity_check_prefix(self):
+        """This function checks whether install succeeded."""
+        def check_paths(path_list, filetype, predicate):
+            for path in path_list:
+                abs_path = os.path.join(self.prefix, path)
+                if not predicate(abs_path):
+                    raise InstallError("Install failed for %s. No such %s in prefix: %s"
+                                       % (self.name, filetype, path))
+
+        check_paths(self.sanity_check_files, 'file', os.path.isfile)
+        check_paths(self.sanity_check_dirs, 'directory', os.path.isdir)
+
         installed = set(os.listdir(self.prefix))
         installed.difference_update(spack.install_layout.hidden_file_paths)
         if not installed:
@@ -1035,7 +1051,7 @@ def do_uninstall(self, force=False):
         # Uninstalling in Spack only requires removing the prefix.
         self.remove_prefix()
         spack.installed_db.remove(self.spec)
-        tty.msg("Successfully uninstalled %s." % self.spec.short_spec)
+        tty.msg("Successfully uninstalled %s" % self.spec.short_spec)
 
         # Once everything else is done, run post install hooks
         spack.hooks.post_uninstall(self)
@@ -1082,7 +1098,7 @@ def do_activate(self, force=False):
         self.extendee_spec.package.activate(self, **self.extendee_args)
 
         spack.install_layout.add_extension(self.extendee_spec, self.spec)
-        tty.msg("Activated extension %s for %s."
+        tty.msg("Activated extension %s for %s"
                 % (self.spec.short_spec, self.extendee_spec.format("$_$@$+$%@")))
 
 
@@ -1134,7 +1150,7 @@ def do_deactivate(self, **kwargs):
         if self.activated:
             spack.install_layout.remove_extension(self.extendee_spec, self.spec)
 
-        tty.msg("Deactivated extension %s for %s."
+        tty.msg("Deactivated extension %s for %s"
                 % (self.spec.short_spec, self.extendee_spec.format("$_$@$+$%@")))
 
 
@@ -1162,7 +1178,6 @@ def do_restage(self):
 
     def do_clean(self):
         """Removes the package's build stage and source tarball."""
-        if os.path.exists(self.stage.path):
         self.stage.destroy()
 
 
@@ -1202,7 +1217,7 @@ def fetch_remote_versions(self):
         try:
             return spack.util.web.find_versions_of_archive(
                 *self.all_urls, list_url=self.list_url, list_depth=self.list_depth)
-        except spack.error.NoNetworkConnectionError, e:
+        except spack.error.NoNetworkConnectionError as e:
             tty.die("Package.fetch_versions couldn't connect to:",
                     e.url, e.message)
 
@@ -1220,8 +1235,8 @@ def rpath(self):
 
     @property
     def rpath_args(self):
-        """Get the rpath args as a string, with -Wl,-rpath= for each element."""
+        """Get the rpath args as a string, with -Wl,-rpath, for each element."""
-        return " ".join("-Wl,-rpath=%s" % p for p in self.rpath)
+        return " ".join("-Wl,-rpath,%s" % p for p in self.rpath)
 
 
 def validate_package_url(url_string):
@@ -1234,6 +1249,52 @@ def validate_package_url(url_string):
         tty.die("Invalid file type in URL: '%s'" % url_string)
 
 
+def dump_packages(spec, path):
+    """Dump all package information for a spec and its dependencies.
+
+       This creates a package repository within path for every
+       namespace in the spec DAG, and fills the repos wtih package
+       files and patch files for every node in the DAG.
+    """
+    mkdirp(path)
+
+    # Copy in package.py files from any dependencies.
+    # Note that we copy them in as they are in the *install* directory
+    # NOT as they are in the repository, because we want a snapshot of
+    # how *this* particular build was done.
+    for node in spec.traverse():
+        if node is not spec:
+            # Locate the dependency package in the install tree and find
+            # its provenance information.
+            source = spack.install_layout.build_packages_path(node)
+            source_repo_root = join_path(source, node.namespace)
+
+            # There's no provenance installed for the source package.  Skip it.
+            # User can always get something current from the builtin repo.
+            if not os.path.isdir(source_repo_root):
+                continue
+
+            # Create a source repo and get the pkg directory out of it.
+            try:
+                source_repo = spack.repository.Repo(source_repo_root)
+                source_pkg_dir = source_repo.dirname_for_package_name(node.name)
+            except RepoError as e:
+                tty.warn("Warning: Couldn't copy in provenance for %s" % node.name)
+
+        # Create a destination repository
+        dest_repo_root = join_path(path, node.namespace)
+        if not os.path.exists(dest_repo_root):
+            spack.repository.create_repo(dest_repo_root)
+        repo = spack.repository.Repo(dest_repo_root)
+
+        # Get the location of the package in the dest repo.
+        dest_pkg_dir = repo.dirname_for_package_name(node.name)
+        if node is not spec:
+            install_tree(source_pkg_dir, dest_pkg_dir)
+        else:
+            spack.repo.dump_provenance(node, dest_pkg_dir)
+
+
 def print_pkg(message):
     """Outputs a message with a package icon."""
     from llnl.util.tty.color import cwrite
@@ -1284,7 +1345,7 @@ class PackageVersionError(PackageError):
     """Raised when a version URL cannot automatically be determined."""
     def __init__(self, version):
         super(PackageVersionError, self).__init__(
-            "Cannot determine a URL automatically for version %s." % version,
+            "Cannot determine a URL automatically for version %s" % version,
             "Please provide a url for this version in the package.py file.")
 
 

lib/spack/spack/preferred_packages.py

@@ -0,0 +1,175 @@
+##############################################################################
+# Copyright (c) 2013, Lawrence Livermore National Security, LLC.
+# Produced at the Lawrence Livermore National Laboratory.
+#
+# This file is part of Spack.
+# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
+# LLNL-CODE-647188
+#
+# For details, see https://scalability-llnl.github.io/spack
+# Please also see the LICENSE file for our notice and the LGPL.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License (as published by
+# the Free Software Foundation) version 2.1 dated February 1999.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
+# conditions of the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+##############################################################################
+
+import spack
+from spack.version import *
+
+class PreferredPackages(object):
+    _default_order = {'compiler' : [ 'gcc', 'intel', 'clang', 'pgi', 'xlc' ] } # Arbitrary, but consistent
+
+    def __init__(self):
+        self.preferred = spack.config.get_config('packages')
+        self._spec_for_pkgname_cache = {}
+
+    # Given a package name, sort component (e.g, version, compiler, ...), and
+    # a second_key (used by providers), return the list
+    def _order_for_package(self, pkgname, component, second_key, test_all=True):
+        pkglist = [pkgname]
+        if test_all:
+            pkglist.append('all')
+        for pkg in pkglist:
+            order = self.preferred.get(pkg, {}).get(component, {})
+            if type(order) is dict:
+                order = order.get(second_key, {})
+            if not order:
+                continue
+            return [str(s).strip() for s in order]
+        return []
+
+
+    # A generic sorting function. Given a package name and sort
+    # component, return less-than-0, 0, or greater-than-0 if
+    # a is respectively less-than, equal to, or greater than b.
+    def _component_compare(self, pkgname, component, a, b, reverse_natural_compare, second_key):
+        if a is None:
+            return -1
+        if b is None:
+            return 1
+        orderlist = self._order_for_package(pkgname, component, second_key)
+        a_in_list = str(a) in orderlist
+        b_in_list = str(b) in orderlist
+        if a_in_list and not b_in_list:
+            return -1
+        elif b_in_list and not a_in_list:
+            return 1
+
+        cmp_a = None
+        cmp_b = None
+        reverse = None
+        if not a_in_list and not b_in_list:
+            cmp_a = a
+            cmp_b = b
+            reverse = -1 if reverse_natural_compare else 1
+        else:
+            cmp_a = orderlist.index(str(a))
+            cmp_b = orderlist.index(str(b))
+            reverse = 1
+
+        if cmp_a < cmp_b:
+            return -1 * reverse
+        elif cmp_a > cmp_b:
+            return 1 * reverse
+        else:
+            return 0
+
+
+    # A sorting function for specs.  Similar to component_compare, but
+    # a and b are considered to match entries in the sorting list if they
+    # satisfy the list component.
+    def _spec_compare(self, pkgname, component, a, b, reverse_natural_compare, second_key):
+        if not a or not a.concrete:
+            return -1
+        if not b or not b.concrete:
+            return 1
+        specs = self._spec_for_pkgname(pkgname, component, second_key)
+        a_index = None
+        b_index = None
+        reverse = -1 if reverse_natural_compare else 1
+        for i, cspec in enumerate(specs):
+            if a_index == None and (cspec.satisfies(a) or a.satisfies(cspec)):
+                a_index = i
+                if b_index:
+                    break
+            if b_index == None and (cspec.satisfies(b) or b.satisfies(cspec)):
+                b_index = i
+                if a_index:
+                    break
+
+        if   a_index != None and b_index == None: return -1
+        elif a_index == None and b_index != None: return 1
+        elif a_index != None and b_index == a_index: return -1 * cmp(a, b)
+        elif a_index != None and b_index != None and a_index != b_index: return cmp(a_index, b_index)
+        else: return cmp(a, b) * reverse
+
+
+
+    # Given a sort order specified by the pkgname/component/second_key, return
+    # a list of CompilerSpecs, VersionLists, or Specs for that sorting list.
+    def _spec_for_pkgname(self, pkgname, component, second_key):
+        key = (pkgname, component, second_key)
+        if not key in self._spec_for_pkgname_cache:
+            pkglist = self._order_for_package(pkgname, component, second_key)
+            if not pkglist:
+                if component in self._default_order:
+                    pkglist = self._default_order[component]
+            if component == 'compiler':
+                self._spec_for_pkgname_cache[key] = [spack.spec.CompilerSpec(s) for s in pkglist]
+            elif component == 'version':
+                self._spec_for_pkgname_cache[key] = [VersionList(s) for s in pkglist]
+            else:
+                self._spec_for_pkgname_cache[key] = [spack.spec.Spec(s) for s in pkglist]
+        return self._spec_for_pkgname_cache[key]
+
+
+    def provider_compare(self, pkgname, provider_str, a, b):
+        """Return less-than-0, 0, or greater than 0 if a is respecively less-than, equal-to, or
+           greater-than b. A and b are possible implementations of provider_str.
+           One provider is less-than another if it is preferred over the other.
+           For example, provider_compare('scorep', 'mpi', 'mvapich', 'openmpi') would return -1 if
+           mvapich should be preferred over openmpi for scorep."""
+        return self._spec_compare(pkgname, 'providers', a, b, False, provider_str)
+
+
+    def spec_has_preferred_provider(self, pkgname, provider_str):
+        """Return True iff the named package has a list of preferred provider"""
+        return bool(self._order_for_package(pkgname, 'providers', provider_str, False))
+
+
+    def version_compare(self, pkgname, a, b):
+        """Return less-than-0, 0, or greater than 0 if version a of pkgname is
+           respecively less-than, equal-to, or greater-than version b of pkgname.
+           One version is less-than another if it is preferred over the other."""
+        return self._spec_compare(pkgname, 'version', a, b, True, None)
+
+
+    def variant_compare(self, pkgname, a, b):
+        """Return less-than-0, 0, or greater than 0 if variant a of pkgname is
+           respecively less-than, equal-to, or greater-than variant b of pkgname.
+           One variant is less-than another if it is preferred over the other."""
+        return self._component_compare(pkgname, 'variant', a, b, False, None)
+
+
+    def architecture_compare(self, pkgname, a, b):
+        """Return less-than-0, 0, or greater than 0 if architecture a of pkgname is
+           respecively less-than, equal-to, or greater-than architecture b of pkgname.
+           One architecture is less-than another if it is preferred over the other."""
+        return self._component_compare(pkgname, 'architecture', a, b, False, None)
+
+
+    def compiler_compare(self, pkgname, a, b):
+        """Return less-than-0, 0, or greater than 0 if compiler a of pkgname is
+           respecively less-than, equal-to, or greater-than compiler b of pkgname.
+           One compiler is less-than another if it is preferred over the other."""
+        return self._spec_compare(pkgname, 'compiler', a, b, False, None)

lib/spack/spack/repository.py

@@ -6,7 +6,7 @@
 # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
 # LLNL-CODE-647188
 #
-# For details, see https://llnl.github.io/spack
+# For details, see https://software.llnl.gov/spack
 # Please also see the LICENSE file for our notice and the LGPL.
 #
 # This program is free software; you can redistribute it and/or modify
@@ -33,7 +33,7 @@
 from external import yaml
 
 import llnl.util.tty as tty
-from llnl.util.filesystem import join_path
+from llnl.util.filesystem import *
 
 import spack.error
 import spack.config
@@ -156,7 +156,7 @@ def _add(self, repo):
 
         if repo.namespace in self.by_namespace:
             raise DuplicateRepoError(
-                "Package repos '%s' and '%s' both provide namespace %s."
+                "Package repos '%s' and '%s' both provide namespace %s"
                 % (repo.root, self.by_namespace[repo.namespace].root, repo.namespace))
 
         # Add repo to the pkg indexes
@@ -316,6 +316,21 @@ def get(self, spec, new=False):
         return self.repo_for_pkg(spec).get(spec)
 
 
+    def get_pkg_class(self, pkg_name):
+        """Find a class for the spec's package and return the class object."""
+        return self.repo_for_pkg(pkg_name).get_pkg_class(pkg_name)
+
+
+    @_autospec
+    def dump_provenance(self, spec, path):
+        """Dump provenance information for a spec to a particular path.
+
+           This dumps the package file and any associated patch files.
+           Raises UnknownPackageError if not found.
+        """
+        return self.repo_for_pkg(spec).dump_provenance(spec, path)
+
+
     def dirname_for_package_name(self, pkg_name):
         return self.repo_for_pkg(pkg_name).dirname_for_package_name(pkg_name)
 
@@ -535,12 +550,12 @@ def get(self, spec, new=False):
             raise UnknownPackageError(spec.name)
 
         if spec.namespace and spec.namespace != self.namespace:
-            raise UnknownPackageError("Repository %s does not contain package %s."
+            raise UnknownPackageError("Repository %s does not contain package %s"
                                       % (self.namespace, spec.fullname))
 
         key = hash(spec)
         if new or key not in self._instances:
-            package_class = self._get_pkg_class(spec.name)
+            package_class = self.get_pkg_class(spec.name)
             try:
                 copy = spec.copy() # defensive copy.  Package owns its spec.
                 self._instances[key] = package_class(copy)
@@ -552,6 +567,35 @@ def get(self, spec, new=False):
         return self._instances[key]
 
 
+    @_autospec
+    def dump_provenance(self, spec, path):
+        """Dump provenance information for a spec to a particular path.
+
+           This dumps the package file and any associated patch files.
+           Raises UnknownPackageError if not found.
+        """
+        # Some preliminary checks.
+        if spec.virtual:
+            raise UnknownPackageError(spec.name)
+
+        if spec.namespace and spec.namespace != self.namespace:
+            raise UnknownPackageError("Repository %s does not contain package %s."
+                                      % (self.namespace, spec.fullname))
+
+        # Install any patch files needed by packages.
+        mkdirp(path)
+        for spec, patches in spec.package.patches.items():
+            for patch in patches:
+                if patch.path:
+                    if os.path.exists(patch.path):
+                        install(patch.path, path)
+                    else:
+                        tty.warn("Patch file did not exist: %s" % patch.path)
+
+        # Install the package.py file itself.
+        install(self.filename_for_package_name(spec), path)
+
+
     def purge(self):
         """Clear entire package instance cache."""
         self._instances.clear()
@@ -676,7 +720,7 @@ def _get_pkg_module(self, pkg_name):
         return self._modules[pkg_name]
 
 
-    def _get_pkg_class(self, pkg_name):
+    def get_pkg_class(self, pkg_name):
         """Get the class for the package out of its module.
 
         First loads (or fetches from cache) a module for the
@@ -705,6 +749,58 @@ def __contains__(self, pkg_name):
         return self.exists(pkg_name)
 
 
+def create_repo(root, namespace=None):
+    """Create a new repository in root with the specified namespace.
+
+       If the namespace is not provided, use basename of root.
+       Return the canonicalized path and the namespace of the created repository.
+    """
+    root = canonicalize_path(root)
+    if not namespace:
+        namespace = os.path.basename(root)
+
+    if not re.match(r'\w[\.\w-]*', namespace):
+        raise InvalidNamespaceError("'%s' is not a valid namespace." % namespace)
+
+    existed = False
+    if os.path.exists(root):
+        if os.path.isfile(root):
+            raise BadRepoError('File %s already exists and is not a directory' % root)
+        elif os.path.isdir(root):
+            if not os.access(root, os.R_OK | os.W_OK):
+                raise BadRepoError('Cannot create new repo in %s: cannot access directory.' % root)
+            if os.listdir(root):
+                raise BadRepoError('Cannot create new repo in %s: directory is not empty.' % root)
+        existed = True
+
+    full_path = os.path.realpath(root)
+    parent = os.path.dirname(full_path)
+    if not os.access(parent, os.R_OK | os.W_OK):
+        raise BadRepoError("Cannot create repository in %s: can't access parent!" % root)
+
+    try:
+        config_path = os.path.join(root, repo_config_name)
+        packages_path = os.path.join(root, packages_dir_name)
+
+        mkdirp(packages_path)
+        with open(config_path, 'w') as config:
+            config.write("repo:\n")
+            config.write("  namespace: '%s'\n" % namespace)
+
+    except (IOError, OSError) as e:
+        raise BadRepoError('Failed to create new repository in %s.' % root,
+                           "Caused by %s: %s" % (type(e), e))
+
+        # try to clean up.
+        if existed:
+            shutil.rmtree(config_path, ignore_errors=True)
+            shutil.rmtree(packages_path, ignore_errors=True)
+        else:
+            shutil.rmtree(root, ignore_errors=True)
+
+    return full_path, namespace
+
+
 class RepoError(spack.error.SpackError):
     """Superclass for repository-related errors."""
 
@@ -713,6 +809,10 @@ class NoRepoConfiguredError(RepoError):
     """Raised when there are no repositories configured."""
 
 
+class InvalidNamespaceError(RepoError):
+    """Raised when an invalid namespace is encountered."""
+
+
 class BadRepoError(RepoError):
     """Raised when repo layout is invalid."""
 
@@ -730,7 +830,7 @@ class UnknownPackageError(PackageLoadError):
     def __init__(self, name, repo=None):
         msg = None
         if repo:
-            msg = "Package %s not found in repository %s." % (name, repo)
+            msg = "Package %s not found in repository %s" % (name, repo)
         else:
             msg = "Package %s not found." % name
         super(UnknownPackageError, self).__init__(msg)

lib/spack/spack/resource.py

@@ -6,7 +6,7 @@
 # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
 # LLNL-CODE-647188
 #
-# For details, see https://llnl.github.io/spack
+# For details, see https://software.llnl.gov/spack
 # Please also see the LICENSE file for our notice and the LGPL.
 #
 # This program is free software; you can redistribute it and/or modify

lib/spack/spack/spec.py

@@ -353,7 +353,7 @@ def constrain(self, other):
     @property
     def concrete(self):
         return self.spec._concrete or all(
-            v in self for v in self.spec.package.variants)
+            v in self for v in self.spec.package_class.variants)
 
 
     def copy(self):
@@ -421,6 +421,9 @@ def __init__(self, spec_like, *dep_like, **kwargs):
         self._normal   = kwargs.get('normal', False)
         self._concrete = kwargs.get('concrete', False)
 
+        # Allow a spec to be constructed with an external path.
+        self.external  = kwargs.get('external', None)
+
         # This allows users to construct a spec DAG with literals.
         # Note that given two specs a and b, Spec(a) copies a, but
         # Spec(a, b) will copy a but just add b as a dep.
@@ -497,6 +500,14 @@ def package(self):
         return spack.repo.get(self)
 
 
+    @property
+    def package_class(self):
+        """Internal package call gets only the class object for a package.
+           Use this to just get package metadata.
+        """
+        return spack.repo.get_pkg_class(self.name)
+
+
     @property
     def virtual(self):
         """Right now, a spec is virtual if no package exists with its name.
@@ -770,7 +781,6 @@ def _concretize_helper(self, presets=None, visited=None):
             # Concretize virtual dependencies last.  Because they're added
             # to presets below, their constraints will all be merged, but we'll
             # still need to select a concrete package later.
-            if not self.virtual:
             changed |= any(
                 (spack.concretizer.concretize_architecture(self),
                  spack.concretizer.concretize_compiler(self),
@@ -786,10 +796,32 @@ def _replace_with(self, concrete):
         """Replace this virtual spec with a concrete spec."""
         assert(self.virtual)
         for name, dependent in self.dependents.items():
+            # remove self from all dependents.
             del dependent.dependencies[self.name]
+
+            # add the replacement, unless it is already a dep of dependent.
+            if concrete.name not in dependent.dependencies:
                 dependent._add_dependency(concrete)
 
 
+    def _replace_node(self, replacement):
+        """Replace this spec with another.
+
+        Connects all dependents of this spec to its replacement, and
+        disconnects this spec from any dependencies it has. New spec
+        will have any dependencies the replacement had, and may need
+        to be normalized.
+
+        """
+        for name, dependent in self.dependents.items():
+            del dependent.dependencies[self.name]
+            dependent._add_dependency(replacement)
+
+        for name, dep in self.dependencies.items():
+            del dep.dependents[self.name]
+            del self.dependencies[dep.name]
+
+
     def _expand_virtual_packages(self):
         """Find virtual packages in this spec, replace them with providers,
            and normalize again to include the provider's (potentially virtual)
@@ -807,22 +839,81 @@ def _expand_virtual_packages(self):
               this are infrequent, but should implement this before it is
               a problem.
         """
-        changed = False
+        # Make an index of stuff this spec already provides
-        while True:
+        self_index = ProviderIndex(self.traverse(), restrict=True)
-            virtuals =[v for v in self.traverse() if v.virtual]
-            if not virtuals:
-                return changed
 
-            for spec in virtuals:
+        changed = False
-                providers = spack.repo.providers_for(spec)
+        done = False
-                concrete = spack.concretizer.choose_provider(spec, providers)
+        while not done:
-                concrete = concrete.copy()
+            done = True
-                spec._replace_with(concrete)
+            for spec in list(self.traverse()):
+                replacement = None
+                if spec.virtual:
+                    replacement = self._find_provider(spec, self_index)
+                    if replacement:
+                        # TODO: may break if in-place on self but
+                        # shouldn't happen if root is traversed first.
+                        spec._replace_with(replacement)
+                        done=False
+                        break
+
+                if not replacement:
+                    # Get a list of possible replacements in order of preference.
+                    candidates = spack.concretizer.choose_virtual_or_external(spec)
+
+                    # Try the replacements in order, skipping any that cause
+                    # satisfiability problems.
+                    for replacement in candidates:
+                        if replacement is spec:
+                            break
+
+                        # Replace spec with the candidate and normalize
+                        copy = self.copy()
+                        copy[spec.name]._dup(replacement.copy(deps=False))
+
+                        try:
+                            # If there are duplicate providers or duplicate provider
+                            # deps, consolidate them and merge constraints.
+                            copy.normalize(force=True)
+                            break
+                        except SpecError as e:
+                            # On error, we'll try the next replacement.
+                            continue
+
+                # If replacement is external then trim the dependencies
+                if replacement.external:
+                    if (spec.dependencies):
+                        changed = True
+                        spec.dependencies = DependencyMap()
+                    replacement.dependencies = DependencyMap()
+
+                # TODO: could this and the stuff in _dup be cleaned up?
+                def feq(cfield, sfield):
+                    return (not cfield) or (cfield == sfield)
+
+                if replacement is spec or (feq(replacement.name, spec.name) and
+                    feq(replacement.versions, spec.versions) and
+                    feq(replacement.compiler, spec.compiler) and
+                    feq(replacement.architecture, spec.architecture) and
+                    feq(replacement.dependencies, spec.dependencies) and
+                    feq(replacement.variants, spec.variants) and
+                    feq(replacement.external, spec.external)):
+                    continue
+
+                # Refine this spec to the candidate. This uses
+                # replace_with AND dup so that it can work in
+                # place. TODO: make this more efficient.
+                if spec.virtual:
+                    spec._replace_with(replacement)
+                    changed = True
+                if spec._dup(replacement, deps=False, cleardeps=False):
                     changed = True
 
-            # If there are duplicate providers or duplicate provider deps, this
+                self_index.update(spec)
-            # consolidates them and merge constraints.
+                done=False
-            changed |= self.normalize(force=True)
+                break
+
+        return changed
 
 
     def concretize(self):
@@ -837,6 +928,7 @@ def concretize(self):
            with requirements of its pacakges.  See flatten() and normalize() for
            more details on this.
         """
+
         if self._concrete:
             return
 
@@ -844,7 +936,7 @@ def concretize(self):
         force = False
 
         while changed:
-            changes = (self.normalize(force=force),
+            changes = (self.normalize(force),
                        self._expand_virtual_packages(),
                        self._concretize_helper())
             changed = any(changes)
@@ -1012,17 +1104,14 @@ def _merge_dependency(self, dep, visited, spec_deps, provider_index):
         """
         changed = False
 
-        # If it's a virtual dependency, try to find a provider and
+        # If it's a virtual dependency, try to find an existing
-        # merge that.
+        # provider in the spec, and merge that.
         if dep.virtual:
             visited.add(dep.name)
             provider = self._find_provider(dep, provider_index)
             if provider:
                 dep = provider
-
         else:
-            # if it's a real dependency, check whether it provides
-            # something already required in the spec.
             index = ProviderIndex([dep], restrict=True)
             for vspec in (v for v in spec_deps.values() if v.virtual):
                 if index.providers_for(vspec):
@@ -1069,7 +1158,7 @@ def _normalize_helper(self, visited, spec_deps, provider_index):
 
         # if we descend into a virtual spec, there's nothing more
         # to normalize.  Concretize will finish resolving it later.
-        if self.virtual:
+        if self.virtual or self.external:
             return False
 
         # Combine constraints from package deps with constraints from
@@ -1119,13 +1208,14 @@ def normalize(self, force=False):
         # Get all the dependencies into one DependencyMap
         spec_deps = self.flat_dependencies(copy=False)
 
-        # Initialize index of virtual dependency providers
+        # Initialize index of virtual dependency providers if
-        index = ProviderIndex(spec_deps.values(), restrict=True)
+        # concretize didn't pass us one already
+        provider_index = ProviderIndex(spec_deps.values(), restrict=True)
 
         # traverse the package DAG and fill out dependencies according
         # to package files & their 'when' specs
         visited = set()
-        any_change = self._normalize_helper(visited, spec_deps, index)
+        any_change = self._normalize_helper(visited, spec_deps, provider_index)
 
         # If there are deps specified but not visited, they're not
         # actually deps of this package.  Raise an error.
@@ -1163,7 +1253,7 @@ def validate_names(self):
 
             # Ensure that variants all exist.
             for vname, variant in spec.variants.items():
-                if vname not in spec.package.variants:
+                if vname not in spec.package_class.variants:
                     raise UnknownVariantError(spec.name, vname)
 
 
@@ -1404,15 +1494,25 @@ def _dup(self, other, **kwargs):
                Whether deps should be copied too.  Set to false to copy a
                spec but not its dependencies.
         """
+        # We don't count dependencies as changes here
+        changed = True
+        if hasattr(self, 'name'):
+            changed = (self.name != other.name and self.versions != other.versions and
+                       self.architecture != other.architecture and self.compiler != other.compiler and
+                       self.variants != other.variants and self._normal != other._normal and
+                       self.concrete != other.concrete and self.external != other.external)
+
         # Local node attributes get copied first.
         self.name = other.name
         self.versions = other.versions.copy()
         self.architecture = other.architecture
         self.compiler = other.compiler.copy() if other.compiler else None
+        if kwargs.get('cleardeps', True):
             self.dependents = DependencyMap()
             self.dependencies = DependencyMap()
         self.variants = other.variants.copy()
         self.variants.spec = self
+        self.external = other.external
         self.namespace = other.namespace
 
         # If we copy dependencies, preserve DAG structure in the new spec
@@ -1431,6 +1531,8 @@ def _dup(self, other, **kwargs):
         # Since we preserved structure, we can copy _normal safely.
         self._normal = other._normal
         self._concrete = other._concrete
+        self.external = other.external
+        return changed
 
 
     def copy(self, **kwargs):
@@ -1571,14 +1673,28 @@ def format(self, format_string='$_$@$%@$+$=', **kwargs):
 
                $_   Package name
                $.   Full package name (with namespace)
-               $@   Version
+               $@   Version with '@' prefix
-               $%   Compiler
+               $%   Compiler with '%' prefix
-               $%@  Compiler & compiler version
+               $%@  Compiler with '%' prefix & compiler version with '@' prefix
                $+   Options
-               $=   Architecture
+               $=   Architecture with '=' prefix
-               $#   7-char prefix of DAG hash
+               $#   7-char prefix of DAG hash with '-' prefix
                $$   $
 
+               You can also use full-string versions, which leave off the prefixes:
+
+               ${PACKAGE}       Package name
+               ${VERSION}       Version
+               ${COMPILER}      Full compiler string
+               ${COMPILERNAME}  Compiler name
+               ${COMPILERVER}   Compiler version
+               ${OPTIONS}       Options
+               ${ARCHITECTURE}  Architecture
+               ${SHA1}          Dependencies 8-char sha1 prefix
+
+               ${SPACK_ROOT}    The spack root directory
+               ${SPACK_INSTALL} The default spack install directory, ${SPACK_PREFIX}/opt
+
            Optionally you can provide a width, e.g. $20_ for a 20-wide name.
            Like printf, you can provide '-' for left justification, e.g.
            $-20_ for a left-justified name.
@@ -1594,7 +1710,8 @@ def format(self, format_string='$_$@$%@$+$=', **kwargs):
         color    = kwargs.get('color', False)
         length = len(format_string)
         out = StringIO()
-        escape = compiler = False
+        named = escape = compiler = False
+        named_str = fmt = ''
 
         def write(s, c):
             if color:
@@ -1636,9 +1753,12 @@ def write(s, c):
                 elif c == '#':
                     out.write('-' + fmt % (self.dag_hash(7)))
                 elif c == '$':
-                    if fmt != '':
+                    if fmt != '%s':
                         raise ValueError("Can't use format width with $$.")
                     out.write('$')
+                elif c == '{':
+                    named = True
+                    named_str = ''
                 escape = False
 
             elif compiler:
@@ -1652,6 +1772,43 @@ def write(s, c):
                     out.write(c)
                 compiler = False
 
+            elif named:
+                if not c == '}':
+                    if i == length - 1:
+                        raise ValueError("Error: unterminated ${ in format: '%s'"
+                                         % format_string)
+                    named_str += c
+                    continue;
+                if named_str == 'PACKAGE':
+                    write(fmt % self.name, '@')
+                if named_str == 'VERSION':
+                    if self.versions and self.versions != _any_version:
+                        write(fmt % str(self.versions), '@')
+                elif named_str == 'COMPILER':
+                    if self.compiler:
+                        write(fmt % self.compiler, '%')
+                elif named_str == 'COMPILERNAME':
+                    if self.compiler:
+                        write(fmt % self.compiler.name, '%')
+                elif named_str == 'COMPILERVER':
+                    if self.compiler:
+                        write(fmt % self.compiler.versions, '%')
+                elif named_str == 'OPTIONS':
+                    if self.variants:
+                        write(fmt % str(self.variants), '+')
+                elif named_str == 'ARCHITECTURE':
+                    if self.architecture:
+                        write(fmt % str(self.architecture), '=')
+                elif named_str == 'SHA1':
+                    if self.dependencies:
+                        out.write(fmt % str(self.dag_hash(7)))
+                elif named_str == 'SPACK_ROOT':
+                    out.write(fmt % spack.prefix)
+                elif named_str == 'SPACK_INSTALL':
+                    out.write(fmt % spack.install_path)
+
+                named = False
+
             elif c == '$':
                 escape = True
                 if i == length - 1:
@@ -1782,6 +1939,7 @@ def spec(self):
         spec.variants = VariantMap(spec)
         spec.architecture = None
         spec.compiler = None
+        spec.external = None
         spec.dependents   = DependencyMap()
         spec.dependencies = DependencyMap()
         spec.namespace = spec_namespace

lib/spack/spack/stage.py

@@ -23,7 +23,7 @@
 # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 ##############################################################################
 import os
-import re
+import errno
 import shutil
 import tempfile
 from urlparse import urljoin
@@ -31,46 +31,65 @@
 import llnl.util.tty as tty
 from llnl.util.filesystem import *
 
+import spack.util.pattern as pattern
+
 import spack
 import spack.config
 import spack.fetch_strategy as fs
 import spack.error
 
-
 STAGE_PREFIX = 'spack-stage-'
 
 
 class Stage(object):
-    """A Stage object manaages a directory where some source code is
+    """Manages a temporary stage directory for building.
-       downloaded and built before being installed.  It handles
-       fetching the source code, either as an archive to be expanded
-       or by checking it out of a repository.  A stage's lifecycle
-       looks like this:
 
-       Stage()
+    A Stage object is a context manager that handles a directory where
-         Constructor creates the stage directory.
+    some source code is downloaded and built before being installed.
-       fetch()
+    It handles fetching the source code, either as an archive to be
-         Fetch a source archive into the stage.
+    expanded or by checking it out of a repository.  A stage's
-       expand_archive()
+    lifecycle looks like this:
-         Expand the source archive.
-       <install>
-         Build and install the archive.  This is handled by the Package class.
-       destroy()
-         Remove the stage once the package has been installed.
 
-       If spack.use_tmp_stage is True, spack will attempt to create stages
+    ```
-       in a tmp directory.  Otherwise, stages are created directly in
+    with Stage() as stage:      # Context manager creates and destroys the stage directory
-       spack.stage_path.
+        stage.fetch()           # Fetch a source archive into the stage.
+        stage.expand_archive()  # Expand the source archive.
+        <install>               # Build and install the archive. (handled by user of Stage)
+    ```
 
-       There are two kinds of stages: named and unnamed.  Named stages can
+    When used as a context manager, the stage is automatically
-       persist between runs of spack, e.g. if you fetched a tarball but
+    destroyed if no exception is raised by the context. If an
-       didn't finish building it, you won't have to fetch it again.
+    excpetion is raised, the stage is left in the filesystem and NOT
+    destroyed, for potential reuse later.
+
+    You can also use the stage's create/destroy functions manually,
+    like this:
+
+    ```
+    stage = Stage()
+    try:
+        stage.create()          # Explicitly create the stage directory.
+        stage.fetch()           # Fetch a source archive into the stage.
+        stage.expand_archive()  # Expand the source archive.
+        <install>               # Build and install the archive. (handled by user of Stage)
+    finally:
+        stage.destroy()         # Explicitly destroy the stage directory.
+    ```
+
+    If spack.use_tmp_stage is True, spack will attempt to create
+    stages in a tmp directory.  Otherwise, stages are created directly
+    in spack.stage_path.
+
+    There are two kinds of stages: named and unnamed.  Named stages
+    can persist between runs of spack, e.g. if you fetched a tarball
+    but didn't finish building it, you won't have to fetch it again.
 
     Unnamed stages are created using standard mkdtemp mechanisms or
     similar, and are intended to persist for only one run of spack.
     """
 
-    def __init__(self, url_or_fetch_strategy, **kwargs):
+    def __init__(self, url_or_fetch_strategy,
+                 name=None, mirror_path=None, keep=False):
         """Create a stage object.
            Parameters:
              url_or_fetch_strategy
@@ -82,6 +101,17 @@ def __init__(self, url_or_fetch_strategy, **kwargs):
                  and will persist between runs (or if you construct another
                  stage object later).  If name is not provided, then this
                  stage will be given a unique name automatically.
+
+             mirror_path
+                 If provided, Stage will search Spack's mirrors for
+                 this archive at the mirror_path, before using the
+                 default fetch strategy.
+
+             keep
+                 By default, when used as a context manager, the Stage
+                 is deleted on exit when no exceptions are raised.
+                 Pass True to keep the stage intact even if no
+                 exceptions are raised.
         """
         # TODO: fetch/stage coupling needs to be reworked -- the logic
         # TODO: here is convoluted and not modular enough.
@@ -95,22 +125,50 @@ def __init__(self, url_or_fetch_strategy, **kwargs):
         self.default_fetcher = self.fetcher  # self.fetcher can change with mirrors.
         self.skip_checksum_for_mirror = True  # used for mirrored archives of repositories.
 
-        self.name = kwargs.get('name')
+        # TODO : this uses a protected member of tempfile, but seemed the only way to get a temporary name
-        self.mirror_path = kwargs.get('mirror_path')
+        # TODO : besides, the temporary link name won't be the same as the temporary stage area in tmp_root
+        self.name = name
+        if name is None:
+            self.name = STAGE_PREFIX + next(tempfile._get_candidate_names())
+        self.mirror_path = mirror_path
         self.tmp_root = find_tmp_root()
 
-        self.path = None
+        # Try to construct here a temporary name for the stage directory
-        self._setup()
+        # If this is a named stage, then construct a named path.
+        self.path = join_path(spack.stage_path, self.name)
+
+        # Flag to decide whether to delete the stage folder on exit or not
+        self.keep = keep
 
 
-    def _cleanup_dead_links(self):
+    def __enter__(self):
-        """Remove any dead links in the stage directory."""
+        """
-        for file in os.listdir(spack.stage_path):
+        Entering a stage context will create the stage directory
-            path = join_path(spack.stage_path, file)
+
-            if os.path.islink(path):
+        Returns:
-                real_path = os.path.realpath(path)
+            self
-                if not os.path.exists(path):
+        """
-                    os.unlink(path)
+        self.create()
+        return self
+
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        """
+        Exiting from a stage context will delete the stage directory unless:
+        - it was explicitly requested not to do so
+        - an exception has been raised
+
+        Args:
+            exc_type: exception type
+            exc_val: exception value
+            exc_tb: exception traceback
+
+        Returns:
+            Boolean
+        """
+        # Delete when there are no exceptions, unless asked to keep.
+        if exc_type is None and not self.keep:
+            self.destroy()
 
 
     def _need_to_create_path(self):
@@ -133,8 +191,8 @@ def _need_to_create_path(self):
             real_tmp = os.path.realpath(self.tmp_root)
 
             if spack.use_tmp_stage:
-                # If we're using a tmp dir, it's a link, and it points at the right spot,
+                # If we're using a tmp dir, it's a link, and it points at the
-                # then keep it.
+                # right spot, then keep it.
                 if (real_path.startswith(real_tmp) and os.path.exists(real_path)):
                     return False
                 else:
@@ -149,56 +207,6 @@ def _need_to_create_path(self):
 
         return False
 
-
-    def _setup(self):
-        """Creates the stage directory.
-           If spack.use_tmp_stage is False, the stage directory is created
-           directly under spack.stage_path.
-
-           If spack.use_tmp_stage is True, this will attempt to create a
-           stage in a temporary directory and link it into spack.stage_path.
-           Spack will use the first writable location in spack.tmp_dirs to
-           create a stage.  If there is no valid location in tmp_dirs, fall
-           back to making the stage inside spack.stage_path.
-        """
-        # Create the top-level stage directory
-        mkdirp(spack.stage_path)
-        self._cleanup_dead_links()
-
-        # If this is a named stage, then construct a named path.
-        if self.name is not None:
-            self.path = join_path(spack.stage_path, self.name)
-
-        # If this is a temporary stage, them make the temp directory
-        tmp_dir = None
-        if self.tmp_root:
-            if self.name is None:
-                # Unnamed tmp root.  Link the path in
-                tmp_dir = tempfile.mkdtemp('', STAGE_PREFIX, self.tmp_root)
-                self.name = os.path.basename(tmp_dir)
-                self.path = join_path(spack.stage_path, self.name)
-                if self._need_to_create_path():
-                    os.symlink(tmp_dir, self.path)
-
-            else:
-                if self._need_to_create_path():
-                    tmp_dir = tempfile.mkdtemp('', STAGE_PREFIX, self.tmp_root)
-                    os.symlink(tmp_dir, self.path)
-
-        # if we're not using a tmp dir, create the stage directly in the
-        # stage dir, rather than linking to it.
-        else:
-            if self.name is None:
-                self.path = tempfile.mkdtemp('', STAGE_PREFIX, spack.stage_path)
-                self.name = os.path.basename(self.path)
-            else:
-                if self._need_to_create_path():
-                    mkdirp(self.path)
-
-        # Make sure we can actually do something with the stage we made.
-        ensure_access(self.path)
-
-
     @property
     def archive_file(self):
         """Path to the source archive within this stage directory."""
@@ -215,29 +223,35 @@ def archive_file(self):
         else:
             return None
 
-
     @property
     def source_path(self):
-        """Returns the path to the expanded/checked out source code
+        """Returns the path to the expanded/checked out source code.
-           within this fetch strategy's path.
 
-           This assumes nothing else is going ot be put in the
+        To find the source code, this method searches for the first
-           FetchStrategy's path.  It searches for the first
+        subdirectory of the stage that it can find, and returns it.
-           subdirectory of the path it can find, then returns that.
+        This assumes nothing besides the archive file will be in the
+        stage path, but it has the advantage that we don't need to
+        know the name of the archive or its contents.
+
+        If the fetch strategy is not supposed to expand the downloaded
+        file, it will just return the stage path. If the archive needs
+        to be expanded, it will return None when no archive is found.
         """
+        if isinstance(self.fetcher, fs.URLFetchStrategy):
+            if not self.fetcher.expand_archive:
+                return self.path
+
         for p in [os.path.join(self.path, f) for f in os.listdir(self.path)]:
             if os.path.isdir(p):
                 return p
         return None
 
-
     def chdir(self):
         """Changes directory to the stage path.  Or dies if it is not set up."""
         if os.path.isdir(self.path):
             os.chdir(self.path)
         else:
-            tty.die("Setup failed: no such directory: " + self.path)
+            raise ChdirError("Setup failed: no such directory: " + self.path)
-
 
     def fetch(self, mirror_only=False):
         """Downloads an archive or checks out code from a repository."""
@@ -282,7 +296,7 @@ def fetch(self, mirror_only=False):
                 self.fetcher = fetcher
                 self.fetcher.fetch()
                 break
-            except spack.error.SpackError, e:
+            except spack.error.SpackError as e:
                 tty.msg("Fetching from %s failed." % fetcher)
                 tty.debug(e)
                 continue
@@ -291,7 +305,6 @@ def fetch(self, mirror_only=False):
             self.fetcher = self.default_fetcher
             raise fs.FetchError(errMessage, None)
 
-
     def check(self):
         """Check the downloaded archive against a checksum digest.
            No-op if this stage checks code out of a repository."""
@@ -305,14 +318,17 @@ def check(self):
         else:
             self.fetcher.check()
 
-
     def expand_archive(self):
         """Changes to the stage directory and attempt to expand the downloaded
            archive.  Fail if the stage is not set up or if the archive is not yet
            downloaded.
         """
+        archive_dir = self.source_path
+        if not archive_dir:
             self.fetcher.expand()
-
+            tty.msg("Created stage in %s" % self.path)
+        else:
+            tty.msg("Already staged %s in %s" % (self.name, self.path))
 
     def chdir_to_source(self):
         """Changes directory to the expanded archive directory.
@@ -326,16 +342,41 @@ def chdir_to_source(self):
             if not os.listdir(path):
                 tty.die("Archive was empty for %s" % self.name)
 
-
     def restage(self):
         """Removes the expanded archive path if it exists, then re-expands
            the archive.
         """
         self.fetcher.reset()
 
+    def create(self):
+        """
+        Creates the stage directory
+
+        If self.tmp_root evaluates to False, the stage directory is
+        created directly under spack.stage_path, otherwise this will
+        attempt to create a stage in a temporary directory and link it
+        into spack.stage_path.
+
+        Spack will use the first writable location in spack.tmp_dirs
+        to create a stage. If there is no valid location in tmp_dirs,
+        fall back to making the stage inside spack.stage_path.
+        """
+        # Create the top-level stage directory
+        mkdirp(spack.stage_path)
+        remove_dead_links(spack.stage_path)
+        # If a tmp_root exists then create a directory there and then link it in the stage area,
+        # otherwise create the stage directory in self.path
+        if self._need_to_create_path():
+            if self.tmp_root:
+                tmp_dir = tempfile.mkdtemp('', STAGE_PREFIX, self.tmp_root)
+                os.symlink(tmp_dir, self.path)
+            else:
+                mkdirp(self.path)
+        # Make sure we can actually do something with the stage we made.
+        ensure_access(self.path)
 
     def destroy(self):
-        """Remove this stage directory."""
+        """Removes this stage directory."""
         remove_linked_tree(self.path)
 
         # Make sure we don't end up in a removed directory
@@ -345,8 +386,82 @@ def destroy(self):
             os.chdir(os.path.dirname(self.path))
 
 
+class ResourceStage(Stage):
+    def __init__(self, url_or_fetch_strategy, root, resource, **kwargs):
+        super(ResourceStage, self).__init__(url_or_fetch_strategy, **kwargs)
+        self.root_stage = root
+        self.resource = resource
+
+    def expand_archive(self):
+        super(ResourceStage, self).expand_archive()
+        root_stage = self.root_stage
+        resource = self.resource
+        placement = os.path.basename(self.source_path) if resource.placement is None else resource.placement
+        if not isinstance(placement, dict):
+            placement = {'': placement}
+        # Make the paths in the dictionary absolute and link
+        for key, value in placement.iteritems():
+            target_path = join_path(root_stage.source_path, resource.destination)
+            destination_path = join_path(target_path, value)
+            source_path = join_path(self.source_path, key)
+
+            try:
+                os.makedirs(target_path)
+            except OSError as err:
+                if err.errno == errno.EEXIST and os.path.isdir(target_path):
+                    pass
+                else:
+                    raise
+
+            if not os.path.exists(destination_path):
+                # Create a symlink
+                tty.info('Moving resource stage\n\tsource : {stage}\n\tdestination : {destination}'.format(
+                    stage=source_path, destination=destination_path
+                ))
+                shutil.move(source_path, destination_path)
+
+
+@pattern.composite(method_list=['fetch', 'create', 'check', 'expand_archive',  'restage', 'destroy'])
+class StageComposite:
+    """
+    Composite for Stage type objects. The first item in this composite is considered to be the root package, and
+    operations that return a value are forwarded to it.
+    """
+    #
+    # __enter__ and __exit__ delegate to all stages in the composite.
+    #
+    def __enter__(self):
+        for item in self:
+            item.__enter__()
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        for item in reversed(self):
+            item.keep = getattr(self, 'keep', False)
+            item.__exit__(exc_type, exc_val, exc_tb)
+
+    #
+    # Below functions act only on the *first* stage in the composite.
+    #
+    @property
+    def source_path(self):
+        return self[0].source_path
+
+    @property
+    def path(self):
+        return self[0].path
+
+    def chdir_to_source(self):
+        return self[0].chdir_to_source()
+
+    @property
+    def archive_file(self):
+        return self[0].archive_file
+
+
 class DIYStage(object):
     """Simple class that allows any directory to be a spack stage."""
+
     def __init__(self, path):
         self.archive_file = None
         self.path = path
@@ -356,12 +471,16 @@ def chdir(self):
         if os.path.isdir(self.path):
             os.chdir(self.path)
         else:
-            tty.die("Setup failed: no such directory: " + self.path)
+            raise ChdirError("Setup failed: no such directory: " + self.path)
+
+    # DIY stages do nothing as context managers.
+    def __enter__(self): pass
+    def __exit__(self, exc_type, exc_val, exc_tb): pass
 
     def chdir_to_source(self):
         self.chdir()
 
-    def fetch(self):
+    def fetch(self, mirror_only):
         tty.msg("No need to fetch for DIY.")
 
     def check(self):
@@ -384,26 +503,12 @@ def _get_mirrors():
     return [val for name, val in config.iteritems()]
 
 
-
 def ensure_access(file=spack.stage_path):
     """Ensure we can access a directory and die with an error if we can't."""
     if not can_access(file):
         tty.die("Insufficient permissions for %s" % file)
 
 
-def remove_linked_tree(path):
-    """Removes a directory and its contents.  If the directory is a symlink,
-       follows the link and reamoves the real directory before removing the
-       link.
-    """
-    if os.path.exists(path):
-        if os.path.islink(path):
-            shutil.rmtree(os.path.realpath(path), True)
-            os.unlink(path)
-        else:
-            shutil.rmtree(path, True)
-
-
 def purge():
     """Remove all build directories in the top-level stage path."""
     if os.path.isdir(spack.stage_path):
@@ -432,19 +537,15 @@ def find_tmp_root():
 
 
 class StageError(spack.error.SpackError):
-    def __init__(self, message, long_message=None):
+    """"Superclass for all errors encountered during staging."""
-        super(self, StageError).__init__(message, long_message)
 
 
 class RestageError(StageError):
-    def __init__(self, message, long_msg=None):
+    """"Error encountered during restaging."""
-        super(RestageError, self).__init__(message, long_msg)
 
 
 class ChdirError(StageError):
-    def __init__(self, message, long_msg=None):
+    """Raised when Spack can't change directories."""
-        super(ChdirError, self).__init__(message, long_msg)
-
 
 # Keep this in namespace for convenience
 FailedDownloadError = fs.FailedDownloadError

lib/spack/spack/test/__init__.py

@@ -48,6 +48,7 @@
               'package_sanity',
               'config',
               'directory_layout',
+              'pattern',
               'python_version',
               'git_fetch',
               'svn_fetch',
@@ -64,7 +65,8 @@
               'lock',
               'database',
               'namespace_trie',
-              'yaml']
+              'yaml',
+              'sbang']
 
 
 def list_tests():

lib/spack/spack/test/cc.py

@@ -39,11 +39,11 @@
     'arg1',
     '-Wl,--start-group',
     'arg2',
-    '-Wl,-rpath=/first/rpath', 'arg3', '-Wl,-rpath', '-Wl,/second/rpath',
+    '-Wl,-rpath,/first/rpath', 'arg3', '-Wl,-rpath', '-Wl,/second/rpath',
     '-llib1', '-llib2',
     'arg4',
     '-Wl,--end-group',
-    '-Xlinker,-rpath', '-Xlinker,/third/rpath', '-Xlinker,-rpath=/fourth/rpath',
+    '-Xlinker', '-rpath', '-Xlinker', '/third/rpath', '-Xlinker', '-rpath', '-Xlinker', '/fourth/rpath',
     '-llib3', '-llib4',
     'arg5', 'arg6']
 
@@ -95,13 +95,13 @@ def test_cpp_mode(self):
     def test_ccld_mode(self):
         self.check_cc('dump-mode', [], "ccld")
         self.check_cc('dump-mode', ['foo.c', '-o', 'foo'], "ccld")
-        self.check_cc('dump-mode', ['foo.c', '-o', 'foo', '-Wl,-rpath=foo'], "ccld")
+        self.check_cc('dump-mode', ['foo.c', '-o', 'foo', '-Wl,-rpath,foo'], "ccld")
-        self.check_cc('dump-mode', ['foo.o', 'bar.o', 'baz.o', '-o', 'foo', '-Wl,-rpath=foo'], "ccld")
+        self.check_cc('dump-mode', ['foo.o', 'bar.o', 'baz.o', '-o', 'foo', '-Wl,-rpath,foo'], "ccld")
 
 
     def test_ld_mode(self):
         self.check_ld('dump-mode', [], "ld")
-        self.check_ld('dump-mode', ['foo.o', 'bar.o', 'baz.o', '-o', 'foo', '-Wl,-rpath=foo'], "ld")
+        self.check_ld('dump-mode', ['foo.o', 'bar.o', 'baz.o', '-o', 'foo', '-Wl,-rpath,foo'], "ld")
 
 
     def test_includes(self):

lib/spack/spack/test/concretize.py

@@ -22,10 +22,9 @@
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 ##############################################################################
-import unittest
-
 import spack
 from spack.spec import Spec, CompilerSpec
+from spack.concretize import find_spec
 from spack.test.mock_packages_test import *
 
 class ConcretizeTest(MockPackagesTest):
@@ -143,6 +142,34 @@ def test_concretize_with_provides_when(self):
                                 for spec in spack.repo.providers_for('mpi@3')))
 
 
+    def test_concretize_two_virtuals(self):
+        """Test a package with multiple virtual dependencies."""
+        s = Spec('hypre').concretize()
+
+
+    def test_concretize_two_virtuals_with_one_bound(self):
+        """Test a package with multiple virtual dependencies and one preset."""
+        s = Spec('hypre ^openblas').concretize()
+
+
+    def test_concretize_two_virtuals_with_two_bound(self):
+        """Test a package with multiple virtual dependencies and two of them preset."""
+        s = Spec('hypre ^openblas ^netlib-lapack').concretize()
+
+
+    def test_concretize_two_virtuals_with_dual_provider(self):
+        """Test a package with multiple virtual dependencies and force a provider
+           that provides both."""
+        s = Spec('hypre ^openblas-with-lapack').concretize()
+
+
+    def test_concretize_two_virtuals_with_dual_provider_and_a_conflict(self):
+        """Test a package with multiple virtual dependencies and force a provider
+           that provides both, and another conflicting package that provides one."""
+        s = Spec('hypre ^openblas-with-lapack ^netlib-lapack')
+        self.assertRaises(spack.spec.MultipleProviderError, s.concretize)
+
+
     def test_virtual_is_fully_expanded_for_callpath(self):
         # force dependence on fake "zmpi" by asking for MPI 10.0
         spec = Spec('callpath ^mpi@10.0')
@@ -192,3 +219,93 @@ def test_compiler_inheritance(self):
         # TODO: not exactly the syntax I would like.
         self.assertTrue(spec['libdwarf'].compiler.satisfies('clang'))
         self.assertTrue(spec['libelf'].compiler.satisfies('clang'))
+
+
+    def test_external_package(self):
+        spec = Spec('externaltool')
+        spec.concretize()
+
+        self.assertEqual(spec['externaltool'].external, '/path/to/external_tool')
+        self.assertFalse('externalprereq' in spec)
+        self.assertTrue(spec['externaltool'].compiler.satisfies('gcc'))
+
+
+    def test_nobuild_package(self):
+        got_error = False
+        spec = Spec('externaltool%clang')
+        try:
+            spec.concretize()
+        except spack.concretize.NoBuildError:
+            got_error = True
+        self.assertTrue(got_error)
+
+
+    def test_external_and_virtual(self):
+        spec = Spec('externaltest')
+        spec.concretize()
+        self.assertEqual(spec['externaltool'].external, '/path/to/external_tool')
+        self.assertEqual(spec['stuff'].external, '/path/to/external_virtual_gcc')
+        self.assertTrue(spec['externaltool'].compiler.satisfies('gcc'))
+        self.assertTrue(spec['stuff'].compiler.satisfies('gcc'))
+
+
+    def test_find_spec_parents(self):
+        """Tests the spec finding logic used by concretization. """
+        s = Spec('a +foo',
+                 Spec('b +foo',
+                      Spec('c'),
+                      Spec('d +foo')),
+                 Spec('e +foo'))
+
+        self.assertEqual('a', find_spec(s['b'], lambda s: '+foo' in s).name)
+
+
+    def test_find_spec_children(self):
+        s = Spec('a',
+                 Spec('b +foo',
+                      Spec('c'),
+                      Spec('d +foo')),
+                 Spec('e +foo'))
+        self.assertEqual('d', find_spec(s['b'], lambda s: '+foo' in s).name)
+        s = Spec('a',
+                 Spec('b +foo',
+                      Spec('c +foo'),
+                      Spec('d')),
+                 Spec('e +foo'))
+        self.assertEqual('c', find_spec(s['b'], lambda s: '+foo' in s).name)
+
+
+    def test_find_spec_sibling(self):
+        s = Spec('a',
+                 Spec('b +foo',
+                      Spec('c'),
+                      Spec('d')),
+                 Spec('e +foo'))
+        self.assertEqual('e', find_spec(s['b'], lambda s: '+foo' in s).name)
+        self.assertEqual('b', find_spec(s['e'], lambda s: '+foo' in s).name)
+
+        s = Spec('a',
+                 Spec('b +foo',
+                      Spec('c'),
+                      Spec('d')),
+                 Spec('e',
+                      Spec('f +foo')))
+        self.assertEqual('f', find_spec(s['b'], lambda s: '+foo' in s).name)
+
+
+    def test_find_spec_self(self):
+        s = Spec('a',
+                 Spec('b +foo',
+                      Spec('c'),
+                      Spec('d')),
+                 Spec('e'))
+        self.assertEqual('b', find_spec(s['b'], lambda s: '+foo' in s).name)
+
+
+    def test_find_spec_none(self):
+        s = Spec('a',
+                 Spec('b',
+                      Spec('c'),
+                      Spec('d')),
+                 Spec('e'))
+        self.assertEqual(None, find_spec(s['b'], lambda s: '+foo' in s))

lib/spack/spack/test/config.py

@@ -22,13 +22,13 @@
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 ##############################################################################
-import unittest
-import shutil
 import os
+import shutil
 from tempfile import mkdtemp
-from ordereddict_backport import OrderedDict
+
 import spack
 import spack.config
+from ordereddict_backport import OrderedDict
 from spack.test.mock_packages_test import *
 
 # Some sample compiler config data

lib/spack/spack/test/configure_guess.py

@@ -23,20 +23,15 @@
 # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 ##############################################################################
 import os
-import unittest
 import shutil
 import tempfile
+import unittest
 
 from llnl.util.filesystem import *
-
 from spack.cmd.create import ConfigureGuesser
 from spack.stage import Stage
-
-from spack.fetch_strategy import URLFetchStrategy
-from spack.directory_layout import YamlDirectoryLayout
-from spack.util.executable import which
 from spack.test.mock_packages_test import *
-from spack.test.mock_repo import MockArchive
+from spack.util.executable import which
 
 
 class InstallTest(unittest.TestCase):
@@ -52,8 +47,6 @@ def setUp(self):
 
     def tearDown(self):
         shutil.rmtree(self.tmpdir, ignore_errors=True)
-        if self.stage:
-            self.stage.destroy()
         os.chdir(self.orig_dir)
 
 
@@ -64,11 +57,11 @@ def check_archive(self, filename, system):
 
         url = 'file://' + join_path(os.getcwd(), 'archive.tar.gz')
         print url
-        self.stage = Stage(url)
+        with Stage(url) as stage:
-        self.stage.fetch()
+            stage.fetch()
 
             guesser = ConfigureGuesser()
-        guesser(self.stage)
+            guesser(stage)
             self.assertEqual(system, guesser.build_system)
 
 

lib/spack/spack/test/database.py

@@ -26,19 +26,18 @@
 These tests check the database is functioning properly,
 both in memory and in its file
 """
-import tempfile
-import shutil
 import multiprocessing
-
+import shutil
-from llnl.util.lock import *
+import tempfile
-from llnl.util.filesystem import join_path
 
 import spack
+from llnl.util.filesystem import join_path
+from llnl.util.lock import *
+from llnl.util.tty.colify import colify
 from spack.database import Database
 from spack.directory_layout import YamlDirectoryLayout
 from spack.test.mock_packages_test import *
 
-from llnl.util.tty.colify import colify
 
 def _print_ref_counts():
     """Print out all ref counts for the graph used here, for debugging"""

lib/spack/spack/test/directory_layout.py

@@ -25,20 +25,17 @@
 """\
 This test verifies that the Spack directory layout works properly.
 """
-import unittest
-import tempfile
-import shutil
 import os
-
+import shutil
-from llnl.util.filesystem import *
+import tempfile
 
 import spack
-from spack.spec import Spec
+from llnl.util.filesystem import *
-from spack.repository import RepoPath
 from spack.directory_layout import YamlDirectoryLayout
+from spack.repository import RepoPath
+from spack.spec import Spec
 from spack.test.mock_packages_test import *
 
-
 # number of packages to test (to reduce test time)
 max_packages = 10
 
@@ -69,6 +66,9 @@ def test_read_and_write_spec(self):
         packages = list(spack.repo.all_packages())[:max_packages]
 
         for pkg in packages:
+            if pkg.name.startswith('external'):
+                #External package tests cannot be installed
+                continue            
             spec = pkg.spec
 
             # If a spec fails to concretize, just skip it.  If it is a
@@ -174,6 +174,9 @@ def test_find(self):
         # Create install prefixes for all packages in the list
         installed_specs = {}
         for pkg in packages:
+            if pkg.name.startswith('external'):
+                #External package tests cannot be installed
+                continue
             spec = pkg.spec.concretized()
             installed_specs[spec.name] = spec
             self.layout.create_install_directory(spec)

lib/spack/spack/test/git_fetch.py

@@ -23,19 +23,12 @@
 # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 ##############################################################################
 import os
-import unittest
-import shutil
-import tempfile
-
-from llnl.util.filesystem import *
 
 import spack
-from spack.version import ver
+from llnl.util.filesystem import *
-from spack.stage import Stage
-from spack.util.executable import which
-
 from spack.test.mock_packages_test import *
 from spack.test.mock_repo import MockGitRepo
+from spack.version import ver
 
 
 class GitFetchTest(MockPackagesTest):
@@ -52,19 +45,15 @@ def setUp(self):
         spec.concretize()
         self.pkg = spack.repo.get(spec, new=True)
 
-
     def tearDown(self):
         """Destroy the stage space used by this test."""
         super(GitFetchTest, self).tearDown()
         self.repo.destroy()
-        self.pkg.do_clean()
-
 
     def assert_rev(self, rev):
         """Check that the current git revision is equal to the supplied rev."""
         self.assertEqual(self.repo.rev_hash('HEAD'), self.repo.rev_hash(rev))
 
-
     def try_fetch(self, rev, test_file, args):
         """Tries to:
            1. Fetch the repo using a fetch strategy constructed with
@@ -76,6 +65,7 @@ def try_fetch(self, rev, test_file, args):
         """
         self.pkg.versions[ver('git')] = args
 
+        with self.pkg.stage:
             self.pkg.do_stage()
             self.assert_rev(rev)
 

lib/spack/spack/test/hg_fetch.py

@@ -23,16 +23,12 @@
 # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 ##############################################################################
 import os
-import unittest
-
-from llnl.util.filesystem import *
-
 import spack
+
 from spack.version import ver
-from spack.stage import Stage
-from spack.util.executable import which
-from spack.test.mock_packages_test import *
 from spack.test.mock_repo import MockHgRepo
+from llnl.util.filesystem import *
+from spack.test.mock_packages_test import *
 
 
 class HgFetchTest(MockPackagesTest):
@@ -49,13 +45,10 @@ def setUp(self):
         spec.concretize()
         self.pkg = spack.repo.get(spec, new=True)
 
-
     def tearDown(self):
         """Destroy the stage space used by this test."""
         super(HgFetchTest, self).tearDown()
         self.repo.destroy()
-        self.pkg.do_clean()
-
 
     def try_fetch(self, rev, test_file, args):
         """Tries to:
@@ -68,6 +61,7 @@ def try_fetch(self, rev, test_file, args):
         """
         self.pkg.versions[ver('hg')] = args
 
+        with self.pkg.stage:
             self.pkg.do_stage()
             self.assertEqual(self.repo.get_rev(), rev)
 

lib/spack/spack/test/install.py

@@ -22,18 +22,13 @@
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 ##############################################################################
-import os
-import unittest
 import shutil
 import tempfile
 
-from llnl.util.filesystem import *
-
 import spack
-from spack.stage import Stage
+from llnl.util.filesystem import *
-from spack.fetch_strategy import URLFetchStrategy
 from spack.directory_layout import YamlDirectoryLayout
-from spack.util.executable import which
+from spack.fetch_strategy import URLFetchStrategy, FetchStrategyComposite
 from spack.test.mock_packages_test import *
 from spack.test.mock_repo import MockArchive
 
@@ -79,7 +74,10 @@ def test_install_and_uninstall(self):
         pkg = spack.repo.get(spec)
 
         # Fake the URL for the package so it downloads from a file.
-        pkg.fetcher = URLFetchStrategy(self.repo.url)
+
+        fetcher = FetchStrategyComposite()
+        fetcher.append(URLFetchStrategy(self.repo.url))
+        pkg.fetcher = fetcher
 
         try:
             pkg.do_install()

lib/spack/spack/test/link_tree.py

@@ -24,8 +24,6 @@
 ##############################################################################
 import os
 import unittest
-import shutil
-import tempfile
 
 from llnl.util.filesystem import *
 from llnl.util.link_tree import LinkTree
@@ -38,6 +36,7 @@ class LinkTreeTest(unittest.TestCase):
 
     def setUp(self):
         self.stage = Stage('link-tree-test')
+        self.stage.create()
 
         with working_dir(self.stage.path):
             touchp('source/1')
@@ -51,9 +50,7 @@ def setUp(self):
         source_path = os.path.join(self.stage.path, 'source')
         self.link_tree = LinkTree(source_path)
 
-
     def tearDown(self):
-        if self.stage:
         self.stage.destroy()
 
 

lib/spack/spack/test/lock.py

@@ -25,15 +25,13 @@
 """
 These tests ensure that our lock works correctly.
 """
-import unittest
-import os
-import tempfile
 import shutil
+import tempfile
+import unittest
 from multiprocessing import Process
 
-from llnl.util.lock import *
 from llnl.util.filesystem import join_path, touch
-
+from llnl.util.lock import *
 from spack.util.multiproc import Barrier
 
 # This is the longest a failed test will take, as the barriers will

lib/spack/spack/test/make_executable.py

@@ -28,13 +28,13 @@
 This just tests whether the right args are getting passed to make.
 """
 import os
-import unittest
-import tempfile
 import shutil
+import tempfile
+import unittest
 
 from llnl.util.filesystem import *
-from spack.util.environment import path_put_first
 from spack.build_environment import MakeExecutable
+from spack.util.environment import path_put_first
 
 
 class MakeExecutableTest(unittest.TestCase):

lib/spack/spack/test/mirror.py

@@ -23,11 +23,10 @@
 # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 ##############################################################################
 import os
-from filecmp import dircmp
-
 import spack
 import spack.mirror
-from spack.util.compression import decompressor_for
+
+from filecmp import dircmp
 from spack.test.mock_packages_test import *
 from spack.test.mock_repo import *
 
@@ -74,14 +73,14 @@ def set_up_package(self, name, MockRepoClass, url_attr):
 
 
     def check_mirror(self):
-        stage = Stage('spack-mirror-test')
+        with Stage('spack-mirror-test') as stage:
             mirror_root = join_path(stage.path, 'test-mirror')
 
             # register mirror with spack config
             mirrors = { 'spack-mirror-test' : 'file://' + mirror_root }
             spack.config.update_config('mirrors', mirrors)
 
-        try:
+
             os.chdir(stage.path)
             spack.mirror.create(
                 mirror_root, self.repos, no_checksum=True)
@@ -103,31 +102,22 @@ def check_mirror(self):
                     pkg = spec.package
 
                     saved_checksum_setting = spack.do_checksum
-                try:
+                    with pkg.stage:
                         # Stage the archive from the mirror and cd to it.
                         spack.do_checksum = False
                         pkg.do_stage(mirror_only=True)
-
                         # Compare the original repo with the expanded archive
                         original_path = mock_repo.path
                         if 'svn' in name:
                             # have to check out the svn repo to compare.
                             original_path = join_path(mock_repo.path, 'checked_out')
                             svn('checkout', mock_repo.url, original_path)
-
                         dcmp = dircmp(original_path, pkg.stage.source_path)
-
                         # make sure there are no new files in the expanded tarball
                         self.assertFalse(dcmp.right_only)
-
                         # and that all original files are present.
                         self.assertTrue(all(l in exclude for l in dcmp.left_only))
-
-                finally:
                         spack.do_checksum = saved_checksum_setting
-                    pkg.do_clean()
-        finally:
-            stage.destroy()
 
 
     def test_git_mirror(self):

lib/spack/spack/test/mock_packages_test.py

@@ -22,17 +22,15 @@
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 ##############################################################################
-import sys
 import os
 import shutil
-import unittest
 import tempfile
-from ordereddict_backport import OrderedDict
+import unittest
-
-from llnl.util.filesystem import mkdirp
 
 import spack
 import spack.config
+from llnl.util.filesystem import mkdirp
+from ordereddict_backport import OrderedDict
 from spack.repository import RepoPath
 from spack.spec import Spec
 
@@ -51,6 +49,19 @@
       fc: /path/to/gfortran
 """
 
+mock_packages_config = """\
+packages:
+  externaltool:
+    buildable: False
+    paths:
+      externaltool@1.0%gcc@4.5.0: /path/to/external_tool
+  externalvirtual:
+    buildable: False
+    paths:
+      externalvirtual@2.0%clang@3.3: /path/to/external_virtual_clang
+      externalvirtual@1.0%gcc@4.5.0: /path/to/external_virtual_gcc
+"""
+
 class MockPackagesTest(unittest.TestCase):
     def initmock(self):
         # Use the mock packages database for these tests.  This allows
@@ -68,9 +79,10 @@ def initmock(self):
         self.mock_user_config = os.path.join(self.temp_config, 'user')
         mkdirp(self.mock_site_config)
         mkdirp(self.mock_user_config)
-        comp_yaml = os.path.join(self.mock_site_config, 'compilers.yaml')
+        for confs in [('compilers.yaml', mock_compiler_config), ('packages.yaml', mock_packages_config)]:
-        with open(comp_yaml, 'w') as f:
+            conf_yaml = os.path.join(self.mock_site_config, confs[0])
-            f.write(mock_compiler_config)
+            with open(conf_yaml, 'w') as f:
+                f.write(confs[1])
 
         # TODO: Mocking this up is kind of brittle b/c ConfigScope
         # TODO: constructor modifies config_scopes.  Make it cleaner.

lib/spack/spack/test/mock_repo.py

@@ -26,13 +26,9 @@
 import shutil
 
 from llnl.util.filesystem import *
-
-import spack
-from spack.version import ver
 from spack.stage import Stage
 from spack.util.executable import which
 
-
 #
 # VCS Systems used by mock repo code.
 #

lib/spack/spack/test/multimethod.py

@@ -25,14 +25,11 @@
 """
 Test for multi_method dispatch.
 """
-import unittest
 
 import spack
 from spack.multimethod import *
-from spack.version import *
-from spack.spec import Spec
-from spack.multimethod import when
 from spack.test.mock_packages_test import *
+from spack.version import *
 
 
 class MultiMethodTest(MockPackagesTest):

lib/spack/spack/test/namespace_trie.py

@@ -6,7 +6,7 @@
 # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
 # LLNL-CODE-647188
 #
-# For details, see https://llnl.github.io/spack
+# For details, see https://software.llnl.gov/spack
 # Please also see the LICENSE file for our notice and the LGPL.
 #
 # This program is free software; you can redistribute it and/or modify
@@ -23,6 +23,7 @@
 # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 ##############################################################################
 import unittest
+
 from spack.util.naming import NamespaceTrie
 
 

lib/spack/spack/test/optional_deps.py

@@ -22,10 +22,8 @@
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 ##############################################################################
-import unittest
 
-import spack
+from spack.spec import Spec
-from spack.spec import Spec, CompilerSpec
 from spack.test.mock_packages_test import *
 
 class ConcretizeTest(MockPackagesTest):

lib/spack/spack/test/packages.py

@@ -22,14 +22,12 @@
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 ##############################################################################
-import unittest
-
-from llnl.util.filesystem import join_path
 
 import spack
+from llnl.util.filesystem import join_path
 from spack.repository import Repo
-from spack.util.naming import mod_to_class
 from spack.test.mock_packages_test import *
+from spack.util.naming import mod_to_class
 
 
 class PackagesTest(MockPackagesTest):

lib/spack/spack/test/pattern.py

@@ -0,0 +1,104 @@
+##############################################################################
+# Copyright (c) 2013, Lawrence Livermore National Security, LLC.
+# Produced at the Lawrence Livermore National Laboratory.
+#
+# This file is part of Spack.
+# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
+# LLNL-CODE-647188
+#
+# For details, see https://github.com/llnl/spack
+# Please also see the LICENSE file for our notice and the LGPL.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License (as published by
+# the Free Software Foundation) version 2.1 dated February 1999.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
+# conditions of the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+##############################################################################
+
+import unittest
+
+import spack.util.pattern as pattern
+
+
+class CompositeTest(unittest.TestCase):
+
+    def setUp(self):
+        class Base:
+            counter = 0
+
+            def add(self):
+                raise NotImplemented('add not implemented')
+
+            def subtract(self):
+                raise NotImplemented('subtract not implemented')
+
+        class One(Base):
+            def add(self):
+                Base.counter += 1
+
+            def subtract(self):
+                Base.counter -= 1
+
+        class Two(Base):
+            def add(self):
+                Base.counter += 2
+
+            def subtract(self):
+                Base.counter -= 2
+
+        self.Base = Base
+        self.One = One
+        self.Two = Two
+
+    def test_composite_from_method_list(self):
+
+        @pattern.composite(method_list=['add', 'subtract'])
+        class CompositeFromMethodList:
+            pass
+
+        composite = CompositeFromMethodList()
+        composite.append(self.One())
+        composite.append(self.Two())
+        composite.add()
+        self.assertEqual(self.Base.counter, 3)
+        composite.pop()
+        composite.subtract()
+        self.assertEqual(self.Base.counter, 2)
+
+    def test_composite_from_interface(self):
+
+        @pattern.composite(interface=self.Base)
+        class CompositeFromInterface:
+            pass
+
+        composite = CompositeFromInterface()
+        composite.append(self.One())
+        composite.append(self.Two())
+        composite.add()
+        self.assertEqual(self.Base.counter, 3)
+        composite.pop()
+        composite.subtract()
+        self.assertEqual(self.Base.counter, 2)
+
+    def test_error_conditions(self):
+
+        def wrong_container():
+            @pattern.composite(interface=self.Base, container=2)
+            class CompositeFromInterface:
+                pass
+
+        def no_methods():
+            @pattern.composite()
+            class CompositeFromInterface:
+                pass
+
+        self.assertRaises(TypeError, wrong_container)
+        self.assertRaises(TypeError, no_methods)

lib/spack/spack/test/python_version.py

@@ -28,12 +28,11 @@
 Spack was originally 2.7, but enough systems in 2014 are still using
 2.6 on their frontend nodes that we need 2.6 to get adopted.
 """
-import unittest
 import os
 import re
+import unittest
 
 import llnl.util.tty as tty
-
 import pyqver2
 import spack
 

lib/spack/spack/test/sbang.py

@@ -0,0 +1,93 @@
+##############################################################################
+# Copyright (c) 2013-2015, Lawrence Livermore National Security, LLC.
+# Produced at the Lawrence Livermore National Laboratory.
+#
+# This file is part of Spack.
+# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
+# LLNL-CODE-647188
+#
+# For details, see https://github.com/llnl/spack
+# Please also see the LICENSE file for our notice and the LGPL.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License (as published by
+# the Free Software Foundation) version 2.1 dated February 1999.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
+# conditions of the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+##############################################################################
+"""\
+Test that Spack's shebang filtering works correctly.
+"""
+import os
+import unittest
+import tempfile
+import shutil
+
+from llnl.util.filesystem import *
+from spack.hooks.sbang import filter_shebangs_in_directory
+import spack
+
+short_line = "#!/this/is/short/bin/bash\n"
+long_line  = "#!/this/" + ('x' * 200) + "/is/long\n"
+sbang_line = '#!/bin/bash %s/bin/sbang\n' % spack.spack_root
+last_line = "last!\n"
+
+class SbangTest(unittest.TestCase):
+    def setUp(self):
+        self.tempdir = tempfile.mkdtemp()
+
+        # make sure we can ignore non-files
+        directory = os.path.join(self.tempdir, 'dir')
+        mkdirp(directory)
+
+        # Script with short shebang
+        self.short_shebang = os.path.join(self.tempdir, 'short')
+        with open(self.short_shebang, 'w') as f:
+            f.write(short_line)
+            f.write(last_line)
+
+        # Script with long shebang
+        self.long_shebang = os.path.join(self.tempdir, 'long')
+        with open(self.long_shebang, 'w') as f:
+            f.write(long_line)
+            f.write(last_line)
+
+        # Script already using sbang.
+        self.has_shebang = os.path.join(self.tempdir, 'shebang')
+        with open(self.has_shebang, 'w') as f:
+            f.write(sbang_line)
+            f.write(long_line)
+            f.write(last_line)
+
+
+    def tearDown(self):
+         shutil.rmtree(self.tempdir, ignore_errors=True)
+
+
+
+    def test_shebang_handling(self):
+        filter_shebangs_in_directory(self.tempdir)
+
+        # Make sure this is untouched
+        with open(self.short_shebang, 'r') as f:
+            self.assertEqual(f.readline(), short_line)
+            self.assertEqual(f.readline(), last_line)
+
+        # Make sure this got patched.
+        with open(self.long_shebang, 'r') as f:
+            self.assertEqual(f.readline(), sbang_line)
+            self.assertEqual(f.readline(), long_line)
+            self.assertEqual(f.readline(), last_line)
+
+        # Make sure this is untouched
+        with open(self.has_shebang, 'r') as f:
+            self.assertEqual(f.readline(), sbang_line)
+            self.assertEqual(f.readline(), long_line)
+            self.assertEqual(f.readline(), last_line)

lib/spack/spack/test/spec_dag.py

@@ -31,8 +31,6 @@
 import spack
 import spack.package
 
-from llnl.util.lang import list_modules
-
 from spack.spec import Spec
 from spack.test.mock_packages_test import *
 

lib/spack/spack/test/spec_semantics.py

@@ -22,7 +22,6 @@
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 ##############################################################################
-import unittest
 from spack.spec import *
 from spack.test.mock_packages_test import *
 

lib/spack/spack/test/spec_syntax.py

@@ -23,9 +23,10 @@
 # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 ##############################################################################
 import unittest
+
 import spack.spec
-from spack.spec import *
 from spack.parse import Token
+from spack.spec import *
 
 # Sample output for a complex lexing.
 complex_lex = [Token(ID, 'mvapich_foo'),

lib/spack/spack/test/stage.py

@@ -25,15 +25,13 @@
 """\
 Test that the Stage class works correctly.
 """
-import unittest
-import shutil
 import os
-import getpass
+import shutil
+import unittest
 from contextlib import *
 
-from llnl.util.filesystem import *
-
 import spack
+from llnl.util.filesystem import *
 from spack.stage import Stage
 from spack.util.executable import which
 
@@ -192,96 +190,73 @@ def check_destroy(self, stage, stage_name):
 
     def test_setup_and_destroy_name_with_tmp(self):
         with use_tmp(True):
-            stage = Stage(archive_url, name=stage_name)
+            with Stage(archive_url, name=stage_name) as stage:
                 self.check_setup(stage, stage_name)
-
-            stage.destroy()
             self.check_destroy(stage, stage_name)
 
 
     def test_setup_and_destroy_name_without_tmp(self):
         with use_tmp(False):
-            stage = Stage(archive_url, name=stage_name)
+            with Stage(archive_url, name=stage_name) as stage:
                 self.check_setup(stage, stage_name)
-
-            stage.destroy()
             self.check_destroy(stage, stage_name)
 
 
     def test_setup_and_destroy_no_name_with_tmp(self):
         with use_tmp(True):
-            stage = Stage(archive_url)
+            with Stage(archive_url) as stage:
                 self.check_setup(stage, None)
-
-            stage.destroy()
             self.check_destroy(stage, None)
 
 
     def test_setup_and_destroy_no_name_without_tmp(self):
         with use_tmp(False):
-            stage = Stage(archive_url)
+            with Stage(archive_url) as stage:
                 self.check_setup(stage, None)
-
-            stage.destroy()
             self.check_destroy(stage, None)
 
 
     def test_chdir(self):
-        stage = Stage(archive_url, name=stage_name)
+        with Stage(archive_url, name=stage_name) as stage:
-
             stage.chdir()
             self.check_setup(stage, stage_name)
             self.check_chdir(stage, stage_name)
-
-        stage.destroy()
         self.check_destroy(stage, stage_name)
 
 
     def test_fetch(self):
-        stage = Stage(archive_url, name=stage_name)
+        with Stage(archive_url, name=stage_name) as stage:
-
             stage.fetch()
             self.check_setup(stage, stage_name)
             self.check_chdir(stage, stage_name)
             self.check_fetch(stage, stage_name)
-
-        stage.destroy()
         self.check_destroy(stage, stage_name)
 
 
     def test_expand_archive(self):
-        stage = Stage(archive_url, name=stage_name)
+        with Stage(archive_url, name=stage_name) as stage:
-
             stage.fetch()
             self.check_setup(stage, stage_name)
             self.check_fetch(stage, stage_name)
-
             stage.expand_archive()
             self.check_expand_archive(stage, stage_name)
-
-        stage.destroy()
         self.check_destroy(stage, stage_name)
 
 
     def test_expand_archive(self):
-        stage = Stage(archive_url, name=stage_name)
+        with Stage(archive_url, name=stage_name) as stage:
-
             stage.fetch()
             self.check_setup(stage, stage_name)
             self.check_fetch(stage, stage_name)
-
             stage.expand_archive()
             stage.chdir_to_source()
             self.check_expand_archive(stage, stage_name)
             self.check_chdir_to_source(stage, stage_name)
-
-        stage.destroy()
         self.check_destroy(stage, stage_name)
 
 
     def test_restage(self):
-        stage = Stage(archive_url, name=stage_name)
+        with Stage(archive_url, name=stage_name) as stage:
-
             stage.fetch()
             stage.expand_archive()
             stage.chdir_to_source()
@@ -298,10 +273,42 @@ def test_restage(self):
             stage.restage()
             self.check_chdir(stage, stage_name)
             self.check_fetch(stage, stage_name)
-
             stage.chdir_to_source()
             self.check_chdir_to_source(stage, stage_name)
             self.assertFalse('foobar' in os.listdir(stage.source_path))
-
-        stage.destroy()
         self.check_destroy(stage, stage_name)
+
+
+    def test_no_keep_without_exceptions(self):
+        with Stage(archive_url, name=stage_name, keep=False) as stage:
+            pass
+        self.check_destroy(stage, stage_name)
+
+
+    def test_keep_without_exceptions(self):
+        with Stage(archive_url, name=stage_name, keep=True) as stage:
+            pass
+        path = self.get_stage_path(stage, stage_name)
+        self.assertTrue(os.path.isdir(path))
+
+
+    def test_no_keep_with_exceptions(self):
+        try:
+            with Stage(archive_url, name=stage_name, keep=False) as stage:
+                raise Exception()
+
+            path = self.get_stage_path(stage, stage_name)
+            self.assertTrue(os.path.isdir(path))
+        except:
+            pass # ignore here.
+
+
+    def test_keep_exceptions(self):
+        try:
+            with Stage(archive_url, name=stage_name, keep=True) as stage:
+                raise Exception()
+
+            path = self.get_stage_path(stage, stage_name)
+            self.assertTrue(os.path.isdir(path))
+        except:
+            pass # ignore here.

lib/spack/spack/test/svn_fetch.py

@@ -24,18 +24,12 @@
 ##############################################################################
 import os
 import re
-import unittest
-import shutil
-import tempfile
-
-from llnl.util.filesystem import *
-
 import spack
-from spack.version import ver
+
-from spack.stage import Stage
-from spack.util.executable import which
-from spack.test.mock_packages_test import *
 from spack.test.mock_repo import svn, MockSvnRepo
+from spack.version import ver
+from spack.test.mock_packages_test import *
+from llnl.util.filesystem import *
 
 
 class SvnFetchTest(MockPackagesTest):
@@ -51,13 +45,10 @@ def setUp(self):
         spec.concretize()
         self.pkg = spack.repo.get(spec, new=True)
 
-
     def tearDown(self):
         """Destroy the stage space used by this test."""
         super(SvnFetchTest, self).tearDown()
         self.repo.destroy()
-        self.pkg.do_clean()
-
 
     def assert_rev(self, rev):
         """Check that the current revision is equal to the supplied rev."""
@@ -70,7 +61,6 @@ def get_rev():
                     return match.group(1)
         self.assertEqual(get_rev(), rev)
 
-
     def try_fetch(self, rev, test_file, args):
         """Tries to:
            1. Fetch the repo using a fetch strategy constructed with
@@ -82,6 +72,7 @@ def try_fetch(self, rev, test_file, args):
         """
         self.pkg.versions[ver('svn')] = args
 
+        with self.pkg.stage:
             self.pkg.do_stage()
             self.assert_rev(rev)
 

lib/spack/spack/test/tally_plugin.py

@@ -6,7 +6,7 @@
 # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
 # LLNL-CODE-647188
 #
-# For details, see https://scalability-llnl.github.io/spack
+# For details, see https://scalability-software.llnl.gov/spack
 # Please also see the LICENSE file for our notice and the LGPL.
 #
 # This program is free software; you can redistribute it and/or modify
@@ -22,10 +22,10 @@
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 ##############################################################################
-from nose.plugins import Plugin
-
 import os
 
+from nose.plugins import Plugin
+
 class Tally(Plugin):
     name = 'tally'
 

lib/spack/spack/test/unit_install.py

@@ -22,10 +22,11 @@
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 ##############################################################################
-import unittest
 import itertools
+import unittest
 
 import spack
+
 test_install = __import__("spack.cmd.test-install",
     fromlist=["BuildId", "create_test_output", "TestResult"])
 

lib/spack/spack/test/url_extrapolate.py

@@ -25,10 +25,7 @@
 """\
 Tests ability of spack to extrapolate URL versions from existing versions.
 """
-import spack
 import spack.url as url
-from spack.spec import Spec
-from spack.version import ver
 from spack.test.mock_packages_test import *
 
 

lib/spack/spack/test/url_parse.py

@@ -27,8 +27,8 @@
 detection in Homebrew.
 """
 import unittest
+
 import spack.url as url
-from pprint import pprint
 
 
 class UrlParseTest(unittest.TestCase):

lib/spack/spack/test/url_substitution.py

@@ -27,7 +27,6 @@
 """
 import unittest
 
-import spack
 import spack.url as url
 
 

lib/spack/spack/test/versions.py

@@ -28,6 +28,7 @@
 where it makes sense.
 """
 import unittest
+
 from spack.version import *
 
 

lib/spack/spack/test/yaml.py

@@ -26,6 +26,7 @@
 Test Spack's custom YAML format.
 """
 import unittest
+
 import spack.util.spack_yaml as syaml
 
 test_file = """\

lib/spack/spack/url.py

@@ -225,7 +225,7 @@ def parse_version_offset(path):
         (r'_((\d+\.)+\d+[a-z]?)[.]orig$', stem),
 
         # e.g. http://www.openssl.org/source/openssl-0.9.8s.tar.gz
-        (r'-([^-]+(-alpha|-beta)?)', stem),
+        (r'-v?([^-]+(-alpha|-beta)?)', stem),
 
         # e.g. astyle_1.23_macosx.tar.gz
         (r'_([^_]+(_alpha|_beta)?)', stem),

lib/spack/spack/util/environment.py

@@ -63,3 +63,10 @@ def pop_keys(dictionary, *keys):
     for key in keys:
         if key in dictionary:
             dictionary.pop(key)
+
+
+def dump_environment(path):
+    """Dump the current environment out to a file."""
+    with open(path, 'w') as env_file:
+        for key,val in sorted(os.environ.items()):
+            env_file.write("%s=%s\n" % (key, val))

lib/spack/spack/util/pattern.py

@@ -0,0 +1,116 @@
+##############################################################################
+# Copyright (c) 2013, Lawrence Livermore National Security, LLC.
+# Produced at the Lawrence Livermore National Laboratory.
+#
+# This file is part of Spack.
+# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
+# LLNL-CODE-647188
+#
+# For details, see https://github.com/llnl/spack
+# Please also see the LICENSE file for our notice and the LGPL.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License (as published by
+# the Free Software Foundation) version 2.1 dated February 1999.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
+# conditions of the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+##############################################################################
+import inspect
+import collections
+import functools
+
+
+def composite(interface=None, method_list=None, container=list):
+    """
+    Returns a class decorator that patches a class adding all the methods it needs to be a composite for a given
+    interface.
+
+    :param interface: class exposing the interface to which the composite object must conform. Only non-private and
+    non-special methods will be taken into account
+
+    :param method_list: names of methods that should be part of the composite
+
+    :param container: container for the composite object (default = list). Must fulfill the MutableSequence contract.
+    The composite class will expose the container API to manage object composition
+
+    :return: class decorator
+    """
+    # Check if container fulfills the MutableSequence contract and raise an exception if it doesn't
+    # The patched class returned by the decorator will inherit from the container class to expose the
+    # interface needed to manage objects composition
+    if not issubclass(container, collections.MutableSequence):
+        raise TypeError("Container must fulfill the MutableSequence contract")
+
+    # Check if at least one of the 'interface' or the 'method_list' arguments are defined
+    if interface is None and method_list is None:
+        raise TypeError("Either 'interface' or 'method_list' must be defined on a call to composite")
+
+    def cls_decorator(cls):
+        # Retrieve the base class of the composite. Inspect its methods and decide which ones will be overridden
+        def no_special_no_private(x):
+            return inspect.ismethod(x) and not x.__name__.startswith('_')
+
+        # Patch the behavior of each of the methods in the previous list. This is done associating an instance of the
+        # descriptor below to any method that needs to be patched.
+        class IterateOver(object):
+            """
+            Decorator used to patch methods in a composite. It iterates over all the items in the instance containing the
+            associated attribute and calls for each of them an attribute with the same name
+            """
+            def __init__(self, name, func=None):
+                self.name = name
+                self.func = func
+
+            def __get__(self, instance, owner):
+                def getter(*args, **kwargs):
+                    for item in instance:
+                        getattr(item, self.name)(*args, **kwargs)
+                # If we are using this descriptor to wrap a method from an interface, then we must conditionally
+                # use the `functools.wraps` decorator to set the appropriate fields.
+                if self.func is not None:
+                    getter = functools.wraps(self.func)(getter)
+                return getter
+
+        dictionary_for_type_call = {}
+        # Construct a dictionary with the methods explicitly passed as name
+        if method_list is not None:
+            # python@2.7: method_list_dict = {name: IterateOver(name) for name in method_list}
+            method_list_dict = {}
+            for name in method_list:
+                method_list_dict[name] = IterateOver(name)
+            dictionary_for_type_call.update(method_list_dict)
+        # Construct a dictionary with the methods inspected from the interface
+        if interface is not None:
+            ##########
+            # python@2.7: interface_methods = {name: method for name, method in inspect.getmembers(interface, predicate=no_special_no_private)}
+            interface_methods = {}
+            for name, method in inspect.getmembers(interface, predicate=no_special_no_private):
+                interface_methods[name] = method
+            ##########
+            # python@2.7: interface_methods_dict = {name: IterateOver(name, method) for name, method in interface_methods.iteritems()}
+            interface_methods_dict = {}
+            for name, method in interface_methods.iteritems():
+                interface_methods_dict[name] = IterateOver(name, method)
+            ##########
+            dictionary_for_type_call.update(interface_methods_dict)
+        # Get the methods that are defined in the scope of the composite class and override any previous definition
+        ##########
+        # python@2.7: cls_method = {name: method for name, method in inspect.getmembers(cls, predicate=inspect.ismethod)}
+        cls_method = {}
+        for name, method in inspect.getmembers(cls, predicate=inspect.ismethod):
+            cls_method[name] = method
+        ##########
+        dictionary_for_type_call.update(cls_method)
+        # Generate the new class on the fly and return it
+        # FIXME : inherit from interface if we start to use ABC classes?
+        wrapper_class = type(cls.__name__, (cls, container), dictionary_for_type_call)
+        return wrapper_class
+
+    return cls_decorator

lib/spack/spack/util/web.py

@@ -86,12 +86,12 @@ def _spider(args):
 
         if not "Content-type" in resp.headers:
             tty.debug("ignoring page " + url)
-            return pages
+            return pages, links
 
         if not resp.headers["Content-type"].startswith('text/html'):
             tty.debug("ignoring page " + url + " with content type " +
                       resp.headers["Content-type"])
-            return pages
+            return pages, links
 
         # Do the real GET request when we know it's just HTML.
         req.get_method = lambda: "GET"

var/spack/repos/builtin.mock/packages/externalprereq/package.py

@@ -0,0 +1,34 @@
+##############################################################################
+# Copyright (c) 2013, Lawrence Livermore National Security, LLC.
+# Produced at the Lawrence Livermore National Laboratory.
+#
+# This file is part of Spack.
+# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
+# LLNL-CODE-647188
+#
+# For details, see https://scalability-llnl.github.io/spack
+# Please also see the LICENSE file for our notice and the LGPL.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License (as published by
+# the Free Software Foundation) version 2.1 dated February 1999.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
+# conditions of the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+##############################################################################
+from spack import *
+
+class Externalprereq(Package):
+    homepage = "http://somewhere.com"
+    url      = "http://somewhere.com/prereq-1.0.tar.gz"
+
+    version('1.4', 'f1234567890abcdef1234567890abcde')
+
+    def install(self, spec, prefix):
+        pass

var/spack/repos/builtin.mock/packages/externaltest/package.py

@@ -0,0 +1,37 @@
+##############################################################################
+# Copyright (c) 2013, Lawrence Livermore National Security, LLC.
+# Produced at the Lawrence Livermore National Laboratory.
+#
+# This file is part of Spack.
+# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
+# LLNL-CODE-647188
+#
+# For details, see https://scalability-llnl.github.io/spack
+# Please also see the LICENSE file for our notice and the LGPL.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License (as published by
+# the Free Software Foundation) version 2.1 dated February 1999.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
+# conditions of the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+##############################################################################
+from spack import *
+
+class Externaltest(Package):
+    homepage = "http://somewhere.com"
+    url      = "http://somewhere.com/test-1.0.tar.gz"
+
+    version('1.0', '1234567890abcdef1234567890abcdef')
+
+    depends_on('stuff')
+    depends_on('externaltool')
+
+    def install(self, spec, prefix):
+        pass

var/spack/repos/builtin.mock/packages/externaltool/package.py

@@ -0,0 +1,36 @@
+##############################################################################
+# Copyright (c) 2013, Lawrence Livermore National Security, LLC.
+# Produced at the Lawrence Livermore National Laboratory.
+#
+# This file is part of Spack.
+# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
+# LLNL-CODE-647188
+#
+# For details, see https://scalability-llnl.github.io/spack
+# Please also see the LICENSE file for our notice and the LGPL.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License (as published by
+# the Free Software Foundation) version 2.1 dated February 1999.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
+# conditions of the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+##############################################################################
+from spack import *
+
+class Externaltool(Package):
+    homepage = "http://somewhere.com"
+    url      = "http://somewhere.com/tool-1.0.tar.gz"
+
+    version('1.0', '1234567890abcdef1234567890abcdef')
+
+    depends_on('externalprereq')
+
+    def install(self, spec, prefix):
+        pass

var/spack/repos/builtin.mock/packages/externalvirtual/package.py

@@ -0,0 +1,37 @@
+##############################################################################
+# Copyright (c) 2013, Lawrence Livermore National Security, LLC.
+# Produced at the Lawrence Livermore National Laboratory.
+#
+# This file is part of Spack.
+# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
+# LLNL-CODE-647188
+#
+# For details, see https://scalability-llnl.github.io/spack
+# Please also see the LICENSE file for our notice and the LGPL.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License (as published by
+# the Free Software Foundation) version 2.1 dated February 1999.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
+# conditions of the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+##############################################################################
+from spack import *
+
+class Externalvirtual(Package):
+    homepage = "http://somewhere.com"
+    url      = "http://somewhere.com/stuff-1.0.tar.gz"
+
+    version('1.0', '1234567890abcdef1234567890abcdef')
+    version('2.0', '234567890abcdef1234567890abcdef1')
+
+    provides('stuff')
+
+    def install(self, spec, prefix):
+        pass

var/spack/repos/builtin.mock/packages/hypre/package.py

@@ -0,0 +1,39 @@
+##############################################################################
+# Copyright (c) 2013-2015, Lawrence Livermore National Security, LLC.
+# Produced at the Lawrence Livermore National Laboratory.
+#
+# This file is part of Spack.
+# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
+# LLNL-CODE-647188
+#
+# For details, see https://github.com/llnl/spack
+# Please also see the LICENSE file for our notice and the LGPL.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License (as published by
+# the Free Software Foundation) version 2.1 dated February 1999.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
+# conditions of the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+##############################################################################
+from spack import *
+
+class Hypre(Package):
+    """Hypre is included here as an example of a package that depends on
+       both LAPACK and BLAS."""
+    homepage = "http://www.openblas.net"
+    url      = "http://github.com/xianyi/OpenBLAS/archive/v0.2.15.tar.gz"
+
+    version('0.2.15', 'b1190f3d3471685f17cfd1ec1d252ac9')
+
+    depends_on('lapack')
+    depends_on('blas')
+
+    def install(self, spec, prefix):
+        pass

var/spack/repos/builtin.mock/packages/mpich/package.py

@@ -38,6 +38,7 @@ class Mpich(Package):
     version('3.0.2', 'foobarbaz')
     version('3.0.1', 'foobarbaz')
     version('3.0', 'foobarbaz')
+    version('1.0', 'foobarbas')
 
     provides('mpi@:3', when='@3:')
     provides('mpi@:1', when='@:1')

var/spack/repos/builtin.mock/packages/openblas-with-lapack/package.py

@@ -0,0 +1,38 @@
+##############################################################################
+# Copyright (c) 2013-2015, Lawrence Livermore National Security, LLC.
+# Produced at the Lawrence Livermore National Laboratory.
+#
+# This file is part of Spack.
+# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
+# LLNL-CODE-647188
+#
+# For details, see https://github.com/llnl/spack
+# Please also see the LICENSE file for our notice and the LGPL.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License (as published by
+# the Free Software Foundation) version 2.1 dated February 1999.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
+# conditions of the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+##############################################################################
+from spack import *
+
+class OpenblasWithLapack(Package):
+    """Dummy version of OpenBLAS that also provides LAPACK, for testing."""
+    homepage = "http://www.openblas.net"
+    url      = "http://github.com/xianyi/OpenBLAS/archive/v0.2.15.tar.gz"
+
+    version('0.2.15', 'b1190f3d3471685f17cfd1ec1d252ac9')
+
+    provides('lapack')
+    provides('blas')
+
+    def install(self, spec, prefix):
+        pass

var/spack/repos/builtin/packages/autoconf/package.py

@@ -6,6 +6,7 @@ class Autoconf(Package):
     url      = "http://ftp.gnu.org/gnu/autoconf/autoconf-2.69.tar.gz"
 
     version('2.69', '82d05e03b93e45f5a39b828dc9c6c29b')
+    version('2.62', '6c1f3b3734999035d77da5024aab4fbd')
 
     def install(self, spec, prefix):
         configure("--prefix=%s" % prefix)

var/spack/repos/builtin/packages/automake/package.py

@@ -5,7 +5,9 @@ class Automake(Package):
     homepage = "http://www.gnu.org/software/automake/"
     url      = "http://ftp.gnu.org/gnu/automake/automake-1.14.tar.gz"
 
+    version('1.15',   '716946a105ca228ab545fc37a70df3a3')
     version('1.14.1', 'd052a3e884631b9c7892f2efce542d75')
+    version('1.11.6', '0286dc30295b62985ca51919202ecfcc')
 
     depends_on('autoconf')
 

var/spack/repos/builtin/packages/binutils/package.py

@@ -4,10 +4,13 @@ class Binutils(Package):
     """GNU binutils, which contain the linker, assembler, objdump and others"""
     homepage = "http://www.gnu.org/software/binutils/"
 
-    version('2.25', 'd9f3303f802a5b6b0bb73a335ab89d66',url="ftp://ftp.gnu.org/gnu/binutils/binutils-2.25.tar.bz2")
+    url="https://ftp.gnu.org/gnu/binutils/binutils-2.25.tar.bz2"
-    version('2.24', 'e0f71a7b2ddab0f8612336ac81d9636b',url="ftp://ftp.gnu.org/gnu/binutils/binutils-2.24.tar.bz2")
+
-    version('2.23.2', '4f8fa651e35ef262edc01d60fb45702e',url="ftp://ftp.gnu.org/gnu/binutils/binutils-2.23.2.tar.bz2")
+    version('2.26', '64146a0faa3b411ba774f47d41de239f')
-    version('2.20.1', '2b9dc8f2b7dbd5ec5992c6e29de0b764',url="ftp://ftp.gnu.org/gnu/binutils/binutils-2.20.1.tar.bz2")
+    version('2.25', 'd9f3303f802a5b6b0bb73a335ab89d66')
+    version('2.24', 'e0f71a7b2ddab0f8612336ac81d9636b')
+    version('2.23.2', '4f8fa651e35ef262edc01d60fb45702e')
+    version('2.20.1', '2b9dc8f2b7dbd5ec5992c6e29de0b764')
 
     # Add a patch that creates binutils libiberty_pic.a which is preferred by OpenSpeedShop and cbtf-krell
     variant('krellpatch', default=False, description="build with openspeedshop based patch.")

var/spack/repos/builtin/packages/blitz/package.py

@@ -0,0 +1,15 @@
+from spack import *
+
+class Blitz(Package):
+    """N-dimensional arrays for C++"""
+    homepage = "http://github.com/blitzpp/blitz"
+    url      = "https://github.com/blitzpp/blitz/tarball/1.0.0"
+
+    version('1.0.0', '9f040b9827fe22228a892603671a77af')
+
+    # No dependencies
+
+    def install(self, spec, prefix):
+        configure('--prefix=%s' % prefix)
+        make()
+        make("install")

var/spack/repos/builtin/packages/cgal/package.py

@@ -46,6 +46,7 @@ class Cgal(Package):
     depends_on('mpfr')
     depends_on('gmp')
     depends_on('zlib')
+    depends_on('cmake')
 
     # FIXME : Qt5 dependency missing (needs Qt5 and OpenGL)
     # FIXME : Optional third party libraries missing

var/spack/repos/builtin/packages/cmake/package.py

@@ -30,6 +30,7 @@ class Cmake(Package):
     homepage  = 'https://www.cmake.org'
     url       = 'https://cmake.org/files/v3.4/cmake-3.4.3.tar.gz'
 
+    version('3.5.0',    '33c5d09d4c33d4ffcc63578a6ba8777e')
     version('3.4.3',    '4cb3ff35b2472aae70f542116d616e63')
     version('3.4.0',    'cd3034e0a44256a0917e254167217fc8')
     version('3.3.1',    '52638576f4e1e621fed6c3410d3a1b12')
@@ -37,16 +38,48 @@ class Cmake(Package):
     version('2.8.10.2', '097278785da7182ec0aea8769d06860c')
 
     variant('ncurses', default=True, description='Enables the build of the ncurses gui')
+    variant('qt', default=False, description='Enables the build of cmake-gui')
+    variant('doc', default=False, description='Enables the generation of html and man page documentation')
+
     depends_on('ncurses', when='+ncurses')
+    depends_on('qt', when='+qt')
+    depends_on('python@2.7.11:', when='+doc')
+    depends_on('py-sphinx', when='+doc')
 
     def url_for_version(self, version):
         """Handle CMake's version-based custom URLs."""
         return 'https://cmake.org/files/v%s/cmake-%s.tar.gz' % (version.up_to(2), version)
 
+    def validate(self, spec):
+        """
+        Checks if incompatible versions of qt were specified
+
+        :param spec: spec of the package
+        :raises RuntimeError: in case of inconsistencies
+        """
+
+        if '+qt' in spec and spec.satisfies('^qt@5.4.0'):
+            msg = 'qt-5.4.0 has broken CMake modules.'
+            raise RuntimeError(msg)
 
     def install(self, spec, prefix):
-        configure('--prefix='   + prefix,
+        # Consistency check
-                  '--parallel=' + str(make_jobs),
+        self.validate(spec)
-                  '--', '-DCMAKE_USE_OPENSSL=ON')
+
+        # configure, build, install:
+        options = ['--prefix=%s' % prefix]
+        options.append('--parallel=%s' % str(make_jobs))
+
+        if '+qt' in spec:
+            options.append('--qt-gui')
+
+        if '+doc' in spec:
+            options.append('--sphinx-html')
+            options.append('--sphinx-man')
+
+        options.append('--')
+        options.append('-DCMAKE_USE_OPENSSL=ON')
+
+        configure(*options)
         make()
         make('install')