From 59f89dd3bee0ee2b668554c537d1d18404108ec4 Mon Sep 17 00:00:00 2001
From: Matthew LeGendre <legendre1@llnl.gov>
Date: Fri, 29 May 2015 11:04:57 -0700
Subject: [PATCH 01/11] Allow long names in format string variables

---
 lib/spack/spack/spec.py | 71 ++++++++++++++++++++++++++++++++++++-----
 1 file changed, 63 insertions(+), 8 deletions(-)

diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py
index e1fbb84423..3b5d16c7a7 100644
--- a/lib/spack/spack/spec.py
+++ b/lib/spack/spack/spec.py
@@ -1503,14 +1503,28 @@ def format(self, format_string='$_$@$%@$+$=', **kwargs):
            in the format string.  The format strings you can provide are::
 
                $_   Package name
-               $@   Version
-               $%   Compiler
-               $%@  Compiler & compiler version
-               $+   Options
-               $=   Architecture
-               $#   7-char prefix of DAG hash
+               $@   Version with '@' prefix
+               $%   Compiler with '%' prefix
+               $%@  Compiler with '%' prefix & compiler version with '@' prefix
+               $+   Options 
+               $=   Architecture with '=' prefix
+               $#   7-char prefix of DAG hash with '-' prefix
                $$   $
 
+               You can also use full-string versions, which leave off the prefixes:
+
+               ${PACKAGE}       Package name
+               ${VERSION}       Version
+               ${COMPILER}      Full compiler string
+               ${COMPILERNAME}  Compiler name
+               ${COMPILERVER}   Compiler version
+               ${OPTIONS}       Options
+               ${ARCHITECTURE}  Architecture
+               ${SHA1}          Dependencies 8-char sha1 prefix
+
+               ${SPACK_ROOT}    The spack root directory
+               ${SPACK_INSTALL} The default spack install directory, ${SPACK_PREFIX}/opt
+
            Optionally you can provide a width, e.g. $20_ for a 20-wide name.
            Like printf, you can provide '-' for left justification, e.g.
            $-20_ for a left-justified name.
@@ -1526,7 +1540,8 @@ def format(self, format_string='$_$@$%@$+$=', **kwargs):
         color    = kwargs.get('color', False)
         length = len(format_string)
         out = StringIO()
-        escape = compiler = False
+        named = escape = compiler = False
+        named_str = fmt = ''
 
         def write(s, c):
             if color:
@@ -1566,9 +1581,12 @@ def write(s, c):
                 elif c == '#':
                     out.write('-' + fmt % (self.dag_hash(7)))
                 elif c == '$':
-                    if fmt != '':
+                    if fmt != '%s':
                         raise ValueError("Can't use format width with $$.")
                     out.write('$')
+                elif c == '{':
+                    named = True
+                    named_str = ''
                 escape = False
 
             elif compiler:
@@ -1582,6 +1600,43 @@ def write(s, c):
                     out.write(c)
                 compiler = False
 
+            elif named:
+                if not c == '}':
+                    if i == length - 1:
+                        raise ValueError("Error: unterminated ${ in format: '%s'"
+                                         % format_string)
+                    named_str += c
+                    continue;
+                if named_str == 'PACKAGE':
+                    write(fmt % self.name, '@')
+                if named_str == 'VERSION':
+                    if self.versions and self.versions != _any_version:
+                        write(fmt % str(self.versions), '@')
+                elif named_str == 'COMPILER':
+                    if self.compiler:
+                        write(fmt % self.compiler, '%')
+                elif named_str == 'COMPILERNAME':
+                    if self.compiler:
+                        write(fmt % self.compiler.name, '%')
+                elif named_str == 'COMPILERVER':
+                    if self.compiler:
+                        write(fmt % self.compiler.versions, '%')
+                elif named_str == 'OPTIONS':
+                    if self.variants:
+                        write(fmt % str(self.variants), '+')
+                elif named_str == 'ARCHITECTURE':
+                    if self.architecture:
+                        write(fmt % str(self.architecture), '=')
+                elif named_str == 'SHA1':
+                    if self.dependencies:
+                        out.write(fmt % str(self.dep_hash(8)))
+                elif named_str == 'SPACK_ROOT':
+                    out.write(fmt % spack.prefix)
+                elif named_str == 'SPACK_INSTALL':
+                    out.write(fmt % spack.install_path)
+
+                named = False
+
             elif c == '$':
                 escape = True
                 if i == length - 1:

From b5c597b31864826050162b358998e240761c5d7e Mon Sep 17 00:00:00 2001
From: Matthew LeGendre <legendre1@llnl.gov>
Date: Fri, 29 May 2015 12:19:57 -0700
Subject: [PATCH 02/11] Allow specs to be sorted based on preferred packages,
 versions, compilers, variants and dependencies.

---
 lib/spack/spack/__init__.py           |   8 ++
 lib/spack/spack/config.py             |  12 +-
 lib/spack/spack/preferred_packages.py | 172 ++++++++++++++++++++++++++
 lib/spack/spack/spec.py               |  34 +++++
 4 files changed, 222 insertions(+), 4 deletions(-)
 create mode 100644 lib/spack/spack/preferred_packages.py

diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py
index caa09eb6e0..bd8478fb98 100644
--- a/lib/spack/spack/__init__.py
+++ b/lib/spack/spack/__init__.py
@@ -69,6 +69,14 @@
 from spack.directory_layout import YamlDirectoryLayout
 install_layout = YamlDirectoryLayout(install_path)
 
+#
+# This controls how packages are sorted when trying to choose
+# the most preferred package.  More preferred packages are sorted
+# first.
+#
+from spack.preferred_packages import PreferredPackages
+pkgsort = PreferredPackages()
+
 #
 # This controls how things are concretized in spack.
 # Replace it with a subclass if you want different
diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py
index 3e91958c2c..dbe225960a 100644
--- a/lib/spack/spack/config.py
+++ b/lib/spack/spack/config.py
@@ -89,8 +89,8 @@
 import os
 import exceptions
 import sys
+import copy
 
-from external.ordereddict import OrderedDict
 from llnl.util.lang import memoized
 import spack.error
 
@@ -114,8 +114,7 @@ def __init__(self, n, f, m):
 
 _ConfigCategory('compilers', 'compilers.yaml', True)
 _ConfigCategory('mirrors', 'mirrors.yaml', True)
-_ConfigCategory('view', 'views.yaml', True)
-_ConfigCategory('order', 'orders.yaml', True)
+_ConfigCategory('preferred', 'preferred.yaml', True)
 
 """Names of scopes and their corresponding configuration files."""
 config_scopes = [('site', os.path.join(spack.etc_path, 'spack')),
@@ -156,7 +155,7 @@ def _merge_dicts(d1, d2):
     """Recursively merges two configuration trees, with entries
        in d2 taking precedence over d1"""
     if not d1:
-        return d2.copy()
+        return copy.copy(d2)
     if not d2:
         return d1
 
@@ -230,6 +229,11 @@ def get_mirror_config():
     return get_config('mirrors')
 
 
+def get_preferred_config():
+    """Get the preferred configuration from config files"""
+    return get_config('preferred')
+
+
 def get_config_scope_dirname(scope):
     """For a scope return the config directory"""
     global config_scopes
diff --git a/lib/spack/spack/preferred_packages.py b/lib/spack/spack/preferred_packages.py
new file mode 100644
index 0000000000..248508fe80
--- /dev/null
+++ b/lib/spack/spack/preferred_packages.py
@@ -0,0 +1,172 @@
+##############################################################################
+# Copyright (c) 2013, Lawrence Livermore National Security, LLC.
+# Produced at the Lawrence Livermore National Laboratory.
+#
+# This file is part of Spack.
+# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
+# LLNL-CODE-647188
+#
+# For details, see https://scalability-llnl.github.io/spack
+# Please also see the LICENSE file for our notice and the LGPL.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License (as published by
+# the Free Software Foundation) version 2.1 dated February 1999.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
+# conditions of the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+##############################################################################
+
+import spack
+from spack.version import *
+
+class PreferredPackages(object):
+    _default_order = {'compiler' : [ 'gcc', 'intel', 'clang', 'pgi', 'xlc' ] }, #Arbitrary, but consistent
+
+    def __init__(self):
+        self.preferred = spack.config.get_preferred_config()
+        self._spec_for_pkgname_cache = {}
+
+    #Given a package name, sort component (e.g, version, compiler, ...), and
+    # a second_key (used by providers), return the list 
+    def _order_for_package(self, pkgname, component, second_key):
+        pkglist = [pkgname]
+        pkglist.append('all')
+        for pkg in pkglist:
+            if not pkg in self.preferred:
+                continue
+            orders = self.preferred[pkg]
+            if not type(orders) is dict:
+                continue
+            if not component in orders:
+                continue
+            order = orders[component]
+            if type(order) is dict:
+                if not second_key in order:
+                    continue;
+                order = order[second_key]
+            if not type(order) is str:
+                tty.die('Expected version list in preferred config, but got %s' % str(order))
+            order_list = order.split(',')
+            return [s.strip() for s in order_list]
+        return []
+            
+
+    # A generic sorting function. Given a package name and sort 
+    # component, return less-than-0, 0, or greater-than-0 if 
+    # a is respectively less-than, equal to, or greater than b.
+    def _component_compare(self, pkgname, component, a, b, reverse_natural_compare, second_key):
+        orderlist = self._order_for_package(pkgname, component, second_key)
+        a_in_list = str(a) in orderlist
+        b_in_list = str(b) in orderlist
+        if a_in_list and not b_in_list:
+            return -1
+        elif b_in_list and not a_in_list:
+            return 1
+
+        cmp_a = None
+        cmp_b = None
+        reverse = None
+        if not a_in_list and not b_in_list:
+            cmp_a = a
+            cmp_b = b
+            reverse = -1 if reverse_natural_compare else 1
+        else:
+            cmp_a = orderlist.index(str(a))
+            cmp_b = orderlist.index(str(b))
+            reverse = 1
+        
+        if cmp_a < cmp_b:
+            return -1 * reverse
+        elif cmp_a > cmp_b:
+            return 1 * reverse
+        else:
+            return 0
+
+
+    # A sorting function for specs.  Similar to component_compare, but
+    # a and b are considered to match entries in the sorting list if they
+    # satisfy the list component.  
+    def _spec_compare(self, pkgname, component, a, b, reverse_natural_compare, second_key):
+        specs = self._spec_for_pkgname(pkgname, component, second_key)
+        a_index = None
+        b_index = None
+        reverse = -1 if reverse_natural_compare else 1
+        for i, cspec in enumerate(specs):
+            if a_index == None and cspec.satisfies(a):
+                a_index = i
+                if b_index:
+                    break
+            if b_index == None and cspec.satisfies(b):
+                b_index = i
+                if a_index:
+                    break
+
+        if   a_index != None and b_index == None: return -1
+        elif a_index == None and b_index != None: return 1
+        elif a_index != None and b_index == a_index: return -1 * cmp(a, b)
+        elif a_index != None and b_index != None and a_index != b_index: return cmp(a_index, b_index)
+        elif a < b: return 1 * reverse
+        elif b < a: return -1 * reverse
+        else: return 0
+
+
+    # Given a sort order specified by the pkgname/component/second_key, return
+    # a list of CompilerSpecs, VersionLists, or Specs for that sorting list.
+    def _spec_for_pkgname(self, pkgname, component, second_key):
+        key = (pkgname, component, second_key)
+        if not key in self._spec_for_pkgname_cache:
+            pkglist = self._order_for_package(pkgname, component, second_key)
+            if not pkglist: 
+                if component in self._default_order:
+                    pkglist = self._default_order[component]
+            if component == 'compiler':
+                self._spec_for_pkgname_cache[key] = [spack.spec.CompilerSpec(s) for s in pkglist]
+            elif component == 'version':
+                self._spec_for_pkgname_cache[key] = [VersionList(s) for s in pkglist]
+            else:
+                self._spec_for_pkgname_cache[key] = [spack.spec.Spec(s) for s in pkglist]
+        return self._spec_for_pkgname_cache[key]
+
+    
+    def provider_compare(self, pkgname, provider_str, a, b):
+        """Return less-than-0, 0, or greater than 0 if a is respecively less-than, equal-to, or 
+           greater-than b. A and b are possible implementations of provider_str.
+           One provider is less-than another if it is preferred over the other.
+           For example, provider_compare('scorep', 'mpi', 'mvapich', 'openmpi') would return -1 if
+           mvapich should be preferred over openmpi for scorep."""
+        return self._spec_compare(pkgname, 'providers', a, b, False, provider_str)
+
+
+    def version_compare(self, pkgname, a, b):
+        """Return less-than-0, 0, or greater than 0 if version a of pkgname is 
+           respecively less-than, equal-to, or greater-than version b of pkgname. 
+           One version is less-than another if it is preferred over the other."""
+        return self._spec_compare(pkgname, 'version', a, b, False, None)
+
+    
+    def variant_compare(self, pkgname, a, b):
+        """Return less-than-0, 0, or greater than 0 if variant a of pkgname is 
+           respecively less-than, equal-to, or greater-than variant b of pkgname. 
+           One variant is less-than another if it is preferred over the other."""
+        return self._component_compare(pkgname, 'variant', a, b, False, None)
+
+
+    def architecture_compare(self, pkgname, a, b):
+        """Return less-than-0, 0, or greater than 0 if architecture a of pkgname is 
+           respecively less-than, equal-to, or greater-than architecture b of pkgname. 
+           One architecture is less-than another if it is preferred over the other."""
+        return self._component_compare(pkgname, 'architecture', a, b, False, None)
+
+
+    def compiler_compare(self, pkgname, a, b):
+        """Return less-than-0, 0, or greater than 0 if compiler a of pkgname is 
+           respecively less-than, equal-to, or greater-than compiler b of pkgname. 
+           One compiler is less-than another if it is preferred over the other."""
+        return self._spec_compare(pkgname, 'compiler', a, b, False, None)
diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py
index 3b5d16c7a7..83b1416e36 100644
--- a/lib/spack/spack/spec.py
+++ b/lib/spack/spack/spec.py
@@ -1653,6 +1653,40 @@ def dep_string(self):
         return ''.join("^" + dep.format() for dep in self.sorted_deps())
 
 
+    def __cmp__(self, other):
+        #Package name sort order is not configurable, always goes alphabetical
+        if self.name != other.name:
+            return cmp(self.name, other.name)
+        
+        #Package version is second in compare order
+        pkgname = self.name
+        if self.versions != other.versions:
+            return spack.pkgsort.version_compare(pkgname,
+                         self.versions, other.versions)
+
+        #Compiler is third
+        if self.compiler != other.compiler:
+            return spack.pkgsort.compiler_compare(pkgname,
+                         self.compiler, other.compiler)
+
+        #Variants
+        if self.variants != other.variants:
+            return spack.pkgsort.variant_compare(pkgname,
+                         self.variants, other.variants)
+
+        #Architecture
+        if self.architecture != other.architecture:
+            return spack.pkgsort.architecture_compare(pkgname,
+                         self.architecture, other.architecture)
+
+        #Dependency is not configurable
+        if self.dep_hash() != other.dep_hash():
+            return -1 if self.dep_hash() < other.dep_hash() else 1
+
+        #Equal specs
+        return 0
+
+
     def __str__(self):
         return self.format() + self.dep_string()
 

From 8d7b7e5d5dadcec9b997b94d95898a4134e122b2 Mon Sep 17 00:00:00 2001
From: Matthew LeGendre <legendre1@llnl.gov>
Date: Fri, 29 May 2015 12:20:32 -0700
Subject: [PATCH 03/11] Use preferred package rules when concretize'ing specs

---
 lib/spack/spack/concretize.py | 68 +++++++++++++++++++----------------
 lib/spack/spack/spec.py       |  2 +-
 2 files changed, 38 insertions(+), 32 deletions(-)

diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py
index 66002492cb..0f258c9096 100644
--- a/lib/spack/spack/concretize.py
+++ b/lib/spack/spack/concretize.py
@@ -38,6 +38,7 @@
 import spack.architecture
 import spack.error
 from spack.version import *
+from functools import partial
 
 
 
@@ -49,8 +50,8 @@ class DefaultConcretizer(object):
 
     def concretize_version(self, spec):
         """If the spec is already concrete, return.  Otherwise take
-           the most recent available version, and default to the package's
-           version if there are no avaialble versions.
+           the preferred version from spackconfig, and default to the package's
+           version if there are no available versions.
 
            TODO: In many cases we probably want to look for installed
                  versions of each package and use an installed version
@@ -68,12 +69,14 @@ def concretize_version(self, spec):
         # If there are known available versions, return the most recent
         # version that satisfies the spec
         pkg = spec.package
+        cmp_versions = partial(spack.pkgsort.version_compare, spec.name)
         valid_versions = sorted(
             [v for v in pkg.versions
-             if any(v.satisfies(sv) for sv in spec.versions)])
+             if any(v.satisfies(sv) for sv in spec.versions)],
+            cmp=cmp_versions)
 
         if valid_versions:
-            spec.versions = ver([valid_versions[-1]])
+            spec.versions = ver([valid_versions[0]])
         else:
             # We don't know of any SAFE versions that match the given
             # spec.  Grab the spec's versions and grab the highest
@@ -138,10 +141,10 @@ def concretize_compiler(self, spec):
         """If the spec already has a compiler, we're done.  If not, then take
            the compiler used for the nearest ancestor with a compiler
            spec and use that.  If the ancestor's compiler is not
-           concrete, then give it a valid version.  If there is no
-           ancestor with a compiler, use the system default compiler.
+           concrete, then used the preferred compiler as specified in 
+           spackconfig.
 
-           Intuition: Use the system default if no package that depends on
+           Intuition: Use the spackconfig default if no package that depends on
            this one has a strict compiler requirement.  Otherwise, try to
            build with the compiler that will be used by libraries that
            link to this one, to maximize compatibility.
@@ -153,40 +156,43 @@ def concretize_compiler(self, spec):
             spec.compiler in all_compilers):
             return False
 
-        try:
-            nearest = next(p for p in spec.traverse(direction='parents')
-                           if p.compiler is not None).compiler
-
-            if not nearest in all_compilers:
-                # Take the newest compiler that saisfies the spec
-                matches = sorted(spack.compilers.find(nearest))
-                if not matches:
-                    raise UnavailableCompilerVersionError(nearest)
-
-                # copy concrete version into nearest spec
-                nearest.versions = matches[-1].versions.copy()
-                assert(nearest.concrete)
-
-            spec.compiler = nearest.copy()
-
-        except StopIteration:
-            spec.compiler = spack.compilers.default_compiler().copy()
-
+        # Find the parent spec that has a compiler, or the root if none do
+        parent_spec = next(p for p in spec.traverse(direction='parents')
+                           if p.compiler is not None or not p.dependents)
+        parent_compiler = parent_spec.compiler
+        assert(parent_spec)
+        
+        # Check if the compiler is already fully specified
+        if parent_compiler in all_compilers:
+            spec.compiler = parent_compiler.copy()
+            return True
+            
+        # Filter the compilers into a sorted list based on the compiler_order from spackconfig
+        compiler_list = all_compilers if not parent_compiler else spack.compilers.find(parent_compiler)
+        cmp_compilers = partial(spack.pkgsort.compiler_compare, parent_spec.name)
+        matches = sorted(compiler_list, cmp=cmp_compilers)
+        if not matches:
+            raise UnavailableCompilerVersionError(parent_compiler)
+            
+        # copy concrete version into parent_compiler
+        spec.compiler = matches[0].copy()
+        assert(spec.compiler.concrete)
         return True  # things changed.
 
 
-    def choose_provider(self, spec, providers):
+    def choose_provider(self, package_spec, spec, providers):
         """This is invoked for virtual specs.  Given a spec with a virtual name,
            say "mpi", and a list of specs of possible providers of that spec,
            select a provider and return it.
         """
         assert(spec.virtual)
         assert(providers)
+        
+        provider_cmp = partial(spack.pkgsort.provider_compare, package_spec.name, spec.name)
+        sorted_providers = sorted(providers, cmp=provider_cmp)
+        first_key = sorted_providers[0]
 
-        index = spack.spec.index_specs(providers)
-        first_key = sorted(index.keys())[0]
-        latest_version = sorted(index[first_key])[-1]
-        return latest_version
+        return first_key
 
 
 class UnavailableCompilerVersionError(spack.error.SpackError):
diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py
index 83b1416e36..41496b0e9d 100644
--- a/lib/spack/spack/spec.py
+++ b/lib/spack/spack/spec.py
@@ -796,7 +796,7 @@ def _expand_virtual_packages(self):
 
             for spec in virtuals:
                 providers = spack.db.providers_for(spec)
-                concrete = spack.concretizer.choose_provider(spec, providers)
+                concrete = spack.concretizer.choose_provider(self, spec, providers)
                 concrete = concrete.copy()
                 spec._replace_with(concrete)
                 changed = True

From ee68a76a193890231d6df7fa7934d42e3708540b Mon Sep 17 00:00:00 2001
From: Matthew LeGendre <legendre1@llnl.gov>
Date: Fri, 29 May 2015 14:57:24 -0700
Subject: [PATCH 04/11] Bug fixes from testing spack preferred packages

---
 lib/spack/spack/preferred_packages.py    | 7 +++----
 var/spack/mock_packages/mpich/package.py | 1 +
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/lib/spack/spack/preferred_packages.py b/lib/spack/spack/preferred_packages.py
index 248508fe80..bc5271f693 100644
--- a/lib/spack/spack/preferred_packages.py
+++ b/lib/spack/spack/preferred_packages.py
@@ -112,9 +112,8 @@ def _spec_compare(self, pkgname, component, a, b, reverse_natural_compare, secon
         elif a_index == None and b_index != None: return 1
         elif a_index != None and b_index == a_index: return -1 * cmp(a, b)
         elif a_index != None and b_index != None and a_index != b_index: return cmp(a_index, b_index)
-        elif a < b: return 1 * reverse
-        elif b < a: return -1 * reverse
-        else: return 0
+        else: return cmp(a, b) * reverse
+
 
 
     # Given a sort order specified by the pkgname/component/second_key, return
@@ -148,7 +147,7 @@ def version_compare(self, pkgname, a, b):
         """Return less-than-0, 0, or greater than 0 if version a of pkgname is 
            respecively less-than, equal-to, or greater-than version b of pkgname. 
            One version is less-than another if it is preferred over the other."""
-        return self._spec_compare(pkgname, 'version', a, b, False, None)
+        return self._spec_compare(pkgname, 'version', a, b, True, None)
 
     
     def variant_compare(self, pkgname, a, b):
diff --git a/var/spack/mock_packages/mpich/package.py b/var/spack/mock_packages/mpich/package.py
index f77d3efc5d..e4110ad530 100644
--- a/var/spack/mock_packages/mpich/package.py
+++ b/var/spack/mock_packages/mpich/package.py
@@ -38,6 +38,7 @@ class Mpich(Package):
     version('3.0.2', 'foobarbaz')
     version('3.0.1', 'foobarbaz')
     version('3.0', 'foobarbaz')
+    version('1.0', 'foobarbas')
 
     provides('mpi@:3', when='@3:')
     provides('mpi@:1', when='@:1')

From 987cd9e78f99aa6ee6f3a48a4b8f556a68cb2965 Mon Sep 17 00:00:00 2001
From: Matthew LeGendre <legendre1@llnl.gov>
Date: Tue, 7 Jul 2015 16:27:13 -0700
Subject: [PATCH 05/11] Update docs for YAML configuration files and preferred
 concretization

---
 lib/spack/docs/basic_usage.rst        | 37 +++++++-----
 lib/spack/docs/mirrors.rst            |  9 ++-
 lib/spack/docs/packaging_guide.rst    | 66 ++++++++++++++++++++-
 lib/spack/docs/site_configuration.rst | 83 ---------------------------
 4 files changed, 91 insertions(+), 104 deletions(-)

diff --git a/lib/spack/docs/basic_usage.rst b/lib/spack/docs/basic_usage.rst
index 0578f0c8db..5d5438220c 100644
--- a/lib/spack/docs/basic_usage.rst
+++ b/lib/spack/docs/basic_usage.rst
@@ -357,7 +357,7 @@ Spack, you can simply run ``spack compiler add`` with the path to
 where the compiler is installed.  For example::
 
     $ spack compiler add /usr/local/tools/ic-13.0.079
-    ==> Added 1 new compiler to /Users/gamblin2/.spackconfig
+    ==> Added 1 new compiler to /Users/gamblin2/.spack/compilers.yaml
         intel@13.0.079
 
 Or you can run ``spack compiler add`` with no arguments to force
@@ -367,7 +367,7 @@ installed, but you know that new compilers have been added to your
 
     $ module load gcc-4.9.0
     $ spack compiler add
-    ==> Added 1 new compiler to /Users/gamblin2/.spackconfig
+    ==> Added 1 new compiler to /Users/gamblin2/.spack/compilers.yaml
         gcc@4.9.0
 
 This loads the environment module for gcc-4.9.0 to get it into the
@@ -398,27 +398,34 @@ Manual compiler configuration
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 If auto-detection fails, you can manually configure a compiler by
-editing your ``~/.spackconfig`` file.  You can do this by running
-``spack config edit``, which will open the file in your ``$EDITOR``.
+editing your ``~/.spack/compilers.yaml`` file.  You can do this by running
+``spack config edit compilers``, which will open the file in your ``$EDITOR``.
 
 Each compiler configuration in the file looks like this::
 
     ...
-    [compiler "intel@15.0.0"]
-        cc = /usr/local/bin/icc-15.0.024-beta
-        cxx = /usr/local/bin/icpc-15.0.024-beta
-        f77 = /usr/local/bin/ifort-15.0.024-beta
-        fc = /usr/local/bin/ifort-15.0.024-beta
-    ...
+    chaos_5_x86_64_ib:
+      ...
+      intel@15.0.0:
+          cc: /usr/local/bin/icc-15.0.024-beta
+          cxx: /usr/local/bin/icpc-15.0.024-beta
+          f77: /usr/local/bin/ifort-15.0.024-beta
+          fc: /usr/local/bin/ifort-15.0.024-beta
+      ...
+
+The chaos_5_x86_64_ib string is an architecture string, and multiple
+compilers can be listed underneath an architecture.  The architecture
+string may be replaced with the string 'all' to signify compilers that
+work on all architectures.
 
 For compilers, like ``clang``, that do not support Fortran, put
 ``None`` for ``f77`` and ``fc``::
 
-    [compiler "clang@3.3svn"]
-        cc = /usr/bin/clang
-        cxx = /usr/bin/clang++
-        f77 = None
-        fc = None
+    clang@3.3svn:
+        cc: /usr/bin/clang
+        cxx: /usr/bin/clang++
+        f77: None
+        fc: None
 
 Once you save the file, the configured compilers will show up in the
 list displayed by ``spack compilers``.
diff --git a/lib/spack/docs/mirrors.rst b/lib/spack/docs/mirrors.rst
index d732a3dd54..7581a0e9ed 100644
--- a/lib/spack/docs/mirrors.rst
+++ b/lib/spack/docs/mirrors.rst
@@ -205,12 +205,11 @@ And, if you want to remove a mirror, just remove it by name::
 Mirror precedence
 ----------------------------
 
-Adding a mirror really just adds a section in ``~/.spackconfig``::
+Adding a mirror really just adds a section in ``~/.spack/mirrors.yaml``::
 
-   [mirror "local_filesystem"]
-       url = file:///Users/gamblin2/spack-mirror-2014-06-24
-   [mirror "remote_server"]
-       url = https://example.com/some/web-hosted/directory/spack-mirror-2014-06-24
+   mirrors:
+   - local_filesystem: file:///Users/gamblin2/spack-mirror-2014-06-24
+   - remote_server: https://example.com/some/web-hosted/directory/spack-mirror-2014-06-24
 
 If you want to change the order in which mirrors are searched for
 packages, you can edit this file and reorder the sections.  Spack will
diff --git a/lib/spack/docs/packaging_guide.rst b/lib/spack/docs/packaging_guide.rst
index 59ba63fa35..5094f739c4 100644
--- a/lib/spack/docs/packaging_guide.rst
+++ b/lib/spack/docs/packaging_guide.rst
@@ -632,7 +632,7 @@ Default
   revision instead.
 
 Revisions
-  Add ``hg`` and ``revision``parameters:
+  Add ``hg`` and ``revision`` parameters:
 
   .. code-block:: python
 
@@ -1524,6 +1524,70 @@ This is useful when you want to know exactly what Spack will do when
 you ask for a particular spec.
 
 
+``Concretization Policies``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+A user may have certain perferrences for how packages should
+be concretized on their system.  For example, one user may prefer packages
+built with OpenMPI and the Intel compiler.  Another user may prefer
+packages be built with MVAPICH and GCC.  
+
+Spack's ``preferred`` configuration can be used to set defaults for sites or users.
+Spack uses this configuration to make decisions about which compilers, package
+versions, depends_on, and variants it should prefer during concretization.
+
+The preferred configuration can be controlled by editing the 
+``~/.spack/preferred.yaml`` file for user configuations, or the
+
+
+Here's an example preferred.yaml file:
+
+.. code-block:: sh
+
+    preferred:
+      dyninst:
+        compiler: gcc@4.9
+        variants: +debug
+      gperftools:
+        version: 2.2, 2.4, 2.3
+      all:
+        compiler: gcc@4.4.7, gcc@4.6:, intel, clang, pgi
+        providers: 
+          mpi: mvapich, mpich, openmpi
+
+At a high level, this example is specifying how packages should be
+concretized.  The dyninst package should prefer using gcc 4.9 and 
+be built with debug options.  The gperftools package should prefer version
+2.2 over 2.4.  Every package on the system should prefer mvapich for
+its MPI and gcc 4.4.7 (except for Dyninst, which perfers gcc 4.9).  
+These options are used to fill in implicit defaults.  Any of them can be overwritten 
+on the command line if explicitly requested.
+
+Each preferred.yaml file begin with the string ``preferred:`` and 
+each subsequent entry is indented underneath it.  The next layer contains
+package names or the special string ``all`` (which applies to 
+every package).  Underneath each package name is 
+one or more components: ``compiler``, ``variants``, ``version``, 
+or ``providers``.  Each component has an ordered list of spec 
+``constraints``, with earlier entries in the list being prefered over
+latter entries.
+
+Sometimes a package installation may have constraints that forbid 
+the first concretization rule, in which case Spack will use the first
+legal concretization rule.  Going back to the example, if a user
+requests gperftools 2.3 or latter, then Spack will install version 2.4 
+as the 2.4 version of gperftools is preferred over 2.3.
+
+An explicit concretization rule in the preferred section will always 
+take preference over unlisted concretizations.  In the above example, 
+xlc isn't listed in the compiler list.  Every listed compiler from
+gcc to pgi will thus be preferred over the xlc compiler.
+
+The syntax for the ``providers`` section differs slightly from other
+concretization rules.  A provider lists a value that packages may
+``depend_on`` (e.g, mpi) and a list of rules for fulfilling that
+dependency.
+
 .. _install-method:
 
 Implementing the ``install`` method
diff --git a/lib/spack/docs/site_configuration.rst b/lib/spack/docs/site_configuration.rst
index b03df29573..1e6740a434 100644
--- a/lib/spack/docs/site_configuration.rst
+++ b/lib/spack/docs/site_configuration.rst
@@ -54,89 +54,6 @@ more elements to the list to indicate where your own site's temporary
 directory is.
 
 
-.. _concretization-policies:
-
-Concretization policies
-----------------------------
-
-When a user asks for a package like ``mpileaks`` to be installed,
-Spack has to make decisions like what version should be installed,
-what compiler to use, and how its dependencies should be configured.
-This process is called *concretization*, and it's covered in detail in
-:ref:`its own section <abstract-and-concrete>`.
-
-The default concretization policies are in the
-:py:mod:`spack.concretize` module, specifically in the
-:py:class:`spack.concretize.DefaultConcretizer` class.  These are the
-important methods used in the concretization process:
-
-* :py:meth:`concretize_version(self, spec) <spack.concretize.DefaultConcretizer.concretize_version>`
-* :py:meth:`concretize_architecture(self, spec) <spack.concretize.DefaultConcretizer.concretize_architecture>`
-* :py:meth:`concretize_compiler(self, spec) <spack.concretize.DefaultConcretizer.concretize_compiler>`
-* :py:meth:`choose_provider(self, spec, providers) <spack.concretize.DefaultConcretizer.choose_provider>`
-
-The first three take a :py:class:`Spec <spack.spec.Spec>` object and
-modify it by adding constraints for the version.  For example, if the
-input spec had a version range like `1.0:5.0.3`, then the
-``concretize_version`` method should set the spec's version to a
-*single* version in that range.  Likewise, ``concretize_architecture``
-selects an architecture when the input spec does not have one, and
-``concretize_compiler`` needs to set both a concrete compiler and a
-concrete compiler version.
-
-``choose_provider()`` affects how concrete implementations are chosen
-based on a virtual dependency spec.  The input spec is some virtual
-dependency and the ``providers`` index is a :py:class:`ProviderIndex
-<spack.packages.ProviderIndex>` object.  The ``ProviderIndex`` maps
-the virtual spec to specs for possible implementations, and
-``choose_provider()`` should simply choose one of these.  The
-``concretize_*`` methods will be called on the chosen implementation
-later, so there is no need to fully concretize the spec when returning
-it.
-
-The ``DefaultConcretizer`` is intended to provide sensible defaults
-for each policy, but there are certain choices that it can't know
-about.  For example, one site might prefer ``OpenMPI`` over ``MPICH``,
-or another might prefer an old version of some packages.  These types
-of special cases can be integrated with custom concretizers.
-
-Writing a custom concretizer
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-To write your own concretizer, you need only subclass
-``DefaultConcretizer`` and override the methods you want to change.
-For example, you might write a class like this to change *only* the
-``concretize_version()`` behavior:
-
-.. code-block:: python
-
-   from spack.concretize import DefaultConcretizer
-
-   class MyConcretizer(DefaultConcretizer):
-       def concretize_version(self, spec):
-           # implement custom logic here.
-
-Once you have written your custom concretizer, you can make Spack use
-it by editing ``globals.py``.  Find this part of the file:
-
-.. code-block:: python
-
-   #
-   # This controls how things are concretized in spack.
-   # Replace it with a subclass if you want different
-   # policies.
-   #
-   concretizer = DefaultConcretizer()
-
-Set concretizer to *your own* class instead of the default:
-
-.. code-block:: python
-
-   concretizer = MyConcretizer()
-
-The next time you run Spack, your changes should take effect.
-
-
 Profiling
 ~~~~~~~~~~~~~~~~~~~~~
 

From 53cde110b1d25f8fcf89d7c26ef02bb073cb6a29 Mon Sep 17 00:00:00 2001
From: Matthew LeGendre <legendre1@llnl.gov>
Date: Tue, 7 Jul 2015 16:27:46 -0700
Subject: [PATCH 06/11] Update Spack mirror command to match docs

---
 lib/spack/spack/cmd/mirror.py | 12 ++++++------
 lib/spack/spack/stage.py      |  5 ++---
 2 files changed, 8 insertions(+), 9 deletions(-)

diff --git a/lib/spack/spack/cmd/mirror.py b/lib/spack/spack/cmd/mirror.py
index 2356170a9a..baf64d30fc 100644
--- a/lib/spack/spack/cmd/mirror.py
+++ b/lib/spack/spack/cmd/mirror.py
@@ -75,8 +75,8 @@ def mirror_add(args):
     if url.startswith('/'):
         url = 'file://' + url
 
-    mirror_dict = { args.name : url }
-    spack.config.add_to_mirror_config({ args.name : url })
+    newmirror = [ { args.name : url } ]
+    spack.config.add_to_mirror_config(newmirror)
 
 
 def mirror_remove(args):
@@ -90,15 +90,15 @@ def mirror_remove(args):
 
 def mirror_list(args):
     """Print out available mirrors to the console."""
-    sec_names = spack.config.get_mirror_config()
-    if not sec_names:
+    mirrors = spack.config.get_mirror_config()
+    if not mirrors:
         tty.msg("No mirrors configured.")
         return
 
-    max_len = max(len(s) for s in sec_names)
+    max_len = max(len(name) for name,path in mirrors)
     fmt = "%%-%ds%%s" % (max_len + 4)
 
-    for name, val in sec_names.iteritems():
+    for name, val in mirrors:
         print fmt % (name, val)
 
 
diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py
index 008c5f0429..c70c7a84a4 100644
--- a/lib/spack/spack/stage.py
+++ b/lib/spack/spack/stage.py
@@ -26,6 +26,7 @@
 import re
 import shutil
 import tempfile
+import sys
 
 import llnl.util.tty as tty
 from llnl.util.filesystem import *
@@ -344,9 +345,7 @@ def destroy(self):
 
 def _get_mirrors():
     """Get mirrors from spack configuration."""
-    config = spack.config.get_mirror_config()
-    return [val for name, val in config.iteritems()]
-
+    return [path for name, path in spack.config.get_mirror_config()]
 
 
 def ensure_access(file=spack.stage_path):

From 53d70fff0121a05fb21c02363570f81573bbeffa Mon Sep 17 00:00:00 2001
From: Matthew LeGendre <legendre1@llnl.gov>
Date: Tue, 7 Jul 2015 16:28:51 -0700
Subject: [PATCH 07/11] Fix type error with YAML config when merging lists from
 different configs.

---
 lib/spack/spack/config.py | 33 ++++++++++++++++-----------------
 1 file changed, 16 insertions(+), 17 deletions(-)

diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py
index dbe225960a..f3526b19fa 100644
--- a/lib/spack/spack/config.py
+++ b/lib/spack/spack/config.py
@@ -98,6 +98,7 @@
 from external.yaml.error import MarkedYAMLError
 import llnl.util.tty as tty
 from llnl.util.filesystem import mkdirp
+import copy
 
 _config_sections = {}
 class _ConfigCategory:
@@ -159,21 +160,19 @@ def _merge_dicts(d1, d2):
     if not d2:
         return d1
 
-    for key2, val2 in d2.iteritems():
-        if not key2 in d1:
-            d1[key2] = val2
-            continue
-        val1 = d1[key2]
-        if isinstance(val1, dict) and isinstance(val2, dict):
-            d1[key2] = _merge_dicts(val1, val2)
-            continue
-        if isinstance(val1, list) and isinstance(val2, list):
-            val1.extend(val2)
-            seen = set()
-            d1[key2] = [ x for x in val1 if not (x in seen or seen.add(x)) ]
-            continue
-        d1[key2] = val2
-    return d1
+    if (type(d1) is list) and (type(d2) is list):
+        d1.extend(d2)
+        return d1
+
+    if (type(d1) is dict) and (type(d2) is dict):
+        for key2, val2 in d2.iteritems():
+            if not key2 in d1:
+                d1[key2] = val2
+            else:
+                d1[key2] = _merge_dicts(d1[key2], val2)
+        return d1
+
+    return d2
 
 
 def get_config(category_name):
@@ -225,8 +224,8 @@ def get_compilers_config(arch=None):
 
 
 def get_mirror_config():
-    """Get the mirror configuration from config files"""
-    return get_config('mirrors')
+    """Get the mirror configuration from config files as a list of name/location tuples"""
+    return [x.items()[0] for x in get_config('mirrors')]
 
 
 def get_preferred_config():

From 650c9d4e36c6a58cf6bca0e6abd580ee54d8e175 Mon Sep 17 00:00:00 2001
From: Matthew LeGendre <legendre1@llnl.gov>
Date: Wed, 16 Sep 2015 10:56:11 -0700
Subject: [PATCH 08/11] Allow spack to build against external
 non-spack-installed packages.

---
 lib/spack/spack/__init__.py            |   6 ++
 lib/spack/spack/abi.py                 | 128 ++++++++++++++++++++++
 lib/spack/spack/concretize.py          | 140 ++++++++++++++++++++++---
 lib/spack/spack/config.py              |  59 ++++++++++-
 lib/spack/spack/directory_layout.py    |   8 ++
 lib/spack/spack/package.py             |   3 +
 lib/spack/spack/preferred_packages.py  |  10 +-
 lib/spack/spack/spec.py                |  58 +++++-----
 var/spack/packages/mpich/package.py    |   1 +
 var/spack/packages/mvapich2/package.py |  17 ++-
 10 files changed, 387 insertions(+), 43 deletions(-)
 create mode 100644 lib/spack/spack/abi.py

diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py
index bd8478fb98..5783005b5b 100644
--- a/lib/spack/spack/__init__.py
+++ b/lib/spack/spack/__init__.py
@@ -77,6 +77,12 @@
 from spack.preferred_packages import PreferredPackages
 pkgsort = PreferredPackages()
 
+#
+# This tests ABI compatibility between packages
+#
+from spack.abi import ABI
+abi = ABI()
+
 #
 # This controls how things are concretized in spack.
 # Replace it with a subclass if you want different
diff --git a/lib/spack/spack/abi.py b/lib/spack/spack/abi.py
new file mode 100644
index 0000000000..f0a997703c
--- /dev/null
+++ b/lib/spack/spack/abi.py
@@ -0,0 +1,128 @@
+##############################################################################
+# Copyright (c) 2015, Lawrence Livermore National Security, LLC.
+# Produced at the Lawrence Livermore National Laboratory.
+#
+# This file is part of Spack.
+# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
+# LLNL-CODE-647188
+#
+# For details, see https://scalability-llnl.github.io/spack
+# Please also see the LICENSE file for our notice and the LGPL.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License (as published by
+# the Free Software Foundation) version 2.1 dated February 1999.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
+# conditions of the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+##############################################################################
+
+import os
+import spack
+import spack.spec
+from spack.spec import CompilerSpec
+from spack.util.executable import Executable, ProcessError
+from llnl.util.lang import memoized
+
+class ABI(object):
+    """This class provides methods to test ABI compatibility between specs.
+       The current implementation is rather rough and could be improved."""
+
+    def architecture_compatible(self, parent, child):
+        """Returns true iff the parent and child specs have ABI compatible architectures."""
+        return not parent.architecture or not child.architecture or parent.architecture == child.architecture
+
+
+    @memoized
+    def _gcc_get_libstdcxx_version(self, version):
+        """Returns gcc ABI compatibility info by getting the library version of
+           a compiler's libstdc++.so or libgcc_s.so"""
+        spec = CompilerSpec("gcc", version)
+        compilers = spack.compilers.compilers_for_spec(spec)
+        if not compilers:
+            return None
+        compiler = compilers[0]
+        rungcc = None
+        libname = None
+        output = None
+        if compiler.cxx:
+            rungcc = Executable(compiler.cxx)
+            libname = "libstdc++.so"
+        elif compiler.cc:
+            rungcc = Executable(compiler.cc)
+            libname = "libgcc_s.so"
+        else:
+            return None
+        try:
+            output = rungcc("--print-file-name=%s" % libname, return_output=True)
+        except ProcessError, e:
+            return None
+        if not output:
+            return None
+        libpath = os.readlink(output.strip())
+        if not libpath:
+            return None
+        return os.path.basename(libpath)
+        
+        
+    @memoized
+    def _gcc_compiler_compare(self, pversion, cversion):
+        """Returns true iff the gcc version pversion and cversion 
+          are ABI compatible."""
+        plib = self._gcc_get_libstdcxx_version(pversion)
+        clib = self._gcc_get_libstdcxx_version(cversion)
+        if not plib or not clib:
+            return False
+        return plib == clib
+
+
+    def _intel_compiler_compare(self, pversion, cversion):
+        """Returns true iff the intel version pversion and cversion
+           are ABI compatible"""
+
+        #Test major and minor versions.  Ignore build version.
+        if (len(pversion.version) < 2 or len(cversion.version) < 2):
+            return False
+        return (pversion.version[0] == cversion.version[0]) and \
+            (pversion.version[1] == cversion.version[1])
+        
+    
+    def compiler_compatible(self, parent, child, **kwargs):
+        """Returns true iff the compilers for parent and child specs are ABI compatible"""
+        if not parent.compiler or not child.compiler:
+            return True
+        
+        if parent.compiler.name != child.compiler.name:
+            #Different compiler families are assumed ABI incompatible
+            return False
+        
+        if kwargs.get('loose', False):
+            return True
+
+        for pversion in parent.compiler.versions:
+            for cversion in child.compiler.versions:
+                #For a few compilers use specialized comparisons.  Otherwise
+                # match on version match.
+                if pversion.satisfies(cversion):
+                    return True
+                elif parent.compiler.name == "gcc" and \
+                     self._gcc_compiler_compare(pversion, cversion):
+                    return True
+                elif parent.compiler.name == "intel" and \
+                     self._intel_compiler_compare(pversion, cversion):
+                    return True
+        return False
+
+    
+    def compatible(self, parent, child, **kwargs):
+        """Returns true iff a parent and child spec are ABI compatible"""
+        loosematch = kwargs.get('loose', False)
+        return self.architecture_compatible(parent, child) and \
+               self.compiler_compatible(parent, child, loose=loosematch)
+    
diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py
index 0f258c9096..01ff163493 100644
--- a/lib/spack/spack/concretize.py
+++ b/lib/spack/spack/concretize.py
@@ -33,13 +33,16 @@
 TODO: make this customizable and allow users to configure
       concretization  policies.
 """
+import spack
 import spack.spec
 import spack.compilers
 import spack.architecture
 import spack.error
 from spack.version import *
 from functools import partial
-
+from spec import DependencyMap
+from itertools import chain
+from spack.config import *
 
 
 class DefaultConcretizer(object):
@@ -48,6 +51,107 @@ class DefaultConcretizer(object):
        default concretization strategies, or you can override all of them.
     """
 
+    def _find_other_spec(self, spec, condition):
+        """Searches the dag from spec in an intelligent order and looks
+           for a spec that matches a condition"""
+        dagiter = chain(spec.traverse(direction='parents'), spec.traverse(direction='children'))
+        found = next((x for x in dagiter if x is not spec and condition(x)), None)
+        if found:
+            return found
+        dagiter = chain(spec.traverse(direction='parents'), spec.traverse(direction='children'))
+        searched = list(dagiter)
+        found = next((x for x in spec.root.traverse() if x not in searched and x is not spec and condition(x)), None)
+        if found:
+            return found
+        if condition(spec):
+            return spec
+        return None
+                                   
+
+    def _valid_virtuals_and_externals(self, spec):
+        """Returns a list of spec/external-path pairs for both virtuals and externals
+           that can concretize this spec.""" 
+
+        # Get a list of candidate packages that could satisfy this spec
+        packages = []
+        if spec.virtual:
+            providers = spack.db.providers_for(spec)
+            if not providers:
+                raise UnsatisfiableProviderSpecError(providers[0], spec)
+            spec_w_preferred_providers = self._find_other_spec(spec, \
+                    lambda(x): spack.pkgsort.spec_has_preferred_provider(x.name, spec.name))
+            if not spec_w_preferred_providers:
+                spec_w_preferred_providers = spec
+            provider_cmp = partial(spack.pkgsort.provider_compare, spec_w_preferred_providers.name, spec.name)
+            packages = sorted(providers, cmp=provider_cmp)
+        else:
+            if not spec_externals(spec) or spec.external:
+                return None
+            packages = [spec]
+
+        # For each candidate package, if it has externals add those to the candidates
+        # if it's a nobuild, then only add the externals.
+        result = []
+        all_compilers = spack.compilers.all_compilers()
+        for pkg in packages:
+            externals = spec_externals(pkg)
+            buildable = not is_spec_nobuild(pkg)
+            if buildable:
+                result.append((pkg, None))
+            if externals:
+                sorted_externals = sorted(externals, cmp=lambda a,b: a[0].__cmp__(b[0]))
+                for external in sorted_externals:
+                    if external[0].satisfies(spec):
+                        result.append(external)
+        if not result:
+            raise NoBuildError(spec)
+        return result
+
+
+    def concretize_virtual_and_external(self, spec):
+        """From a list of candidate virtual and external packages, concretize to one that
+           is ABI compatible with the rest of the DAG."""
+        candidates = self._valid_virtuals_and_externals(spec)
+        if not candidates:
+            return False
+
+        #Find the another spec in the dag that has a compiler.  We'll use that
+        # spec to test compiler compatibility.
+        other_spec = self._find_other_spec(spec, lambda(x): x.compiler)
+        if not other_spec:
+            other_spec = spec.root
+
+        #Choose an ABI-compatible candidate, or the first match otherwise.
+        candidate = None
+        if other_spec:
+            candidate = next((c for c in candidates if spack.abi.compatible(c[0], other_spec)), None)
+            if not candidate:
+                #Try a looser ABI matching
+                candidate = next((c for c in candidates if spack.abi.compatible(c[0], other_spec, loose=True)), None)
+        if not candidate:
+            #Pick the first choice
+            candidate = candidates[0]
+        external = candidate[1]
+        candidate_spec = candidate[0]
+        
+        #Refine this spec to the candidate.
+        changed = False
+        if spec.virtual:
+            spec._replace_with(candidate_spec)
+            changed = True
+        if spec._dup(candidate_spec, deps=False, cleardeps=False):
+            changed = True
+        if not spec.external and external:
+            spec.external = external
+            changed = True
+        #If we're external then trim the dependencies
+        if external and spec.dependencies:
+            changed = True
+            spec.depencencies = DependencyMap()
+        
+        return changed
+        
+        
     def concretize_version(self, spec):
         """If the spec is already concrete, return.  Otherwise take
            the preferred version from spackconfig, and default to the package's
@@ -150,31 +254,32 @@ def concretize_compiler(self, spec):
            link to this one, to maximize compatibility.
         """
         all_compilers = spack.compilers.all_compilers()
-
+        
         if (spec.compiler and
             spec.compiler.concrete and
             spec.compiler in all_compilers):
             return False
 
-        # Find the parent spec that has a compiler, or the root if none do
-        parent_spec = next(p for p in spec.traverse(direction='parents')
-                           if p.compiler is not None or not p.dependents)
-        parent_compiler = parent_spec.compiler
-        assert(parent_spec)
+        #Find the another spec that has a compiler, or the root if none do
+        other_spec = self._find_other_spec(spec, lambda(x) : x.compiler)
+        if not other_spec:
+            other_spec = spec.root
+        other_compiler = other_spec.compiler
+        assert(other_spec)
         
         # Check if the compiler is already fully specified
-        if parent_compiler in all_compilers:
-            spec.compiler = parent_compiler.copy()
+        if other_compiler in all_compilers:
+            spec.compiler = other_compiler.copy()
             return True
             
         # Filter the compilers into a sorted list based on the compiler_order from spackconfig
-        compiler_list = all_compilers if not parent_compiler else spack.compilers.find(parent_compiler)
-        cmp_compilers = partial(spack.pkgsort.compiler_compare, parent_spec.name)
+        compiler_list = all_compilers if not other_compiler else spack.compilers.find(other_compiler)
+        cmp_compilers = partial(spack.pkgsort.compiler_compare, other_spec.name)
         matches = sorted(compiler_list, cmp=cmp_compilers)
         if not matches:
-            raise UnavailableCompilerVersionError(parent_compiler)
+            raise UnavailableCompilerVersionError(other_compiler)
             
-        # copy concrete version into parent_compiler
+        # copy concrete version into other_compiler
         spec.compiler = matches[0].copy()
         assert(spec.compiler.concrete)
         return True  # things changed.
@@ -210,3 +315,12 @@ class NoValidVersionError(spack.error.SpackError):
     def __init__(self, spec):
         super(NoValidVersionError, self).__init__(
             "There are no valid versions for %s that match '%s'" % (spec.name, spec.versions))
+
+
+class NoBuildError(spack.error.SpackError):
+    """Raised when a package is configured with the nobuild option, but
+       no satisfactory external versions can be found"""
+    def __init__(self, spec):
+        super(NoBuildError, self).__init__(
+            "The spec '%s' is configured as nobuild, and no matching external installs were found" % spec.name)
+       
diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py
index f3526b19fa..60577c45b3 100644
--- a/lib/spack/spack/config.py
+++ b/lib/spack/spack/config.py
@@ -90,9 +90,12 @@
 import exceptions
 import sys
 import copy
-
-from llnl.util.lang import memoized
+import inspect
+import glob
+import imp
+import spack.spec
 import spack.error
+from llnl.util.lang import memoized
 
 from external import yaml
 from external.yaml.error import MarkedYAMLError
@@ -116,6 +119,9 @@ def __init__(self, n, f, m):
 _ConfigCategory('compilers', 'compilers.yaml', True)
 _ConfigCategory('mirrors', 'mirrors.yaml', True)
 _ConfigCategory('preferred', 'preferred.yaml', True)
+_ConfigCategory('view', 'views.yaml', True)
+_ConfigCategory('preferred', 'preferred.yaml', True)
+_ConfigCategory('packages', 'packages.yaml', True)
 
 """Names of scopes and their corresponding configuration files."""
 config_scopes = [('site', os.path.join(spack.etc_path, 'spack')),
@@ -233,6 +239,55 @@ def get_preferred_config():
     return get_config('preferred')
 
 
+@memoized
+def get_packages_config():
+    """Get the externals configuration from config files"""
+    package_config = get_config('packages')
+    if not package_config:
+        return {}
+    indexed_packages = {}
+    for p in package_config:
+        package_name = spack.spec.Spec(p.keys()[0]).name
+        if package_name not in indexed_packages:
+            indexed_packages[package_name] = []
+        indexed_packages[package_name].append({ spack.spec.Spec(key) : val for key, val in p.iteritems() })
+    return indexed_packages
+
+
+def is_spec_nobuild(spec):
+    """Return true if the spec pkgspec is configured as nobuild"""
+    allpkgs = get_packages_config()
+    name = spec.name
+    if not name in allpkgs:
+        return False
+    for itm in allpkgs[name]:
+        for pkg,conf in itm.iteritems():
+            if pkg.satisfies(spec):
+                if conf.get('nobuild', False):
+                    return True
+    return False
+
+
+def spec_externals(spec):
+    """Return a list of spec, directory pairs for each external location for spec"""
+    allpkgs = get_packages_config()
+    name = spec.name
+    spec_locations = []
+    
+    if not name in allpkgs:
+        return []
+    for itm in allpkgs[name]:
+        for pkg,conf in itm.iteritems():
+            if not pkg.satisfies(spec):
+                continue
+            path = conf.get('path', None)
+            if not path:
+                continue
+            spec_locations.append( (pkg, path) )
+    return spec_locations
+        
+    
+
 def get_config_scope_dirname(scope):
     """For a scope return the config directory"""
     global config_scopes
diff --git a/lib/spack/spack/directory_layout.py b/lib/spack/spack/directory_layout.py
index 85ecc1ce2b..83e6eb566a 100644
--- a/lib/spack/spack/directory_layout.py
+++ b/lib/spack/spack/directory_layout.py
@@ -187,6 +187,14 @@ def hidden_file_paths(self):
 
     def relative_path_for_spec(self, spec):
         _check_concrete(spec)
+        
+        if spec.external:
+            return spec.external
+
+        enabled_variants = (
+            '-' + v.name for v in spec.variants.values()
+            if v.enabled)
+
         dir_name = "%s-%s-%s" % (
             spec.name,
             spec.version,
diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py
index 61606d0590..1e2f0378c8 100644
--- a/lib/spack/spack/package.py
+++ b/lib/spack/spack/package.py
@@ -752,6 +752,9 @@ def do_install(self,
         if not self.spec.concrete:
             raise ValueError("Can only install concrete packages.")
 
+        if self.spec.external:
+            return
+
         if os.path.exists(self.prefix):
             tty.msg("%s is already installed in %s." % (self.name, self.prefix))
             return
diff --git a/lib/spack/spack/preferred_packages.py b/lib/spack/spack/preferred_packages.py
index bc5271f693..bc2a4ac234 100644
--- a/lib/spack/spack/preferred_packages.py
+++ b/lib/spack/spack/preferred_packages.py
@@ -35,9 +35,10 @@ def __init__(self):
 
     #Given a package name, sort component (e.g, version, compiler, ...), and
     # a second_key (used by providers), return the list 
-    def _order_for_package(self, pkgname, component, second_key):
+    def _order_for_package(self, pkgname, component, second_key, test_all=True):
         pkglist = [pkgname]
-        pkglist.append('all')
+        if test_all:
+            pkglist.append('all')
         for pkg in pkglist:
             if not pkg in self.preferred:
                 continue
@@ -143,6 +144,11 @@ def provider_compare(self, pkgname, provider_str, a, b):
         return self._spec_compare(pkgname, 'providers', a, b, False, provider_str)
 
 
+    def spec_has_preferred_provider(self, pkgname, provider_str):
+        """Return True iff the named package has a list of preferred provider"""
+        return bool(self._order_for_package(pkgname, 'providers', provider_str, False))
+
+
     def version_compare(self, pkgname, a, b):
         """Return less-than-0, 0, or greater than 0 if version a of pkgname is 
            respecively less-than, equal-to, or greater-than version b of pkgname. 
diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py
index 41496b0e9d..6984b4a174 100644
--- a/lib/spack/spack/spec.py
+++ b/lib/spack/spack/spec.py
@@ -419,6 +419,7 @@ def __init__(self, spec_like, *dep_like, **kwargs):
         # package.py files for.
         self._normal = kwargs.get('normal', False)
         self._concrete = kwargs.get('concrete', False)
+        self.external = None
 
         # This allows users to construct a spec DAG with literals.
         # Note that given two specs a and b, Spec(a) copies a, but
@@ -426,7 +427,7 @@ def __init__(self, spec_like, *dep_like, **kwargs):
         for dep in dep_like:
             spec = dep if isinstance(dep, Spec) else Spec(dep)
             self._add_dependency(spec)
-
+        
 
     #
     # Private routines here are called by the parser when building a spec.
@@ -751,12 +752,11 @@ def _concretize_helper(self, presets=None, visited=None):
             # Concretize virtual dependencies last.  Because they're added
             # to presets below, their constraints will all be merged, but we'll
             # still need to select a concrete package later.
-            if not self.virtual:
-                changed |= any(
-                    (spack.concretizer.concretize_architecture(self),
-                     spack.concretizer.concretize_compiler(self),
-                     spack.concretizer.concretize_version(self),
-                     spack.concretizer.concretize_variants(self)))
+            changed |= any(
+                (spack.concretizer.concretize_architecture(self),
+                 spack.concretizer.concretize_compiler(self),
+                 spack.concretizer.concretize_version(self),
+                 spack.concretizer.concretize_variants(self)))
             presets[self.name] = self
 
         visited.add(self.name)
@@ -789,21 +789,18 @@ def _expand_virtual_packages(self):
               a problem.
         """
         changed = False
-        while True:
-            virtuals =[v for v in self.traverse() if v.virtual]
-            if not virtuals:
-                return changed
+        done = False
+        while not done:
+            done = True
+            for spec in list(self.traverse()):
+                if spack.concretizer.concretize_virtual_and_external(spec):
+                    done = False
+                    changed = True
 
-            for spec in virtuals:
-                providers = spack.db.providers_for(spec)
-                concrete = spack.concretizer.choose_provider(self, spec, providers)
-                concrete = concrete.copy()
-                spec._replace_with(concrete)
-                changed = True
-
-            # If there are duplicate providers or duplicate provider deps, this
-            # consolidates them and merge constraints.
-            changed |= self.normalize(force=True)
+        # If there are duplicate providers or duplicate provider deps, this
+        # consolidates them and merge constraints.
+        changed |= self.normalize(force=True)
+        return changed
 
 
     def concretize(self):
@@ -830,7 +827,6 @@ def concretize(self):
                        self._concretize_helper())
             changed = any(changes)
             force=True
-
         self._concrete = True
 
 
@@ -1346,15 +1342,26 @@ def _dup(self, other, **kwargs):
                Whether deps should be copied too.  Set to false to copy a
                spec but not its dependencies.
         """
+
+        # We don't count dependencies as changes here
+        changed = True
+        if hasattr(self, 'name'):
+            changed = (self.name != other.name and self.versions != other.versions and \
+                       self.architecture != other.architecture and self.compiler != other.compiler and \
+                       self.variants != other.variants and self._normal != other._normal and \
+                       self.concrete != other.concrete and self.external != other.external)
+                                       
         # Local node attributes get copied first.
         self.name = other.name
         self.versions = other.versions.copy()
         self.architecture = other.architecture
         self.compiler = other.compiler.copy() if other.compiler else None
-        self.dependents = DependencyMap()
-        self.dependencies = DependencyMap()
+        if kwargs.get('cleardeps', True):
+            self.dependents = DependencyMap()
+            self.dependencies = DependencyMap()
         self.variants = other.variants.copy()
         self.variants.spec = self
+        self.external = other.external
 
         # If we copy dependencies, preserve DAG structure in the new spec
         if kwargs.get('deps', True):
@@ -1372,6 +1379,8 @@ def _dup(self, other, **kwargs):
         # Since we preserved structure, we can copy _normal safely.
         self._normal = other._normal
         self._concrete = other._concrete
+        self.external = other.external
+        return changed
 
 
     def copy(self, **kwargs):
@@ -1796,6 +1805,7 @@ def spec(self):
         spec.variants = VariantMap(spec)
         spec.architecture = None
         spec.compiler = None
+        spec.external = None
         spec.dependents   = DependencyMap()
         spec.dependencies = DependencyMap()
 
diff --git a/var/spack/packages/mpich/package.py b/var/spack/packages/mpich/package.py
index b6b2dfde21..dfff22152d 100644
--- a/var/spack/packages/mpich/package.py
+++ b/var/spack/packages/mpich/package.py
@@ -45,6 +45,7 @@ def setup_dependent_environment(self, module, spec, dep_spec):
         os.environ['MPICH_F77'] = 'f77'
         os.environ['MPICH_F90'] = 'f90'
 
+        module.mpicc = join_path(self.prefix.bin, 'mpicc')
 
     def install(self, spec, prefix):
         config_args = ["--prefix=" + prefix,
diff --git a/var/spack/packages/mvapich2/package.py b/var/spack/packages/mvapich2/package.py
index ca0b1287c1..93bce011b7 100644
--- a/var/spack/packages/mvapich2/package.py
+++ b/var/spack/packages/mvapich2/package.py
@@ -11,10 +11,17 @@ class Mvapich2(Package):
 
     version('2.0', '9fbb68a4111a8b6338e476dc657388b4',
             url='http://mvapich.cse.ohio-state.edu/download/mvapich/mv2/mvapich2-2.0.tar.gz')
+    
+    version('2.1', '0095ceecb19bbb7fb262131cb9c2cdd6', 
+            url='http://mvapich.cse.ohio-state.edu/download/mvapich/mv2/mvapich2-2.1.tar.gz')
 
     provides('mpi@:2.2', when='@1.9')    # MVAPICH2-1.9 supports MPI 2.2
     provides('mpi@:3.0', when='@2.0')    # MVAPICH2-2.0 supports MPI 3.0
 
+    variant('psm', default=False, description="build with psm")
+
+    variant('pmi', default=False, description="build with pmi")
+    depends_on('pmgr_collective', when='+pmi')
 
     def install(self, spec, prefix):
         # we'll set different configure flags depending on our environment
@@ -80,7 +87,13 @@ def install(self, spec, prefix):
             configure_args.append("--with-device=ch3:psm")
         else:
             # throw this flag on IB systems
-            configure_args.append("--with-device=ch3:mrail", "--with-rdma=gen2")
+            configure_args.append("--with-device=ch3:mrail")
+            configure_args.append("--with-rdma=gen2")
+
+        if "+pmi" in spec:
+            configure_args.append("--with-pmi=pmgr_collective" % spec['pmgr_collective'].prefix)
+        else:
+            configure_args.append("--with-pmi=slurm")
 
         # TODO: shared-memory build
 
@@ -93,7 +106,7 @@ def install(self, spec, prefix):
             "--enable-f77", "--enable-fc", "--enable-cxx",
             "--enable-shared", "--enable-sharedlibs=gcc",
             "--enable-debuginfo",
-            "--with-pm=no", "--with-pmi=slurm",
+            "--with-pm=no",
             "--enable-romio", "--with-file-system=lustre+nfs+ufs",
             "--disable-mpe", "--without-mpe",
             "--disable-silent-rules",

From e4d2ba30b57618da388a1a990381d149b33d7aba Mon Sep 17 00:00:00 2001
From: Matthew LeGendre <legendre1@llnl.gov>
Date: Fri, 2 Oct 2015 11:00:41 -0700
Subject: [PATCH 09/11] Fix failure in spack.test.config.ConfigTest from
 incorrect compiler config merging

---
 lib/spack/spack/config.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py
index 60577c45b3..712a2b78fc 100644
--- a/lib/spack/spack/config.py
+++ b/lib/spack/spack/config.py
@@ -174,8 +174,12 @@ def _merge_dicts(d1, d2):
         for key2, val2 in d2.iteritems():
             if not key2 in d1:
                 d1[key2] = val2
-            else:
+            elif type(d1[key2]) is dict and type(val2) is dict:
                 d1[key2] = _merge_dicts(d1[key2], val2)
+            elif (type(d1) is list) and (type(d2) is list):
+                d1.extend(d2)
+            else:
+                d1[key2] = val2
         return d1
 
     return d2
@@ -360,7 +364,7 @@ def add_to_mirror_config(addition_dict, scope=None):
 
 
 def add_to_compiler_config(addition_dict, scope=None, arch=None):
-    """Add compilerss to the configuration files"""
+    """Add compilers to the configuration files"""
     if not arch:
         arch = spack.architecture.sys_type()
     add_to_config('compilers', { arch : addition_dict }, scope)

From 18f0b24a7f21ec7b46510f45867386b7600bbc55 Mon Sep 17 00:00:00 2001
From: Matthew LeGendre <legendre1@llnl.gov>
Date: Mon, 5 Oct 2015 11:30:48 -0700
Subject: [PATCH 10/11] Add tests for spack external dependencies, plus fixes
 for issues found by those tests.

---
 lib/spack/spack/concretize.py                 |  9 +++--
 lib/spack/spack/spec.py                       |  2 +-
 lib/spack/spack/test/concretize.py            | 28 ++++++++++++++
 .../site_spackconfig/packages.yaml            | 13 +++++++
 .../mock_packages/externalprereq/package.py   | 34 +++++++++++++++++
 .../mock_packages/externaltest/package.py     | 37 +++++++++++++++++++
 .../mock_packages/externaltool/package.py     | 36 ++++++++++++++++++
 .../mock_packages/externalvirtual/package.py  | 37 +++++++++++++++++++
 8 files changed, 191 insertions(+), 5 deletions(-)
 create mode 100644 var/spack/mock_configs/site_spackconfig/packages.yaml
 create mode 100644 var/spack/mock_packages/externalprereq/package.py
 create mode 100644 var/spack/mock_packages/externaltest/package.py
 create mode 100644 var/spack/mock_packages/externaltool/package.py
 create mode 100644 var/spack/mock_packages/externalvirtual/package.py

diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py
index 01ff163493..c27a023136 100644
--- a/lib/spack/spack/concretize.py
+++ b/lib/spack/spack/concretize.py
@@ -85,8 +85,8 @@ def _valid_virtuals_and_externals(self, spec):
             provider_cmp = partial(spack.pkgsort.provider_compare, spec_w_preferred_providers.name, spec.name)
             packages = sorted(providers, cmp=provider_cmp)
         else:
-            if not spec_externals(spec) or spec.external:
-                return None
+            if spec.external:
+                return False
             packages = [spec]
 
         # For each candidate package, if it has externals add those to the candidates
@@ -129,7 +129,7 @@ def concretize_virtual_and_external(self, spec):
                 #Try a looser ABI matching
                 candidate = next((c for c in candidates if spack.abi.compatible(c[0], other_spec, loose=True)), None)
         if not candidate:
-            #Pick the first choice
+            #No ABI matches. Pick the top choice based on the orignal preferences.
             candidate = candidates[0]
         external = candidate[1]
         candidate_spec = candidate[0]
@@ -144,10 +144,11 @@ def concretize_virtual_and_external(self, spec):
         if not spec.external and external:
             spec.external = external
             changed = True
+
         #If we're external then trim the dependencies
         if external and spec.dependencies:
             changed = True
-            spec.depencencies = DependencyMap()
+            spec.dependencies = DependencyMap()
         
         return changed
         
diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py
index 6984b4a174..49b67cd361 100644
--- a/lib/spack/spack/spec.py
+++ b/lib/spack/spack/spec.py
@@ -1022,7 +1022,7 @@ def _normalize_helper(self, visited, spec_deps, provider_index):
 
         # if we descend into a virtual spec, there's nothing more
         # to normalize.  Concretize will finish resolving it later.
-        if self.virtual:
+        if self.virtual or self.external:
             return False
 
         # Combine constraints from package deps with constraints from
diff --git a/lib/spack/spack/test/concretize.py b/lib/spack/spack/test/concretize.py
index b3a77d076a..f81a2f5af8 100644
--- a/lib/spack/spack/test/concretize.py
+++ b/lib/spack/spack/test/concretize.py
@@ -192,3 +192,31 @@ def test_compiler_inheritance(self):
         # TODO: not exactly the syntax I would like.
         self.assertTrue(spec['libdwarf'].compiler.satisfies('clang'))
         self.assertTrue(spec['libelf'].compiler.satisfies('clang'))
+
+
+    def test_external_package(self):
+        spec = Spec('externaltool')
+        spec.concretize()
+
+        self.assertEqual(spec['externaltool'].external, '/path/to/external_tool')
+        self.assertFalse('externalprereq' in spec)
+        self.assertTrue(spec['externaltool'].compiler.satisfies('gcc'))
+
+
+    def test_nobuild_package(self):
+        got_error = False
+        spec = Spec('externaltool%clang')
+        try:
+            spec.concretize()
+        except spack.concretize.NoBuildError:
+            got_error = True
+        self.assertTrue(got_error)
+
+
+    def test_external_and_virtual(self):
+        spec = Spec('externaltest')
+        spec.concretize()
+        self.assertTrue(spec['externaltool'].external, '/path/to/external_tool')
+        self.assertTrue(spec['stuff'].external, '/path/to/external_virtual_gcc')
+        self.assertTrue(spec['externaltool'].compiler.satisfies('gcc'))
+        self.assertTrue(spec['stuff'].compiler.satisfies('gcc'))
diff --git a/var/spack/mock_configs/site_spackconfig/packages.yaml b/var/spack/mock_configs/site_spackconfig/packages.yaml
new file mode 100644
index 0000000000..eb52c6cf11
--- /dev/null
+++ b/var/spack/mock_configs/site_spackconfig/packages.yaml
@@ -0,0 +1,13 @@
+packages:
+  - externaltool:
+      nobuild: True
+  - externaltool@1.0%gcc@4.5.0:
+      path: /path/to/external_tool
+  - externalvirtual@2.0%clang@3.3:
+      path: /path/to/external_virtual_clang
+      nobuild: True
+  - externalvirtual@1.0%gcc@4.5.0:
+      path: /path/to/external_virtual_gcc
+      nobuild: True
+
+
diff --git a/var/spack/mock_packages/externalprereq/package.py b/var/spack/mock_packages/externalprereq/package.py
new file mode 100644
index 0000000000..7d63925693
--- /dev/null
+++ b/var/spack/mock_packages/externalprereq/package.py
@@ -0,0 +1,34 @@
+##############################################################################
+# Copyright (c) 2013, Lawrence Livermore National Security, LLC.
+# Produced at the Lawrence Livermore National Laboratory.
+#
+# This file is part of Spack.
+# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
+# LLNL-CODE-647188
+#
+# For details, see https://scalability-llnl.github.io/spack
+# Please also see the LICENSE file for our notice and the LGPL.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License (as published by
+# the Free Software Foundation) version 2.1 dated February 1999.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
+# conditions of the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+##############################################################################
+from spack import *
+
+class Externalprereq(Package):
+    homepage = "http://somewhere.com"
+    url      = "http://somewhere.com/prereq-1.0.tar.gz"
+
+    version('1.4', 'f1234567890abcdef1234567890abcde')
+
+    def install(self, spec, prefix):
+        pass
diff --git a/var/spack/mock_packages/externaltest/package.py b/var/spack/mock_packages/externaltest/package.py
new file mode 100644
index 0000000000..c546922f87
--- /dev/null
+++ b/var/spack/mock_packages/externaltest/package.py
@@ -0,0 +1,37 @@
+##############################################################################
+# Copyright (c) 2013, Lawrence Livermore National Security, LLC.
+# Produced at the Lawrence Livermore National Laboratory.
+#
+# This file is part of Spack.
+# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
+# LLNL-CODE-647188
+#
+# For details, see https://scalability-llnl.github.io/spack
+# Please also see the LICENSE file for our notice and the LGPL.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License (as published by
+# the Free Software Foundation) version 2.1 dated February 1999.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
+# conditions of the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+##############################################################################
+from spack import *
+
+class Externaltest(Package):
+    homepage = "http://somewhere.com"
+    url      = "http://somewhere.com/test-1.0.tar.gz"
+
+    version('1.0', '1234567890abcdef1234567890abcdef')
+
+    depends_on('stuff')
+    depends_on('externaltool')
+
+    def install(self, spec, prefix):
+        pass
diff --git a/var/spack/mock_packages/externaltool/package.py b/var/spack/mock_packages/externaltool/package.py
new file mode 100644
index 0000000000..af902bd70e
--- /dev/null
+++ b/var/spack/mock_packages/externaltool/package.py
@@ -0,0 +1,36 @@
+##############################################################################
+# Copyright (c) 2013, Lawrence Livermore National Security, LLC.
+# Produced at the Lawrence Livermore National Laboratory.
+#
+# This file is part of Spack.
+# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
+# LLNL-CODE-647188
+#
+# For details, see https://scalability-llnl.github.io/spack
+# Please also see the LICENSE file for our notice and the LGPL.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License (as published by
+# the Free Software Foundation) version 2.1 dated February 1999.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
+# conditions of the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+##############################################################################
+from spack import *
+
+class Externaltool(Package):
+    homepage = "http://somewhere.com"
+    url      = "http://somewhere.com/tool-1.0.tar.gz"
+
+    version('1.0', '1234567890abcdef1234567890abcdef')
+
+    depends_on('externalprereq')
+
+    def install(self, spec, prefix):
+        pass
diff --git a/var/spack/mock_packages/externalvirtual/package.py b/var/spack/mock_packages/externalvirtual/package.py
new file mode 100644
index 0000000000..722c1e1c53
--- /dev/null
+++ b/var/spack/mock_packages/externalvirtual/package.py
@@ -0,0 +1,37 @@
+##############################################################################
+# Copyright (c) 2013, Lawrence Livermore National Security, LLC.
+# Produced at the Lawrence Livermore National Laboratory.
+#
+# This file is part of Spack.
+# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
+# LLNL-CODE-647188
+#
+# For details, see https://scalability-llnl.github.io/spack
+# Please also see the LICENSE file for our notice and the LGPL.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License (as published by
+# the Free Software Foundation) version 2.1 dated February 1999.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
+# conditions of the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+##############################################################################
+from spack import *
+
+class Externalvirtual(Package):
+    homepage = "http://somewhere.com"
+    url      = "http://somewhere.com/stuff-1.0.tar.gz"
+
+    version('1.0', '1234567890abcdef1234567890abcdef')
+    version('2.0', '234567890abcdef1234567890abcdef1')
+
+    provides('stuff')
+
+    def install(self, spec, prefix):
+        pass

From fac4428766fb0a6b6cd357b654215f55df1220d4 Mon Sep 17 00:00:00 2001
From: Matthew LeGendre <legendre1@llnl.gov>
Date: Mon, 5 Oct 2015 14:04:33 -0700
Subject: [PATCH 11/11] Documentation for external packages.

---
 lib/spack/docs/site_configuration.rst | 73 +++++++++++++++++++++++++++
 1 file changed, 73 insertions(+)

diff --git a/lib/spack/docs/site_configuration.rst b/lib/spack/docs/site_configuration.rst
index 1e6740a434..a7211a9d95 100644
--- a/lib/spack/docs/site_configuration.rst
+++ b/lib/spack/docs/site_configuration.rst
@@ -54,6 +54,79 @@ more elements to the list to indicate where your own site's temporary
 directory is.
 
 
+External Packages
+~~~~~~~~~~~~~~~~~~~~~
+It's possible for Spack to use certain externally-installed 
+packages rather than always rebuilding packages. This may be desirable
+if machines ship with system packages, such as a customized MPI
+that should be used instead of Spack building its own MPI.
+
+External packages are configured through the ``packages.yaml`` file found
+in a Spack installation's ``etc/spack/`` or a user's ``~/.spack/``
+directory. Here's an example of an external configuration::
+
+.. code-block:: yaml
+
+  packages:
+    - openmpi@1.4.3%gcc@4.4.7=chaos_5_x86_64_ib:
+        path: /opt/openmpi-1.4.3
+    - openmpi@1.4.3%gcc@4.4.7=chaos_5_x86_64_ib+debug:
+        path: /opt/openmpi-1.4.3-debug
+    - openmpi@1.6.5%intel@10.1=chaos_5_x86_64_ib:
+        path: /opt/openmpi-1.6.5-intel
+
+This example lists three installations of OpenMPI, one built with gcc,
+one built with gcc and debug information, and another built with OpenMPI.
+If Spack is asked to build a package that uses one of these MPIs as a 
+dependency, it link the package to the pre-installed OpenMPI in
+the given directory.  
+
+Each ``packages.yaml`` should begin with a ``packages:`` token, followed
+by a list of package specs.  Specs in the ``packages.yaml`` have at most
+one ``path`` tag, which specifies the top-level directory where the 
+spec is installed.  
+
+Each spec should be as well-defined as reasonably possible.  If a 
+package lacks a spec component, such as missing a compiler or 
+package version, then Spack will guess the missing component based 
+on its most-favored packages, and it may guess incorrectly.
+
+All package versions and compilers listed in ``packages.yaml`` should 
+have entries in Spack's packages and compiler configuration, even
+the package and compiler may not actually be used.
+
+The packages configuration can tell Spack to use an external location
+for certain package versions, but it does not restrict Spack to using
+external packages.  In the above example, if an OpenMPI 1.8.4 became
+available Spack may choose to start building and linking with that version
+rather than continue using the pre-installed OpenMPI versions.
+
+To prevent this, the ``packages.yaml`` configuration also allows packages
+to be flagged as non-buildable.  The previous example could be modified to
+be::
+
+.. code-block:: yaml
+
+  packages:
+    - openmpi:
+        nobuild: True
+    - openmpi@1.4.3%gcc@4.4.7=chaos_5_x86_64_ib:
+        path: /opt/openmpi-1.4.3
+    - openmpi@1.4.3%gcc@4.4.7=chaos_5_x86_64_ib+debug:
+        path: /opt/openmpi-1.4.3-debug
+    - openmpi@1.6.5%intel@10.1=chaos_5_x86_64_ib:
+        path: /opt/openmpi-1.6.5-intel
+
+The addition of the ``nobuild`` flag tells Spack that it should never build
+its own version of OpenMPI, and it will instead always rely on a pre-built
+OpenMPI.  Similar to ``path``, ``nobuild`` is specified as a property under
+a spec and will prevent building of anything that satisfies that spec.
+
+The ``nobuild`` does not need to be paired with external packages.  
+It could also be used alone to forbid versions of packages that may be 
+buggy or otherwise undesirable.
+  
+
 Profiling
 ~~~~~~~~~~~~~~~~~~~~~