Added a function that concretizes specs together (#11158)

* Added a function that concretizes specs together

* Specs concretized together are copied instead of being referenced

This makes the specs different objects and removes any reference to the
fake root package that is needed currently for concretization.

* Factored creating a repository for concretization into its own function

* Added a test on overlapping dependencies

lib/spack/spack/concretize.py

@@ -15,6 +15,12 @@
       concretization  policies.
 """
 from __future__ import print_function
+
+import os.path
+import tempfile
+import llnl.util.filesystem as fs
+import llnl.util.tty as tty
+
 from itertools import chain
 from functools_backport import reverse_order
 from contextlib import contextmanager
@@ -28,6 +34,7 @@
 import spack.compilers
 import spack.architecture
 import spack.error
+import spack.tengine
 from spack.config import config
 from spack.version import ver, Version, VersionList, VersionRange
 from spack.package_prefs import PackagePrefs, spec_externals, is_spec_buildable
@@ -465,6 +472,57 @@ def _compiler_concretization_failure(compiler_spec, arch):
         raise UnavailableCompilerVersionError(compiler_spec, arch)
 
 
+def concretize_specs_together(*abstract_specs):
+    """Given a number of specs as input, tries to concretize them together.
+
+    Args:
+        *abstract_specs: abstract specs to be concretized, given either
+            as Specs or strings
+
+    Returns:
+        List of concretized specs
+    """
+    def make_concretization_repository(abstract_specs):
+        """Returns the path to a temporary repository created to contain
+        a fake package that depends on all of the abstract specs.
+        """
+        tmpdir = tempfile.mkdtemp()
+        repo_path, _ = spack.repo.create_repo(tmpdir)
+
+        debug_msg = '[CONCRETIZATION]: Creating helper repository in {0}'
+        tty.debug(debug_msg.format(repo_path))
+
+        pkg_dir = os.path.join(repo_path, 'packages', 'concretizationroot')
+        fs.mkdirp(pkg_dir)
+        environment = spack.tengine.make_environment()
+        template = environment.get_template('misc/coconcretization.pyt')
+
+        # Split recursive specs, as it seems the concretizer has issue
+        # respecting conditions on dependents expressed like
+        # depends_on('foo ^bar@1.0'), see issue #11160
+        split_specs = [dep for spec in abstract_specs
+                       for dep in spec.traverse(root=True)]
+
+        with open(os.path.join(pkg_dir, 'package.py'), 'w') as f:
+            f.write(template.render(specs=[str(s) for s in split_specs]))
+
+        return spack.repo.Repo(repo_path)
+
+    abstract_specs = [spack.spec.Spec(s) for s in abstract_specs]
+    concretization_repository = make_concretization_repository(abstract_specs)
+
+    with spack.repo.additional_repository(concretization_repository):
+        # Spec from a helper package that depends on all the abstract_specs
+        concretization_root = spack.spec.Spec('concretizationroot')
+        concretization_root.concretize()
+        # Retrieve the direct dependencies
+        concrete_specs = [
+            concretization_root[spec.name].copy() for spec in abstract_specs
+        ]
+
+    return concrete_specs
+
+
 class NoCompilersForArchError(spack.error.SpackError):
     def __init__(self, arch, available_os_targets):
         err_msg = ("No compilers found"
@@ -474,8 +532,9 @@ def __init__(self, arch, available_os_targets):
                    (arch.os, arch.target))
 
         available_os_target_strs = list()
-        for os, t in available_os_targets:
-            os_target_str = "%s-%s" % (os, t) if t else os
+        for operating_system, t in available_os_targets:
+            os_target_str = "%s-%s" % (operating_system, t) if t \
+                else operating_system
             available_os_target_strs.append(os_target_str)
         err_msg += (
             "\nCompilers are defined for the following"

lib/spack/spack/repo.py

@@ -1223,6 +1223,18 @@ def swap(repo_path):
     path = saved
 
 
+@contextlib.contextmanager
+def additional_repository(repository):
+    """Adds temporarily a repository to the default one.
+
+    Args:
+        repository: repository to be added
+    """
+    path.put_first(repository)
+    yield
+    path.remove(repository)
+
+
 class RepoError(spack.error.SpackError):
     """Superclass for repository-related errors."""
 

lib/spack/spack/test/concretize.py

@@ -7,6 +7,7 @@
 import llnl.util.lang
 
 import spack.architecture
+import spack.concretize
 import spack.repo
 
 from spack.concretize import find_spec
@@ -525,3 +526,41 @@ def test_regression_issue_7941(self):
         t.concretize()
 
         assert s.dag_hash() == t.dag_hash()
+
+    @pytest.mark.parametrize('abstract_specs', [
+        # Establish a baseline - concretize a single spec
+        ('mpileaks',),
+        # When concretized together with older version of callpath
+        # and dyninst it uses those older versions
+        ('mpileaks', 'callpath@0.9', 'dyninst@8.1.1'),
+        # Handle recursive syntax within specs
+        ('mpileaks', 'callpath@0.9 ^dyninst@8.1.1', 'dyninst'),
+        # Test specs that have overlapping dependencies but are not
+        # one a dependency of the other
+        ('mpileaks', 'direct-mpich')
+    ])
+    def test_simultaneous_concretization_of_specs(self, abstract_specs):
+
+        abstract_specs = [Spec(x) for x in abstract_specs]
+        concrete_specs = spack.concretize.concretize_specs_together(
+            *abstract_specs
+        )
+
+        # Check there's only one configuration of each package in the DAG
+        names = set(
+            dep.name for spec in concrete_specs for dep in spec.traverse()
+        )
+        for name in names:
+            name_specs = set(
+                spec[name] for spec in concrete_specs if name in spec
+            )
+            assert len(name_specs) == 1
+
+        # Check that there's at least one Spec that satisfies the
+        # initial abstract request
+        for aspec in abstract_specs:
+            assert any(cspec.satisfies(aspec) for cspec in concrete_specs)
+
+        # Make sure the concrete spec are top-level specs with no dependents
+        for spec in concrete_specs:
+            assert not spec.dependents()

share/spack/templates/misc/coconcretization.pyt

@@ -0,0 +1,15 @@
+# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+
+class Concretizationroot(Package):
+    url = 'fake_url'
+
+    version('1.0')
+
+{% for dep in specs %}
+    depends_on('{{ dep }}')
+{% endfor %}
+