Enable creation of mirrors for systems with different compilers (#5153)

* Create mirror for system with different compilers

Spack concretizes the spec provided by the user in
"spack mirror create" to ensure downloading the right
dependencies. Under normal circumstances concretization
requires that the chosen compiler exists on the system,
but this is not required when creating download mirrors
for other systems, so this requirement is removed in that
case.

* Add test for disabling compiler existence check

* Update compiler existence checking logic

* improve test for disabling compiler existence check

lib/spack/spack/cmd/mirror.py

@@ -167,6 +167,7 @@ def mirror_create(args):
     """Create a directory to be used as a spack mirror, and fill it with
        package archives."""
     # try to parse specs from the command line first.
+    spack.concretizer.disable_compiler_existence_check()
     specs = spack.cmd.parse_specs(args.specs, concretize=True)
 
     # If there is a file, parse each line as a spec and add it to the list.

lib/spack/spack/concretize.py

@@ -48,10 +48,18 @@
 
 
 class DefaultConcretizer(object):
-    """This class doesn't have any state, it just provides some methods for
-       concretization.  You can subclass it to override just some of the
-       default concretization strategies, or you can override all of them.
+    """You can subclass this class to override some of the default
+       concretization strategies, or you can override all of them.
     """
+    def __init__(self):
+        self.check_for_compiler_existence = True
+
+    def disable_compiler_existence_check(self):
+        self.check_for_compiler_existence = False
+
+    def enable_compiler_existence_check(self):
+        self.check_for_compiler_existence = True
+
     def _valid_virtuals_and_externals(self, spec):
         """Returns a list of candidate virtual dep providers and external
            packages that coiuld be used to concretize a spec.
@@ -283,14 +291,9 @@ def concretize_compiler(self, spec):
         def _proper_compiler_style(cspec, aspec):
             return spack.compilers.compilers_for_spec(cspec, arch_spec=aspec)
 
-        all_compiler_specs = spack.compilers.all_compiler_specs()
-        if not all_compiler_specs:
-            raise spack.compilers.NoCompilersError()
-
-        if (spec.compiler and
-            spec.compiler.concrete and
-                spec.compiler in all_compiler_specs):
-            if not _proper_compiler_style(spec.compiler, spec.architecture):
+        if spec.compiler and spec.compiler.concrete:
+            if (self.check_for_compiler_existence and not
+                    _proper_compiler_style(spec.compiler, spec.architecture)):
                 _compiler_concretization_failure(
                     spec.compiler, spec.architecture)
             return False
@@ -302,6 +305,22 @@ def _proper_compiler_style(cspec, aspec):
         assert(other_spec)
 
         # Check if the compiler is already fully specified
+        if (other_compiler and other_compiler.concrete and
+                not self.check_for_compiler_existence):
+            spec.compiler = other_compiler.copy()
+            return True
+
+        all_compiler_specs = spack.compilers.all_compiler_specs()
+        if not all_compiler_specs:
+            # If compiler existence checking is disabled, then we would have
+            # exited by now if there were sufficient hints to form a full
+            # compiler spec. Therefore even if compiler existence checking is
+            # disabled, compilers must be available at this point because the
+            # available compilers are used to choose a compiler. If compiler
+            # existence checking is enabled then some compiler must exist in
+            # order to complete the spec.
+            raise spack.compilers.NoCompilersError()
+
         if other_compiler in all_compiler_specs:
             spec.compiler = other_compiler.copy()
             if not _proper_compiler_style(spec.compiler, spec.architecture):
@@ -377,8 +396,13 @@ def concretize_compiler_flags(self, spec):
         # Include the compiler flag defaults from the config files
         # This ensures that spack will detect conflicts that stem from a change
         # in default compiler flags.
-        compiler = spack.compilers.compiler_for_spec(
-            spec.compiler, spec.architecture)
+        try:
+            compiler = spack.compilers.compiler_for_spec(
+                spec.compiler, spec.architecture)
+        except spack.compilers.NoCompilerForSpecError:
+            if self.check_for_compiler_existence:
+                raise
+            return ret
         for flag in compiler.flags:
             config_flags = set(compiler.flags.get(flag, []))
             flags = set(spec.compiler_flags.get(flag, []))

lib/spack/spack/test/concretize.py

@@ -147,6 +147,18 @@ def test_concretize_with_restricted_virtual(self):
         concrete = check_concretize('mpileaks   ^mpich2@1.3.1:1.4')
         assert concrete['mpich2'].satisfies('mpich2@1.3.1:1.4')
 
+    def test_concretize_disable_compiler_existence_check(self):
+        with pytest.raises(spack.concretize.UnavailableCompilerVersionError):
+            check_concretize('dttop %gcc@100.100')
+
+        try:
+            spack.concretizer.disable_compiler_existence_check()
+            spec = check_concretize('dttop %gcc@100.100')
+            assert spec.satisfies('%gcc@100.100')
+            assert spec['dtlink3'].satisfies('%gcc@100.100')
+        finally:
+            spack.concretizer.enable_compiler_existence_check()
+
     def test_concretize_with_provides_when(self):
         """Make sure insufficient versions of MPI are not in providers list when
         we ask for some advanced version.