From 6548f2db5c89280496e8bf6cbd5b992f77740330 Mon Sep 17 00:00:00 2001 From: scheibelp Date: Wed, 21 Mar 2018 14:55:26 -0700 Subject: [PATCH] Enable creation of mirrors for systems with different compilers (#5153) * Create mirror for system with different compilers Spack concretizes the spec provided by the user in "spack mirror create" to ensure downloading the right dependencies. Under normal circumstances concretization requires that the chosen compiler exists on the system, but this is not required when creating download mirrors for other systems, so this requirement is removed in that case. * Add test for disabling compiler existence check * Update compiler existence checking logic * improve test for disabling compiler existence check --- lib/spack/spack/cmd/mirror.py | 1 + lib/spack/spack/concretize.py | 50 ++++++++++++++++++++++-------- lib/spack/spack/test/concretize.py | 12 +++++++ 3 files changed, 50 insertions(+), 13 deletions(-) diff --git a/lib/spack/spack/cmd/mirror.py b/lib/spack/spack/cmd/mirror.py index cd59be5782..f00df323f4 100644 --- a/lib/spack/spack/cmd/mirror.py +++ b/lib/spack/spack/cmd/mirror.py @@ -167,6 +167,7 @@ def mirror_create(args): """Create a directory to be used as a spack mirror, and fill it with package archives.""" # try to parse specs from the command line first. + spack.concretizer.disable_compiler_existence_check() specs = spack.cmd.parse_specs(args.specs, concretize=True) # If there is a file, parse each line as a spec and add it to the list. diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 3b1e83cb5b..4d1ddf7ac9 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -48,10 +48,18 @@ class DefaultConcretizer(object): - """This class doesn't have any state, it just provides some methods for - concretization. You can subclass it to override just some of the - default concretization strategies, or you can override all of them. + """You can subclass this class to override some of the default + concretization strategies, or you can override all of them. """ + def __init__(self): + self.check_for_compiler_existence = True + + def disable_compiler_existence_check(self): + self.check_for_compiler_existence = False + + def enable_compiler_existence_check(self): + self.check_for_compiler_existence = True + def _valid_virtuals_and_externals(self, spec): """Returns a list of candidate virtual dep providers and external packages that coiuld be used to concretize a spec. @@ -283,14 +291,9 @@ def concretize_compiler(self, spec): def _proper_compiler_style(cspec, aspec): return spack.compilers.compilers_for_spec(cspec, arch_spec=aspec) - all_compiler_specs = spack.compilers.all_compiler_specs() - if not all_compiler_specs: - raise spack.compilers.NoCompilersError() - - if (spec.compiler and - spec.compiler.concrete and - spec.compiler in all_compiler_specs): - if not _proper_compiler_style(spec.compiler, spec.architecture): + if spec.compiler and spec.compiler.concrete: + if (self.check_for_compiler_existence and not + _proper_compiler_style(spec.compiler, spec.architecture)): _compiler_concretization_failure( spec.compiler, spec.architecture) return False @@ -302,6 +305,22 @@ def _proper_compiler_style(cspec, aspec): assert(other_spec) # Check if the compiler is already fully specified + if (other_compiler and other_compiler.concrete and + not self.check_for_compiler_existence): + spec.compiler = other_compiler.copy() + return True + + all_compiler_specs = spack.compilers.all_compiler_specs() + if not all_compiler_specs: + # If compiler existence checking is disabled, then we would have + # exited by now if there were sufficient hints to form a full + # compiler spec. Therefore even if compiler existence checking is + # disabled, compilers must be available at this point because the + # available compilers are used to choose a compiler. If compiler + # existence checking is enabled then some compiler must exist in + # order to complete the spec. + raise spack.compilers.NoCompilersError() + if other_compiler in all_compiler_specs: spec.compiler = other_compiler.copy() if not _proper_compiler_style(spec.compiler, spec.architecture): @@ -377,8 +396,13 @@ def concretize_compiler_flags(self, spec): # Include the compiler flag defaults from the config files # This ensures that spack will detect conflicts that stem from a change # in default compiler flags. - compiler = spack.compilers.compiler_for_spec( - spec.compiler, spec.architecture) + try: + compiler = spack.compilers.compiler_for_spec( + spec.compiler, spec.architecture) + except spack.compilers.NoCompilerForSpecError: + if self.check_for_compiler_existence: + raise + return ret for flag in compiler.flags: config_flags = set(compiler.flags.get(flag, [])) flags = set(spec.compiler_flags.get(flag, [])) diff --git a/lib/spack/spack/test/concretize.py b/lib/spack/spack/test/concretize.py index 2482452208..7885ca279f 100644 --- a/lib/spack/spack/test/concretize.py +++ b/lib/spack/spack/test/concretize.py @@ -147,6 +147,18 @@ def test_concretize_with_restricted_virtual(self): concrete = check_concretize('mpileaks ^mpich2@1.3.1:1.4') assert concrete['mpich2'].satisfies('mpich2@1.3.1:1.4') + def test_concretize_disable_compiler_existence_check(self): + with pytest.raises(spack.concretize.UnavailableCompilerVersionError): + check_concretize('dttop %gcc@100.100') + + try: + spack.concretizer.disable_compiler_existence_check() + spec = check_concretize('dttop %gcc@100.100') + assert spec.satisfies('%gcc@100.100') + assert spec['dtlink3'].satisfies('%gcc@100.100') + finally: + spack.concretizer.enable_compiler_existence_check() + def test_concretize_with_provides_when(self): """Make sure insufficient versions of MPI are not in providers list when we ask for some advanced version.