diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index 9c8cf33fb1..4bd15a3219 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -12,7 +12,6 @@ import shutil import sys import tempfile -from subprocess import PIPE, run from typing import List, Optional, Sequence import llnl.path @@ -24,6 +23,7 @@ import spack.error import spack.spec import spack.util.executable +import spack.util.libc import spack.util.module_cmd import spack.version from spack.util.environment import filter_system_paths @@ -197,98 +197,6 @@ def _parse_dynamic_linker(output: str): return arg.split("=", 1)[1] -def _libc_from_ldd(ldd: str) -> Optional["spack.spec.Spec"]: - try: - result = run([ldd, "--version"], stdout=PIPE, stderr=PIPE, check=False) - stdout = result.stdout.decode("utf-8") - except Exception: - return None - - if not re.search("gnu|glibc", stdout, re.IGNORECASE): - return None - - version_str = re.match(r".+\(.+\) (.+)", stdout) - if not version_str: - return None - try: - return spack.spec.Spec(f"glibc@={version_str.group(1)}") - except Exception: - return None - - -def _libc_from_dynamic_linker(dynamic_linker: str) -> Optional["spack.spec.Spec"]: - if not os.path.exists(dynamic_linker): - return None - - # The dynamic linker is usually installed in the same /lib(64)?/ld-*.so path across all - # distros. The rest of libc is elsewhere, e.g. /usr. Typically the dynamic linker is then - # a symlink into /usr/lib, which we use to for determining the actual install prefix of - # libc. - realpath = os.path.realpath(dynamic_linker) - - prefix = os.path.dirname(realpath) - # Remove the multiarch suffix if it exists - if os.path.basename(prefix) not in ("lib", "lib64"): - prefix = os.path.dirname(prefix) - - # Non-standard install layout -- just bail. - if os.path.basename(prefix) not in ("lib", "lib64"): - return None - - prefix = os.path.dirname(prefix) - - # Now try to figure out if glibc or musl, which is the only ones we support. - # In recent glibc we can simply execute the dynamic loader. In musl that's always the case. - try: - result = run([dynamic_linker, "--version"], stdout=PIPE, stderr=PIPE, check=False) - stdout = result.stdout.decode("utf-8") - stderr = result.stderr.decode("utf-8") - except Exception: - return None - - # musl prints to stderr - if stderr.startswith("musl libc"): - version_str = re.search(r"^Version (.+)$", stderr, re.MULTILINE) - if not version_str: - return None - try: - spec = spack.spec.Spec(f"musl@={version_str.group(1)}") - spec.external_path = prefix - return spec - except Exception: - return None - elif re.search("gnu|glibc", stdout, re.IGNORECASE): - # output is like "ld.so (...) stable release version 2.33." write a regex for it - match = re.search(r"version (\d+\.\d+(?:\.\d+)?)", stdout) - if not match: - return None - try: - version = match.group(1) - spec = spack.spec.Spec(f"glibc@={version}") - spec.external_path = prefix - return spec - except Exception: - return None - else: - # Could not get the version by running the dynamic linker directly. Instead locate `ldd` - # relative to the dynamic linker. - ldd = os.path.join(prefix, "bin", "ldd") - if not os.path.exists(ldd): - # If `/lib64/ld.so` was not a symlink to `/usr/lib/ld.so` we can try to use /usr as - # prefix. This is the case on ubuntu 18.04 where /lib != /usr/lib. - if prefix != "/": - return None - prefix = "/usr" - ldd = os.path.join(prefix, "bin", "ldd") - if not os.path.exists(ldd): - return None - maybe_spec = _libc_from_ldd(ldd) - if not maybe_spec: - return None - maybe_spec.external_path = prefix - return maybe_spec - - def in_system_subdirectory(path): system_dirs = [ "/lib/", @@ -536,7 +444,9 @@ def implicit_rpaths(self) -> List[str]: all_required_libs = list(self.required_libs) + Compiler._all_compiler_rpath_libraries return list(paths_containing_libs(link_dirs, all_required_libs)) + @property def default_libc(self) -> Optional["spack.spec.Spec"]: + """Determine libc targeted by the compiler from link line""" output = self.compiler_verbose_output if not output: @@ -547,7 +457,7 @@ def default_libc(self) -> Optional["spack.spec.Spec"]: if not dynamic_linker: return None - return _libc_from_dynamic_linker(dynamic_linker) + return spack.util.libc.libc_from_dynamic_linker(dynamic_linker) @property def required_libs(self): diff --git a/lib/spack/spack/solver/asp.py b/lib/spack/spack/solver/asp.py index dc79a7eead..2b86200a30 100644 --- a/lib/spack/spack/solver/asp.py +++ b/lib/spack/spack/solver/asp.py @@ -41,6 +41,8 @@ import spack.spec import spack.store import spack.util.crypto +import spack.util.elf +import spack.util.libc import spack.util.path import spack.util.timer import spack.variant @@ -283,20 +285,27 @@ def all_compilers_in_config(configuration): return spack.compilers.all_compilers_from(configuration) -def compatible_libc(candidate_libc_spec): - """Returns a list of libc specs that are compatible with the one passed as argument""" - result = set() - for compiler in all_compilers_in_config(spack.config.CONFIG): - libc = compiler.default_libc() - if not libc: - continue - if ( - libc.name == candidate_libc_spec.name - and libc.version >= candidate_libc_spec.version - and libc.external_path == candidate_libc_spec.external_path - ): - result.add(libc) - return sorted(result) +def all_libcs() -> Set[spack.spec.Spec]: + """Return a set of all libc specs targeted by any configured compiler. If none, fall back to + libc determined from the current Python process if dynamically linked.""" + + libcs = { + c.default_libc for c in all_compilers_in_config(spack.config.CONFIG) if c.default_libc + } + + if libcs: + return libcs + + libc = spack.util.libc.libc_from_current_python_process() + return {libc} if libc else set() + + +def libc_is_compatible(lhs: spack.spec.Spec, rhs: spack.spec.Spec) -> List[spack.spec.Spec]: + return ( + lhs.name == rhs.name + and lhs.external_path == rhs.external_path + and lhs.version >= rhs.version + ) def using_libc_compatibility() -> bool: @@ -597,7 +606,7 @@ def _external_config_with_implicit_externals(configuration): return packages_yaml for compiler in all_compilers_in_config(configuration): - libc = compiler.default_libc() + libc = compiler.default_libc if libc: entry = {"spec": f"{libc} %{compiler.spec}", "prefix": libc.external_path} packages_yaml.setdefault(libc.name, {}).setdefault("externals", []).append(entry) @@ -1028,6 +1037,9 @@ def __init__(self, tests: bool = False): self.pkgs: Set[str] = set() self.explicitly_required_namespaces: Dict[str, str] = {} + # list of unique libc specs targeted by compilers (or an educated guess if no compiler) + self.libcs: List[spack.spec.Spec] = [] + def pkg_version_rules(self, pkg): """Output declared versions of a package. @@ -1872,13 +1884,14 @@ def _spec_clauses( if dep.name == "gcc-runtime": continue - # LIBC is also solved again by clingo, but in this case the compatibility + # libc is also solved again by clingo, but in this case the compatibility # is not encoded in the parent node - so we need to emit explicit facts if "libc" in dspec.virtuals: - for x in compatible_libc(dep): - clauses.append( - fn.attr("compatible_libc", spec.name, x.name, x.version) - ) + for libc in self.libcs: + if libc_is_compatible(libc, dep): + clauses.append( + fn.attr("compatible_libc", spec.name, libc.name, libc.version) + ) continue # We know dependencies are real for concrete specs. For abstract @@ -2336,6 +2349,7 @@ def setup( node_counter = _create_counter(specs, tests=self.tests) self.possible_virtuals = node_counter.possible_virtuals() self.pkgs = node_counter.possible_dependencies() + self.libcs = sorted(all_libcs()) # type: ignore[type-var] # Fail if we already know an unreachable node is requested for spec in specs: @@ -2345,16 +2359,16 @@ def setup( if missing_deps: raise spack.spec.InvalidDependencyError(spec.name, missing_deps) - for node in spack.traverse.traverse_nodes(specs): + for node in traverse.traverse_nodes(specs): if node.namespace is not None: self.explicitly_required_namespaces[node.name] = node.namespace self.gen = ProblemInstanceBuilder() compiler_parser = CompilerParser(configuration=spack.config.CONFIG).with_input_specs(specs) - # Only relevant for linux - for libc in compiler_parser.allowed_libcs: - self.gen.fact(fn.allowed_libc(libc.name, libc.version)) + if using_libc_compatibility(): + for libc in self.libcs: + self.gen.fact(fn.allowed_libc(libc.name, libc.version)) if not allow_deprecated: self.gen.fact(fn.deprecated_versions_not_allowed()) @@ -2505,15 +2519,16 @@ def define_runtime_constraints(self): if not compiler.available: continue - if using_libc_compatibility(): - libc = compiler.compiler_obj.default_libc() - if libc: - recorder("*").depends_on( - "libc", when=f"%{compiler.spec}", type="link", description="Add libc" - ) - recorder("*").depends_on( - str(libc), when=f"%{compiler.spec}", type="link", description="Add libc" - ) + if using_libc_compatibility() and compiler.compiler_obj.default_libc: + recorder("*").depends_on( + "libc", when=f"%{compiler.spec}", type="link", description="Add libc" + ) + recorder("*").depends_on( + str(compiler.compiler_obj.default_libc), + when=f"%{compiler.spec}", + type="link", + description="Add libc", + ) recorder.consume_facts() @@ -2890,18 +2905,13 @@ class CompilerParser: def __init__(self, configuration) -> None: self.compilers: Set[KnownCompiler] = set() - self.allowed_libcs = set() for c in all_compilers_in_config(configuration): - if using_libc_compatibility(): - libc = c.default_libc() - if not libc: - warnings.warn( - f"cannot detect libc from {c.spec}. The compiler will not be used " - f"during concretization." - ) - continue - - self.allowed_libcs.add(libc) + if using_libc_compatibility() and not c.default_libc: + warnings.warn( + f"cannot detect libc from {c.spec}. The compiler will not be used " + f"during concretization." + ) + continue target = c.target if c.target != "any" else None candidate = KnownCompiler( diff --git a/lib/spack/spack/solver/concretize.lp b/lib/spack/spack/solver/concretize.lp index ba1da1cfe6..85cc697a2d 100644 --- a/lib/spack/spack/solver/concretize.lp +++ b/lib/spack/spack/solver/concretize.lp @@ -1082,6 +1082,9 @@ error(100, "{0} compiler '{2}@{3}' incompatible with 'target={1}'", Package, Tar compiler_version(CompilerID, Version), build(node(X, Package)). +#defined compiler_supports_target/2. +#defined compiler_available/1. + % if a target is set explicitly, respect it attr("node_target", PackageNode, Target) :- attr("node", PackageNode), attr("node_target_set", PackageNode, Target). diff --git a/lib/spack/spack/test/concretize.py b/lib/spack/spack/test/concretize.py index 9faaa08f06..3bbd9e5bb8 100644 --- a/lib/spack/spack/test/concretize.py +++ b/lib/spack/spack/test/concretize.py @@ -83,7 +83,7 @@ def binary_compatibility(monkeypatch, request): return monkeypatch.setattr(spack.solver.asp, "using_libc_compatibility", lambda: True) - monkeypatch.setattr(spack.compiler.Compiler, "default_libc", lambda x: Spec("glibc@=2.28")) + monkeypatch.setattr(spack.compiler.Compiler, "default_libc", Spec("glibc@=2.28")) @pytest.fixture( diff --git a/lib/spack/spack/util/elf.py b/lib/spack/spack/util/elf.py index 6047c2f4da..64577bf8fb 100644 --- a/lib/spack/spack/util/elf.py +++ b/lib/spack/spack/util/elf.py @@ -641,6 +641,20 @@ def substitute_rpath_and_pt_interp_in_place_or_raise( return False +def pt_interp(path: str) -> Optional[str]: + """Retrieve the interpreter of an executable at `path`.""" + try: + with open(path, "rb") as f: + elf = parse_elf(f, interpreter=True) + except (OSError, ElfParsingError): + return None + + if not elf.has_pt_interp: + return None + + return elf.pt_interp_str.decode("utf-8") + + class ElfCStringUpdatesFailed(Exception): def __init__( self, rpath: Optional[UpdateCStringAction], pt_interp: Optional[UpdateCStringAction] diff --git a/lib/spack/spack/util/libc.py b/lib/spack/spack/util/libc.py new file mode 100644 index 0000000000..df0101bd46 --- /dev/null +++ b/lib/spack/spack/util/libc.py @@ -0,0 +1,117 @@ +# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +import os +import re +import sys +from subprocess import PIPE, run +from typing import Optional + +import spack.spec +import spack.util.elf + + +def _libc_from_ldd(ldd: str) -> Optional["spack.spec.Spec"]: + try: + result = run([ldd, "--version"], stdout=PIPE, stderr=PIPE, check=False) + stdout = result.stdout.decode("utf-8") + except Exception: + return None + + if not re.search("gnu|glibc", stdout, re.IGNORECASE): + return None + + version_str = re.match(r".+\(.+\) (.+)", stdout) + if not version_str: + return None + try: + return spack.spec.Spec(f"glibc@={version_str.group(1)}") + except Exception: + return None + + +def libc_from_dynamic_linker(dynamic_linker: str) -> Optional["spack.spec.Spec"]: + if not os.path.exists(dynamic_linker): + return None + + # The dynamic linker is usually installed in the same /lib(64)?/ld-*.so path across all + # distros. The rest of libc is elsewhere, e.g. /usr. Typically the dynamic linker is then + # a symlink into /usr/lib, which we use to for determining the actual install prefix of + # libc. + realpath = os.path.realpath(dynamic_linker) + + prefix = os.path.dirname(realpath) + # Remove the multiarch suffix if it exists + if os.path.basename(prefix) not in ("lib", "lib64"): + prefix = os.path.dirname(prefix) + + # Non-standard install layout -- just bail. + if os.path.basename(prefix) not in ("lib", "lib64"): + return None + + prefix = os.path.dirname(prefix) + + # Now try to figure out if glibc or musl, which is the only ones we support. + # In recent glibc we can simply execute the dynamic loader. In musl that's always the case. + try: + result = run([dynamic_linker, "--version"], stdout=PIPE, stderr=PIPE, check=False) + stdout = result.stdout.decode("utf-8") + stderr = result.stderr.decode("utf-8") + except Exception: + return None + + # musl prints to stderr + if stderr.startswith("musl libc"): + version_str = re.search(r"^Version (.+)$", stderr, re.MULTILINE) + if not version_str: + return None + try: + spec = spack.spec.Spec(f"musl@={version_str.group(1)}") + spec.external_path = prefix + return spec + except Exception: + return None + elif re.search("gnu|glibc", stdout, re.IGNORECASE): + # output is like "ld.so (...) stable release version 2.33." write a regex for it + match = re.search(r"version (\d+\.\d+(?:\.\d+)?)", stdout) + if not match: + return None + try: + version = match.group(1) + spec = spack.spec.Spec(f"glibc@={version}") + spec.external_path = prefix + return spec + except Exception: + return None + else: + # Could not get the version by running the dynamic linker directly. Instead locate `ldd` + # relative to the dynamic linker. + ldd = os.path.join(prefix, "bin", "ldd") + if not os.path.exists(ldd): + # If `/lib64/ld.so` was not a symlink to `/usr/lib/ld.so` we can try to use /usr as + # prefix. This is the case on ubuntu 18.04 where /lib != /usr/lib. + if prefix != "/": + return None + prefix = "/usr" + ldd = os.path.join(prefix, "bin", "ldd") + if not os.path.exists(ldd): + return None + maybe_spec = _libc_from_ldd(ldd) + if not maybe_spec: + return None + maybe_spec.external_path = prefix + return maybe_spec + + +def libc_from_current_python_process() -> Optional["spack.spec.Spec"]: + if not sys.executable: + return None + + dynamic_linker = spack.util.elf.pt_interp(sys.executable) + + if not dynamic_linker: + return None + + return libc_from_dynamic_linker(dynamic_linker)