diff --git a/etc/spack/defaults/packages.yaml b/etc/spack/defaults/packages.yaml index 654875a575..0484a1a559 100644 --- a/etc/spack/defaults/packages.yaml +++ b/etc/spack/defaults/packages.yaml @@ -35,6 +35,7 @@ packages: java: [openjdk, jdk, ibm-java] jpeg: [libjpeg-turbo, libjpeg] lapack: [openblas, amdlibflame] + libc: [glibc, musl] libgfortran: [ gcc-runtime ] libglx: [mesa+glx, mesa18+glx] libifcore: [ intel-oneapi-runtime ] diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index 15c11995a7..7b7e0f8298 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -8,9 +8,11 @@ import os import platform import re +import shlex import shutil import sys import tempfile +from subprocess import PIPE, run from typing import List, Optional, Sequence import llnl.path @@ -184,6 +186,113 @@ def _parse_non_system_link_dirs(string: str) -> List[str]: return list(p for p in link_dirs if not in_system_subdirectory(p)) +def _parse_dynamic_linker(output: str): + """Parse -dynamic-linker /path/to/ld.so from compiler output""" + for line in reversed(output.splitlines()): + if "-dynamic-linker" not in line: + continue + args = shlex.split(line) + + for idx in reversed(range(1, len(args))): + arg = args[idx] + if arg == "-dynamic-linker" or args == "--dynamic-linker": + return args[idx + 1] + elif arg.startswith("--dynamic-linker=") or arg.startswith("-dynamic-linker="): + return arg.split("=", 1)[1] + + +def _libc_from_ldd(ldd: str) -> Optional["spack.spec.Spec"]: + try: + result = run([ldd, "--version"], stdout=PIPE, stderr=PIPE, check=False) + stdout = result.stdout.decode("utf-8") + except Exception: + return None + + if not re.search("gnu|glibc", stdout, re.IGNORECASE): + return None + + version_str = re.match(r".+\(.+\) (.+)", stdout) + if not version_str: + return None + try: + return spack.spec.Spec(f"glibc@={version_str.group(1)}") + except Exception: + return None + + +def _libc_from_dynamic_linker(dynamic_linker: str) -> Optional["spack.spec.Spec"]: + if not os.path.exists(dynamic_linker): + return None + + # The dynamic linker is usually installed in the same /lib(64)?/ld-*.so path across all + # distros. The rest of libc is elsewhere, e.g. /usr. Typically the dynamic linker is then + # a symlink into /usr/lib, which we use to for determining the actual install prefix of + # libc. + realpath = os.path.realpath(dynamic_linker) + + prefix = os.path.dirname(realpath) + # Remove the multiarch suffix if it exists + if os.path.basename(prefix) not in ("lib", "lib64"): + prefix = os.path.dirname(prefix) + + # Non-standard install layout -- just bail. + if os.path.basename(prefix) not in ("lib", "lib64"): + return None + + prefix = os.path.dirname(prefix) + + # Now try to figure out if glibc or musl, which is the only ones we support. + # In recent glibc we can simply execute the dynamic loader. In musl that's always the case. + try: + result = run([dynamic_linker, "--version"], stdout=PIPE, stderr=PIPE, check=False) + stdout = result.stdout.decode("utf-8") + stderr = result.stderr.decode("utf-8") + except Exception: + return None + + # musl prints to stderr + if stderr.startswith("musl libc"): + version_str = re.search(r"^Version (.+)$", stderr, re.MULTILINE) + if not version_str: + return None + try: + spec = spack.spec.Spec(f"musl@={version_str.group(1)}") + spec.external_path = prefix + return spec + except Exception: + return None + elif re.search("gnu|glibc", stdout, re.IGNORECASE): + # output is like "ld.so (...) stable release version 2.33." write a regex for it + match = re.search(r"version (\d+\.\d+(?:\.\d+)?)", stdout) + if not match: + return None + try: + version = match.group(1) + spec = spack.spec.Spec(f"glibc@={version}") + spec.external_path = prefix + return spec + except Exception: + return None + else: + # Could not get the version by running the dynamic linker directly. Instead locate `ldd` + # relative to the dynamic linker. + ldd = os.path.join(prefix, "bin", "ldd") + if not os.path.exists(ldd): + # If `/lib64/ld.so` was not a symlink to `/usr/lib/ld.so` we can try to use /usr as + # prefix. This is the case on ubuntu 18.04 where /lib != /usr/lib. + if prefix != "/": + return None + prefix = "/usr" + ldd = os.path.join(prefix, "bin", "ldd") + if not os.path.exists(ldd): + return None + maybe_spec = _libc_from_ldd(ldd) + if not maybe_spec: + return None + maybe_spec.external_path = prefix + return maybe_spec + + def in_system_subdirectory(path): system_dirs = [ "/lib/", @@ -417,17 +526,33 @@ def real_version(self): self._real_version = self.version return self._real_version - def implicit_rpaths(self): + def implicit_rpaths(self) -> List[str]: if self.enable_implicit_rpaths is False: return [] - # Put CXX first since it has the most linking issues - # And because it has flags that affect linking - link_dirs = self._get_compiler_link_paths() + output = self.compiler_verbose_output + + if not output: + return [] + + link_dirs = _parse_non_system_link_dirs(output) all_required_libs = list(self.required_libs) + Compiler._all_compiler_rpath_libraries return list(paths_containing_libs(link_dirs, all_required_libs)) + def default_libc(self) -> Optional["spack.spec.Spec"]: + output = self.compiler_verbose_output + + if not output: + return None + + dynamic_linker = _parse_dynamic_linker(output) + + if not dynamic_linker: + return None + + return _libc_from_dynamic_linker(dynamic_linker) + @property def required_libs(self): """For executables created with this compiler, the compiler libraries @@ -436,17 +561,17 @@ def required_libs(self): # By default every compiler returns the empty list return [] - def _get_compiler_link_paths(self): + @property + def compiler_verbose_output(self) -> Optional[str]: + """Verbose output from compiling a dummy C source file. Output is cached.""" + if not hasattr(self, "_compile_c_source_output"): + self._compile_c_source_output = self._compile_dummy_c_source() + return self._compile_c_source_output + + def _compile_dummy_c_source(self) -> Optional[str]: cc = self.cc if self.cc else self.cxx if not cc or not self.verbose_flag: - # Cannot determine implicit link paths without a compiler / verbose flag - return [] - - # What flag types apply to first_compiler, in what order - if cc == self.cc: - flags = ["cflags", "cppflags", "ldflags"] - else: - flags = ["cxxflags", "cppflags", "ldflags"] + return None try: tmpdir = tempfile.mkdtemp(prefix="spack-implicit-link-info") @@ -458,20 +583,19 @@ def _get_compiler_link_paths(self): "int main(int argc, char* argv[]) { (void)argc; (void)argv; return 0; }\n" ) cc_exe = spack.util.executable.Executable(cc) - for flag_type in flags: + for flag_type in ["cflags" if cc == self.cc else "cxxflags", "cppflags", "ldflags"]: cc_exe.add_default_arg(*self.flags.get(flag_type, [])) with self.compiler_environment(): - output = cc_exe(self.verbose_flag, fin, "-o", fout, output=str, error=str) - return _parse_non_system_link_dirs(output) + return cc_exe(self.verbose_flag, fin, "-o", fout, output=str, error=str) except spack.util.executable.ProcessError as pe: tty.debug("ProcessError: Command exited with non-zero status: " + pe.long_message) - return [] + return None finally: shutil.rmtree(tmpdir, ignore_errors=True) @property - def verbose_flag(self): + def verbose_flag(self) -> Optional[str]: """ This property should be overridden in the compiler subclass if a verbose flag is available. diff --git a/lib/spack/spack/compilers/nag.py b/lib/spack/spack/compilers/nag.py index c12ccec7bf..6040b74a14 100644 --- a/lib/spack/spack/compilers/nag.py +++ b/lib/spack/spack/compilers/nag.py @@ -64,7 +64,7 @@ def verbose_flag(self): # # This way, we at least enable the implicit rpath detection, which is # based on compilation of a C file (see method - # spack.compiler._get_compiler_link_paths): in the case of a mixed + # spack.compiler._compile_dummy_c_source): in the case of a mixed # NAG/GCC toolchain, the flag will be passed to g++ (e.g. # 'g++ -Wl,-v ./main.c'), otherwise, the flag will be passed to nagfor # (e.g. 'nagfor -Wl,-v ./main.c' - note that nagfor recognizes '.c' diff --git a/lib/spack/spack/solver/asp.py b/lib/spack/spack/solver/asp.py index 22d6994e4c..a77e92a51d 100644 --- a/lib/spack/spack/solver/asp.py +++ b/lib/spack/spack/solver/asp.py @@ -566,6 +566,23 @@ def _spec_with_default_name(spec_str, name): return spec +def _external_config_with_implictit_externals(): + # Read packages.yaml and normalize it, so that it will not contain entries referring to + # virtual packages. + packages_yaml = _normalize_packages_yaml(spack.config.get("packages")) + + # Add externals for libc from compilers on Linux + if spack.platforms.host().name != "linux": + return packages_yaml + + for compiler in all_compilers_in_config(): + libc = compiler.default_libc() + if libc: + entry = {"spec": f"{libc} %{compiler.spec}", "prefix": libc.external_path} + packages_yaml.setdefault(libc.name, {}).setdefault("externals", []).append(entry) + return packages_yaml + + class ErrorHandler: def __init__(self, model): self.model = model @@ -1554,12 +1571,8 @@ def emit_facts_from_requirement_rules(self, rules: List[RequirementRule]): requirement_weight += 1 def external_packages(self): - """Facts on external packages, as read from packages.yaml""" - # Read packages.yaml and normalize it, so that it - # will not contain entries referring to virtual - # packages. - packages_yaml = spack.config.get("packages") - packages_yaml = _normalize_packages_yaml(packages_yaml) + """Facts on external packages, from packages.yaml and implicit externals.""" + packages_yaml = _external_config_with_implictit_externals() self.gen.h1("External packages") for pkg_name, data in packages_yaml.items(): @@ -3185,12 +3198,8 @@ def no_flags(self, node, flag_type): self._specs[node].compiler_flags[flag_type] = [] def external_spec_selected(self, node, idx): - """This means that the external spec and index idx - has been selected for this package. - """ - - packages_yaml = spack.config.get("packages") - packages_yaml = _normalize_packages_yaml(packages_yaml) + """This means that the external spec and index idx has been selected for this package.""" + packages_yaml = _external_config_with_implictit_externals() spec_info = packages_yaml[node.pkg]["externals"][int(idx)] self._specs[node].external_path = spec_info.get("prefix", None) self._specs[node].external_modules = spack.spec.Spec._format_module_list( diff --git a/lib/spack/spack/solver/os_compatibility.lp b/lib/spack/spack/solver/os_compatibility.lp index 2ed8a15388..df312acf6d 100644 --- a/lib/spack/spack/solver/os_compatibility.lp +++ b/lib/spack/spack/solver/os_compatibility.lp @@ -12,16 +12,3 @@ % macOS os_compatible("monterey", "bigsur"). os_compatible("bigsur", "catalina"). - -% Ubuntu -os_compatible("ubuntu22.04", "ubuntu21.10"). -os_compatible("ubuntu21.10", "ubuntu21.04"). -os_compatible("ubuntu21.04", "ubuntu20.10"). -os_compatible("ubuntu20.10", "ubuntu20.04"). -os_compatible("ubuntu20.04", "ubuntu19.10"). -os_compatible("ubuntu19.10", "ubuntu19.04"). -os_compatible("ubuntu19.04", "ubuntu18.10"). -os_compatible("ubuntu18.10", "ubuntu18.04"). - -%EL8 -os_compatible("rhel8", "rocky8"). diff --git a/lib/spack/spack/test/compilers/basics.py b/lib/spack/spack/test/compilers/basics.py index 2d99b53466..a5a5b8f662 100644 --- a/lib/spack/spack/test/compilers/basics.py +++ b/lib/spack/spack/test/compilers/basics.py @@ -14,6 +14,7 @@ import spack.compilers import spack.spec import spack.util.environment +import spack.util.module_cmd from spack.compiler import Compiler from spack.util.executable import Executable, ProcessError @@ -137,14 +138,6 @@ def __init__(self): environment={}, ) - def _get_compiler_link_paths(self): - # Mock os.path.isdir so the link paths don't have to exist - old_isdir = os.path.isdir - os.path.isdir = lambda x: True - ret = super()._get_compiler_link_paths() - os.path.isdir = old_isdir - return ret - @property def name(self): return "mockcompiler" @@ -162,34 +155,25 @@ def verbose_flag(self): required_libs = ["libgfortran"] -def test_implicit_rpaths(dirs_with_libfiles, monkeypatch): +@pytest.mark.not_on_windows("Not supported on Windows (yet)") +def test_implicit_rpaths(dirs_with_libfiles): lib_to_dirs, all_dirs = dirs_with_libfiles - - def try_all_dirs(*args): - return all_dirs - - monkeypatch.setattr(MockCompiler, "_get_compiler_link_paths", try_all_dirs) - - expected_rpaths = set(lib_to_dirs["libstdc++"] + lib_to_dirs["libgfortran"]) - compiler = MockCompiler() + compiler._compile_c_source_output = "ld " + " ".join(f"-L{d}" for d in all_dirs) retrieved_rpaths = compiler.implicit_rpaths() - assert set(retrieved_rpaths) == expected_rpaths + assert set(retrieved_rpaths) == set(lib_to_dirs["libstdc++"] + lib_to_dirs["libgfortran"]) -no_flag_dirs = ["/path/to/first/lib", "/path/to/second/lib64"] -no_flag_output = "ld -L%s -L%s" % tuple(no_flag_dirs) - -flag_dirs = ["/path/to/first/with/flag/lib", "/path/to/second/lib64"] -flag_output = "ld -L%s -L%s" % tuple(flag_dirs) +without_flag_output = "ld -L/path/to/first/lib -L/path/to/second/lib64" +with_flag_output = "ld -L/path/to/first/with/flag/lib -L/path/to/second/lib64" def call_compiler(exe, *args, **kwargs): # This method can replace Executable.__call__ to emulate a compiler that # changes libraries depending on a flag. if "--correct-flag" in exe.exe: - return flag_output - return no_flag_output + return with_flag_output + return without_flag_output @pytest.mark.not_on_windows("Not supported on Windows (yet)") @@ -203,8 +187,8 @@ def call_compiler(exe, *args, **kwargs): ("cc", "cppflags"), ], ) -@pytest.mark.enable_compiler_link_paths -def test_get_compiler_link_paths(monkeypatch, exe, flagname): +@pytest.mark.enable_compiler_execution +def test_compile_dummy_c_source_adds_flags(monkeypatch, exe, flagname): # create fake compiler that emits mock verbose output compiler = MockCompiler() monkeypatch.setattr(Executable, "__call__", call_compiler) @@ -221,40 +205,38 @@ def test_get_compiler_link_paths(monkeypatch, exe, flagname): assert False # Test without flags - assert compiler._get_compiler_link_paths() == no_flag_dirs + assert compiler._compile_dummy_c_source() == without_flag_output if flagname: # set flags and test compiler.flags = {flagname: ["--correct-flag"]} - assert compiler._get_compiler_link_paths() == flag_dirs + assert compiler._compile_dummy_c_source() == with_flag_output -def test_get_compiler_link_paths_no_path(): +@pytest.mark.enable_compiler_execution +def test_compile_dummy_c_source_no_path(): compiler = MockCompiler() compiler.cc = None compiler.cxx = None - compiler.f77 = None - compiler.fc = None - assert compiler._get_compiler_link_paths() == [] + assert compiler._compile_dummy_c_source() is None -def test_get_compiler_link_paths_no_verbose_flag(): +@pytest.mark.enable_compiler_execution +def test_compile_dummy_c_source_no_verbose_flag(): compiler = MockCompiler() compiler._verbose_flag = None - assert compiler._get_compiler_link_paths() == [] + assert compiler._compile_dummy_c_source() is None @pytest.mark.not_on_windows("Not supported on Windows (yet)") -@pytest.mark.enable_compiler_link_paths -def test_get_compiler_link_paths_load_env(working_env, monkeypatch, tmpdir): +@pytest.mark.enable_compiler_execution +def test_compile_dummy_c_source_load_env(working_env, monkeypatch, tmpdir): gcc = str(tmpdir.join("gcc")) with open(gcc, "w") as f: f.write( - """#!/bin/sh + f"""#!/bin/sh if [ "$ENV_SET" = "1" ] && [ "$MODULE_LOADED" = "1" ]; then - echo '""" - + no_flag_output - + """' + printf '{without_flag_output}' fi """ ) @@ -274,7 +256,7 @@ def module(*args): compiler.environment = {"set": {"ENV_SET": "1"}} compiler.modules = ["turn_on"] - assert compiler._get_compiler_link_paths() == no_flag_dirs + assert compiler._compile_dummy_c_source() == without_flag_output # Get the desired flag from the specified compiler spec. diff --git a/lib/spack/spack/test/conftest.py b/lib/spack/spack/test/conftest.py index f25cf42e6c..c8c300ca95 100644 --- a/lib/spack/spack/test/conftest.py +++ b/lib/spack/spack/test/conftest.py @@ -34,6 +34,7 @@ import spack.binary_distribution import spack.caches import spack.cmd.buildcache +import spack.compiler import spack.compilers import spack.config import spack.database @@ -269,10 +270,6 @@ def clean_test_environment(): ev.deactivate() -def _verify_executables_noop(*args): - return None - - def _host(): """Mock archspec host so there is no inconsistency on the Windows platform This function cannot be local as it needs to be pickleable""" @@ -298,9 +295,7 @@ def mock_compiler_executable_verification(request, monkeypatch): If a test is marked in that way this is a no-op.""" if "enable_compiler_verification" not in request.keywords: - monkeypatch.setattr( - spack.compiler.Compiler, "verify_executables", _verify_executables_noop - ) + monkeypatch.setattr(spack.compiler.Compiler, "verify_executables", _return_none) # Hooks to add command line options or set other custom behaviors. @@ -934,26 +929,16 @@ def dirs_with_libfiles(tmpdir_factory): yield lib_to_dirs, all_dirs -def _compiler_link_paths_noop(*args): - return [] +def _return_none(*args): + return None @pytest.fixture(scope="function", autouse=True) def disable_compiler_execution(monkeypatch, request): - """ - This fixture can be disabled for tests of the compiler link path - functionality by:: - - @pytest.mark.enable_compiler_link_paths - - If a test is marked in that way this is a no-op.""" - if "enable_compiler_link_paths" not in request.keywords: - # Compiler.determine_implicit_rpaths actually runs the compiler. So - # replace that function with a noop that simulates finding no implicit - # RPATHs - monkeypatch.setattr( - spack.compiler.Compiler, "_get_compiler_link_paths", _compiler_link_paths_noop - ) + """Disable compiler execution to determine implicit link paths and libc flavor and version. + To re-enable use `@pytest.mark.enable_compiler_execution`""" + if "enable_compiler_execution" not in request.keywords: + monkeypatch.setattr(spack.compiler.Compiler, "_compile_dummy_c_source", _return_none) @pytest.fixture(scope="function") diff --git a/pytest.ini b/pytest.ini index 617881d77b..2a3152da3d 100644 --- a/pytest.ini +++ b/pytest.ini @@ -12,7 +12,7 @@ markers = requires_executables: tests that requires certain executables in PATH to run nomockstage: use a stage area specifically created for this test, instead of relying on a common mock stage enable_compiler_verification: enable compiler verification within unit tests - enable_compiler_link_paths: verifies compiler link paths within unit tests + enable_compiler_execution: enable compiler execution to detect link paths and libc disable_clean_stage_check: avoid failing tests if there are leftover files in the stage area only_clingo: mark unit tests that run only with clingo only_original: mark unit tests that are specific to the original concretizer diff --git a/var/spack/repos/builtin/packages/gcc/package.py b/var/spack/repos/builtin/packages/gcc/package.py index f9ff5b9fbe..91f5358b0c 100644 --- a/var/spack/repos/builtin/packages/gcc/package.py +++ b/var/spack/repos/builtin/packages/gcc/package.py @@ -1185,5 +1185,13 @@ def runtime_constraints(cls, *, spec, pkg): description=f"Add a dependency on '{gfortran_str}' for nodes compiled with " f"{str(spec)} and using the 'fortran' language", ) + + libc = compiler.default_libc() + + if libc: + pkg("*").depends_on( + str(libc), when=f"%{str(compiler.spec)}", type="link", description="Add libc" + ) + # The version of gcc-runtime is the same as the %gcc used to "compile" it pkg("gcc-runtime").requires(f"@={str(spec.version)}", when=f"%{str(spec)}") diff --git a/var/spack/repos/builtin/packages/glibc/package.py b/var/spack/repos/builtin/packages/glibc/package.py index c5a1709efa..d878244f87 100644 --- a/var/spack/repos/builtin/packages/glibc/package.py +++ b/var/spack/repos/builtin/packages/glibc/package.py @@ -20,9 +20,12 @@ class Glibc(AutotoolsPackage, GNUMirrorPackage): maintainers("haampie") build_directory = "build" + tags = ["runtime"] license("LGPL-2.1-or-later") + provides("libc") + version("master", branch="master") version("2.39", sha256="97f84f3b7588cd54093a6f6389b0c1a81e70d99708d74963a2e3eab7c7dc942d") version("2.38", sha256="16e51e0455e288f03380b436e41d5927c60945abd86d0c9852b84be57dd6ed5e") diff --git a/var/spack/repos/builtin/packages/musl/package.py b/var/spack/repos/builtin/packages/musl/package.py index 13ce939b41..4b503feb64 100644 --- a/var/spack/repos/builtin/packages/musl/package.py +++ b/var/spack/repos/builtin/packages/musl/package.py @@ -25,9 +25,12 @@ class Musl(MakefilePackage): homepage = "https://www.musl-libc.org" url = "https://www.musl-libc.org/releases/musl-1.1.23.tar.gz" + tags = ["runtime"] license("MIT") + provides("libc") + version("1.2.4", sha256="7a35eae33d5372a7c0da1188de798726f68825513b7ae3ebe97aaaa52114f039") version("1.2.3", sha256="7d5b0b6062521e4627e099e4c9dc8248d32a30285e959b7eecaa780cf8cfd4a4") version("1.2.2", sha256="9b969322012d796dc23dda27a35866034fa67d8fb67e0e2c45c913c3d43219dd")