From 8371bb4e192f876978fd77ea81c728bcd3475768 Mon Sep 17 00:00:00 2001 From: Harmen Stoppels Date: Thu, 16 Nov 2023 10:04:58 +0100 Subject: [PATCH] gcc-runtime: add separate package for gcc runtime libs The gcc-runtime package adds a separate node for gcc's dynamic runtime libraries. This should help with: 1. binary caches where rpaths for compiler support libs cannot be relocated because the compiler is missing on the target system 2. creating "minimal" container images The package is versioned like `gcc` (in principle it could be unversioned, but Spack doesn't always guarantee not mixing compilers) --- lib/spack/spack/main.py | 2 + lib/spack/spack/package_base.py | 18 +- lib/spack/spack/test/conftest.py | 5 + .../stacks/e4s-cray-rhel/spack.yaml | 2 + .../stacks/e4s-oneapi/spack.yaml | 2 + .../builtin/packages/gcc-runtime/package.py | 222 ++++++++++++++++++ 6 files changed, 250 insertions(+), 1 deletion(-) create mode 100644 var/spack/repos/builtin/packages/gcc-runtime/package.py diff --git a/lib/spack/spack/main.py b/lib/spack/spack/main.py index 56a4dc0e33..bcdc7d7599 100644 --- a/lib/spack/spack/main.py +++ b/lib/spack/spack/main.py @@ -36,6 +36,7 @@ import spack.config import spack.environment as ev import spack.modules +import spack.package_base import spack.paths import spack.platforms import spack.repo @@ -607,6 +608,7 @@ def setup_main_options(args): [(key, [spack.paths.mock_packages_path])] ) spack.repo.PATH = spack.repo.create(spack.config.CONFIG) + spack.package_base.WITH_GCC_RUNTIME = False # If the user asked for it, don't check ssl certs. if args.insecure: diff --git a/lib/spack/spack/package_base.py b/lib/spack/spack/package_base.py index 7d8f7104df..bf8ed56d95 100644 --- a/lib/spack/spack/package_base.py +++ b/lib/spack/spack/package_base.py @@ -53,6 +53,7 @@ import spack.util.environment import spack.util.path import spack.util.web +from spack.directives import _depends_on from spack.filesystem_view import YamlFilesystemView from spack.install_test import ( PackageTest, @@ -76,6 +77,7 @@ """Allowed URL schemes for spack packages.""" _ALLOWED_URL_SCHEMES = ["http", "https", "ftp", "file", "git"] +WITH_GCC_RUNTIME = True #: Filename for the Spack build/install log. _spack_build_logfile = "spack-build-out.txt" @@ -371,6 +373,20 @@ def _wrapper(instance, *args, **kwargs): return _execute_under_condition +class BinaryPackage: + """This adds a universal dependency on gcc-runtime.""" + + def maybe_depend_on_gcc_runtime(self): + # Do not depend on itself, and allow tests to disable this universal dep + if self.name == "gcc-runtime" or not WITH_GCC_RUNTIME: + return + for v in ["13", "12", "11", "10", "9", "8", "7", "6", "5", "4"]: + _depends_on(self, f"gcc-runtime@{v}:", type="link", when=f"%gcc@{v} platform=linux") + _depends_on(self, f"gcc-runtime@{v}:", type="link", when=f"%gcc@{v} platform=cray") + + _directives_to_be_executed = [maybe_depend_on_gcc_runtime] + + class PackageViewMixin: """This collects all functionality related to adding installed Spack package to views. Packages can customize how they are added to views by @@ -433,7 +449,7 @@ def remove_files_from_view(self, view, merge_map): Pb = TypeVar("Pb", bound="PackageBase") -class PackageBase(WindowsRPath, PackageViewMixin, metaclass=PackageMeta): +class PackageBase(WindowsRPath, PackageViewMixin, BinaryPackage, metaclass=PackageMeta): """This is the superclass for all spack packages. ***The Package class*** diff --git a/lib/spack/spack/test/conftest.py b/lib/spack/spack/test/conftest.py index 785d986018..df9a43a123 100644 --- a/lib/spack/spack/test/conftest.py +++ b/lib/spack/spack/test/conftest.py @@ -57,6 +57,11 @@ from spack.util.pattern import Bunch +@pytest.fixture(scope="session", autouse=True) +def drop_gcc_runtime(): + spack.package_base.WITH_GCC_RUNTIME = False + + def ensure_configuration_fixture_run_before(request): """Ensure that fixture mutating the configuration run before the one where the function is called. diff --git a/share/spack/gitlab/cloud_pipelines/stacks/e4s-cray-rhel/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/e4s-cray-rhel/spack.yaml index c96ddafc08..5cd3178d66 100644 --- a/share/spack/gitlab/cloud_pipelines/stacks/e4s-cray-rhel/spack.yaml +++ b/share/spack/gitlab/cloud_pipelines/stacks/e4s-cray-rhel/spack.yaml @@ -33,6 +33,8 @@ spack: elfutils: variants: +bzip2 ~nls +xz require: "%gcc" + gcc-runtime: + require: "%gcc" hdf5: variants: +fortran +hl +shared libfabric: diff --git a/share/spack/gitlab/cloud_pipelines/stacks/e4s-oneapi/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/e4s-oneapi/spack.yaml index 85e23abac4..75e11a695e 100644 --- a/share/spack/gitlab/cloud_pipelines/stacks/e4s-oneapi/spack.yaml +++ b/share/spack/gitlab/cloud_pipelines/stacks/e4s-oneapi/spack.yaml @@ -17,6 +17,8 @@ spack: variants: +mpi elfutils: variants: +bzip2 ~nls +xz + gcc-runtime: + require: "%gcc" hdf5: require: "%gcc" variants: +fortran +hl +shared diff --git a/var/spack/repos/builtin/packages/gcc-runtime/package.py b/var/spack/repos/builtin/packages/gcc-runtime/package.py new file mode 100644 index 0000000000..f0cd13dd14 --- /dev/null +++ b/var/spack/repos/builtin/packages/gcc-runtime/package.py @@ -0,0 +1,222 @@ +# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +import os +import re + +from macholib import MachO, mach_o + +from llnl.util import tty + +from spack.package import * +from spack.util.elf import parse_elf + + +class GccRuntime(Package): + """Package for GCC compiler runtime libraries""" + + homepage = "https://gcc.gnu.org" + has_code = False + + maintainers("haampie") + + license("GPL-3.0-or-later WITH GCC-exception-3.1") + + requires("%gcc") + + LIBRARIES = [ + "asan", + "atomic", + "gcc_s", + "gfortran", + "gomp", + "hwasan", + "itm", + "lsan", + "quadmath", + "ssp", + "stdc++", + "tsan", + "ubsan", + ] + + for v in [ + "13.2", + "13.1", + "12.3", + "12.2", + "12.1", + "11.4", + "11.3", + "11.2", + "11.1", + "10.5", + "10.4", + "10.3", + "10.2", + "10.1", + "9.5", + "9.4", + "9.3", + "9.2", + "9.1", + "8.5", + "8.4", + "8.3", + "8.2", + "8.1", + "7.5", + "7.4", + "7.3", + "7.2", + "7.1", + "6.5", + "6.4", + "6.3", + "6.2", + "6.1", + "5.5", + "5.4", + "5.3", + "5.2", + "5.1", + "4.9.4", + "4.9.3", + "4.9.2", + "4.9.1", + "4.9.0", + "4.8.5", + "4.8.4", + "4.8.3", + "4.8.2", + "4.8.1", + "4.8.0", + "4.7.4", + "4.7.3", + "4.7.2", + "4.7.1", + "4.7.0", + "4.6.4", + "4.6.3", + "4.6.2", + "4.6.1", + "4.6.0", + "4.5.4", + "4.5.3", + "4.5.2", + "4.5.1", + "4.5.0", + ]: + version(v) + requires(f"%gcc@{v}", when=f"@{v}") + + def install(self, spec, prefix): + if spec.platform in ["linux", "cray", "freebsd"]: + libraries = self._get_libraries_elf() + elif spec.platform == "darwin": + libraries = self._get_libraries_macho() + else: + raise RuntimeError("Unsupported platform") + + mkdir(prefix.lib) + + if not libraries: + tty.warn("Could not detect any shared GCC runtime libraries") + return + + for path, name in libraries: + install(path, os.path.join(prefix.lib, name)) + + def _get_libraries_elf(self): + """Get the GCC runtime libraries for ELF binaries""" + cc = Executable(self.compiler.cc) + lib_regex = re.compile(rb"\blib[a-z-_]+\.so\.\d+\b") + path_and_install_name = [] + + for name in self.LIBRARIES: + # Look for the dynamic library that gcc would use to link, + # that is with .so extension and without abi suffix. + path = cc(f"-print-file-name=lib{name}.so", output=str).strip() + + # gcc reports an absolute path on success + if not os.path.isabs(path): + continue + + # Now there are two options: + # 1. the file is an ELF file + # 2. the file is a linker script referencing the actual library + with open(path, "rb") as f: + try: + # Try to parse as an ELF file + soname = parse_elf(f, dynamic_section=True).dt_soname_str.decode("utf-8") + except Exception: + # On failure try to "parse" as ld script; the actual + # library needs to be mentioned by filename. + f.seek(0) + script_matches = lib_regex.findall(f.read()) + if len(script_matches) != 1: + continue + soname = script_matches[0].decode("utf-8") + + # Now locate and install the runtime library + runtime_path = cc(f"-print-file-name={soname}", output=str).strip() + + if not os.path.isabs(runtime_path): + continue + + path_and_install_name.append((runtime_path, soname)) + + return path_and_install_name + + def _get_libraries_macho(self): + """Same as _get_libraries_elf but for Mach-O binaries""" + cc = Executable(self.compiler.cc) + path_and_install_name = [] + + for name in self.LIBRARIES: + if name == "gcc_s": + # On darwin, libgcc_s is versioned and can't be linked as -lgcc_s, + # but needs a suffix we don't know, so we parse it from the link line. + match = re.search( + r"\s-l(gcc_s\.[0-9.]+)\s", cc("-xc", "-", "-shared-libgcc", "-###", error=str) + ) + if match is None: + continue + name = match.group(1) + + path = cc(f"-print-file-name=lib{name}.dylib", output=str).strip() + + if not os.path.isabs(path): + continue + + macho = MachO.MachO(path) + + # Get the LC_ID_DYLIB load command + for load_command, _, data in macho.headers[-1].commands: + if load_command.cmd == mach_o.LC_ID_DYLIB: + # Strip off @rpath/ prefix, or even an absolute path. + dylib_name = os.path.basename(data.rstrip(b"\x00").decode()) + break + else: + continue + + # Locate by dylib name + runtime_path = cc(f"-print-file-name={dylib_name}", output=str).strip() + + if not os.path.isabs(runtime_path): + continue + + path_and_install_name.append((runtime_path, dylib_name)) + + return path_and_install_name + + @property + def libs(self): + # Currently these libs are not linkable with -l, they all have a suffix. + return LibraryList([]) + + @property + def headers(self): + return HeaderList([])