hlrs-software-stack/spack
7
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

gcc: generate spec file and fix external libc default paths after install from cache (#43839)

Browse source 
Co-authored-by: Massimiliano Culpo <massimiliano.culpo@gmail.com>
This commit is contained in:
Harmen Stoppels 2024-04-27 16:49:20 +02:00 committed by GitHub
parent ae9f2d4d40
commit 43f3a35150
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
9 changed files with 172 additions and 32 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

18
lib/spack/spack/compiler.py
View file

@ -8,7 +8,6 @@
import os
import platform
import re
import shlex
import shutil
import sys
import tempfile
@ -182,21 +181,6 @@ def _parse_non_system_link_dirs(string: str) -> List[str]:
return list(p for p in link_dirs if not in_system_subdirectory(p))
def _parse_dynamic_linker(output: str):
"""Parse -dynamic-linker /path/to/ld.so from compiler output"""
for line in reversed(output.splitlines()):
if "-dynamic-linker" not in line:
continue
args = shlex.split(line)
for idx in reversed(range(1, len(args))):
arg = args[idx]
if arg == "-dynamic-linker" or args == "--dynamic-linker":
return args[idx + 1]
elif arg.startswith("--dynamic-linker=") or arg.startswith("-dynamic-linker="):
return arg.split("=", 1)[1]
def in_system_subdirectory(path):
system_dirs = [
"/lib/",
@ -452,7 +436,7 @@ def default_libc(self) -> Optional["spack.spec.Spec"]:
if not output:
return None
dynamic_linker = _parse_dynamic_linker(output)
dynamic_linker = spack.util.libc.parse_dynamic_linker(output)
if not dynamic_linker:
return None

19
lib/spack/spack/detection/path.py
View file

@ -83,26 +83,15 @@ def executables_in_path(path_hints: List[str]) -> Dict[str, str]:
return path_to_dict(search_paths)
def get_elf_compat(path):
"""For ELF files, get a triplet (EI_CLASS, EI_DATA, e_machine) and see if
it is host-compatible."""
# On ELF platforms supporting, we try to be a bit smarter when it comes to shared
# libraries, by dropping those that are not host compatible.
with open(path, "rb") as f:
elf = elf_utils.parse_elf(f, only_header=True)
return (elf.is_64_bit, elf.is_little_endian, elf.elf_hdr.e_machine)
def accept_elf(path, host_compat):
"""Accept an ELF file if the header matches the given compat triplet,
obtained with :py:func:`get_elf_compat`. In case it's not an ELF (e.g.
static library, or some arbitrary file, fall back to is_readable_file)."""
"""Accept an ELF file if the header matches the given compat triplet. In case it's not an ELF
(e.g. static library, or some arbitrary file, fall back to is_readable_file)."""
# Fast path: assume libraries at least have .so in their basename.
# Note: don't replace with splitext, because of libsmth.so.1.2.3 file names.
if ".so" not in os.path.basename(path):
return llnl.util.filesystem.is_readable_file(path)
try:
return host_compat == get_elf_compat(path)
return host_compat == elf_utils.get_elf_compat(path)
except (OSError, elf_utils.ElfParsingError):
return llnl.util.filesystem.is_readable_file(path)
@ -155,7 +144,7 @@ def libraries_in_ld_and_system_library_path(
search_paths = list(llnl.util.lang.dedupe(search_paths, key=file_identifier))
try:
host_compat = get_elf_compat(sys.executable)
host_compat = elf_utils.get_elf_compat(sys.executable)
accept = lambda path: accept_elf(path, host_compat)
except (OSError, elf_utils.ElfParsingError):
accept = llnl.util.filesystem.is_readable_file

3
lib/spack/spack/installer.py
View file

@ -489,6 +489,9 @@ def _process_binary_cache_tarball(
with timer.measure("install"), spack.util.path.filter_padding():
binary_distribution.extract_tarball(pkg.spec, download_result, force=False, timer=timer)
if hasattr(pkg, "_post_buildcache_install_hook"):
pkg._post_buildcache_install_hook()
pkg.installed_from_binary_cache = True
spack.store.STORE.db.add(pkg.spec, spack.store.STORE.layout, explicit=explicit)
return True

26
lib/spack/spack/test/util/libc.py Normal file
View file

@ -0,0 +1,26 @@
# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
import pytest
from spack.util import libc
@pytest.mark.parametrize(
"libc_prefix,startfile_prefix,expected",
[
# Ubuntu
("/usr", "/usr/lib/x86_64-linux-gnu", "/usr/include/x86_64-linux-gnu"),
("/usr", "/usr/lib/x86_64-linux-musl", "/usr/include/x86_64-linux-musl"),
("/usr", "/usr/lib/aarch64-linux-gnu", "/usr/include/aarch64-linux-gnu"),
("/usr", "/usr/lib/aarch64-linux-musl", "/usr/include/aarch64-linux-musl"),
# rhel-like
("/usr", "/usr/lib64", "/usr/include"),
("/usr", "/usr/lib", "/usr/include"),
],
)
@pytest.mark.not_on_windows("The unit test deals with unix-like paths")
def test_header_dir_computation(libc_prefix, startfile_prefix, expected):
"""Tests that we compute the correct header directory from the prefix of the libc startfiles"""
assert libc.libc_include_dir_from_startfile_prefix(libc_prefix, startfile_prefix) == expected

10
lib/spack/spack/util/elf.py
View file

@ -655,6 +655,16 @@ def pt_interp(path: str) -> Optional[str]:
return elf.pt_interp_str.decode("utf-8")
def get_elf_compat(path):
"""Get a triplet (EI_CLASS, EI_DATA, e_machine) from an ELF file, which can be used to see if
two ELF files are compatible."""
# On ELF platforms supporting, we try to be a bit smarter when it comes to shared
# libraries, by dropping those that are not host compatible.
with open(path, "rb") as f:
elf = parse_elf(f, only_header=True)
return (elf.is_64_bit, elf.is_little_endian, elf.elf_hdr.e_machine)
class ElfCStringUpdatesFailed(Exception):
def __init__(
self, rpath: Optional[UpdateCStringAction], pt_interp: Optional[UpdateCStringAction]

59
lib/spack/spack/util/libc.py
View file

@ -4,7 +4,9 @@
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
import os
import os.path
import re
import shlex
import sys
from subprocess import PIPE, run
from typing import Optional
@ -115,3 +117,60 @@ def libc_from_current_python_process() -> Optional["spack.spec.Spec"]:
return None
return libc_from_dynamic_linker(dynamic_linker)
def startfile_prefix(prefix: str, compatible_with: str = sys.executable) -> Optional[str]:
# Search for crt1.o at max depth 2 compatible with the ELF file provided in compatible_with.
# This is useful for finding external libc startfiles on a multiarch system.
try:
compat = spack.util.elf.get_elf_compat(compatible_with)
accept = lambda path: spack.util.elf.get_elf_compat(path) == compat
except Exception:
accept = lambda path: True
queue = [(0, prefix)]
while queue:
depth, path = queue.pop()
try:
iterator = os.scandir(path)
except OSError:
continue
with iterator:
for entry in iterator:
try:
if entry.is_dir(follow_symlinks=True):
if depth < 2:
queue.append((depth + 1, entry.path))
elif entry.name == "crt1.o" and accept(entry.path):
return path
except Exception:
continue
return None
def parse_dynamic_linker(output: str):
"""Parse -dynamic-linker /path/to/ld.so from compiler output"""
for line in reversed(output.splitlines()):
if "-dynamic-linker" not in line:
continue
args = shlex.split(line)
for idx in reversed(range(1, len(args))):
arg = args[idx]
if arg == "-dynamic-linker" or args == "--dynamic-linker":
return args[idx + 1]
elif arg.startswith("--dynamic-linker=") or arg.startswith("-dynamic-linker="):
return arg.split("=", 1)[1]
def libc_include_dir_from_startfile_prefix(
libc_prefix: str, startfile_prefix: str
) -> Optional[str]:
"""Heuristic to determine the glibc include directory from the startfile prefix. Replaces
$libc_prefix/lib*/<multiarch> with $libc_prefix/include/<multiarch>. This function does not
check if the include directory actually exists or is correct."""
parts = os.path.relpath(startfile_prefix, libc_prefix).split(os.path.sep)
if parts[0] not in ("lib", "lib64", "libx32", "lib32"):
return None
parts[0] = "include"
return os.path.join(libc_prefix, *parts)

61
var/spack/repos/builtin/packages/gcc/package.py
View file

@ -16,6 +16,7 @@
import spack.platforms
import spack.util.executable
import spack.util.libc
from spack.operating_systems.mac_os import macos_sdk_path, macos_version
from spack.package import *
@ -1152,3 +1153,63 @@ def runtime_constraints(cls, *, spec, pkg):
)
# The version of gcc-runtime is the same as the %gcc used to "compile" it
pkg("gcc-runtime").requires(f"@={str(spec.version)}", when=f"%{str(spec)}")
def _post_buildcache_install_hook(self):
if not self.spec.satisfies("platform=linux"):
return
# Setting up the runtime environment shouldn't be necessary here.
relocation_args = []
gcc = self.spec["gcc"].command
specs_file = os.path.join(self.spec_dir, "specs")
dryrun = gcc("test.c", "-###", output=os.devnull, error=str).strip()
if not dryrun:
tty.warn(f"Cannot relocate {specs_file}, compiler might not be working properly")
return
dynamic_linker = spack.util.libc.parse_dynamic_linker(dryrun)
if not dynamic_linker:
tty.warn(f"Cannot relocate {specs_file}, compiler might not be working properly")
return
libc = spack.util.libc.libc_from_dynamic_linker(dynamic_linker)
# We search for crt1.o ourselves because `gcc -print-prile-name=crt1.o` can give a rather
# convoluted relative path from a different prefix.
startfile_prefix = spack.util.libc.startfile_prefix(libc.external_path, dynamic_linker)
gcc_can_locate = lambda p: os.path.isabs(
gcc(f"-print-file-name={p}", output=str, error=os.devnull).strip()
)
if not gcc_can_locate("crt1.o"):
relocation_args.append(f"-B{startfile_prefix}")
# libc headers may also be in a multiarch subdir.
header_dir = spack.util.libc.libc_include_dir_from_startfile_prefix(
libc.external_path, startfile_prefix
)
if header_dir and all(
os.path.exists(os.path.join(header_dir, h))
for h in libc.package_class.representative_headers
):
relocation_args.append(f"-isystem {header_dir}")
else:
tty.warn(
f"Cannot relocate {specs_file} include directories, "
f"compiler might not be working properly"
)
# Delete current spec files.
try:
os.unlink(specs_file)
except OSError:
pass
# Write a new one and append flags for libc
self.write_specs_file()
if relocation_args:
with open(specs_file, "a") as f:
print("*self_spec:", file=f)
print(f"+ {' '.join(relocation_args)}", file=f)
print(file=f)

4
var/spack/repos/builtin/packages/glibc/package.py
View file

@ -22,6 +22,10 @@ class Glibc(AutotoolsPackage, GNUMirrorPackage):
build_directory = "build"
tags = ["runtime"]
# This is used when the package is external and we need to find the actual default include path
# which may be in a multiarch subdir.
representative_headers = ["ieee754.h"]
license("LGPL-2.1-or-later")
provides("libc")

4
var/spack/repos/builtin/packages/musl/package.py
View file

@ -29,6 +29,10 @@ class Musl(MakefilePackage):
license("MIT")
# This is used when the package is external and we need to find the actual default include path
# which may be in a multiarch subdir.
representative_headers = ["iso646.h"]
provides("libc")
version("1.2.4", sha256="7a35eae33d5372a7c0da1188de798726f68825513b7ae3ebe97aaaa52114f039")