hlrs-software-stack/spack
7
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Add libc dependency to compiled packages and runtime deps

Browse source 
This commit differentiate linux from other platforms by
using libc compatibility as a criterion for deciding
which buildcaches / binaries can be reused. Other
platforms still use OS compatibility.

On linux a libc is injected by all compilers as an implicit
external, and the compatibility criterion is that a libc is
compatible with all other libcs with the same name and a
version that is lesser or equal.

Some concretization unit tests use libc when run on linux.
This commit is contained in:
Massimiliano Culpo 2024-04-03 14:18:47 +02:00 committed by Harmen Stoppels
parent 209a3bf302
commit 34146c197a
18 changed files with 190 additions and 88 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

93
lib/spack/spack/solver/asp.py
View file

@ -283,6 +283,27 @@ def all_compilers_in_config(configuration):
return spack.compilers.all_compilers_from(configuration)
def compatible_libc(candidate_libc_spec):
"""Returns a list of libc specs that are compatible with the one passed as argument"""
result = set()
for compiler in all_compilers_in_config(spack.config.CONFIG):
libc = compiler.default_libc()
if not libc:
continue
if (
libc.name == candidate_libc_spec.name
and libc.version >= candidate_libc_spec.version
and libc.external_path == candidate_libc_spec.external_path
):
result.add(libc)
return sorted(result)
def using_libc_compatibility() -> bool:
"""Returns True if we are currently using libc compatibility"""
return spack.platforms.host().name == "linux"
def extend_flag_list(flag_list, new_flags):
"""Extend a list of flags, preserving order and precedence.
@ -566,16 +587,16 @@ def _spec_with_default_name(spec_str, name):
return spec
def _external_config_with_implictit_externals():
def _external_config_with_implicit_externals(configuration):
# Read packages.yaml and normalize it, so that it will not contain entries referring to
# virtual packages.
packages_yaml = _normalize_packages_yaml(spack.config.get("packages"))
packages_yaml = _normalize_packages_yaml(configuration.get("packages"))
# Add externals for libc from compilers on Linux
if spack.platforms.host().name != "linux":
if not using_libc_compatibility():
return packages_yaml
for compiler in all_compilers_in_config():
for compiler in all_compilers_in_config(configuration):
libc = compiler.default_libc()
if libc:
entry = {"spec": f"{libc} %{compiler.spec}", "prefix": libc.external_path}
@ -801,10 +822,16 @@ def solve(self, setup, specs, reuse=None, output=None, control=None, allow_depre
self.control.load(os.path.join(parent_dir, "heuristic.lp"))
if spack.config.CONFIG.get("concretizer:duplicates:strategy", "none") != "none":
self.control.load(os.path.join(parent_dir, "heuristic_separate.lp"))
self.control.load(os.path.join(parent_dir, "os_compatibility.lp"))
self.control.load(os.path.join(parent_dir, "display.lp"))
if not setup.concretize_everything:
self.control.load(os.path.join(parent_dir, "when_possible.lp"))
# Binary compatibility is based on libc on Linux, and on the os tag elsewhere
if using_libc_compatibility():
self.control.load(os.path.join(parent_dir, "libc_compatibility.lp"))
else:
self.control.load(os.path.join(parent_dir, "os_compatibility.lp"))
timer.stop("load")
# Grounding is the first step in the solve -- it turns our facts
@ -1572,7 +1599,7 @@ def emit_facts_from_requirement_rules(self, rules: List[RequirementRule]):
def external_packages(self):
"""Facts on external packages, from packages.yaml and implicit externals."""
packages_yaml = _external_config_with_implictit_externals()
packages_yaml = _external_config_with_implicit_externals(spack.config.CONFIG)
self.gen.h1("External packages")
for pkg_name, data in packages_yaml.items():
@ -1845,6 +1872,15 @@ def _spec_clauses(
if dep.name == "gcc-runtime":
continue
# LIBC is also solved again by clingo, but in this case the compatibility
# is not encoded in the parent node - so we need to emit explicit facts
if "libc" in dspec.virtuals:
for x in compatible_libc(dep):
clauses.append(
fn.attr("compatible_libc", spec.name, x.name, x.version)
)
continue
# We know dependencies are real for concrete specs. For abstract
# specs they just mean the dep is somehow in the DAG.
for dtype in dt.ALL_FLAGS:
@ -2316,6 +2352,10 @@ def setup(
self.gen = ProblemInstanceBuilder()
compiler_parser = CompilerParser(configuration=spack.config.CONFIG).with_input_specs(specs)
# Only relevant for linux
for libc in compiler_parser.allowed_libcs:
self.gen.fact(fn.allowed_libc(libc.name, libc.version))
if not allow_deprecated:
self.gen.fact(fn.deprecated_versions_not_allowed())
@ -2445,18 +2485,35 @@ def visit(node):
def define_runtime_constraints(self):
"""Define the constraints to be imposed on the runtimes"""
recorder = RuntimePropertyRecorder(self)
# TODO: Use only available compilers ?
for compiler in self.possible_compilers:
compiler_with_different_cls_names = {"oneapi": "intel-oneapi-compilers"}
compiler_with_different_cls_names = {
"oneapi": "intel-oneapi-compilers",
"clang": "llvm",
}
compiler_cls_name = compiler_with_different_cls_names.get(
compiler.spec.name, compiler.spec.name
)
try:
compiler_cls = spack.repo.PATH.get_pkg_class(compiler_cls_name)
if hasattr(compiler_cls, "runtime_constraints"):
compiler_cls.runtime_constraints(spec=compiler.spec, pkg=recorder)
except spack.repo.UnknownPackageError:
pass
# Inject libc from available compilers, on Linux
if not compiler.available:
continue
if hasattr(compiler_cls, "runtime_constraints"):
compiler_cls.runtime_constraints(spec=compiler.spec, pkg=recorder)
if using_libc_compatibility():
libc = compiler.compiler_obj.default_libc()
if libc:
recorder("*").depends_on(
"libc", when=f"%{compiler.spec}", type="link", description="Add libc"
)
recorder("*").depends_on(
str(libc), when=f"%{compiler.spec}", type="link", description="Add libc"
)
recorder.consume_facts()
@ -2833,7 +2890,19 @@ class CompilerParser:
def __init__(self, configuration) -> None:
self.compilers: Set[KnownCompiler] = set()
self.allowed_libcs = set()
for c in all_compilers_in_config(configuration):
if using_libc_compatibility():
libc = c.default_libc()
if not libc:
warnings.warn(
f"cannot detect libc from {c.spec}. The compiler will not be used "
f"during concretization."
)
continue
self.allowed_libcs.add(libc)
target = c.target if c.target != "any" else None
candidate = KnownCompiler(
spec=c.spec, os=c.operating_system, target=target, available=True, compiler_obj=c
@ -3199,7 +3268,7 @@ def no_flags(self, node, flag_type):
def external_spec_selected(self, node, idx):
"""This means that the external spec and index idx has been selected for this package."""
packages_yaml = _external_config_with_implictit_externals()
packages_yaml = _external_config_with_implicit_externals(spack.config.CONFIG)
spec_info = packages_yaml[node.pkg]["externals"][int(idx)]
self._specs[node].external_path = spec_info.get("prefix", None)
self._specs[node].external_modules = spack.spec.Spec._format_module_list(
@ -3514,7 +3583,7 @@ def _check_input_and_extract_concrete_specs(specs):
def _reusable_specs(self, specs):
reusable_specs = []
if self.reuse:
packages = spack.config.get("packages")
packages = _external_config_with_implicit_externals(spack.config.CONFIG)
# Specs from the local Database
with spack.store.STORE.db.read_transaction():
reusable_specs.extend(

15
lib/spack/spack/solver/concretize.lp
View file

@ -1023,14 +1023,6 @@ error(100, "Cannot select '{0} os={1}' (operating system '{1}' is not buildable)
attr("node_os", node(X, Package), OS),
not buildable_os(OS).
% can't have dependencies on incompatible OS's
error(100, "{0} and dependency {1} have incompatible operating systems 'os={2}' and 'os={3}'", Package, Dependency, PackageNodeOS, DependencyOS)
:- depends_on(node(X, Package), node(Y, Dependency)),
attr("node_os", node(X, Package), PackageNodeOS),
attr("node_os", node(Y, Dependency), DependencyOS),
not os_compatible(PackageNodeOS, DependencyOS),
build(node(X, Package)).
% give OS choice weights according to os declarations
node_os_weight(PackageNode, Weight)
:- attr("node", PackageNode),
@ -1043,13 +1035,6 @@ os_compatible(OS, OS) :- os(OS).
% Transitive compatibility among operating systems
os_compatible(OS1, OS3) :- os_compatible(OS1, OS2), os_compatible(OS2, OS3).
% We can select only operating systems compatible with the ones
% for which we can build software. We need a cardinality constraint
% since we might have more than one "buildable_os(OS)" fact.
:- not 1 { os_compatible(CurrentOS, ReusedOS) : buildable_os(CurrentOS) },
attr("node_os", Package, ReusedOS),
internal_error("Reused OS incompatible with build OS").
% If an OS is set explicitly respect the value
attr("node_os", PackageNode, OS) :- attr("node_os_set", PackageNode, OS), attr("node", PackageNode).

37
lib/spack/spack/solver/libc_compatibility.lp Normal file
View file

@ -0,0 +1,37 @@
% Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
% Spack Project Developers. See the top-level COPYRIGHT file for details.
%
% SPDX-License-Identifier: (Apache-2.0 OR MIT)
%=============================================================================
% Libc compatibility rules for reusing solves.
%
% These rules are used on Linux
%=============================================================================
% A package cannot be reused if the libc is not compatible with it
:- provider(node(X, LibcPackage), node(0, "libc")),
attr("version", node(X, LibcPackage), LibcVersion),
attr("hash", node(R, ReusedPackage), Hash),
% Libc packages can be reused without the "compatible_libc" attribute
ReusedPackage != LibcPackage,
not attr("compatible_libc", node(R, ReusedPackage), LibcPackage, LibcVersion).
% Check whether the DAG has any built package
has_built_packages() :- build(X), not external(X).
% A libc is needed in the DAG
:- has_built_packages(), not provider(_, node(0, "libc")).
% The libc must be chosen among available ones
:- has_built_packages(),
provider(node(X, LibcPackage), node(0, "libc")),
attr("node", node(X, LibcPackage)),
attr("version", node(X, LibcPackage), LibcVersion),
not allowed_libc(LibcPackage, LibcVersion).
% A built node must depend on libc
:- build(PackageNode),
provider(LibcNode, node(0, "libc")),
not external(PackageNode),
not depends_on(PackageNode, LibcNode).

16
lib/spack/spack/solver/os_compatibility.lp
View file

@ -7,8 +7,24 @@
% OS compatibility rules for reusing solves.
% os_compatible(RecentOS, OlderOS)
% OlderOS binaries can be used on RecentOS
%
% These rules are used on every platform, but Linux
%=============================================================================
% macOS
os_compatible("monterey", "bigsur").
os_compatible("bigsur", "catalina").
% can't have dependencies on incompatible OS's
error(100, "{0} and dependency {1} have incompatible operating systems 'os={2}' and 'os={3}'", Package, Dependency, PackageNodeOS, DependencyOS)
:- depends_on(node(X, Package), node(Y, Dependency)),
attr("node_os", node(X, Package), PackageNodeOS),
attr("node_os", node(Y, Dependency), DependencyOS),
not os_compatible(PackageNodeOS, DependencyOS),
build(node(X, Package)).
% We can select only operating systems compatible with the ones
% for which we can build software. We need a cardinality constraint
% since we might have more than one "buildable_os(OS)" fact.
:- not 1 { os_compatible(CurrentOS, ReusedOS) : buildable_os(CurrentOS) },
attr("node_os", Package, ReusedOS).

43
lib/spack/spack/test/concretize.py
View file

@ -13,6 +13,7 @@
import llnl.util.lang
import spack.compiler
import spack.compilers
import spack.concretize
import spack.config
@ -67,6 +68,24 @@ def check_concretize(abstract_spec):
return concrete
@pytest.fixture(scope="function", autouse=True)
def binary_compatibility(monkeypatch, request):
"""Selects whether we use OS compatibility for binaries, or libc compatibility."""
if spack.platforms.real_host().name != "linux":
return
if "mock_packages" not in request.fixturenames:
# Only builtin.mock has a mock glibc package
return
if "database" in request.fixturenames or "mutable_database" in request.fixturenames:
# Databases have been created without glibc support
return
monkeypatch.setattr(spack.solver.asp, "using_libc_compatibility", lambda: True)
monkeypatch.setattr(spack.compiler.Compiler, "default_libc", lambda x: Spec("glibc@=2.28"))
@pytest.fixture(
params=[
# no_deps
@ -1452,6 +1471,8 @@ def test_os_selection_when_multiple_choices_are_possible(
):
s = Spec(spec_str).concretized()
for node in s.traverse():
if node.name == "glibc":
continue
assert node.satisfies(expected_os)
@pytest.mark.regression("22718")
@ -1764,7 +1785,8 @@ def test_best_effort_coconcretize(self, specs, expected):
for s in result.specs:
concrete_specs.update(s.traverse())
assert len(concrete_specs) == expected
libc_offset = 1 if spack.solver.asp.using_libc_compatibility() else 0
assert len(concrete_specs) == expected + libc_offset
@pytest.mark.parametrize(
"specs,expected_spec,occurances",
@ -1884,29 +1906,16 @@ def test_version_weight_and_provenance(self):
result_spec = result.specs[0]
num_specs = len(list(result_spec.traverse()))
libc_offset = 1 if spack.solver.asp.using_libc_compatibility() else 0
criteria = [
(num_specs - 1, None, "number of packages to build (vs. reuse)"),
(num_specs - 1 - libc_offset, None, "number of packages to build (vs. reuse)"),
(2, 0, "version badness"),
]
for criterion in criteria:
assert criterion in result.criteria
assert criterion in result.criteria, result_spec
assert result_spec.satisfies("^b@1.0")
@pytest.mark.regression("31169")
@pytest.mark.only_clingo("Use case not supported by the original concretizer")
def test_not_reusing_incompatible_os(self):
root_spec = Spec("b")
s = root_spec.concretized()
wrong_os = s.copy()
wrong_os.architecture = spack.spec.ArchSpec("test-ubuntu2204-x86_64")
with spack.config.override("concretizer:reuse", True):
solver = spack.solver.asp.Solver()
setup = spack.solver.asp.SpackSolverSetup()
result, _, _ = solver.driver.solve(setup, [root_spec], reuse=[wrong_os])
concrete_spec = result.specs[0]
assert concrete_spec.satisfies("os={}".format(s.architecture.os))
@pytest.mark.only_clingo("Use case not supported by the original concretizer")
def test_reuse_succeeds_with_config_compatible_os(self):
root_spec = Spec("b")

21
var/spack/repos/builtin.mock/packages/glibc/package.py Normal file
View file

@ -0,0 +1,21 @@
# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
from spack.package import *
class Glibc(AutotoolsPackage, GNUMirrorPackage):
"""The GNU C Library provides many of the low-level components used
directly by programs written in the C or C++ languages.
"""
homepage = "https://www.gnu.org/software/libc/"
gnu_mirror_path = "libc/glibc-2.33.tar.gz"
git = "https://sourceware.org/git/glibc.git"
tags = ["runtime"]
provides("libc")
version("2.39", sha256="97f84f3b7588cd54093a6f6389b0c1a81e70d99708d74963a2e3eab7c7dc942d")

1
var/spack/repos/builtin/packages/aocc/package.py
View file

@ -28,7 +28,6 @@ class Aocc(Package):
"""
_name = "aocc"
family = "compiler"
homepage = "https://www.amd.com/en/developer/aocc.html"
maintainers("amd-toolchain-support")

2
var/spack/repos/builtin/packages/gcc-runtime/package.py
View file

@ -53,6 +53,8 @@ class GccRuntime(Package):
provides("libgfortran@4", when="%gcc@7")
provides("libgfortran@5", when="%gcc@8:")
depends_on("libc", type="link")
def install(self, spec, prefix):
if spec.platform in ["linux", "cray", "freebsd"]:
libraries = get_elf_libraries(compiler=self.compiler, libraries=self.LIBRARIES)

8
var/spack/repos/builtin/packages/gcc/package.py
View file

@ -1185,13 +1185,5 @@ def runtime_constraints(cls, *, spec, pkg):
description=f"Add a dependency on '{gfortran_str}' for nodes compiled with "
f"{str(spec)} and using the 'fortran' language",
)
libc = compiler.default_libc()
if libc:
pkg("*").depends_on(
str(libc), when=f"%{str(compiler.spec)}", type="link", description="Add libc"
)
# The version of gcc-runtime is the same as the %gcc used to "compile" it
pkg("gcc-runtime").requires(f"@={str(spec.version)}", when=f"%{str(spec)}")

4
var/spack/repos/builtin/packages/glibc/package.py
View file

@ -198,3 +198,7 @@ def build(self, spec, prefix):
make("-C", "..", f"objdir={os.getcwd()}", "lib")
delete_rpath(join_path("elf", "ld.so"))
make()
@property
def libs(self):
return LibraryList([])

2
var/spack/repos/builtin/packages/intel-llvm/package.py
View file

@ -14,8 +14,6 @@ class IntelLlvm(CMakePackage):
homepage = "https://github.com/intel/llvm"
git = "https://github.com/intel/llvm.git"
family = "compiler"
license("Apache-2.0")
version("sycl", branch="sycl")

2
var/spack/repos/builtin/packages/intel-oneapi-compilers/package.py
View file

@ -422,5 +422,5 @@ def runtime_constraints(cls, *, spec, pkg):
description=f"Add a dependency on 'libifcore' for nodes compiled with "
f"{str(spec)} and using the 'fortran' language",
)
# The version of gcc-runtime is the same as the %gcc used to "compile" it
# The version of intel-oneapi-runtime is the same as the %oneapi used to "compile" it
pkg("intel-oneapi-runtime").requires(f"@={str(spec.version)}", when=f"%{str(spec)}")

2
var/spack/repos/builtin/packages/intel-oneapi-runtime/package.py
View file

@ -44,6 +44,8 @@ class IntelOneapiRuntime(Package):
conflicts("platform=windows", msg="IntelOneAPI can only be installed on Linux, and FreeBSD")
conflicts("platform=darwin", msg="IntelOneAPI can only be installed on Linux, and FreeBSD")
depends_on("libc", type="link")
def install(self, spec, prefix):
libraries = get_elf_libraries(compiler=self.compiler, libraries=self.LIBRARIES)
mkdir(prefix.lib)

24
var/spack/repos/builtin/packages/libc/package.py
View file

@ -1,24 +0,0 @@
# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
from spack.package import *
class Libc(BundlePackage):
"""Dummy package to provide interfaces available in libc."""
homepage = "https://en.wikipedia.org/wiki/C_standard_library"
version("1.0") # Dummy
variant("iconv", default=False, description="Provides interfaces for Localization Functions")
variant("rpc", default=False, description="Provides interfaces for RPC")
provides("iconv", when="+iconv")
provides("rpc", when="+rpc")
@property
def libs(self):
return LibraryList([])

2
var/spack/repos/builtin/packages/llvm-doe/package.py
View file

@ -28,8 +28,6 @@ class LlvmDoe(CMakePackage, CudaPackage):
generator("ninja")
family = "compiler" # Used by lmod
version("doe", branch="doe", preferred=True)
version("upstream", branch="llvm.org/main")
version("bolt", branch="bolt/main")

2
var/spack/repos/builtin/packages/llvm/package.py
View file

@ -32,8 +32,6 @@ class Llvm(CMakePackage, CudaPackage):
generator("ninja")
family = "compiler" # Used by lmod
license("Apache-2.0")
version("main", branch="main")

2
var/spack/repos/builtin/packages/sollve/package.py
View file

@ -16,8 +16,6 @@ class Sollve(CMakePackage):
homepage = "https://www.bnl.gov/compsci/projects/SOLLVE/"
git = "https://github.com/SOLLVE/llvm.git"
family = "compiler" # Used by lmod
# NOTE: The debug version of LLVM is an order of magnitude larger than
# the release version, and may take up 20-30 GB of space. If you want
# to save space, build with `build_type=Release`.

2
var/spack/repos/builtin/packages/templight/package.py
View file

@ -18,8 +18,6 @@ class Templight(CMakePackage):
git = "https://github.com/mikael-s-persson/templight.git"
llvm_svn = "http://llvm.org/svn/llvm-project/{0}/trunk"
family = "compiler" # Used by lmod
# Templight is a patch to clang, so we have three versions to care about:
# - The one that will be used in Spack specifications
# - The git branch that we need to fetch from in the templight repo