hlrs-software-stack/spack
7
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Add intel-oneapi-runtime, allow injecting virtual dependencies (#42062)

Browse source 
This PR adds:
- A new runtime for `%oneapi` compilers, called `intel-oneapi-runtime`
- Information to both `gcc-runtime`  and `intel-oneapi-runtime`, to ensure
  that we don't mix compilers using different soname for either `libgfortran`
  or `libifcore`

To do so, the following internal mechanisms have been implemented:
- Possibility to inject virtual dependencies from the `runtime_constraints`
  callback on packages

Information has been added to `gcc-runtime` to provide the correct soname
under different conditions on its `%gcc`.

Rules injected into the solver looks like:

```prolog
% Add a dependency on 'gfortran@5' for nodes compiled with gcc@=13.2.0 and using the 'fortran' language
attr("dependency_holds", node(ID, Package), "gfortran", "link") :-
  attr("node", node(ID, Package)),
  attr("node_compiler", node(ID, Package), "gcc"),
  attr("node_compiler_version", node(ID, Package), "gcc", "13.2.0"),
  not external(node(ID, Package)),
  not runtime(Package),
  attr("language", node(ID, Package), "fortran").

attr("virtual_node", node(RuntimeID, "gfortran")) :-
  attr("depends_on", node(ID, Package), ProviderNode, "link"),
  provider(ProviderNode, node(RuntimeID, "gfortran")),
  attr("node", node(ID, Package)),
  attr("node_compiler", node(ID, Package), "gcc"),
  attr("node_compiler_version", node(ID, Package), "gcc", "13.2.0"),
  not external(node(ID, Package)),
  not runtime(Package),
  attr("language", node(ID, Package), "fortran").

attr("node_version_satisfies", node(RuntimeID, "gfortran"), "5") :-
  attr("depends_on", node(ID, Package), ProviderNode, "link"),
  provider(ProviderNode, node(RuntimeID, "gfortran")),
  attr("node", node(ID, Package)),
  attr("node_compiler", node(ID, Package), "gcc"),
  attr("node_compiler_version", node(ID, Package), "gcc", "13.2.0"),
  not external(node(ID, Package)),
  not runtime(Package),
  attr("language", node(ID, Package), "fortran").
```
This commit is contained in:
Massimiliano Culpo 2024-03-25 06:59:21 +01:00 committed by GitHub
parent 1fd4353289
commit 0c9a53ba3a
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
12 changed files with 269 additions and 70 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
etc/spack/defaults/packages.yaml
View file

@ -24,6 +24,7 @@ packages:
elf: [elfutils]
fftw-api: [fftw, amdfftw]
flame: [libflame, amdlibflame]
fortran-rt: [gcc-runtime, intel-oneapi-runtime]
fuse: [libfuse]
gl: [glx, osmesa]
glu: [mesa-glu, openglu]
@ -34,7 +35,9 @@ packages:
java: [openjdk, jdk, ibm-java]
jpeg: [libjpeg-turbo, libjpeg]
lapack: [openblas, amdlibflame]
libgfortran: [ gcc-runtime ]
libglx: [mesa+glx, mesa18+glx]
libifcore: [ intel-oneapi-runtime ]
libllvm: [llvm]
libosmesa: [mesa+osmesa, mesa18+osmesa]
lua-lang: [lua, lua-luajit-openresty, lua-luajit]

22
lib/spack/spack/directives.py
View file

@ -94,6 +94,9 @@ class OpenMpi(Package):
PatchesType = Optional[Union[Patcher, str, List[Union[Patcher, str]]]]
SUPPORTED_LANGUAGES = ("fortran", "cxx")
def _make_when_spec(value: WhenType) -> Optional["spack.spec.Spec"]:
"""Create a ``Spec`` that indicates when a directive should be applied.
@ -585,6 +588,9 @@ def depends_on(
@see The section "Dependency specs" in the Spack Packaging Guide.
"""
if spack.spec.Spec(spec).name in SUPPORTED_LANGUAGES:
assert type == "build", "languages must be of 'build' type"
return _language(lang_spec_str=spec, when=when)
def _execute_depends_on(pkg: "spack.package_base.PackageBase"):
_depends_on(pkg, spec, when=when, type=type, patches=patches)
@ -967,7 +973,6 @@ def license(
checked_by: string or list of strings indicating which github user checked the
license (if any).
when: A spec specifying when the license applies.
when: A spec specifying when the license applies.
"""
return lambda pkg: _execute_license(pkg, license_identifier, when)
@ -1014,6 +1019,21 @@ def _execute_requires(pkg: "spack.package_base.PackageBase"):
return _execute_requires
@directive("languages")
def _language(lang_spec_str: str, *, when: Optional[Union[str, bool]] = None):
"""Temporary implementation of language virtuals, until compilers are proper dependencies."""
def _execute_languages(pkg: "spack.package_base.PackageBase"):
when_spec = _make_when_spec(when)
if not when_spec:
return
languages = pkg.languages.setdefault(when_spec, set())
languages.add(lang_spec_str)
return _execute_languages
class DirectiveError(spack.error.SpackError):
"""This is raised when something is wrong with a package directive."""

1
lib/spack/spack/package_base.py
View file

@ -567,6 +567,7 @@ class PackageBase(WindowsRPath, PackageViewMixin, metaclass=PackageMeta):
provided_together: Dict["spack.spec.Spec", List[Set[str]]]
patches: Dict["spack.spec.Spec", List["spack.patch.Patch"]]
variants: Dict[str, Tuple["spack.variant.Variant", "spack.spec.Spec"]]
languages: Dict["spack.spec.Spec", Set[str]]
#: By default, packages are not virtual
#: Virtual packages override this attribute

64
lib/spack/spack/solver/asp.py
View file

@ -1040,6 +1040,16 @@ def conflict_rules(self, pkg):
)
self.gen.newline()
def package_languages(self, pkg):
for when_spec, languages in pkg.languages.items():
condition_msg = f"{pkg.name} needs the {', '.join(sorted(languages))} language"
if when_spec != spack.spec.Spec():
condition_msg += f" when {when_spec}"
condition_id = self.condition(when_spec, name=pkg.name, msg=condition_msg)
for language in sorted(languages):
self.gen.fact(fn.pkg_fact(pkg.name, fn.language(condition_id, language)))
self.gen.newline()
def compiler_facts(self):
"""Facts about available compilers."""
@ -1089,6 +1099,9 @@ def pkg_rules(self, pkg, tests):
self.pkg_version_rules(pkg)
self.gen.newline()
# languages
self.package_languages(pkg)
# variants
self.variant_rules(pkg)
@ -2294,8 +2307,6 @@ def setup(
self.possible_virtuals = node_counter.possible_virtuals()
self.pkgs = node_counter.possible_dependencies()
self.pkgs.update(spack.repo.PATH.packages_with_tags("runtime"))
# Fail if we already know an unreachable node is requested
for spec in specs:
missing_deps = [
@ -2309,7 +2320,6 @@ def setup(
self.explicitly_required_namespaces[node.name] = node.namespace
self.gen = ProblemInstanceBuilder()
if not allow_deprecated:
self.gen.fact(fn.deprecated_versions_not_allowed())
@ -2439,14 +2449,14 @@ def define_runtime_constraints(self):
"""Define the constraints to be imposed on the runtimes"""
recorder = RuntimePropertyRecorder(self)
for compiler in self.possible_compilers:
if compiler.name != "gcc":
continue
compiler_with_different_cls_names = {"oneapi": "intel-oneapi-compilers"}
compiler_cls_name = compiler_with_different_cls_names.get(compiler.name, compiler.name)
try:
compiler_cls = spack.repo.PATH.get_pkg_class(compiler.name)
compiler_cls = spack.repo.PATH.get_pkg_class(compiler_cls_name)
except spack.repo.UnknownPackageError:
continue
if hasattr(compiler_cls, "runtime_constraints"):
compiler_cls.runtime_constraints(compiler=compiler, pkg=recorder)
compiler_cls.runtime_constraints(spec=compiler.spec, pkg=recorder)
recorder.consume_facts()
@ -2858,13 +2868,24 @@ def reset(self):
"""Resets the current state."""
self.current_package = None
def depends_on(self, dependency_str: str, *, when: str, type: str, description: str) -> None:
def depends_on(
self,
dependency_str: str,
*,
when: str,
type: str,
description: str,
languages: Optional[List[str]] = None,
) -> None:
"""Injects conditional dependencies on packages.
Conditional dependencies can be either "real" packages or virtual dependencies.
Args:
dependency_str: the dependency spec to inject
when: anonymous condition to be met on a package to have the dependency
type: dependency type
languages: languages needed by the package for the dependency to be considered
description: human-readable description of the rule for adding the dependency
"""
# TODO: The API for this function is not final, and is still subject to change. At
@ -2890,26 +2911,45 @@ def depends_on(self, dependency_str: str, *, when: str, type: str, description:
f" not external({node_variable}),\n"
f" not runtime(Package)"
).replace(f'"{placeholder}"', f"{node_variable}")
if languages:
body_str += ",\n"
for language in languages:
body_str += f' attr("language", {node_variable}, "{language}")'
head_clauses = self._setup.spec_clauses(dependency_spec, body=False)
runtime_pkg = dependency_spec.name
is_virtual = head_clauses[0].args[0] == "virtual_node"
main_rule = (
f"% {description}\n"
f'1 {{ attr("depends_on", {node_variable}, node(0..X-1, "{runtime_pkg}"), "{type}") :'
f' max_dupes("gcc-runtime", X)}} 1:-\n'
f' max_dupes("{runtime_pkg}", X)}} 1:-\n'
f"{body_str}.\n\n"
)
if is_virtual:
main_rule = (
f"% {description}\n"
f'attr("dependency_holds", {node_variable}, "{runtime_pkg}", "{type}") :-\n'
f"{body_str}.\n\n"
)
self.rules.append(main_rule)
for clause in head_clauses:
if clause.args[0] == "node":
continue
runtime_node = f'node(RuntimeID, "{runtime_pkg}")'
head_str = str(clause).replace(f'"{runtime_pkg}"', runtime_node)
rule = (
f"{head_str} :-\n"
depends_on_constraint = (
f' attr("depends_on", {node_variable}, {runtime_node}, "{type}"),\n'
f"{body_str}.\n\n"
)
if is_virtual:
depends_on_constraint = (
f' attr("depends_on", {node_variable}, ProviderNode, "{type}"),\n'
f" provider(ProviderNode, {runtime_node}),\n"
)
rule = f"{head_str} :-\n" f"{depends_on_constraint}" f"{body_str}.\n\n"
self.rules.append(rule)
self.reset()

8
lib/spack/spack/solver/concretize.lp
View file

@ -158,6 +158,14 @@ error(100, multiple_values_error, Attribute, Package)
attr_single_value(Attribute),
2 { attr(Attribute, node(ID, Package), Value) }.
%-----------------------------------------------------------------------------
% Languages used
%-----------------------------------------------------------------------------
attr("language", node(X, Package), Language) :-
condition_holds(ConditionID, node(X, Package)),
pkg_fact(Package,language(ConditionID, Language)).
%-----------------------------------------------------------------------------
% Version semantics
%-----------------------------------------------------------------------------

13
lib/spack/spack/solver/counter.py
View file

@ -10,6 +10,7 @@
import spack.deptypes as dt
import spack.package_base
import spack.repo
import spack.spec
PossibleDependencies = Set[str]
@ -24,7 +25,13 @@ class Counter:
"""
def __init__(self, specs: List["spack.spec.Spec"], tests: bool) -> None:
self.specs = specs
runtime_pkgs = spack.repo.PATH.packages_with_tags("runtime")
runtime_virtuals = set()
for x in runtime_pkgs:
pkg_class = spack.repo.PATH.get_pkg_class(x)
runtime_virtuals.update(pkg_class.provided_virtual_names())
self.specs = specs + [spack.spec.Spec(x) for x in runtime_pkgs]
self.link_run_types: dt.DepFlag = dt.LINK | dt.RUN | dt.TEST
self.all_types: dt.DepFlag = dt.ALL
@ -33,7 +40,9 @@ def __init__(self, specs: List["spack.spec.Spec"], tests: bool) -> None:
self.all_types = dt.LINK | dt.RUN | dt.BUILD
self._possible_dependencies: PossibleDependencies = set()
self._possible_virtuals: Set[str] = set(x.name for x in specs if x.virtual)
self._possible_virtuals: Set[str] = (
set(x.name for x in specs if x.virtual) | runtime_virtuals
)
def possible_dependencies(self) -> PossibleDependencies:
"""Returns the list of possible dependencies"""

94
var/spack/repos/builtin/packages/gcc-runtime/package.py
View file

@ -22,6 +22,9 @@ class GccRuntime(Package):
tags = ["runtime"]
# gcc-runtime versions are declared dynamically
skip_version_audit = ["platform=linux", "platform=darwin"]
maintainers("haampie")
license("GPL-3.0-or-later WITH GCC-exception-3.1")
@ -44,9 +47,15 @@ class GccRuntime(Package):
"ubsan",
]
# libgfortran ABI
provides("fortran-rt", "libgfortran")
provides("libgfortran@3", when="%gcc@:6")
provides("libgfortran@4", when="%gcc@7")
provides("libgfortran@5", when="%gcc@8:")
def install(self, spec, prefix):
if spec.platform in ["linux", "cray", "freebsd"]:
libraries = self._get_libraries_elf()
libraries = get_elf_libraries(compiler=self.compiler, libraries=self.LIBRARIES)
elif spec.platform == "darwin":
libraries = self._get_libraries_macho()
else:
@ -61,47 +70,6 @@ def install(self, spec, prefix):
for path, name in libraries:
install(path, os.path.join(prefix.lib, name))
def _get_libraries_elf(self):
"""Get the GCC runtime libraries for ELF binaries"""
cc = Executable(self.compiler.cc)
lib_regex = re.compile(rb"\blib[a-z-_]+\.so\.\d+\b")
path_and_install_name = []
for name in self.LIBRARIES:
# Look for the dynamic library that gcc would use to link,
# that is with .so extension and without abi suffix.
path = cc(f"-print-file-name=lib{name}.so", output=str).strip()
# gcc reports an absolute path on success
if not os.path.isabs(path):
continue
# Now there are two options:
# 1. the file is an ELF file
# 2. the file is a linker script referencing the actual library
with open(path, "rb") as f:
try:
# Try to parse as an ELF file
soname = parse_elf(f, dynamic_section=True).dt_soname_str.decode("utf-8")
except Exception:
# On failure try to "parse" as ld script; the actual
# library needs to be mentioned by filename.
f.seek(0)
script_matches = lib_regex.findall(f.read())
if len(script_matches) != 1:
continue
soname = script_matches[0].decode("utf-8")
# Now locate and install the runtime library
runtime_path = cc(f"-print-file-name={soname}", output=str).strip()
if not os.path.isabs(runtime_path):
continue
path_and_install_name.append((runtime_path, soname))
return path_and_install_name
def _get_libraries_macho(self):
"""Same as _get_libraries_elf but for Mach-O binaries"""
cc = Executable(self.compiler.cc)
@ -152,3 +120,45 @@ def libs(self):
@property
def headers(self):
return HeaderList([])
def get_elf_libraries(compiler, libraries):
"""Get the GCC runtime libraries for ELF binaries"""
cc = Executable(compiler.cc)
lib_regex = re.compile(rb"\blib[a-z-_]+\.so\.\d+\b")
path_and_install_name = []
for name in libraries:
# Look for the dynamic library that gcc would use to link,
# that is with .so extension and without abi suffix.
path = cc(f"-print-file-name=lib{name}.so", output=str).strip()
# gcc reports an absolute path on success
if not os.path.isabs(path):
continue
# Now there are two options:
# 1. the file is an ELF file
# 2. the file is a linker script referencing the actual library
with open(path, "rb") as f:
try:
# Try to parse as an ELF file
soname = parse_elf(f, dynamic_section=True).dt_soname_str.decode("utf-8")
except Exception:
# On failure try to "parse" as ld script; the actual
# library needs to be mentioned by filename.
f.seek(0)
script_matches = lib_regex.findall(f.read())
if len(script_matches) != 1:
continue
soname = script_matches[0].decode("utf-8")
# Now locate and install the runtime library
runtime_path = cc(f"-print-file-name={soname}", output=str).strip()
if not os.path.isabs(runtime_path):
continue
path_and_install_name.append((runtime_path, soname))
return path_and_install_name

30
var/spack/repos/builtin/packages/gcc/package.py
View file

@ -1144,7 +1144,7 @@ def detect_gdc(self):
)
@classmethod
def runtime_constraints(cls, *, compiler, pkg):
def runtime_constraints(cls, *, spec, pkg):
"""Callback function to inject runtime-related rules into the solver.
Rule-injection is obtained through method calls of the ``pkg`` argument.
@ -1153,7 +1153,7 @@ def runtime_constraints(cls, *, compiler, pkg):
we'll document the behavior at https://spack.readthedocs.io/en/latest/
Args:
compiler: compiler object (node attribute) currently considered
spec: spec that will inject runtime dependencies
pkg: object used to forward information to the solver
"""
pkg("*").depends_on(
@ -1163,11 +1163,27 @@ def runtime_constraints(cls, *, compiler, pkg):
description="If any package uses %gcc, it depends on gcc-runtime",
)
pkg("*").depends_on(
f"gcc-runtime@{str(compiler.version)}:",
when=f"%{str(compiler.spec)}",
f"gcc-runtime@{str(spec.version)}:",
when=f"%{str(spec)}",
type="link",
description=f"If any package uses %{str(compiler.spec)}, "
f"it depends on gcc-runtime@{str(compiler.version)}:",
description=f"If any package uses %{str(spec)}, "
f"it depends on gcc-runtime@{str(spec.version)}:",
)
gfortran_str = "libgfortran@5"
if spec.satisfies("gcc@:6"):
gfortran_str = "libgfortran@3"
elif spec.satisfies("gcc@7"):
gfortran_str = "libgfortran@4"
for fortran_virtual in ("fortran-rt", gfortran_str):
pkg("*").depends_on(
fortran_virtual,
when=f"%{str(spec)}",
languages=["fortran"],
type="link",
description=f"Add a dependency on '{gfortran_str}' for nodes compiled with "
f"{str(spec)} and using the 'fortran' language",
)
# The version of gcc-runtime is the same as the %gcc used to "compile" it
pkg("gcc-runtime").requires(f"@={str(compiler.version)}", when=f"%{str(compiler.spec)}")
pkg("gcc-runtime").requires(f"@={str(spec.version)}", when=f"%{str(spec)}")

3
var/spack/repos/builtin/packages/hdf5/package.py
View file

@ -33,6 +33,9 @@ class Hdf5(CMakePackage):
license("custom")
depends_on("cxx", type="build", when="+cxx")
depends_on("fortran", type="build", when="+fortran")
# The 'develop' version is renamed so that we could uninstall (or patch) it
# without affecting other develop version.
version("develop-1.15", branch="develop")

28
var/spack/repos/builtin/packages/intel-oneapi-compilers/package.py
View file

@ -388,3 +388,31 @@ def _ld_library_path(self):
p = join_path(self.component_prefix.linux, d)
if find(p, "*." + dso_suffix, recursive=False):
yield p
@classmethod
def runtime_constraints(cls, *, spec, pkg):
pkg("*").depends_on(
"intel-oneapi-runtime",
when="%oneapi",
type="link",
description="If any package uses %oneapi, it depends on intel-oneapi-runtime",
)
pkg("*").depends_on(
f"intel-oneapi-runtime@{str(spec.version)}:",
when=f"%{str(spec)}",
type="link",
description=f"If any package uses %{str(spec)}, "
f"it depends on intel-oneapi-runtime@{str(spec.version)}:",
)
for fortran_virtual in ("fortran-rt", "libifcore@5"):
pkg("*").depends_on(
fortran_virtual,
when=f"%{str(spec)}",
languages=["fortran"],
type="link",
description=f"Add a dependency on 'libifcore' for nodes compiled with "
f"{str(spec)} and using the 'fortran' language",
)
# The version of gcc-runtime is the same as the %gcc used to "compile" it
pkg("intel-oneapi-runtime").requires(f"@={str(spec.version)}", when=f"%{str(spec)}")

61
var/spack/repos/builtin/packages/intel-oneapi-runtime/package.py Normal file
View file

@ -0,0 +1,61 @@
# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
import os
from llnl.util import tty
from spack.package import *
from spack.pkg.builtin.gcc_runtime import get_elf_libraries
class IntelOneapiRuntime(Package):
"""Package for OneAPI compiler runtime libraries"""
homepage = "https://software.intel.com/content/www/us/en/develop/tools/oneapi.html"
has_code = False
tags = ["runtime"]
requires("%oneapi")
depends_on("gcc-runtime", type="link")
LIBRARIES = [
"imf",
"intlc",
"irng",
"svml",
"ifcore", # Fortran
"ifcoremt", # Fortran
"ifport", # Fortran
"iomp5",
"sycl",
]
# libifcore ABI
provides("fortran-rt", "libifcore@5", when="%oneapi@2021:")
provides("sycl")
conflicts("platform=windows", msg="IntelOneAPI can only be installed on Linux, and FreeBSD")
conflicts("platform=darwin", msg="IntelOneAPI can only be installed on Linux, and FreeBSD")
def install(self, spec, prefix):
libraries = get_elf_libraries(compiler=self.compiler, libraries=self.LIBRARIES)
mkdir(prefix.lib)
if not libraries:
tty.warn("Could not detect any shared OneAPI runtime libraries")
return
for path, name in libraries:
install(path, os.path.join(prefix.lib, name))
@property
def libs(self):
return LibraryList([])
@property
def headers(self):
return HeaderList([])

12
var/spack/repos/compiler_runtime.test/packages/gcc/package.py
View file

@ -14,7 +14,7 @@ class Gcc(Package):
version("12.3.0")
@classmethod
def runtime_constraints(cls, *, compiler, pkg):
def runtime_constraints(cls, *, spec, pkg):
pkg("*").depends_on(
"gcc-runtime",
when="%gcc",
@ -22,11 +22,11 @@ def runtime_constraints(cls, *, compiler, pkg):
description="If any package uses %gcc, it depends on gcc-runtime",
)
pkg("*").depends_on(
f"gcc-runtime@{str(compiler.version)}:",
when=f"%{str(compiler.spec)}",
f"gcc-runtime@{str(spec.version)}:",
when=f"%{str(spec)}",
type="link",
description=f"If any package uses %{str(compiler.spec)}, "
f"it depends on gcc-runtime@{str(compiler.version)}:",
description=f"If any package uses %{str(spec)}, "
f"it depends on gcc-runtime@{str(spec.version)}:",
)
# The version of gcc-runtime is the same as the %gcc used to "compile" it
pkg("gcc-runtime").requires(f"@={str(compiler.version)}", when=f"%{str(compiler.spec)}")
pkg("gcc-runtime").requires(f"@={str(spec.version)}", when=f"%{str(spec)}")