From 1f31c3374ccfb843cc36e6c967fc61c8b832f10f Mon Sep 17 00:00:00 2001 From: Greg Becker Date: Mon, 6 May 2024 01:33:33 -0700 Subject: [PATCH] External package detection for compilers (#43464) This creates shared infrastructure for compiler packages to implement the detailed search capabilities from the `spack compiler find` command for the `spack external find` command. After this commit, `spack compiler find` can be replaced with `spack external find --tag compiler`, with the exception of mixed toolchains. --- lib/spack/docs/getting_started.rst | 14 +- lib/spack/spack/build_systems/compiler.py | 144 ++++++++++++++ lib/spack/spack/compilers/__init__.py | 63 ++++-- lib/spack/spack/package.py | 1 + lib/spack/spack/test/cmd/compiler.py | 30 +-- lib/spack/spack/test/conftest.py | 2 +- .../builtin/packages/acfl/detection_test.yaml | 83 ++++++++ .../repos/builtin/packages/acfl/package.py | 37 +--- .../builtin/packages/aocc/detection_test.yaml | 75 +++++++ .../repos/builtin/packages/aocc/package.py | 8 +- .../packages/apple-clang/detection_test.yaml | 84 ++++---- .../builtin/packages/apple-clang/package.py | 46 +---- .../builtin/packages/cce/detection_test.yaml | 44 +++++ .../repos/builtin/packages/cce/package.py | 30 +++ .../builtin/packages/fj/detection_test.yaml | 27 +++ .../repos/builtin/packages/fj/package.py | 40 +--- .../builtin/packages/gcc/detection_test.yaml | 187 +++++++++--------- .../repos/builtin/packages/gcc/package.py | 95 ++++----- .../detection_test.yaml | 89 +++++++++ .../intel-oneapi-compilers-classic/package.py | 21 +- .../detection_test.yaml | 23 +++ .../intel-oneapi-compilers/package.py | 11 +- .../packages/intel/detection_test.yaml | 43 ++-- .../packages/llvm-amdgpu/detection_test.yaml | 103 ++++++++++ .../builtin/packages/llvm-amdgpu/package.py | 23 +-- .../builtin/packages/llvm/detection_test.yaml | 132 +++++++------ .../repos/builtin/packages/llvm/package.py | 72 ++++--- .../builtin/packages/msvc/detection_test.yaml | 14 ++ .../repos/builtin/packages/msvc/package.py | 66 +++++++ .../builtin/packages/nag/detection_test.yaml | 13 ++ .../repos/builtin/packages/nag/package.py | 58 +----- .../packages/nvhpc/detection_test.yaml | 82 ++++++++ .../repos/builtin/packages/nvhpc/package.py | 16 +- .../builtin/packages/pgi/detection_test.yaml | 11 ++ .../repos/builtin/packages/pgi/package.py | 9 +- .../builtin/packages/xl/detection_test.yaml | 31 +++ .../repos/builtin/packages/xl/package.py | 65 ++++++ .../builtin/packages/xlc/detection_test.yaml | 19 -- .../repos/builtin/packages/xlc/package.py | 81 -------- .../builtin/packages/xlf/detection_test.yaml | 12 -- .../repos/builtin/packages/xlf/package.py | 69 ------- 41 files changed, 1375 insertions(+), 698 deletions(-) create mode 100644 lib/spack/spack/build_systems/compiler.py create mode 100644 var/spack/repos/builtin/packages/acfl/detection_test.yaml create mode 100644 var/spack/repos/builtin/packages/aocc/detection_test.yaml create mode 100644 var/spack/repos/builtin/packages/cce/detection_test.yaml create mode 100644 var/spack/repos/builtin/packages/cce/package.py create mode 100644 var/spack/repos/builtin/packages/fj/detection_test.yaml create mode 100644 var/spack/repos/builtin/packages/intel-oneapi-compilers-classic/detection_test.yaml create mode 100644 var/spack/repos/builtin/packages/intel-oneapi-compilers/detection_test.yaml create mode 100644 var/spack/repos/builtin/packages/llvm-amdgpu/detection_test.yaml create mode 100644 var/spack/repos/builtin/packages/msvc/detection_test.yaml create mode 100644 var/spack/repos/builtin/packages/msvc/package.py create mode 100644 var/spack/repos/builtin/packages/nag/detection_test.yaml create mode 100644 var/spack/repos/builtin/packages/nvhpc/detection_test.yaml create mode 100644 var/spack/repos/builtin/packages/pgi/detection_test.yaml create mode 100644 var/spack/repos/builtin/packages/xl/detection_test.yaml create mode 100644 var/spack/repos/builtin/packages/xl/package.py delete mode 100644 var/spack/repos/builtin/packages/xlc/detection_test.yaml delete mode 100644 var/spack/repos/builtin/packages/xlc/package.py delete mode 100644 var/spack/repos/builtin/packages/xlf/detection_test.yaml delete mode 100644 var/spack/repos/builtin/packages/xlf/package.py diff --git a/lib/spack/docs/getting_started.rst b/lib/spack/docs/getting_started.rst index 25dfc95ee5..9435111c8e 100644 --- a/lib/spack/docs/getting_started.rst +++ b/lib/spack/docs/getting_started.rst @@ -478,6 +478,13 @@ prefix, you can add them to the ``extra_attributes`` field. Similarly, all other fields from the compilers config can be added to the ``extra_attributes`` field for an external representing a compiler. +Note that the format for the ``paths`` field in the +``extra_attributes`` section is different than in the ``compilers`` +config. For compilers configured as external packages, the section is +named ``compilers`` and the dictionary maps language names (``c``, +``cxx``, ``fortran``) to paths, rather than using the names ``cc``, +``fc``, and ``f77``. + .. code-block:: yaml packages: @@ -493,11 +500,10 @@ all other fields from the compilers config can be added to the - spec: llvm+clang@15.0.0 arch=linux-rhel8-skylake prefix: /usr extra_attributes: - paths: - cc: /usr/bin/clang-with-suffix + compilers: + c: /usr/bin/clang-with-suffix cxx: /usr/bin/clang++-with-extra-info - fc: /usr/bin/gfortran - f77: /usr/bin/gfortran + fortran: /usr/bin/gfortran extra_rpaths: - /usr/lib/llvm/ diff --git a/lib/spack/spack/build_systems/compiler.py b/lib/spack/spack/build_systems/compiler.py new file mode 100644 index 0000000000..d441b57b2e --- /dev/null +++ b/lib/spack/spack/build_systems/compiler.py @@ -0,0 +1,144 @@ +# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +import itertools +import os +import pathlib +import re +import sys +from typing import Dict, List, Sequence, Tuple, Union + +import llnl.util.tty as tty +from llnl.util.lang import classproperty + +import spack.compiler +import spack.package_base + +# Local "type" for type hints +Path = Union[str, pathlib.Path] + + +class CompilerPackage(spack.package_base.PackageBase): + """A Package mixin for all common logic for packages that implement compilers""" + + # TODO: how do these play nicely with other tags + tags: Sequence[str] = ["compiler"] + + #: Optional suffix regexes for searching for this type of compiler. + #: Suffixes are used by some frameworks, e.g. macports uses an '-mp-X.Y' + #: version suffix for gcc. + compiler_suffixes: List[str] = [r"-.*"] + + #: Optional prefix regexes for searching for this compiler + compiler_prefixes: List[str] = [] + + #: Compiler argument(s) that produces version information + #: If multiple arguments, the earlier arguments must produce errors when invalid + compiler_version_argument: Union[str, Tuple[str]] = "-dumpversion" + + #: Regex used to extract version from compiler's output + compiler_version_regex: str = "(.*)" + + #: Static definition of languages supported by this class + compiler_languages: Sequence[str] = ["c", "cxx", "fortran"] + + def __init__(self, spec: "spack.spec.Spec"): + super().__init__(spec) + msg = f"Supported languages for {spec} are not a subset of possible supported languages" + msg += f" supports: {self.supported_languages}, valid values: {self.compiler_languages}" + assert set(self.supported_languages) <= set(self.compiler_languages), msg + + @property + def supported_languages(self) -> Sequence[str]: + """Dynamic definition of languages supported by this package""" + return self.compiler_languages + + @classproperty + def compiler_names(cls) -> Sequence[str]: + """Construct list of compiler names from per-language names""" + names = [] + for language in cls.compiler_languages: + names.extend(getattr(cls, f"{language}_names")) + return names + + @classproperty + def executables(cls) -> Sequence[str]: + """Construct executables for external detection from names, prefixes, and suffixes.""" + regexp_fmt = r"^({0}){1}({2})$" + prefixes = [""] + cls.compiler_prefixes + suffixes = [""] + cls.compiler_suffixes + if sys.platform == "win32": + ext = r"\.(?:exe|bat)" + suffixes += [suf + ext for suf in suffixes] + return [ + regexp_fmt.format(prefix, re.escape(name), suffix) + for prefix, name, suffix in itertools.product(prefixes, cls.compiler_names, suffixes) + ] + + @classmethod + def determine_version(cls, exe: Path): + version_argument = cls.compiler_version_argument + if isinstance(version_argument, str): + version_argument = (version_argument,) + + for va in version_argument: + try: + output = spack.compiler.get_compiler_version_output(exe, va) + match = re.search(cls.compiler_version_regex, output) + if match: + return ".".join(match.groups()) + except spack.util.executable.ProcessError: + pass + except Exception as e: + tty.debug( + f"[{__file__}] Cannot detect a valid version for the executable " + f"{str(exe)}, for package '{cls.name}': {e}" + ) + + @classmethod + def compiler_bindir(cls, prefix: Path) -> Path: + """Overridable method for the location of the compiler bindir within the preifx""" + return os.path.join(prefix, "bin") + + @classmethod + def determine_compiler_paths(cls, exes: Sequence[Path]) -> Dict[str, Path]: + """Compute the paths to compiler executables associated with this package + + This is a helper method for ``determine_variants`` to compute the ``extra_attributes`` + to include with each spec object.""" + # There are often at least two copies (not symlinks) of each compiler executable in the + # same directory: one with a canonical name, e.g. "gfortran", and another one with the + # target prefix, e.g. "x86_64-pc-linux-gnu-gfortran". There also might be a copy of "gcc" + # with the version suffix, e.g. "x86_64-pc-linux-gnu-gcc-6.3.0". To ensure the consistency + # of values in the "paths" dictionary (i.e. we prefer all of them to reference copies + # with canonical names if possible), we iterate over the executables in the reversed sorted + # order: + # First pass over languages identifies exes that are perfect matches for canonical names + # Second pass checks for names with prefix/suffix + # Second pass is sorted by language name length because longer named languages + # e.g. cxx can often contain the names of shorter named languages + # e.g. c (e.g. clang/clang++) + paths = {} + exes = sorted(exes, reverse=True) + languages = { + lang: getattr(cls, f"{lang}_names") + for lang in sorted(cls.compiler_languages, key=len, reverse=True) + } + for exe in exes: + for lang, names in languages.items(): + if os.path.basename(exe) in names: + paths[lang] = exe + break + else: + for lang, names in languages.items(): + if any(name in os.path.basename(exe) for name in names): + paths[lang] = exe + break + + return paths + + @classmethod + def determine_variants(cls, exes: Sequence[Path], version_str: str) -> Tuple: + # path determination is separated so it can be reused in subclasses + return "", {"compilers": cls.determine_compiler_paths(exes=exes)} diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index a52d787f02..8ce9d81120 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -164,33 +164,56 @@ def _compiler_config_from_package_config(config): def _compiler_config_from_external(config): + extra_attributes_key = "extra_attributes" + compilers_key = "compilers" + c_key, cxx_key, fortran_key = "c", "cxx", "fortran" + + # Allow `@x.y.z` instead of `@=x.y.z` spec = spack.spec.parse_with_version_concrete(config["spec"]) - # use str(spec.versions) to allow `@x.y.z` instead of `@=x.y.z` + compiler_spec = spack.spec.CompilerSpec( package_name_to_compiler_name.get(spec.name, spec.name), spec.version ) - extra_attributes = config.get("extra_attributes", {}) - prefix = config.get("prefix", None) + err_header = f"The external spec '{spec}' cannot be used as a compiler" - compiler_class = class_for_compiler_name(compiler_spec.name) - paths = extra_attributes.get("paths", {}) - compiler_langs = ["cc", "cxx", "fc", "f77"] - for lang in compiler_langs: - if paths.setdefault(lang, None): - continue - - if not prefix: - continue - - # Check for files that satisfy the naming scheme for this compiler - bindir = os.path.join(prefix, "bin") - for f, regex in itertools.product(os.listdir(bindir), compiler_class.search_regexps(lang)): - if regex.match(f): - paths[lang] = os.path.join(bindir, f) - - if all(v is None for v in paths.values()): + # If extra_attributes is not there I might not want to use this entry as a compiler, + # therefore just leave a debug message, but don't be loud with a warning. + if extra_attributes_key not in config: + tty.debug(f"[{__file__}] {err_header}: missing the '{extra_attributes_key}' key") return None + extra_attributes = config[extra_attributes_key] + + # If I have 'extra_attributes' warn if 'compilers' is missing, or we don't have a C compiler + if compilers_key not in extra_attributes: + warnings.warn( + f"{err_header}: missing the '{compilers_key}' key under '{extra_attributes_key}'" + ) + return None + attribute_compilers = extra_attributes[compilers_key] + + if c_key not in attribute_compilers: + warnings.warn( + f"{err_header}: missing the C compiler path under " + f"'{extra_attributes_key}:{compilers_key}'" + ) + return None + c_compiler = attribute_compilers[c_key] + + # C++ and Fortran compilers are not mandatory, so let's just leave a debug trace + if cxx_key not in attribute_compilers: + tty.debug(f"[{__file__}] The external spec {spec} does not have a C++ compiler") + + if fortran_key not in attribute_compilers: + tty.debug(f"[{__file__}] The external spec {spec} does not have a Fortran compiler") + + # compilers format has cc/fc/f77, externals format has "c/fortran" + paths = { + "cc": c_compiler, + "cxx": attribute_compilers.get(cxx_key, None), + "fc": attribute_compilers.get(fortran_key, None), + "f77": attribute_compilers.get(fortran_key, None), + } if not spec.architecture: host_platform = spack.platforms.host() diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index ac56cf1e1a..d0b7beda1d 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -39,6 +39,7 @@ ) from spack.build_systems.cargo import CargoPackage from spack.build_systems.cmake import CMakePackage, generator +from spack.build_systems.compiler import CompilerPackage from spack.build_systems.cuda import CudaPackage from spack.build_systems.generic import Package from spack.build_systems.gnu import GNUMirrorPackage diff --git a/lib/spack/spack/test/cmd/compiler.py b/lib/spack/spack/test/cmd/compiler.py index 150fd1af54..2fde7fbc92 100644 --- a/lib/spack/spack/test/cmd/compiler.py +++ b/lib/spack/spack/test/cmd/compiler.py @@ -261,15 +261,14 @@ def test_compiler_list_empty(no_compilers_yaml, working_env, compilers_dir): [ ( { - "spec": "gcc@=7.7.7 os=foobar target=x86_64", + "spec": "gcc@=7.7.7 languages=c,cxx,fortran os=foobar target=x86_64", "prefix": "/path/to/fake", "modules": ["gcc/7.7.7", "foobar"], "extra_attributes": { - "paths": { - "cc": "/path/to/fake/gcc", + "compilers": { + "c": "/path/to/fake/gcc", "cxx": "/path/to/fake/g++", - "fc": "/path/to/fake/gfortran", - "f77": "/path/to/fake/gfortran", + "fortran": "/path/to/fake/gfortran", }, "flags": {"fflags": "-ffree-form"}, }, @@ -285,26 +284,7 @@ def test_compiler_list_empty(no_compilers_yaml, working_env, compilers_dir): \tmodules = ['gcc/7.7.7', 'foobar'] \toperating system = foobar """, - ), - ( - { - "spec": "gcc@7.7.7", - "prefix": "{prefix}", - "modules": ["gcc/7.7.7", "foobar"], - "extra_attributes": {"flags": {"fflags": "-ffree-form"}}, - }, - """gcc@7.7.7: -\tpaths: -\t\tcc = {compilers_dir}{sep}gcc-8{suffix} -\t\tcxx = {compilers_dir}{sep}g++-8{suffix} -\t\tf77 = {compilers_dir}{sep}gfortran-8{suffix} -\t\tfc = {compilers_dir}{sep}gfortran-8{suffix} -\tflags: -\t\tfflags = ['-ffree-form'] -\tmodules = ['gcc/7.7.7', 'foobar'] -\toperating system = debian6 -""", - ), + ) ], ) def test_compilers_shows_packages_yaml( diff --git a/lib/spack/spack/test/conftest.py b/lib/spack/spack/test/conftest.py index 59d6171e3d..29c10fb2e3 100644 --- a/lib/spack/spack/test/conftest.py +++ b/lib/spack/spack/test/conftest.py @@ -1694,7 +1694,7 @@ def mock_executable(tmp_path): """Factory to create a mock executable in a temporary directory that output a custom string when run. """ - shebang = "#!/bin/sh\n" if sys.platform != "win32" else "@ECHO OFF" + shebang = "#!/bin/sh\n" if sys.platform != "win32" else "@ECHO OFF\n" def _factory(name, output, subdir=("bin",)): executable_dir = tmp_path.joinpath(*subdir) diff --git a/var/spack/repos/builtin/packages/acfl/detection_test.yaml b/var/spack/repos/builtin/packages/acfl/detection_test.yaml new file mode 100644 index 0000000000..23fc1930f2 --- /dev/null +++ b/var/spack/repos/builtin/packages/acfl/detection_test.yaml @@ -0,0 +1,83 @@ +paths: +- layout: + - executables: + - "bin/armclang" + - "bin/armclang++" + - "bin/armflang" + script: | + echo "Arm C/C++/Fortran Compiler version 19.0 (build number 73) (based on LLVM 7.0.2)" + echo "Target: aarch64--linux-gnu" + echo "Thread model: posix" + echo "InstalledDir:" + echo "/opt/arm/arm-hpc-compiler-19.0_Generic-AArch64_RHEL-7_aarch64-linux/bin" + platforms: [linux] + results: + - spec: acfl@19.0 + extra_attributes: + compilers: + c: ".*/bin/armclang" + cxx: ".*/bin/armclang[+][+]" + fortran: ".*/bin/armflang" +- layout: + - executables: + - "bin/armclang" + - "bin/armclang++" + script: | + echo "Arm C/C++/Fortran Compiler version 19.0 (build number 73) (based on LLVM 7.0.2)" + echo "Target: aarch64--linux-gnu" + echo "Thread model: posix" + echo "InstalledDir:" + echo "/opt/arm/arm-hpc-compiler-19.0_Generic-AArch64_RHEL-7_aarch64-linux/bin" + platforms: [linux] + results: + - spec: acfl@19.0 + extra_attributes: + compilers: + c: ".*/bin/armclang" + cxx: ".*/bin/armclang[+][+]" +- layout: + - executables: + - "bin/armclang" + - "bin/armclang++" + - "bin/armflang" + script: | + echo "Arm C/C++/Fortran Compiler version 19.3.1 (build number 75) (based on LLVM 7.0.2)" + echo "Target: aarch64--linux-gnu" + echo "Thread model: posix" + echo "InstalledDir:" + echo "/opt/arm/arm-hpc-compiler-19.3.5_Generic-AArch64_RHEL-7_aarch64-linux/bin" + - executables: + - "bin/armclang-18" + - "bin/armclang++-18" + - "bin/armflang-18" + script: | + echo "Arm C/C++/Fortran Compiler version 18.0 (build number 27) (based on LLVM 7.0.0)" + echo "Target: aarch64--linux-gnu" + echo "Thread model: posix" + echo "InstalledDir:" + echo "/opt/arm/arm-hpc-compiler-19_Generic-AArch64_RHEL-7_aarch64-linux/bin" + platforms: [linux] + results: + - spec: acfl@19.3.1 + extra_attributes: + compilers: + c: ".*/bin/armclang$" + cxx: ".*/bin/armclang[+][+]$" + fortran: ".*/bin/armflang$" + - spec: acfl@18.0 + extra_attributes: + compilers: + c: ".*/bin/armclang-18" + cxx: ".*/bin/armclang[+][+]-18" + fortran: ".*/bin/armflang-18" +- layout: # does not detect upstream clang + - executables: + - "bin/clang" + - "bin/clang++" + script: | + echo "clang version 8.0.0 (tags/RELEASE_800/final" + echo "Target: x86_64-unknown-linux-gnu\n" + echo "Thread model: posix\n" + echo "InstalledDir: /usr/bin" + platforms: [linux] + results: [] diff --git a/var/spack/repos/builtin/packages/acfl/package.py b/var/spack/repos/builtin/packages/acfl/package.py index fefab9b574..66d2ddc0b5 100644 --- a/var/spack/repos/builtin/packages/acfl/package.py +++ b/var/spack/repos/builtin/packages/acfl/package.py @@ -3,7 +3,6 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os -import re from spack.package import * @@ -228,7 +227,7 @@ def get_gcc_prefix(spec): return join_path(spec.prefix, next(dir for dir in dirlist if dir.startswith("gcc"))) -class Acfl(Package): +class Acfl(Package, CompilerPackage): """Arm Compiler combines the optimized tools and libraries from Arm with a modern LLVM-based compiler framework. """ @@ -275,33 +274,15 @@ def install(self, spec, prefix): ) exe("--accept", "--force", "--install-to", prefix) - @classmethod - def determine_version(cls, exe): - regex_str = r"Arm C\/C\+\+\/Fortran Compiler version ([\d\.]+) " r"\(build number (\d+)\) " - version_regex = re.compile(regex_str) - try: - output = spack.compiler.get_compiler_version_output(exe, "--version") - match = version_regex.search(output) - if match: - if match.group(1).count(".") == 1: - return match.group(1) + ".0." + match.group(2) - return match.group(1) + "." + match.group(2) - except spack.util.executable.ProcessError: - pass - except Exception as e: - tty.debug(e) + compiler_languages = ["c", "cxx", "fortran"] + c_names = ["armclang"] + cxx_names = ["armclang++"] + fortran_names = ["armflang"] - @classmethod - def determine_variants(cls, exes, version_str): - compilers = {} - for exe in exes: - if "armclang" in exe: - compilers["c"] = exe - if "armclang++" in exe: - compilers["cxx"] = exe - if "armflang" in exe: - compilers["fortran"] = exe - return "", {"compilers": compilers} + compiler_version_argument = "--version" + compiler_version_regex = ( + r"Arm C\/C\+\+\/Fortran Compiler version ([\d\.]+) \(build number \d+\) " + ) @property def cc(self): diff --git a/var/spack/repos/builtin/packages/aocc/detection_test.yaml b/var/spack/repos/builtin/packages/aocc/detection_test.yaml new file mode 100644 index 0000000000..331ba0ba4a --- /dev/null +++ b/var/spack/repos/builtin/packages/aocc/detection_test.yaml @@ -0,0 +1,75 @@ +paths: +- layout: + - executables: + - "bin/clang" + - "bin/clang++" + - "bin/flang" + script: | + echo "AMD clang version 12.0.0 (CLANG: AOCC_3_1_0-Build#126 2021_06_07)(based on LLVM Mirror.Version.12.0.0)\n" + echo "Target: x86_64-unknown-linux-gnu\n" + echo "Thread model: posix\n" + platforms: [linux] + results: + - spec: aocc@3.1.0 + extra_attributes: + compilers: + c: ".*/bin/clang" + cxx: ".*/bin/clang[+][+]" + fortran: ".*/bin/flang" +- layout: + - executables: + - "bin/clang" + - "bin/clang++" + script: | + echo "AMD clang version 12.0.0 (CLANG: AOCC_3_1_0-Build#126 2021_06_07)(based on LLVM Mirror.Version.12.0.0)\n" + echo "Target: x86_64-unknown-linux-gnu\n" + echo "Thread model: posix\n" + platforms: [linux] + results: + - spec: aocc@3.1.0 + extra_attributes: + compilers: + c: ".*/bin/clang" + cxx: ".*/bin/clang[+][+]" +- layout: + - executables: + - "bin/clang" + - "bin/clang++" + - "bin/flang" + script: | + echo "AMD clang version 12.0.0 (CLANG: AOCC_3_0_0-Build#78 2020_12_10)(based on LLVM Mirror.Version.12.0.0)\n" + echo "Target: x86_64-unknown-linux-gnu\n" + echo "Thread model: posix\n" + - executables: + - "bin/clang-11" + - "bin/clang++-11" + - "bin/flang-11" + script: | + echo "AMD clang version 11.0.0 (CLANG: AOCC_2_3_0-Build#85 2020_11_10)(based on LLVM Mirror.Version.11.0.0)\n" + echo "Target: x86_64-unknown-linux-gnu\n" + echo "Thread model: posix\n" + platforms: [linux] + results: + - spec: aocc@3.0.0 + extra_attributes: + compilers: + c: ".*/bin/clang$" + cxx: ".*/bin/clang[+][+]$" + fortran: ".*/bin/flang$" + - spec: aocc@2.3.0 + extra_attributes: + compilers: + c: ".*/bin/clang-11" + cxx: ".*/bin/clang[+][+]-11" + fortran: ".*/bin/flang-11" +- layout: + - executables: + - "bin/clang" + - "bin/clang++" + script: | + echo "clang version 8.0.0 (tags/RELEASE_800/final" + echo "Target: x86_64-unknown-linux-gnu\n" + echo "Thread model: posix\n" + echo "InstalledDir: /usr/bin" + platforms: [linux] + results: [] diff --git a/var/spack/repos/builtin/packages/aocc/package.py b/var/spack/repos/builtin/packages/aocc/package.py index f51b5d6795..f2451d1f63 100644 --- a/var/spack/repos/builtin/packages/aocc/package.py +++ b/var/spack/repos/builtin/packages/aocc/package.py @@ -8,7 +8,7 @@ from spack.package import * -class Aocc(Package): +class Aocc(Package, CompilerPackage): """ The AOCC compiler system is a high performance, production quality code generation tool. The AOCC environment provides various options to developers @@ -104,3 +104,9 @@ def cfg_files(self): for compiler in ["clang", "clang++"]: with open(join_path(self.prefix.bin, "{}.cfg".format(compiler)), "w") as f: f.write(compiler_options) + + compiler_version_argument = "--version" + compiler_version_regex = r"AOCC_(\d+[._]\d+[._]\d+)" + c_names = ["clang"] + cxx_names = ["clang++"] + fortran_names = ["flang"] diff --git a/var/spack/repos/builtin/packages/apple-clang/detection_test.yaml b/var/spack/repos/builtin/packages/apple-clang/detection_test.yaml index dffdf88286..1e52329116 100644 --- a/var/spack/repos/builtin/packages/apple-clang/detection_test.yaml +++ b/var/spack/repos/builtin/packages/apple-clang/detection_test.yaml @@ -1,38 +1,48 @@ paths: - # Apple Clang on MacBook Pro (Catalina) - - layout: - - executables: - - "bin/clang" - - "bin/clang++" - script: | - echo "Apple clang version 11.0.0 (clang-1100.0.33.8)" - echo "Target: x86_64-apple-darwin19.5.0" - echo "Thread model: posix" - echo "InstalledDir: /Library/Developer/CommandLineTools/usr/bin" - platforms: ["darwin"] - results: - - spec: 'apple-clang@11.0.0' - # Apple Clang on Apple M1 (Ventura) - - layout: - - executables: - - "bin/clang" - - "bin/clang++" - script: | - echo "Apple clang version 15.0.0 (clang-1500.0.40.1)" - echo "Target: arm64-apple-darwin22.6.0" - echo "Thread model: posix" - echo "InstalledDir: /Library/Developer/CommandLineTools/usr/bin" - platforms: ["darwin"] - results: - - spec: 'apple-clang@15.0.0' - # Test that missing a compiler prevents the package from being detected - - layout: - - executables: - - "bin/clang" - script: | - echo "Apple clang version 11.0.0 (clang-1100.0.33.8)" - echo "Target: x86_64-apple-darwin19.5.0" - echo "Thread model: posix" - echo "InstalledDir: /Library/Developer/CommandLineTools/usr/bin" - platforms: ["darwin"] - results: [ ] +# Apple Clang on MacBook Pro (Catalina) +- layout: + - executables: + - "bin/clang" + - "bin/clang++" + script: | + echo "Apple clang version 11.0.0 (clang-1100.0.33.8)" + echo "Target: x86_64-apple-darwin19.5.0" + echo "Thread model: posix" + echo "InstalledDir: /Library/Developer/CommandLineTools/usr/bin" + platforms: ["darwin"] + results: + - spec: 'apple-clang@11.0.0' + extra_attributes: + compilers: + c: ".*/bin/clang" + cxx: ".*/bin/clang[+][+]" + +# Apple Clang on Apple M1 (Ventura) +- layout: + - executables: + - "bin/clang" + - "bin/clang++" + script: | + echo "Apple clang version 15.0.0 (clang-1500.0.40.1)" + echo "Target: arm64-apple-darwin22.6.0" + echo "Thread model: posix" + echo "InstalledDir: /Library/Developer/CommandLineTools/usr/bin" + platforms: ["darwin"] + results: + - spec: 'apple-clang@15.0.0' + extra_attributes: + compilers: + c: ".*/bin/clang" + cxx: ".*/bin/clang[+][+]" + +# Test that missing a compiler prevents the package from being detected +- layout: + - executables: + - "bin/clang" + script: | + echo "Apple clang version 11.0.0 (clang-1100.0.33.8)" + echo "Target: x86_64-apple-darwin19.5.0" + echo "Thread model: posix" + echo "InstalledDir: /Library/Developer/CommandLineTools/usr/bin" + platforms: ["darwin"] + results: [] diff --git a/var/spack/repos/builtin/packages/apple-clang/package.py b/var/spack/repos/builtin/packages/apple-clang/package.py index 634aedf4eb..95c3eb54c0 100644 --- a/var/spack/repos/builtin/packages/apple-clang/package.py +++ b/var/spack/repos/builtin/packages/apple-clang/package.py @@ -2,12 +2,10 @@ # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) -import re - from spack.package import * -class AppleClang(BundlePackage): +class AppleClang(BundlePackage, CompilerPackage): """Apple's Clang compiler""" homepage = "https://developer.apple.com/videos/developer-tools/compiler-and-llvm" @@ -15,44 +13,12 @@ class AppleClang(BundlePackage): maintainers("alalazo") - executables = ["^clang$", r"^clang\+\+$", "^ld.lld$", "^lldb$"] + compiler_languages = ["c", "cxx"] + c_names = ["clang"] + cxx_names = ["clang++"] - @classmethod - def determine_version(cls, exe): - version_regex = re.compile( - # Apple's LLVM compiler has its own versions, which are - # different from vanilla LLVM - r"^Apple (?:LLVM|clang) version ([^ )]+)", - # Multi-line, since 'Apple clang' may not be on the first line - # in particular, when run as gcc, it seems to output - # "Configured with: --prefix=..." as the first line - re.M, - ) - try: - compiler = Executable(exe) - output = compiler("--version", output=str, error=str) - match = version_regex.search(output) - if match: - return match.group(match.lastindex) - except Exception: - pass - - return None - - @classmethod - def determine_variants(cls, exes, version_str): - compilers = {} - for exe in exes: - if "clang++" in exe: - compilers["cxx"] = exe - elif "clang" in exe: - compilers["c"] = exe - elif "ld.lld" in exe: - compilers["ld"] = exe - elif "lldb" in exe: - compilers["lldb"] = exe - - return "", {"compilers": compilers} + compiler_version_regex = r"^Apple (?:LLVM|clang) version ([^ )]+)" + compiler_version_argument = "--version" @classmethod def validate_detected_spec(cls, spec, extra_attributes): diff --git a/var/spack/repos/builtin/packages/cce/detection_test.yaml b/var/spack/repos/builtin/packages/cce/detection_test.yaml new file mode 100644 index 0000000000..b425ab439a --- /dev/null +++ b/var/spack/repos/builtin/packages/cce/detection_test.yaml @@ -0,0 +1,44 @@ +paths: +- layout: + - executables: + - "bin/craycc" + script: | + echo "Cray C : Version 8.4.6 Mon Apr 15, 2019 12:13:39" + - executables: + - "bin/crayCC" + script: | + echo "Cray C++ : Version 8.4.6 Mon Apr 15, 2019 12:13:39" + - executables: + - "bin/crayftn" + script: | + echo "Cray Fortran : Version 8.4.6 Mon Apr 15, 2019 12:13:39" + platforms: [linux] + results: + - spec: cce@8.4.6 + extra_attributes: + compilers: + c: ".*/bin/craycc" + cxx: ".*/bin/crayCC" + fortran: ".*/bin/crayftn" +- layout: + - executables: + - "bin/craycc" + - "bin/crayCC" + script: | + echo "Cray clang version 17.0.1 (5ec9405551a8c8845cf14e81dc28bff7aa3935cb)" + echo "Target: x86_64-unknown-linux-gnu" + echo "Thread model: posix" + echo "InstalledDir: /opt/cray/pe/cce/17.0.1/cce-clang/x86_64/share/../bin" + echo "Configuration file: /opt/cray/pe/cce/17.0.1/cce-clang/x86_64/bin/clang.cfg" + - executables: + - "bin/crayftn" + script: | + echo "Cray Fortran : Version 17.0.1" + platforms: [linux] + results: + - spec: cce@17.0.1 + extra_attributes: + compilers: + c: ".*/bin/craycc" + cxx: ".*/bin/crayCC" + fortran: ".*/bin/crayftn" diff --git a/var/spack/repos/builtin/packages/cce/package.py b/var/spack/repos/builtin/packages/cce/package.py new file mode 100644 index 0000000000..0772be6ab1 --- /dev/null +++ b/var/spack/repos/builtin/packages/cce/package.py @@ -0,0 +1,30 @@ +# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +from spack.package import * + + +class Cce(Package, CompilerPackage): + """Stub package for external detection of the Cray compiler package.""" + + homepage = "https://cpe.ext.hpe.com/docs/cce/index.html" + url = "https://cpe.ext.hpe.com/docs/cce/index.html" + + compiler_languages = ["c", "cxx", "fortran"] + c_names = ["craycc"] + cxx_names = ["crayCC"] + fortran_names = ["crayftn"] + + compiler_version_argument = "--version" + compiler_version_regex = ( + r"[Cc]ray (?:clang|C :|C\+\+ :|Fortran :) [Vv]ersion.*?(\d+(?:\.\d+)+)" + ) + + # notify when the package is updated. + maintainers("becker33") + + version("16.0.0") + + def install(self, spec, prefix): + raise NotImplementedError("cray compiler must be configured as external") diff --git a/var/spack/repos/builtin/packages/fj/detection_test.yaml b/var/spack/repos/builtin/packages/fj/detection_test.yaml new file mode 100644 index 0000000000..9159d20d91 --- /dev/null +++ b/var/spack/repos/builtin/packages/fj/detection_test.yaml @@ -0,0 +1,27 @@ +paths: +- layout: + - executables: + - "bin/fcc" + script: | + echo "fcc (FCC) 4.0.0a 20190314" + echo "simulating gcc version 6.1" + echo "Copyright FUJITSU LIMITED 2019" + - executables: + - "bin/FCC" + script: | + echo "FCC (FCC) 4.0.0a 20190314" + echo "simulating gcc version 6.1" + echo "Copyright FUJITSU LIMITED 2019" + - executables: + - "bin/frt" + script: | + echo "frt (FRT) 4.0.0a 20190314" + echo "Copyright FUJITSU LIMITED 2019" + platforms: [linux] + results: + - spec: fj@4.0.0a + extra_attributes: + compilers: + c: ".*/bin/fcc" + cxx: ".*/bin/FCC" + fortran: ".*/bin/frt" diff --git a/var/spack/repos/builtin/packages/fj/package.py b/var/spack/repos/builtin/packages/fj/package.py index 899f85aa78..9ebcecab2e 100644 --- a/var/spack/repos/builtin/packages/fj/package.py +++ b/var/spack/repos/builtin/packages/fj/package.py @@ -2,16 +2,10 @@ # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) -import re - -import llnl.util.tty as tty - -import spack.compiler -import spack.util.executable from spack.package import * -class Fj(Package): +class Fj(Package, CompilerPackage): """The Fujitsu compiler system is a high performance, production quality code generation tool designed for high performance parallel computing workloads. @@ -27,29 +21,9 @@ def install(self, spec, prefix): "detected on a system where they are supplied by vendor" ) - executables = ["^fcc", "^FCC", "^frt"] - - @classmethod - def determine_version(cls, exe): - version_regex = re.compile(r"\((?:FCC|FRT)\) ([a-z\d.]+)") - try: - output = spack.compiler.get_compiler_version_output(exe, "--version") - match = version_regex.search(output) - if match: - return match.group(1) - except spack.util.executable.ProcessError: - pass - except Exception as e: - tty.debug(e) - - @classmethod - def determine_variants(cls, exes, version_str): - compilers = {} - for exe in exes: - if "fcc" in exe: - compilers["c"] = exe - if "FCC" in exe: - compilers["cxx"] = exe - if "frt" in exe: - compilers["fortran"] = exe - return "", {"compilers": compilers} + compiler_languages = ["c", "cxx", "fortran"] + c_names = ["fcc"] + cxx_names = ["FCC"] + fortran_names = ["frt"] + compiler_version_regex = r"\((?:FCC|FRT)\) ([a-z\d.]+)" + compiler_version_argument = "--version" diff --git a/var/spack/repos/builtin/packages/gcc/detection_test.yaml b/var/spack/repos/builtin/packages/gcc/detection_test.yaml index 4677a8cdab..7269c0301c 100644 --- a/var/spack/repos/builtin/packages/gcc/detection_test.yaml +++ b/var/spack/repos/builtin/packages/gcc/detection_test.yaml @@ -1,93 +1,96 @@ paths: - # Ubuntu 20.04, system compilers without Fortran. This - # test also covers which flags are expected to be used - # during the detection of gcc. - - layout: - - executables: - - "bin/gcc" - - "bin/g++" - script: | - if [ "$1" = "-dumpversion" ] ; then - echo "9" - elif [ "$1" = "-dumpfullversion" ] ; then - echo "9.4.0" - elif [ "$1" = "--version" ] ; then - echo "gcc (Ubuntu 9.4.0-1ubuntu1~20.04.2) 9.4.0" - echo "Copyright (C) 2019 Free Software Foundation, Inc." - echo "This is free software; see the source for copying conditions. There is NO" - echo "warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE." - else - echo "mock executable got an unexpected flag: $1" - exit 1 - fi - platforms: ["darwin", "linux"] - results: - - spec: "gcc@9.4.0 languages=c,c++" - extra_attributes: - compilers: - c: ".*/bin/gcc" - cxx: ".*/bin/g++" - # Mock a version < 7 of GCC that requires -dumpversion and - # errors with -dumpfullversion - - layout: - - executables: - - "bin/gcc-5" - - "bin/g++-5" - - "bin/gfortran-5" - script: | - if [ "$1" = "-dumpversion" ] ; then - echo "5.5.0" - else - echo "gcc-5: fatal error: no input files" - echo "compilation terminated." - exit 1 - fi - platforms: ["darwin", "linux"] - results: - - spec: "gcc@5.5.0 languages=c,c++,fortran" - extra_attributes: - compilers: - c: ".*/bin/gcc-5$" - cxx: ".*/bin/g[+][+]-5$" - fortran: ".*/bin/gfortran-5$" - # Multiple compilers present at the same time - - layout: - - executables: - - "bin/x86_64-linux-gnu-gcc-6" - script: 'echo 6.5.0' - - executables: - - "bin/x86_64-linux-gnu-gcc-10" - - "bin/x86_64-linux-gnu-g++-10" - script: "echo 10.1.0" - platforms: [darwin, linux] - results: - - spec: "gcc@6.5.0 languages=c" - extra_attributes: - compilers: - c: ".*/bin/x86_64-linux-gnu-gcc-6$" - - spec: "gcc@10.1.0 languages=c,c++" - extra_attributes: - compilers: - c: ".*/bin/x86_64-linux-gnu-gcc-10$" - cxx: ".*/bin/x86_64-linux-gnu-g[+][+]-10$" - # Apple clang under disguise as gcc should not be detected - - layout: - - executables: - - "bin/gcc" - script: | - if [ "$1" = "-dumpversion" ] ; then - echo "15.0.0" - elif [ "$1" = "-dumpfullversion" ] ; then - echo "clang: error: no input files" >&2 - exit 1 - elif [ "$1" = "--version" ] ; then - echo "Apple clang version 15.0.0 (clang-1500.3.9.4)" - echo "Target: x86_64-apple-darwin23.4.0" - echo "Thread model: posix" - echo "InstalledDir: /Library/Developer/CommandLineTools/usr/bin" - else - echo "mock executable got an unexpected flag: $1" - exit 1 - fi - platforms: ["darwin"] - results: [] +# Ubuntu 20.04, system compilers without Fortran. This +# test also covers which flags are expected to be used +# during the detection of gcc. +- layout: + - executables: + - "bin/gcc" + - "bin/g++" + script: | + if [ "$1" = "-dumpversion" ] ; then + echo "9" + elif [ "$1" = "-dumpfullversion" ] ; then + echo "9.4.0" + elif [ "$1" = "--version" ] ; then + echo "gcc (Ubuntu 9.4.0-1ubuntu1~20.04.2) 9.4.0" + echo "Copyright (C) 2019 Free Software Foundation, Inc." + echo "This is free software; see the source for copying conditions. There is NO" + echo "warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE." + else + echo "mock executable got an unexpected flag: $1" + exit 1 + fi + platforms: ["darwin", "linux"] + results: + - spec: "gcc@9.4.0 languages=c,c++" + extra_attributes: + compilers: + c: ".*/bin/gcc" + cxx: ".*/bin/g++" + +# Mock a version < 7 of GCC that requires -dumpversion and +# errors with -dumpfullversion +- layout: + - executables: + - "bin/gcc-5" + - "bin/g++-5" + - "bin/gfortran-5" + script: | + if [ "$1" = "-dumpversion" ] ; then + echo "5.5.0" + else + echo "gcc-5: fatal error: no input files" + echo "compilation terminated." + exit 1 + fi + platforms: ["darwin", "linux"] + results: + - spec: "gcc@5.5.0 languages=c,c++,fortran" + extra_attributes: + compilers: + c: ".*/bin/gcc-5$" + cxx: ".*/bin/g[+][+]-5$" + fortran: ".*/bin/gfortran-5$" + +# Multiple compilers present at the same time +- layout: + - executables: + - "bin/x86_64-linux-gnu-gcc-6" + script: 'echo 6.5.0' + - executables: + - "bin/x86_64-linux-gnu-gcc-10" + - "bin/x86_64-linux-gnu-g++-10" + script: "echo 10.1.0" + platforms: [darwin, linux] + results: + - spec: "gcc@6.5.0 languages=c" + extra_attributes: + compilers: + c: ".*/bin/x86_64-linux-gnu-gcc-6$" + - spec: "gcc@10.1.0 languages=c,c++" + extra_attributes: + compilers: + c: ".*/bin/x86_64-linux-gnu-gcc-10$" + cxx: ".*/bin/x86_64-linux-gnu-g[+][+]-10$" + +# Apple clang under disguise as gcc should not be detected +- layout: + - executables: + - "bin/gcc" + script: | + if [ "$1" = "-dumpversion" ] ; then + echo "15.0.0" + elif [ "$1" = "-dumpfullversion" ] ; then + echo "clang: error: no input files" >&2 + exit 1 + elif [ "$1" = "--version" ] ; then + echo "Apple clang version 15.0.0 (clang-1500.3.9.4)" + echo "Target: x86_64-apple-darwin23.4.0" + echo "Thread model: posix" + echo "InstalledDir: /Library/Developer/CommandLineTools/usr/bin" + else + echo "mock executable got an unexpected flag: $1" + exit 1 + fi + platforms: ["darwin"] + results: [] diff --git a/var/spack/repos/builtin/packages/gcc/package.py b/var/spack/repos/builtin/packages/gcc/package.py index 0ed2f2f3b8..f10ff1623a 100644 --- a/var/spack/repos/builtin/packages/gcc/package.py +++ b/var/spack/repos/builtin/packages/gcc/package.py @@ -5,13 +5,11 @@ import glob import itertools import os -import re import sys from archspec.cpu import UnsupportedMicroarchitecture import llnl.util.tty as tty -from llnl.util.lang import classproperty from llnl.util.symlink import readlink import spack.platforms @@ -21,7 +19,7 @@ from spack.package import * -class Gcc(AutotoolsPackage, GNUMirrorPackage): +class Gcc(AutotoolsPackage, GNUMirrorPackage, CompilerPackage): """The GNU Compiler Collection includes front ends for C, C++, Objective-C, Fortran, Ada, and Go, as well as libraries for these languages.""" @@ -503,11 +501,36 @@ class Gcc(AutotoolsPackage, GNUMirrorPackage): build_directory = "spack-build" - @classproperty - def executables(cls): - names = [r"gcc", r"[^\w]?g\+\+", r"gfortran", r"gdc", r"gccgo"] - suffixes = [r"", r"-mp-\d+\.\d", r"-\d+\.\d", r"-\d+", r"\d\d"] - return [r"".join(x) for x in itertools.product(names, suffixes)] + compiler_languages = ["c", "cxx", "fortran", "d", "go"] + + @property + def supported_languages(self): + # This weirdness is because it could be called on an abstract spec + if "languages" not in self.spec.variants: + return self.compiler_languages + return [x for x in self.compiler_languages if x in self.spec.variants["languages"].value] + + c_names = ["gcc"] + cxx_names = ["g++"] + fortran_names = ["gfortran"] + d_names = ["gdc"] + go_names = ["gccgo"] + compiler_prefixes = [r"\w+-\w+-\w+-"] + compiler_suffixes = [r"-mp-\d+(?:\.\d+)?", r"-\d+(?:\.\d+)?", r"\d\d"] + compiler_version_regex = r"(?