From 80389911cce2daae5b0e4df069a3be9575954dc3 Mon Sep 17 00:00:00 2001 From: Harmen Stoppels Date: Fri, 2 Sep 2022 12:48:46 +0200 Subject: [PATCH] Update bootstrap buildcache to v0.3 (#32262) This release allow to bootstrap patchelf from binaries. --- etc/spack/defaults/bootstrap.yaml | 4 +- lib/spack/spack/bootstrap.py | 51 +++++++++++++++---- lib/spack/spack/relocate.py | 8 +-- .../bootstrap/github-actions-v0.3/clingo.json | 1 + .../bootstrap/github-actions-v0.3/gnupg.json | 1 + .../github-actions-v0.3/metadata.yaml | 8 +++ .../github-actions-v0.3/patchelf.json | 34 +++++++++++++ 7 files changed, 91 insertions(+), 16 deletions(-) create mode 120000 share/spack/bootstrap/github-actions-v0.3/clingo.json create mode 120000 share/spack/bootstrap/github-actions-v0.3/gnupg.json create mode 100644 share/spack/bootstrap/github-actions-v0.3/metadata.yaml create mode 100644 share/spack/bootstrap/github-actions-v0.3/patchelf.json diff --git a/etc/spack/defaults/bootstrap.yaml b/etc/spack/defaults/bootstrap.yaml index b3ab1c99df..a4a9b23515 100644 --- a/etc/spack/defaults/bootstrap.yaml +++ b/etc/spack/defaults/bootstrap.yaml @@ -9,6 +9,8 @@ bootstrap: # may not be able to bootstrap all the software that Spack needs, # depending on its type. sources: + - name: 'github-actions-v0.3' + metadata: $spack/share/spack/bootstrap/github-actions-v0.3 - name: 'github-actions-v0.2' metadata: $spack/share/spack/bootstrap/github-actions-v0.2 - name: 'github-actions-v0.1' @@ -18,5 +20,5 @@ bootstrap: trusted: # By default we trust bootstrapping from sources and from binaries # produced on Github via the workflow - github-actions-v0.2: true + github-actions-v0.3: true spack-install: true diff --git a/lib/spack/spack/bootstrap.py b/lib/spack/spack/bootstrap.py index 4acb8aa6f8..6951ad4946 100644 --- a/lib/spack/spack/bootstrap.py +++ b/lib/spack/spack/bootstrap.py @@ -15,6 +15,7 @@ import re import sys import sysconfig +import uuid import six @@ -40,6 +41,7 @@ import spack.util.path import spack.util.spack_yaml import spack.util.url +import spack.version #: Name of the file containing metadata about the bootstrapping source METADATA_YAML_FILENAME = "metadata.yaml" @@ -260,12 +262,11 @@ def mirror_scope(self): class _BuildcacheBootstrapper(_BootstrapperBase): """Install the software needed during bootstrapping from a buildcache.""" - config_scope_name = "bootstrap_buildcache" - def __init__(self, conf): super(_BuildcacheBootstrapper, self).__init__(conf) self.metadata_dir = spack.util.path.canonicalize_path(conf["metadata"]) self.last_search = None + self.config_scope_name = "bootstrap_buildcache-{}".format(uuid.uuid4()) @staticmethod def _spec_and_platform(abstract_spec_str): @@ -378,13 +379,12 @@ def try_search_path(self, executables, abstract_spec_str): class _SourceBootstrapper(_BootstrapperBase): """Install the software needed during bootstrapping from sources.""" - config_scope_name = "bootstrap_source" - def __init__(self, conf): super(_SourceBootstrapper, self).__init__(conf) self.metadata_dir = spack.util.path.canonicalize_path(conf["metadata"]) self.conf = conf self.last_search = None + self.config_scope_name = "bootstrap_source-{}".format(uuid.uuid4()) def try_import(self, module, abstract_spec_str): info = {} @@ -788,17 +788,46 @@ def ensure_gpg_in_path_or_raise(): def patchelf_root_spec(): """Return the root spec used to bootstrap patchelf""" - # TODO: patchelf is restricted to v0.13 since earlier versions have - # TODO: bugs that we don't to deal with, while v0.14 requires a C++17 - # TODO: which may not be available on all platforms. - return _root_spec("patchelf@0.13.1:0.13.99") + # 0.13.1 is the last version not to require C++17. + return _root_spec("patchelf@0.13.1:") + + +def verify_patchelf(patchelf): + """Older patchelf versions can produce broken binaries, so we + verify the version here. + + Arguments: + + patchelf (spack.util.executable.Executable): patchelf executable + """ + out = patchelf("--version", output=str, error=os.devnull, fail_on_error=False).strip() + if patchelf.returncode != 0: + return False + parts = out.split(" ") + if len(parts) < 2: + return False + try: + version = spack.version.Version(parts[1]) + except ValueError: + return False + return version >= spack.version.Version("0.13.1") def ensure_patchelf_in_path_or_raise(): """Ensure patchelf is in the PATH or raise.""" - return ensure_executables_in_path_or_raise( - executables=["patchelf"], abstract_spec=patchelf_root_spec() - ) + # The old concretizer is not smart and we're doing its job: if the latest patchelf + # does not concretize because the compiler doesn't support C++17, we try to + # concretize again with an upperbound @:13. + try: + return ensure_executables_in_path_or_raise( + executables=["patchelf"], abstract_spec=patchelf_root_spec(), cmd_check=verify_patchelf + ) + except RuntimeError: + return ensure_executables_in_path_or_raise( + executables=["patchelf"], + abstract_spec=_root_spec("patchelf@0.13.1:0.13"), + cmd_check=verify_patchelf, + ) ### diff --git a/lib/spack/spack/relocate.py b/lib/spack/spack/relocate.py index e2d362fe05..8212093a12 100644 --- a/lib/spack/spack/relocate.py +++ b/lib/spack/spack/relocate.py @@ -13,6 +13,7 @@ import llnl.util.lang import llnl.util.tty as tty +from llnl.util.lang import memoized from llnl.util.symlink import symlink import spack.bootstrap @@ -76,15 +77,14 @@ def __init__(self, old_path, new_path): super(BinaryTextReplaceError, self).__init__(msg, err_msg) +@memoized def _patchelf(): """Return the full path to the patchelf binary, if available, else None.""" if is_macos: return None - patchelf = executable.which("patchelf") - if patchelf is None: - with spack.bootstrap.ensure_bootstrap_configuration(): - patchelf = spack.bootstrap.ensure_patchelf_in_path_or_raise() + with spack.bootstrap.ensure_bootstrap_configuration(): + patchelf = spack.bootstrap.ensure_patchelf_in_path_or_raise() return patchelf.path diff --git a/share/spack/bootstrap/github-actions-v0.3/clingo.json b/share/spack/bootstrap/github-actions-v0.3/clingo.json new file mode 120000 index 0000000000..049ba5f7ce --- /dev/null +++ b/share/spack/bootstrap/github-actions-v0.3/clingo.json @@ -0,0 +1 @@ +../github-actions-v0.2/clingo.json \ No newline at end of file diff --git a/share/spack/bootstrap/github-actions-v0.3/gnupg.json b/share/spack/bootstrap/github-actions-v0.3/gnupg.json new file mode 120000 index 0000000000..1d6273a6d9 --- /dev/null +++ b/share/spack/bootstrap/github-actions-v0.3/gnupg.json @@ -0,0 +1 @@ +../github-actions-v0.2/gnupg.json \ No newline at end of file diff --git a/share/spack/bootstrap/github-actions-v0.3/metadata.yaml b/share/spack/bootstrap/github-actions-v0.3/metadata.yaml new file mode 100644 index 0000000000..d27e261721 --- /dev/null +++ b/share/spack/bootstrap/github-actions-v0.3/metadata.yaml @@ -0,0 +1,8 @@ +type: buildcache +description: | + Buildcache generated from a public workflow using Github Actions. + The sha256 checksum of binaries is checked before installation. +info: + url: https://mirror.spack.io/bootstrap/github-actions/v0.3 + homepage: https://github.com/spack/spack-bootstrap-mirrors + releases: https://github.com/spack/spack-bootstrap-mirrors/releases diff --git a/share/spack/bootstrap/github-actions-v0.3/patchelf.json b/share/spack/bootstrap/github-actions-v0.3/patchelf.json new file mode 100644 index 0000000000..699c51c8ab --- /dev/null +++ b/share/spack/bootstrap/github-actions-v0.3/patchelf.json @@ -0,0 +1,34 @@ +{ + "verified": [ + { + "binaries": [ + [ + "patchelf", + "cn4gsqzdnnffk7ynvbcai6wrt5ehqqrl", + "8c6a28cbe8133d719be27ded11159f0aa2c97ed1d0881119ae0ebd71f8ccc755" + ] + ], + "spec": "patchelf@0.13: %gcc platform=linux target=aarch64" + }, + { + "binaries": [ + [ + "patchelf", + "mgq6n2heyvcx2ebdpchkbknwwn3u63s6", + "1d4ea9167fb8345a178c1352e0377cc37ef2b421935cf2b48fb6fa03a94fca3d" + ] + ], + "spec": "patchelf@0.13: %gcc platform=linux target=ppc64le" + }, + { + "binaries": [ + [ + "patchelf", + "htk62k7efo2z22kh6kmhaselru7bfkuc", + "833df21b20eaa7999ac4c5779ae26aa90397d9027aebaa686a428589befda693" + ] + ], + "spec": "patchelf@0.13: %gcc platform=linux target=x86_64" + } + ] +} \ No newline at end of file