Update bootstrap buildcache to v0.3 (#32262)

This release allow to bootstrap patchelf from binaries.

etc/spack/defaults/bootstrap.yaml

@@ -9,6 +9,8 @@ bootstrap:
   # may not be able to bootstrap all the software that Spack needs,
   # depending on its type.
   sources:
+  - name: 'github-actions-v0.3'
+    metadata: $spack/share/spack/bootstrap/github-actions-v0.3
   - name: 'github-actions-v0.2'
     metadata: $spack/share/spack/bootstrap/github-actions-v0.2
   - name: 'github-actions-v0.1'
@@ -18,5 +20,5 @@ bootstrap:
   trusted:
     # By default we trust bootstrapping from sources and from binaries
     # produced on Github via the workflow
-    github-actions-v0.2: true
+    github-actions-v0.3: true
     spack-install: true

lib/spack/spack/bootstrap.py

@@ -15,6 +15,7 @@
 import re
 import sys
 import sysconfig
+import uuid
 
 import six
 
@@ -40,6 +41,7 @@
 import spack.util.path
 import spack.util.spack_yaml
 import spack.util.url
+import spack.version
 
 #: Name of the file containing metadata about the bootstrapping source
 METADATA_YAML_FILENAME = "metadata.yaml"
@@ -260,12 +262,11 @@ def mirror_scope(self):
 class _BuildcacheBootstrapper(_BootstrapperBase):
     """Install the software needed during bootstrapping from a buildcache."""
 
-    config_scope_name = "bootstrap_buildcache"
-
     def __init__(self, conf):
         super(_BuildcacheBootstrapper, self).__init__(conf)
         self.metadata_dir = spack.util.path.canonicalize_path(conf["metadata"])
         self.last_search = None
+        self.config_scope_name = "bootstrap_buildcache-{}".format(uuid.uuid4())
 
     @staticmethod
     def _spec_and_platform(abstract_spec_str):
@@ -378,13 +379,12 @@ def try_search_path(self, executables, abstract_spec_str):
 class _SourceBootstrapper(_BootstrapperBase):
     """Install the software needed during bootstrapping from sources."""
 
-    config_scope_name = "bootstrap_source"
-
     def __init__(self, conf):
         super(_SourceBootstrapper, self).__init__(conf)
         self.metadata_dir = spack.util.path.canonicalize_path(conf["metadata"])
         self.conf = conf
         self.last_search = None
+        self.config_scope_name = "bootstrap_source-{}".format(uuid.uuid4())
 
     def try_import(self, module, abstract_spec_str):
         info = {}
@@ -788,17 +788,46 @@ def ensure_gpg_in_path_or_raise():
 
 def patchelf_root_spec():
     """Return the root spec used to bootstrap patchelf"""
-    # TODO: patchelf is restricted to v0.13 since earlier versions have
-    # TODO: bugs that we don't to deal with, while v0.14 requires a C++17
-    # TODO: which may not be available on all platforms.
-    return _root_spec("patchelf@0.13.1:0.13.99")
+    # 0.13.1 is the last version not to require C++17.
+    return _root_spec("patchelf@0.13.1:")
+
+
+def verify_patchelf(patchelf):
+    """Older patchelf versions can produce broken binaries, so we
+    verify the version here.
+
+    Arguments:
+
+        patchelf (spack.util.executable.Executable): patchelf executable
+    """
+    out = patchelf("--version", output=str, error=os.devnull, fail_on_error=False).strip()
+    if patchelf.returncode != 0:
+        return False
+    parts = out.split(" ")
+    if len(parts) < 2:
+        return False
+    try:
+        version = spack.version.Version(parts[1])
+    except ValueError:
+        return False
+    return version >= spack.version.Version("0.13.1")
 
 
 def ensure_patchelf_in_path_or_raise():
     """Ensure patchelf is in the PATH or raise."""
-    return ensure_executables_in_path_or_raise(
-        executables=["patchelf"], abstract_spec=patchelf_root_spec()
-    )
+    # The old concretizer is not smart and we're doing its job: if the latest patchelf
+    # does not concretize because the compiler doesn't support C++17, we try to
+    # concretize again with an upperbound @:13.
+    try:
+        return ensure_executables_in_path_or_raise(
+            executables=["patchelf"], abstract_spec=patchelf_root_spec(), cmd_check=verify_patchelf
+        )
+    except RuntimeError:
+        return ensure_executables_in_path_or_raise(
+            executables=["patchelf"],
+            abstract_spec=_root_spec("patchelf@0.13.1:0.13"),
+            cmd_check=verify_patchelf,
+        )
 
 
 ###

lib/spack/spack/relocate.py

@@ -13,6 +13,7 @@
 
 import llnl.util.lang
 import llnl.util.tty as tty
+from llnl.util.lang import memoized
 from llnl.util.symlink import symlink
 
 import spack.bootstrap
@@ -76,15 +77,14 @@ def __init__(self, old_path, new_path):
         super(BinaryTextReplaceError, self).__init__(msg, err_msg)
 
 
+@memoized
 def _patchelf():
     """Return the full path to the patchelf binary, if available, else None."""
     if is_macos:
         return None
 
-    patchelf = executable.which("patchelf")
-    if patchelf is None:
-        with spack.bootstrap.ensure_bootstrap_configuration():
-            patchelf = spack.bootstrap.ensure_patchelf_in_path_or_raise()
+    with spack.bootstrap.ensure_bootstrap_configuration():
+        patchelf = spack.bootstrap.ensure_patchelf_in_path_or_raise()
 
     return patchelf.path
 

share/spack/bootstrap/github-actions-v0.3/clingo.json

@@ -0,0 +1 @@
+../github-actions-v0.2/clingo.json

share/spack/bootstrap/github-actions-v0.3/gnupg.json

@@ -0,0 +1 @@
+../github-actions-v0.2/gnupg.json

share/spack/bootstrap/github-actions-v0.3/metadata.yaml

@@ -0,0 +1,8 @@
+type: buildcache
+description: |
+  Buildcache generated from a public workflow using Github Actions.
+  The sha256 checksum of binaries is checked before installation.
+info:
+  url: https://mirror.spack.io/bootstrap/github-actions/v0.3
+  homepage: https://github.com/spack/spack-bootstrap-mirrors
+  releases: https://github.com/spack/spack-bootstrap-mirrors/releases

share/spack/bootstrap/github-actions-v0.3/patchelf.json

@@ -0,0 +1,34 @@
+{
+  "verified": [
+    {
+      "binaries": [
+        [
+          "patchelf",
+          "cn4gsqzdnnffk7ynvbcai6wrt5ehqqrl",
+          "8c6a28cbe8133d719be27ded11159f0aa2c97ed1d0881119ae0ebd71f8ccc755"
+        ]
+      ],
+      "spec": "patchelf@0.13: %gcc platform=linux target=aarch64"
+    },
+    {
+      "binaries": [
+        [
+          "patchelf",
+          "mgq6n2heyvcx2ebdpchkbknwwn3u63s6",
+          "1d4ea9167fb8345a178c1352e0377cc37ef2b421935cf2b48fb6fa03a94fca3d"
+        ]
+      ],
+      "spec": "patchelf@0.13: %gcc platform=linux target=ppc64le"
+    },
+    {
+      "binaries": [
+        [
+          "patchelf",
+          "htk62k7efo2z22kh6kmhaselru7bfkuc",
+          "833df21b20eaa7999ac4c5779ae26aa90397d9027aebaa686a428589befda693"
+        ]
+      ],
+      "spec": "patchelf@0.13: %gcc platform=linux target=x86_64"
+    }
+  ]
+}