hlrs-software-stack/spack
7
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Bootstrap clingo from binaries (#22720)

Browse source 
* Bootstrap clingo from binaries

* Move information on clingo binaries to a JSON file

* Add support to bootstrap on Cray

Bootstrapping on Cray requires, at the moment, to
swap the platform when looking for binaries - due
to #22800.

* Add SHA256 verification for bootstrapped software

Use sha256 verification for binaries necessary to bootstrap
the concretizer and gpg for signature verification

* patchelf: use Spec._old_concretize() to bootstrap

As noted in #24450 we may happen to need the
concretizer when bootstrapping clingo. In that case
only the old concretizer is available.

* Add a schema for bootstrapping methods

Two fields have been added to bootstrap.yaml:
  "sources" which lists the methods available for
       bootstrapping software
  "trusted" which records if a source is trusted or not

A subcommand has been added to "spack bootstrap" to list
the sources currently available.

* Methods used for bootstrapping are configurable from bootstrap:sources

The function that tries to ensure a given Python module
is importable now tries bootstrapping methods in the same
order as they are defined in `bootstrap.yaml`

* Permit to trust/untrust bootstrapping methods

* Add binary tests for MacOS, Ubuntu

* Add documentation

* Add a note on bash
This commit is contained in:
Massimiliano Culpo 2021-08-18 20:14:02 +02:00 committed by GitHub
parent 8a32f72829
commit 4318ceb2b3
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
17 changed files with 1001 additions and 148 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

55
.github/workflows/bootstrap.yml vendored
View file

@ -18,7 +18,7 @@ on:
jobs:
fedora:
fedora-sources:
runs-on: ubuntu-latest
container: "fedora:latest"
steps:
@ -40,11 +40,12 @@ jobs:
shell: runuser -u spack-test -- bash {0}
run: |
source share/spack/setup-env.sh
spack bootstrap untrust github-actions
spack external find cmake bison
spack -d solve zlib
tree ~/.spack/bootstrap/store/
ubuntu:
ubuntu-sources:
runs-on: ubuntu-latest
container: "ubuntu:latest"
steps:
@ -69,11 +70,12 @@ jobs:
shell: runuser -u spack-test -- bash {0}
run: |
source share/spack/setup-env.sh
spack bootstrap untrust github-actions
spack external find cmake bison
spack -d solve zlib
tree ~/.spack/bootstrap/store/
opensuse:
opensuse-sources:
runs-on: ubuntu-latest
container: "opensuse/tumbleweed:latest"
steps:
@ -93,11 +95,12 @@ jobs:
- name: Bootstrap clingo
run: |
source share/spack/setup-env.sh
spack bootstrap untrust github-actions
spack external find cmake bison
spack -d solve zlib
tree ~/.spack/bootstrap/store/
macos:
macos-sources:
runs-on: macos-latest
steps:
- name: Install dependencies
@ -108,6 +111,50 @@ jobs:
run: |
source share/spack/setup-env.sh
export PATH=/usr/local/opt/bison@2.7/bin:$PATH
spack bootstrap untrust github-actions
spack external find --not-buildable cmake bison
spack -d solve zlib
tree ~/.spack/bootstrap/store/
macos-clingo-binaries:
runs-on: macos-latest
strategy:
matrix:
python-version: ['3.5', '3.6', '3.7', '3.8', '3.9']
steps:
- name: Install dependencies
run: |
brew install tree
- uses: actions/checkout@v2
- uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- name: Bootstrap clingo
run: |
source share/spack/setup-env.sh
spack bootstrap untrust spack-install
spack -d solve zlib
tree ~/.spack/bootstrap/store/
ubuntu-clingo-binaries:
runs-on: ubuntu-latest
strategy:
matrix:
python-version: ['2.7', '3.5', '3.6', '3.7', '3.8', '3.9']
steps:
- uses: actions/checkout@v2
- uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- name: Setup repo and non-root user
run: |
git --version
git fetch --unshallow
. .github/workflows/setup_git.sh
- name: Bootstrap clingo
run: |
source share/spack/setup-env.sh
spack bootstrap untrust spack-install
spack -d solve zlib
tree ~/.spack/bootstrap/store/

4
.github/workflows/unit_tests.yaml vendored
View file

@ -159,13 +159,13 @@ jobs:
mkdir -p ${KCOV_ROOT}/build
cd ${KCOV_ROOT}/build && cmake -Wno-dev ${KCOV_ROOT}/kcov-${KCOV_VERSION} && cd -
make -C ${KCOV_ROOT}/build && sudo make -C ${KCOV_ROOT}/build install
- name: Bootstrap clingo from sources
- name: Bootstrap clingo
if: ${{ matrix.concretizer == 'clingo' }}
env:
SPACK_PYTHON: python
run: |
. share/spack/setup-env.sh
spack external find --not-buildable cmake bison
spack bootstrap untrust spack-install
spack -v solve zlib
- name: Run unit tests (full suite with coverage)
if: ${{ needs.changes.outputs.with_coverage == 'true' }}

25
etc/spack/defaults/bootstrap.yaml
View file

@ -5,3 +5,28 @@ bootstrap:
# Root directory for bootstrapping work. The software bootstrapped
# by Spack is installed in a "store" subfolder of this root directory
root: ~/.spack/bootstrap
# Methods that can be used to bootstrap software. Each method may or
# may not be able to bootstrap all of the software that Spack needs,
# depending on its type.
sources:
- name: 'github-actions'
type: buildcache
description: |
Buildcache generated from a public workflow using Github Actions.
The sha256 checksum of binaries is checked before installation.
info:
url: https://mirror.spack.io/bootstrap/github-actions/v0.1
homepage: https://github.com/alalazo/spack-bootstrap-mirrors
releases: https://github.com/alalazo/spack-bootstrap-mirrors/releases
# This method is just Spack bootstrapping the software it needs from sources.
# It has been added here so that users can selectively disable bootstrapping
# from sources by "untrusting" it.
- name: spack-install
type: install
description: |
Specs built from sources by Spack. May take a long time.
trusted:
# By default we trust bootstrapping from sources and from binaries
# produced on Github via the workflow
github-actions: true
spack-install: true

214
lib/spack/docs/getting_started.rst
View file

@ -9,22 +9,16 @@
Getting Started
===============
-------------
Prerequisites
-------------
--------------------
System Prerequisites
--------------------
Spack has the following minimum requirements, which must be installed
before Spack is run:
Spack has the following minimum system requirements, which are assumed to
be present on the machine where Spack is run:
#. Python 2 (2.6 or 2.7) or 3 (3.5 - 3.9) to run Spack
#. A C/C++ compiler for building and the ``bash`` shell for Spack's compiler
wrapper
#. The ``make`` executable for building
#. The ``tar``, ``gzip``, ``unzip``, ``bzip2``, ``xz`` and optionally ``zstd``
executables for extracting source code
#. The ``patch`` command to apply patches
#. The ``git`` and ``curl`` commands for fetching
#. If using the ``gpg`` subcommand, ``gnupg2`` is required
.. csv-table:: System prerequisites for Spack
:file: tables/system_prerequisites.csv
:header-rows: 1
These requirements can be easily installed on most modern Linux systems;
on macOS, XCode is required. Spack is designed to run on HPC
@ -90,6 +84,151 @@ sourcing time, ensuring future invocations of the ``spack`` command will
continue to use the same consistent python version regardless of changes in
the environment.
^^^^^^^^^^^^^^^^^^^^
Bootstrapping clingo
^^^^^^^^^^^^^^^^^^^^
Spack supports using ``clingo`` as an external solver to compute which software
needs to be installed. The default configuration allows Spack to install
``clingo`` from a public buildcache, created by a Github Action workflow. In this
case the bootstrapping procedure is transparent to the user, except for a
slightly long waiting time on the first concretization of a spec:
.. code-block:: console
$ spack find -b
==> Showing internal bootstrap store at "/home/spack/.spack/bootstrap/store"
==> 0 installed packages
$ time spack solve zlib
==> Best of 2 considered solutions.
==> Optimization Criteria:
Priority Criterion Value
1 deprecated versions used 0
2 version weight 0
3 number of non-default variants (roots) 0
4 multi-valued variants 0
5 preferred providers for roots 0
6 number of non-default variants (non-roots) 0
7 preferred providers (non-roots) 0
8 compiler mismatches 0
9 version badness 0
10 count of non-root multi-valued variants 0
11 non-preferred compilers 0
12 target mismatches 0
13 non-preferred targets 0
zlib@1.2.11%gcc@11.1.0+optimize+pic+shared arch=linux-ubuntu18.04-broadwell
real 0m30,618s
user 0m27,278s
sys 0m1,549s
After this command you'll see that ``clingo`` has been installed for Spack's own use:
.. code-block:: console
$ spack find -b
==> Showing internal bootstrap store at "/home/spack/.spack/bootstrap/store"
==> 2 installed packages
-- linux-rhel5-x86_64 / gcc@9.3.0 -------------------------------
clingo-bootstrap@spack python@3.6
Subsequent calls to the concretizer will then be much faster:
.. code-block:: console
$ time spack solve zlib
[ ... ]
real 0m1,222s
user 0m1,146s
sys 0m0,059s
If for security or for other reasons you don't want to or can't install precompiled
binaries, Spack can fall-back to bootstrap ``clingo`` from source files. To forbid
Spack from retrieving binaries from the bootstrapping buildcache, the following
command must be given:
.. code-block:: console
$ spack bootstrap untrust github-actions
==> "github-actions" is now untrusted and will not be used for bootstrapping
since an "untrusted" way of bootstrapping software will not be considered
by Spack. You can verify the new settings are effective with:
.. code-block:: console
$ spack bootstrap list
Name: github-actions UNTRUSTED
Type: buildcache
Info:
url: https://mirror.spack.io/bootstrap/github-actions/v0.1
homepage: https://github.com/alalazo/spack-bootstrap-mirrors
releases: https://github.com/alalazo/spack-bootstrap-mirrors/releases
Description:
Buildcache generated from a public workflow using Github Actions.
The sha256 checksum of binaries is checked before installation.
Name: spack-install TRUSTED
Type: install
Description:
Specs built from sources by Spack. May take a long time.
When bootstrapping from sources, Spack requires a compiler with support
for C++14 (GCC on ``linux``, Apple Clang on ``darwin``) and static C++
standard libraries on ``linux``. Spack will build the required software
on the first request to concretize a spec:
.. code-block:: console
$ spack solve zlib
[+] /usr (external bison-3.0.4-wu5pgjchxzemk5ya2l3ddqug2d7jv6eb)
[+] /usr (external cmake-3.19.4-a4kmcfzxxy45mzku4ipmj5kdiiz5a57b)
[+] /usr (external python-3.6.9-x4fou4iqqlh5ydwddx3pvfcwznfrqztv)
==> Installing re2c-1.2.1-e3x6nxtk3ahgd63ykgy44mpuva6jhtdt
[ ... ]
==> Optimization: [0, 0, 0, 0, 0, 1, 0, 0, 0]
zlib@1.2.11%gcc@10.1.0+optimize+pic+shared arch=linux-ubuntu18.04-broadwell
.. tip::
If you want to speed-up bootstrapping ``clingo`` from sources, you may try to
search for ``cmake`` and ``bison`` on your system:
.. code-block:: console
$ spack external find cmake bison
==> The following specs have been detected on this system and added to /home/spack/.spack/packages.yaml
bison@3.0.4 cmake@3.19.4
"""""""""""""""""""
The Bootstrap Store
"""""""""""""""""""
All the tools Spack needs for its own functioning are installed in a separate store, which lives
under the ``${HOME}/.spack`` directory. The software installed there can be queried with:
.. code-block:: console
$ spack find --bootstrap
==> Showing internal bootstrap store at "/home/spack/.spack/bootstrap/store"
==> 3 installed packages
-- linux-ubuntu18.04-x86_64 / gcc@10.1.0 ------------------------
clingo-bootstrap@spack python@3.6.9 re2c@1.2.1
In case it's needed the bootstrap store can also be cleaned with:
.. code-block:: console
$ spack clean -b
==> Removing software in "/home/spack/.spack/bootstrap/store"
^^^^^^^^^^^^^^^^^^
Check Installation
@ -118,53 +257,6 @@ environment*, especially for ``PATH``. Only software that comes with
the system, or that you know you wish to use with Spack, should be
included. This procedure will avoid many strange build errors.
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
Optional: Bootstrapping clingo
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
Spack supports using clingo as an external solver to compute which software
needs to be installed. If you have a default compiler supporting C++14 Spack
can automatically bootstrap this tool from sources the first time it is
needed:
.. code-block:: console
$ spack solve zlib
[+] /usr (external bison-3.0.4-wu5pgjchxzemk5ya2l3ddqug2d7jv6eb)
[+] /usr (external cmake-3.19.4-a4kmcfzxxy45mzku4ipmj5kdiiz5a57b)
[+] /usr (external python-3.6.9-x4fou4iqqlh5ydwddx3pvfcwznfrqztv)
==> Installing re2c-1.2.1-e3x6nxtk3ahgd63ykgy44mpuva6jhtdt
[ ... ]
==> Optimization: [0, 0, 0, 0, 0, 1, 0, 0, 0]
zlib@1.2.11%gcc@10.1.0+optimize+pic+shared arch=linux-ubuntu18.04-broadwell
If you want to speed-up bootstrapping, you may try to search for ``cmake`` and ``bison``
on your system:
.. code-block:: console
$ spack external find cmake bison
==> The following specs have been detected on this system and added to /home/spack/.spack/packages.yaml
bison@3.0.4 cmake@3.19.4
All the tools Spack needs for its own functioning are installed in a separate store, which lives
under the ``${HOME}/.spack`` directory. The software installed there can be queried with:
.. code-block:: console
$ spack find --bootstrap
==> Showing internal bootstrap store at "/home/spack/.spack/bootstrap/store"
==> 3 installed packages
-- linux-ubuntu18.04-x86_64 / gcc@10.1.0 ------------------------
clingo-bootstrap@spack python@3.6.9 re2c@1.2.1
In case it's needed the bootstrap store can also be cleaned with:
.. code-block:: console
$ spack clean -b
==> Removing software in "/home/spack/.spack/bootstrap/store"
^^^^^^^^^^^^^^^^^^^^^^^^^^
Optional: Alternate Prefix
^^^^^^^^^^^^^^^^^^^^^^^^^^

17
lib/spack/docs/tables/system_prerequisites.csv Normal file
View file

@ -0,0 +1,17 @@
Name, Supported Versions, Notes, Requirement Reason
Python, 2.6/2.7/3.5-3.9, , Interpreter for Spack
C/C++ Compilers, , , Building software
make, , , Build software
patch, , , Build software
bash, , , Compiler wrappers
tar, , , Extract/create archives
gzip, , , Compress/Decompress archives
unzip, , , Compress/Decompress archives
bzip, , , Compress/Decompress archives
xz, , , Compress/Decompress archives
zstd, , Optional, Compress/Decompress archives
file, , , Create/Use Buildcaches
gnupg2, , , Sign/Verify Buildcaches
git, , , Manage Software Repositories
svn, , Optional, Manage Software Repositories
hg, , Optional, Manage Software Repositories
1 Name Supported Versions Notes Requirement Reason
2 Python 2.6/2.7/3.5-3.9 Interpreter for Spack
3 C/C++ Compilers Building software
4 make Build software
5 patch Build software
6 bash Compiler wrappers
7 tar Extract/create archives
8 gzip Compress/Decompress archives
9 unzip Compress/Decompress archives
10 bzip Compress/Decompress archives
11 xz Compress/Decompress archives
12 zstd Optional Compress/Decompress archives
13 file Create/Use Buildcaches
14 gnupg2 Sign/Verify Buildcaches
15 git Manage Software Repositories
16 svn Optional Manage Software Repositories
17 hg Optional Manage Software Repositories

11
lib/spack/spack/binary_distribution.py
View file

@ -29,6 +29,7 @@
import spack.database as spack_db
import spack.fetch_strategy as fs
import spack.hash_types as ht
import spack.hooks.sbang
import spack.mirror
import spack.relocate as relocate
import spack.util.file_cache as file_cache
@ -615,9 +616,8 @@ def write_buildinfo_file(spec, workdir, rel=False):
prefix_to_hash[str(d.prefix)] = d.dag_hash()
# Create buildinfo data and write it to disk
import spack.hooks.sbang as sbang
buildinfo = {}
buildinfo['sbang_install_path'] = sbang.sbang_install_path()
buildinfo['sbang_install_path'] = spack.hooks.sbang.sbang_install_path()
buildinfo['relative_rpaths'] = rel
buildinfo['buildpath'] = spack.store.layout.root
buildinfo['spackprefix'] = spack.paths.prefix
@ -1169,8 +1169,6 @@ def relocate_package(spec, allow_root):
"""
Relocate the given package
"""
import spack.hooks.sbang as sbang
workdir = str(spec.prefix)
buildinfo = read_buildinfo_file(workdir)
new_layout_root = str(spack.store.layout.root)
@ -1209,7 +1207,8 @@ def relocate_package(spec, allow_root):
prefix_to_prefix_bin = OrderedDict({})
if old_sbang_install_path:
prefix_to_prefix_text[old_sbang_install_path] = sbang.sbang_install_path()
install_path = spack.hooks.sbang.sbang_install_path()
prefix_to_prefix_text[old_sbang_install_path] = install_path
prefix_to_prefix_text[old_prefix] = new_prefix
prefix_to_prefix_bin[old_prefix] = new_prefix
@ -1223,7 +1222,7 @@ def relocate_package(spec, allow_root):
# now a POSIX script that lives in the install prefix. Old packages
# will have the old sbang location in their shebangs.
orig_sbang = '#!/bin/bash {0}/bin/sbang'.format(old_spack_prefix)
new_sbang = sbang.sbang_shebang_line()
new_sbang = spack.hooks.sbang.sbang_shebang_line()
prefix_to_prefix_text[orig_sbang] = new_sbang
tty.debug("Relocating package from",

325
lib/spack/spack/bootstrap.py
View file

@ -3,6 +3,7 @@
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
import contextlib
import json
import os
import sys
@ -18,7 +19,9 @@
import llnl.util.tty as tty
import spack.architecture
import spack.binary_distribution
import spack.config
import spack.main
import spack.paths
import spack.repo
import spack.spec
@ -28,6 +31,214 @@
import spack.util.path
from spack.util.environment import EnvironmentModifications
#: Map a bootstrapper type to the corresponding class
_bootstrap_methods = {}
def _bootstrapper(type):
"""Decorator to register classes implementing bootstrapping
methods.
Args:
type (str): string identifying the class
"""
def _register(cls):
_bootstrap_methods[type] = cls
return cls
return _register
def _try_import_from_store(module, abstract_spec_str):
"""Return True if the module can be imported from an already
installed spec, False otherwise.
Args:
module: Python module to be imported
abstract_spec_str: abstract spec that may provide the module
"""
bincache_platform = spack.architecture.real_platform()
if str(bincache_platform) == 'cray':
bincache_platform = spack.platforms.linux.Linux()
with spack.architecture.use_platform(bincache_platform):
abstract_spec_str = str(spack.spec.Spec(abstract_spec_str))
# We have to run as part of this python interpreter
abstract_spec_str += ' ^' + spec_for_current_python()
installed_specs = spack.store.db.query(abstract_spec_str, installed=True)
for candidate_spec in installed_specs:
lib_spd = candidate_spec['python'].package.default_site_packages_dir
lib64_spd = lib_spd.replace('lib/', 'lib64/')
module_paths = [
os.path.join(candidate_spec.prefix, lib_spd),
os.path.join(candidate_spec.prefix, lib64_spd)
]
sys.path.extend(module_paths)
try:
if _python_import(module):
msg = ('[BOOTSTRAP MODULE {0}] The installed spec "{1}/{2}" '
'provides the "{0}" Python module').format(
module, abstract_spec_str, candidate_spec.dag_hash()
)
tty.debug(msg)
return True
except Exception as e:
msg = ('unexpected error while trying to import module '
'"{0}" from spec "{1}" [error="{2}"]')
tty.warn(msg.format(module, candidate_spec, str(e)))
else:
msg = "Spec {0} did not provide module {1}"
tty.warn(msg.format(candidate_spec, module))
sys.path = sys.path[:-2]
return False
@_bootstrapper(type='buildcache')
class _BuildcacheBootstrapper(object):
"""Install the software needed during bootstrapping from a buildcache."""
def __init__(self, conf):
self.name = conf['name']
self.url = conf['info']['url']
def try_import(self, module, abstract_spec_str):
# This import is local since it is needed only on Cray
import spack.platforms.linux
if _try_import_from_store(module, abstract_spec_str):
return True
# Try to install from an unsigned binary cache
abstract_spec = spack.spec.Spec(
abstract_spec_str + ' ^' + spec_for_current_python()
)
# On Cray we want to use Linux binaries if available from mirrors
bincache_platform = spack.architecture.real_platform()
if str(bincache_platform) == 'cray':
bincache_platform = spack.platforms.linux.Linux()
with spack.architecture.use_platform(bincache_platform):
abstract_spec = spack.spec.Spec(
abstract_spec_str + ' ^' + spec_for_current_python()
)
# Read information on verified clingo binaries
json_filename = '{0}.json'.format(module)
json_path = os.path.join(
spack.paths.share_path, 'bootstrap', self.name, json_filename
)
with open(json_path) as f:
data = json.load(f)
buildcache = spack.main.SpackCommand('buildcache')
# Ensure we see only the buildcache being used to bootstrap
mirror_scope = spack.config.InternalConfigScope(
'bootstrap', {'mirrors:': {self.name: self.url}}
)
with spack.config.override(mirror_scope):
# This index is currently needed to get the compiler used to build some
# specs that wwe know by dag hash.
spack.binary_distribution.binary_index.regenerate_spec_cache()
index = spack.binary_distribution.update_cache_and_get_specs()
for item in data['verified']:
candidate_spec = item['spec']
python_spec = item['python']
# Skip specs which are not compatible
if not abstract_spec.satisfies(candidate_spec):
continue
if python_spec not in abstract_spec:
continue
for pkg_name, pkg_hash, pkg_sha256 in item['binaries']:
msg = ('[BOOTSTRAP MODULE {0}] Try installing "{1}" from binary '
'cache at "{2}"')
tty.debug(msg.format(module, pkg_name, self.url))
index_spec = next(x for x in index if x.dag_hash() == pkg_hash)
# Reconstruct the compiler that we need to use for bootstrapping
compiler_entry = {
"modules": [],
"operating_system": str(index_spec.os),
"paths": {
"cc": "/dev/null",
"cxx": "/dev/null",
"f77": "/dev/null",
"fc": "/dev/null"
},
"spec": str(index_spec.compiler),
"target": str(index_spec.target.family)
}
with spack.architecture.use_platform(bincache_platform):
with spack.config.override(
'compilers', [{'compiler': compiler_entry}]
):
spec_str = '/' + pkg_hash
install_args = [
'install',
'--sha256', pkg_sha256,
'-a', '-u', '-o', '-f', spec_str
]
buildcache(*install_args, fail_on_error=False)
# TODO: undo installations that didn't complete?
if _try_import_from_store(module, abstract_spec_str):
return True
return False
@_bootstrapper(type='install')
class _SourceBootstrapper(object):
"""Install the software needed during bootstrapping from sources."""
def __init__(self, conf):
self.conf = conf
@staticmethod
def try_import(module, abstract_spec_str):
if _try_import_from_store(module, abstract_spec_str):
return True
# Try to build and install from sources
with spack_python_interpreter():
# Add hint to use frontend operating system on Cray
if str(spack.architecture.platform()) == 'cray':
abstract_spec_str += ' os=fe'
concrete_spec = spack.spec.Spec(
abstract_spec_str + ' ^' + spec_for_current_python()
)
if module == 'clingo':
# TODO: remove when the old concretizer is deprecated
concrete_spec._old_concretize()
else:
concrete_spec.concretize()
msg = "[BOOTSTRAP MODULE {0}] Try installing '{1}' from sources"
tty.debug(msg.format(module, abstract_spec_str))
# Install the spec that should make the module importable
concrete_spec.package.do_install()
return _try_import_from_store(module, abstract_spec_str=abstract_spec_str)
def _make_bootstrapper(conf):
"""Return a bootstrap object built according to the
configuration argument
"""
btype = conf['type']
return _bootstrap_methods[btype](conf)
def _source_is_trusted(conf):
trusted, name = spack.config.get('bootstrap:trusted'), conf['name']
if name not in trusted:
return False
return trusted[name]
def spec_for_current_python():
"""For bootstrapping purposes we are just interested in the Python
@ -68,72 +279,58 @@ def spack_python_interpreter():
yield
def make_module_available(module, spec=None, install=False):
"""Ensure module is importable"""
# If we already can import it, that's great
try:
__import__(module)
def ensure_module_importable_or_raise(module, abstract_spec=None):
"""Make the requested module available for import, or raise.
This function tries to import a Python module in the current interpreter
using, in order, the methods configured in bootstrap.yaml.
If none of the methods succeed, an exception is raised. The function exits
on first success.
Args:
module (str): module to be imported in the current interpreter
abstract_spec (str): abstract spec that might provide the module. If not
given it defaults to "module"
Raises:
ImportError: if the module couldn't be imported
"""
# If we can import it already, that's great
tty.debug("[BOOTSTRAP MODULE {0}] Try importing from Python".format(module))
if _python_import(module):
return
except ImportError:
pass
# If it's already installed, use it
# Search by spec
spec = spack.spec.Spec(spec or module)
abstract_spec = abstract_spec or module
source_configs = spack.config.get('bootstrap:sources', [])
for current_config in source_configs:
if not _source_is_trusted(current_config):
msg = ('[BOOTSTRAP MODULE {0}] Skipping source "{1}" since it is '
'not trusted').format(module, current_config['name'])
tty.debug(msg)
continue
# We have to run as part of this python
# We can constrain by a shortened version in place of a version range
# because this spec is only used for querying or as a placeholder to be
# replaced by an external that already has a concrete version. This syntax
# is not sufficient when concretizing without an external, as it will
# concretize to python@X.Y instead of python@X.Y.Z
python_requirement = '^' + spec_for_current_python()
spec.constrain(python_requirement)
installed_specs = spack.store.db.query(spec, installed=True)
for ispec in installed_specs:
lib_spd = ispec['python'].package.default_site_packages_dir
lib64_spd = lib_spd.replace('lib/', 'lib64/')
module_paths = [
os.path.join(ispec.prefix, lib_spd),
os.path.join(ispec.prefix, lib64_spd)
]
b = _make_bootstrapper(current_config)
try:
sys.path.extend(module_paths)
__import__(module)
return
except ImportError:
tty.warn("Spec %s did not provide module %s" % (ispec, module))
sys.path = sys.path[:-2]
if b.try_import(module, abstract_spec):
return
except Exception as e:
msg = '[BOOTSTRAP MODULE {0}] Unexpected error "{1}"'
tty.debug(msg.format(module, str(e)))
def _raise_error(module_name, module_spec):
error_msg = 'cannot import module "{0}"'.format(module_name)
if module_spec:
error_msg += ' from spec "{0}'.format(module_spec)
raise ImportError(error_msg)
# We couldn't import in any way, so raise an import error
msg = 'cannot bootstrap the "{0}" Python module'.format(module)
if abstract_spec:
msg += ' from spec "{0}"'.format(abstract_spec)
raise ImportError(msg)
if not install:
_raise_error(module, spec)
with spack_python_interpreter():
# We will install for ourselves, using this python if needed
# Concretize the spec
spec.concretize()
spec.package.do_install()
lib_spd = spec['python'].package.default_site_packages_dir
lib64_spd = lib_spd.replace('lib/', 'lib64/')
module_paths = [
os.path.join(spec.prefix, lib_spd),
os.path.join(spec.prefix, lib64_spd)
]
def _python_import(module):
try:
sys.path.extend(module_paths)
__import__(module)
return
except ImportError:
sys.path = sys.path[:-2]
_raise_error(module, spec)
return False
return True
def get_executable(exe, spec=None, install=False):
@ -147,7 +344,8 @@ def get_executable(exe, spec=None, install=False):
When ``install`` is True, Spack will use the python used to run Spack as an
external. The ``install`` option should only be used with packages that
install quickly (when using external python) or are guaranteed by Spack
organization to be in a binary mirror (clingo)."""
organization to be in a binary mirror (clingo).
"""
# Search the system first
runner = spack.util.executable.which(exe)
if runner:
@ -260,14 +458,17 @@ def clingo_root_spec():
else:
spec_str += ' %gcc'
# Add hint to use frontend operating system on Cray
if str(spack.architecture.platform()) == 'cray':
spec_str += ' os=fe'
# Add the generic target
generic_target = archspec.cpu.host().family
spec_str += ' target={0}'.format(str(generic_target))
tty.debug('[BOOTSTRAP ROOT SPEC] clingo: {0}'.format(spec_str))
return spack.spec.Spec(spec_str)
return spec_str
def ensure_clingo_importable_or_raise():
"""Ensure that the clingo module is available for import."""
ensure_module_importable_or_raise(
module='clingo', abstract_spec=clingo_root_spec()
)

110
lib/spack/spack/cmd/bootstrap.py
View file

@ -6,6 +6,7 @@
import shutil
import llnl.util.tty
import llnl.util.tty.color
import spack.cmd.common.arguments
import spack.config
@ -51,6 +52,27 @@ def setup_parser(subparser):
help='set the bootstrap directory to this value'
)
list = sp.add_parser(
'list', help='list the methods available for bootstrapping'
)
_add_scope_option(list)
trust = sp.add_parser(
'trust', help='trust a bootstrapping method'
)
_add_scope_option(trust)
trust.add_argument(
'name', help='name of the method to be trusted'
)
untrust = sp.add_parser(
'untrust', help='untrust a bootstrapping method'
)
_add_scope_option(untrust)
untrust.add_argument(
'name', help='name of the method to be untrusted'
)
def _enable_or_disable(args):
# Set to True if we called "enable", otherwise set to false
@ -100,11 +122,97 @@ def _root(args):
print(root)
def _list(args):
sources = spack.config.get(
'bootstrap:sources', default=None, scope=args.scope
)
if not sources:
llnl.util.tty.msg(
"No method available for bootstrapping Spack's dependencies"
)
return
def _print_method(source, trusted):
color = llnl.util.tty.color
def fmt(header, content):
header_fmt = "@*b{{{0}:}} {1}"
color.cprint(header_fmt.format(header, content))
trust_str = "@*y{UNKNOWN}"
if trusted is True:
trust_str = "@*g{TRUSTED}"
elif trusted is False:
trust_str = "@*r{UNTRUSTED}"
fmt("Name", source['name'] + ' ' + trust_str)
print()
fmt(" Type", source['type'])
print()
info_lines = ['\n']
for key, value in source.get('info', {}).items():
info_lines.append(' ' * 4 + '@*{{{0}}}: {1}\n'.format(key, value))
if len(info_lines) > 1:
fmt(" Info", ''.join(info_lines))
description_lines = ['\n']
for line in source['description'].split('\n'):
description_lines.append(' ' * 4 + line + '\n')
fmt(" Description", ''.join(description_lines))
trusted = spack.config.get('bootstrap:trusted', {})
for s in sources:
_print_method(s, trusted.get(s['name'], None))
def _write_trust_state(args, value):
name = args.name
sources = spack.config.get('bootstrap:sources')
matches = [s for s in sources if s['name'] == name]
if not matches:
names = [s['name'] for s in sources]
msg = ('there is no bootstrapping method named "{0}". Valid '
'method names are: {1}'.format(name, ', '.join(names)))
raise RuntimeError(msg)
if len(matches) > 1:
msg = ('there is more than one bootstrapping method named "{0}". '
'Please delete all methods but one from bootstrap.yaml '
'before proceeding').format(name)
raise RuntimeError(msg)
# Setting the scope explicitly is needed to not copy over to a new scope
# the entire default configuration for bootstrap.yaml
scope = args.scope or spack.config.default_modify_scope('bootstrap')
spack.config.add(
'bootstrap:trusted:{0}:{1}'.format(name, str(value)), scope=scope
)
def _trust(args):
_write_trust_state(args, value=True)
msg = '"{0}" is now trusted for bootstrapping'
llnl.util.tty.msg(msg.format(args.name))
def _untrust(args):
_write_trust_state(args, value=False)
msg = '"{0}" is now untrusted and will not be used for bootstrapping'
llnl.util.tty.msg(msg.format(args.name))
def bootstrap(parser, args):
callbacks = {
'enable': _enable_or_disable,
'disable': _enable_or_disable,
'reset': _reset,
'root': _root
'root': _root,
'list': _list,
'trust': _trust,
'untrust': _untrust
}
callbacks[args.subcommand](args)

14
lib/spack/spack/cmd/buildcache.py
View file

@ -2,7 +2,7 @@
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
import argparse
import os
import shutil
import sys
@ -21,6 +21,7 @@
import spack.repo
import spack.spec
import spack.store
import spack.util.crypto
import spack.util.url as url_util
from spack.cmd import display_specs
from spack.error import SpecError
@ -97,6 +98,8 @@ def setup_parser(subparser):
install.add_argument('-o', '--otherarch', action='store_true',
help="install specs from other architectures" +
" instead of default platform and OS")
# This argument is needed by the bootstrapping logic to verify checksums
install.add_argument('--sha256', help=argparse.SUPPRESS)
arguments.add_common_arguments(install, ['specs'])
install.set_defaults(func=installtarball)
@ -495,6 +498,15 @@ def install_tarball(spec, args):
else:
tarball = bindist.download_tarball(spec)
if tarball:
if args.sha256:
checker = spack.util.crypto.Checker(args.sha256)
msg = ('cannot verify checksum for "{0}"'
' [expected={1}]')
msg = msg.format(tarball, args.sha256)
if not checker.check(tarball):
raise spack.binary_distribution.NoChecksumException(msg)
tty.debug('Verified SHA256 checksum of the build cache')
tty.msg('Installing buildcache for spec %s' % spec.format())
bindist.extract_tarball(spec, tarball, args.allow_root,
args.unsigned, args.force)

1
lib/spack/spack/hooks/sbang.py
View file

@ -12,7 +12,6 @@
import llnl.util.filesystem as fs
import llnl.util.tty as tty
import spack.modules
import spack.paths
import spack.store

3
lib/spack/spack/relocate.py
View file

@ -88,7 +88,8 @@ def _patchelf():
return patchelf.path
# Check if patchelf spec is installed
spec = spack.spec.Spec('patchelf').concretized()
spec = spack.spec.Spec('patchelf')
spec._old_concretize()
exe_path = os.path.join(spec.prefix.bin, "patchelf")
if spec.package.installed and os.path.exists(exe_path):
return exe_path

21
lib/spack/spack/schema/bootstrap.py
View file

@ -4,6 +4,19 @@
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
"""Schema for bootstrap.yaml configuration file."""
#: Schema of a single source
_source_schema = {
'type': 'object',
'properties': {
'name': {'type': 'string'},
'description': {'type': 'string'},
'type': {'type': 'string'},
'info': {'type': 'object'}
},
'additionalProperties': False,
'required': ['name', 'description', 'type']
}
properties = {
'bootstrap': {
'type': 'object',
@ -12,6 +25,14 @@
'root': {
'type': 'string'
},
'sources': {
'type': 'array',
'items': _source_schema
},
'trusted': {
'type': 'object',
'patternProperties': {r'\w[\w-]*': {'type': 'boolean'}}
}
}
}
}

8
lib/spack/spack/solver/asp.py
View file

@ -267,14 +267,8 @@ def __init__(self, cores=True, asp=None):
"""
global clingo
if not clingo:
# TODO: Find a way to vendor the concrete spec
# in a cross-platform way
with spack.bootstrap.ensure_bootstrap_configuration():
clingo_spec = spack.bootstrap.clingo_root_spec()
clingo_spec._old_concretize()
spack.bootstrap.make_module_available(
'clingo', spec=clingo_spec, install=True
)
spack.bootstrap.ensure_clingo_importable_or_raise()
import clingo
self.out = asp or llnl.util.lang.Devnull()
self.cores = cores

46
lib/spack/spack/test/cmd/bootstrap.py
View file

@ -99,3 +99,49 @@ def test_reset_in_file_scopes_overwrites_backup_files(mutable_config):
_bootstrap('reset', '-y')
assert not os.path.exists(bootstrap_yaml)
assert os.path.exists(backup_file)
def test_list_sources(capsys):
# Get the merged list and ensure we get our defaults
with capsys.disabled():
output = _bootstrap('list')
assert "github-actions" in output
# Ask for a specific scope and check that the list of sources is empty
with capsys.disabled():
output = _bootstrap('list', '--scope', 'user')
assert "No method available" in output
@pytest.mark.parametrize('command,value', [
('trust', True),
('untrust', False)
])
def test_trust_or_untrust_sources(mutable_config, command, value):
key = 'bootstrap:trusted:github-actions'
trusted = spack.config.get(key, default=None)
assert trusted is None
_bootstrap(command, 'github-actions')
trusted = spack.config.get(key, default=None)
assert trusted is value
def test_trust_or_untrust_fails_with_no_method(mutable_config):
with pytest.raises(RuntimeError, match='no bootstrapping method'):
_bootstrap('trust', 'foo')
def test_trust_or_untrust_fails_with_more_than_one_method(mutable_config):
wrong_config = {'sources': [
{'name': 'github-actions',
'type': 'buildcache',
'description': ''},
{'name': 'github-actions',
'type': 'buildcache',
'description': 'Another entry'}],
'trusted': {}
}
with spack.config.override('bootstrap', wrong_config):
with pytest.raises(RuntimeError, match='more than one'):
_bootstrap('trust', 'github-actions')

12
lib/spack/spack/test/data/config/bootstrap.yaml Normal file
View file

@ -0,0 +1,12 @@
bootstrap:
sources:
- name: 'github-actions'
type: buildcache
description: |
Buildcache generated from a public workflow using Github Actions.
The sha256 checksum of binaries is checked before installation.
info:
url: file:///home/culpo/production/spack/mirrors/clingo
homepage: https://github.com/alalazo/spack-bootstrap-mirrors
releases: https://github.com/alalazo/spack-bootstrap-mirrors/releases
trusted: {}

257
share/spack/bootstrap/github-actions/clingo.json Normal file
View file

@ -0,0 +1,257 @@
{
"verified": [
{
"binaries": [
[
"clingo-bootstrap",
"m4ertsh3ooxdisc5tigglublasu4udfe",
"094548672362306d75012398a6f9b1e8c0f796c833163ca77cf644d84822f25f"
]
],
"python": "python@3.5",
"spec": "clingo-bootstrap%apple-clang platform=darwin target=x86_64"
},
{
"binaries": [
[
"clingo-bootstrap",
"iv4gr5vscx2io23ljgdueybwatxlk6wo",
"c8110c68ec339d05155392818b21ba87b27905ad798f5f3f194d6312385dbdc2"
]
],
"python": "python@3.6",
"spec": "clingo-bootstrap%apple-clang platform=darwin target=x86_64"
},
{
"binaries": [
[
"clingo-bootstrap",
"rwxcxsohrkp5iai3yushsltkdprjmexb",
"fbee764cac890a29bc03c472d3ba0401e915d6924a7cedac9fd8d961159b70e7"
]
],
"python": "python@3.7",
"spec": "clingo-bootstrap%apple-clang platform=darwin target=x86_64"
},
{
"binaries": [
[
"clingo-bootstrap",
"p5on7i4hejl775ezndzfdkhvwra3hatn",
"35e32f7c1f80e99da450b52643800fd2895ee2f895109f708b5cf0da6afbedff"
]
],
"python": "python@3.8",
"spec": "clingo-bootstrap%apple-clang platform=darwin target=x86_64"
},
{
"binaries": [
[
"clingo-bootstrap",
"omsvlh5v6fi2saw5qyqvzsbvqpvrf5yw",
"cef0e554737dbf22655094d8ae072c67539cce2a37cba1577aeb5aea18b5747c"
]
],
"python": "python@3.9",
"spec": "clingo-bootstrap%apple-clang platform=darwin target=x86_64"
},
{
"binaries": [
[
"clingo-bootstrap",
"qguh44xegb72y4q4lar3ufjddissoumv",
"68d2d0c06690d75a794aa2c50be9d6d501fec1b566784bf87b1fc5611f84f3c9"
]
],
"python": "python@3.5",
"spec": "clingo-bootstrap%gcc platform=linux target=aarch64"
},
{
"binaries": [
[
"clingo-bootstrap",
"tsypkz7hyylmh5pwpykaf7wcmdunrdiv",
"3b8e3e6e21e399a90c4128776cc591734f9d533f0a7e64ed8babd9cbcf616e3d"
]
],
"python": "python@3.6",
"spec": "clingo-bootstrap%gcc platform=linux target=aarch64"
},
{
"binaries": [
[
"clingo-bootstrap",
"2w6b4q5j2zgra6t3zfrjfbvsnoi5yqmk",
"4f335e02641f6ecc7ec7d9d2b8293d07d6e7a7234034531713b760aaa507fa7c"
]
],
"python": "python@3.7",
"spec": "clingo-bootstrap%gcc platform=linux target=aarch64"
},
{
"binaries": [
[
"clingo-bootstrap",
"ij7udwpgeghdfoswljwdtwwaylvudze7",
"b5eedf593f198e523aebf6ce127fd8ffcf3d1c980a920cdf1c5a0d2a335e4892"
]
],
"python": "python@3.8",
"spec": "clingo-bootstrap%gcc platform=linux target=aarch64"
},
{
"binaries": [
[
"clingo-bootstrap",
"a5ppijpzmk7ubjem4i3zttbxp545vjuz",
"8f9755c16c0c99b5c40c420f2c1c6aec2bdff99b25444e001506527af18dd94e"
]
],
"python": "python@3.9",
"spec": "clingo-bootstrap%gcc platform=linux target=aarch64"
},
{
"binaries": [
[
"clingo-bootstrap",
"tsnva2bxjguosntz3tk5mqbdgrjvhfcc",
"bcd093c08110309e705beebccd012260a61215eda12c1d47f3a89d4734ec7170"
]
],
"python": "python@3.5",
"spec": "clingo-bootstrap%gcc platform=linux target=ppc64le"
},
{
"binaries": [
[
"clingo-bootstrap",
"rfu4la457mdbpoffk2g5hikj2hhoek4s",
"b6417b9b90f3f4e98caaa869393edee08fad3d3c7db37fad8b332c785d0e81e6"
]
],
"python": "python@3.6",
"spec": "clingo-bootstrap%gcc platform=linux target=ppc64le"
},
{
"binaries": [
[
"clingo-bootstrap",
"qg3utj2csbgtdwc2slqbjmwn25rkor5r",
"a1760e064d41d364cdf53f89248f7824dad9bf97c6b999df18343b57d21c06ed"
]
],
"python": "python@3.7",
"spec": "clingo-bootstrap%gcc platform=linux target=ppc64le"
},
{
"binaries": [
[
"clingo-bootstrap",
"uxkpc2euofdkgveonftklstnxyha5wsf",
"d6a04e7b15dae05eacce54806fa18356b392a5e2d212a55bf0960116b8e9dfef"
]
],
"python": "python@3.8",
"spec": "clingo-bootstrap%gcc platform=linux target=ppc64le"
},
{
"binaries": [
[
"clingo-bootstrap",
"i2uoawqsmmkr2uwb42oxfsnt4djzejrj",
"6511837f87e50c40a7f4aab2ec7454e5200594821e4d1fc4a441d3be647b9acb"
]
],
"python": "python@3.9",
"spec": "clingo-bootstrap%gcc platform=linux target=ppc64le"
},
{
"binaries": [
[
"clingo-bootstrap",
"p3bqplnwqo66lvvolmtaezck77jafrc4",
"436f9483e4028c12c32ba9f4e7e91e944bf9819ef487dfe4e42ddd1d487c93ee"
]
],
"python": "python@2.6",
"spec": "clingo-bootstrap%gcc platform=linux target=x86_64"
},
{
"binaries": [
[
"clingo-bootstrap",
"rpa6yyujr7ilnfcowq2pbmkmh7uzrijp",
"3be0f4ccd412d45506f9f305ef7f6621cd246fbde97aed081595d01dafe3c397"
]
],
"python": "python@2.7+ucs4",
"spec": "clingo-bootstrap%gcc platform=linux target=x86_64"
},
{
"binaries": [
[
"clingo-bootstrap",
"y32mbn7swer3yxvgf6tmkgekpo23uo5z",
"f6e0716bd97f2df123abcd96ec8884c525a9fd10b81e0062784e7b0d2df3f622"
]
],
"python": "python@2.7~ucs4",
"spec": "clingo-bootstrap%gcc platform=linux target=x86_64"
},
{
"binaries": [
[
"clingo-bootstrap",
"ohtmyp5c74xt75csx4llbjs5anobryp6",
"7d613ddbca1640d761311fb00403c0cb65e279534c44a2129b8d9610f6146e78"
]
],
"python": "python@3.5",
"spec": "clingo-bootstrap%gcc platform=linux target=x86_64"
},
{
"binaries": [
[
"clingo-bootstrap",
"vcipwnf57slgoo7busvvkzjkk7vydeb5",
"db5222760045f20ad1e5c194179d31273b8e4bfa6ade38e15cd3182d685cc05b"
]
],
"python": "python@3.6",
"spec": "clingo-bootstrap%gcc platform=linux target=x86_64"
},
{
"binaries": [
[
"clingo-bootstrap",
"ffoyoewfd6pdwbjniodfkqusyvkrbhyi",
"14cea5f6cfd86bcb8de38ad8c1a5e44cc22955de2e7c78b825b617dccd107dbe"
]
],
"python": "python@3.7",
"spec": "clingo-bootstrap%gcc platform=linux target=x86_64"
},
{
"binaries": [
[
"clingo-bootstrap",
"kt74l7kjzrlp3cgtj2576o33mhsrgyrw",
"e71de4beb68bb3e58bd2dcb98dc3be3a375c82781b6f7cb01bc5d552c2240bd2"
]
],
"python": "python@3.8",
"spec": "clingo-bootstrap%gcc platform=linux target=x86_64"
},
{
"binaries": [
[
"clingo-bootstrap",
"hmnv6gk5wha64k6r3s7hid35mzvhkuot",
"b08ff59357fa184ce39b8cc0a17aaf7f0a925a449ab389e1afa4eab6ae026f2e"
]
],
"python": "python@3.9",
"spec": "clingo-bootstrap%gcc platform=linux target=x86_64"
}
]
}

26
share/spack/spack-completion.bash
View file

@ -421,7 +421,7 @@ _spack_bootstrap() {
then
SPACK_COMPREPLY="-h --help"
else
SPACK_COMPREPLY="enable disable reset root"
SPACK_COMPREPLY="enable disable reset root list trust untrust"
fi
}
@ -446,6 +446,28 @@ _spack_bootstrap_root() {
fi
}
_spack_bootstrap_list() {
SPACK_COMPREPLY="-h --help --scope"
}
_spack_bootstrap_trust() {
if $list_options
then
SPACK_COMPREPLY="-h --help --scope"
else
SPACK_COMPREPLY=""
fi
}
_spack_bootstrap_untrust() {
if $list_options
then
SPACK_COMPREPLY="-h --help --scope"
else
SPACK_COMPREPLY=""
fi
}
_spack_build_env() {
if $list_options
then
@ -476,7 +498,7 @@ _spack_buildcache_create() {
_spack_buildcache_install() {
if $list_options
then
SPACK_COMPREPLY="-h --help -f --force -m --multiple -a --allow-root -u --unsigned -o --otherarch"
SPACK_COMPREPLY="-h --help -f --force -m --multiple -a --allow-root -u --unsigned -o --otherarch --sha256"
else
_all_packages
fi