New subcommand: spack bootstrap status (#28004)

This command pokes the environment, Python interpreter
and bootstrap store to check if dependencies needed by
Spack are available.

If any are missing, it shows a comprehensible message.

lib/spack/spack/bootstrap.py

@@ -10,6 +10,7 @@
 import json
 import os
 import os.path
+import platform
 import re
 import sys
 import sysconfig
@@ -837,3 +838,142 @@ def ensure_flake8_in_path_or_raise():
     """Ensure that flake8 is in the PATH or raise."""
     executable, root_spec = 'flake8', flake8_root_spec()
     return ensure_executables_in_path_or_raise([executable], abstract_spec=root_spec)
+
+
+def _missing(name, purpose, system_only=True):
+    """Message to be printed if an executable is not found"""
+    msg = '[{2}] MISSING "{0}": {1}'
+    if not system_only:
+        return msg.format(name, purpose, '@*y{{B}}')
+    return msg.format(name, purpose, '@*y{{-}}')
+
+
+def _required_system_executable(exes, msg):
+    """Search for an executable is the system path only."""
+    if isinstance(exes, six.string_types):
+        exes = (exes,)
+    if spack.util.executable.which_string(*exes):
+        return True, None
+    return False, msg
+
+
+def _required_python_module(module, query_spec, msg):
+    """Check if a Python module is available in the current interpreter or
+    if it can be loaded from the bootstrap store
+    """
+    if _python_import(module) or _try_import_from_store(module, query_spec):
+        return True, None
+    return False, msg
+
+
+def _required_executable(exes, query_spec, msg):
+    """Search for an executable in the system path or in the bootstrap store."""
+    if isinstance(exes, six.string_types):
+        exes = (exes,)
+    if (spack.util.executable.which_string(*exes) or
+            _executables_in_store(exes, query_spec)):
+        return True, None
+    return False, msg
+
+
+def _core_requirements():
+    _core_system_exes = {
+        'make': _missing('make', 'required to build software from sources'),
+        'patch': _missing('patch', 'required to patch source code before building'),
+        'bash': _missing('bash', 'required for Spack compiler wrapper'),
+        'tar': _missing('tar', 'required to manage code archives'),
+        'gzip': _missing('gzip', 'required to compress/decompress code archives'),
+        'unzip': _missing('unzip', 'required to compress/decompress code archives'),
+        'bzip2': _missing('bzip2', 'required to compress/decompress code archives'),
+        'git': _missing('git', 'required to fetch/manage git repositories')
+    }
+    if platform.system().lower() == 'linux':
+        _core_system_exes['xz'] = _missing(
+            'xz', 'required to compress/decompress code archives'
+        )
+
+    # Executables that are not bootstrapped yet
+    result = [_required_system_executable(exe, msg)
+              for exe, msg in _core_system_exes.items()]
+    # Python modules
+    result.append(_required_python_module(
+        'clingo', clingo_root_spec(),
+        _missing('clingo', 'required to concretize specs', False)
+    ))
+    return result
+
+
+def _buildcache_requirements():
+    _buildcache_exes = {
+        'file': _missing('file', 'required to analyze files for buildcaches'),
+        ('gpg2', 'gpg'): _missing('gpg2', 'required to sign/verify buildcaches', False)
+    }
+    if platform.system().lower() == 'darwin':
+        _buildcache_exes['otool'] = _missing('otool', 'required to relocate binaries')
+
+    # Executables that are not bootstrapped yet
+    result = [_required_system_executable(exe, msg)
+              for exe, msg in _buildcache_exes.items()]
+
+    if platform.system().lower() == 'linux':
+        result.append(_required_executable(
+            'patchelf', patchelf_root_spec(),
+            _missing('patchelf', 'required to relocate binaries', False)
+        ))
+
+    return result
+
+
+def _optional_requirements():
+    _optional_exes = {
+        'zstd': _missing('zstd', 'required to compress/decompress code archives'),
+        'svn': _missing('svn', 'required to manage subversion repositories'),
+        'hg': _missing('hg', 'required to manage mercurial repositories')
+    }
+    # Executables that are not bootstrapped yet
+    result = [_required_system_executable(exe, msg)
+              for exe, msg in _optional_exes.items()]
+    return result
+
+
+def _development_requirements():
+    return [
+        _required_executable('isort', isort_root_spec(),
+                             _missing('isort', 'required for style checks', False)),
+        _required_executable('mypy', mypy_root_spec(),
+                             _missing('mypy', 'required for style checks', False)),
+        _required_executable('flake8', flake8_root_spec(),
+                             _missing('flake8', 'required for style checks', False)),
+        _required_executable('black', black_root_spec(),
+                             _missing('black', 'required for code formatting', False))
+    ]
+
+
+def status_message(section):
+    """Return a status message to be printed to screen that refers to the
+    section passed as argument and a bool which is True if there are missing
+    dependencies.
+
+    Args:
+        section (str): either 'core' or 'buildcache' or 'optional' or 'develop'
+    """
+    pass_token, fail_token = '@*g{[PASS]}', '@*r{[FAIL]}'
+
+    # Contain the header of the section and a list of requirements
+    spack_sections = {
+        'core': ("{0} @*{{Core Functionalities}}", _core_requirements),
+        'buildcache': ("{0} @*{{Binary packages}}", _buildcache_requirements),
+        'optional': ("{0} @*{{Optional Features}}", _optional_requirements),
+        'develop': ("{0} @*{{Development Dependencies}}", _development_requirements)
+    }
+    msg, required_software = spack_sections[section]
+
+    with ensure_bootstrap_configuration():
+        missing_software = False
+        for found, err_msg in required_software():
+            if not found:
+                missing_software = True
+                msg += "\n  " + err_msg
+        msg += '\n'
+        msg = msg.format(pass_token if not missing_software else fail_token)
+    return msg, missing_software

lib/spack/spack/cmd/bootstrap.py

@@ -10,6 +10,8 @@
 import llnl.util.tty
 import llnl.util.tty.color
 
+import spack
+import spack.bootstrap
 import spack.cmd.common.arguments
 import spack.config
 import spack.main
@@ -32,6 +34,16 @@ def _add_scope_option(parser):
 def setup_parser(subparser):
     sp = subparser.add_subparsers(dest='subcommand')
 
+    status = sp.add_parser('status', help='get the status of Spack')
+    status.add_argument(
+        '--optional', action='store_true', default=False,
+        help='show the status of rarely used optional dependencies'
+    )
+    status.add_argument(
+        '--dev', action='store_true', default=False,
+        help='show the status of dependencies needed to develop Spack'
+    )
+
     enable = sp.add_parser('enable', help='enable bootstrapping')
     _add_scope_option(enable)
 
@@ -207,8 +219,39 @@ def _untrust(args):
     llnl.util.tty.msg(msg.format(args.name))
 
 
+def _status(args):
+    sections = ['core', 'buildcache']
+    if args.optional:
+        sections.append('optional')
+    if args.dev:
+        sections.append('develop')
+
+    header = "@*b{{Spack v{0} - {1}}}".format(
+        spack.spack_version, spack.bootstrap.spec_for_current_python()
+    )
+    print(llnl.util.tty.color.colorize(header))
+    print()
+    # Use the context manager here to avoid swapping between user and
+    # bootstrap config many times
+    missing = False
+    with spack.bootstrap.ensure_bootstrap_configuration():
+        for current_section in sections:
+            status_msg, fail = spack.bootstrap.status_message(section=current_section)
+            missing = missing or fail
+            if status_msg:
+                print(llnl.util.tty.color.colorize(status_msg))
+        print()
+    legend = ('Spack will take care of bootstrapping any missing dependency marked'
+              ' as [@*y{B}]. Dependencies marked as [@*y{-}] are instead required'
+              ' to be found on the system.')
+    if missing:
+        print(llnl.util.tty.color.colorize(legend))
+        print()
+
+
 def bootstrap(parser, args):
     callbacks = {
+        'status': _status,
         'enable': _enable_or_disable,
         'disable': _enable_or_disable,
         'reset': _reset,

lib/spack/spack/test/bootstrap.py

@@ -150,3 +150,20 @@ def test_nested_use_of_context_manager(mutable_config):
         with spack.bootstrap.ensure_bootstrap_configuration():
             assert spack.config.config != user_config
     assert spack.config.config == user_config
+
+
+@pytest.mark.parametrize('expected_missing', [False, True])
+def test_status_function_find_files(
+        mutable_config, mock_executable, tmpdir, monkeypatch, expected_missing
+):
+    if not expected_missing:
+        mock_executable('foo', 'echo Hello WWorld!')
+
+    monkeypatch.setattr(
+        spack.bootstrap, '_optional_requirements',
+        lambda: [spack.bootstrap._required_system_executable('foo', 'NOT FOUND')]
+    )
+    monkeypatch.setenv('PATH', str(tmpdir.join('bin')))
+
+    _, missing = spack.bootstrap.status_message('optional')
+    assert missing is expected_missing

share/spack/qa/run-unit-tests

@@ -38,6 +38,7 @@ bin/spack help -a
 
 # Profile and print top 20 lines for a simple call to spack spec
 spack -p --lines 20 spec mpileaks%gcc ^dyninst@10.0.0 ^elfutils@0.170
+$coverage_run $(which spack) bootstrap status --dev --optional
 
 #-----------------------------------------------------------
 # Run unit tests with code coverage

share/spack/spack-completion.bash

@@ -434,10 +434,14 @@ _spack_bootstrap() {
     then
         SPACK_COMPREPLY="-h --help"
     else
-        SPACK_COMPREPLY="enable disable reset root list trust untrust"
+        SPACK_COMPREPLY="status enable disable reset root list trust untrust"
     fi
 }
 
+_spack_bootstrap_status() {
+    SPACK_COMPREPLY="-h --help --optional --dev"
+}
+
 _spack_bootstrap_enable() {
     SPACK_COMPREPLY="-h --help --scope"
 }