Allow loading extensions through python entry-points (#42370)

This PR adds the ability to load spack extensions through `importlib.metadata` entry 
points, in addition to the regular configuration variable.

It requires Python 3.8 or greater to be properly supported.

lib/spack/docs/configuration.rst

@@ -73,9 +73,12 @@ are six configuration scopes. From lowest to highest:
    Spack instance per project) or for site-wide settings on a multi-user
    machine (e.g., for a common Spack instance).
 
+#. **plugin**: Read from a Python project's entry points. Settings here affect
+   all instances of Spack running with the same Python installation.  This scope takes higher precedence than site, system, and default scopes.
+
 #. **user**: Stored in the home directory: ``~/.spack/``. These settings
    affect all instances of Spack and take higher precedence than site,
-   system, or defaults scopes.
+   system, plugin, or defaults scopes.
 
 #. **custom**: Stored in a custom directory specified by ``--config-scope``.
    If multiple scopes are listed on the command line, they are ordered
@@ -196,6 +199,45 @@ with MPICH. You can create different configuration scopes for use with
                mpi: [mpich]
 
 
+.. _plugin-scopes:
+
+^^^^^^^^^^^^^
+Plugin scopes
+^^^^^^^^^^^^^
+
+.. note::
+   Python version >= 3.8 is required to enable plugin configuration.
+
+Spack can be made aware of configuration scopes that are installed as part of a python package.  To do so, register a function that returns the scope's path to the ``"spack.config"`` entry point.  Consider the Python package ``my_package`` that includes Spack configurations:
+
+.. code-block:: console
+
+  my-package/
+  ├── src
+  │   ├── my_package
+  │   │   ├── __init__.py
+  │   │   └── spack/
+  │   │   │   └── config.yaml
+  └── pyproject.toml
+
+adding the following to ``my_package``'s ``pyproject.toml`` will make ``my_package``'s ``spack/`` configurations visible to Spack when ``my_package`` is installed:
+
+.. code-block:: toml
+
+   [project.entry_points."spack.config"]
+   my_package = "my_package:get_config_path"
+
+The function ``my_package.get_extension_path`` in ``my_package/__init__.py`` might look like
+
+.. code-block:: python
+
+   import importlib.resources
+
+   def get_config_path():
+       dirname = importlib.resources.files("my_package").joinpath("spack")
+       if dirname.exists():
+           return str(dirname)
+
 .. _platform-scopes:
 
 ------------------------

lib/spack/docs/extensions.rst

@@ -111,3 +111,39 @@ The corresponding unit tests can be run giving the appropriate options to ``spac
 
    (5 durations < 0.005s hidden.  Use -vv to show these durations.)
    =========================================== 5 passed in 5.06s ============================================
+
+---------------------------------------
+Registering Extensions via Entry Points
+---------------------------------------
+
+.. note::
+   Python version >= 3.8 is required to register extensions via entry points.
+
+Spack can be made aware of extensions that are installed as part of a python package.  To do so, register a function that returns the extension path, or paths, to the ``"spack.extensions"`` entry point.  Consider the Python package ``my_package`` that includes a Spack extension:
+
+.. code-block:: console
+
+  my-package/
+  ├── src
+  │   ├── my_package
+  │   │   └── __init__.py
+  │   └── spack-scripting/  # the spack extensions
+  └── pyproject.toml
+
+adding the following to ``my_package``'s ``pyproject.toml`` will make the ``spack-scripting`` extension visible to Spack when ``my_package`` is installed:
+
+.. code-block:: toml
+
+   [project.entry_points."spack.extenions"]
+   my_package = "my_package:get_extension_path"
+
+The function ``my_package.get_extension_path`` in ``my_package/__init__.py`` might look like
+
+.. code-block:: python
+
+   import importlib.resources
+
+   def get_extension_path():
+       dirname = importlib.resources.files("my_package").joinpath("spack-scripting")
+       if dirname.exists():
+           return str(dirname)

lib/spack/llnl/util/lang.py

@@ -12,6 +12,7 @@
 import re
 import sys
 import traceback
+import warnings
 from datetime import datetime, timedelta
 from typing import Any, Callable, Iterable, List, Tuple
 
@@ -843,6 +844,48 @@ def __repr__(self):
         return repr(self.instance)
 
 
+def get_entry_points(*, group: str):
+    """Wrapper for ``importlib.metadata.entry_points``
+
+    Adapted from https://github.com/HypothesisWorks/hypothesis/blob/0a90ed6edf56319149956c7321d4110078a5c228/hypothesis-python/src/hypothesis/entry_points.py
+
+    Args:
+        group (str): the group of entry points to select
+
+    Returns:
+        EntryPoints for ``group``
+
+    """
+
+    try:
+        try:
+            from importlib import metadata as importlib_metadata  # type: ignore  # novermin
+        except ImportError:
+            import importlib_metadata  # type: ignore  # mypy thinks this is a redefinition
+        try:
+            entry_points = importlib_metadata.entry_points(group=group)
+        except TypeError:
+            # Prior to Python 3.10, entry_points accepted no parameters and always
+            # returned a dictionary of entry points, keyed by group.  See
+            # https://docs.python.org/3/library/importlib.metadata.html#entry-points
+            entry_points = importlib_metadata.entry_points().get(group, [])
+        yield from entry_points
+    except ImportError:
+        # But if we're not on Python >= 3.8 and the importlib_metadata backport
+        # is not installed, we fall back to pkg_resources anyway.
+        try:
+            import pkg_resources  # type: ignore
+        except ImportError:
+            warnings.warn(
+                "Under Python <= 3.7, Spack requires either the importlib_metadata "
+                "or setuptools package in order to load extensions via entrypoints.",
+                ImportWarning,
+            )
+            yield from ()
+        else:
+            yield from pkg_resources.iter_entry_points(group)
+
+
 def load_module_from_file(module_name, module_path):
     """Loads a python module from the path of the corresponding file.
 

lib/spack/spack/config.py

@@ -764,6 +764,31 @@ def _add_platform_scope(
     cfg.push_scope(scope_type(plat_name, plat_path))
 
 
+def config_paths_from_entry_points() -> List[Tuple[str, str]]:
+    """Load configuration paths from entry points
+
+    A python package can register entry point metadata so that Spack can find
+    its configuration by adding the following to the project's pyproject.toml:
+
+    .. code-block:: toml
+
+       [project.entry-points."spack.config"]
+       baz = "baz:get_spack_config_path"
+
+    The function ``get_spack_config_path`` returns the path to the package's
+    spack configuration scope
+
+    """
+    config_paths: List[Tuple[str, str]] = []
+    for entry_point in lang.get_entry_points(group="spack.config"):
+        hook = entry_point.load()
+        if callable(hook):
+            config_path = hook()
+            if config_path and os.path.exists(config_path):
+                config_paths.append(("plugin-%s" % entry_point.name, str(config_path)))
+    return config_paths
+
+
 def _add_command_line_scopes(
     cfg: Union[Configuration, lang.Singleton], command_line_scopes: List[str]
 ) -> None:
@@ -816,6 +841,9 @@ def create() -> Configuration:
     # No site-level configs should be checked into spack by default.
     configuration_paths.append(("site", os.path.join(spack.paths.etc_path)))
 
+    # Python package's can register configuration scopes via entry_points
+    configuration_paths.extend(config_paths_from_entry_points())
+
     # User configuration can override both spack defaults and site config
     # This is disabled if user asks for no local configuration.
     if not disable_local_config:

lib/spack/spack/extensions.py

@@ -12,6 +12,7 @@
 import re
 import sys
 import types
+from pathlib import Path
 from typing import List
 
 import llnl.util.lang
@@ -132,10 +133,38 @@ def load_extension(name: str) -> str:
 def get_extension_paths():
     """Return the list of canonicalized extension paths from config:extensions."""
     extension_paths = spack.config.get("config:extensions") or []
+    extension_paths.extend(extension_paths_from_entry_points())
     paths = [spack.util.path.canonicalize_path(p) for p in extension_paths]
     return paths
 
 
+def extension_paths_from_entry_points() -> List[str]:
+    """Load extensions from a Python package's entry points.
+
+    A python package can register entry point metadata so that Spack can find
+    its extensions by adding the following to the project's pyproject.toml:
+
+    .. code-block:: toml
+
+       [project.entry-points."spack.extensions"]
+       baz = "baz:get_spack_extensions"
+
+    The function ``get_spack_extensions`` returns paths to the package's
+    spack extensions
+
+    """
+    extension_paths: List[str] = []
+    for entry_point in llnl.util.lang.get_entry_points(group="spack.extensions"):
+        hook = entry_point.load()
+        if callable(hook):
+            paths = hook() or []
+            if isinstance(paths, (Path, str)):
+                extension_paths.append(str(paths))
+            else:
+                extension_paths.extend(paths)
+    return extension_paths
+
+
 def get_command_paths():
     """Return the list of paths where to search for command files."""
     command_paths = []

lib/spack/spack/test/entry_points.py

@@ -0,0 +1,112 @@
+# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+import os
+import sys
+
+import pytest
+
+import spack.config
+import spack.extensions
+
+
+class MockConfigEntryPoint:
+    def __init__(self, tmp_path):
+        self.dir = tmp_path
+        self.name = "mypackage_config"
+
+    def load(self):
+        etc_path = self.dir.joinpath("spack/etc")
+        etc_path.mkdir(exist_ok=True, parents=True)
+        f = self.dir / "spack/etc/config.yaml"
+        with open(f, "w") as fh:
+            fh.write("config:\n  install_tree:\n    root: /spam/opt\n")
+
+        def ep():
+            return self.dir / "spack/etc"
+
+        return ep
+
+
+class MockExtensionsEntryPoint:
+    def __init__(self, tmp_path):
+        self.dir = tmp_path
+        self.name = "mypackage_extensions"
+
+    def load(self):
+        cmd_path = self.dir.joinpath("spack/spack-myext/myext/cmd")
+        cmd_path.mkdir(exist_ok=True, parents=True)
+        f = self.dir / "spack/spack-myext/myext/cmd/spam.py"
+        with open(f, "w") as fh:
+            fh.write("description = 'hello world extension command'\n")
+            fh.write("section = 'test command'\n")
+            fh.write("level = 'long'\n")
+            fh.write("def setup_parser(subparser):\n    pass\n")
+            fh.write("def spam(parser, args):\n    print('spam for all!')\n")
+
+        def ep():
+            return self.dir / "spack/spack-myext"
+
+        return ep
+
+
+def entry_points_factory(tmp_path):
+    def entry_points(group=None):
+        if group == "spack.config":
+            return (MockConfigEntryPoint(tmp_path),)
+        elif group == "spack.extensions":
+            return (MockExtensionsEntryPoint(tmp_path),)
+        return ()
+
+    return entry_points
+
+
+@pytest.fixture()
+def mock_entry_points(tmp_path, monkeypatch):
+    entry_points = entry_points_factory(tmp_path)
+    try:
+        try:
+            import importlib.metadata as importlib_metadata  # type: ignore # novermin
+        except ImportError:
+            import importlib_metadata
+        monkeypatch.setattr(importlib_metadata, "entry_points", entry_points)
+    except ImportError:
+        try:
+            import pkg_resources  # type: ignore
+        except ImportError:
+            return
+        monkeypatch.setattr(pkg_resources, "iter_entry_points", entry_points)
+
+
+@pytest.mark.skipif(sys.version_info[:2] < (3, 8), reason="Python>=3.8 required")
+def test_spack_entry_point_config(tmp_path, mock_entry_points):
+    """Test config scope entry point"""
+    config_paths = dict(spack.config.config_paths_from_entry_points())
+    config_path = config_paths.get("plugin-mypackage_config")
+    my_config_path = tmp_path / "spack/etc"
+    if config_path is None:
+        raise ValueError("Did not find entry point config in %s" % str(config_paths))
+    else:
+        assert os.path.samefile(config_path, my_config_path)
+    config = spack.config.create()
+    assert config.get("config:install_tree:root", scope="plugin-mypackage_config") == "/spam/opt"
+
+
+@pytest.mark.skipif(sys.version_info[:2] < (3, 8), reason="Python>=3.8 required")
+def test_spack_entry_point_extension(tmp_path, mock_entry_points):
+    """Test config scope entry point"""
+    my_ext = tmp_path / "spack/spack-myext"
+    extensions = spack.extensions.get_extension_paths()
+    found = bool([ext for ext in extensions if os.path.samefile(ext, my_ext)])
+    if not found:
+        raise ValueError("Did not find extension in %s" % ", ".join(extensions))
+    extensions = spack.extensions.extension_paths_from_entry_points()
+    found = bool([ext for ext in extensions if os.path.samefile(ext, my_ext)])
+    if not found:
+        raise ValueError("Did not find extension in %s" % ", ".join(extensions))
+    root = spack.extensions.load_extension("myext")
+    assert os.path.samefile(root, my_ext)
+    module = spack.extensions.get_module("spam")
+    assert module is not None

pyproject.toml

@@ -154,11 +154,13 @@ ignore_missing_imports = true
       'boto3',
       'botocore',
       'distro',
+      'importlib.metadata',
       'jinja2',
       'jsonschema',
       'macholib',
       'markupsafe',
       'numpy',
+      'pkg_resources',
       'pyristent',
       'pytest',
       'ruamel.yaml',