From 7e468aefd5dcc8b12c4e543ba3f9411dd3775759 Mon Sep 17 00:00:00 2001 From: Tim Fuller Date: Wed, 6 Mar 2024 03:18:49 -0700 Subject: [PATCH] Allow loading extensions through python entry-points (#42370) This PR adds the ability to load spack extensions through `importlib.metadata` entry points, in addition to the regular configuration variable. It requires Python 3.8 or greater to be properly supported. --- lib/spack/docs/configuration.rst | 44 ++++++++++- lib/spack/docs/extensions.rst | 36 +++++++++ lib/spack/llnl/util/lang.py | 43 ++++++++++ lib/spack/spack/config.py | 28 +++++++ lib/spack/spack/extensions.py | 29 +++++++ lib/spack/spack/test/entry_points.py | 112 +++++++++++++++++++++++++++ pyproject.toml | 2 + 7 files changed, 293 insertions(+), 1 deletion(-) create mode 100644 lib/spack/spack/test/entry_points.py diff --git a/lib/spack/docs/configuration.rst b/lib/spack/docs/configuration.rst index a4b60e43e3..1d216b6925 100644 --- a/lib/spack/docs/configuration.rst +++ b/lib/spack/docs/configuration.rst @@ -73,9 +73,12 @@ are six configuration scopes. From lowest to highest: Spack instance per project) or for site-wide settings on a multi-user machine (e.g., for a common Spack instance). +#. **plugin**: Read from a Python project's entry points. Settings here affect + all instances of Spack running with the same Python installation. This scope takes higher precedence than site, system, and default scopes. + #. **user**: Stored in the home directory: ``~/.spack/``. These settings affect all instances of Spack and take higher precedence than site, - system, or defaults scopes. + system, plugin, or defaults scopes. #. **custom**: Stored in a custom directory specified by ``--config-scope``. If multiple scopes are listed on the command line, they are ordered @@ -196,6 +199,45 @@ with MPICH. You can create different configuration scopes for use with mpi: [mpich] +.. _plugin-scopes: + +^^^^^^^^^^^^^ +Plugin scopes +^^^^^^^^^^^^^ + +.. note:: + Python version >= 3.8 is required to enable plugin configuration. + +Spack can be made aware of configuration scopes that are installed as part of a python package. To do so, register a function that returns the scope's path to the ``"spack.config"`` entry point. Consider the Python package ``my_package`` that includes Spack configurations: + +.. code-block:: console + + my-package/ + ├── src + │   ├── my_package + │   │   ├── __init__.py + │   │   └── spack/ + │   │   │   └── config.yaml + └── pyproject.toml + +adding the following to ``my_package``'s ``pyproject.toml`` will make ``my_package``'s ``spack/`` configurations visible to Spack when ``my_package`` is installed: + +.. code-block:: toml + + [project.entry_points."spack.config"] + my_package = "my_package:get_config_path" + +The function ``my_package.get_extension_path`` in ``my_package/__init__.py`` might look like + +.. code-block:: python + + import importlib.resources + + def get_config_path(): + dirname = importlib.resources.files("my_package").joinpath("spack") + if dirname.exists(): + return str(dirname) + .. _platform-scopes: ------------------------ diff --git a/lib/spack/docs/extensions.rst b/lib/spack/docs/extensions.rst index 2d82c2ba84..0879645b73 100644 --- a/lib/spack/docs/extensions.rst +++ b/lib/spack/docs/extensions.rst @@ -111,3 +111,39 @@ The corresponding unit tests can be run giving the appropriate options to ``spac (5 durations < 0.005s hidden. Use -vv to show these durations.) =========================================== 5 passed in 5.06s ============================================ + +--------------------------------------- +Registering Extensions via Entry Points +--------------------------------------- + +.. note:: + Python version >= 3.8 is required to register extensions via entry points. + +Spack can be made aware of extensions that are installed as part of a python package. To do so, register a function that returns the extension path, or paths, to the ``"spack.extensions"`` entry point. Consider the Python package ``my_package`` that includes a Spack extension: + +.. code-block:: console + + my-package/ + ├── src + │   ├── my_package + │   │   └── __init__.py + │   └── spack-scripting/ # the spack extensions + └── pyproject.toml + +adding the following to ``my_package``'s ``pyproject.toml`` will make the ``spack-scripting`` extension visible to Spack when ``my_package`` is installed: + +.. code-block:: toml + + [project.entry_points."spack.extenions"] + my_package = "my_package:get_extension_path" + +The function ``my_package.get_extension_path`` in ``my_package/__init__.py`` might look like + +.. code-block:: python + + import importlib.resources + + def get_extension_path(): + dirname = importlib.resources.files("my_package").joinpath("spack-scripting") + if dirname.exists(): + return str(dirname) diff --git a/lib/spack/llnl/util/lang.py b/lib/spack/llnl/util/lang.py index 4f14a29ef8..ddca65c381 100644 --- a/lib/spack/llnl/util/lang.py +++ b/lib/spack/llnl/util/lang.py @@ -12,6 +12,7 @@ import re import sys import traceback +import warnings from datetime import datetime, timedelta from typing import Any, Callable, Iterable, List, Tuple @@ -843,6 +844,48 @@ def __repr__(self): return repr(self.instance) +def get_entry_points(*, group: str): + """Wrapper for ``importlib.metadata.entry_points`` + + Adapted from https://github.com/HypothesisWorks/hypothesis/blob/0a90ed6edf56319149956c7321d4110078a5c228/hypothesis-python/src/hypothesis/entry_points.py + + Args: + group (str): the group of entry points to select + + Returns: + EntryPoints for ``group`` + + """ + + try: + try: + from importlib import metadata as importlib_metadata # type: ignore # novermin + except ImportError: + import importlib_metadata # type: ignore # mypy thinks this is a redefinition + try: + entry_points = importlib_metadata.entry_points(group=group) + except TypeError: + # Prior to Python 3.10, entry_points accepted no parameters and always + # returned a dictionary of entry points, keyed by group. See + # https://docs.python.org/3/library/importlib.metadata.html#entry-points + entry_points = importlib_metadata.entry_points().get(group, []) + yield from entry_points + except ImportError: + # But if we're not on Python >= 3.8 and the importlib_metadata backport + # is not installed, we fall back to pkg_resources anyway. + try: + import pkg_resources # type: ignore + except ImportError: + warnings.warn( + "Under Python <= 3.7, Spack requires either the importlib_metadata " + "or setuptools package in order to load extensions via entrypoints.", + ImportWarning, + ) + yield from () + else: + yield from pkg_resources.iter_entry_points(group) + + def load_module_from_file(module_name, module_path): """Loads a python module from the path of the corresponding file. diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index cc15a7d36e..982b730f16 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -764,6 +764,31 @@ def _add_platform_scope( cfg.push_scope(scope_type(plat_name, plat_path)) +def config_paths_from_entry_points() -> List[Tuple[str, str]]: + """Load configuration paths from entry points + + A python package can register entry point metadata so that Spack can find + its configuration by adding the following to the project's pyproject.toml: + + .. code-block:: toml + + [project.entry-points."spack.config"] + baz = "baz:get_spack_config_path" + + The function ``get_spack_config_path`` returns the path to the package's + spack configuration scope + + """ + config_paths: List[Tuple[str, str]] = [] + for entry_point in lang.get_entry_points(group="spack.config"): + hook = entry_point.load() + if callable(hook): + config_path = hook() + if config_path and os.path.exists(config_path): + config_paths.append(("plugin-%s" % entry_point.name, str(config_path))) + return config_paths + + def _add_command_line_scopes( cfg: Union[Configuration, lang.Singleton], command_line_scopes: List[str] ) -> None: @@ -816,6 +841,9 @@ def create() -> Configuration: # No site-level configs should be checked into spack by default. configuration_paths.append(("site", os.path.join(spack.paths.etc_path))) + # Python package's can register configuration scopes via entry_points + configuration_paths.extend(config_paths_from_entry_points()) + # User configuration can override both spack defaults and site config # This is disabled if user asks for no local configuration. if not disable_local_config: diff --git a/lib/spack/spack/extensions.py b/lib/spack/spack/extensions.py index b7b30e135c..a561c50ecf 100644 --- a/lib/spack/spack/extensions.py +++ b/lib/spack/spack/extensions.py @@ -12,6 +12,7 @@ import re import sys import types +from pathlib import Path from typing import List import llnl.util.lang @@ -132,10 +133,38 @@ def load_extension(name: str) -> str: def get_extension_paths(): """Return the list of canonicalized extension paths from config:extensions.""" extension_paths = spack.config.get("config:extensions") or [] + extension_paths.extend(extension_paths_from_entry_points()) paths = [spack.util.path.canonicalize_path(p) for p in extension_paths] return paths +def extension_paths_from_entry_points() -> List[str]: + """Load extensions from a Python package's entry points. + + A python package can register entry point metadata so that Spack can find + its extensions by adding the following to the project's pyproject.toml: + + .. code-block:: toml + + [project.entry-points."spack.extensions"] + baz = "baz:get_spack_extensions" + + The function ``get_spack_extensions`` returns paths to the package's + spack extensions + + """ + extension_paths: List[str] = [] + for entry_point in llnl.util.lang.get_entry_points(group="spack.extensions"): + hook = entry_point.load() + if callable(hook): + paths = hook() or [] + if isinstance(paths, (Path, str)): + extension_paths.append(str(paths)) + else: + extension_paths.extend(paths) + return extension_paths + + def get_command_paths(): """Return the list of paths where to search for command files.""" command_paths = [] diff --git a/lib/spack/spack/test/entry_points.py b/lib/spack/spack/test/entry_points.py new file mode 100644 index 0000000000..6a7c543850 --- /dev/null +++ b/lib/spack/spack/test/entry_points.py @@ -0,0 +1,112 @@ +# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +import os +import sys + +import pytest + +import spack.config +import spack.extensions + + +class MockConfigEntryPoint: + def __init__(self, tmp_path): + self.dir = tmp_path + self.name = "mypackage_config" + + def load(self): + etc_path = self.dir.joinpath("spack/etc") + etc_path.mkdir(exist_ok=True, parents=True) + f = self.dir / "spack/etc/config.yaml" + with open(f, "w") as fh: + fh.write("config:\n install_tree:\n root: /spam/opt\n") + + def ep(): + return self.dir / "spack/etc" + + return ep + + +class MockExtensionsEntryPoint: + def __init__(self, tmp_path): + self.dir = tmp_path + self.name = "mypackage_extensions" + + def load(self): + cmd_path = self.dir.joinpath("spack/spack-myext/myext/cmd") + cmd_path.mkdir(exist_ok=True, parents=True) + f = self.dir / "spack/spack-myext/myext/cmd/spam.py" + with open(f, "w") as fh: + fh.write("description = 'hello world extension command'\n") + fh.write("section = 'test command'\n") + fh.write("level = 'long'\n") + fh.write("def setup_parser(subparser):\n pass\n") + fh.write("def spam(parser, args):\n print('spam for all!')\n") + + def ep(): + return self.dir / "spack/spack-myext" + + return ep + + +def entry_points_factory(tmp_path): + def entry_points(group=None): + if group == "spack.config": + return (MockConfigEntryPoint(tmp_path),) + elif group == "spack.extensions": + return (MockExtensionsEntryPoint(tmp_path),) + return () + + return entry_points + + +@pytest.fixture() +def mock_entry_points(tmp_path, monkeypatch): + entry_points = entry_points_factory(tmp_path) + try: + try: + import importlib.metadata as importlib_metadata # type: ignore # novermin + except ImportError: + import importlib_metadata + monkeypatch.setattr(importlib_metadata, "entry_points", entry_points) + except ImportError: + try: + import pkg_resources # type: ignore + except ImportError: + return + monkeypatch.setattr(pkg_resources, "iter_entry_points", entry_points) + + +@pytest.mark.skipif(sys.version_info[:2] < (3, 8), reason="Python>=3.8 required") +def test_spack_entry_point_config(tmp_path, mock_entry_points): + """Test config scope entry point""" + config_paths = dict(spack.config.config_paths_from_entry_points()) + config_path = config_paths.get("plugin-mypackage_config") + my_config_path = tmp_path / "spack/etc" + if config_path is None: + raise ValueError("Did not find entry point config in %s" % str(config_paths)) + else: + assert os.path.samefile(config_path, my_config_path) + config = spack.config.create() + assert config.get("config:install_tree:root", scope="plugin-mypackage_config") == "/spam/opt" + + +@pytest.mark.skipif(sys.version_info[:2] < (3, 8), reason="Python>=3.8 required") +def test_spack_entry_point_extension(tmp_path, mock_entry_points): + """Test config scope entry point""" + my_ext = tmp_path / "spack/spack-myext" + extensions = spack.extensions.get_extension_paths() + found = bool([ext for ext in extensions if os.path.samefile(ext, my_ext)]) + if not found: + raise ValueError("Did not find extension in %s" % ", ".join(extensions)) + extensions = spack.extensions.extension_paths_from_entry_points() + found = bool([ext for ext in extensions if os.path.samefile(ext, my_ext)]) + if not found: + raise ValueError("Did not find extension in %s" % ", ".join(extensions)) + root = spack.extensions.load_extension("myext") + assert os.path.samefile(root, my_ext) + module = spack.extensions.get_module("spam") + assert module is not None diff --git a/pyproject.toml b/pyproject.toml index 0a4d094d1a..06aec6fc43 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -154,11 +154,13 @@ ignore_missing_imports = true 'boto3', 'botocore', 'distro', + 'importlib.metadata', 'jinja2', 'jsonschema', 'macholib', 'markupsafe', 'numpy', + 'pkg_resources', 'pyristent', 'pytest', 'ruamel.yaml',