From 86378502f9fda95725c535614d587d4c345aad08 Mon Sep 17 00:00:00 2001 From: Massimiliano Culpo Date: Wed, 16 Nov 2022 15:39:33 +0100 Subject: [PATCH] Use "vendoring" to manage 3rd party dependencies --- lib/spack/docs/conf.py | 2 +- lib/spack/external/__init__.py | 55 +++---------------- lib/spack/external/patches/jsonschema.patch | 15 +++++ lib/spack/external/vendor.txt | 9 +++ .../spack/operating_systems/cray_backend.py | 2 +- .../spack/operating_systems/linux_distro.py | 8 +-- lib/spack/spack/test/cmd/ci.py | 18 +++--- lib/spack/spack/test/database.py | 6 +- lib/spack/spack_installable/main.py | 1 + pyproject.toml | 46 ++++++++++++++-- 10 files changed, 92 insertions(+), 70 deletions(-) create mode 100644 lib/spack/external/patches/jsonschema.patch create mode 100644 lib/spack/external/vendor.txt diff --git a/lib/spack/docs/conf.py b/lib/spack/docs/conf.py index fe6e081c7d..842c84c4e5 100644 --- a/lib/spack/docs/conf.py +++ b/lib/spack/docs/conf.py @@ -36,7 +36,7 @@ if not os.path.exists(link_name): os.symlink(os.path.abspath("../../.."), link_name, target_is_directory=True) sys.path.insert(0, os.path.abspath("_spack_root/lib/spack/external")) -sys.path.insert(0, os.path.abspath("_spack_root/lib/spack/external/pytest-fallback")) +sys.path.insert(0, os.path.abspath("_spack_root/lib/spack/external/_vendoring")) sys.path.append(os.path.abspath("_spack_root/lib/spack/")) # Add the Spack bin directory to the path so that we can use its output in docs. diff --git a/lib/spack/external/__init__.py b/lib/spack/external/__init__.py index ce5924a9a5..7a41e77059 100644 --- a/lib/spack/external/__init__.py +++ b/lib/spack/external/__init__.py @@ -11,7 +11,7 @@ * Homepage: https://altgraph.readthedocs.io/en/latest/index.html * Usage: dependency of macholib -* Version: 0.17.2 +* Version: 0.17.3 archspec -------- @@ -20,17 +20,6 @@ * Usage: Labeling, comparison and detection of microarchitectures * Version: 0.2.0 (commit e44bad9c7b6defac73696f64078b2fe634719b62) -argparse --------- - -* Homepage: https://pypi.python.org/pypi/argparse -* Usage: We include our own version to be Python 3.X compatible. -* Version: 1.4.0 -* Note: This package has been slightly modified to improve - error message formatting. See the following commit if the - vendored copy ever needs to be updated again: - https://github.com/spack/spack/pull/6786/commits/dfcef577b77249106ea4e4c69a6cd9e64fa6c418 - astunparse ---------------- @@ -52,7 +41,7 @@ * Homepage: https://github.com/python-attrs/attrs * Usage: Needed by jsonschema. -* Version: 21.2.0 (83d3cd70f90a3f4d19ee8b508e58d1c58821c0ad) +* Version: 22.1.0 ctest_log_parser ---------------- @@ -67,21 +56,14 @@ * Homepage: https://pypi.python.org/pypi/distro * Usage: Provides a more stable linux distribution detection. -* Version: 1.6.0 (64946a1e2a9ff529047070657728600e006c99ff) -* Note: Last version supporting Python 2.7 - -functools32 ------------ -* Homepage: https://github.com/MiCHiLU/python-functools32 -* Usage: Needed by jsonschema when using Python 2.7. -* Version: 3.2.3-2 +* Version: 1.8.0 jinja2 ------ * Homepage: https://pypi.python.org/pypi/Jinja2 * Usage: A modern and designer-friendly templating language for Python. -* Version: 2.11.3 (last version supporting Python 2.7) +* Version: 3.0.3 (last version supporting Python 3.6) jsonschema ---------- @@ -96,44 +78,21 @@ * Homepage: https://macholib.readthedocs.io/en/latest/index.html# * Usage: Manipulation of Mach-o binaries for relocating macOS buildcaches on Linux -* Version: 1.15.2 +* Version: 1.16.2 markupsafe ---------- * Homepage: https://pypi.python.org/pypi/MarkupSafe * Usage: Implements a XML/HTML/XHTML Markup safe string for Python. -* Version: 1.1.1 (last version supporting Python 2.7) - -py --- - -* Homepage: https://pypi.python.org/pypi/py -* Usage: Needed by pytest. Library with cross-python path, - ini-parsing, io, code, and log facilities. -* Version: 1.4.34 (last version supporting Python 2.6) -* Note: This packages has been modified: - * https://github.com/pytest-dev/py/pull/186 was backported +* Version: 2.0.1 (last version supporting Python 3.6) pyrsistent ---------- * Homepage: http://github.com/tobgu/pyrsistent/ * Usage: Needed by `jsonschema` -* Version: 0.16.1 (last version supporting Python 2.7) -* Note: We only include the parts needed for `jsonschema`. - -pytest ------- - -* Homepage: https://pypi.python.org/pypi/pytest -* Usage: Testing framework used by Spack. -* Version: 3.2.5 (last version supporting Python 2.6) -* Note: This package has been slightly modified: - * We improve Python 2.6 compatibility. See: - https://github.com/spack/spack/pull/6801. - * We have patched pytest not to depend on setuptools. See: - https://github.com/spack/spack/pull/15612 +* Version: 0.18.0 ruamel.yaml ------ diff --git a/lib/spack/external/patches/jsonschema.patch b/lib/spack/external/patches/jsonschema.patch new file mode 100644 index 0000000000..d22d87a3d6 --- /dev/null +++ b/lib/spack/external/patches/jsonschema.patch @@ -0,0 +1,15 @@ +diff --git a/lib/spack/external/_vendoring/jsonschema/__init__.py b/lib/spack/external/_vendoring/jsonschema/__init__.py +index 6b630cdfbb..1791fe7fbf 100644 +--- a/lib/spack/external/_vendoring/jsonschema/__init__.py ++++ b/lib/spack/external/_vendoring/jsonschema/__init__.py +@@ -27,8 +27,5 @@ + RefResolver, + validate, + ) +-try: +- from importlib import metadata +-except ImportError: # for Python<3.8 +- import importlib_metadata as metadata +-__version__ = metadata.version("jsonschema") ++ ++__version__ = "3.2.0" diff --git a/lib/spack/external/vendor.txt b/lib/spack/external/vendor.txt new file mode 100644 index 0000000000..3080ef110e --- /dev/null +++ b/lib/spack/external/vendor.txt @@ -0,0 +1,9 @@ +distro==1.8.0 +jsonschema==3.2.0 + attrs==22.1.0 + pyrsistent==0.18.0 +jinja2==3.0.3 + markupsafe==2.0.1 +six==1.16.0 +macholib==1.16.2 + altgraph==0.17.3 diff --git a/lib/spack/spack/operating_systems/cray_backend.py b/lib/spack/spack/operating_systems/cray_backend.py index 0076bc7df5..9220520926 100644 --- a/lib/spack/spack/operating_systems/cray_backend.py +++ b/lib/spack/spack/operating_systems/cray_backend.py @@ -83,7 +83,7 @@ def __init__(self): if version: # If we found a CrayOS version, we do not want the information # from LinuxDistro. In order to skip the logic from - # external.distro.linux_distribution, while still calling __init__ + # distro.linux_distribution, while still calling __init__ # methods further up the MRO, we skip LinuxDistro in the MRO and # call the OperatingSystem superclass __init__ method super(LinuxDistro, self).__init__(name, version) diff --git a/lib/spack/spack/operating_systems/linux_distro.py b/lib/spack/spack/operating_systems/linux_distro.py index 403d3a0d1c..4eaf5da2ef 100644 --- a/lib/spack/spack/operating_systems/linux_distro.py +++ b/lib/spack/spack/operating_systems/linux_distro.py @@ -15,9 +15,9 @@ def kernel_version(): """Return the kernel version as a Version object. Note that the kernel version is distinct from OS and/or distribution versions. For instance: - >>> external.distro.id() + >>> distro.id() 'centos' - >>> external.distro.version() + >>> distro.version() '7' >>> platform.release() '5.10.84+' @@ -39,9 +39,9 @@ class LinuxDistro(OperatingSystem): def __init__(self): try: # This will throw an error if imported on a non-Linux platform. - import external.distro + import distro - distname, version = external.distro.id(), external.distro.version() + distname, version = distro.id(), distro.version() except ImportError: distname, version = "unknown", "" diff --git a/lib/spack/spack/test/cmd/ci.py b/lib/spack/spack/test/cmd/ci.py index 034ea89c22..3033862c9b 100644 --- a/lib/spack/spack/test/cmd/ci.py +++ b/lib/spack/spack/test/cmd/ci.py @@ -9,8 +9,8 @@ import shutil import sys +import jsonschema import pytest -from jsonschema import ValidationError, validate from llnl.util.filesystem import mkdirp, working_dir @@ -1313,7 +1313,7 @@ def test_push_mirror_contents( index_path = os.path.join(buildcache_path, "index.json") with open(index_path) as idx_fd: index_object = json.load(idx_fd) - validate(index_object, db_idx_schema) + jsonschema.validate(index_object, db_idx_schema) # Now that index is regenerated, validate "buildcache list" output buildcache_list_output = buildcache_cmd("list", output=str) @@ -1325,7 +1325,7 @@ def test_push_mirror_contents( spec_json_path = os.path.join(buildcache_path, file_name) with open(spec_json_path) as json_fd: json_object = Spec.extract_json_from_clearsig(json_fd.read()) - validate(json_object, specfile_schema) + jsonschema.validate(json_object, specfile_schema) logs_dir = working_dir.join("logs_dir") if not os.path.exists(logs_dir.strpath): @@ -1630,7 +1630,7 @@ def test_ci_rebuild_index( index_path = os.path.join(buildcache_path, "index.json") with open(index_path) as idx_fd: index_object = json.load(idx_fd) - validate(index_object, db_idx_schema) + jsonschema.validate(index_object, db_idx_schema) def test_ci_generate_bootstrap_prune_dag( @@ -1911,21 +1911,21 @@ def test_ensure_only_one_temporary_storage(): # User can specify "enable-artifacts-buildcache" (boolean) yaml_obj = syaml.load(gitlab_ci_template.format(enable_artifacts)) - validate(yaml_obj, gitlab_ci_schema) + jsonschema.validate(yaml_obj, gitlab_ci_schema) # User can also specify "temporary-storage-url-prefix" (string) yaml_obj = syaml.load(gitlab_ci_template.format(temp_storage)) - validate(yaml_obj, gitlab_ci_schema) + jsonschema.validate(yaml_obj, gitlab_ci_schema) # However, specifying both should fail to validate yaml_obj = syaml.load(gitlab_ci_template.format(specify_both)) - with pytest.raises(ValidationError): - validate(yaml_obj, gitlab_ci_schema) + with pytest.raises(jsonschema.ValidationError): + jsonschema.validate(yaml_obj, gitlab_ci_schema) # Specifying neither should be fine too, as neither of these properties # should be required yaml_obj = syaml.load(gitlab_ci_template.format(specify_neither)) - validate(yaml_obj, gitlab_ci_schema) + jsonschema.validate(yaml_obj, gitlab_ci_schema) def test_ci_generate_temp_storage_url( diff --git a/lib/spack/spack/test/database.py b/lib/spack/spack/test/database.py index 387daba1b5..2d37f80dba 100644 --- a/lib/spack/spack/test/database.py +++ b/lib/spack/spack/test/database.py @@ -20,7 +20,7 @@ _use_uuid = False pass -from jsonschema import validate +import jsonschema import llnl.util.lock as lk from llnl.util.tty.colify import colify @@ -456,7 +456,7 @@ def test_005_db_exists(database): with open(index_file) as fd: index_object = json.load(fd) - validate(index_object, schema) + jsonschema.validate(index_object, schema) def test_010_all_install_sanity(database): @@ -750,7 +750,7 @@ def test_old_external_entries_prefix(mutable_database): with open(spack.store.db._index_path, "r") as f: db_obj = json.loads(f.read()) - validate(db_obj, schema) + jsonschema.validate(db_obj, schema) s = spack.spec.Spec("externaltool") s.concretize() diff --git a/lib/spack/spack_installable/main.py b/lib/spack/spack_installable/main.py index 7b4c40b8d9..8aca9d0ed7 100644 --- a/lib/spack/spack_installable/main.py +++ b/lib/spack/spack_installable/main.py @@ -18,6 +18,7 @@ def main(argv=None): # Add external libs spack_external_libs = os.path.join(spack_lib_path, "external") + sys.path.insert(0, os.path.join(spack_external_libs, "_vendoring")) sys.path.insert(0, spack_external_libs) # Here we delete ruamel.yaml in case it has been already imported from site # (see #9206 for a broader description of the issue). diff --git a/pyproject.toml b/pyproject.toml index dda109cff5..fa7bd81d69 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,8 +4,6 @@ description="The spack package manager" dependencies=[ "clingo", "setuptools", - "six", - "types-six", ] dynamic = ["version"] @@ -21,8 +19,8 @@ dev = [ "pytest", "pytest-xdist", "setuptools", - "click==8.0.2", - 'black==21.12b0', + "click", + 'black', "mypy", "isort", "flake8", @@ -195,3 +193,43 @@ ignore_errors = true [tool.coverage.html] directory = "htmlcov" + +[tool.vendoring] +destination = "lib/spack/external/_vendoring" +requirements = "lib/spack/external/vendor.txt" +namespace = "" + +protected-files = ["__init__.py", "README.rst", "vendor.txt"] +patches-dir = "lib/spack/external/patches" + +[tool.vendoring.transformations] +substitute = [ +] +drop = [ + # contains unnecessary scripts + "bin/", + # interpreter and OS specific msgpack libs + "msgpack/*.so", + # unneeded parts of setuptools + "easy_install.py", + "setuptools", + "pkg_resources/_vendor/", + "pkg_resources/extern/", + # trim vendored pygments styles and lexers + "pygments/styles/[!_]*.py", + '^pygments/lexers/(?!python|__init__|_mapping).*\.py$', + # trim rich's markdown support + "rich/markdown.py", +] + +[tool.vendoring.typing-stubs] +six = ["six.__init__", "six.moves.__init__", "six.moves.configparser"] +distro = [] + +[tool.vendoring.license.directories] +setuptools = "pkg_resources" + +[tool.vendoring.license.fallback-urls] +CacheControl = "https://raw.githubusercontent.com/ionrock/cachecontrol/v0.12.6/LICENSE.txt" +distlib = "https://bitbucket.org/pypa/distlib/raw/master/LICENSE.txt" +webencodings = "https://github.com/SimonSapin/python-webencodings/raw/master/LICENSE" \ No newline at end of file