Use "vendoring" to manage 3rd party dependencies

lib/spack/docs/conf.py

@@ -36,7 +36,7 @@
 if not os.path.exists(link_name):
     os.symlink(os.path.abspath("../../.."), link_name, target_is_directory=True)
 sys.path.insert(0, os.path.abspath("_spack_root/lib/spack/external"))
-sys.path.insert(0, os.path.abspath("_spack_root/lib/spack/external/pytest-fallback"))
+sys.path.insert(0, os.path.abspath("_spack_root/lib/spack/external/_vendoring"))
 sys.path.append(os.path.abspath("_spack_root/lib/spack/"))
 
 # Add the Spack bin directory to the path so that we can use its output in docs.

lib/spack/external/__init__.py

@@ -11,7 +11,7 @@
 
 * Homepage: https://altgraph.readthedocs.io/en/latest/index.html
 * Usage: dependency of macholib
-* Version: 0.17.2
+* Version: 0.17.3
 
 archspec
 --------
@@ -20,17 +20,6 @@
 * Usage: Labeling, comparison and detection of microarchitectures
 * Version: 0.2.0 (commit e44bad9c7b6defac73696f64078b2fe634719b62)
 
-argparse
---------
-
-* Homepage: https://pypi.python.org/pypi/argparse
-* Usage: We include our own version to be Python 3.X compatible.
-* Version: 1.4.0
-* Note: This package has been slightly modified to improve
-  error message formatting. See the following commit if the
-  vendored copy ever needs to be updated again:
-  https://github.com/spack/spack/pull/6786/commits/dfcef577b77249106ea4e4c69a6cd9e64fa6c418
-
 astunparse
 ----------------
 
@@ -52,7 +41,7 @@
 
 * Homepage: https://github.com/python-attrs/attrs
 * Usage: Needed by jsonschema.
-* Version: 21.2.0 (83d3cd70f90a3f4d19ee8b508e58d1c58821c0ad)
+* Version: 22.1.0
 
 ctest_log_parser
 ----------------
@@ -67,21 +56,14 @@
 
 * Homepage: https://pypi.python.org/pypi/distro
 * Usage: Provides a more stable linux distribution detection.
-* Version: 1.6.0 (64946a1e2a9ff529047070657728600e006c99ff)
+* Version: 1.8.0
-* Note: Last version supporting Python 2.7
-
-functools32
------------
-* Homepage: https://github.com/MiCHiLU/python-functools32
-* Usage: Needed by jsonschema when using Python 2.7.
-* Version: 3.2.3-2
 
 jinja2
 ------
 
 * Homepage: https://pypi.python.org/pypi/Jinja2
 * Usage: A modern and designer-friendly templating language for Python.
-* Version: 2.11.3 (last version supporting Python 2.7)
+* Version: 3.0.3 (last version supporting Python 3.6)
 
 jsonschema
 ----------
@@ -96,44 +78,21 @@
 
 * Homepage: https://macholib.readthedocs.io/en/latest/index.html#
 * Usage: Manipulation of Mach-o binaries for relocating macOS buildcaches on Linux
-* Version: 1.15.2
+* Version: 1.16.2
 
 markupsafe
 ----------
 
 * Homepage: https://pypi.python.org/pypi/MarkupSafe
 * Usage: Implements a XML/HTML/XHTML Markup safe string for Python.
-* Version: 1.1.1 (last version supporting Python 2.7)
+* Version: 2.0.1 (last version supporting Python 3.6)
-
-py
---
-
-* Homepage: https://pypi.python.org/pypi/py
-* Usage: Needed by pytest. Library with cross-python path,
-  ini-parsing, io, code, and log facilities.
-* Version: 1.4.34 (last version supporting Python 2.6)
-* Note: This packages has been modified:
-  * https://github.com/pytest-dev/py/pull/186 was backported
 
 pyrsistent
 ----------
 
 * Homepage: http://github.com/tobgu/pyrsistent/
 * Usage: Needed by `jsonschema`
-* Version: 0.16.1 (last version supporting Python 2.7)
+* Version: 0.18.0
-* Note: We only include the parts needed for `jsonschema`.
-
-pytest
-------
-
-* Homepage: https://pypi.python.org/pypi/pytest
-* Usage: Testing framework used by Spack.
-* Version: 3.2.5 (last version supporting Python 2.6)
-* Note: This package has been slightly modified:
-  * We improve Python 2.6 compatibility. See:
-    https://github.com/spack/spack/pull/6801.
-  * We have patched pytest not to depend on setuptools. See:
-    https://github.com/spack/spack/pull/15612
 
 ruamel.yaml
 ------

lib/spack/external/patches/jsonschema.patch

@@ -0,0 +1,15 @@
+diff --git a/lib/spack/external/_vendoring/jsonschema/__init__.py b/lib/spack/external/_vendoring/jsonschema/__init__.py
+index 6b630cdfbb..1791fe7fbf 100644
+--- a/lib/spack/external/_vendoring/jsonschema/__init__.py
++++ b/lib/spack/external/_vendoring/jsonschema/__init__.py
+@@ -27,8 +27,5 @@
+     RefResolver,
+     validate,
+ )
+-try:
+-    from importlib import metadata
+-except ImportError: # for Python<3.8
+-    import importlib_metadata as metadata
+-__version__ = metadata.version("jsonschema")
++
++__version__ = "3.2.0"

lib/spack/external/vendor.txt

@@ -0,0 +1,9 @@
+distro==1.8.0
+jsonschema==3.2.0
+  attrs==22.1.0
+  pyrsistent==0.18.0
+jinja2==3.0.3
+  markupsafe==2.0.1
+six==1.16.0
+macholib==1.16.2
+  altgraph==0.17.3

lib/spack/spack/operating_systems/cray_backend.py

@@ -83,7 +83,7 @@ def __init__(self):
         if version:
             # If we found a CrayOS version, we do not want the information
             # from LinuxDistro. In order to skip the logic from
-            # external.distro.linux_distribution, while still calling __init__
+            # distro.linux_distribution, while still calling __init__
             # methods further up the MRO, we skip LinuxDistro in the MRO and
             # call the OperatingSystem superclass __init__ method
             super(LinuxDistro, self).__init__(name, version)

lib/spack/spack/operating_systems/linux_distro.py

@@ -15,9 +15,9 @@ def kernel_version():
     """Return the kernel version as a Version object.
     Note that the kernel version is distinct from OS and/or
     distribution versions. For instance:
-    >>> external.distro.id()
+    >>> distro.id()
     'centos'
-    >>> external.distro.version()
+    >>> distro.version()
     '7'
     >>> platform.release()
     '5.10.84+'
@@ -39,9 +39,9 @@ class LinuxDistro(OperatingSystem):
     def __init__(self):
         try:
             # This will throw an error if imported on a non-Linux platform.
-            import external.distro
+            import distro
 
-            distname, version = external.distro.id(), external.distro.version()
+            distname, version = distro.id(), distro.version()
         except ImportError:
             distname, version = "unknown", ""
 

lib/spack/spack/test/cmd/ci.py

@@ -9,8 +9,8 @@
 import shutil
 import sys
 
+import jsonschema
 import pytest
-from jsonschema import ValidationError, validate
 
 from llnl.util.filesystem import mkdirp, working_dir
 
@@ -1313,7 +1313,7 @@ def test_push_mirror_contents(
             index_path = os.path.join(buildcache_path, "index.json")
             with open(index_path) as idx_fd:
                 index_object = json.load(idx_fd)
-                validate(index_object, db_idx_schema)
+                jsonschema.validate(index_object, db_idx_schema)
 
             # Now that index is regenerated, validate "buildcache list" output
             buildcache_list_output = buildcache_cmd("list", output=str)
@@ -1325,7 +1325,7 @@ def test_push_mirror_contents(
                     spec_json_path = os.path.join(buildcache_path, file_name)
                     with open(spec_json_path) as json_fd:
                         json_object = Spec.extract_json_from_clearsig(json_fd.read())
-                        validate(json_object, specfile_schema)
+                        jsonschema.validate(json_object, specfile_schema)
 
             logs_dir = working_dir.join("logs_dir")
             if not os.path.exists(logs_dir.strpath):
@@ -1630,7 +1630,7 @@ def test_ci_rebuild_index(
             index_path = os.path.join(buildcache_path, "index.json")
             with open(index_path) as idx_fd:
                 index_object = json.load(idx_fd)
-                validate(index_object, db_idx_schema)
+                jsonschema.validate(index_object, db_idx_schema)
 
 
 def test_ci_generate_bootstrap_prune_dag(
@@ -1911,21 +1911,21 @@ def test_ensure_only_one_temporary_storage():
 
     # User can specify "enable-artifacts-buildcache" (boolean)
     yaml_obj = syaml.load(gitlab_ci_template.format(enable_artifacts))
-    validate(yaml_obj, gitlab_ci_schema)
+    jsonschema.validate(yaml_obj, gitlab_ci_schema)
 
     # User can also specify "temporary-storage-url-prefix" (string)
     yaml_obj = syaml.load(gitlab_ci_template.format(temp_storage))
-    validate(yaml_obj, gitlab_ci_schema)
+    jsonschema.validate(yaml_obj, gitlab_ci_schema)
 
     # However, specifying both should fail to validate
     yaml_obj = syaml.load(gitlab_ci_template.format(specify_both))
-    with pytest.raises(ValidationError):
+    with pytest.raises(jsonschema.ValidationError):
-        validate(yaml_obj, gitlab_ci_schema)
+        jsonschema.validate(yaml_obj, gitlab_ci_schema)
 
     # Specifying neither should be fine too, as neither of these properties
     # should be required
     yaml_obj = syaml.load(gitlab_ci_template.format(specify_neither))
-    validate(yaml_obj, gitlab_ci_schema)
+    jsonschema.validate(yaml_obj, gitlab_ci_schema)
 
 
 def test_ci_generate_temp_storage_url(

lib/spack/spack/test/database.py

@@ -20,7 +20,7 @@
     _use_uuid = False
     pass
 
-from jsonschema import validate
+import jsonschema
 
 import llnl.util.lock as lk
 from llnl.util.tty.colify import colify
@@ -456,7 +456,7 @@ def test_005_db_exists(database):
 
     with open(index_file) as fd:
         index_object = json.load(fd)
-        validate(index_object, schema)
+        jsonschema.validate(index_object, schema)
 
 
 def test_010_all_install_sanity(database):
@@ -750,7 +750,7 @@ def test_old_external_entries_prefix(mutable_database):
     with open(spack.store.db._index_path, "r") as f:
         db_obj = json.loads(f.read())
 
-    validate(db_obj, schema)
+    jsonschema.validate(db_obj, schema)
 
     s = spack.spec.Spec("externaltool")
     s.concretize()

lib/spack/spack_installable/main.py

@@ -18,6 +18,7 @@ def main(argv=None):
 
     # Add external libs
     spack_external_libs = os.path.join(spack_lib_path, "external")
+    sys.path.insert(0, os.path.join(spack_external_libs, "_vendoring"))
     sys.path.insert(0, spack_external_libs)
     # Here we delete ruamel.yaml in case it has been already imported from site
     # (see #9206 for a broader description of the issue).

pyproject.toml

@@ -4,8 +4,6 @@ description="The spack package manager"
 dependencies=[
   "clingo",
   "setuptools",
-  "six",
-  "types-six",
 ]
 dynamic = ["version"]
 
@@ -21,8 +19,8 @@ dev = [
   "pytest",
   "pytest-xdist",
   "setuptools",
-  "click==8.0.2",
+  "click",
-  'black==21.12b0',
+  'black',
   "mypy",
   "isort",
   "flake8",
@@ -195,3 +193,43 @@ ignore_errors = true
 
 [tool.coverage.html]
 directory = "htmlcov"
+
+[tool.vendoring]
+destination = "lib/spack/external/_vendoring"
+requirements = "lib/spack/external/vendor.txt"
+namespace = ""
+
+protected-files = ["__init__.py", "README.rst", "vendor.txt"]
+patches-dir = "lib/spack/external/patches"
+
+[tool.vendoring.transformations]
+substitute = [
+]
+drop = [
+  # contains unnecessary scripts
+  "bin/",
+  # interpreter and OS specific msgpack libs
+  "msgpack/*.so",
+  # unneeded parts of setuptools
+  "easy_install.py",
+  "setuptools",
+  "pkg_resources/_vendor/",
+  "pkg_resources/extern/",
+  # trim vendored pygments styles and lexers
+  "pygments/styles/[!_]*.py",
+  '^pygments/lexers/(?!python|__init__|_mapping).*\.py$',
+  # trim rich's markdown support
+  "rich/markdown.py",
+]
+
+[tool.vendoring.typing-stubs]
+six = ["six.__init__", "six.moves.__init__", "six.moves.configparser"]
+distro = []
+
+[tool.vendoring.license.directories]
+setuptools = "pkg_resources"
+
+[tool.vendoring.license.fallback-urls]
+CacheControl = "https://raw.githubusercontent.com/ionrock/cachecontrol/v0.12.6/LICENSE.txt"
+distlib = "https://bitbucket.org/pypa/distlib/raw/master/LICENSE.txt"
+webencodings = "https://github.com/SimonSapin/python-webencodings/raw/master/LICENSE"