introduce llnl.util.compat to remove sys.version_info checks (#21720)

- also split typing.py into typing_extensions and add py2 shims

lib/spack/external/py2/typing.py

@@ -6,79 +6,77 @@
 This is a fake set of symbols to allow spack to import typing in python
 versions where we do not support type checking (<3)
 """
-Annotated = None
-Any = None
-Callable = None
-ForwardRef = None
-Generic = None
-Literal = None
-Optional = None
-Tuple = None
-TypeVar = None
-Union = None
-AbstractSet = None
-ByteString = None
-Container = None
-Hashable = None
-ItemsView = None
-Iterable = None
-Iterator = None
-KeysView = None
-Mapping = None
-MappingView = None
-MutableMapping = None
-MutableSequence = None
-MutableSet = None
-Sequence = None
-Sized = None
-ValuesView = None
-Awaitable = None
-AsyncIterator = None
-AsyncIterable = None
-Coroutine = None
-Collection = None
-AsyncGenerator = None
-AsyncContextManager = None
-Reversible = None
-SupportsAbs = None
-SupportsBytes = None
-SupportsComplex = None
-SupportsFloat = None
-SupportsInt = None
-SupportsRound = None
-ChainMap = None
-Dict = None
-List = None
-OrderedDict = None
-Set = None
-FrozenSet = None
-NamedTuple = None
-Generator = None
-AnyStr = None
-cast = None
+from collections import defaultdict
+
+# (1) Unparameterized types.
+Annotated = object
+Any = object
+AnyStr = object
+ByteString = object
+Counter = object
+Final = object
+Hashable = object
+NoReturn = object
+Sized = object
+SupportsAbs = object
+SupportsBytes = object
+SupportsComplex = object
+SupportsFloat = object
+SupportsIndex = object
+SupportsInt = object
+SupportsRound = object
+
+# (2) Parameterized types.
+AbstractSet = defaultdict(lambda: object)
+AsyncContextManager = defaultdict(lambda: object)
+AsyncGenerator = defaultdict(lambda: object)
+AsyncIterable = defaultdict(lambda: object)
+AsyncIterator = defaultdict(lambda: object)
+Awaitable = defaultdict(lambda: object)
+Callable = defaultdict(lambda: object)
+ChainMap = defaultdict(lambda: object)
+ClassVar = defaultdict(lambda: object)
+Collection = defaultdict(lambda: object)
+Container = defaultdict(lambda: object)
+ContextManager = defaultdict(lambda: object)
+Coroutine = defaultdict(lambda: object)
+DefaultDict = defaultdict(lambda: object)
+Deque = defaultdict(lambda: object)
+Dict = defaultdict(lambda: object)
+ForwardRef = defaultdict(lambda: object)
+FrozenSet = defaultdict(lambda: object)
+Generator = defaultdict(lambda: object)
+Generic = defaultdict(lambda: object)
+ItemsView = defaultdict(lambda: object)
+Iterable = defaultdict(lambda: object)
+Iterator = defaultdict(lambda: object)
+KeysView = defaultdict(lambda: object)
+List = defaultdict(lambda: object)
+Literal = defaultdict(lambda: object)
+Mapping = defaultdict(lambda: object)
+MappingView = defaultdict(lambda: object)
+MutableMapping = defaultdict(lambda: object)
+MutableSequence = defaultdict(lambda: object)
+MutableSet = defaultdict(lambda: object)
+NamedTuple = defaultdict(lambda: object)
+Optional = defaultdict(lambda: object)
+OrderedDict = defaultdict(lambda: object)
+Reversible = defaultdict(lambda: object)
+Sequence = defaultdict(lambda: object)
+Set = defaultdict(lambda: object)
+Tuple = defaultdict(lambda: object)
+Type = defaultdict(lambda: object)
+TypedDict = defaultdict(lambda: object)
+Union = defaultdict(lambda: object)
+ValuesView = defaultdict(lambda: object)
+
+# (3) Type variable declarations.
+TypeVar = lambda *args, **kwargs: None
+
+# (4) Functions.
+cast = lambda _type, x: x
 get_args = None
 get_origin = None
 get_type_hints = None
 no_type_check = None
 no_type_check_decorator = None
-NoReturn = None
-
-# these are the typing extension symbols
-ClassVar = None
-Final = None
-Protocol = None
-Type = None
-TypedDict = None
-ContextManager = None
-Counter = None
-Deque = None
-DefaultDict = None
-SupportsIndex = None
-final = None
-IntVar = None
-Literal = None
-NewType = None
-overload = None
-runtime_checkable = None
-Text = None
-TYPE_CHECKING = None

lib/spack/external/py2/typing_extensions.py

@@ -0,0 +1,26 @@
+# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+"""
+This is a fake set of symbols to allow spack to import typing in python
+versions where we do not support type checking (<3)
+"""
+from collections import defaultdict
+
+# (1) Unparameterized types.
+IntVar = object
+Literal = object
+NewType = object
+Text = object
+
+# (2) Parameterized types.
+Protocol = defaultdict(lambda: object)
+
+# (3) Macro for avoiding evaluation except during type checking.
+TYPE_CHECKING = False
+
+# (4) Decorators.
+final = lambda x: x
+overload = lambda x: x
+runtime_checkable = lambda x: x

lib/spack/llnl/util/compat.py

@@ -0,0 +1,39 @@
+# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+# isort: off
+
+import sys
+
+if sys.version_info < (3,):
+    from itertools import ifilter as filter
+    from itertools import imap as map
+    from itertools import izip as zip
+    from itertools import izip_longest as zip_longest  # novm
+    from urllib import urlencode as urlencode
+    from urllib import urlopen as urlopen
+else:
+    filter = filter
+    map = map
+    zip = zip
+    from itertools import zip_longest as zip_longest  # novm # noqa: F401
+    from urllib.parse import urlencode as urlencode   # novm # noqa: F401
+    from urllib.request import urlopen as urlopen     # novm # noqa: F401
+
+if sys.version_info >= (3, 3):
+    from collections.abc import Hashable as Hashable                # novm
+    from collections.abc import Iterable as Iterable                # novm
+    from collections.abc import Mapping as Mapping                  # novm
+    from collections.abc import MutableMapping as MutableMapping    # novm
+    from collections.abc import MutableSequence as MutableSequence  # novm
+    from collections.abc import MutableSet as MutableSet            # novm
+    from collections.abc import Sequence as Sequence                # novm
+else:
+    from collections import Hashable as Hashable                # noqa: F401
+    from collections import Iterable as Iterable                # noqa: F401
+    from collections import Mapping as Mapping                  # noqa: F401
+    from collections import MutableMapping as MutableMapping    # noqa: F401
+    from collections import MutableSequence as MutableSequence  # noqa: F401
+    from collections import MutableSet as MutableSet            # noqa: F401
+    from collections import Sequence as Sequence                # noqa: F401

lib/spack/llnl/util/filesystem.py

@@ -21,15 +21,11 @@
 import six
 
 from llnl.util import tty
+from llnl.util.compat import Sequence
 from llnl.util.lang import dedupe, memoized
 
 from spack.util.executable import Executable
 
-if sys.version_info >= (3, 3):
-    from collections.abc import Sequence  # novm
-else:
-    from collections import Sequence
-
 __all__ = [
     'FileFilter',
     'FileList',

lib/spack/llnl/util/lang.py

@@ -14,17 +14,7 @@
 
 from six import string_types
 
-if sys.version_info < (3, 0):
-    from itertools import izip_longest  # novm
-    zip_longest = izip_longest
-else:
-    from itertools import zip_longest  # novm
-
-if sys.version_info >= (3, 3):
-    from collections.abc import Hashable, MutableMapping  # novm
-else:
-    from collections import Hashable, MutableMapping
-
+from llnl.util.compat import Hashable, MutableMapping, zip_longest
 
 # Ignore emacs backups when listing modules
 ignore_modules = [r'^\.#', '~$']

lib/spack/spack/ci_needs_workaround.py

@@ -3,18 +3,11 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
-import collections
-
-try:
-    # dynamically import to keep vermin from complaining
-    collections_abc = __import__('collections.abc')
-except ImportError:
-    collections_abc = collections
-
+from llnl.util.compat import Mapping
 
 get_job_name = lambda needs_entry: (
     needs_entry.get('job') if (
-        isinstance(needs_entry, collections_abc.Mapping) and
+        isinstance(needs_entry, Mapping) and
         needs_entry.get('artifacts', True))
 
     else
@@ -25,7 +18,7 @@
 
 
 def convert_job(job_entry):
-    if not isinstance(job_entry, collections_abc.Mapping):
+    if not isinstance(job_entry, Mapping):
         return job_entry
 
     needs = job_entry.get('needs')

lib/spack/spack/ci_optimization.py

@@ -3,28 +3,23 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
-import collections
-
-try:
-    # dynamically import to keep vermin from complaining
-    collections_abc = __import__('collections.abc')
-except ImportError:
-    collections_abc = collections
-
 import copy
 import hashlib
+from collections import defaultdict
+
+from llnl.util.compat import Mapping, Sequence
 
 import spack.util.spack_yaml as syaml
 
 
 def sort_yaml_obj(obj):
-    if isinstance(obj, collections_abc.Mapping):
+    if isinstance(obj, Mapping):
         return syaml.syaml_dict(
             (k, sort_yaml_obj(v))
             for k, v in
             sorted(obj.items(), key=(lambda item: str(item[0]))))
 
-    if isinstance(obj, collections_abc.Sequence) and not isinstance(obj, str):
+    if isinstance(obj, Sequence) and not isinstance(obj, str):
         return syaml.syaml_list(sort_yaml_obj(x) for x in obj)
 
     return obj
@@ -44,8 +39,8 @@ def matches(obj, proto):
 
     Precondition: proto must not have any reference cycles
     """
-    if isinstance(obj, collections_abc.Mapping):
-        if not isinstance(proto, collections_abc.Mapping):
+    if isinstance(obj, Mapping):
+        if not isinstance(proto, Mapping):
             return False
 
         return all(
@@ -53,10 +48,10 @@ def matches(obj, proto):
             for key, val in proto.items()
         )
 
-    if (isinstance(obj, collections_abc.Sequence) and
+    if (isinstance(obj, Sequence) and
             not isinstance(obj, str)):
 
-        if not (isinstance(proto, collections_abc.Sequence) and
+        if not (isinstance(proto, Sequence) and
                 not isinstance(proto, str)):
             return False
 
@@ -90,8 +85,8 @@ def subkeys(obj, proto):
 
     Otherwise, obj is returned.
     """
-    if not (isinstance(obj, collections_abc.Mapping) and
-            isinstance(proto, collections_abc.Mapping)):
+    if not (isinstance(obj, Mapping) and
+            isinstance(proto, Mapping)):
         return obj
 
     new_obj = {}
@@ -104,7 +99,7 @@ def subkeys(obj, proto):
             matches(proto[key], value)):
             continue
 
-        if isinstance(value, collections_abc.Mapping):
+        if isinstance(value, Mapping):
             new_obj[key] = subkeys(value, proto[key])
             continue
 
@@ -132,7 +127,7 @@ def add_extends(yaml, key):
     has_key = ('extends' in yaml)
     extends = yaml.get('extends')
 
-    if has_key and not isinstance(extends, (str, collections_abc.Sequence)):
+    if has_key and not isinstance(extends, (str, Sequence)):
         return
 
     if extends is None:
@@ -283,7 +278,7 @@ def build_histogram(iterator, key):
     The list is sorted in descending order by count, yielding the most
     frequently occuring hashes first.
     """
-    buckets = collections.defaultdict(int)
+    buckets = defaultdict(int)
     values = {}
 
     num_objects = 0

lib/spack/spack/cmd/python.py

@@ -73,7 +73,7 @@ def ipython_interpreter(args):
     support running a script or arguments
     """
     try:
-        import IPython
+        import IPython  # type: ignore[import]
     except ImportError:
         tty.die("ipython is not installed, install and try again.")
 

lib/spack/spack/directives.py

@@ -30,13 +30,13 @@ class OpenMpi(Package):
 import functools
 import os.path
 import re
-import sys
 from typing import List, Set  # novm
 
 import six
 
 import llnl.util.lang
 import llnl.util.tty.color
+from llnl.util.compat import Sequence
 
 import spack.error
 import spack.patch
@@ -48,12 +48,6 @@ class OpenMpi(Package):
 from spack.resource import Resource
 from spack.version import Version, VersionChecksumError
 
-if sys.version_info >= (3, 3):
-    from collections.abc import Sequence  # novm
-else:
-    from collections import Sequence
-
-
 __all__ = ['DirectiveError', 'DirectiveMeta']
 
 #: These are variant names used by Spack internally; packages can't use them

lib/spack/spack/filesystem_view.py

@@ -11,6 +11,7 @@
 import sys
 
 from llnl.util import tty
+from llnl.util.compat import filter, map, zip
 from llnl.util.filesystem import mkdirp, remove_dead_links, remove_empty_directories
 from llnl.util.lang import index_by, match_predicate
 from llnl.util.link_tree import LinkTree, MergeConflictError
@@ -29,12 +30,6 @@
 )
 from spack.error import SpackError
 
-# compatability
-if sys.version_info < (3, 0):
-    from itertools import ifilter as filter
-    from itertools import imap as map
-    from itertools import izip as zip
-
 __all__ = ["FilesystemView", "YamlFilesystemView"]
 
 

lib/spack/spack/mirror.py

@@ -21,12 +21,8 @@
 import ruamel.yaml.error as yaml_error
 import six
 
-if sys.version_info >= (3, 5):
-    from collections.abc import Mapping  # novm
-else:
-    from collections import Mapping
-
 import llnl.util.tty as tty
+from llnl.util.compat import Mapping
 from llnl.util.filesystem import mkdirp
 
 import spack.config

lib/spack/spack/package.py

@@ -1293,7 +1293,7 @@ def prefix(self):
         """Get the prefix into which this package should be installed."""
         return self.spec.prefix
 
-    @property  # type: ignore
+    @property  # type: ignore[misc]
     @memoized
     def compiler(self):
         """Get the spack.compiler.Compiler object used to build this package"""

lib/spack/spack/repo.py

@@ -18,18 +18,13 @@
 import types
 from typing import Dict  # novm
 
-import six
-
-if sys.version_info >= (3, 5):
-    from collections.abc import Mapping  # novm
-else:
-    from collections import Mapping
-
 import ruamel.yaml as yaml
+import six
 
 import llnl.util.filesystem as fs
 import llnl.util.lang
 import llnl.util.tty as tty
+from llnl.util.compat import Mapping
 
 import spack.caches
 import spack.config

lib/spack/spack/s3_handler.py

@@ -63,7 +63,7 @@ def _s3_open(url):
 class UrllibS3Handler(urllib_request.HTTPSHandler):
     def s3_open(self, req):
         orig_url = req.get_full_url()
-        from botocore.exceptions import ClientError
+        from botocore.exceptions import ClientError  # type: ignore[import]
         try:
             url, headers, stream = _s3_open(orig_url)
             return urllib_response.addinfourl(stream, headers, url)

lib/spack/spack/schema/environment.py

@@ -38,13 +38,9 @@ def parse(config_obj):
         config_obj: a configuration dictionary conforming to the
             schema definition for environment modifications
     """
-    import sys
+    from llnl.util.compat import Sequence
 
     import spack.util.environment as ev
-    if sys.version_info >= (3, 5):
-        from collections.abc import Sequence  # novm
-    else:
-        from collections import Sequence  # novm
 
     env = ev.EnvironmentModifications()
     for command, variable in config_obj.items():

lib/spack/spack/solver/asp.py

@@ -17,13 +17,15 @@
 
 import archspec.cpu
 
+from llnl.util.compat import Sequence
+
 try:
-    import clingo
+    import clingo  # type: ignore[import]
 
     # There may be a better way to detect this
     clingo_cffi = hasattr(clingo.Symbol, '_rep')
 except ImportError:
-    clingo = None  # type: ignore
+    clingo = None
     clingo_cffi = False
 
 import llnl.util.lang
@@ -49,11 +51,6 @@
 import spack.variant
 import spack.version
 
-if sys.version_info >= (3, 3):
-    from collections.abc import Sequence  # novm
-else:
-    from collections import Sequence
-
 # these are from clingo.ast and bootstrapped later
 ASTType = None
 parse_files = None

lib/spack/spack/spec.py

@@ -81,7 +81,6 @@
 import operator
 import os
 import re
-import sys
 import warnings
 
 import ruamel.yaml as yaml
@@ -91,6 +90,7 @@
 import llnl.util.lang as lang
 import llnl.util.tty as tty
 import llnl.util.tty.color as clr
+from llnl.util.compat import Mapping
 
 import spack.compiler
 import spack.compilers
@@ -117,12 +117,6 @@
 import spack.variant as vt
 import spack.version as vn
 
-if sys.version_info >= (3, 3):
-    from collections.abc import Mapping  # novm
-else:
-    from collections import Mapping
-
-
 __all__ = [
     'CompilerSpec',
     'Spec',

lib/spack/spack/test/ci.py

@@ -3,7 +3,6 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
-import collections
 import itertools as it
 import json
 import os
@@ -24,12 +23,6 @@
 import spack.util.gpg
 import spack.util.spack_yaml as syaml
 
-try:
-    # dynamically import to keep vermin from complaining
-    collections_abc = __import__('collections.abc')
-except ImportError:
-    collections_abc = collections
-
 
 @pytest.fixture
 def tmp_scope():

lib/spack/spack/test/cmd/versions.py

@@ -41,7 +41,7 @@ def test_remote_versions_only():
 @pytest.mark.usefixtures('mock_packages')
 def test_new_versions_only(monkeypatch):
     """Test a package for which new versions should be available."""
-    from spack.pkg.builtin.mock.brillig import Brillig
+    from spack.pkg.builtin.mock.brillig import Brillig  # type: ignore[import]
 
     def mock_fetch_remote_versions(*args, **kwargs):
         mock_remote_versions = {

lib/spack/spack/test/conftest.py

@@ -15,6 +15,7 @@
 import shutil
 import tempfile
 import xml.etree.ElementTree
+from typing import Dict  # novm
 
 import py
 import pytest
@@ -50,8 +51,8 @@
 #
 # Return list of shas for latest two git commits in local spack repo
 #
-@pytest.fixture
-def last_two_git_commits(scope='session'):
+@pytest.fixture(scope='session')
+def last_two_git_commits():
     git = spack.util.executable.which('git', required=True)
     spack_git_path = spack.paths.prefix
     with working_dir(spack_git_path):
@@ -1488,7 +1489,7 @@ def mock_test_repo(tmpdir_factory):
 class MockBundle(object):
     has_code = False
     name = 'mock-bundle'
-    versions = {}  # type: ignore
+    versions = {}  # type: Dict
 
 
 @pytest.fixture

lib/spack/spack/test/packages.py

@@ -55,13 +55,13 @@ def test_package_class_names(self):
     # Below tests target direct imports of spack packages from the
     # spack.pkg namespace
     def test_import_package(self):
-        import spack.pkg.builtin.mock.mpich             # noqa
+        import spack.pkg.builtin.mock.mpich  # type: ignore[import] # noqa
 
     def test_import_package_as(self):
-        import spack.pkg.builtin.mock.mpich as mp       # noqa
-        import spack.pkg.builtin.mock                   # noqa
-        import spack.pkg.builtin.mock as m              # noqa
-        from spack.pkg.builtin import mock              # noqa
+        import spack.pkg.builtin.mock  # noqa
+        import spack.pkg.builtin.mock as m  # noqa
+        import spack.pkg.builtin.mock.mpich as mp  # noqa
+        from spack.pkg.builtin import mock  # noqa
 
     def test_inheritance_of_diretives(self):
         p = spack.repo.get('simple-inheritance')
@@ -104,18 +104,18 @@ def test_import_class_from_package(self):
         from spack.pkg.builtin.mock.mpich import Mpich  # noqa
 
     def test_import_module_from_package(self):
-        from spack.pkg.builtin.mock import mpich        # noqa
+        from spack.pkg.builtin.mock import mpich  # noqa
 
     def test_import_namespace_container_modules(self):
-        import spack.pkg                                # noqa
-        import spack.pkg as p                           # noqa
-        from spack import pkg                           # noqa
-        import spack.pkg.builtin                        # noqa
-        import spack.pkg.builtin as b                   # noqa
-        from spack.pkg import builtin                   # noqa
-        import spack.pkg.builtin.mock                   # noqa
-        import spack.pkg.builtin.mock as m              # noqa
-        from spack.pkg.builtin import mock              # noqa
+        import spack.pkg  # noqa
+        import spack.pkg as p  # noqa
+        import spack.pkg.builtin  # noqa
+        import spack.pkg.builtin as b  # noqa
+        import spack.pkg.builtin.mock  # noqa
+        import spack.pkg.builtin.mock as m  # noqa
+        from spack import pkg  # noqa
+        from spack.pkg import builtin  # noqa
+        from spack.pkg.builtin import mock  # noqa
 
 
 @pytest.mark.regression('2737')

lib/spack/spack/test/spec_yaml.py

@@ -11,10 +11,11 @@
 import ast
 import inspect
 import os
-import sys
 
 import pytest
 
+from llnl.util.compat import Iterable, Mapping
+
 import spack.hash_types as ht
 import spack.spec
 import spack.util.spack_json as sjson
@@ -25,11 +26,6 @@
 from spack.util.mock_package import MockPackageMultiRepo
 from spack.util.spack_yaml import SpackYAMLError, syaml_dict
 
-if sys.version_info >= (3, 3):
-    from collections.abc import Iterable, Mapping  # novm
-else:
-    from collections import Iterable, Mapping
-
 
 def check_yaml_round_trip(spec):
     yaml_text = spec.to_yaml()

lib/spack/spack/util/environment.py

@@ -25,6 +25,7 @@
 import spack.platforms
 import spack.spec
 import spack.util.executable as executable
+import spack.util.spack_json as sjson
 
 system_paths = ['/', '/usr', '/usr/local']
 suffixes = ['bin', 'bin64', 'include', 'lib', 'lib64']
@@ -1027,13 +1028,7 @@ def _source_single_file(file_and_args, environment):
 
         # If we're in python2, convert to str objects instead of unicode
         # like json gives us.  We can't put unicode in os.environ anyway.
-        if sys.version_info[0] < 3:
-            environment = dict(
-                (k.encode('utf-8'), v.encode('utf-8'))
-                for k, v in environment.items()
-            )
-
-        return environment
+        return sjson.encode_json_dict(environment)
 
     current_environment = kwargs.get('env', dict(os.environ))
     for f in files:
@@ -1074,7 +1069,7 @@ def set_intersection(fullset, *args):
         return subset
 
     # Don't modify input, make a copy instead
-    environment = dict(environment)
+    environment = sjson.decode_json_dict(dict(environment))
 
     # Retain (whitelist) has priority over prune (blacklist)
     prune = set_intersection(set(environment), *blacklist)

lib/spack/spack/util/mock_package.py

@@ -103,7 +103,6 @@ def is_virtual(self, name, use_index=True):
         return False
 
     def repo_for_pkg(self, name):
-        import collections
         Repo = collections.namedtuple('Repo', ['namespace'])
         return Repo('mockrepo')
 

lib/spack/spack/util/pattern.py

@@ -5,12 +5,8 @@
 
 import functools
 import inspect
-import sys
 
-if sys.version_info >= (3, 3):
-    from collections.abc import MutableSequence  # novm
-else:
-    from collections import MutableSequence
+from llnl.util.compat import MutableSequence
 
 
 class Delegate(object):

lib/spack/spack/util/py2.py

@@ -0,0 +1,16 @@
+# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+import base64
+
+from six import PY3, binary_type, text_type
+
+
+def b32encode(digest):
+    # type: (binary_type) -> text_type
+    b32 = base64.b32encode(digest)
+    if PY3:
+        return b32.decode()
+    return b32

lib/spack/spack/util/s3.py

@@ -61,16 +61,16 @@ def create_s3_session(url, connection={}):
     # NOTE(opadron): import boto and friends as late as possible.  We don't
     # want to require boto as a dependency unless the user actually wants to
     # access S3 mirrors.
-    from boto3 import Session
-    from botocore.exceptions import ClientError
+    from boto3 import Session  # type: ignore[import]
+    from botocore.exceptions import ClientError  # type: ignore[import]
 
     s3_connection, s3_client_args = get_mirror_s3_connection_info(connection)
 
     session = Session(**s3_connection)
     # if no access credentials provided above, then access anonymously
     if not session.get_credentials():
-        from botocore import UNSIGNED
-        from botocore.client import Config
+        from botocore import UNSIGNED  # type: ignore[import]
+        from botocore.client import Config  # type: ignore[import]
 
         s3_client_args["config"] = Config(signature_version=UNSIGNED)
 

lib/spack/spack/util/spack_json.py

@@ -5,13 +5,13 @@
 
 """Simple wrapper around JSON to guarantee consistent use of load/dump. """
 import json
-import sys
+from typing import Any, Dict, Optional  # novm
 
-from six import iteritems, string_types
+from six import PY3, iteritems, string_types
 
 import spack.error
 
-__all__ = ['load', 'dump', 'SpackJSONError']
+__all__ = ['load', 'dump', 'SpackJSONError', 'encode_json_dict', 'decode_json_dict']
 
 _json_dump_args = {
     'indent': 2,
@@ -20,27 +20,45 @@
 
 
 def load(stream):
+    # type: (Any) -> Dict
     """Spack JSON needs to be ordered to support specs."""
     if isinstance(stream, string_types):
-        load = json.loads
+        load = json.loads       # type: ignore[assignment]
     else:
-        load = json.load
+        load = json.load        # type: ignore[assignment]
 
     return _strify(load(stream, object_hook=_strify), ignore_dicts=True)
 
 
+def encode_json_dict(data):
+    # type: (Dict) -> Dict
+    """Converts python 2 unicodes to str in JSON data."""
+    return _strify(data)
+
+
 def dump(data, stream=None):
+    # type: (Dict, Optional[Any]) -> Optional[str]
     """Dump JSON with a reasonable amount of indentation and separation."""
+    data = _strify(data)
     if stream is None:
-        return json.dumps(data, **_json_dump_args)
-    else:
-        return json.dump(data, stream, **_json_dump_args)
+        return json.dumps(data, **_json_dump_args)     # type: ignore[arg-type]
+    json.dump(data, stream, **_json_dump_args)         # type: ignore[arg-type]
+    return None
+
+
+def decode_json_dict(data):
+    # type: (Dict) -> Dict
+    """Converts str to python 2 unicodes in JSON data."""
+    return _strify(data)
 
 
 def _strify(data, ignore_dicts=False):
-    """Converts python 2 unicodes to str in JSON data."""
+    # type: (Dict, bool) -> Dict
+    """Helper method for ``encode_json_dict()`` and ``decode_json_dict()``.
+
+    Converts python 2 unicodes to str in JSON data, or the other way around."""
     # this is a no-op in python 3
-    if sys.version_info[0] >= 3:
+    if PY3:
         return data
 
     # if this is a unicode string in python 2, return its string representation
@@ -66,4 +84,5 @@ class SpackJSONError(spack.error.SpackError):
     """Raised when there are issues with JSON parsing."""
 
     def __init__(self, msg, json_error):
+        # type: (str, BaseException) -> None
         super(SpackJSONError, self).__init__(msg, str(json_error))

lib/spack/spack/util/spack_yaml.py

@@ -15,30 +15,23 @@
 import collections
 import ctypes
 import re
-import sys
 from typing import List  # novm
 
 import ruamel.yaml as yaml
 from ruamel.yaml import RoundTripDumper, RoundTripLoader
 from six import StringIO, string_types
 
+from llnl.util.compat import Mapping
 from llnl.util.tty.color import cextra, clen, colorize
 
 import spack.error
 
-if sys.version_info >= (3, 3):
-    from collections.abc import Mapping  # novm
-else:
-    from collections import Mapping
-
-
 # Only export load and dump
 __all__ = ['load', 'dump', 'SpackYAMLError']
 
+
 # Make new classes so we can add custom attributes.
 # Also, use OrderedDict instead of just dict.
-
-
 class syaml_dict(collections.OrderedDict):
     def __repr__(self):
         mappings = ('%r: %r' % (k, v) for k, v in self.items())

lib/spack/spack/variant.py

@@ -11,17 +11,12 @@
 import inspect
 import itertools
 import re
-import sys
 
 from six import StringIO
 
-if sys.version_info >= (3, 5):
-    from collections.abc import Sequence  # novm
-else:
-    from collections import Sequence
-
 import llnl.util.lang as lang
 import llnl.util.tty.color
+from llnl.util.compat import Sequence
 
 import spack.directives
 import spack.error as error

lib/spack/spack/verify.py

@@ -2,28 +2,22 @@
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
-import base64
 import hashlib
 import os
-import sys
 
 import llnl.util.tty as tty
 
 import spack.filesystem_view
 import spack.store
 import spack.util.file_permissions as fp
+import spack.util.py2 as compat
 import spack.util.spack_json as sjson
 
 
 def compute_hash(path):
     with open(path, 'rb') as f:
         sha1 = hashlib.sha1(f.read()).digest()
-        b32 = base64.b32encode(sha1)
-
-        if sys.version_info[0] >= 3:
-            b32 = b32.decode()
-
-        return b32
+        return compat.b32encode(sha1)
 
 
 def create_manifest_entry(path):

pyproject.toml

@@ -10,6 +10,7 @@ sections = [
 known_first_party = "spack"
 known_archspec = "archspec"
 known_llnl = "llnl"
+known_third_party = ["ruamel", "six"]
 src_paths = "lib"
 honor_noqa = true