diff --git a/lib/spack/external/py2/typing.py b/lib/spack/external/py2/typing.py index cc32de8844..7c36962d9f 100644 --- a/lib/spack/external/py2/typing.py +++ b/lib/spack/external/py2/typing.py @@ -6,79 +6,77 @@ This is a fake set of symbols to allow spack to import typing in python versions where we do not support type checking (<3) """ -Annotated = None -Any = None -Callable = None -ForwardRef = None -Generic = None -Literal = None -Optional = None -Tuple = None -TypeVar = None -Union = None -AbstractSet = None -ByteString = None -Container = None -Hashable = None -ItemsView = None -Iterable = None -Iterator = None -KeysView = None -Mapping = None -MappingView = None -MutableMapping = None -MutableSequence = None -MutableSet = None -Sequence = None -Sized = None -ValuesView = None -Awaitable = None -AsyncIterator = None -AsyncIterable = None -Coroutine = None -Collection = None -AsyncGenerator = None -AsyncContextManager = None -Reversible = None -SupportsAbs = None -SupportsBytes = None -SupportsComplex = None -SupportsFloat = None -SupportsInt = None -SupportsRound = None -ChainMap = None -Dict = None -List = None -OrderedDict = None -Set = None -FrozenSet = None -NamedTuple = None -Generator = None -AnyStr = None -cast = None +from collections import defaultdict + +# (1) Unparameterized types. +Annotated = object +Any = object +AnyStr = object +ByteString = object +Counter = object +Final = object +Hashable = object +NoReturn = object +Sized = object +SupportsAbs = object +SupportsBytes = object +SupportsComplex = object +SupportsFloat = object +SupportsIndex = object +SupportsInt = object +SupportsRound = object + +# (2) Parameterized types. +AbstractSet = defaultdict(lambda: object) +AsyncContextManager = defaultdict(lambda: object) +AsyncGenerator = defaultdict(lambda: object) +AsyncIterable = defaultdict(lambda: object) +AsyncIterator = defaultdict(lambda: object) +Awaitable = defaultdict(lambda: object) +Callable = defaultdict(lambda: object) +ChainMap = defaultdict(lambda: object) +ClassVar = defaultdict(lambda: object) +Collection = defaultdict(lambda: object) +Container = defaultdict(lambda: object) +ContextManager = defaultdict(lambda: object) +Coroutine = defaultdict(lambda: object) +DefaultDict = defaultdict(lambda: object) +Deque = defaultdict(lambda: object) +Dict = defaultdict(lambda: object) +ForwardRef = defaultdict(lambda: object) +FrozenSet = defaultdict(lambda: object) +Generator = defaultdict(lambda: object) +Generic = defaultdict(lambda: object) +ItemsView = defaultdict(lambda: object) +Iterable = defaultdict(lambda: object) +Iterator = defaultdict(lambda: object) +KeysView = defaultdict(lambda: object) +List = defaultdict(lambda: object) +Literal = defaultdict(lambda: object) +Mapping = defaultdict(lambda: object) +MappingView = defaultdict(lambda: object) +MutableMapping = defaultdict(lambda: object) +MutableSequence = defaultdict(lambda: object) +MutableSet = defaultdict(lambda: object) +NamedTuple = defaultdict(lambda: object) +Optional = defaultdict(lambda: object) +OrderedDict = defaultdict(lambda: object) +Reversible = defaultdict(lambda: object) +Sequence = defaultdict(lambda: object) +Set = defaultdict(lambda: object) +Tuple = defaultdict(lambda: object) +Type = defaultdict(lambda: object) +TypedDict = defaultdict(lambda: object) +Union = defaultdict(lambda: object) +ValuesView = defaultdict(lambda: object) + +# (3) Type variable declarations. +TypeVar = lambda *args, **kwargs: None + +# (4) Functions. +cast = lambda _type, x: x get_args = None get_origin = None get_type_hints = None no_type_check = None no_type_check_decorator = None -NoReturn = None - -# these are the typing extension symbols -ClassVar = None -Final = None -Protocol = None -Type = None -TypedDict = None -ContextManager = None -Counter = None -Deque = None -DefaultDict = None -SupportsIndex = None -final = None -IntVar = None -Literal = None -NewType = None -overload = None -runtime_checkable = None -Text = None -TYPE_CHECKING = None diff --git a/lib/spack/external/py2/typing_extensions.py b/lib/spack/external/py2/typing_extensions.py new file mode 100644 index 0000000000..ca6bc10999 --- /dev/null +++ b/lib/spack/external/py2/typing_extensions.py @@ -0,0 +1,26 @@ +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +""" +This is a fake set of symbols to allow spack to import typing in python +versions where we do not support type checking (<3) +""" +from collections import defaultdict + +# (1) Unparameterized types. +IntVar = object +Literal = object +NewType = object +Text = object + +# (2) Parameterized types. +Protocol = defaultdict(lambda: object) + +# (3) Macro for avoiding evaluation except during type checking. +TYPE_CHECKING = False + +# (4) Decorators. +final = lambda x: x +overload = lambda x: x +runtime_checkable = lambda x: x diff --git a/lib/spack/llnl/util/compat.py b/lib/spack/llnl/util/compat.py new file mode 100644 index 0000000000..ca914d0fb6 --- /dev/null +++ b/lib/spack/llnl/util/compat.py @@ -0,0 +1,39 @@ +# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +# isort: off + +import sys + +if sys.version_info < (3,): + from itertools import ifilter as filter + from itertools import imap as map + from itertools import izip as zip + from itertools import izip_longest as zip_longest # novm + from urllib import urlencode as urlencode + from urllib import urlopen as urlopen +else: + filter = filter + map = map + zip = zip + from itertools import zip_longest as zip_longest # novm # noqa: F401 + from urllib.parse import urlencode as urlencode # novm # noqa: F401 + from urllib.request import urlopen as urlopen # novm # noqa: F401 + +if sys.version_info >= (3, 3): + from collections.abc import Hashable as Hashable # novm + from collections.abc import Iterable as Iterable # novm + from collections.abc import Mapping as Mapping # novm + from collections.abc import MutableMapping as MutableMapping # novm + from collections.abc import MutableSequence as MutableSequence # novm + from collections.abc import MutableSet as MutableSet # novm + from collections.abc import Sequence as Sequence # novm +else: + from collections import Hashable as Hashable # noqa: F401 + from collections import Iterable as Iterable # noqa: F401 + from collections import Mapping as Mapping # noqa: F401 + from collections import MutableMapping as MutableMapping # noqa: F401 + from collections import MutableSequence as MutableSequence # noqa: F401 + from collections import MutableSet as MutableSet # noqa: F401 + from collections import Sequence as Sequence # noqa: F401 diff --git a/lib/spack/llnl/util/filesystem.py b/lib/spack/llnl/util/filesystem.py index d44812e5ae..f9f2b3162f 100644 --- a/lib/spack/llnl/util/filesystem.py +++ b/lib/spack/llnl/util/filesystem.py @@ -21,15 +21,11 @@ import six from llnl.util import tty +from llnl.util.compat import Sequence from llnl.util.lang import dedupe, memoized from spack.util.executable import Executable -if sys.version_info >= (3, 3): - from collections.abc import Sequence # novm -else: - from collections import Sequence - __all__ = [ 'FileFilter', 'FileList', diff --git a/lib/spack/llnl/util/lang.py b/lib/spack/llnl/util/lang.py index 180cedac88..c806cab1d7 100644 --- a/lib/spack/llnl/util/lang.py +++ b/lib/spack/llnl/util/lang.py @@ -14,17 +14,7 @@ from six import string_types -if sys.version_info < (3, 0): - from itertools import izip_longest # novm - zip_longest = izip_longest -else: - from itertools import zip_longest # novm - -if sys.version_info >= (3, 3): - from collections.abc import Hashable, MutableMapping # novm -else: - from collections import Hashable, MutableMapping - +from llnl.util.compat import Hashable, MutableMapping, zip_longest # Ignore emacs backups when listing modules ignore_modules = [r'^\.#', '~$'] diff --git a/lib/spack/spack/ci_needs_workaround.py b/lib/spack/spack/ci_needs_workaround.py index 6252baa323..ae4cf33cd3 100644 --- a/lib/spack/spack/ci_needs_workaround.py +++ b/lib/spack/spack/ci_needs_workaround.py @@ -3,18 +3,11 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) -import collections - -try: - # dynamically import to keep vermin from complaining - collections_abc = __import__('collections.abc') -except ImportError: - collections_abc = collections - +from llnl.util.compat import Mapping get_job_name = lambda needs_entry: ( needs_entry.get('job') if ( - isinstance(needs_entry, collections_abc.Mapping) and + isinstance(needs_entry, Mapping) and needs_entry.get('artifacts', True)) else @@ -25,7 +18,7 @@ def convert_job(job_entry): - if not isinstance(job_entry, collections_abc.Mapping): + if not isinstance(job_entry, Mapping): return job_entry needs = job_entry.get('needs') diff --git a/lib/spack/spack/ci_optimization.py b/lib/spack/spack/ci_optimization.py index f32429a5fa..b63abb3446 100644 --- a/lib/spack/spack/ci_optimization.py +++ b/lib/spack/spack/ci_optimization.py @@ -3,28 +3,23 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) -import collections - -try: - # dynamically import to keep vermin from complaining - collections_abc = __import__('collections.abc') -except ImportError: - collections_abc = collections - import copy import hashlib +from collections import defaultdict + +from llnl.util.compat import Mapping, Sequence import spack.util.spack_yaml as syaml def sort_yaml_obj(obj): - if isinstance(obj, collections_abc.Mapping): + if isinstance(obj, Mapping): return syaml.syaml_dict( (k, sort_yaml_obj(v)) for k, v in sorted(obj.items(), key=(lambda item: str(item[0])))) - if isinstance(obj, collections_abc.Sequence) and not isinstance(obj, str): + if isinstance(obj, Sequence) and not isinstance(obj, str): return syaml.syaml_list(sort_yaml_obj(x) for x in obj) return obj @@ -44,8 +39,8 @@ def matches(obj, proto): Precondition: proto must not have any reference cycles """ - if isinstance(obj, collections_abc.Mapping): - if not isinstance(proto, collections_abc.Mapping): + if isinstance(obj, Mapping): + if not isinstance(proto, Mapping): return False return all( @@ -53,10 +48,10 @@ def matches(obj, proto): for key, val in proto.items() ) - if (isinstance(obj, collections_abc.Sequence) and + if (isinstance(obj, Sequence) and not isinstance(obj, str)): - if not (isinstance(proto, collections_abc.Sequence) and + if not (isinstance(proto, Sequence) and not isinstance(proto, str)): return False @@ -90,8 +85,8 @@ def subkeys(obj, proto): Otherwise, obj is returned. """ - if not (isinstance(obj, collections_abc.Mapping) and - isinstance(proto, collections_abc.Mapping)): + if not (isinstance(obj, Mapping) and + isinstance(proto, Mapping)): return obj new_obj = {} @@ -104,7 +99,7 @@ def subkeys(obj, proto): matches(proto[key], value)): continue - if isinstance(value, collections_abc.Mapping): + if isinstance(value, Mapping): new_obj[key] = subkeys(value, proto[key]) continue @@ -132,7 +127,7 @@ def add_extends(yaml, key): has_key = ('extends' in yaml) extends = yaml.get('extends') - if has_key and not isinstance(extends, (str, collections_abc.Sequence)): + if has_key and not isinstance(extends, (str, Sequence)): return if extends is None: @@ -283,7 +278,7 @@ def build_histogram(iterator, key): The list is sorted in descending order by count, yielding the most frequently occuring hashes first. """ - buckets = collections.defaultdict(int) + buckets = defaultdict(int) values = {} num_objects = 0 diff --git a/lib/spack/spack/cmd/python.py b/lib/spack/spack/cmd/python.py index 5224e8bbc9..901574c12d 100644 --- a/lib/spack/spack/cmd/python.py +++ b/lib/spack/spack/cmd/python.py @@ -73,7 +73,7 @@ def ipython_interpreter(args): support running a script or arguments """ try: - import IPython + import IPython # type: ignore[import] except ImportError: tty.die("ipython is not installed, install and try again.") diff --git a/lib/spack/spack/directives.py b/lib/spack/spack/directives.py index 5de64ebf52..81f58f542d 100644 --- a/lib/spack/spack/directives.py +++ b/lib/spack/spack/directives.py @@ -30,13 +30,13 @@ class OpenMpi(Package): import functools import os.path import re -import sys from typing import List, Set # novm import six import llnl.util.lang import llnl.util.tty.color +from llnl.util.compat import Sequence import spack.error import spack.patch @@ -48,12 +48,6 @@ class OpenMpi(Package): from spack.resource import Resource from spack.version import Version, VersionChecksumError -if sys.version_info >= (3, 3): - from collections.abc import Sequence # novm -else: - from collections import Sequence - - __all__ = ['DirectiveError', 'DirectiveMeta'] #: These are variant names used by Spack internally; packages can't use them diff --git a/lib/spack/spack/filesystem_view.py b/lib/spack/spack/filesystem_view.py index d1ec998f94..a95f9f9f6d 100644 --- a/lib/spack/spack/filesystem_view.py +++ b/lib/spack/spack/filesystem_view.py @@ -11,6 +11,7 @@ import sys from llnl.util import tty +from llnl.util.compat import filter, map, zip from llnl.util.filesystem import mkdirp, remove_dead_links, remove_empty_directories from llnl.util.lang import index_by, match_predicate from llnl.util.link_tree import LinkTree, MergeConflictError @@ -29,12 +30,6 @@ ) from spack.error import SpackError -# compatability -if sys.version_info < (3, 0): - from itertools import ifilter as filter - from itertools import imap as map - from itertools import izip as zip - __all__ = ["FilesystemView", "YamlFilesystemView"] diff --git a/lib/spack/spack/mirror.py b/lib/spack/spack/mirror.py index 3b8a60992c..0085d920b8 100644 --- a/lib/spack/spack/mirror.py +++ b/lib/spack/spack/mirror.py @@ -21,12 +21,8 @@ import ruamel.yaml.error as yaml_error import six -if sys.version_info >= (3, 5): - from collections.abc import Mapping # novm -else: - from collections import Mapping - import llnl.util.tty as tty +from llnl.util.compat import Mapping from llnl.util.filesystem import mkdirp import spack.config diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 6f7e7a71b5..5df80f1e03 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -1293,7 +1293,7 @@ def prefix(self): """Get the prefix into which this package should be installed.""" return self.spec.prefix - @property # type: ignore + @property # type: ignore[misc] @memoized def compiler(self): """Get the spack.compiler.Compiler object used to build this package""" diff --git a/lib/spack/spack/repo.py b/lib/spack/spack/repo.py index 63d8dfafbc..42a7aab5b6 100644 --- a/lib/spack/spack/repo.py +++ b/lib/spack/spack/repo.py @@ -18,18 +18,13 @@ import types from typing import Dict # novm -import six - -if sys.version_info >= (3, 5): - from collections.abc import Mapping # novm -else: - from collections import Mapping - import ruamel.yaml as yaml +import six import llnl.util.filesystem as fs import llnl.util.lang import llnl.util.tty as tty +from llnl.util.compat import Mapping import spack.caches import spack.config diff --git a/lib/spack/spack/s3_handler.py b/lib/spack/spack/s3_handler.py index 66fa73e07a..9f775abafb 100644 --- a/lib/spack/spack/s3_handler.py +++ b/lib/spack/spack/s3_handler.py @@ -63,7 +63,7 @@ def _s3_open(url): class UrllibS3Handler(urllib_request.HTTPSHandler): def s3_open(self, req): orig_url = req.get_full_url() - from botocore.exceptions import ClientError + from botocore.exceptions import ClientError # type: ignore[import] try: url, headers, stream = _s3_open(orig_url) return urllib_response.addinfourl(stream, headers, url) diff --git a/lib/spack/spack/schema/environment.py b/lib/spack/spack/schema/environment.py index 90632c266c..3bb02d33d0 100644 --- a/lib/spack/spack/schema/environment.py +++ b/lib/spack/spack/schema/environment.py @@ -38,13 +38,9 @@ def parse(config_obj): config_obj: a configuration dictionary conforming to the schema definition for environment modifications """ - import sys + from llnl.util.compat import Sequence import spack.util.environment as ev - if sys.version_info >= (3, 5): - from collections.abc import Sequence # novm - else: - from collections import Sequence # novm env = ev.EnvironmentModifications() for command, variable in config_obj.items(): diff --git a/lib/spack/spack/solver/asp.py b/lib/spack/spack/solver/asp.py index d6c8fc8d3a..4f727cdf48 100644 --- a/lib/spack/spack/solver/asp.py +++ b/lib/spack/spack/solver/asp.py @@ -17,13 +17,15 @@ import archspec.cpu +from llnl.util.compat import Sequence + try: - import clingo + import clingo # type: ignore[import] # There may be a better way to detect this clingo_cffi = hasattr(clingo.Symbol, '_rep') except ImportError: - clingo = None # type: ignore + clingo = None clingo_cffi = False import llnl.util.lang @@ -49,11 +51,6 @@ import spack.variant import spack.version -if sys.version_info >= (3, 3): - from collections.abc import Sequence # novm -else: - from collections import Sequence - # these are from clingo.ast and bootstrapped later ASTType = None parse_files = None diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index aa9cd6a709..e97a3ca12d 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -81,7 +81,6 @@ import operator import os import re -import sys import warnings import ruamel.yaml as yaml @@ -91,6 +90,7 @@ import llnl.util.lang as lang import llnl.util.tty as tty import llnl.util.tty.color as clr +from llnl.util.compat import Mapping import spack.compiler import spack.compilers @@ -117,12 +117,6 @@ import spack.variant as vt import spack.version as vn -if sys.version_info >= (3, 3): - from collections.abc import Mapping # novm -else: - from collections import Mapping - - __all__ = [ 'CompilerSpec', 'Spec', diff --git a/lib/spack/spack/test/ci.py b/lib/spack/spack/test/ci.py index e054ccf14f..7bff8f8447 100644 --- a/lib/spack/spack/test/ci.py +++ b/lib/spack/spack/test/ci.py @@ -3,7 +3,6 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) -import collections import itertools as it import json import os @@ -24,12 +23,6 @@ import spack.util.gpg import spack.util.spack_yaml as syaml -try: - # dynamically import to keep vermin from complaining - collections_abc = __import__('collections.abc') -except ImportError: - collections_abc = collections - @pytest.fixture def tmp_scope(): diff --git a/lib/spack/spack/test/cmd/versions.py b/lib/spack/spack/test/cmd/versions.py index 0482c654a7..d931a3fe9b 100644 --- a/lib/spack/spack/test/cmd/versions.py +++ b/lib/spack/spack/test/cmd/versions.py @@ -41,7 +41,7 @@ def test_remote_versions_only(): @pytest.mark.usefixtures('mock_packages') def test_new_versions_only(monkeypatch): """Test a package for which new versions should be available.""" - from spack.pkg.builtin.mock.brillig import Brillig + from spack.pkg.builtin.mock.brillig import Brillig # type: ignore[import] def mock_fetch_remote_versions(*args, **kwargs): mock_remote_versions = { diff --git a/lib/spack/spack/test/conftest.py b/lib/spack/spack/test/conftest.py index c3bd724517..d805112e5f 100644 --- a/lib/spack/spack/test/conftest.py +++ b/lib/spack/spack/test/conftest.py @@ -15,6 +15,7 @@ import shutil import tempfile import xml.etree.ElementTree +from typing import Dict # novm import py import pytest @@ -50,8 +51,8 @@ # # Return list of shas for latest two git commits in local spack repo # -@pytest.fixture -def last_two_git_commits(scope='session'): +@pytest.fixture(scope='session') +def last_two_git_commits(): git = spack.util.executable.which('git', required=True) spack_git_path = spack.paths.prefix with working_dir(spack_git_path): @@ -1488,7 +1489,7 @@ def mock_test_repo(tmpdir_factory): class MockBundle(object): has_code = False name = 'mock-bundle' - versions = {} # type: ignore + versions = {} # type: Dict @pytest.fixture diff --git a/lib/spack/spack/test/packages.py b/lib/spack/spack/test/packages.py index beec23f0be..7170a08aae 100644 --- a/lib/spack/spack/test/packages.py +++ b/lib/spack/spack/test/packages.py @@ -55,13 +55,13 @@ def test_package_class_names(self): # Below tests target direct imports of spack packages from the # spack.pkg namespace def test_import_package(self): - import spack.pkg.builtin.mock.mpich # noqa + import spack.pkg.builtin.mock.mpich # type: ignore[import] # noqa def test_import_package_as(self): - import spack.pkg.builtin.mock.mpich as mp # noqa - import spack.pkg.builtin.mock # noqa - import spack.pkg.builtin.mock as m # noqa - from spack.pkg.builtin import mock # noqa + import spack.pkg.builtin.mock # noqa + import spack.pkg.builtin.mock as m # noqa + import spack.pkg.builtin.mock.mpich as mp # noqa + from spack.pkg.builtin import mock # noqa def test_inheritance_of_diretives(self): p = spack.repo.get('simple-inheritance') @@ -104,18 +104,18 @@ def test_import_class_from_package(self): from spack.pkg.builtin.mock.mpich import Mpich # noqa def test_import_module_from_package(self): - from spack.pkg.builtin.mock import mpich # noqa + from spack.pkg.builtin.mock import mpich # noqa def test_import_namespace_container_modules(self): - import spack.pkg # noqa - import spack.pkg as p # noqa - from spack import pkg # noqa - import spack.pkg.builtin # noqa - import spack.pkg.builtin as b # noqa - from spack.pkg import builtin # noqa - import spack.pkg.builtin.mock # noqa - import spack.pkg.builtin.mock as m # noqa - from spack.pkg.builtin import mock # noqa + import spack.pkg # noqa + import spack.pkg as p # noqa + import spack.pkg.builtin # noqa + import spack.pkg.builtin as b # noqa + import spack.pkg.builtin.mock # noqa + import spack.pkg.builtin.mock as m # noqa + from spack import pkg # noqa + from spack.pkg import builtin # noqa + from spack.pkg.builtin import mock # noqa @pytest.mark.regression('2737') diff --git a/lib/spack/spack/test/spec_yaml.py b/lib/spack/spack/test/spec_yaml.py index ce27c24629..8aa14b9230 100644 --- a/lib/spack/spack/test/spec_yaml.py +++ b/lib/spack/spack/test/spec_yaml.py @@ -11,10 +11,11 @@ import ast import inspect import os -import sys import pytest +from llnl.util.compat import Iterable, Mapping + import spack.hash_types as ht import spack.spec import spack.util.spack_json as sjson @@ -25,11 +26,6 @@ from spack.util.mock_package import MockPackageMultiRepo from spack.util.spack_yaml import SpackYAMLError, syaml_dict -if sys.version_info >= (3, 3): - from collections.abc import Iterable, Mapping # novm -else: - from collections import Iterable, Mapping - def check_yaml_round_trip(spec): yaml_text = spec.to_yaml() diff --git a/lib/spack/spack/util/environment.py b/lib/spack/spack/util/environment.py index b6c3ac359b..9b143b0b6b 100644 --- a/lib/spack/spack/util/environment.py +++ b/lib/spack/spack/util/environment.py @@ -25,6 +25,7 @@ import spack.platforms import spack.spec import spack.util.executable as executable +import spack.util.spack_json as sjson system_paths = ['/', '/usr', '/usr/local'] suffixes = ['bin', 'bin64', 'include', 'lib', 'lib64'] @@ -1027,13 +1028,7 @@ def _source_single_file(file_and_args, environment): # If we're in python2, convert to str objects instead of unicode # like json gives us. We can't put unicode in os.environ anyway. - if sys.version_info[0] < 3: - environment = dict( - (k.encode('utf-8'), v.encode('utf-8')) - for k, v in environment.items() - ) - - return environment + return sjson.encode_json_dict(environment) current_environment = kwargs.get('env', dict(os.environ)) for f in files: @@ -1074,7 +1069,7 @@ def set_intersection(fullset, *args): return subset # Don't modify input, make a copy instead - environment = dict(environment) + environment = sjson.decode_json_dict(dict(environment)) # Retain (whitelist) has priority over prune (blacklist) prune = set_intersection(set(environment), *blacklist) diff --git a/lib/spack/spack/util/mock_package.py b/lib/spack/spack/util/mock_package.py index c600933cc9..b5f942aedf 100644 --- a/lib/spack/spack/util/mock_package.py +++ b/lib/spack/spack/util/mock_package.py @@ -103,7 +103,6 @@ def is_virtual(self, name, use_index=True): return False def repo_for_pkg(self, name): - import collections Repo = collections.namedtuple('Repo', ['namespace']) return Repo('mockrepo') diff --git a/lib/spack/spack/util/pattern.py b/lib/spack/spack/util/pattern.py index 1eb8b6a7a1..220e18b4a3 100644 --- a/lib/spack/spack/util/pattern.py +++ b/lib/spack/spack/util/pattern.py @@ -5,12 +5,8 @@ import functools import inspect -import sys -if sys.version_info >= (3, 3): - from collections.abc import MutableSequence # novm -else: - from collections import MutableSequence +from llnl.util.compat import MutableSequence class Delegate(object): diff --git a/lib/spack/spack/util/py2.py b/lib/spack/spack/util/py2.py new file mode 100644 index 0000000000..b92a6d66d9 --- /dev/null +++ b/lib/spack/spack/util/py2.py @@ -0,0 +1,16 @@ +# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +import base64 + +from six import PY3, binary_type, text_type + + +def b32encode(digest): + # type: (binary_type) -> text_type + b32 = base64.b32encode(digest) + if PY3: + return b32.decode() + return b32 diff --git a/lib/spack/spack/util/s3.py b/lib/spack/spack/util/s3.py index c8c885c5ab..b52a879e21 100644 --- a/lib/spack/spack/util/s3.py +++ b/lib/spack/spack/util/s3.py @@ -61,16 +61,16 @@ def create_s3_session(url, connection={}): # NOTE(opadron): import boto and friends as late as possible. We don't # want to require boto as a dependency unless the user actually wants to # access S3 mirrors. - from boto3 import Session - from botocore.exceptions import ClientError + from boto3 import Session # type: ignore[import] + from botocore.exceptions import ClientError # type: ignore[import] s3_connection, s3_client_args = get_mirror_s3_connection_info(connection) session = Session(**s3_connection) # if no access credentials provided above, then access anonymously if not session.get_credentials(): - from botocore import UNSIGNED - from botocore.client import Config + from botocore import UNSIGNED # type: ignore[import] + from botocore.client import Config # type: ignore[import] s3_client_args["config"] = Config(signature_version=UNSIGNED) diff --git a/lib/spack/spack/util/spack_json.py b/lib/spack/spack/util/spack_json.py index 38bae8f387..7083b3c80a 100644 --- a/lib/spack/spack/util/spack_json.py +++ b/lib/spack/spack/util/spack_json.py @@ -5,13 +5,13 @@ """Simple wrapper around JSON to guarantee consistent use of load/dump. """ import json -import sys +from typing import Any, Dict, Optional # novm -from six import iteritems, string_types +from six import PY3, iteritems, string_types import spack.error -__all__ = ['load', 'dump', 'SpackJSONError'] +__all__ = ['load', 'dump', 'SpackJSONError', 'encode_json_dict', 'decode_json_dict'] _json_dump_args = { 'indent': 2, @@ -20,27 +20,45 @@ def load(stream): + # type: (Any) -> Dict """Spack JSON needs to be ordered to support specs.""" if isinstance(stream, string_types): - load = json.loads + load = json.loads # type: ignore[assignment] else: - load = json.load + load = json.load # type: ignore[assignment] return _strify(load(stream, object_hook=_strify), ignore_dicts=True) +def encode_json_dict(data): + # type: (Dict) -> Dict + """Converts python 2 unicodes to str in JSON data.""" + return _strify(data) + + def dump(data, stream=None): + # type: (Dict, Optional[Any]) -> Optional[str] """Dump JSON with a reasonable amount of indentation and separation.""" + data = _strify(data) if stream is None: - return json.dumps(data, **_json_dump_args) - else: - return json.dump(data, stream, **_json_dump_args) + return json.dumps(data, **_json_dump_args) # type: ignore[arg-type] + json.dump(data, stream, **_json_dump_args) # type: ignore[arg-type] + return None + + +def decode_json_dict(data): + # type: (Dict) -> Dict + """Converts str to python 2 unicodes in JSON data.""" + return _strify(data) def _strify(data, ignore_dicts=False): - """Converts python 2 unicodes to str in JSON data.""" + # type: (Dict, bool) -> Dict + """Helper method for ``encode_json_dict()`` and ``decode_json_dict()``. + + Converts python 2 unicodes to str in JSON data, or the other way around.""" # this is a no-op in python 3 - if sys.version_info[0] >= 3: + if PY3: return data # if this is a unicode string in python 2, return its string representation @@ -66,4 +84,5 @@ class SpackJSONError(spack.error.SpackError): """Raised when there are issues with JSON parsing.""" def __init__(self, msg, json_error): + # type: (str, BaseException) -> None super(SpackJSONError, self).__init__(msg, str(json_error)) diff --git a/lib/spack/spack/util/spack_yaml.py b/lib/spack/spack/util/spack_yaml.py index d869b19dc7..227b9d7449 100644 --- a/lib/spack/spack/util/spack_yaml.py +++ b/lib/spack/spack/util/spack_yaml.py @@ -15,30 +15,23 @@ import collections import ctypes import re -import sys from typing import List # novm import ruamel.yaml as yaml from ruamel.yaml import RoundTripDumper, RoundTripLoader from six import StringIO, string_types +from llnl.util.compat import Mapping from llnl.util.tty.color import cextra, clen, colorize import spack.error -if sys.version_info >= (3, 3): - from collections.abc import Mapping # novm -else: - from collections import Mapping - - # Only export load and dump __all__ = ['load', 'dump', 'SpackYAMLError'] + # Make new classes so we can add custom attributes. # Also, use OrderedDict instead of just dict. - - class syaml_dict(collections.OrderedDict): def __repr__(self): mappings = ('%r: %r' % (k, v) for k, v in self.items()) diff --git a/lib/spack/spack/variant.py b/lib/spack/spack/variant.py index 1be147ff6f..ccbc5db33e 100644 --- a/lib/spack/spack/variant.py +++ b/lib/spack/spack/variant.py @@ -11,17 +11,12 @@ import inspect import itertools import re -import sys from six import StringIO -if sys.version_info >= (3, 5): - from collections.abc import Sequence # novm -else: - from collections import Sequence - import llnl.util.lang as lang import llnl.util.tty.color +from llnl.util.compat import Sequence import spack.directives import spack.error as error diff --git a/lib/spack/spack/verify.py b/lib/spack/spack/verify.py index 382592fcdc..dca908f345 100644 --- a/lib/spack/spack/verify.py +++ b/lib/spack/spack/verify.py @@ -2,28 +2,22 @@ # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) -import base64 import hashlib import os -import sys import llnl.util.tty as tty import spack.filesystem_view import spack.store import spack.util.file_permissions as fp +import spack.util.py2 as compat import spack.util.spack_json as sjson def compute_hash(path): with open(path, 'rb') as f: sha1 = hashlib.sha1(f.read()).digest() - b32 = base64.b32encode(sha1) - - if sys.version_info[0] >= 3: - b32 = b32.decode() - - return b32 + return compat.b32encode(sha1) def create_manifest_entry(path): diff --git a/pyproject.toml b/pyproject.toml index d09e589bed..4b6da081c2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,6 +10,7 @@ sections = [ known_first_party = "spack" known_archspec = "archspec" known_llnl = "llnl" +known_third_party = ["ruamel", "six"] src_paths = "lib" honor_noqa = true