From 1fb38c9e042f9fb349718616e9522a8d99f80128 Mon Sep 17 00:00:00 2001 From: Massimiliano Culpo Date: Tue, 16 Jan 2018 07:00:39 +0100 Subject: [PATCH] Vendor ordereddict for python2.6 only (#6931) * Vendor ordereddict for python2.6 only This commit substitutes the custom module 'ordereddict_backport' with the more known 'ordereddict' and vendors it only for python 2.6. Other supported versions of python will use 'collections.OrderedDict'. * Use absolute imports also for python 2.6 See PEP-328 for more information on the subject * Added provenance of vendored ordereddict --- bin/spack | 4 + lib/spack/external/__init__.py | 7 +- lib/spack/external/ordereddict_backport.py | 268 --------------------- lib/spack/external/py26/ordereddict.py | 127 ++++++++++ lib/spack/spack/config.py | 4 +- lib/spack/spack/test/config.py | 4 +- lib/spack/spack/test/conftest.py | 6 +- lib/spack/spack/test/spack_yaml.py | 4 +- lib/spack/spack/util/ordereddict.py | 40 +++ lib/spack/spack/util/spack_yaml.py | 2 +- 10 files changed, 185 insertions(+), 281 deletions(-) delete mode 100644 lib/spack/external/ordereddict_backport.py create mode 100644 lib/spack/external/py26/ordereddict.py create mode 100644 lib/spack/spack/util/ordereddict.py diff --git a/bin/spack b/bin/spack index 51dbce695b..e9278ddc4c 100755 --- a/bin/spack +++ b/bin/spack @@ -43,6 +43,10 @@ sys.path.insert(0, spack_lib_path) # Add external libs spack_external_libs = os.path.join(spack_lib_path, "external") + +if sys.version_info[:2] == (2, 6): + sys.path.insert(0, os.path.join(spack_external_libs, 'py26')) + sys.path.insert(0, spack_external_libs) # Handle vendoring of YAML specially, as it has two versions. diff --git a/lib/spack/external/__init__.py b/lib/spack/external/__init__.py index bee09fdebc..004339d7e3 100644 --- a/lib/spack/external/__init__.py +++ b/lib/spack/external/__init__.py @@ -86,9 +86,10 @@ orderddict ---------- -* Homepage: http://code.activestate.com/recipes/576693-ordered-dictionary-for-py24/ -* Usage: We include our own version to be Python 2.6 compatible. -* Version: Unversioned +* Homepage: https://pypi.org/project/ordereddict/ +* Usage: A drop-in substitute for Py2.7's new collections.OrderedDict + that works in Python 2.4-2.6. +* Version: 1.1 py -- diff --git a/lib/spack/external/ordereddict_backport.py b/lib/spack/external/ordereddict_backport.py deleted file mode 100644 index 154e5d1872..0000000000 --- a/lib/spack/external/ordereddict_backport.py +++ /dev/null @@ -1,268 +0,0 @@ -# -# Backport of OrderedDict() class that runs on Python 2.4, 2.5, 2.6, 2.7 and pypy. -# Passes Python2.7's test suite and incorporates all the latest updates. -# -# From http://code.activestate.com/recipes/576693-ordered-dictionary-for-py24/ -# This file is in the public domain, and has no particular license. -# -try: - from thread import get_ident as _get_ident -except ImportError: - try: - from dummy_thread import get_ident as _get_ident - except ImportError: - try: - from _dummy_thread import get_ident as _get_ident - except ImportError: - from threading import get_ident as _get_ident # nopyqver - -try: - from _abcoll import KeysView, ValuesView, ItemsView -except ImportError: - pass - - -class OrderedDict(dict): - 'Dictionary that remembers insertion order' - # An inherited dict maps keys to values. - # The inherited dict provides __getitem__, __len__, __contains__, and get. - # The remaining methods are order-aware. - # Big-O running times for all methods are the same as for regular dictionaries. - - # The internal self.__map dictionary maps keys to links in a doubly linked list. - # The circular doubly linked list starts and ends with a sentinel element. - # The sentinel element never gets deleted (this simplifies the algorithm). - # Each link is stored as a list of length three: [PREV, NEXT, KEY]. - - def __init__(self, *args, **kwds): - '''Initialize an ordered dictionary. Signature is the same as for - regular dictionaries, but keyword arguments are not recommended - because their insertion order is arbitrary. - - ''' - if len(args) > 1: - raise TypeError('expected at most 1 arguments, got %d' % len(args)) - try: - self.__root - except AttributeError: - self.__root = root = [] # sentinel node - root[:] = [root, root, None] - self.__map = {} - self.__update(*args, **kwds) - - def __setitem__(self, key, value, dict_setitem=dict.__setitem__): - 'od.__setitem__(i, y) <==> od[i]=y' - # Setting a new item creates a new link which goes at the end of the linked - # list, and the inherited dictionary is updated with the new key/value pair. - if key not in self: - root = self.__root - last = root[0] - last[1] = root[0] = self.__map[key] = [last, root, key] - dict_setitem(self, key, value) - - def __delitem__(self, key, dict_delitem=dict.__delitem__): - 'od.__delitem__(y) <==> del od[y]' - # Deleting an existing item uses self.__map to find the link which is - # then removed by updating the links in the predecessor and successor nodes. - dict_delitem(self, key) - link_prev, link_next, key = self.__map.pop(key) - link_prev[1] = link_next - link_next[0] = link_prev - - def __iter__(self): - 'od.__iter__() <==> iter(od)' - root = self.__root - curr = root[1] - while curr is not root: - yield curr[2] - curr = curr[1] - - def __reversed__(self): - 'od.__reversed__() <==> reversed(od)' - root = self.__root - curr = root[0] - while curr is not root: - yield curr[2] - curr = curr[0] - - def clear(self): - 'od.clear() -> None. Remove all items from od.' - try: - for node in self.__map.itervalues(): - del node[:] - root = self.__root - root[:] = [root, root, None] - self.__map.clear() - except AttributeError: - pass - dict.clear(self) - - def popitem(self, last=True): - '''od.popitem() -> (k, v), return and remove a (key, value) pair. - Pairs are returned in LIFO order if last is true or FIFO order if false. - - ''' - if not self: - raise KeyError('dictionary is empty') - root = self.__root - if last: - link = root[0] - link_prev = link[0] - link_prev[1] = root - root[0] = link_prev - else: - link = root[1] - link_next = link[1] - root[1] = link_next - link_next[0] = root - key = link[2] - del self.__map[key] - value = dict.pop(self, key) - return key, value - - # -- the following methods do not depend on the internal structure -- - - def keys(self): - 'od.keys() -> list of keys in od' - return list(self) - - def values(self): - 'od.values() -> list of values in od' - return [self[key] for key in self] - - def items(self): - 'od.items() -> list of (key, value) pairs in od' - return [(key, self[key]) for key in self] - - def iterkeys(self): - 'od.iterkeys() -> an iterator over the keys in od' - return iter(self) - - def itervalues(self): - 'od.itervalues -> an iterator over the values in od' - for k in self: - yield self[k] - - def iteritems(self): - 'od.iteritems -> an iterator over the (key, value) items in od' - for k in self: - yield (k, self[k]) - - def update(*args, **kwds): - '''od.update(E, **F) -> None. Update od from dict/iterable E and F. - - If E is a dict instance, does: for k in E: od[k] = E[k] - If E has a .keys() method, does: for k in E.keys(): od[k] = E[k] - Or if E is an iterable of items, does: for k, v in E: od[k] = v - In either case, this is followed by: for k, v in F.items(): od[k] = v - - ''' - if len(args) > 2: - raise TypeError('update() takes at most 2 positional ' - 'arguments (%d given)' % (len(args),)) - elif not args: - raise TypeError('update() takes at least 1 argument (0 given)') - self = args[0] - # Make progressively weaker assumptions about "other" - other = () - if len(args) == 2: - other = args[1] - if isinstance(other, dict): - for key in other: - self[key] = other[key] - elif hasattr(other, 'keys'): - for key in other.keys(): - self[key] = other[key] - else: - for key, value in other: - self[key] = value - for key, value in kwds.items(): - self[key] = value - - __update = update # let subclasses override update without breaking __init__ - - __marker = object() - - def pop(self, key, default=__marker): - '''od.pop(k[,d]) -> v, remove specified key and return the corresponding value. - If key is not found, d is returned if given, otherwise KeyError is raised. - - ''' - if key in self: - result = self[key] - del self[key] - return result - if default is self.__marker: - raise KeyError(key) - return default - - def setdefault(self, key, default=None): - 'od.setdefault(k[,d]) -> od.get(k,d), also set od[k]=d if k not in od' - if key in self: - return self[key] - self[key] = default - return default - - def __repr__(self, _repr_running={}): - 'od.__repr__() <==> repr(od)' - call_key = id(self), _get_ident() - if call_key in _repr_running: - return '...' - _repr_running[call_key] = 1 - try: - if not self: - return '%s()' % (self.__class__.__name__,) - return '%s(%r)' % (self.__class__.__name__, self.items()) - finally: - del _repr_running[call_key] - - def __reduce__(self): - 'Return state information for pickling' - items = [[k, self[k]] for k in self] - inst_dict = vars(self).copy() - for k in vars(OrderedDict()): - inst_dict.pop(k, None) - if inst_dict: - return (self.__class__, (items,), inst_dict) - return self.__class__, (items,) - - def copy(self): - 'od.copy() -> a shallow copy of od' - return self.__class__(self) - - @classmethod - def fromkeys(cls, iterable, value=None): - '''OD.fromkeys(S[, v]) -> New ordered dictionary with keys from S - and values equal to v (which defaults to None). - - ''' - d = cls() - for key in iterable: - d[key] = value - return d - - def __eq__(self, other): - '''od.__eq__(y) <==> od==y. Comparison to another OD is order-sensitive - while comparison to a regular mapping is order-insensitive. - - ''' - if isinstance(other, OrderedDict): - return len(self)==len(other) and self.items() == other.items() - return dict.__eq__(self, other) - - def __ne__(self, other): - return not self == other - - # -- the following methods are only used in Python 2.7 -- - - def viewkeys(self): - "od.viewkeys() -> a set-like object providing a view on od's keys" - return KeysView(self) - - def viewvalues(self): - "od.viewvalues() -> an object providing a view on od's values" - return ValuesView(self) - - def viewitems(self): - "od.viewitems() -> a set-like object providing a view on od's items" - return ItemsView(self) diff --git a/lib/spack/external/py26/ordereddict.py b/lib/spack/external/py26/ordereddict.py new file mode 100644 index 0000000000..7242b5060d --- /dev/null +++ b/lib/spack/external/py26/ordereddict.py @@ -0,0 +1,127 @@ +# Copyright (c) 2009 Raymond Hettinger +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation files +# (the "Software"), to deal in the Software without restriction, +# including without limitation the rights to use, copy, modify, merge, +# publish, distribute, sublicense, and/or sell copies of the Software, +# and to permit persons to whom the Software is furnished to do so, +# subject to the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +from UserDict import DictMixin + +class OrderedDict(dict, DictMixin): + + def __init__(self, *args, **kwds): + if len(args) > 1: + raise TypeError('expected at most 1 arguments, got %d' % len(args)) + try: + self.__end + except AttributeError: + self.clear() + self.update(*args, **kwds) + + def clear(self): + self.__end = end = [] + end += [None, end, end] # sentinel node for doubly linked list + self.__map = {} # key --> [key, prev, next] + dict.clear(self) + + def __setitem__(self, key, value): + if key not in self: + end = self.__end + curr = end[1] + curr[2] = end[1] = self.__map[key] = [key, curr, end] + dict.__setitem__(self, key, value) + + def __delitem__(self, key): + dict.__delitem__(self, key) + key, prev, next = self.__map.pop(key) + prev[2] = next + next[1] = prev + + def __iter__(self): + end = self.__end + curr = end[2] + while curr is not end: + yield curr[0] + curr = curr[2] + + def __reversed__(self): + end = self.__end + curr = end[1] + while curr is not end: + yield curr[0] + curr = curr[1] + + def popitem(self, last=True): + if not self: + raise KeyError('dictionary is empty') + if last: + key = reversed(self).next() + else: + key = iter(self).next() + value = self.pop(key) + return key, value + + def __reduce__(self): + items = [[k, self[k]] for k in self] + tmp = self.__map, self.__end + del self.__map, self.__end + inst_dict = vars(self).copy() + self.__map, self.__end = tmp + if inst_dict: + return (self.__class__, (items,), inst_dict) + return self.__class__, (items,) + + def keys(self): + return list(self) + + setdefault = DictMixin.setdefault + update = DictMixin.update + pop = DictMixin.pop + values = DictMixin.values + items = DictMixin.items + iterkeys = DictMixin.iterkeys + itervalues = DictMixin.itervalues + iteritems = DictMixin.iteritems + + def __repr__(self): + if not self: + return '%s()' % (self.__class__.__name__,) + return '%s(%r)' % (self.__class__.__name__, self.items()) + + def copy(self): + return self.__class__(self) + + @classmethod + def fromkeys(cls, iterable, value=None): + d = cls() + for key in iterable: + d[key] = value + return d + + def __eq__(self, other): + if isinstance(other, OrderedDict): + if len(self) != len(other): + return False + for p, q in zip(self.items(), other.items()): + if p != q: + return False + return True + return dict.__eq__(self, other) + + def __ne__(self, other): + return not self == other diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 3a34db13e0..53e5eccf3b 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -60,7 +60,7 @@ import jsonschema from yaml.error import MarkedYAMLError from jsonschema import Draft4Validator, validators -from ordereddict_backport import OrderedDict +from spack.util.ordereddict import OrderedDict import llnl.util.tty as tty from llnl.util.filesystem import mkdirp @@ -236,7 +236,7 @@ def __repr__(self): def highest_precedence_scope(): """Get the scope with highest precedence (prefs will override others).""" - return config_scopes.values()[-1] + return list(config_scopes.values())[-1] def validate_scope(scope): diff --git a/lib/spack/spack/test/config.py b/lib/spack/spack/test/config.py index 2b7a7a5c08..e79d829ba0 100644 --- a/lib/spack/spack/test/config.py +++ b/lib/spack/spack/test/config.py @@ -27,7 +27,7 @@ import os import tempfile -import ordereddict_backport +import spack.util.ordereddict import pytest import spack import spack.config @@ -198,7 +198,7 @@ def config(tmpdir): """Mocks the configuration scope.""" spack.config.clear_config_caches() real_scope = spack.config.config_scopes - spack.config.config_scopes = ordereddict_backport.OrderedDict() + spack.config.config_scopes = spack.util.ordereddict.OrderedDict() for priority in ['low', 'high']: spack.config.ConfigScope(priority, str(tmpdir.join(priority))) Config = collections.namedtuple('Config', ['real', 'mock']) diff --git a/lib/spack/spack/test/conftest.py b/lib/spack/spack/test/conftest.py index 8d83d43ee0..0bdbf72f12 100644 --- a/lib/spack/spack/test/conftest.py +++ b/lib/spack/spack/test/conftest.py @@ -28,7 +28,7 @@ import shutil import re -import ordereddict_backport +import spack.util.ordereddict import py import pytest @@ -240,7 +240,7 @@ def config(configuration_dir): spack.package_prefs.PackagePrefs.clear_caches() spack.config.clear_config_caches() real_scope = spack.config.config_scopes - spack.config.config_scopes = ordereddict_backport.OrderedDict() + spack.config.config_scopes = spack.util.ordereddict.OrderedDict() spack.config.ConfigScope('site', str(configuration_dir.join('site'))) spack.config.ConfigScope('system', str(configuration_dir.join('system'))) spack.config.ConfigScope('user', str(configuration_dir.join('user'))) @@ -644,7 +644,7 @@ def __init__(self, name, dependencies, dependency_types, conditions=None, versions=None): self.name = name self.spec = None - self.dependencies = ordereddict_backport.OrderedDict() + self.dependencies = spack.util.ordereddict.OrderedDict() assert len(dependencies) == len(dependency_types) for dep, dtype in zip(dependencies, dependency_types): diff --git a/lib/spack/spack/test/spack_yaml.py b/lib/spack/spack/test/spack_yaml.py index 2bcb2b4ce9..7bf4e4e041 100644 --- a/lib/spack/spack/test/spack_yaml.py +++ b/lib/spack/spack/test/spack_yaml.py @@ -68,10 +68,10 @@ def test_parse(data): def test_dict_order(data): expected_order = ['x86_64', 'some_list', 'another_list', 'some_key'] - assert data['config_file'].keys() == expected_order + assert list(data['config_file'].keys()) == expected_order expected_order = ['foo', 'bar', 'baz'] - assert data['config_file']['x86_64'].keys() == expected_order + assert list(data['config_file']['x86_64'].keys()) == expected_order def test_line_numbers(data): diff --git a/lib/spack/spack/util/ordereddict.py b/lib/spack/spack/util/ordereddict.py new file mode 100644 index 0000000000..5d9f716005 --- /dev/null +++ b/lib/spack/spack/util/ordereddict.py @@ -0,0 +1,40 @@ +############################################################################## +# Copyright (c) 2013-2017, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/spack/spack +# Please also see the NOTICE and LICENSE files for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +"""This file dispatches to the correct implementation of OrderedDict.""" + +# TODO: this file can be removed when support for python 2.6 will be dropped + +# Removing this import will make python 2.6 +# fail on import of ordereddict +from __future__ import absolute_import + +import sys + +if sys.version_info[:2] == (2, 6): + import ordereddict + OrderedDict = ordereddict.OrderedDict +else: + import collections + OrderedDict = collections.OrderedDict diff --git a/lib/spack/spack/util/spack_yaml.py b/lib/spack/spack/util/spack_yaml.py index 476ed464f5..8ac2efc229 100644 --- a/lib/spack/spack/util/spack_yaml.py +++ b/lib/spack/spack/util/spack_yaml.py @@ -35,7 +35,7 @@ from yaml import Loader, Dumper from yaml.nodes import MappingNode, SequenceNode, ScalarNode from yaml.constructor import ConstructorError -from ordereddict_backport import OrderedDict +from spack.util.ordereddict import OrderedDict import spack.error