externals: add astunparse

Add `astunparse` as `spack_astunparse`. This library unparses Python ASTs and we're
adding it under our own name so that we can make modifications to it.

Ultimately this will be used to make `package_hash` consistent across Python versions.

COPYRIGHT

@@ -34,6 +34,10 @@ includes the sbang tool directly in bin/sbang. These packages are covered
 by various permissive licenses. A summary listing follows. See the
 license included with each package for full details.
 
+PackageName: altgraph
+PackageHomePage: https://altgraph.readthedocs.io/en/latest/index.html
+PackageLicenseDeclared: MIT
+
 PackageName: argparse
 PackageHomePage: https://pypi.python.org/pypi/argparse
 PackageLicenseDeclared: Python-2.0
@@ -62,6 +66,10 @@ PackageName: jsonschema
 PackageHomePage: https://pypi.python.org/pypi/jsonschema
 PackageLicenseDeclared: MIT
 
+PackageName: macholib
+PackageHomePage: https://macholib.readthedocs.io/en/latest/index.html
+PackageLicenseDeclared: MIT
+
 PackageName: markupsafe
 PackageHomePage: https://pypi.python.org/pypi/MarkupSafe
 PackageLicenseDeclared: BSD-3-Clause
@@ -94,10 +102,6 @@ PackageName: six
 PackageHomePage: https://pypi.python.org/pypi/six
 PackageLicenseDeclared: MIT
 
-PackageName: macholib
-PackageHomePage: https://macholib.readthedocs.io/en/latest/index.html
-PackageLicenseDeclared: MIT
-
-PackageName: altgraph
-PackageHomePage: https://altgraph.readthedocs.io/en/latest/index.html
-PackageLicenseDeclared: MIT
+PackageName: spack_astunparse
+PackageHomePage: https://github.com/simonpercivall/astunparse
+PackageLicenseDeclared: Python-2.0

lib/spack/external/__init__.py

@@ -6,6 +6,13 @@
 """This module contains the following external, potentially separately
 licensed, packages that are included in Spack:
 
+altgraph
+--------
+
+* Homepage: https://altgraph.readthedocs.io/en/latest/index.html
+* Usage: dependency of macholib
+* Version: 0.16.1
+
 archspec
 --------
 
@@ -68,6 +75,13 @@
 * Version: 3.2.0 (last version before 2.7 and 3.6 support was dropped)
 * Note: We don't include tests or benchmarks; just what Spack needs.
 
+macholib
+--------
+
+* Homepage: https://macholib.readthedocs.io/en/latest/index.html#
+* Usage: Manipulation of Mach-o binaries for relocating macOS buildcaches on Linux
+* Version: 1.12
+
 markupsafe
 ----------
 
@@ -124,18 +138,21 @@
 * Usage: Python 2 and 3 compatibility utilities.
 * Version: 1.16.0
 
-macholib
---------
 
-* Homepage: https://macholib.readthedocs.io/en/latest/index.html#
-* Usage: Manipulation of Mach-o binaries for relocating macOS buildcaches on Linux
-* Version: 1.12
+spack_astunparse
+----------------
 
-altgraph
---------
-
-* Homepage: https://altgraph.readthedocs.io/en/latest/index.html
-* Usage: dependency of macholib
-* Version: 0.16.1
+* Homepage: https://github.com/simonpercivall/astunparse
+* Usage: Unparsing Python ASTs for package hashes in Spack
+* Version: 1.6.3 (plus modifications)
+* Note: We have modified this library to generate consistent unparsed ASTs
+  regardless of the Python version. It contains the original ``astunparse``
+  library, as well as modifications for consistency. It also contains
+  backports from the ``ast.unparse`` function in Python 3.9 and later, so
+  that it will generate output consistent with the builtin ``ast.unparse``
+  function, in case we ever want to drop astunparse as an external
+  dependency.  Because we have modified the parsing (potentially at the
+  cost of round-trippability of the code), we call this ``spack_astunparse``
+  to avoid confusion with ``astunparse``.
 
 """

lib/spack/external/spack_astunparse/LICENSE

@@ -0,0 +1,64 @@
+LICENSE
+=======
+
+Copyright (c) 2014, Simon Percivall
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
+
+* Neither the name of AST Unparser nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
+--------------------------------------------
+
+1. This LICENSE AGREEMENT is between the Python Software Foundation
+("PSF"), and the Individual or Organization ("Licensee") accessing and
+otherwise using this software ("Python") in source or binary form and
+its associated documentation.
+
+2. Subject to the terms and conditions of this License Agreement, PSF hereby
+grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce,
+analyze, test, perform and/or display publicly, prepare derivative works,
+distribute, and otherwise use Python alone or in any derivative version,
+provided, however, that PSF's License Agreement and PSF's notice of copyright,
+i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010,
+2011, 2012, 2013, 2014 Python Software Foundation; All Rights Reserved" are retained
+in Python alone or in any derivative version prepared by Licensee.
+
+3. In the event Licensee prepares a derivative work that is based on
+or incorporates Python or any part thereof, and wants to make
+the derivative work available to others as provided herein, then
+Licensee hereby agrees to include in any such work a brief summary of
+the changes made to Python.
+
+4. PSF is making Python available to Licensee on an "AS IS"
+basis.  PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR
+IMPLIED.  BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND
+DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS
+FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT
+INFRINGE ANY THIRD PARTY RIGHTS.
+
+5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON
+FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS
+A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON,
+OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF.
+
+6. This License Agreement will automatically terminate upon a material
+breach of its terms and conditions.
+
+7. Nothing in this License Agreement shall be deemed to create any
+relationship of agency, partnership, or joint venture between PSF and
+Licensee.  This License Agreement does not grant permission to use PSF
+trademarks or trade name in a trademark sense to endorse or promote
+products or services of Licensee, or any third party.
+
+8. By copying, installing or otherwise using Python, Licensee
+agrees to be bound by the terms and conditions of this License
+Agreement.

lib/spack/external/spack_astunparse/__init__.py

@@ -0,0 +1,13 @@
+# coding: utf-8
+from __future__ import absolute_import
+from six.moves import cStringIO
+from .unparser import Unparser
+
+
+__version__ = '1.6.3'
+
+
+def unparse(tree):
+    v = cStringIO()
+    Unparser(tree, file=v)
+    return v.getvalue()

lib/spack/external/spack_astunparse/unparser.py

@@ -0,0 +1,906 @@
+"Usage: unparse.py <path to source file>"
+from __future__ import print_function, unicode_literals
+import six
+import sys
+import ast
+import os
+import tokenize
+from six import StringIO
+
+# Large float and imaginary literals get turned into infinities in the AST.
+# We unparse those infinities to INFSTR.
+INFSTR = "1e" + repr(sys.float_info.max_10_exp + 1)
+
+def interleave(inter, f, seq):
+    """Call f on each item in seq, calling inter() in between.
+    """
+    seq = iter(seq)
+    try:
+        f(next(seq))
+    except StopIteration:
+        pass
+    else:
+        for x in seq:
+            inter()
+            f(x)
+
+class Unparser:
+    """Methods in this class recursively traverse an AST and
+    output source code for the abstract syntax; original formatting
+    is disregarded. """
+
+    def __init__(self, tree, file = sys.stdout):
+        """Unparser(tree, file=sys.stdout) -> None.
+         Print the source for tree to file."""
+        self.f = file
+        self.future_imports = []
+        self._indent = 0
+        self.dispatch(tree)
+        print("", file=self.f)
+        self.f.flush()
+
+    def fill(self, text = ""):
+        "Indent a piece of text, according to the current indentation level"
+        self.f.write("\n"+"    "*self._indent + text)
+
+    def write(self, text):
+        "Append a piece of text to the current line."
+        self.f.write(six.text_type(text))
+
+    def enter(self):
+        "Print ':', and increase the indentation."
+        self.write(":")
+        self._indent += 1
+
+    def leave(self):
+        "Decrease the indentation level."
+        self._indent -= 1
+
+    def dispatch(self, tree):
+        "Dispatcher function, dispatching tree type T to method _T."
+        if isinstance(tree, list):
+            for t in tree:
+                self.dispatch(t)
+            return
+        meth = getattr(self, "_"+tree.__class__.__name__)
+        meth(tree)
+
+
+    ############### Unparsing methods ######################
+    # There should be one method per concrete grammar type #
+    # Constructors should be grouped by sum type. Ideally, #
+    # this would follow the order in the grammar, but      #
+    # currently doesn't.                                   #
+    ########################################################
+
+    def _Module(self, tree):
+        for stmt in tree.body:
+            self.dispatch(stmt)
+
+    def _Interactive(self, tree):
+        for stmt in tree.body:
+            self.dispatch(stmt)
+
+    def _Expression(self, tree):
+        self.dispatch(tree.body)
+
+    # stmt
+    def _Expr(self, tree):
+        self.fill()
+        self.dispatch(tree.value)
+
+    def _NamedExpr(self, tree):
+        self.write("(")
+        self.dispatch(tree.target)
+        self.write(" := ")
+        self.dispatch(tree.value)
+        self.write(")")
+
+    def _Import(self, t):
+        self.fill("import ")
+        interleave(lambda: self.write(", "), self.dispatch, t.names)
+
+    def _ImportFrom(self, t):
+        # A from __future__ import may affect unparsing, so record it.
+        if t.module and t.module == '__future__':
+            self.future_imports.extend(n.name for n in t.names)
+
+        self.fill("from ")
+        self.write("." * t.level)
+        if t.module:
+            self.write(t.module)
+        self.write(" import ")
+        interleave(lambda: self.write(", "), self.dispatch, t.names)
+
+    def _Assign(self, t):
+        self.fill()
+        for target in t.targets:
+            self.dispatch(target)
+            self.write(" = ")
+        self.dispatch(t.value)
+
+    def _AugAssign(self, t):
+        self.fill()
+        self.dispatch(t.target)
+        self.write(" "+self.binop[t.op.__class__.__name__]+"= ")
+        self.dispatch(t.value)
+
+    def _AnnAssign(self, t):
+        self.fill()
+        if not t.simple and isinstance(t.target, ast.Name):
+            self.write('(')
+        self.dispatch(t.target)
+        if not t.simple and isinstance(t.target, ast.Name):
+            self.write(')')
+        self.write(": ")
+        self.dispatch(t.annotation)
+        if t.value:
+            self.write(" = ")
+            self.dispatch(t.value)
+
+    def _Return(self, t):
+        self.fill("return")
+        if t.value:
+            self.write(" ")
+            self.dispatch(t.value)
+
+    def _Pass(self, t):
+        self.fill("pass")
+
+    def _Break(self, t):
+        self.fill("break")
+
+    def _Continue(self, t):
+        self.fill("continue")
+
+    def _Delete(self, t):
+        self.fill("del ")
+        interleave(lambda: self.write(", "), self.dispatch, t.targets)
+
+    def _Assert(self, t):
+        self.fill("assert ")
+        self.dispatch(t.test)
+        if t.msg:
+            self.write(", ")
+            self.dispatch(t.msg)
+
+    def _Exec(self, t):
+        self.fill("exec ")
+        self.dispatch(t.body)
+        if t.globals:
+            self.write(" in ")
+            self.dispatch(t.globals)
+        if t.locals:
+            self.write(", ")
+            self.dispatch(t.locals)
+
+    def _Print(self, t):
+        self.fill("print ")
+        do_comma = False
+        if t.dest:
+            self.write(">>")
+            self.dispatch(t.dest)
+            do_comma = True
+        for e in t.values:
+            if do_comma:self.write(", ")
+            else:do_comma=True
+            self.dispatch(e)
+        if not t.nl:
+            self.write(",")
+
+    def _Global(self, t):
+        self.fill("global ")
+        interleave(lambda: self.write(", "), self.write, t.names)
+
+    def _Nonlocal(self, t):
+        self.fill("nonlocal ")
+        interleave(lambda: self.write(", "), self.write, t.names)
+
+    def _Await(self, t):
+        self.write("(")
+        self.write("await")
+        if t.value:
+            self.write(" ")
+            self.dispatch(t.value)
+        self.write(")")
+
+    def _Yield(self, t):
+        self.write("(")
+        self.write("yield")
+        if t.value:
+            self.write(" ")
+            self.dispatch(t.value)
+        self.write(")")
+
+    def _YieldFrom(self, t):
+        self.write("(")
+        self.write("yield from")
+        if t.value:
+            self.write(" ")
+            self.dispatch(t.value)
+        self.write(")")
+
+    def _Raise(self, t):
+        self.fill("raise")
+        if six.PY3:
+            if not t.exc:
+                assert not t.cause
+                return
+            self.write(" ")
+            self.dispatch(t.exc)
+            if t.cause:
+                self.write(" from ")
+                self.dispatch(t.cause)
+        else:
+            self.write(" ")
+            if t.type:
+                self.dispatch(t.type)
+            if t.inst:
+                self.write(", ")
+                self.dispatch(t.inst)
+            if t.tback:
+                self.write(", ")
+                self.dispatch(t.tback)
+
+    def _Try(self, t):
+        self.fill("try")
+        self.enter()
+        self.dispatch(t.body)
+        self.leave()
+        for ex in t.handlers:
+            self.dispatch(ex)
+        if t.orelse:
+            self.fill("else")
+            self.enter()
+            self.dispatch(t.orelse)
+            self.leave()
+        if t.finalbody:
+            self.fill("finally")
+            self.enter()
+            self.dispatch(t.finalbody)
+            self.leave()
+
+    def _TryExcept(self, t):
+        self.fill("try")
+        self.enter()
+        self.dispatch(t.body)
+        self.leave()
+
+        for ex in t.handlers:
+            self.dispatch(ex)
+        if t.orelse:
+            self.fill("else")
+            self.enter()
+            self.dispatch(t.orelse)
+            self.leave()
+
+    def _TryFinally(self, t):
+        if len(t.body) == 1 and isinstance(t.body[0], ast.TryExcept):
+            # try-except-finally
+            self.dispatch(t.body)
+        else:
+            self.fill("try")
+            self.enter()
+            self.dispatch(t.body)
+            self.leave()
+
+        self.fill("finally")
+        self.enter()
+        self.dispatch(t.finalbody)
+        self.leave()
+
+    def _ExceptHandler(self, t):
+        self.fill("except")
+        if t.type:
+            self.write(" ")
+            self.dispatch(t.type)
+        if t.name:
+            self.write(" as ")
+            if six.PY3:
+                self.write(t.name)
+            else:
+                self.dispatch(t.name)
+        self.enter()
+        self.dispatch(t.body)
+        self.leave()
+
+    def _ClassDef(self, t):
+        self.write("\n")
+        for deco in t.decorator_list:
+            self.fill("@")
+            self.dispatch(deco)
+        self.fill("class "+t.name)
+        if six.PY3:
+            self.write("(")
+            comma = False
+            for e in t.bases:
+                if comma: self.write(", ")
+                else: comma = True
+                self.dispatch(e)
+            for e in t.keywords:
+                if comma: self.write(", ")
+                else: comma = True
+                self.dispatch(e)
+            if sys.version_info[:2] < (3, 5):
+                if t.starargs:
+                    if comma: self.write(", ")
+                    else: comma = True
+                    self.write("*")
+                    self.dispatch(t.starargs)
+                if t.kwargs:
+                    if comma: self.write(", ")
+                    else: comma = True
+                    self.write("**")
+                    self.dispatch(t.kwargs)
+            self.write(")")
+        elif t.bases:
+                self.write("(")
+                for a in t.bases:
+                    self.dispatch(a)
+                    self.write(", ")
+                self.write(")")
+        self.enter()
+        self.dispatch(t.body)
+        self.leave()
+
+    def _FunctionDef(self, t):
+        self.__FunctionDef_helper(t, "def")
+
+    def _AsyncFunctionDef(self, t):
+        self.__FunctionDef_helper(t, "async def")
+
+    def __FunctionDef_helper(self, t, fill_suffix):
+        self.write("\n")
+        for deco in t.decorator_list:
+            self.fill("@")
+            self.dispatch(deco)
+        def_str = fill_suffix+" "+t.name + "("
+        self.fill(def_str)
+        self.dispatch(t.args)
+        self.write(")")
+        if getattr(t, "returns", False):
+            self.write(" -> ")
+            self.dispatch(t.returns)
+        self.enter()
+        self.dispatch(t.body)
+        self.leave()
+
+    def _For(self, t):
+        self.__For_helper("for ", t)
+
+    def _AsyncFor(self, t):
+        self.__For_helper("async for ", t)
+
+    def __For_helper(self, fill, t):
+        self.fill(fill)
+        self.dispatch(t.target)
+        self.write(" in ")
+        self.dispatch(t.iter)
+        self.enter()
+        self.dispatch(t.body)
+        self.leave()
+        if t.orelse:
+            self.fill("else")
+            self.enter()
+            self.dispatch(t.orelse)
+            self.leave()
+
+    def _If(self, t):
+        self.fill("if ")
+        self.dispatch(t.test)
+        self.enter()
+        self.dispatch(t.body)
+        self.leave()
+        # collapse nested ifs into equivalent elifs.
+        while (t.orelse and len(t.orelse) == 1 and
+               isinstance(t.orelse[0], ast.If)):
+            t = t.orelse[0]
+            self.fill("elif ")
+            self.dispatch(t.test)
+            self.enter()
+            self.dispatch(t.body)
+            self.leave()
+        # final else
+        if t.orelse:
+            self.fill("else")
+            self.enter()
+            self.dispatch(t.orelse)
+            self.leave()
+
+    def _While(self, t):
+        self.fill("while ")
+        self.dispatch(t.test)
+        self.enter()
+        self.dispatch(t.body)
+        self.leave()
+        if t.orelse:
+            self.fill("else")
+            self.enter()
+            self.dispatch(t.orelse)
+            self.leave()
+
+    def _generic_With(self, t, async_=False):
+        self.fill("async with " if async_ else "with ")
+        if hasattr(t, 'items'):
+            interleave(lambda: self.write(", "), self.dispatch, t.items)
+        else:
+            self.dispatch(t.context_expr)
+            if t.optional_vars:
+                self.write(" as ")
+                self.dispatch(t.optional_vars)
+        self.enter()
+        self.dispatch(t.body)
+        self.leave()
+
+    def _With(self, t):
+        self._generic_With(t)
+
+    def _AsyncWith(self, t):
+        self._generic_With(t, async_=True)
+
+    # expr
+    def _Bytes(self, t):
+        self.write(repr(t.s))
+
+    def _Str(self, tree):
+        if six.PY3:
+            self.write(repr(tree.s))
+        else:
+            # if from __future__ import unicode_literals is in effect,
+            # then we want to output string literals using a 'b' prefix
+            # and unicode literals with no prefix.
+            if "unicode_literals" not in self.future_imports:
+                self.write(repr(tree.s))
+            elif isinstance(tree.s, str):
+                self.write("b" + repr(tree.s))
+            elif isinstance(tree.s, unicode):
+                self.write(repr(tree.s).lstrip("u"))
+            else:
+                assert False, "shouldn't get here"
+
+    def _JoinedStr(self, t):
+        # JoinedStr(expr* values)
+        self.write("f")
+        string = StringIO()
+        self._fstring_JoinedStr(t, string.write)
+        # Deviation from `unparse.py`: Try to find an unused quote.
+        # This change is made to handle _very_ complex f-strings.
+        v = string.getvalue()
+        if '\n' in v or '\r' in v:
+            quote_types = ["'''", '"""']
+        else:
+            quote_types = ["'", '"', '"""', "'''"]
+        for quote_type in quote_types:
+            if quote_type not in v:
+                v = "{quote_type}{v}{quote_type}".format(quote_type=quote_type, v=v)
+                break
+        else:
+            v = repr(v)
+        self.write(v)
+
+    def _FormattedValue(self, t):
+        # FormattedValue(expr value, int? conversion, expr? format_spec)
+        self.write("f")
+        string = StringIO()
+        self._fstring_JoinedStr(t, string.write)
+        self.write(repr(string.getvalue()))
+
+    def _fstring_JoinedStr(self, t, write):
+        for value in t.values:
+            meth = getattr(self, "_fstring_" + type(value).__name__)
+            meth(value, write)
+
+    def _fstring_Str(self, t, write):
+        value = t.s.replace("{", "{{").replace("}", "}}")
+        write(value)
+
+    def _fstring_Constant(self, t, write):
+        assert isinstance(t.value, str)
+        value = t.value.replace("{", "{{").replace("}", "}}")
+        write(value)
+
+    def _fstring_FormattedValue(self, t, write):
+        write("{")
+        expr = StringIO()
+        Unparser(t.value, expr)
+        expr = expr.getvalue().rstrip("\n")
+        if expr.startswith("{"):
+            write(" ")  # Separate pair of opening brackets as "{ {"
+        write(expr)
+        if t.conversion != -1:
+            conversion = chr(t.conversion)
+            assert conversion in "sra"
+            write("!{conversion}".format(conversion=conversion))
+        if t.format_spec:
+            write(":")
+            meth = getattr(self, "_fstring_" + type(t.format_spec).__name__)
+            meth(t.format_spec, write)
+        write("}")
+
+    def _Name(self, t):
+        self.write(t.id)
+
+    def _NameConstant(self, t):
+        self.write(repr(t.value))
+
+    def _Repr(self, t):
+        self.write("`")
+        self.dispatch(t.value)
+        self.write("`")
+
+    def _write_constant(self, value):
+        if isinstance(value, (float, complex)):
+            # Substitute overflowing decimal literal for AST infinities.
+            self.write(repr(value).replace("inf", INFSTR))
+        else:
+            self.write(repr(value))
+
+    def _Constant(self, t):
+        value = t.value
+        if isinstance(value, tuple):
+            self.write("(")
+            if len(value) == 1:
+                self._write_constant(value[0])
+                self.write(",")
+            else:
+                interleave(lambda: self.write(", "), self._write_constant, value)
+            self.write(")")
+        elif value is Ellipsis: # instead of `...` for Py2 compatibility
+            self.write("...")
+        else:
+            if t.kind == "u":
+                self.write("u")
+            self._write_constant(t.value)
+
+    def _Num(self, t):
+        repr_n = repr(t.n)
+        if six.PY3:
+            self.write(repr_n.replace("inf", INFSTR))
+        else:
+            # Parenthesize negative numbers, to avoid turning (-1)**2 into -1**2.
+            if repr_n.startswith("-"):
+                self.write("(")
+            if "inf" in repr_n and repr_n.endswith("*j"):
+                repr_n = repr_n.replace("*j", "j")
+            # Substitute overflowing decimal literal for AST infinities.
+            self.write(repr_n.replace("inf", INFSTR))
+            if repr_n.startswith("-"):
+                self.write(")")
+
+    def _List(self, t):
+        self.write("[")
+        interleave(lambda: self.write(", "), self.dispatch, t.elts)
+        self.write("]")
+
+    def _ListComp(self, t):
+        self.write("[")
+        self.dispatch(t.elt)
+        for gen in t.generators:
+            self.dispatch(gen)
+        self.write("]")
+
+    def _GeneratorExp(self, t):
+        self.write("(")
+        self.dispatch(t.elt)
+        for gen in t.generators:
+            self.dispatch(gen)
+        self.write(")")
+
+    def _SetComp(self, t):
+        self.write("{")
+        self.dispatch(t.elt)
+        for gen in t.generators:
+            self.dispatch(gen)
+        self.write("}")
+
+    def _DictComp(self, t):
+        self.write("{")
+        self.dispatch(t.key)
+        self.write(": ")
+        self.dispatch(t.value)
+        for gen in t.generators:
+            self.dispatch(gen)
+        self.write("}")
+
+    def _comprehension(self, t):
+        if getattr(t, 'is_async', False):
+            self.write(" async for ")
+        else:
+            self.write(" for ")
+        self.dispatch(t.target)
+        self.write(" in ")
+        self.dispatch(t.iter)
+        for if_clause in t.ifs:
+            self.write(" if ")
+            self.dispatch(if_clause)
+
+    def _IfExp(self, t):
+        self.write("(")
+        self.dispatch(t.body)
+        self.write(" if ")
+        self.dispatch(t.test)
+        self.write(" else ")
+        self.dispatch(t.orelse)
+        self.write(")")
+
+    def _Set(self, t):
+        assert(t.elts) # should be at least one element
+        self.write("{")
+        interleave(lambda: self.write(", "), self.dispatch, t.elts)
+        self.write("}")
+
+    def _Dict(self, t):
+        self.write("{")
+        def write_key_value_pair(k, v):
+            self.dispatch(k)
+            self.write(": ")
+            self.dispatch(v)
+
+        def write_item(item):
+            k, v = item
+            if k is None:
+                # for dictionary unpacking operator in dicts {**{'y': 2}}
+                # see PEP 448 for details
+                self.write("**")
+                self.dispatch(v)
+            else:
+                write_key_value_pair(k, v)
+        interleave(lambda: self.write(", "), write_item, zip(t.keys, t.values))
+        self.write("}")
+
+    def _Tuple(self, t):
+        self.write("(")
+        if len(t.elts) == 1:
+            elt = t.elts[0]
+            self.dispatch(elt)
+            self.write(",")
+        else:
+            interleave(lambda: self.write(", "), self.dispatch, t.elts)
+        self.write(")")
+
+    unop = {"Invert":"~", "Not": "not", "UAdd":"+", "USub":"-"}
+    def _UnaryOp(self, t):
+        self.write("(")
+        self.write(self.unop[t.op.__class__.__name__])
+        self.write(" ")
+        if six.PY2 and isinstance(t.op, ast.USub) and isinstance(t.operand, ast.Num):
+            # If we're applying unary minus to a number, parenthesize the number.
+            # This is necessary: -2147483648 is different from -(2147483648) on
+            # a 32-bit machine (the first is an int, the second a long), and
+            # -7j is different from -(7j).  (The first has real part 0.0, the second
+            # has real part -0.0.)
+            self.write("(")
+            self.dispatch(t.operand)
+            self.write(")")
+        else:
+            self.dispatch(t.operand)
+        self.write(")")
+
+    binop = { "Add":"+", "Sub":"-", "Mult":"*", "MatMult":"@", "Div":"/", "Mod":"%",
+                    "LShift":"<<", "RShift":">>", "BitOr":"|", "BitXor":"^", "BitAnd":"&",
+                    "FloorDiv":"//", "Pow": "**"}
+    def _BinOp(self, t):
+        self.write("(")
+        self.dispatch(t.left)
+        self.write(" " + self.binop[t.op.__class__.__name__] + " ")
+        self.dispatch(t.right)
+        self.write(")")
+
+    cmpops = {"Eq":"==", "NotEq":"!=", "Lt":"<", "LtE":"<=", "Gt":">", "GtE":">=",
+                        "Is":"is", "IsNot":"is not", "In":"in", "NotIn":"not in"}
+    def _Compare(self, t):
+        self.write("(")
+        self.dispatch(t.left)
+        for o, e in zip(t.ops, t.comparators):
+            self.write(" " + self.cmpops[o.__class__.__name__] + " ")
+            self.dispatch(e)
+        self.write(")")
+
+    boolops = {ast.And: 'and', ast.Or: 'or'}
+    def _BoolOp(self, t):
+        self.write("(")
+        s = " %s " % self.boolops[t.op.__class__]
+        interleave(lambda: self.write(s), self.dispatch, t.values)
+        self.write(")")
+
+    def _Attribute(self,t):
+        self.dispatch(t.value)
+        # Special case: 3.__abs__() is a syntax error, so if t.value
+        # is an integer literal then we need to either parenthesize
+        # it or add an extra space to get 3 .__abs__().
+        if isinstance(t.value, getattr(ast, 'Constant', getattr(ast, 'Num', None))) and isinstance(t.value.n, int):
+            self.write(" ")
+        self.write(".")
+        self.write(t.attr)
+
+    def _Call(self, t):
+        self.dispatch(t.func)
+        self.write("(")
+        comma = False
+        for e in t.args:
+            if comma: self.write(", ")
+            else: comma = True
+            self.dispatch(e)
+        for e in t.keywords:
+            if comma: self.write(", ")
+            else: comma = True
+            self.dispatch(e)
+        if sys.version_info[:2] < (3, 5):
+            if t.starargs:
+                if comma: self.write(", ")
+                else: comma = True
+                self.write("*")
+                self.dispatch(t.starargs)
+            if t.kwargs:
+                if comma: self.write(", ")
+                else: comma = True
+                self.write("**")
+                self.dispatch(t.kwargs)
+        self.write(")")
+
+    def _Subscript(self, t):
+        self.dispatch(t.value)
+        self.write("[")
+        self.dispatch(t.slice)
+        self.write("]")
+
+    def _Starred(self, t):
+        self.write("*")
+        self.dispatch(t.value)
+
+    # slice
+    def _Ellipsis(self, t):
+        self.write("...")
+
+    def _Index(self, t):
+        self.dispatch(t.value)
+
+    def _Slice(self, t):
+        if t.lower:
+            self.dispatch(t.lower)
+        self.write(":")
+        if t.upper:
+            self.dispatch(t.upper)
+        if t.step:
+            self.write(":")
+            self.dispatch(t.step)
+
+    def _ExtSlice(self, t):
+        interleave(lambda: self.write(', '), self.dispatch, t.dims)
+
+    # argument
+    def _arg(self, t):
+        self.write(t.arg)
+        if t.annotation:
+            self.write(": ")
+            self.dispatch(t.annotation)
+
+    # others
+    def _arguments(self, t):
+        first = True
+        # normal arguments
+        all_args = getattr(t, 'posonlyargs', []) + t.args
+        defaults = [None] * (len(all_args) - len(t.defaults)) + t.defaults
+        for index, elements in enumerate(zip(all_args, defaults), 1):
+            a, d = elements
+            if first:first = False
+            else: self.write(", ")
+            self.dispatch(a)
+            if d:
+                self.write("=")
+                self.dispatch(d)
+            if index == len(getattr(t, 'posonlyargs', ())):
+                self.write(", /")
+
+        # varargs, or bare '*' if no varargs but keyword-only arguments present
+        if t.vararg or getattr(t, "kwonlyargs", False):
+            if first:first = False
+            else: self.write(", ")
+            self.write("*")
+            if t.vararg:
+                if hasattr(t.vararg, 'arg'):
+                    self.write(t.vararg.arg)
+                    if t.vararg.annotation:
+                        self.write(": ")
+                        self.dispatch(t.vararg.annotation)
+                else:
+                    self.write(t.vararg)
+                    if getattr(t, 'varargannotation', None):
+                        self.write(": ")
+                        self.dispatch(t.varargannotation)
+
+        # keyword-only arguments
+        if getattr(t, "kwonlyargs", False):
+            for a, d in zip(t.kwonlyargs, t.kw_defaults):
+                if first:first = False
+                else: self.write(", ")
+                self.dispatch(a),
+                if d:
+                    self.write("=")
+                    self.dispatch(d)
+
+        # kwargs
+        if t.kwarg:
+            if first:first = False
+            else: self.write(", ")
+            if hasattr(t.kwarg, 'arg'):
+                self.write("**"+t.kwarg.arg)
+                if t.kwarg.annotation:
+                    self.write(": ")
+                    self.dispatch(t.kwarg.annotation)
+            else:
+                self.write("**"+t.kwarg)
+                if getattr(t, 'kwargannotation', None):
+                    self.write(": ")
+                    self.dispatch(t.kwargannotation)
+
+    def _keyword(self, t):
+        if t.arg is None:
+            # starting from Python 3.5 this denotes a kwargs part of the invocation
+            self.write("**")
+        else:
+            self.write(t.arg)
+            self.write("=")
+        self.dispatch(t.value)
+
+    def _Lambda(self, t):
+        self.write("(")
+        self.write("lambda ")
+        self.dispatch(t.args)
+        self.write(": ")
+        self.dispatch(t.body)
+        self.write(")")
+
+    def _alias(self, t):
+        self.write(t.name)
+        if t.asname:
+            self.write(" as "+t.asname)
+
+    def _withitem(self, t):
+        self.dispatch(t.context_expr)
+        if t.optional_vars:
+            self.write(" as ")
+            self.dispatch(t.optional_vars)
+
+def roundtrip(filename, output=sys.stdout):
+    if six.PY3:
+        with open(filename, "rb") as pyfile:
+            encoding = tokenize.detect_encoding(pyfile.readline)[0]
+        with open(filename, "r", encoding=encoding) as pyfile:
+            source = pyfile.read()
+    else:
+        with open(filename, "r") as pyfile:
+            source = pyfile.read()
+    tree = compile(source, filename, "exec", ast.PyCF_ONLY_AST, dont_inherit=True)
+    Unparser(tree, output)
+
+
+
+def testdir(a):
+    try:
+        names = [n for n in os.listdir(a) if n.endswith('.py')]
+    except OSError:
+        print("Directory not readable: %s" % a, file=sys.stderr)
+    else:
+        for n in names:
+            fullname = os.path.join(a, n)
+            if os.path.isfile(fullname):
+                output = StringIO()
+                print('Testing %s' % fullname)
+                try:
+                    roundtrip(fullname, output)
+                except Exception as e:
+                    print('  Failed to compile, exception is %s' % repr(e))
+            elif os.path.isdir(fullname):
+                testdir(fullname)
+
+def main(args):
+    if args[0] == '--testdir':
+        for a in args[1:]:
+            testdir(a)
+    else:
+        for a in args:
+            roundtrip(a)
+
+if __name__=='__main__':
+    main(sys.argv[1:])