hlrs-software-stack/spack
7
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

unparser: implement operator precedence algorithm for unparser

Browse source 
Backport operator precedence algorithm from here:
    397b96f6d7

This eliminates unnecessary parentheses from our unparsed output and makes Spack's unparser
consistent with the one in upstream Python 3.9+, with one exception.

Our parser normalizes argument order when `py_ver_consistent` is set, so that star arguments
in function calls come last.  We have to do this because Python 2's AST doesn't have information
about their actual order.

If we ever support only Python 3.9 and higher, we can easily switch over to `ast.unparse`, as
the unparsing is consistent except for this detail (modulo future changes to `ast.unparse`)
This commit is contained in:
Todd Gamblin 2021-12-22 12:44:42 -08:00 committed by Greg Becker
parent afb358313a
commit 396c37d82f
7 changed files with 315 additions and 170 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
COPYRIGHT
View file

@ -42,6 +42,10 @@ PackageName: argparse
PackageHomePage: https://pypi.python.org/pypi/argparse PackageHomePage: https://pypi.python.org/pypi/argparse
PackageLicenseDeclared: Python-2.0 PackageLicenseDeclared: Python-2.0
PackageName: astunparse
PackageHomePage: https://github.com/simonpercivall/astunparse
PackageLicenseDeclared: Python-2.0
PackageName: attrs PackageName: attrs
PackageHomePage: https://github.com/python-attrs/attrs PackageHomePage: https://github.com/python-attrs/attrs
PackageLicenseDeclared: MIT PackageLicenseDeclared: MIT
@ -101,7 +105,3 @@ PackageLicenseDeclared: Apache-2.0 OR MIT
PackageName: six PackageName: six
PackageHomePage: https://pypi.python.org/pypi/six PackageHomePage: https://pypi.python.org/pypi/six
PackageLicenseDeclared: MIT PackageLicenseDeclared: MIT
PackageName: spack_astunparse
PackageHomePage: https://github.com/simonpercivall/astunparse
PackageLicenseDeclared: Python-2.0

33
lib/spack/external/__init__.py vendored
View file

@ -31,6 +31,22 @@
vendored copy ever needs to be updated again: vendored copy ever needs to be updated again:
https://github.com/spack/spack/pull/6786/commits/dfcef577b77249106ea4e4c69a6cd9e64fa6c418 https://github.com/spack/spack/pull/6786/commits/dfcef577b77249106ea4e4c69a6cd9e64fa6c418
astunparse
----------------
* Homepage: https://github.com/simonpercivall/astunparse
* Usage: Unparsing Python ASTs for package hashes in Spack
* Version: 1.6.3 (plus modifications)
* Note: This is in ``spack.util.unparse`` because it's very heavily
modified, and we want to track coverage for it.
Specifically, we have modified this library to generate consistent unparsed ASTs
regardless of the Python version. It is based on:
1. The original ``astunparse`` library;
2. Modifications for consistency;
3. Backports from the ``ast.unparse`` function in Python 3.9 and later
The unparsing is now mostly consistent with upstream ``ast.unparse``, so if
we ever require Python 3.9 or higher, we can drop this external package.
attrs attrs
---------------- ----------------
@ -138,21 +154,4 @@
* Usage: Python 2 and 3 compatibility utilities. * Usage: Python 2 and 3 compatibility utilities.
* Version: 1.16.0 * Version: 1.16.0
spack_astunparse
----------------
* Homepage: https://github.com/simonpercivall/astunparse
* Usage: Unparsing Python ASTs for package hashes in Spack
* Version: 1.6.3 (plus modifications)
* Note: We have modified this library to generate consistent unparsed ASTs
regardless of the Python version. It contains the original ``astunparse``
library, as well as modifications for consistency. It also contains
backports from the ``ast.unparse`` function in Python 3.9 and later, so
that it will generate output consistent with the builtin ``ast.unparse``
function, in case we ever want to drop astunparse as an external
dependency. Because we have modified the parsing (potentially at the
cost of round-trippability of the code), we call this ``spack_astunparse``
to avoid confusion with ``astunparse``.
""" """

13
lib/spack/external/spack_astunparse/__init__.py vendored
View file

@ -1,13 +0,0 @@
# coding: utf-8
from __future__ import absolute_import
from six.moves import cStringIO
from .unparser import Unparser
__version__ = '1.6.3'
def unparse(tree, py_ver_consistent=False):
v = cStringIO()
Unparser(tree, file=v, py_ver_consistent=py_ver_consistent)
return v.getvalue()

4
lib/spack/spack/cmd/license.py
View file

@ -34,8 +34,8 @@
r'^bin/spack$', r'^bin/spack$',
r'^bin/spack-python$', r'^bin/spack-python$',
# all of spack core # all of spack core except unparse
r'^lib/spack/spack/.*\.py$', r'^lib/spack/spack/(?!util/unparse).*\.py$',
r'^lib/spack/spack/.*\.sh$', r'^lib/spack/spack/.*\.sh$',
r'^lib/spack/spack/.*\.lp$', r'^lib/spack/spack/.*\.lp$',
r'^lib/spack/llnl/.*\.py$', r'^lib/spack/llnl/.*\.py$',

0
lib/spack/external/spack_astunparse/LICENSE → lib/spack/spack/util/unparse/LICENSE
View file

19
lib/spack/spack/util/unparse/__init__.py Normal file
View file

@ -0,0 +1,19 @@
# Copyright (c) 2014-2021, Simon Percivall and Spack Project Developers.
#
# SPDX-License-Identifier: Python-2.0
# coding: utf-8
from __future__ import absolute_import
from six.moves import cStringIO
from .unparser import Unparser
__version__ = '1.6.3'
def unparse(tree, py_ver_consistent=False):
v = cStringIO()
unparser = Unparser(py_ver_consistent=py_ver_consistent)
unparser.visit(tree, v)
return v.getvalue().strip() + "\n"

408
lib/spack/external/spack_astunparse/unparser.py → lib/spack/spack/util/unparse/unparser.py
View file

@ -1,11 +1,12 @@
# Copyright (c) 2014-2021, Simon Percivall and Spack Project Developers.
#
# SPDX-License-Identifier: Python-2.0
"Usage: unparse.py <path to source file>" "Usage: unparse.py <path to source file>"
from __future__ import print_function, unicode_literals from __future__ import print_function, unicode_literals
import ast import ast
import os
import sys import sys
import tokenize
from contextlib import contextmanager from contextlib import contextmanager
import six import six
@ -22,6 +23,34 @@ def nullcontext():
# We unparse those infinities to INFSTR. # We unparse those infinities to INFSTR.
INFSTR = "1e" + repr(sys.float_info.max_10_exp + 1) INFSTR = "1e" + repr(sys.float_info.max_10_exp + 1)
class _Precedence:
"""Precedence table that originated from python grammar."""
TUPLE = 0
YIELD = 1 # 'yield', 'yield from'
TEST = 2 # 'if'-'else', 'lambda'
OR = 3 # 'or'
AND = 4 # 'and'
NOT = 5 # 'not'
CMP = 6 # '<', '>', '==', '>=', '<=', '!=', 'in', 'not in', 'is', 'is not'
EXPR = 7
BOR = EXPR # '|'
BXOR = 8 # '^'
BAND = 9 # '&'
SHIFT = 10 # '<<', '>>'
ARITH = 11 # '+', '-'
TERM = 12 # '*', '@', '/', '%', '//'
FACTOR = 13 # unary '+', '-', '~'
POWER = 14 # '**'
AWAIT = 15 # 'await'
ATOM = 16
def pnext(precedence):
return min(precedence + 1, _Precedence.ATOM)
def interleave(inter, f, seq): def interleave(inter, f, seq):
"""Call f on each item in seq, calling inter() in between. """Call f on each item in seq, calling inter() in between.
""" """
@ -35,29 +64,28 @@ def interleave(inter, f, seq):
inter() inter()
f(x) f(x)
class Unparser: class Unparser:
"""Methods in this class recursively traverse an AST and """Methods in this class recursively traverse an AST and
output source code for the abstract syntax; original formatting output source code for the abstract syntax; original formatting
is disregarded. """ is disregarded. """
def __init__(self, tree, file = sys.stdout, py_ver_consistent=False): def __init__(self, py_ver_consistent=False):
"""Unparser(tree, file=sys.stdout) -> None. """Traverse an AST and generate its source.
Print the source for tree to file.
Arguments: Arguments:
py_ver_consistent (bool): if True, generate unparsed code that is py_ver_consistent (bool): if True, generate unparsed code that is
consistent between Python 2.7 and 3.5-3.10. consistent between Python 2.7 and 3.5-3.10.
Consistency is achieved by: Consistency is achieved by:
1. Ensuring there are no spaces between unary operators and 1. Ensuring that *args and **kwargs are always the last arguments,
their operands. regardless of the python version, because Python 2's AST does not
2. Ensuring that *args and **kwargs are always the last arguments, have sufficient information to reconstruct star-arg order.
regardless of the python version. 2. Always unparsing print as a function.
3. Always unparsing print as a function.
4. Not putting an extra comma after Python 2 class definitions.
Without these changes, the same source can generate different code for different Without these changes, the same source can generate different code for Python 2
Python versions, depending on subtle AST differences. and Python 3, depending on subtle AST differences. The first of these two
causes this module to behave differently from Python 3.8+'s `ast.unparse()`
One place where single source will generate an inconsistent AST is with One place where single source will generate an inconsistent AST is with
multi-argument print statements, e.g.:: multi-argument print statements, e.g.::
@ -69,17 +97,20 @@ def __init__(self, tree, file = sys.stdout, py_ver_consistent=False):
this inconsistency. this inconsistency.
""" """
self.f = file
self.future_imports = [] self.future_imports = []
self._indent = 0 self._indent = 0
self._py_ver_consistent = py_ver_consistent self._py_ver_consistent = py_ver_consistent
self._precedences = {}
def visit(self, tree, output_file):
"""Traverse tree and write source code to output_file."""
self.f = output_file
self.dispatch(tree) self.dispatch(tree)
print("", file=self.f)
self.f.flush() self.f.flush()
def fill(self, text = ""): def fill(self, text=""):
"Indent a piece of text, according to the current indentation level" "Indent a piece of text, according to the current indentation level"
self.f.write("\n"+" "*self._indent + text) self.f.write("\n" + " " * self._indent + text)
def write(self, text): def write(self, text):
"Append a piece of text to the current line." "Append a piece of text to the current line."
@ -117,22 +148,32 @@ def delimit_if(self, start, end, condition):
else: else:
return nullcontext() return nullcontext()
def require_parens(self, precedence, t):
"""Shortcut to adding precedence related parens"""
return self.delimit_if("(", ")", self.get_precedence(t) > precedence)
def get_precedence(self, t):
return self._precedences.get(t, _Precedence.TEST)
def set_precedence(self, precedence, *nodes):
for t in nodes:
self._precedences[t] = precedence
def dispatch(self, tree): def dispatch(self, tree):
"Dispatcher function, dispatching tree type T to method _T." "Dispatcher function, dispatching tree type T to method _T."
if isinstance(tree, list): if isinstance(tree, list):
for t in tree: for t in tree:
self.dispatch(t) self.dispatch(t)
return return
meth = getattr(self, "_"+tree.__class__.__name__) meth = getattr(self, "_" + tree.__class__.__name__)
meth(tree) meth(tree)
#
############### Unparsing methods ###################### # Unparsing methods
# There should be one method per concrete grammar type # #
# Constructors should be grouped by sum type. Ideally, # # There should be one method per concrete grammar type Constructors
# this would follow the order in the grammar, but # # should be # grouped by sum type. Ideally, this would follow the order
# currently doesn't. # # in the grammar, but currently doesn't.
########################################################
def _Module(self, tree): def _Module(self, tree):
for stmt in tree.body: for stmt in tree.body:
@ -148,10 +189,12 @@ def _Expression(self, tree):
# stmt # stmt
def _Expr(self, tree): def _Expr(self, tree):
self.fill() self.fill()
self.set_precedence(_Precedence.YIELD, tree.value)
self.dispatch(tree.value) self.dispatch(tree.value)
def _NamedExpr(self, tree): def _NamedExpr(self, tree):
with self.delimit("(", ")"): with self.require_parens(_Precedence.TUPLE, tree):
self.set_precedence(_Precedence.ATOM, tree.target, tree.value)
self.dispatch(tree.target) self.dispatch(tree.target)
self.write(" := ") self.write(" := ")
self.dispatch(tree.value) self.dispatch(tree.value)
@ -182,13 +225,13 @@ def _Assign(self, t):
def _AugAssign(self, t): def _AugAssign(self, t):
self.fill() self.fill()
self.dispatch(t.target) self.dispatch(t.target)
self.write(" "+self.binop[t.op.__class__.__name__]+"= ") self.write(" " + self.binop[t.op.__class__.__name__] + "= ")
self.dispatch(t.value) self.dispatch(t.value)
def _AnnAssign(self, t): def _AnnAssign(self, t):
self.fill() self.fill()
with self.delimit_if( with self.delimit_if(
"(", ")", not node.simple and isinstance(t.target, ast.Name)): "(", ")", not t.simple and isinstance(t.target, ast.Name)):
self.dispatch(t.target) self.dispatch(t.target)
self.write(": ") self.write(": ")
self.dispatch(t.annotation) self.dispatch(t.annotation)
@ -245,8 +288,10 @@ def _Print(self, t):
self.dispatch(t.dest) self.dispatch(t.dest)
do_comma = True do_comma = True
for e in t.values: for e in t.values:
if do_comma:self.write(", ") if do_comma:
else:do_comma=True self.write(", ")
else:
do_comma = True
self.dispatch(e) self.dispatch(e)
if not t.nl: if not t.nl:
self.write(",") self.write(",")
@ -263,24 +308,27 @@ def _Nonlocal(self, t):
interleave(lambda: self.write(", "), self.write, t.names) interleave(lambda: self.write(", "), self.write, t.names)
def _Await(self, t): def _Await(self, t):
with self.delimit("(", ")"): with self.require_parens(_Precedence.AWAIT, t):
self.write("await") self.write("await")
if t.value: if t.value:
self.write(" ") self.write(" ")
self.set_precedence(_Precedence.ATOM, t.value)
self.dispatch(t.value) self.dispatch(t.value)
def _Yield(self, t): def _Yield(self, t):
with self.delimit("(", ")"): with self.require_parens(_Precedence.YIELD, t):
self.write("yield") self.write("yield")
if t.value: if t.value:
self.write(" ") self.write(" ")
self.set_precedence(_Precedence.ATOM, t.value)
self.dispatch(t.value) self.dispatch(t.value)
def _YieldFrom(self, t): def _YieldFrom(self, t):
with self.delimit("(", ")"): with self.require_parens(_Precedence.YIELD, t):
self.write("yield from") self.write("yield from")
if t.value: if t.value:
self.write(" ") self.write(" ")
self.set_precedence(_Precedence.ATOM, t.value)
self.dispatch(t.value) self.dispatch(t.value)
def _Raise(self, t): def _Raise(self, t):
@ -364,27 +412,35 @@ def _ClassDef(self, t):
for deco in t.decorator_list: for deco in t.decorator_list:
self.fill("@") self.fill("@")
self.dispatch(deco) self.dispatch(deco)
self.fill("class "+t.name) self.fill("class " + t.name)
if six.PY3: if six.PY3:
with self.delimit("(", ")"): with self.delimit("(", ")"):
comma = False comma = False
for e in t.bases: for e in t.bases:
if comma: self.write(", ") if comma:
else: comma = True self.write(", ")
else:
comma = True
self.dispatch(e) self.dispatch(e)
for e in t.keywords: for e in t.keywords:
if comma: self.write(", ") if comma:
else: comma = True self.write(", ")
else:
comma = True
self.dispatch(e) self.dispatch(e)
if sys.version_info[:2] < (3, 5): if sys.version_info[:2] < (3, 5):
if t.starargs: if t.starargs:
if comma: self.write(", ") if comma:
else: comma = True self.write(", ")
else:
comma = True
self.write("*") self.write("*")
self.dispatch(t.starargs) self.dispatch(t.starargs)
if t.kwargs: if t.kwargs:
if comma: self.write(", ") if comma:
else: comma = True self.write(", ")
else:
comma = True
self.write("**") self.write("**")
self.dispatch(t.kwargs) self.dispatch(t.kwargs)
elif t.bases: elif t.bases:
@ -497,7 +553,7 @@ def _Str(self, tree):
self.write(repr(tree.s)) self.write(repr(tree.s))
elif isinstance(tree.s, str): elif isinstance(tree.s, str):
self.write("b" + repr(tree.s)) self.write("b" + repr(tree.s))
elif isinstance(tree.s, unicode): elif isinstance(tree.s, unicode): # noqa
self.write(repr(tree.s).lstrip("u")) self.write(repr(tree.s).lstrip("u"))
else: else:
assert False, "shouldn't get here" assert False, "shouldn't get here"
@ -545,9 +601,13 @@ def _fstring_Constant(self, t, write):
def _fstring_FormattedValue(self, t, write): def _fstring_FormattedValue(self, t, write):
write("{") write("{")
expr = StringIO() expr = StringIO()
Unparser(t.value, expr) unparser = type(self)(py_ver_consistent=self.py_ver_consistent)
unparser.set_precedence(pnext(_Precedence.TEST), t.value)
unparser.visit(t.value, expr)
expr = expr.getvalue().rstrip("\n") expr = expr.getvalue().rstrip("\n")
if expr.startswith("{"): if expr.startswith("{"):
write(" ") # Separate pair of opening brackets as "{ {" write(" ") # Separate pair of opening brackets as "{ {"
write(expr) write(expr)
@ -588,7 +648,7 @@ def _Constant(self, t):
self.write(",") self.write(",")
else: else:
interleave(lambda: self.write(", "), self._write_constant, value) interleave(lambda: self.write(", "), self._write_constant, value)
elif value is Ellipsis: # instead of `...` for Py2 compatibility elif value is Ellipsis: # instead of `...` for Py2 compatibility
self.write("...") self.write("...")
else: else:
if t.kind == "u": if t.kind == "u":
@ -601,7 +661,7 @@ def _Num(self, t):
self.write(repr_n.replace("inf", INFSTR)) self.write(repr_n.replace("inf", INFSTR))
else: else:
# Parenthesize negative numbers, to avoid turning (-1)**2 into -1**2. # Parenthesize negative numbers, to avoid turning (-1)**2 into -1**2.
with self.delimit_if("(", ")", repr_n.startswith("-")): with self.require_parens(pnext(_Precedence.FACTOR), t):
if "inf" in repr_n and repr_n.endswith("*j"): if "inf" in repr_n and repr_n.endswith("*j"):
repr_n = repr_n.replace("*j", "j") repr_n = repr_n.replace("*j", "j")
# Substitute overflowing decimal literal for AST infinities. # Substitute overflowing decimal literal for AST infinities.
@ -642,23 +702,27 @@ def _comprehension(self, t):
self.write(" async for ") self.write(" async for ")
else: else:
self.write(" for ") self.write(" for ")
self.set_precedence(_Precedence.TUPLE, t.target)
self.dispatch(t.target) self.dispatch(t.target)
self.write(" in ") self.write(" in ")
self.set_precedence(pnext(_Precedence.TEST), t.iter, *t.ifs)
self.dispatch(t.iter) self.dispatch(t.iter)
for if_clause in t.ifs: for if_clause in t.ifs:
self.write(" if ") self.write(" if ")
self.dispatch(if_clause) self.dispatch(if_clause)
def _IfExp(self, t): def _IfExp(self, t):
with self.delimit("(", ")"): with self.require_parens(_Precedence.TEST, t):
self.set_precedence(pnext(_Precedence.TEST), t.body, t.test)
self.dispatch(t.body) self.dispatch(t.body)
self.write(" if ") self.write(" if ")
self.dispatch(t.test) self.dispatch(t.test)
self.write(" else ") self.write(" else ")
self.set_precedence(_Precedence.TEST, t.orelse)
self.dispatch(t.orelse) self.dispatch(t.orelse)
def _Set(self, t): def _Set(self, t):
assert(t.elts) # should be at least one element assert(t.elts) # should be at least one element
with self.delimit("{", "}"): with self.delimit("{", "}"):
interleave(lambda: self.write(", "), self.dispatch, t.elts) interleave(lambda: self.write(", "), self.dispatch, t.elts)
@ -674,6 +738,7 @@ def write_item(item):
# for dictionary unpacking operator in dicts {**{'y': 2}} # for dictionary unpacking operator in dicts {**{'y': 2}}
# see PEP 448 for details # see PEP 448 for details
self.write("**") self.write("**")
self.set_precedence(_Precedence.EXPR, v)
self.dispatch(v) self.dispatch(v)
else: else:
write_key_value_pair(k, v) write_key_value_pair(k, v)
@ -690,13 +755,33 @@ def _Tuple(self, t):
else: else:
interleave(lambda: self.write(", "), self.dispatch, t.elts) interleave(lambda: self.write(", "), self.dispatch, t.elts)
unop = {"Invert":"~", "Not": "not", "UAdd":"+", "USub":"-"} unop = {
"Invert": "~",
"Not": "not",
"UAdd": "+",
"USub": "-"
}
unop_precedence = {
"~": _Precedence.FACTOR,
"not": _Precedence.NOT,
"+": _Precedence.FACTOR,
"-": _Precedence.FACTOR,
}
def _UnaryOp(self, t): def _UnaryOp(self, t):
with self.delimit("(", ")"): operator = self.unop[t.op.__class__.__name__]
self.write(self.unop[t.op.__class__.__name__]) operator_precedence = self.unop_precedence[operator]
if not self._py_ver_consistent: with self.require_parens(operator_precedence, t):
self.write(operator)
# factor prefixes (+, -, ~) shouldn't be separated
# from the value they belong, (e.g: +1 instead of + 1)
if operator_precedence != _Precedence.FACTOR:
self.write(" ") self.write(" ")
if six.PY2 and isinstance(t.op, ast.USub) and isinstance(t.operand, ast.Num): self.set_precedence(operator_precedence, t.operand)
if (six.PY2 and
isinstance(t.op, ast.USub) and isinstance(t.operand, ast.Num)):
# If we're applying unary minus to a number, parenthesize the number. # If we're applying unary minus to a number, parenthesize the number.
# This is necessary: -2147483648 is different from -(2147483648) on # This is necessary: -2147483648 is different from -(2147483648) on
# a 32-bit machine (the first is an int, the second a long), and # a 32-bit machine (the first is an int, the second a long), and
@ -707,41 +792,117 @@ def _UnaryOp(self, t):
else: else:
self.dispatch(t.operand) self.dispatch(t.operand)
binop = { "Add":"+", "Sub":"-", "Mult":"*", "MatMult":"@", "Div":"/", "Mod":"%", binop = {
"LShift":"<<", "RShift":">>", "BitOr":"|", "BitXor":"^", "BitAnd":"&", "Add": "+",
"FloorDiv":"//", "Pow": "**"} "Sub": "-",
"Mult": "*",
"MatMult": "@",
"Div": "/",
"Mod": "%",
"LShift": "<<",
"RShift": ">>",
"BitOr": "|",
"BitXor": "^",
"BitAnd": "&",
"FloorDiv": "//",
"Pow": "**",
}
binop_precedence = {
"+": _Precedence.ARITH,
"-": _Precedence.ARITH,
"*": _Precedence.TERM,
"@": _Precedence.TERM,
"/": _Precedence.TERM,
"%": _Precedence.TERM,
"<<": _Precedence.SHIFT,
">>": _Precedence.SHIFT,
"|": _Precedence.BOR,
"^": _Precedence.BXOR,
"&": _Precedence.BAND,
"//": _Precedence.TERM,
"**": _Precedence.POWER,
}
binop_rassoc = frozenset(("**",))
def _BinOp(self, t): def _BinOp(self, t):
with self.delimit("(", ")"): operator = self.binop[t.op.__class__.__name__]
operator_precedence = self.binop_precedence[operator]
with self.require_parens(operator_precedence, t):
if operator in self.binop_rassoc:
left_precedence = pnext(operator_precedence)
right_precedence = operator_precedence
else:
left_precedence = operator_precedence
right_precedence = pnext(operator_precedence)
self.set_precedence(left_precedence, t.left)
self.dispatch(t.left) self.dispatch(t.left)
self.write(" " + self.binop[t.op.__class__.__name__] + " ") self.write(" %s " % operator)
self.set_precedence(right_precedence, t.right)
self.dispatch(t.right) self.dispatch(t.right)
cmpops = {"Eq":"==", "NotEq":"!=", "Lt":"<", "LtE":"<=", "Gt":">", "GtE":">=", cmpops = {
"Is":"is", "IsNot":"is not", "In":"in", "NotIn":"not in"} "Eq": "==",
"NotEq": "!=",
"Lt": "<",
"LtE": "<=",
"Gt": ">",
"GtE": ">=",
"Is": "is",
"IsNot": "is not",
"In": "in",
"NotIn": "not in",
}
def _Compare(self, t): def _Compare(self, t):
with self.delimit("(", ")"): with self.require_parens(_Precedence.CMP, t):
self.set_precedence(pnext(_Precedence.CMP), t.left, *t.comparators)
self.dispatch(t.left) self.dispatch(t.left)
for o, e in zip(t.ops, t.comparators): for o, e in zip(t.ops, t.comparators):
self.write(" " + self.cmpops[o.__class__.__name__] + " ") self.write(" " + self.cmpops[o.__class__.__name__] + " ")
self.dispatch(e) self.dispatch(e)
boolops = {ast.And: 'and', ast.Or: 'or'} boolops = {
def _BoolOp(self, t): "And": "and",
with self.delimit("(", ")"): "Or": "or",
s = " %s " % self.boolops[t.op.__class__] }
interleave(lambda: self.write(s), self.dispatch, t.values)
def _Attribute(self,t): boolop_precedence = {
"and": _Precedence.AND,
"or": _Precedence.OR,
}
def _BoolOp(self, t):
operator = self.boolops[t.op.__class__.__name__]
# use a dict instead of nonlocal for Python 2 compatibility
op = {"precedence": self.boolop_precedence[operator]}
def increasing_level_dispatch(t):
op["precedence"] = pnext(op["precedence"])
self.set_precedence(op["precedence"], t)
self.dispatch(t)
with self.require_parens(op["precedence"], t):
s = " %s " % operator
interleave(lambda: self.write(s), increasing_level_dispatch, t.values)
def _Attribute(self, t):
self.set_precedence(_Precedence.ATOM, t.value)
self.dispatch(t.value) self.dispatch(t.value)
# Special case: 3.__abs__() is a syntax error, so if t.value # Special case: 3.__abs__() is a syntax error, so if t.value
# is an integer literal then we need to either parenthesize # is an integer literal then we need to either parenthesize
# it or add an extra space to get 3 .__abs__(). # it or add an extra space to get 3 .__abs__().
if isinstance(t.value, getattr(ast, 'Constant', getattr(ast, 'Num', None))) and isinstance(t.value.n, int): if (isinstance(t.value, getattr(ast, 'Constant', getattr(ast, 'Num', None))) and
isinstance(t.value.n, int)):
self.write(" ") self.write(" ")
self.write(".") self.write(".")
self.write(t.attr) self.write(t.attr)
def _Call(self, t): def _Call(self, t):
self.set_precedence(_Precedence.ATOM, t.func)
self.dispatch(t.func) self.dispatch(t.func)
with self.delimit("(", ")"): with self.delimit("(", ")"):
comma = False comma = False
@ -754,8 +915,10 @@ def _Call(self, t):
if move_stars_last and isinstance(e, ast.Starred): if move_stars_last and isinstance(e, ast.Starred):
star_and_kwargs.append(e) star_and_kwargs.append(e)
else: else:
if comma: self.write(", ") if comma:
else: comma = True self.write(", ")
else:
comma = True
self.dispatch(e) self.dispatch(e)
for e in t.keywords: for e in t.keywords:
@ -763,35 +926,45 @@ def _Call(self, t):
if e.arg is None and move_stars_last: if e.arg is None and move_stars_last:
star_and_kwargs.append(e) star_and_kwargs.append(e)
else: else:
if comma: self.write(", ") if comma:
else: comma = True self.write(", ")
else:
comma = True
self.dispatch(e) self.dispatch(e)
if move_stars_last: if move_stars_last:
for e in star_and_kwargs: for e in star_and_kwargs:
if comma: self.write(", ") if comma:
else: comma = True self.write(", ")
else:
comma = True
self.dispatch(e) self.dispatch(e)
if sys.version_info[:2] < (3, 5): if sys.version_info[:2] < (3, 5):
if t.starargs: if t.starargs:
if comma: self.write(", ") if comma:
else: comma = True self.write(", ")
else:
comma = True
self.write("*") self.write("*")
self.dispatch(t.starargs) self.dispatch(t.starargs)
if t.kwargs: if t.kwargs:
if comma: self.write(", ") if comma:
else: comma = True self.write(", ")
else:
comma = True
self.write("**") self.write("**")
self.dispatch(t.kwargs) self.dispatch(t.kwargs)
def _Subscript(self, t): def _Subscript(self, t):
self.set_precedence(_Precedence.ATOM, t.value)
self.dispatch(t.value) self.dispatch(t.value)
with self.delimit("[", "]"): with self.delimit("[", "]"):
self.dispatch(t.slice) self.dispatch(t.slice)
def _Starred(self, t): def _Starred(self, t):
self.write("*") self.write("*")
self.set_precedence(_Precedence.EXPR, t.value)
self.dispatch(t.value) self.dispatch(t.value)
# slice # slice
@ -799,6 +972,7 @@ def _Ellipsis(self, t):
self.write("...") self.write("...")
def _Index(self, t): def _Index(self, t):
self.set_precedence(_Precedence.TUPLE, t.value)
self.dispatch(t.value) self.dispatch(t.value)
def _Slice(self, t): def _Slice(self, t):
@ -829,8 +1003,10 @@ def _arguments(self, t):
defaults = [None] * (len(all_args) - len(t.defaults)) + t.defaults defaults = [None] * (len(all_args) - len(t.defaults)) + t.defaults
for index, elements in enumerate(zip(all_args, defaults), 1): for index, elements in enumerate(zip(all_args, defaults), 1):
a, d = elements a, d = elements
if first:first = False if first:
else: self.write(", ") first = False
else:
self.write(", ")
self.dispatch(a) self.dispatch(a)
if d: if d:
self.write("=") self.write("=")
@ -840,8 +1016,10 @@ def _arguments(self, t):
# varargs, or bare '*' if no varargs but keyword-only arguments present # varargs, or bare '*' if no varargs but keyword-only arguments present
if t.vararg or getattr(t, "kwonlyargs", False): if t.vararg or getattr(t, "kwonlyargs", False):
if first:first = False if first:
else: self.write(", ") first = False
else:
self.write(", ")
self.write("*") self.write("*")
if t.vararg: if t.vararg:
if hasattr(t.vararg, 'arg'): if hasattr(t.vararg, 'arg'):
@ -858,8 +1036,10 @@ def _arguments(self, t):
# keyword-only arguments # keyword-only arguments
if getattr(t, "kwonlyargs", False): if getattr(t, "kwonlyargs", False):
for a, d in zip(t.kwonlyargs, t.kw_defaults): for a, d in zip(t.kwonlyargs, t.kw_defaults):
if first:first = False if first:
else: self.write(", ") first = False
else:
self.write(", ")
self.dispatch(a), self.dispatch(a),
if d: if d:
self.write("=") self.write("=")
@ -867,15 +1047,17 @@ def _arguments(self, t):
# kwargs # kwargs
if t.kwarg: if t.kwarg:
if first:first = False if first:
else: self.write(", ") first = False
else:
self.write(", ")
if hasattr(t.kwarg, 'arg'): if hasattr(t.kwarg, 'arg'):
self.write("**"+t.kwarg.arg) self.write("**" + t.kwarg.arg)
if t.kwarg.annotation: if t.kwarg.annotation:
self.write(": ") self.write(": ")
self.dispatch(t.kwarg.annotation) self.dispatch(t.kwarg.annotation)
else: else:
self.write("**"+t.kwarg) self.write("**" + t.kwarg)
if getattr(t, 'kwargannotation', None): if getattr(t, 'kwargannotation', None):
self.write(": ") self.write(": ")
self.dispatch(t.kwargannotation) self.dispatch(t.kwargannotation)
@ -890,62 +1072,20 @@ def _keyword(self, t):
self.dispatch(t.value) self.dispatch(t.value)
def _Lambda(self, t): def _Lambda(self, t):
with self.delimit("(", ")"): with self.require_parens(_Precedence.TEST, t):
self.write("lambda ") self.write("lambda ")
self.dispatch(t.args) self.dispatch(t.args)
self.write(": ") self.write(": ")
self.set_precedence(_Precedence.TEST, t.body)
self.dispatch(t.body) self.dispatch(t.body)
def _alias(self, t): def _alias(self, t):
self.write(t.name) self.write(t.name)
if t.asname: if t.asname:
self.write(" as "+t.asname) self.write(" as " + t.asname)
def _withitem(self, t): def _withitem(self, t):
self.dispatch(t.context_expr) self.dispatch(t.context_expr)
if t.optional_vars: if t.optional_vars:
self.write(" as ") self.write(" as ")
self.dispatch(t.optional_vars) self.dispatch(t.optional_vars)
def roundtrip(filename, output=sys.stdout):
if six.PY3:
with open(filename, "rb") as pyfile:
encoding = tokenize.detect_encoding(pyfile.readline)[0]
with open(filename, "r", encoding=encoding) as pyfile:
source = pyfile.read()
else:
with open(filename, "r") as pyfile:
source = pyfile.read()
tree = compile(source, filename, "exec", ast.PyCF_ONLY_AST, dont_inherit=True)
Unparser(tree, output)
def testdir(a):
try:
names = [n for n in os.listdir(a) if n.endswith('.py')]
except OSError:
print("Directory not readable: %s" % a, file=sys.stderr)
else:
for n in names:
fullname = os.path.join(a, n)
if os.path.isfile(fullname):
output = StringIO()
print('Testing %s' % fullname)
try:
roundtrip(fullname, output)
except Exception as e:
print(' Failed to compile, exception is %s' % repr(e))
elif os.path.isdir(fullname):
testdir(fullname)
def main(args):
if args[0] == '--testdir':
for a in args[1:]:
testdir(a)
else:
for a in args:
roundtrip(a)
if __name__=='__main__':
main(sys.argv[1:])