diff options
Diffstat (limited to 'Lib/lib2to3')
68 files changed, 1092 insertions, 187 deletions
diff --git a/Lib/lib2to3/Grammar.txt b/Lib/lib2to3/Grammar.txt index be034e1219..8a96726ecc 100644 --- a/Lib/lib2to3/Grammar.txt +++ b/Lib/lib2to3/Grammar.txt @@ -128,7 +128,7 @@ atom: ('(' [yield_expr|testlist_gexp] ')' | '`' testlist1 '`' | NAME | NUMBER | STRING+ | '.' '.' '.') listmaker: (test|star_expr) ( comp_for | (',' (test|star_expr))* [','] ) -testlist_gexp: test ( comp_for | (',' (test|star_expr))* [','] ) +testlist_gexp: (test|star_expr) ( comp_for | (',' (test|star_expr))* [','] ) lambdef: 'lambda' [varargslist] ':' test trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME subscriptlist: subscript (',' subscript)* [','] diff --git a/Lib/lib2to3/__main__.py b/Lib/lib2to3/__main__.py new file mode 100644 index 0000000000..80688baf27 --- /dev/null +++ b/Lib/lib2to3/__main__.py @@ -0,0 +1,4 @@ +import sys +from .main import main + +sys.exit(main("lib2to3.fixes")) diff --git a/Lib/lib2to3/btm_matcher.py b/Lib/lib2to3/btm_matcher.py new file mode 100644 index 0000000000..736ba2b9d8 --- /dev/null +++ b/Lib/lib2to3/btm_matcher.py @@ -0,0 +1,168 @@ +"""A bottom-up tree matching algorithm implementation meant to speed +up 2to3's matching process. After the tree patterns are reduced to +their rarest linear path, a linear Aho-Corasick automaton is +created. The linear automaton traverses the linear paths from the +leaves to the root of the AST and returns a set of nodes for further +matching. This reduces significantly the number of candidate nodes.""" + +__author__ = "George Boutsioukis <gboutsioukis@gmail.com>" + +import logging +import itertools +from collections import defaultdict + +from . import pytree +from .btm_utils import reduce_tree + +class BMNode(object): + """Class for a node of the Aho-Corasick automaton used in matching""" + count = itertools.count() + def __init__(self): + self.transition_table = {} + self.fixers = [] + self.id = next(BMNode.count) + self.content = '' + +class BottomMatcher(object): + """The main matcher class. After instantiating the patterns should + be added using the add_fixer method""" + + def __init__(self): + self.match = set() + self.root = BMNode() + self.nodes = [self.root] + self.fixers = [] + self.logger = logging.getLogger("RefactoringTool") + + def add_fixer(self, fixer): + """Reduces a fixer's pattern tree to a linear path and adds it + to the matcher(a common Aho-Corasick automaton). The fixer is + appended on the matching states and called when they are + reached""" + self.fixers.append(fixer) + tree = reduce_tree(fixer.pattern_tree) + linear = tree.get_linear_subpattern() + match_nodes = self.add(linear, start=self.root) + for match_node in match_nodes: + match_node.fixers.append(fixer) + + def add(self, pattern, start): + "Recursively adds a linear pattern to the AC automaton" + #print("adding pattern", pattern, "to", start) + if not pattern: + #print("empty pattern") + return [start] + if isinstance(pattern[0], tuple): + #alternatives + #print("alternatives") + match_nodes = [] + for alternative in pattern[0]: + #add all alternatives, and add the rest of the pattern + #to each end node + end_nodes = self.add(alternative, start=start) + for end in end_nodes: + match_nodes.extend(self.add(pattern[1:], end)) + return match_nodes + else: + #single token + #not last + if pattern[0] not in start.transition_table: + #transition did not exist, create new + next_node = BMNode() + start.transition_table[pattern[0]] = next_node + else: + #transition exists already, follow + next_node = start.transition_table[pattern[0]] + + if pattern[1:]: + end_nodes = self.add(pattern[1:], start=next_node) + else: + end_nodes = [next_node] + return end_nodes + + def run(self, leaves): + """The main interface with the bottom matcher. The tree is + traversed from the bottom using the constructed + automaton. Nodes are only checked once as the tree is + retraversed. When the automaton fails, we give it one more + shot(in case the above tree matches as a whole with the + rejected leaf), then we break for the next leaf. There is the + special case of multiple arguments(see code comments) where we + recheck the nodes + + Args: + The leaves of the AST tree to be matched + + Returns: + A dictionary of node matches with fixers as the keys + """ + current_ac_node = self.root + results = defaultdict(list) + for leaf in leaves: + current_ast_node = leaf + while current_ast_node: + current_ast_node.was_checked = True + for child in current_ast_node.children: + # multiple statements, recheck + if isinstance(child, pytree.Leaf) and child.value == u";": + current_ast_node.was_checked = False + break + if current_ast_node.type == 1: + #name + node_token = current_ast_node.value + else: + node_token = current_ast_node.type + + if node_token in current_ac_node.transition_table: + #token matches + current_ac_node = current_ac_node.transition_table[node_token] + for fixer in current_ac_node.fixers: + if not fixer in results: + results[fixer] = [] + results[fixer].append(current_ast_node) + + else: + #matching failed, reset automaton + current_ac_node = self.root + if (current_ast_node.parent is not None + and current_ast_node.parent.was_checked): + #the rest of the tree upwards has been checked, next leaf + break + + #recheck the rejected node once from the root + if node_token in current_ac_node.transition_table: + #token matches + current_ac_node = current_ac_node.transition_table[node_token] + for fixer in current_ac_node.fixers: + if not fixer in results.keys(): + results[fixer] = [] + results[fixer].append(current_ast_node) + + current_ast_node = current_ast_node.parent + return results + + def print_ac(self): + "Prints a graphviz diagram of the BM automaton(for debugging)" + print("digraph g{") + def print_node(node): + for subnode_key in node.transition_table.keys(): + subnode = node.transition_table[subnode_key] + print("%d -> %d [label=%s] //%s" % + (node.id, subnode.id, type_repr(subnode_key), str(subnode.fixers))) + if subnode_key == 1: + print(subnode.content) + print_node(subnode) + print_node(self.root) + print("}") + +# taken from pytree.py for debugging; only used by print_ac +_type_reprs = {} +def type_repr(type_num): + global _type_reprs + if not _type_reprs: + from .pygram import python_symbols + # printing tokens is possible but not as useful + # from .pgen2 import token // token.__dict__.items(): + for name, val in python_symbols.__dict__.items(): + if type(val) == int: _type_reprs[val] = name + return _type_reprs.setdefault(type_num, type_num) diff --git a/Lib/lib2to3/btm_utils.py b/Lib/lib2to3/btm_utils.py new file mode 100644 index 0000000000..2276dc9e96 --- /dev/null +++ b/Lib/lib2to3/btm_utils.py @@ -0,0 +1,283 @@ +"Utility functions used by the btm_matcher module" + +from . import pytree +from .pgen2 import grammar, token +from .pygram import pattern_symbols, python_symbols + +syms = pattern_symbols +pysyms = python_symbols +tokens = grammar.opmap +token_labels = token + +TYPE_ANY = -1 +TYPE_ALTERNATIVES = -2 +TYPE_GROUP = -3 + +class MinNode(object): + """This class serves as an intermediate representation of the + pattern tree during the conversion to sets of leaf-to-root + subpatterns""" + + def __init__(self, type=None, name=None): + self.type = type + self.name = name + self.children = [] + self.leaf = False + self.parent = None + self.alternatives = [] + self.group = [] + + def __repr__(self): + return str(self.type) + ' ' + str(self.name) + + def leaf_to_root(self): + """Internal method. Returns a characteristic path of the + pattern tree. This method must be run for all leaves until the + linear subpatterns are merged into a single""" + node = self + subp = [] + while node: + if node.type == TYPE_ALTERNATIVES: + node.alternatives.append(subp) + if len(node.alternatives) == len(node.children): + #last alternative + subp = [tuple(node.alternatives)] + node.alternatives = [] + node = node.parent + continue + else: + node = node.parent + subp = None + break + + if node.type == TYPE_GROUP: + node.group.append(subp) + #probably should check the number of leaves + if len(node.group) == len(node.children): + subp = get_characteristic_subpattern(node.group) + node.group = [] + node = node.parent + continue + else: + node = node.parent + subp = None + break + + if node.type == token_labels.NAME and node.name: + #in case of type=name, use the name instead + subp.append(node.name) + else: + subp.append(node.type) + + node = node.parent + return subp + + def get_linear_subpattern(self): + """Drives the leaf_to_root method. The reason that + leaf_to_root must be run multiple times is because we need to + reject 'group' matches; for example the alternative form + (a | b c) creates a group [b c] that needs to be matched. Since + matching multiple linear patterns overcomes the automaton's + capabilities, leaf_to_root merges each group into a single + choice based on 'characteristic'ity, + + i.e. (a|b c) -> (a|b) if b more characteristic than c + + Returns: The most 'characteristic'(as defined by + get_characteristic_subpattern) path for the compiled pattern + tree. + """ + + for l in self.leaves(): + subp = l.leaf_to_root() + if subp: + return subp + + def leaves(self): + "Generator that returns the leaves of the tree" + for child in self.children: + for x in child.leaves(): + yield x + if not self.children: + yield self + +def reduce_tree(node, parent=None): + """ + Internal function. Reduces a compiled pattern tree to an + intermediate representation suitable for feeding the + automaton. This also trims off any optional pattern elements(like + [a], a*). + """ + + new_node = None + #switch on the node type + if node.type == syms.Matcher: + #skip + node = node.children[0] + + if node.type == syms.Alternatives : + #2 cases + if len(node.children) <= 2: + #just a single 'Alternative', skip this node + new_node = reduce_tree(node.children[0], parent) + else: + #real alternatives + new_node = MinNode(type=TYPE_ALTERNATIVES) + #skip odd children('|' tokens) + for child in node.children: + if node.children.index(child)%2: + continue + reduced = reduce_tree(child, new_node) + if reduced is not None: + new_node.children.append(reduced) + elif node.type == syms.Alternative: + if len(node.children) > 1: + + new_node = MinNode(type=TYPE_GROUP) + for child in node.children: + reduced = reduce_tree(child, new_node) + if reduced: + new_node.children.append(reduced) + if not new_node.children: + # delete the group if all of the children were reduced to None + new_node = None + + else: + new_node = reduce_tree(node.children[0], parent) + + elif node.type == syms.Unit: + if (isinstance(node.children[0], pytree.Leaf) and + node.children[0].value == '('): + #skip parentheses + return reduce_tree(node.children[1], parent) + if ((isinstance(node.children[0], pytree.Leaf) and + node.children[0].value == '[') + or + (len(node.children)>1 and + hasattr(node.children[1], "value") and + node.children[1].value == '[')): + #skip whole unit if its optional + return None + + leaf = True + details_node = None + alternatives_node = None + has_repeater = False + repeater_node = None + has_variable_name = False + + for child in node.children: + if child.type == syms.Details: + leaf = False + details_node = child + elif child.type == syms.Repeater: + has_repeater = True + repeater_node = child + elif child.type == syms.Alternatives: + alternatives_node = child + if hasattr(child, 'value') and child.value == '=': # variable name + has_variable_name = True + + #skip variable name + if has_variable_name: + #skip variable name, '=' + name_leaf = node.children[2] + if hasattr(name_leaf, 'value') and name_leaf.value == '(': + # skip parenthesis + name_leaf = node.children[3] + else: + name_leaf = node.children[0] + + #set node type + if name_leaf.type == token_labels.NAME: + #(python) non-name or wildcard + if name_leaf.value == 'any': + new_node = MinNode(type=TYPE_ANY) + else: + if hasattr(token_labels, name_leaf.value): + new_node = MinNode(type=getattr(token_labels, name_leaf.value)) + else: + new_node = MinNode(type=getattr(pysyms, name_leaf.value)) + + elif name_leaf.type == token_labels.STRING: + #(python) name or character; remove the apostrophes from + #the string value + name = name_leaf.value.strip("'") + if name in tokens: + new_node = MinNode(type=tokens[name]) + else: + new_node = MinNode(type=token_labels.NAME, name=name) + elif name_leaf.type == syms.Alternatives: + new_node = reduce_tree(alternatives_node, parent) + + #handle repeaters + if has_repeater: + if repeater_node.children[0].value == '*': + #reduce to None + new_node = None + elif repeater_node.children[0].value == '+': + #reduce to a single occurence i.e. do nothing + pass + else: + #TODO: handle {min, max} repeaters + raise NotImplementedError + pass + + #add children + if details_node and new_node is not None: + for child in details_node.children[1:-1]: + #skip '<', '>' markers + reduced = reduce_tree(child, new_node) + if reduced is not None: + new_node.children.append(reduced) + if new_node: + new_node.parent = parent + return new_node + + +def get_characteristic_subpattern(subpatterns): + """Picks the most characteristic from a list of linear patterns + Current order used is: + names > common_names > common_chars + """ + if not isinstance(subpatterns, list): + return subpatterns + if len(subpatterns)==1: + return subpatterns[0] + + # first pick out the ones containing variable names + subpatterns_with_names = [] + subpatterns_with_common_names = [] + common_names = ['in', 'for', 'if' , 'not', 'None'] + subpatterns_with_common_chars = [] + common_chars = "[]().,:" + for subpattern in subpatterns: + if any(rec_test(subpattern, lambda x: type(x) is str)): + if any(rec_test(subpattern, + lambda x: isinstance(x, str) and x in common_chars)): + subpatterns_with_common_chars.append(subpattern) + elif any(rec_test(subpattern, + lambda x: isinstance(x, str) and x in common_names)): + subpatterns_with_common_names.append(subpattern) + + else: + subpatterns_with_names.append(subpattern) + + if subpatterns_with_names: + subpatterns = subpatterns_with_names + elif subpatterns_with_common_names: + subpatterns = subpatterns_with_common_names + elif subpatterns_with_common_chars: + subpatterns = subpatterns_with_common_chars + # of the remaining subpatterns pick out the longest one + return max(subpatterns, key=len) + +def rec_test(sequence, test_func): + """Tests test_func on all items of sequence and items of included + sub-iterables""" + for x in sequence: + if isinstance(x, (list, tuple)): + for y in rec_test(x, test_func): + yield y + else: + yield test_func(x) diff --git a/Lib/lib2to3/fixer_base.py b/Lib/lib2to3/fixer_base.py index 16887aabc2..f6421ba3f7 100644 --- a/Lib/lib2to3/fixer_base.py +++ b/Lib/lib2to3/fixer_base.py @@ -24,6 +24,7 @@ class BaseFix(object): PATTERN = None # Most subclasses should override with a string literal pattern = None # Compiled pattern, set by compile_pattern() + pattern_tree = None # Tree representation of the pattern options = None # Options object passed to initializer filename = None # The filename (set by set_filename) logger = None # A logger (set by set_filename) @@ -36,6 +37,12 @@ class BaseFix(object): _accept_type = None # [Advanced and not public] This tells RefactoringTool # which node type to accept when there's not a pattern. + keep_line_order = False # For the bottom matcher: match with the + # original line order + BM_compatible = False # Compatibility with the bottom matching + # module; every fixer should set this + # manually + # Shortcut for access to Python grammar symbols syms = pygram.python_symbols @@ -58,7 +65,9 @@ class BaseFix(object): self.{pattern,PATTERN} in .match(). """ if self.PATTERN is not None: - self.pattern = PatternCompiler().compile_pattern(self.PATTERN) + PC = PatternCompiler() + self.pattern, self.pattern_tree = PC.compile_pattern(self.PATTERN, + with_tree=True) def set_filename(self, filename): """Set the filename, and a logger derived from it. diff --git a/Lib/lib2to3/fixer_util.py b/Lib/lib2to3/fixer_util.py index 3b136694a6..30da893711 100644 --- a/Lib/lib2to3/fixer_util.py +++ b/Lib/lib2to3/fixer_util.py @@ -1,6 +1,8 @@ """Utility functions, node construction macros, etc.""" # Author: Collin Winter +from itertools import islice + # Local imports from .pgen2 import token from .pytree import Leaf, Node @@ -14,7 +16,7 @@ from . import patcomp def KeywordArg(keyword, value): return Node(syms.argument, - [keyword, Leaf(token.EQUAL, u'='), value]) + [keyword, Leaf(token.EQUAL, u"="), value]) def LParen(): return Leaf(token.LPAR, u"(") @@ -76,9 +78,9 @@ def Number(n, prefix=None): def Subscript(index_node): """A numeric or string subscript""" - return Node(syms.trailer, [Leaf(token.LBRACE, u'['), + return Node(syms.trailer, [Leaf(token.LBRACE, u"["), index_node, - Leaf(token.RBRACE, u']')]) + Leaf(token.RBRACE, u"]")]) def String(string, prefix=None): """A string leaf""" @@ -120,9 +122,9 @@ def FromImport(package_name, name_leafs): # Pull the leaves out of their old tree leaf.remove() - children = [Leaf(token.NAME, u'from'), + children = [Leaf(token.NAME, u"from"), Leaf(token.NAME, package_name, prefix=u" "), - Leaf(token.NAME, u'import', prefix=u" "), + Leaf(token.NAME, u"import", prefix=u" "), Node(syms.import_as_names, name_leafs)] imp = Node(syms.import_from, children) return imp @@ -245,6 +247,16 @@ def is_probably_builtin(node): return False return True +def find_indentation(node): + """Find the indentation of *node*.""" + while node is not None: + if node.type == syms.suite and len(node.children) > 2: + indent = node.children[1] + if indent.type == token.INDENT: + return indent.value + node = node.parent + return u"" + ########################################################### ### The following functions are to find bindings in a suite ########################################################### @@ -283,8 +295,8 @@ def touch_import(package, name, node): """ Works like `does_tree_import` but adds an import statement if it was not imported. """ def is_import_stmt(node): - return node.type == syms.simple_stmt and node.children and \ - is_import(node.children[0]) + return (node.type == syms.simple_stmt and node.children and + is_import(node.children[0])) root = find_root(node) @@ -307,18 +319,18 @@ def touch_import(package, name, node): # if that also fails, we stick to the beginning of the file if insert_pos == 0: for idx, node in enumerate(root.children): - if node.type == syms.simple_stmt and node.children and \ - node.children[0].type == token.STRING: + if (node.type == syms.simple_stmt and node.children and + node.children[0].type == token.STRING): insert_pos = idx + 1 break if package is None: import_ = Node(syms.import_name, [ - Leaf(token.NAME, u'import'), - Leaf(token.NAME, name, prefix=u' ') + Leaf(token.NAME, u"import"), + Leaf(token.NAME, name, prefix=u" ") ]) else: - import_ = FromImport(package, [Leaf(token.NAME, name, prefix=u' ')]) + import_ = FromImport(package, [Leaf(token.NAME, name, prefix=u" ")]) children = [import_, Newline()] root.insert_child(insert_pos, Node(syms.simple_stmt, children)) @@ -404,7 +416,7 @@ def _is_import_binding(node, name, package=None): if package and unicode(node.children[1]).strip() != package: return None n = node.children[3] - if package and _find(u'as', n): + if package and _find(u"as", n): # See test_from_import_as for explanation return None elif n.type == syms.import_as_names and _find(name, n): diff --git a/Lib/lib2to3/fixes/fix_apply.py b/Lib/lib2to3/fixes/fix_apply.py index 6d1c5d9ed4..a7dc3a046d 100644 --- a/Lib/lib2to3/fixes/fix_apply.py +++ b/Lib/lib2to3/fixes/fix_apply.py @@ -12,6 +12,7 @@ from .. import fixer_base from ..fixer_util import Call, Comma, parenthesize class FixApply(fixer_base.BaseFix): + BM_compatible = True PATTERN = """ power< 'apply' diff --git a/Lib/lib2to3/fixes/fix_basestring.py b/Lib/lib2to3/fixes/fix_basestring.py index 0de6c26e9d..a3c9a43649 100644 --- a/Lib/lib2to3/fixes/fix_basestring.py +++ b/Lib/lib2to3/fixes/fix_basestring.py @@ -6,6 +6,7 @@ from .. import fixer_base from ..fixer_util import Name class FixBasestring(fixer_base.BaseFix): + BM_compatible = True PATTERN = "'basestring'" diff --git a/Lib/lib2to3/fixes/fix_buffer.py b/Lib/lib2to3/fixes/fix_buffer.py index 1c1c4a128b..c6b0928024 100644 --- a/Lib/lib2to3/fixes/fix_buffer.py +++ b/Lib/lib2to3/fixes/fix_buffer.py @@ -9,6 +9,7 @@ from ..fixer_util import Name class FixBuffer(fixer_base.BaseFix): + BM_compatible = True explicit = True # The user must ask for this fixer diff --git a/Lib/lib2to3/fixes/fix_callable.py b/Lib/lib2to3/fixes/fix_callable.py index ca5d20b3ea..df33d614ba 100644 --- a/Lib/lib2to3/fixes/fix_callable.py +++ b/Lib/lib2to3/fixes/fix_callable.py @@ -11,6 +11,9 @@ from lib2to3 import fixer_base from lib2to3.fixer_util import Call, Name, String, Attr, touch_import class FixCallable(fixer_base.BaseFix): + BM_compatible = True + + order = "pre" # Ignore callable(*args) or use of keywords. # Either could be a hint that the builtin callable() is not being used. diff --git a/Lib/lib2to3/fixes/fix_dict.py b/Lib/lib2to3/fixes/fix_dict.py index 5912cbf611..f681e4d713 100644 --- a/Lib/lib2to3/fixes/fix_dict.py +++ b/Lib/lib2to3/fixes/fix_dict.py @@ -40,6 +40,8 @@ iter_exempt = fixer_util.consuming_calls | set(["iter"]) class FixDict(fixer_base.BaseFix): + BM_compatible = True + PATTERN = """ power< head=any+ trailer< '.' method=('keys'|'items'|'values'| diff --git a/Lib/lib2to3/fixes/fix_except.py b/Lib/lib2to3/fixes/fix_except.py index 8118058b7f..e324718f64 100644 --- a/Lib/lib2to3/fixes/fix_except.py +++ b/Lib/lib2to3/fixes/fix_except.py @@ -34,6 +34,7 @@ def find_excepts(nodes): yield (n, nodes[i+2]) class FixExcept(fixer_base.BaseFix): + BM_compatible = True PATTERN = """ try_stmt< 'try' ':' (simple_stmt | suite) diff --git a/Lib/lib2to3/fixes/fix_exec.py b/Lib/lib2to3/fixes/fix_exec.py index 121431c0c2..50e1854454 100644 --- a/Lib/lib2to3/fixes/fix_exec.py +++ b/Lib/lib2to3/fixes/fix_exec.py @@ -16,6 +16,7 @@ from ..fixer_util import Comma, Name, Call class FixExec(fixer_base.BaseFix): + BM_compatible = True PATTERN = """ exec_stmt< 'exec' a=any 'in' b=any [',' c=any] > diff --git a/Lib/lib2to3/fixes/fix_execfile.py b/Lib/lib2to3/fixes/fix_execfile.py index 38d6abc9b3..2f29d3b281 100644 --- a/Lib/lib2to3/fixes/fix_execfile.py +++ b/Lib/lib2to3/fixes/fix_execfile.py @@ -13,6 +13,7 @@ from ..fixer_util import (Comma, Name, Call, LParen, RParen, Dot, Node, class FixExecfile(fixer_base.BaseFix): + BM_compatible = True PATTERN = """ power< 'execfile' trailer< '(' arglist< filename=any [',' globals=any [',' locals=any ] ] > ')' > > diff --git a/Lib/lib2to3/fixes/fix_exitfunc.py b/Lib/lib2to3/fixes/fix_exitfunc.py index 28d30b8f79..89fb3db533 100644 --- a/Lib/lib2to3/fixes/fix_exitfunc.py +++ b/Lib/lib2to3/fixes/fix_exitfunc.py @@ -9,6 +9,8 @@ from lib2to3.fixer_util import Name, Attr, Call, Comma, Newline, syms class FixExitfunc(fixer_base.BaseFix): + keep_line_order = True + BM_compatible = True PATTERN = """ ( diff --git a/Lib/lib2to3/fixes/fix_filter.py b/Lib/lib2to3/fixes/fix_filter.py index 0e92b14521..18ee2ffc06 100644 --- a/Lib/lib2to3/fixes/fix_filter.py +++ b/Lib/lib2to3/fixes/fix_filter.py @@ -19,6 +19,7 @@ from .. import fixer_base from ..fixer_util import Name, Call, ListComp, in_special_context class FixFilter(fixer_base.ConditionalFix): + BM_compatible = True PATTERN = """ filter_lambda=power< diff --git a/Lib/lib2to3/fixes/fix_funcattrs.py b/Lib/lib2to3/fixes/fix_funcattrs.py index 5d1ba8cfa8..9e45c02857 100644 --- a/Lib/lib2to3/fixes/fix_funcattrs.py +++ b/Lib/lib2to3/fixes/fix_funcattrs.py @@ -7,6 +7,8 @@ from ..fixer_util import Name class FixFuncattrs(fixer_base.BaseFix): + BM_compatible = True + PATTERN = """ power< any+ trailer< '.' attr=('func_closure' | 'func_doc' | 'func_globals' | 'func_name' | 'func_defaults' | 'func_code' diff --git a/Lib/lib2to3/fixes/fix_future.py b/Lib/lib2to3/fixes/fix_future.py index 861e13c62c..fbcb86af07 100644 --- a/Lib/lib2to3/fixes/fix_future.py +++ b/Lib/lib2to3/fixes/fix_future.py @@ -9,6 +9,8 @@ from .. import fixer_base from ..fixer_util import BlankLine class FixFuture(fixer_base.BaseFix): + BM_compatible = True + PATTERN = """import_from< 'from' module_name="__future__" 'import' any >""" # This should be run last -- some things check for the import diff --git a/Lib/lib2to3/fixes/fix_getcwdu.py b/Lib/lib2to3/fixes/fix_getcwdu.py index a5b5aa3778..82233c8993 100644 --- a/Lib/lib2to3/fixes/fix_getcwdu.py +++ b/Lib/lib2to3/fixes/fix_getcwdu.py @@ -8,6 +8,7 @@ from .. import fixer_base from ..fixer_util import Name class FixGetcwdu(fixer_base.BaseFix): + BM_compatible = True PATTERN = """ power< 'os' trailer< dot='.' name='getcwdu' > any* > diff --git a/Lib/lib2to3/fixes/fix_has_key.py b/Lib/lib2to3/fixes/fix_has_key.py index 0915a363bd..bead4cb51c 100644 --- a/Lib/lib2to3/fixes/fix_has_key.py +++ b/Lib/lib2to3/fixes/fix_has_key.py @@ -37,6 +37,7 @@ from ..fixer_util import Name, parenthesize class FixHasKey(fixer_base.BaseFix): + BM_compatible = True PATTERN = """ anchor=power< diff --git a/Lib/lib2to3/fixes/fix_idioms.py b/Lib/lib2to3/fixes/fix_idioms.py index c07b77e55a..37b6eefa51 100644 --- a/Lib/lib2to3/fixes/fix_idioms.py +++ b/Lib/lib2to3/fixes/fix_idioms.py @@ -35,7 +35,6 @@ CMP = "(n='!=' | '==' | 'is' | n=comp_op< 'is' 'not' >)" TYPE = "power< 'type' trailer< '(' x=any ')' > >" class FixIdioms(fixer_base.BaseFix): - explicit = True # The user must ask for this fixer PATTERN = r""" diff --git a/Lib/lib2to3/fixes/fix_import.py b/Lib/lib2to3/fixes/fix_import.py index d35fa0caf8..201e811e61 100644 --- a/Lib/lib2to3/fixes/fix_import.py +++ b/Lib/lib2to3/fixes/fix_import.py @@ -36,6 +36,7 @@ def traverse_imports(names): class FixImport(fixer_base.BaseFix): + BM_compatible = True PATTERN = """ import_from< 'from' imp=any 'import' ['('] any [')'] > diff --git a/Lib/lib2to3/fixes/fix_imports.py b/Lib/lib2to3/fixes/fix_imports.py index e1ad667f7b..93c9e6787b 100644 --- a/Lib/lib2to3/fixes/fix_imports.py +++ b/Lib/lib2to3/fixes/fix_imports.py @@ -84,6 +84,8 @@ def build_pattern(mapping=MAPPING): class FixImports(fixer_base.BaseFix): + BM_compatible = True + keep_line_order = True # This is overridden in fix_imports2. mapping = MAPPING diff --git a/Lib/lib2to3/fixes/fix_input.py b/Lib/lib2to3/fixes/fix_input.py index 291cadf220..fbf4c72f5c 100644 --- a/Lib/lib2to3/fixes/fix_input.py +++ b/Lib/lib2to3/fixes/fix_input.py @@ -11,7 +11,7 @@ context = patcomp.compile_pattern("power< 'eval' trailer< '(' any ')' > >") class FixInput(fixer_base.BaseFix): - + BM_compatible = True PATTERN = """ power< 'input' args=trailer< '(' [any] ')' > > """ diff --git a/Lib/lib2to3/fixes/fix_intern.py b/Lib/lib2to3/fixes/fix_intern.py index 67f98efbed..e7bb5052b4 100644 --- a/Lib/lib2to3/fixes/fix_intern.py +++ b/Lib/lib2to3/fixes/fix_intern.py @@ -12,6 +12,8 @@ from ..fixer_util import Name, Attr, touch_import class FixIntern(fixer_base.BaseFix): + BM_compatible = True + order = "pre" PATTERN = """ power< 'intern' diff --git a/Lib/lib2to3/fixes/fix_isinstance.py b/Lib/lib2to3/fixes/fix_isinstance.py index c720a837e2..4b04c8fd0c 100644 --- a/Lib/lib2to3/fixes/fix_isinstance.py +++ b/Lib/lib2to3/fixes/fix_isinstance.py @@ -14,7 +14,7 @@ from ..fixer_util import token class FixIsinstance(fixer_base.BaseFix): - + BM_compatible = True PATTERN = """ power< 'isinstance' diff --git a/Lib/lib2to3/fixes/fix_itertools.py b/Lib/lib2to3/fixes/fix_itertools.py index 77b3a015f2..27f8a49ec6 100644 --- a/Lib/lib2to3/fixes/fix_itertools.py +++ b/Lib/lib2to3/fixes/fix_itertools.py @@ -12,7 +12,8 @@ from .. import fixer_base from ..fixer_util import Name class FixItertools(fixer_base.BaseFix): - it_funcs = "('imap'|'ifilter'|'izip'|'ifilterfalse')" + BM_compatible = True + it_funcs = "('imap'|'ifilter'|'izip'|'izip_longest'|'ifilterfalse')" PATTERN = """ power< it='itertools' trailer< @@ -27,7 +28,8 @@ class FixItertools(fixer_base.BaseFix): def transform(self, node, results): prefix = None func = results['func'][0] - if 'it' in results and func.value != u'ifilterfalse': + if ('it' in results and + func.value not in (u'ifilterfalse', u'izip_longest')): dot, it = (results['dot'], results['it']) # Remove the 'itertools' prefix = it.prefix diff --git a/Lib/lib2to3/fixes/fix_itertools_imports.py b/Lib/lib2to3/fixes/fix_itertools_imports.py index c7223de747..28610cfcb9 100644 --- a/Lib/lib2to3/fixes/fix_itertools_imports.py +++ b/Lib/lib2to3/fixes/fix_itertools_imports.py @@ -6,6 +6,7 @@ from lib2to3.fixer_util import BlankLine, syms, token class FixItertoolsImports(fixer_base.BaseFix): + BM_compatible = True PATTERN = """ import_from< 'from' 'itertools' 'import' imports=any > """ %(locals()) @@ -20,6 +21,9 @@ class FixItertoolsImports(fixer_base.BaseFix): if child.type == token.NAME: member = child.value name_node = child + elif child.type == token.STAR: + # Just leave the import as is. + return else: assert child.type == syms.import_as_name name_node = child.children[0] @@ -27,9 +31,10 @@ class FixItertoolsImports(fixer_base.BaseFix): if member_name in (u'imap', u'izip', u'ifilter'): child.value = None child.remove() - elif member_name == u'ifilterfalse': + elif member_name in (u'ifilterfalse', u'izip_longest'): node.changed() - name_node.value = u'filterfalse' + name_node.value = (u'filterfalse' if member_name[1] == u'f' + else u'zip_longest') # Make sure the import statement is still sane children = imports.children[:] or [imports] @@ -40,12 +45,12 @@ class FixItertoolsImports(fixer_base.BaseFix): else: remove_comma ^= True - if children[-1].type == token.COMMA: - children[-1].remove() + while children and children[-1].type == token.COMMA: + children.pop().remove() # If there are no imports left, just get rid of the entire statement - if not (imports.children or getattr(imports, 'value', None)) or \ - imports.parent is None: + if (not (imports.children or getattr(imports, 'value', None)) or + imports.parent is None): p = node.prefix node = BlankLine() node.prefix = p diff --git a/Lib/lib2to3/fixes/fix_long.py b/Lib/lib2to3/fixes/fix_long.py index 6f3661cbab..5dddde0d08 100644 --- a/Lib/lib2to3/fixes/fix_long.py +++ b/Lib/lib2to3/fixes/fix_long.py @@ -10,7 +10,7 @@ from lib2to3.fixer_util import is_probably_builtin class FixLong(fixer_base.BaseFix): - + BM_compatible = True PATTERN = "'long'" def transform(self, node, results): diff --git a/Lib/lib2to3/fixes/fix_map.py b/Lib/lib2to3/fixes/fix_map.py index 6d269573ad..7a7d0dbc66 100644 --- a/Lib/lib2to3/fixes/fix_map.py +++ b/Lib/lib2to3/fixes/fix_map.py @@ -26,6 +26,7 @@ from ..fixer_util import Name, Call, ListComp, in_special_context from ..pygram import python_symbols as syms class FixMap(fixer_base.ConditionalFix): + BM_compatible = True PATTERN = """ map_none=power< diff --git a/Lib/lib2to3/fixes/fix_metaclass.py b/Lib/lib2to3/fixes/fix_metaclass.py index 789cb168bb..c86fbeaab2 100644 --- a/Lib/lib2to3/fixes/fix_metaclass.py +++ b/Lib/lib2to3/fixes/fix_metaclass.py @@ -48,7 +48,7 @@ def fixup_parse_tree(cls_node): """ for node in cls_node.children: if node.type == syms.suite: - # already in the prefered format, do nothing + # already in the preferred format, do nothing return # !%@#! oneliners have no suite node, we have to fake one up @@ -143,6 +143,7 @@ def fixup_indent(suite): class FixMetaclass(fixer_base.BaseFix): + BM_compatible = True PATTERN = """ classdef<any*> diff --git a/Lib/lib2to3/fixes/fix_methodattrs.py b/Lib/lib2to3/fixes/fix_methodattrs.py index 5e94a9c526..f3c1ecfec1 100644 --- a/Lib/lib2to3/fixes/fix_methodattrs.py +++ b/Lib/lib2to3/fixes/fix_methodattrs.py @@ -13,6 +13,7 @@ MAP = { } class FixMethodattrs(fixer_base.BaseFix): + BM_compatible = True PATTERN = """ power< any+ trailer< '.' attr=('im_func' | 'im_self' | 'im_class') > any* > """ diff --git a/Lib/lib2to3/fixes/fix_next.py b/Lib/lib2to3/fixes/fix_next.py index 9514798f4c..f021a9bd70 100644 --- a/Lib/lib2to3/fixes/fix_next.py +++ b/Lib/lib2to3/fixes/fix_next.py @@ -15,6 +15,7 @@ bind_warning = "Calls to builtin next() possibly shadowed by global binding" class FixNext(fixer_base.BaseFix): + BM_compatible = True PATTERN = """ power< base=any+ trailer< '.' attr='next' > trailer< '(' ')' > > | diff --git a/Lib/lib2to3/fixes/fix_nonzero.py b/Lib/lib2to3/fixes/fix_nonzero.py index 17be377013..ba83478f81 100644 --- a/Lib/lib2to3/fixes/fix_nonzero.py +++ b/Lib/lib2to3/fixes/fix_nonzero.py @@ -6,6 +6,7 @@ from .. import fixer_base from ..fixer_util import Name, syms class FixNonzero(fixer_base.BaseFix): + BM_compatible = True PATTERN = """ classdef< 'class' any+ ':' suite< any* diff --git a/Lib/lib2to3/fixes/fix_operator.py b/Lib/lib2to3/fixes/fix_operator.py index d9bf160419..7bf2c0dd22 100644 --- a/Lib/lib2to3/fixes/fix_operator.py +++ b/Lib/lib2to3/fixes/fix_operator.py @@ -1,40 +1,96 @@ -"""Fixer for operator.{isCallable,sequenceIncludes} +"""Fixer for operator functions. -operator.isCallable(obj) -> hasattr(obj, '__call__') +operator.isCallable(obj) -> hasattr(obj, '__call__') operator.sequenceIncludes(obj) -> operator.contains(obj) +operator.isSequenceType(obj) -> isinstance(obj, collections.Sequence) +operator.isMappingType(obj) -> isinstance(obj, collections.Mapping) +operator.isNumberType(obj) -> isinstance(obj, numbers.Number) +operator.repeat(obj, n) -> operator.mul(obj, n) +operator.irepeat(obj, n) -> operator.imul(obj, n) """ # Local imports -from .. import fixer_base -from ..fixer_util import Call, Name, String +from lib2to3 import fixer_base +from lib2to3.fixer_util import Call, Name, String, touch_import + + +def invocation(s): + def dec(f): + f.invocation = s + return f + return dec + class FixOperator(fixer_base.BaseFix): + BM_compatible = True + order = "pre" - methods = "method=('isCallable'|'sequenceIncludes')" - func = "'(' func=any ')'" + methods = """ + method=('isCallable'|'sequenceIncludes' + |'isSequenceType'|'isMappingType'|'isNumberType' + |'repeat'|'irepeat') + """ + obj = "'(' obj=any ')'" PATTERN = """ power< module='operator' - trailer< '.' %(methods)s > trailer< %(func)s > > + trailer< '.' %(methods)s > trailer< %(obj)s > > | - power< %(methods)s trailer< %(func)s > > - """ % dict(methods=methods, func=func) + power< %(methods)s trailer< %(obj)s > > + """ % dict(methods=methods, obj=obj) def transform(self, node, results): + method = self._check_method(node, results) + if method is not None: + return method(node, results) + + @invocation("operator.contains(%s)") + def _sequenceIncludes(self, node, results): + return self._handle_rename(node, results, u"contains") + + @invocation("hasattr(%s, '__call__')") + def _isCallable(self, node, results): + obj = results["obj"] + args = [obj.clone(), String(u", "), String(u"'__call__'")] + return Call(Name(u"hasattr"), args, prefix=node.prefix) + + @invocation("operator.mul(%s)") + def _repeat(self, node, results): + return self._handle_rename(node, results, u"mul") + + @invocation("operator.imul(%s)") + def _irepeat(self, node, results): + return self._handle_rename(node, results, u"imul") + + @invocation("isinstance(%s, collections.Sequence)") + def _isSequenceType(self, node, results): + return self._handle_type2abc(node, results, u"collections", u"Sequence") + + @invocation("isinstance(%s, collections.Mapping)") + def _isMappingType(self, node, results): + return self._handle_type2abc(node, results, u"collections", u"Mapping") + + @invocation("isinstance(%s, numbers.Number)") + def _isNumberType(self, node, results): + return self._handle_type2abc(node, results, u"numbers", u"Number") + + def _handle_rename(self, node, results, name): method = results["method"][0] + method.value = name + method.changed() - if method.value == u"sequenceIncludes": - if "module" not in results: - # operator may not be in scope, so we can't make a change. - self.warning(node, "You should use operator.contains here.") - else: - method.value = u"contains" - method.changed() - elif method.value == u"isCallable": - if "module" not in results: - self.warning(node, - "You should use hasattr(%s, '__call__') here." % - results["func"].value) + def _handle_type2abc(self, node, results, module, abc): + touch_import(None, module, node) + obj = results["obj"] + args = [obj.clone(), String(u", " + u".".join([module, abc]))] + return Call(Name(u"isinstance"), args, prefix=node.prefix) + + def _check_method(self, node, results): + method = getattr(self, "_" + results["method"][0].value.encode("ascii")) + if callable(method): + if "module" in results: + return method else: - func = results["func"] - args = [func.clone(), String(u", "), String(u"'__call__'")] - return Call(Name(u"hasattr"), args, prefix=node.prefix) + sub = (unicode(results["obj"]),) + invocation_str = unicode(method.invocation) % sub + self.warning(node, u"You should use '%s' here." % invocation_str) + return None diff --git a/Lib/lib2to3/fixes/fix_paren.py b/Lib/lib2to3/fixes/fix_paren.py index eeb0d4070a..8650cd9078 100644 --- a/Lib/lib2to3/fixes/fix_paren.py +++ b/Lib/lib2to3/fixes/fix_paren.py @@ -10,6 +10,8 @@ from ..fixer_util import LParen, RParen # XXX This doesn't support nested for loops like [x for x in 1, 2 for x in 1, 2] class FixParen(fixer_base.BaseFix): + BM_compatible = True + PATTERN = """ atom< ('[' | '(') (listmaker< any diff --git a/Lib/lib2to3/fixes/fix_print.py b/Lib/lib2to3/fixes/fix_print.py index 6cad8ce59a..98786b3ecd 100644 --- a/Lib/lib2to3/fixes/fix_print.py +++ b/Lib/lib2to3/fixes/fix_print.py @@ -28,6 +28,8 @@ parend_expr = patcomp.compile_pattern( class FixPrint(fixer_base.BaseFix): + BM_compatible = True + PATTERN = """ simple_stmt< any* bare='print' any* > | print_stmt """ diff --git a/Lib/lib2to3/fixes/fix_raise.py b/Lib/lib2to3/fixes/fix_raise.py index 24e123c730..b958ba0129 100644 --- a/Lib/lib2to3/fixes/fix_raise.py +++ b/Lib/lib2to3/fixes/fix_raise.py @@ -4,6 +4,7 @@ raise -> raise raise E -> raise E raise E, V -> raise E(V) raise E, V, T -> raise E(V).with_traceback(T) +raise E, None, T -> raise E.with_traceback(T) raise (((E, E'), E''), E'''), V -> raise E(V) raise "foo", V, T -> warns about string exceptions @@ -29,6 +30,7 @@ from ..fixer_util import Name, Call, Attr, ArgList, is_tuple class FixRaise(fixer_base.BaseFix): + BM_compatible = True PATTERN = """ raise_stmt< 'raise' exc=any [',' val=any [',' tb=any]] > """ @@ -37,8 +39,9 @@ class FixRaise(fixer_base.BaseFix): syms = self.syms exc = results["exc"].clone() - if exc.type is token.STRING: - self.cannot_convert(node, "Python 3 does not support string exceptions") + if exc.type == token.STRING: + msg = "Python 3 does not support string exceptions" + self.cannot_convert(node, msg) return # Python 2 supports @@ -52,7 +55,7 @@ class FixRaise(fixer_base.BaseFix): # exc.children[1:-1] is the unparenthesized tuple # exc.children[1].children[0] is the first element of the tuple exc = exc.children[1].children[0].clone() - exc.prefix = " " + exc.prefix = u" " if "val" not in results: # One-argument raise @@ -71,7 +74,12 @@ class FixRaise(fixer_base.BaseFix): tb = results["tb"].clone() tb.prefix = u"" - e = Call(exc, args) + e = exc + # If there's a traceback and None is passed as the value, then don't + # add a call, since the user probably just wants to add a + # traceback. See issue #9661. + if val.type != token.NAME or val.value != u"None": + e = Call(exc, args) with_tb = Attr(e, Name(u'with_traceback')) + [ArgList([tb])] new = pytree.Node(syms.simple_stmt, [Name(u"raise")] + with_tb) new.prefix = node.prefix diff --git a/Lib/lib2to3/fixes/fix_raw_input.py b/Lib/lib2to3/fixes/fix_raw_input.py index ba045d6341..3a73b81864 100644 --- a/Lib/lib2to3/fixes/fix_raw_input.py +++ b/Lib/lib2to3/fixes/fix_raw_input.py @@ -7,6 +7,7 @@ from ..fixer_util import Name class FixRawInput(fixer_base.BaseFix): + BM_compatible = True PATTERN = """ power< name='raw_input' trailer< '(' [any] ')' > any* > """ diff --git a/Lib/lib2to3/fixes/fix_reduce.py b/Lib/lib2to3/fixes/fix_reduce.py index 194866d6a0..6bd785c1cd 100644 --- a/Lib/lib2to3/fixes/fix_reduce.py +++ b/Lib/lib2to3/fixes/fix_reduce.py @@ -14,6 +14,9 @@ from lib2to3.fixer_util import touch_import class FixReduce(fixer_base.BaseFix): + BM_compatible = True + order = "pre" + PATTERN = """ power< 'reduce' trailer< '(' diff --git a/Lib/lib2to3/fixes/fix_renames.py b/Lib/lib2to3/fixes/fix_renames.py index a45345a981..4bcce8c4eb 100644 --- a/Lib/lib2to3/fixes/fix_renames.py +++ b/Lib/lib2to3/fixes/fix_renames.py @@ -40,6 +40,7 @@ def build_pattern(): class FixRenames(fixer_base.BaseFix): + BM_compatible = True PATTERN = "|".join(build_pattern()) order = "pre" # Pre-order tree traversal diff --git a/Lib/lib2to3/fixes/fix_repr.py b/Lib/lib2to3/fixes/fix_repr.py index 4b37e73259..f34365647f 100644 --- a/Lib/lib2to3/fixes/fix_repr.py +++ b/Lib/lib2to3/fixes/fix_repr.py @@ -10,6 +10,7 @@ from ..fixer_util import Call, Name, parenthesize class FixRepr(fixer_base.BaseFix): + BM_compatible = True PATTERN = """ atom < '`' expr=any '`' > """ diff --git a/Lib/lib2to3/fixes/fix_set_literal.py b/Lib/lib2to3/fixes/fix_set_literal.py index cfc522cb46..d3d38ec4e0 100644 --- a/Lib/lib2to3/fixes/fix_set_literal.py +++ b/Lib/lib2to3/fixes/fix_set_literal.py @@ -11,6 +11,7 @@ from lib2to3.fixer_util import token, syms class FixSetLiteral(fixer_base.BaseFix): + BM_compatible = True explicit = True PATTERN = """power< 'set' trailer< '(' diff --git a/Lib/lib2to3/fixes/fix_standarderror.py b/Lib/lib2to3/fixes/fix_standarderror.py index a5789ba55b..6cad51116d 100644 --- a/Lib/lib2to3/fixes/fix_standarderror.py +++ b/Lib/lib2to3/fixes/fix_standarderror.py @@ -9,7 +9,7 @@ from ..fixer_util import Name class FixStandarderror(fixer_base.BaseFix): - + BM_compatible = True PATTERN = """ 'StandardError' """ diff --git a/Lib/lib2to3/fixes/fix_sys_exc.py b/Lib/lib2to3/fixes/fix_sys_exc.py index 348929bb6e..2ecca2b535 100644 --- a/Lib/lib2to3/fixes/fix_sys_exc.py +++ b/Lib/lib2to3/fixes/fix_sys_exc.py @@ -14,6 +14,7 @@ from ..fixer_util import Attr, Call, Name, Number, Subscript, Node, syms class FixSysExc(fixer_base.BaseFix): # This order matches the ordering of sys.exc_info(). exc_info = [u"exc_type", u"exc_value", u"exc_traceback"] + BM_compatible = True PATTERN = """ power< 'sys' trailer< dot='.' attribute=(%s) > > """ % '|'.join("'%s'" % e for e in exc_info) diff --git a/Lib/lib2to3/fixes/fix_throw.py b/Lib/lib2to3/fixes/fix_throw.py index 63271cf1d3..1468d89a45 100644 --- a/Lib/lib2to3/fixes/fix_throw.py +++ b/Lib/lib2to3/fixes/fix_throw.py @@ -14,7 +14,7 @@ from .. import fixer_base from ..fixer_util import Name, Call, ArgList, Attr, is_tuple class FixThrow(fixer_base.BaseFix): - + BM_compatible = True PATTERN = """ power< any trailer< '.' 'throw' > trailer< '(' args=arglist< exc=any ',' val=any [',' tb=any] > ')' > diff --git a/Lib/lib2to3/fixes/fix_tuple_params.py b/Lib/lib2to3/fixes/fix_tuple_params.py index 647eec0cfc..6361717dc5 100644 --- a/Lib/lib2to3/fixes/fix_tuple_params.py +++ b/Lib/lib2to3/fixes/fix_tuple_params.py @@ -29,6 +29,10 @@ def is_docstring(stmt): stmt.children[0].type == token.STRING class FixTupleParams(fixer_base.BaseFix): + run_order = 4 #use a lower order since lambda is part of other + #patterns + BM_compatible = True + PATTERN = """ funcdef< 'def' any parameters< '(' args=any ')' > ['->' any] ':' suite=any+ > diff --git a/Lib/lib2to3/fixes/fix_types.py b/Lib/lib2to3/fixes/fix_types.py index 2fbbab8792..fc9d495927 100644 --- a/Lib/lib2to3/fixes/fix_types.py +++ b/Lib/lib2to3/fixes/fix_types.py @@ -52,7 +52,7 @@ _TYPE_MAPPING = { _pats = ["power< 'types' trailer< '.' name='%s' > >" % t for t in _TYPE_MAPPING] class FixTypes(fixer_base.BaseFix): - + BM_compatible = True PATTERN = '|'.join(_pats) def transform(self, node, results): diff --git a/Lib/lib2to3/fixes/fix_unicode.py b/Lib/lib2to3/fixes/fix_unicode.py index cd3b89cb17..6c89576540 100644 --- a/Lib/lib2to3/fixes/fix_unicode.py +++ b/Lib/lib2to3/fixes/fix_unicode.py @@ -10,7 +10,7 @@ _mapping = {u"unichr" : u"chr", u"unicode" : u"str"} _literal_re = re.compile(ur"[uU][rR]?[\'\"]") class FixUnicode(fixer_base.BaseFix): - + BM_compatible = True PATTERN = "STRING | 'unicode' | 'unichr'" def transform(self, node, results): diff --git a/Lib/lib2to3/fixes/fix_urllib.py b/Lib/lib2to3/fixes/fix_urllib.py index db18ca84ee..34e1b2702b 100644 --- a/Lib/lib2to3/fixes/fix_urllib.py +++ b/Lib/lib2to3/fixes/fix_urllib.py @@ -5,39 +5,40 @@ # Author: Nick Edds # Local imports -from .fix_imports import alternates, FixImports -from .. import fixer_base -from ..fixer_util import Name, Comma, FromImport, Newline, attr_chain - -MAPPING = {'urllib': [ - ('urllib.request', - ['URLOpener', 'FancyURLOpener', 'urlretrieve', - '_urlopener', 'urlopen', 'urlcleanup', - 'pathname2url', 'url2pathname']), - ('urllib.parse', - ['quote', 'quote_plus', 'unquote', 'unquote_plus', - 'urlencode', 'splitattr', 'splithost', 'splitnport', - 'splitpasswd', 'splitport', 'splitquery', 'splittag', - 'splittype', 'splituser', 'splitvalue', ]), - ('urllib.error', - ['ContentTooShortError'])], - 'urllib2' : [ - ('urllib.request', - ['urlopen', 'install_opener', 'build_opener', - 'Request', 'OpenerDirector', 'BaseHandler', - 'HTTPDefaultErrorHandler', 'HTTPRedirectHandler', - 'HTTPCookieProcessor', 'ProxyHandler', - 'HTTPPasswordMgr', - 'HTTPPasswordMgrWithDefaultRealm', - 'AbstractBasicAuthHandler', - 'HTTPBasicAuthHandler', 'ProxyBasicAuthHandler', - 'AbstractDigestAuthHandler', - 'HTTPDigestAuthHandler', 'ProxyDigestAuthHandler', - 'HTTPHandler', 'HTTPSHandler', 'FileHandler', - 'FTPHandler', 'CacheFTPHandler', - 'UnknownHandler']), - ('urllib.error', - ['URLError', 'HTTPError']), +from lib2to3.fixes.fix_imports import alternates, FixImports +from lib2to3 import fixer_base +from lib2to3.fixer_util import (Name, Comma, FromImport, Newline, + find_indentation, Node, syms) + +MAPPING = {"urllib": [ + ("urllib.request", + ["URLopener", "FancyURLopener", "urlretrieve", + "_urlopener", "urlopen", "urlcleanup", + "pathname2url", "url2pathname"]), + ("urllib.parse", + ["quote", "quote_plus", "unquote", "unquote_plus", + "urlencode", "splitattr", "splithost", "splitnport", + "splitpasswd", "splitport", "splitquery", "splittag", + "splittype", "splituser", "splitvalue", ]), + ("urllib.error", + ["ContentTooShortError"])], + "urllib2" : [ + ("urllib.request", + ["urlopen", "install_opener", "build_opener", + "Request", "OpenerDirector", "BaseHandler", + "HTTPDefaultErrorHandler", "HTTPRedirectHandler", + "HTTPCookieProcessor", "ProxyHandler", + "HTTPPasswordMgr", + "HTTPPasswordMgrWithDefaultRealm", + "AbstractBasicAuthHandler", + "HTTPBasicAuthHandler", "ProxyBasicAuthHandler", + "AbstractDigestAuthHandler", + "HTTPDigestAuthHandler", "ProxyDigestAuthHandler", + "HTTPHandler", "HTTPSHandler", "FileHandler", + "FTPHandler", "CacheFTPHandler", + "UnknownHandler"]), + ("urllib.error", + ["URLError", "HTTPError"]), ] } @@ -78,7 +79,7 @@ class FixUrllib(FixImports): import name with a comma separated list of its replacements. """ - import_mod = results.get('module') + import_mod = results.get("module") pref = import_mod.prefix names = [] @@ -94,9 +95,9 @@ class FixUrllib(FixImports): the module to be imported from with the appropriate new module. """ - mod_member = results.get('mod_member') + mod_member = results.get("mod_member") pref = mod_member.prefix - member = results.get('member') + member = results.get("member") # Simple case with only a single member being imported if member: @@ -111,35 +112,51 @@ class FixUrllib(FixImports): if new_name: mod_member.replace(Name(new_name, prefix=pref)) else: - self.cannot_convert(node, - 'This is an invalid module element') + self.cannot_convert(node, "This is an invalid module element") # Multiple members being imported else: # a dictionary for replacements, order matters modules = [] mod_dict = {} - members = results.get('members') + members = results["members"] for member in members: - member = member.value # we only care about the actual members - if member != ',': + if member.type == syms.import_as_name: + as_name = member.children[2].value + member_name = member.children[0].value + else: + member_name = member.value + as_name = None + if member_name != u",": for change in MAPPING[mod_member.value]: - if member in change[1]: - if change[0] in mod_dict: - mod_dict[change[0]].append(member) - else: - mod_dict[change[0]] = [member] + if member_name in change[1]: + if change[0] not in mod_dict: modules.append(change[0]) + mod_dict.setdefault(change[0], []).append(member) new_nodes = [] + indentation = find_indentation(node) + first = True + def handle_name(name, prefix): + if name.type == syms.import_as_name: + kids = [Name(name.children[0].value, prefix=prefix), + name.children[1].clone(), + name.children[2].clone()] + return [Node(syms.import_as_name, kids)] + return [Name(name.value, prefix=prefix)] for module in modules: elts = mod_dict[module] names = [] for elt in elts[:-1]: - names.extend([Name(elt, prefix=pref), Comma()]) - names.append(Name(elts[-1], prefix=pref)) - new_nodes.append(FromImport(module, names)) + names.extend(handle_name(elt, pref)) + names.append(Comma()) + names.extend(handle_name(elts[-1], pref)) + new = FromImport(module, names) + if not first or node.parent.prefix.endswith(indentation): + new.prefix = indentation + new_nodes.append(new) + first = False if new_nodes: nodes = [] for new_node in new_nodes[:-1]: @@ -147,12 +164,12 @@ class FixUrllib(FixImports): nodes.append(new_nodes[-1]) node.replace(nodes) else: - self.cannot_convert(node, 'All module elements are invalid') + self.cannot_convert(node, "All module elements are invalid") def transform_dot(self, node, results): """Transform for calls to module members in code.""" - module_dot = results.get('bare_with_attr') - member = results.get('member') + module_dot = results.get("bare_with_attr") + member = results.get("member") new_name = None if isinstance(member, list): member = member[0] @@ -164,17 +181,17 @@ class FixUrllib(FixImports): module_dot.replace(Name(new_name, prefix=module_dot.prefix)) else: - self.cannot_convert(node, 'This is an invalid module element') + self.cannot_convert(node, "This is an invalid module element") def transform(self, node, results): - if results.get('module'): + if results.get("module"): self.transform_import(node, results) - elif results.get('mod_member'): + elif results.get("mod_member"): self.transform_member(node, results) - elif results.get('bare_with_attr'): + elif results.get("bare_with_attr"): self.transform_dot(node, results) # Renaming and star imports are not supported for these modules. - elif results.get('module_star'): - self.cannot_convert(node, 'Cannot handle star imports.') - elif results.get('module_as'): - self.cannot_convert(node, 'This module is now multiple modules') + elif results.get("module_star"): + self.cannot_convert(node, "Cannot handle star imports.") + elif results.get("module_as"): + self.cannot_convert(node, "This module is now multiple modules") diff --git a/Lib/lib2to3/fixes/fix_xrange.py b/Lib/lib2to3/fixes/fix_xrange.py index 2eb106504e..f1436724b4 100644 --- a/Lib/lib2to3/fixes/fix_xrange.py +++ b/Lib/lib2to3/fixes/fix_xrange.py @@ -10,7 +10,7 @@ from .. import patcomp class FixXrange(fixer_base.BaseFix): - + BM_compatible = True PATTERN = """ power< (name='range'|name='xrange') trailer< '(' args=any ')' > diff --git a/Lib/lib2to3/fixes/fix_xreadlines.py b/Lib/lib2to3/fixes/fix_xreadlines.py index ee4b356d3d..f50b9a2755 100644 --- a/Lib/lib2to3/fixes/fix_xreadlines.py +++ b/Lib/lib2to3/fixes/fix_xreadlines.py @@ -9,6 +9,7 @@ from ..fixer_util import Name class FixXreadlines(fixer_base.BaseFix): + BM_compatible = True PATTERN = """ power< call=any+ trailer< '.' 'xreadlines' > trailer< '(' ')' > > | diff --git a/Lib/lib2to3/fixes/fix_zip.py b/Lib/lib2to3/fixes/fix_zip.py index 6736a7edc2..c5d7b66d67 100644 --- a/Lib/lib2to3/fixes/fix_zip.py +++ b/Lib/lib2to3/fixes/fix_zip.py @@ -13,6 +13,7 @@ from ..fixer_util import Name, Call, in_special_context class FixZip(fixer_base.ConditionalFix): + BM_compatible = True PATTERN = """ power< 'zip' args=trailer< '(' [any] ')' > > diff --git a/Lib/lib2to3/main.py b/Lib/lib2to3/main.py index ffa42ab76c..3929db70d3 100644 --- a/Lib/lib2to3/main.py +++ b/Lib/lib2to3/main.py @@ -101,7 +101,7 @@ def main(fixer_pkg, args=None): parser.add_option("-j", "--processes", action="store", default=1, type="int", help="Run 2to3 concurrently") parser.add_option("-x", "--nofix", action="append", default=[], - help="Prevent a fixer from being run.") + help="Prevent a transformation from being run") parser.add_option("-l", "--list-fixes", action="store_true", help="List available transformations") parser.add_option("-p", "--print-function", action="store_true", @@ -113,7 +113,7 @@ def main(fixer_pkg, args=None): parser.add_option("-w", "--write", action="store_true", help="Write back modified files") parser.add_option("-n", "--nobackups", action="store_true", default=False, - help="Don't write backups for modified files.") + help="Don't write backups for modified files") # Parse command line arguments refactor_stdin = False diff --git a/Lib/lib2to3/patcomp.py b/Lib/lib2to3/patcomp.py index daa67fa87b..093e5f9f8d 100644 --- a/Lib/lib2to3/patcomp.py +++ b/Lib/lib2to3/patcomp.py @@ -12,6 +12,7 @@ __author__ = "Guido van Rossum <guido@python.org>" # Python imports import os +import StringIO # Fairly local imports from .pgen2 import driver, literals, token, tokenize, parse, grammar @@ -32,7 +33,7 @@ class PatternSyntaxError(Exception): def tokenize_wrapper(input): """Tokenizes a string suppressing significant whitespace.""" skip = set((token.NEWLINE, token.INDENT, token.DEDENT)) - tokens = tokenize.generate_tokens(driver.generate_lines(input).next) + tokens = tokenize.generate_tokens(StringIO.StringIO(input).readline) for quintuple in tokens: type, value, start, end, line_text = quintuple if type not in skip: @@ -52,14 +53,17 @@ class PatternCompiler(object): self.pysyms = pygram.python_symbols self.driver = driver.Driver(self.grammar, convert=pattern_convert) - def compile_pattern(self, input, debug=False): + def compile_pattern(self, input, debug=False, with_tree=False): """Compiles a pattern string to a nested pytree.*Pattern object.""" tokens = tokenize_wrapper(input) try: root = self.driver.parse_tokens(tokens, debug=debug) - except parse.ParseError, e: + except parse.ParseError as e: raise PatternSyntaxError(str(e)) - return self.compile_node(root) + if with_tree: + return self.compile_node(root), root + else: + return self.compile_node(root) def compile_node(self, node): """Compiles a node, recursively. diff --git a/Lib/lib2to3/pgen2/conv.py b/Lib/lib2to3/pgen2/conv.py index 5d788a1d2b..28fbb0b95f 100644 --- a/Lib/lib2to3/pgen2/conv.py +++ b/Lib/lib2to3/pgen2/conv.py @@ -51,7 +51,7 @@ class Converter(grammar.Grammar): self.finish_off() def parse_graminit_h(self, filename): - """Parse the .h file writen by pgen. (Internal) + """Parse the .h file written by pgen. (Internal) This file is a sequence of #define statements defining the nonterminals of the grammar as numbers. We build two tables @@ -82,7 +82,7 @@ class Converter(grammar.Grammar): return True def parse_graminit_c(self, filename): - """Parse the .c file writen by pgen. (Internal) + """Parse the .c file written by pgen. (Internal) The file looks as follows. The first two lines are always this: diff --git a/Lib/lib2to3/pgen2/driver.py b/Lib/lib2to3/pgen2/driver.py index 6b3825e055..16adec057f 100644 --- a/Lib/lib2to3/pgen2/driver.py +++ b/Lib/lib2to3/pgen2/driver.py @@ -19,6 +19,7 @@ __all__ = ["Driver", "load_grammar"] import codecs import os import logging +import StringIO import sys # Pgen imports @@ -101,18 +102,10 @@ class Driver(object): def parse_string(self, text, debug=False): """Parse a string and return the syntax tree.""" - tokens = tokenize.generate_tokens(generate_lines(text).next) + tokens = tokenize.generate_tokens(StringIO.StringIO(text).readline) return self.parse_tokens(tokens, debug) -def generate_lines(text): - """Generator that behaves like readline without using StringIO.""" - for line in text.splitlines(True): - yield line - while True: - yield "" - - def load_grammar(gt="Grammar.txt", gp=None, save=True, force=False, logger=None): """Load the grammar (maybe from a pickle).""" diff --git a/Lib/lib2to3/pygram.py b/Lib/lib2to3/pygram.py index 6cdb3a44e8..621ff24c95 100644 --- a/Lib/lib2to3/pygram.py +++ b/Lib/lib2to3/pygram.py @@ -13,6 +13,8 @@ from . import pytree # The grammar file _GRAMMAR_FILE = os.path.join(os.path.dirname(__file__), "Grammar.txt") +_PATTERN_GRAMMAR_FILE = os.path.join(os.path.dirname(__file__), + "PatternGrammar.txt") class Symbols(object): @@ -33,3 +35,6 @@ python_symbols = Symbols(python_grammar) python_grammar_no_print_statement = python_grammar.copy() del python_grammar_no_print_statement.keywords["print"] + +pattern_grammar = driver.load_grammar(_PATTERN_GRAMMAR_FILE) +pattern_symbols = Symbols(pattern_grammar) diff --git a/Lib/lib2to3/pytree.py b/Lib/lib2to3/pytree.py index 6245c8c520..179caca51f 100644 --- a/Lib/lib2to3/pytree.py +++ b/Lib/lib2to3/pytree.py @@ -16,7 +16,6 @@ import sys import warnings from StringIO import StringIO - HUGE = 0x7FFFFFFF # maximum repeat count, default max _type_reprs = {} @@ -30,7 +29,6 @@ def type_repr(type_num): if type(val) == int: _type_reprs[val] = name return _type_reprs.setdefault(type_num, type_num) - class Base(object): """ @@ -47,6 +45,7 @@ class Base(object): parent = None # Parent node pointer, or None children = () # Tuple of subnodes was_changed = False + was_checked = False def __new__(cls, *args, **kwds): """Constructor that prevents Base from being instantiated.""" @@ -213,6 +212,16 @@ class Base(object): return None return self.parent.children[i-1] + def leaves(self): + for child in self.children: + for x in child.leaves(): + yield x + + def depth(self): + if self.parent is None: + return 0 + return 1 + self.parent.depth() + def get_suffix(self): """ Return the string immediately following the invocant node. This is @@ -227,12 +236,14 @@ class Base(object): def __str__(self): return unicode(self).encode("ascii") - class Node(Base): """Concrete implementation for interior nodes.""" - def __init__(self, type, children, context=None, prefix=None): + def __init__(self,type, children, + context=None, + prefix=None, + fixers_applied=None): """ Initializer. @@ -249,6 +260,10 @@ class Node(Base): ch.parent = self if prefix is not None: self.prefix = prefix + if fixers_applied: + self.fixers_applied = fixers_applied[:] + else: + self.fixers_applied = None def __repr__(self): """Return a canonical string representation.""" @@ -273,7 +288,8 @@ class Node(Base): def clone(self): """Return a cloned (deep) copy of self.""" - return Node(self.type, [ch.clone() for ch in self.children]) + return Node(self.type, [ch.clone() for ch in self.children], + fixers_applied=self.fixers_applied) def post_order(self): """Return a post-order iterator for the tree.""" @@ -286,7 +302,7 @@ class Node(Base): """Return a pre-order iterator for the tree.""" yield self for child in self.children: - for node in child.post_order(): + for node in child.pre_order(): yield node def _prefix_getter(self): @@ -341,7 +357,10 @@ class Leaf(Base): lineno = 0 # Line where this token starts in the input column = 0 # Column where this token tarts in the input - def __init__(self, type, value, context=None, prefix=None): + def __init__(self, type, value, + context=None, + prefix=None, + fixers_applied=[]): """ Initializer. @@ -355,6 +374,7 @@ class Leaf(Base): self.value = value if prefix is not None: self._prefix = prefix + self.fixers_applied = fixers_applied[:] def __repr__(self): """Return a canonical string representation.""" @@ -380,7 +400,11 @@ class Leaf(Base): def clone(self): """Return a cloned (deep) copy of self.""" return Leaf(self.type, self.value, - (self.prefix, (self.lineno, self.column))) + (self.prefix, (self.lineno, self.column)), + fixers_applied=self.fixers_applied) + + def leaves(self): + yield self def post_order(self): """Return a post-order iterator for the tree.""" @@ -634,8 +658,8 @@ class WildcardPattern(BasePattern): content: optional sequence of subsequences of patterns; if absent, matches one node; if present, each subsequence is an alternative [*] - min: optinal minumum number of times to match, default 0 - max: optional maximum number of times tro match, default HUGE + min: optional minimum number of times to match, default 0 + max: optional maximum number of times to match, default HUGE name: optional name assigned to this match [*] Thus, if content is [[a, b, c], [d, e], [f, g, h]] this is @@ -719,9 +743,11 @@ class WildcardPattern(BasePattern): else: # The reason for this is that hitting the recursion limit usually # results in some ugly messages about how RuntimeErrors are being - # ignored. - save_stderr = sys.stderr - sys.stderr = StringIO() + # ignored. We don't do this on non-CPython implementation because + # they don't have this problem. + if hasattr(sys, "getrefcount"): + save_stderr = sys.stderr + sys.stderr = StringIO() try: for count, r in self._recursive_matches(nodes, 0): if self.name: @@ -735,7 +761,8 @@ class WildcardPattern(BasePattern): r[self.name] = nodes[:count] yield count, r finally: - sys.stderr = save_stderr + if hasattr(sys, "getrefcount"): + sys.stderr = save_stderr def _iterative_matches(self, nodes): """Helper to iteratively yield the matches.""" diff --git a/Lib/lib2to3/refactor.py b/Lib/lib2to3/refactor.py index 3e94699f3d..7d00d12f9d 100644 --- a/Lib/lib2to3/refactor.py +++ b/Lib/lib2to3/refactor.py @@ -24,7 +24,10 @@ from itertools import chain # Local imports from .pgen2 import driver, tokenize, token +from .fixer_util import find_root from . import pytree, pygram +from . import btm_utils as bu +from . import btm_matcher as bm def get_all_fix_names(fixer_pkg, remove_prefix=True): @@ -201,11 +204,28 @@ class RefactoringTool(object): logger=self.logger) self.pre_order, self.post_order = self.get_fixers() - self.pre_order_heads = _get_headnode_dict(self.pre_order) - self.post_order_heads = _get_headnode_dict(self.post_order) self.files = [] # List of files that were or should be modified + self.BM = bm.BottomMatcher() + self.bmi_pre_order = [] # Bottom Matcher incompatible fixers + self.bmi_post_order = [] + + for fixer in chain(self.post_order, self.pre_order): + if fixer.BM_compatible: + self.BM.add_fixer(fixer) + # remove fixers that will be handled by the bottom-up + # matcher + elif fixer in self.pre_order: + self.bmi_pre_order.append(fixer) + elif fixer in self.post_order: + self.bmi_post_order.append(fixer) + + self.bmi_pre_order_heads = _get_headnode_dict(self.bmi_pre_order) + self.bmi_post_order_heads = _get_headnode_dict(self.bmi_post_order) + + + def get_fixers(self): """Inspects the options to load the requested patterns and handlers. @@ -268,6 +288,7 @@ class RefactoringTool(object): def refactor(self, items, write=False, doctests_only=False): """Refactor a list of files and directories.""" + for dir_or_file in items: if os.path.isdir(dir_or_file): self.refactor_dir(dir_or_file, write, doctests_only) @@ -281,13 +302,14 @@ class RefactoringTool(object): Files and subdirectories starting with '.' are skipped. """ + py_ext = os.extsep + "py" for dirpath, dirnames, filenames in os.walk(dir_name): self.log_debug("Descending into %s", dirpath) dirnames.sort() filenames.sort() for name in filenames: - if not name.startswith(".") and \ - os.path.splitext(name)[1].endswith("py"): + if (not name.startswith(".") and + os.path.splitext(name)[1] == py_ext): fullname = os.path.join(dirpath, name) self.refactor_file(fullname, write, doctests_only) # Modify dirnames in-place to remove subdirs with leading dots @@ -299,7 +321,7 @@ class RefactoringTool(object): """ try: f = open(filename, "rb") - except IOError, err: + except IOError as err: self.log_error("Can't open %s: %s", filename, err) return None, None try: @@ -348,7 +370,7 @@ class RefactoringTool(object): self.driver.grammar = pygram.python_grammar_no_print_statement try: tree = self.driver.parse_string(data) - except Exception, err: + except Exception as err: self.log_error("Can't parse %s: %s: %s", name, err.__class__.__name__, err) return @@ -378,6 +400,10 @@ class RefactoringTool(object): def refactor_tree(self, tree, name): """Refactors a parse tree (modifying the tree in place). + For compatible patterns the bottom matcher module is + used. Otherwise the tree is traversed node-to-node for + matches. + Args: tree: a pytree.Node instance representing the root of the tree to be refactored. @@ -386,11 +412,65 @@ class RefactoringTool(object): Returns: True if the tree was modified, False otherwise. """ + for fixer in chain(self.pre_order, self.post_order): fixer.start_tree(tree, name) - self.traverse_by(self.pre_order_heads, tree.pre_order()) - self.traverse_by(self.post_order_heads, tree.post_order()) + #use traditional matching for the incompatible fixers + self.traverse_by(self.bmi_pre_order_heads, tree.pre_order()) + self.traverse_by(self.bmi_post_order_heads, tree.post_order()) + + # obtain a set of candidate nodes + match_set = self.BM.run(tree.leaves()) + + while any(match_set.values()): + for fixer in self.BM.fixers: + if fixer in match_set and match_set[fixer]: + #sort by depth; apply fixers from bottom(of the AST) to top + match_set[fixer].sort(key=pytree.Base.depth, reverse=True) + + if fixer.keep_line_order: + #some fixers(eg fix_imports) must be applied + #with the original file's line order + match_set[fixer].sort(key=pytree.Base.get_lineno) + + for node in list(match_set[fixer]): + if node in match_set[fixer]: + match_set[fixer].remove(node) + + try: + find_root(node) + except AssertionError: + # this node has been cut off from a + # previous transformation ; skip + continue + + if node.fixers_applied and fixer in node.fixers_applied: + # do not apply the same fixer again + continue + + results = fixer.match(node) + + if results: + new = fixer.transform(node, results) + if new is not None: + node.replace(new) + #new.fixers_applied.append(fixer) + for node in new.post_order(): + # do not apply the fixer again to + # this or any subnode + if not node.fixers_applied: + node.fixers_applied = [] + node.fixers_applied.append(fixer) + + # update the original match set for + # the added code + new_matches = self.BM.run(new.leaves()) + for fxr in new_matches: + if not fxr in match_set: + match_set[fxr]=[] + + match_set[fxr].extend(new_matches[fxr]) for fixer in chain(self.pre_order, self.post_order): fixer.finish_tree(tree, name) @@ -448,12 +528,12 @@ class RefactoringTool(object): """ try: f = _open_with_encoding(filename, "w", encoding=encoding) - except os.error, err: + except os.error as err: self.log_error("Can't create %s: %s", filename, err) return try: f.write(_to_system_newlines(new_text)) - except os.error, err: + except os.error as err: self.log_error("Can't write %s: %s", filename, err) finally: f.close() @@ -516,8 +596,8 @@ class RefactoringTool(object): """ try: tree = self.parse_block(block, lineno, indent) - except Exception, err: - if self.log.isEnabledFor(logging.DEBUG): + except Exception as err: + if self.logger.isEnabledFor(logging.DEBUG): for line in block: self.log_debug("Source: %s", line.rstrip(u"\n")) self.log_error("Can't parse docstring in %s line %s: %s: %s", diff --git a/Lib/lib2to3/tests/data/bom.py b/Lib/lib2to3/tests/data/bom.py index ecb782a30d..9bc3975a42 100644 --- a/Lib/lib2to3/tests/data/bom.py +++ b/Lib/lib2to3/tests/data/bom.py @@ -1,3 +1,2 @@ # coding: utf-8 print "BOM BOOM!" - diff --git a/Lib/lib2to3/tests/data/py2_test_grammar.py b/Lib/lib2to3/tests/data/py2_test_grammar.py index 17f9f4c51c..b5a4137d16 100644 --- a/Lib/lib2to3/tests/data/py2_test_grammar.py +++ b/Lib/lib2to3/tests/data/py2_test_grammar.py @@ -316,7 +316,7 @@ class GrammarTests(unittest.TestCase): ### simple_stmt: small_stmt (';' small_stmt)* [';'] x = 1; pass; del x def foo(): - # verify statments that end with semi-colons + # verify statements that end with semi-colons x = 1; pass; del x; foo() diff --git a/Lib/lib2to3/tests/data/py3_test_grammar.py b/Lib/lib2to3/tests/data/py3_test_grammar.py index 977f0b8b35..c0bf7f27aa 100644 --- a/Lib/lib2to3/tests/data/py3_test_grammar.py +++ b/Lib/lib2to3/tests/data/py3_test_grammar.py @@ -356,7 +356,7 @@ class GrammarTests(unittest.TestCase): ### simple_stmt: small_stmt (';' small_stmt)* [';'] x = 1; pass; del x def foo(): - # verify statments that end with semi-colons + # verify statements that end with semi-colons x = 1; pass; del x; foo() diff --git a/Lib/lib2to3/tests/test_fixers.py b/Lib/lib2to3/tests/test_fixers.py index e8e8e12440..88679f0976 100644 --- a/Lib/lib2to3/tests/test_fixers.py +++ b/Lib/lib2to3/tests/test_fixers.py @@ -868,6 +868,11 @@ class Test_raise(FixerTestCase): raise Exception(5).with_traceback(6) # foo""" self.check(b, a) + def test_None_value(self): + b = """raise Exception(5), None, tb""" + a = """raise Exception(5).with_traceback(tb)""" + self.check(b, a) + def test_tuple_value(self): b = """raise Exception, (5, 6, 7)""" a = """raise Exception(5, 6, 7)""" @@ -1812,12 +1817,42 @@ class Test_urllib(FixerTestCase): b = "from %s import %s as foo_bar" % (old, member) a = "from %s import %s as foo_bar" % (new, member) self.check(b, a) + b = "from %s import %s as blah, %s" % (old, member, member) + a = "from %s import %s as blah, %s" % (new, member, member) + self.check(b, a) def test_star(self): for old in self.modules: s = "from %s import *" % old self.warns_unchanged(s, "Cannot handle star imports") + def test_indented(self): + b = """ +def foo(): + from urllib import urlencode, urlopen +""" + a = """ +def foo(): + from urllib.parse import urlencode + from urllib.request import urlopen +""" + self.check(b, a) + + b = """ +def foo(): + other() + from urllib import urlencode, urlopen +""" + a = """ +def foo(): + other() + from urllib.parse import urlencode + from urllib.request import urlopen +""" + self.check(b, a) + + + def test_import_module_usage(self): for old, changes in self.modules.items(): for new, members in changes: @@ -3588,16 +3623,24 @@ class Test_itertools(FixerTestCase): a = """%s(f, a)""" self.checkall(b, a) - def test_2(self): + def test_qualified(self): b = """itertools.ifilterfalse(a, b)""" a = """itertools.filterfalse(a, b)""" self.check(b, a) - def test_4(self): + b = """itertools.izip_longest(a, b)""" + a = """itertools.zip_longest(a, b)""" + self.check(b, a) + + def test_2(self): b = """ifilterfalse(a, b)""" a = """filterfalse(a, b)""" self.check(b, a) + b = """izip_longest(a, b)""" + a = """zip_longest(a, b)""" + self.check(b, a) + def test_space_1(self): b = """ %s(f, a)""" a = """ %s(f, a)""" @@ -3608,9 +3651,14 @@ class Test_itertools(FixerTestCase): a = """ itertools.filterfalse(a, b)""" self.check(b, a) + b = """ itertools.izip_longest(a, b)""" + a = """ itertools.zip_longest(a, b)""" + self.check(b, a) + def test_run_order(self): self.assert_runs_after('map', 'zip', 'filter') + class Test_itertools_imports(FixerTestCase): fixer = 'itertools_imports' @@ -3623,6 +3671,10 @@ class Test_itertools_imports(FixerTestCase): a = "from itertools import bar, foo" self.check(b, a) + b = "from itertools import chain, imap, izip" + a = "from itertools import chain" + self.check(b, a) + def test_comments(self): b = "#foo\nfrom itertools import imap, izip" a = "#foo\n" @@ -3657,18 +3709,23 @@ class Test_itertools_imports(FixerTestCase): s = "from itertools import bar as bang" self.unchanged(s) - def test_ifilter(self): - b = "from itertools import ifilterfalse" - a = "from itertools import filterfalse" - self.check(b, a) + def test_ifilter_and_zip_longest(self): + for name in "filterfalse", "zip_longest": + b = "from itertools import i%s" % (name,) + a = "from itertools import %s" % (name,) + self.check(b, a) - b = "from itertools import imap, ifilterfalse, foo" - a = "from itertools import filterfalse, foo" - self.check(b, a) + b = "from itertools import imap, i%s, foo" % (name,) + a = "from itertools import %s, foo" % (name,) + self.check(b, a) - b = "from itertools import bar, ifilterfalse, foo" - a = "from itertools import bar, filterfalse, foo" - self.check(b, a) + b = "from itertools import bar, i%s, foo" % (name,) + a = "from itertools import bar, %s, foo" % (name,) + self.check(b, a) + + def test_import_star(self): + s = "from itertools import *" + self.unchanged(s) def test_unchanged(self): @@ -4299,13 +4356,89 @@ class Test_operator(FixerTestCase): a = "operator.contains(x, y)" self.check(b, a) + b = "operator .sequenceIncludes(x, y)" + a = "operator .contains(x, y)" + self.check(b, a) + + b = "operator. sequenceIncludes(x, y)" + a = "operator. contains(x, y)" + self.check(b, a) + + def test_operator_isSequenceType(self): + b = "operator.isSequenceType(x)" + a = "import collections\nisinstance(x, collections.Sequence)" + self.check(b, a) + + def test_operator_isMappingType(self): + b = "operator.isMappingType(x)" + a = "import collections\nisinstance(x, collections.Mapping)" + self.check(b, a) + + def test_operator_isNumberType(self): + b = "operator.isNumberType(x)" + a = "import numbers\nisinstance(x, numbers.Number)" + self.check(b, a) + + def test_operator_repeat(self): + b = "operator.repeat(x, n)" + a = "operator.mul(x, n)" + self.check(b, a) + + b = "operator .repeat(x, n)" + a = "operator .mul(x, n)" + self.check(b, a) + + b = "operator. repeat(x, n)" + a = "operator. mul(x, n)" + self.check(b, a) + + def test_operator_irepeat(self): + b = "operator.irepeat(x, n)" + a = "operator.imul(x, n)" + self.check(b, a) + + b = "operator .irepeat(x, n)" + a = "operator .imul(x, n)" + self.check(b, a) + + b = "operator. irepeat(x, n)" + a = "operator. imul(x, n)" + self.check(b, a) + def test_bare_isCallable(self): s = "isCallable(x)" - self.warns_unchanged(s, "You should use hasattr(x, '__call__') here.") + t = "You should use 'hasattr(x, '__call__')' here." + self.warns_unchanged(s, t) def test_bare_sequenceIncludes(self): s = "sequenceIncludes(x, y)" - self.warns_unchanged(s, "You should use operator.contains here.") + t = "You should use 'operator.contains(x, y)' here." + self.warns_unchanged(s, t) + + def test_bare_operator_isSequenceType(self): + s = "isSequenceType(z)" + t = "You should use 'isinstance(z, collections.Sequence)' here." + self.warns_unchanged(s, t) + + def test_bare_operator_isMappingType(self): + s = "isMappingType(x)" + t = "You should use 'isinstance(x, collections.Mapping)' here." + self.warns_unchanged(s, t) + + def test_bare_operator_isNumberType(self): + s = "isNumberType(y)" + t = "You should use 'isinstance(y, numbers.Number)' here." + self.warns_unchanged(s, t) + + def test_bare_operator_repeat(self): + s = "repeat(x, n)" + t = "You should use 'operator.mul(x, n)' here." + self.warns_unchanged(s, t) + + def test_bare_operator_irepeat(self): + s = "irepeat(y, 187)" + t = "You should use 'operator.imul(y, 187)' here." + self.warns_unchanged(s, t) class Test_exitfunc(FixerTestCase): diff --git a/Lib/lib2to3/tests/test_parser.py b/Lib/lib2to3/tests/test_parser.py index 703d879af0..2602381351 100644 --- a/Lib/lib2to3/tests/test_parser.py +++ b/Lib/lib2to3/tests/test_parser.py @@ -19,6 +19,16 @@ import sys # Local imports from lib2to3.pgen2 import tokenize from ..pgen2.parse import ParseError +from lib2to3.pygram import python_symbols as syms + + +class TestDriver(support.TestCase): + + def test_formfeed(self): + s = """print 1\n\x0Cprint 2\n""" + t = driver.parse_string(s) + self.assertEqual(t.children[0].children[0].type, syms.print_stmt) + self.assertEqual(t.children[1].children[0].type, syms.print_stmt) class GrammarTest(support.TestCase): diff --git a/Lib/lib2to3/tests/test_pytree.py b/Lib/lib2to3/tests/test_pytree.py index d31f67debd..ac7d9006aa 100644 --- a/Lib/lib2to3/tests/test_pytree.py +++ b/Lib/lib2to3/tests/test_pytree.py @@ -178,17 +178,42 @@ class TestNodes(support.TestCase): self.assertEqual(str(n1), "foo**bar") self.assertTrue(isinstance(n1.children, list)) + def test_leaves(self): + l1 = pytree.Leaf(100, "foo") + l2 = pytree.Leaf(100, "bar") + l3 = pytree.Leaf(100, "fooey") + n2 = pytree.Node(1000, [l1, l2]) + n3 = pytree.Node(1000, [l3]) + n1 = pytree.Node(1000, [n2, n3]) + + self.assertEqual(list(n1.leaves()), [l1, l2, l3]) + + def test_depth(self): + l1 = pytree.Leaf(100, "foo") + l2 = pytree.Leaf(100, "bar") + n2 = pytree.Node(1000, [l1, l2]) + n3 = pytree.Node(1000, []) + n1 = pytree.Node(1000, [n2, n3]) + + self.assertEqual(l1.depth(), 2) + self.assertEqual(n3.depth(), 1) + self.assertEqual(n1.depth(), 0) + def test_post_order(self): l1 = pytree.Leaf(100, "foo") l2 = pytree.Leaf(100, "bar") - n1 = pytree.Node(1000, [l1, l2]) - self.assertEqual(list(n1.post_order()), [l1, l2, n1]) + l3 = pytree.Leaf(100, "fooey") + c1 = pytree.Node(1000, [l1, l2]) + n1 = pytree.Node(1000, [c1, l3]) + self.assertEqual(list(n1.post_order()), [l1, l2, c1, l3, n1]) def test_pre_order(self): l1 = pytree.Leaf(100, "foo") l2 = pytree.Leaf(100, "bar") - n1 = pytree.Node(1000, [l1, l2]) - self.assertEqual(list(n1.pre_order()), [n1, l1, l2]) + l3 = pytree.Leaf(100, "fooey") + c1 = pytree.Node(1000, [l1, l2]) + n1 = pytree.Node(1000, [c1, l3]) + self.assertEqual(list(n1.pre_order()), [n1, c1, l1, l2, l3]) def test_changed(self): l1 = pytree.Leaf(100, "f") diff --git a/Lib/lib2to3/tests/test_refactor.py b/Lib/lib2to3/tests/test_refactor.py index 59e5a74438..3eecde8cfb 100644 --- a/Lib/lib2to3/tests/test_refactor.py +++ b/Lib/lib2to3/tests/test_refactor.py @@ -223,6 +223,7 @@ from __future__ import print_function""" "hi.py", ".dumb", ".after.py", + "notpy.npy", "sappy"] expected = ["hi.py"] check(tree, expected) diff --git a/Lib/lib2to3/tests/test_util.py b/Lib/lib2to3/tests/test_util.py index 6186b4ff74..2fab8b9ad5 100644 --- a/Lib/lib2to3/tests/test_util.py +++ b/Lib/lib2to3/tests/test_util.py @@ -568,10 +568,27 @@ class Test_touch_import(support.TestCase): def test_from_import(self): node = parse('bar()') - fixer_util.touch_import("cgi", "escape", node) - self.assertEqual(str(node), 'from cgi import escape\nbar()\n\n') + fixer_util.touch_import("html", "escape", node) + self.assertEqual(str(node), 'from html import escape\nbar()\n\n') def test_name_import(self): node = parse('bar()') fixer_util.touch_import(None, "cgi", node) self.assertEqual(str(node), 'import cgi\nbar()\n\n') + +class Test_find_indentation(support.TestCase): + + def test_nothing(self): + fi = fixer_util.find_indentation + node = parse("node()") + self.assertEqual(fi(node), u"") + node = parse("") + self.assertEqual(fi(node), u"") + + def test_simple(self): + fi = fixer_util.find_indentation + node = parse("def f():\n x()") + self.assertEqual(fi(node), u"") + self.assertEqual(fi(node.children[0].children[4].children[2]), u" ") + node = parse("def f():\n x()\n y()") + self.assertEqual(fi(node.children[0].children[4].children[4]), u" ") |