diff options
Diffstat (limited to 'Lib/tokenize.py')
| -rw-r--r-- | Lib/tokenize.py | 66 |
1 files changed, 6 insertions, 60 deletions
diff --git a/Lib/tokenize.py b/Lib/tokenize.py index fce010bc5e..cf1ecc99a9 100644 --- a/Lib/tokenize.py +++ b/Lib/tokenize.py @@ -32,6 +32,7 @@ import itertools as _itertools import re import sys from token import * +from token import EXACT_TOKEN_TYPES cookie_re = re.compile(r'^[ \t\f]*#.*?coding[:=][ \t]*([-\w.]+)', re.ASCII) blank_re = re.compile(br'^[ \t\f]*(?:[#\r\n]|$)', re.ASCII) @@ -41,55 +42,6 @@ __all__ = token.__all__ + ["tokenize", "generate_tokens", "detect_encoding", "untokenize", "TokenInfo"] del token -EXACT_TOKEN_TYPES = { - '(': LPAR, - ')': RPAR, - '[': LSQB, - ']': RSQB, - ':': COLON, - ',': COMMA, - ';': SEMI, - '+': PLUS, - '-': MINUS, - '*': STAR, - '/': SLASH, - '|': VBAR, - '&': AMPER, - '<': LESS, - '>': GREATER, - '=': EQUAL, - '.': DOT, - '%': PERCENT, - '{': LBRACE, - '}': RBRACE, - '==': EQEQUAL, - '!=': NOTEQUAL, - '<=': LESSEQUAL, - '>=': GREATEREQUAL, - '~': TILDE, - '^': CIRCUMFLEX, - '<<': LEFTSHIFT, - '>>': RIGHTSHIFT, - '**': DOUBLESTAR, - '+=': PLUSEQUAL, - '-=': MINEQUAL, - '*=': STAREQUAL, - '/=': SLASHEQUAL, - '%=': PERCENTEQUAL, - '&=': AMPEREQUAL, - '|=': VBAREQUAL, - '^=': CIRCUMFLEXEQUAL, - '<<=': LEFTSHIFTEQUAL, - '>>=': RIGHTSHIFTEQUAL, - '**=': DOUBLESTAREQUAL, - '//': DOUBLESLASH, - '//=': DOUBLESLASHEQUAL, - '...': ELLIPSIS, - '->': RARROW, - '@': AT, - '@=': ATEQUAL, -} - class TokenInfo(collections.namedtuple('TokenInfo', 'type string start end line')): def __repr__(self): annotated_type = '%d (%s)' % (self.type, tok_name[self.type]) @@ -163,17 +115,11 @@ Triple = group(StringPrefix + "'''", StringPrefix + '"""') String = group(StringPrefix + r"'[^\n'\\]*(?:\\.[^\n'\\]*)*'", StringPrefix + r'"[^\n"\\]*(?:\\.[^\n"\\]*)*"') -# Because of leftmost-then-longest match semantics, be sure to put the -# longest operators first (e.g., if = came before ==, == would get -# recognized as two instances of =). -Operator = group(r"\*\*=?", r">>=?", r"<<=?", r"!=", - r"//=?", r"->", - r"[+\-*/%&@|^=<>]=?", - r"~") - -Bracket = '[][(){}]' -Special = group(r'\r?\n', r'\.\.\.', r'[:;.,@]') -Funny = group(Operator, Bracket, Special) +# Sorting in reverse order puts the long operators before their prefixes. +# Otherwise if = came before ==, == would get recognized as two instances +# of =. +Special = group(*map(re.escape, sorted(EXACT_TOKEN_TYPES, reverse=True))) +Funny = group(r'\r?\n', Special) PlainToken = group(Number, Funny, String, Name) Token = Ignore + PlainToken |