From 2cc3b4ba9ffa658784da03f14a0a068e2c61d1b3 Mon Sep 17 00:00:00 2001 From: Ezio Melotti Date: Sat, 3 Nov 2012 17:38:43 +0200 Subject: #16152: fix tokenize to ignore whitespace at the end of the code when no newline is found. Patch by Ned Batchelder. --- Lib/tokenize.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'Lib/tokenize.py') diff --git a/Lib/tokenize.py b/Lib/tokenize.py index 59081d3579..29c9e29b30 100644 --- a/Lib/tokenize.py +++ b/Lib/tokenize.py @@ -108,7 +108,7 @@ ContStr = group(r"[bB]?[rR]?'[^\n'\\]*(?:\\.[^\n'\\]*)*" + group("'", r'\\\r?\n'), r'[bB]?[rR]?"[^\n"\\]*(?:\\.[^\n"\\]*)*' + group('"', r'\\\r?\n')) -PseudoExtras = group(r'\\\r?\n', Comment, Triple) +PseudoExtras = group(r'\\\r?\n|\Z', Comment, Triple) PseudoToken = Whitespace + group(PseudoExtras, Number, Funny, ContStr, Name) def _compile(expr): @@ -473,6 +473,8 @@ def _tokenize(readline, encoding): if pseudomatch: # scan for tokens start, end = pseudomatch.span(1) spos, epos, pos = (lnum, start), (lnum, end), end + if start == end: + continue token, initial = line[start:end], line[start] if (initial in numchars or # ordinary number -- cgit v1.2.1