From 428de65ca99492436130165bfbaeb56d6d1daec7 Mon Sep 17 00:00:00 2001 From: Trent Nelson Date: Tue, 18 Mar 2008 22:41:35 +0000 Subject: - Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been renamed tokenize and now works with bytes rather than strings. A new detect_encoding function has been added for determining source file encoding according to PEP-0263. Token sequences returned by tokenize always start with an ENCODING token which specifies the encoding used to decode the file. This token is used to encode the output of untokenize back to bytes. Credit goes to Michael "I'm-going-to-name-my-first-child-unittest" Foord from Resolver Systems for this work. --- Lib/idlelib/EditorWindow.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'Lib/idlelib/EditorWindow.py') diff --git a/Lib/idlelib/EditorWindow.py b/Lib/idlelib/EditorWindow.py index e1d9ba3256..2b5c5270e0 100644 --- a/Lib/idlelib/EditorWindow.py +++ b/Lib/idlelib/EditorWindow.py @@ -1437,7 +1437,9 @@ class IndentSearcher(object): _tokenize.tabsize = self.tabwidth try: try: - _tokenize.tokenize(self.readline, self.tokeneater) + tokens = _tokenize.generate_tokens(self.readline) + for token in tokens: + self.tokeneater(*token) except _tokenize.TokenError: # since we cut off the tokenizer early, we can trigger # spurious errors -- cgit v1.2.1