From 4ad0921cb4e75f0fab120c845d73990075da152d Mon Sep 17 00:00:00 2001 From: Claudiu Popa Date: Sat, 3 Jan 2015 10:00:00 +0200 Subject: Use the new Module.stream, since Module.file_stream is deprecated. --- utils.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) (limited to 'utils.py') diff --git a/utils.py b/utils.py index fcc7904..6ce4bf0 100644 --- a/utils.py +++ b/utils.py @@ -128,16 +128,19 @@ def category_id(cid): return MSG_TYPES_LONG.get(cid) +def _decoding_readline(stream, module): + return lambda: stream.readline().decode(module.file_encoding, + 'replace') + + def tokenize_module(module): - stream = module.file_stream - stream.seek(0) - readline = stream.readline - if sys.version_info < (3, 0): - if module.file_encoding is not None: - readline = lambda: stream.readline().decode(module.file_encoding, - 'replace') - return list(tokenize.generate_tokens(readline)) - return list(tokenize.tokenize(readline)) + with module.stream() as stream: + readline = stream.readline + if sys.version_info < (3, 0): + if module.file_encoding is not None: + readline = _decoding_readline(stream, module) + return list(tokenize.generate_tokens(readline)) + return list(tokenize.tokenize(readline)) def build_message_def(checker, msgid, msg_tuple): if implements(checker, (IRawChecker, ITokenChecker)): -- cgit v1.2.1