From 6d809a8ee479e662d5a614de66f27ff8f0b302b3 Mon Sep 17 00:00:00 2001 From: Torsten Marek Date: Wed, 1 May 2013 13:43:13 +0200 Subject: Tokenize the input source only once and hand it to all checkers that need the token stream. A lot of checkers need access to the token stream, but they all tokenize the source code again in BaseRawChecker.process_module. This change introduces a new checker type ITokenChecker, for which the token stream is created exactly once in PyLinter, and then injected into all registered checkers. --- checkers/__init__.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) (limited to 'checkers/__init__.py') diff --git a/checkers/__init__.py b/checkers/__init__.py index 700a78e..dd868c6 100644 --- a/checkers/__init__.py +++ b/checkers/__init__.py @@ -39,6 +39,7 @@ messages nor reports. XXX not true, emit a 07 report ! """ import tokenize +import warnings from os import listdir from os.path import dirname, join, isdir, splitext @@ -121,6 +122,9 @@ class BaseRawChecker(BaseChecker): stream must implement the readline method """ + warnings.warn("Modules that need access to the tokens should " + "use the ITokenChecker interface.", + DeprecationWarning) stream = node.file_stream stream.seek(0) # XXX may be removed with astng > 0.23 self.process_tokens(tokenize.generate_tokens(stream.readline)) @@ -130,6 +134,14 @@ class BaseRawChecker(BaseChecker): raise NotImplementedError() +class BaseTokenChecker(BaseChecker): + """Base class for checkers that want to have access to the token stream.""" + + def process_tokens(self, tokens): + """Should be overridden by subclasses.""" + raise NotImplementedError() + + PY_EXTS = ('.py', '.pyc', '.pyo', '.pyw', '.so', '.dll') def initialize(linter): -- cgit v1.2.1