Tokenize the input source only once and hand it to all checkers that need the token stream.

A lot of checkers need access to the token stream, but they all tokenize the
source code again in BaseRawChecker.process_module. This change introduces
a new checker type ITokenChecker, for which the token stream is created
exactly once in PyLinter, and then injected into all registered checkers.

1 files changed, 12 insertions, 0 deletions

diff --git a/checkers/__init__.py b/checkers/__init__.py
index 700a78e..dd868c6 100644
--- a/checkers/__init__.py
+++ b/checkers/__init__.py
@@ -39,6 +39,7 @@ messages nor reports. XXX not true, emit a 07 report !
 """
 
 import tokenize
+import warnings
 from os import listdir
 from os.path import dirname, join, isdir, splitext
 
@@ -121,6 +122,9 @@ class BaseRawChecker(BaseChecker):
 
         stream must implement the readline method
         """
+        warnings.warn("Modules that need access to the tokens should "
+                      "use the ITokenChecker interface.",
+                      DeprecationWarning)
         stream = node.file_stream
         stream.seek(0) # XXX may be removed with astng > 0.23
         self.process_tokens(tokenize.generate_tokens(stream.readline))
@@ -130,6 +134,14 @@ class BaseRawChecker(BaseChecker):
         raise NotImplementedError()
 
 
+class BaseTokenChecker(BaseChecker):
+    """Base class for checkers that want to have access to the token stream."""
+    
+    def process_tokens(self, tokens):
+        """Should be overridden by subclasses."""
+        raise NotImplementedError()
+
+
 PY_EXTS = ('.py', '.pyc', '.pyo', '.pyw', '.so', '.dll')
 
 def initialize(linter):