Add support for tokens of a complete file

The `tokens` property of the `FileProcessor` class only contains tokens of the
current line but not all tokens. So if a plugin which is only executed once
per file, that property is useless. To make the tokens also available to
plugins it is now be able to supply all the tokens of a file.

It also updates the documentation to separate which parameters are static and
which are changed on each line. Using the latter parameters on plugins which
are only run once per file isn't very sensible.

1 files changed, 14 insertions, 0 deletions

diff --git a/src/flake8/processor.py b/src/flake8/processor.py
index dee0b15..79844af 100644
--- a/src/flake8/processor.py
+++ b/src/flake8/processor.py
@@ -43,6 +43,7 @@ class FileProcessor(object):
     - :attr:`previous_indent_level`
     - :attr:`previous_logical`
     - :attr:`tokens`
+    - :attr:`file_tokens`
     - :attr:`total_lines`
     - :attr:`verbose`
     """
@@ -101,6 +102,19 @@ class FileProcessor(object):
         self.statistics = {
             'logical lines': 0,
         }
+        self._file_tokens = None
+
+    @property
+    def file_tokens(self):
+        if self._file_tokens is None:
+            line_iter = iter(self.lines)
+            try:
+                self._file_tokens = list(tokenize.generate_tokens(
+                    lambda: next(line_iter)))
+            except tokenize.TokenError as exc:
+                raise exceptions.InvalidSyntax(exc.message, exception=exc)
+
+        return self._file_tokens[:]
 
     @contextlib.contextmanager
     def inside_multiline(self, line_number):