Merge pyflakes-ast-3005

Author: gbrandl, exarkun
Reviewer: exarkun
Fixes: #3005

Convert pyflakes to use the Python 2.5+ _ast module instead of
the older (now essentially unmaintained) compiler package.

Introduce a number of new tests for various edge cases previously
untested, as well, since this involved changing substantial chunks
of pyflakes internals.

Also add support for certain new constructs which will be added
in Python 2.7, including set comprehensions and dict comprehensions.

Because Python 2.4 does not include the _ast module, this change
effectively drops support for running Pyflakes using Python 2.4.

1 files changed, 6 insertions, 15 deletions

diff --git a/pyflakes/scripts/pyflakes.py b/pyflakes/scripts/pyflakes.py
index da8d2ea..6b1dae2 100644
--- a/pyflakes/scripts/pyflakes.py
+++ b/pyflakes/scripts/pyflakes.py
@@ -3,8 +3,9 @@
 Implementation of the command-line I{pyflakes} tool.
 """
 
-import compiler, sys
+import sys
 import os
+import _ast
 
 checker = __import__('pyflakes.checker').checker
 
@@ -22,18 +23,10 @@ def check(codeString, filename):
     @return: The number of warnings emitted.
     @rtype: C{int}
     """
-    # Since compiler.parse does not reliably report syntax errors, use the
-    # built in compiler first to detect those.
+    # First, compile into an AST and handle syntax errors.
     try:
-        try:
-            compile(codeString, filename, "exec")
-        except MemoryError:
-            # Python 2.4 will raise MemoryError if the source can't be
-            # decoded.
-            if sys.version_info[:2] == (2, 4):
-                raise SyntaxError(None)
-            raise
-    except (SyntaxError, IndentationError), value:
+        tree = compile(codeString, filename, "exec", _ast.PyCF_ONLY_AST)
+    except SyntaxError, value:
         msg = value.args[0]
 
         (lineno, offset, text) = value.lineno, value.offset, value.text
@@ -58,9 +51,7 @@ def check(codeString, filename):
 
         return 1
     else:
-        # Okay, it's syntactically valid.  Now parse it into an ast and check
-        # it.
-        tree = compiler.parse(codeString)
+        # Okay, it's syntactically valid.  Now check it.
         w = checker.Checker(tree, filename)
         w.messages.sort(lambda a, b: cmp(a.lineno, b.lineno))
         for warning in w.messages: