Merge branch 'master' into syntaxerror

Conflicts:
	pyflakes/reporter.py
	pyflakes/test/test_api.py

10 files changed, 83 insertions, 32 deletions

diff --git a/.travis.yml b/.travis.yml
index 95d1b36..2289a8a 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,6 +1,5 @@
 language: python
 python:
-  - 2.5
   - 2.6
   - 2.7
   - 3.2
diff --git a/AUTHORS b/AUTHORS
index f8ff41b..6f8355d 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -2,6 +2,7 @@
 Contributors
 ------------
 
+* Phil Frost - Former Divmod Team
 * Moe Aboulkheir - Former Divmod Team
 * Jean-Paul Calderone - Former Divmod Team
 * Glyph Lefkowitz - Former Divmod Team
diff --git a/LICENSE b/LICENSE
index cea6b2e..e4d553a 100644
--- a/LICENSE
+++ b/LICENSE
@@ -1,5 +1,5 @@
 Copyright 2005-2011 Divmod, Inc.
-Copyright 2013 Florent Xicluna
+Copyright 2013-2014 Florent Xicluna
 
 Permission is hereby granted, free of charge, to any person obtaining
 a copy of this software and associated documentation files (the
diff --git a/NEWS.txt b/NEWS.txt
index dd63dfe..56c251c 100644
--- a/NEWS.txt
+++ b/NEWS.txt
@@ -1,3 +1,9 @@
+UNRELEASED:
+  - Adapt for the AST in Python 3.4.
+  - Fix caret position on SyntaxError.
+  - Fix crash on Python 2.x with some doctest SyntaxError.
+  - Add tox.ini.
+
 0.7.3 (2013-07-02):
   - Do not report undefined name for generator expression and dict or
     set comprehension at class level.
diff --git a/README.rst b/README.rst
index 9f4429f..0c5bd04 100644
--- a/README.rst
+++ b/README.rst
@@ -9,7 +9,7 @@ parsing the source file, not importing it, so it is safe to use on
 modules with side effects.  It's also much faster.
 
 It is `available on PyPI <http://pypi.python.org/pypi/pyflakes>`_
-and it supports all active versions of Python from 2.5 to 3.3.
+and it supports all active versions of Python from 2.5 to 3.4.
 
 
 Installation
diff --git a/pyflakes/checker.py b/pyflakes/checker.py
index d59b8c9..04491f9 100644
--- a/pyflakes/checker.py
+++ b/pyflakes/checker.py
@@ -7,12 +7,14 @@ Also, it models the Bindings and Scopes.
 import doctest
 import os
 import sys
-try:
-    builtin_vars = dir(__import__('builtins'))
-    PY2 = False
-except ImportError:
-    builtin_vars = dir(__import__('__builtin__'))
+
+if sys.version_info < (3, 0):
     PY2 = True
+    builtin_vars = dir(__import__('__builtin__'))
+else:
+    PY2 = False
+    builtin_vars = dir(__import__('builtins'))
+PY33 = sys.version_info < (3, 4)    # Python 2.5 to 3.3
 
 try:
     import ast
@@ -578,7 +580,7 @@ class Checker(object):
             except SyntaxError:
                 e = sys.exc_info()[1]
                 position = (node_lineno + example.lineno + e.lineno,
-                            example.indent + 4 + e.offset)
+                            example.indent + 4 + (e.offset or 0))
                 self.report(messages.DoctestSyntaxError, node, position)
             else:
                 self.offset = (node_offset[0] + node_lineno + example.lineno,
@@ -599,7 +601,7 @@ class Checker(object):
     # "expr" type nodes
     BOOLOP = BINOP = UNARYOP = IFEXP = DICT = SET = YIELD = YIELDFROM = \
         COMPARE = CALL = REPR = ATTRIBUTE = SUBSCRIPT = LIST = TUPLE = \
-        STARRED = handleChildren
+        STARRED = NAMECONSTANT = handleChildren
 
     NUM = STR = BYTES = ELLIPSIS = ignore
 
@@ -705,6 +707,7 @@ class Checker(object):
 
     def LAMBDA(self, node):
         args = []
+        annotations = []
 
         if PY2:
             def addArgs(arglist):
@@ -712,34 +715,41 @@ class Checker(object):
                     if isinstance(arg, ast.Tuple):
                         addArgs(arg.elts)
                     else:
-                        if arg.id in args:
-                            self.report(messages.DuplicateArgument,
-                                        node, arg.id)
                         args.append(arg.id)
             addArgs(node.args.args)
             defaults = node.args.defaults
         else:
             for arg in node.args.args + node.args.kwonlyargs:
-                if arg.arg in args:
-                    self.report(messages.DuplicateArgument,
-                                node, arg.arg)
                 args.append(arg.arg)
-                self.handleNode(arg.annotation, node)
-            if hasattr(node, 'returns'):    # Only for FunctionDefs
-                for annotation in (node.args.varargannotation,
-                                   node.args.kwargannotation, node.returns):
-                    self.handleNode(annotation, node)
+                annotations.append(arg.annotation)
             defaults = node.args.defaults + node.args.kw_defaults
 
-        # vararg/kwarg identifiers are not Name nodes
-        for wildcard in (node.args.vararg, node.args.kwarg):
+        # Only for Python3 FunctionDefs
+        is_py3_func = hasattr(node, 'returns')
+
+        for arg_name in ('vararg', 'kwarg'):
+            wildcard = getattr(node.args, arg_name)
             if not wildcard:
                 continue
-            if wildcard in args:
-                self.report(messages.DuplicateArgument, node, wildcard)
-            args.append(wildcard)
-        for default in defaults:
-            self.handleNode(default, node)
+            args.append(wildcard if PY33 else wildcard.arg)
+            if is_py3_func:
+                if PY33:  # Python 2.5 to 3.3
+                    argannotation = arg_name + 'annotation'
+                    annotations.append(getattr(node.args, argannotation))
+                else:     # Python >= 3.4
+                    annotations.append(wildcard.annotation)
+
+        if is_py3_func:
+            annotations.append(node.returns)
+
+        if len(set(args)) < len(args):
+            for (idx, arg) in enumerate(args):
+                if arg in args[:idx]:
+                    self.report(messages.DuplicateArgument, node, arg)
+
+        for child in annotations + defaults:
+            if child:
+                self.handleNode(child, node)
 
         def runFunction():
 
diff --git a/pyflakes/reporter.py b/pyflakes/reporter.py
index 74f9f2e..a4b6840 100644
--- a/pyflakes/reporter.py
+++ b/pyflakes/reporter.py
@@ -2,6 +2,7 @@
 Provide the Reporter class.
 """
 
+import re
 import sys
 
 
@@ -60,7 +61,8 @@ class Reporter(object):
         self._stderr.write(line)
         self._stderr.write('\n')
         if offset is not None:
-            self._stderr.write(" " * offset + "^\n")
+            self._stderr.write(re.sub(r'\S', ' ', line[:offset]) +
+                               "^\n")
 
     def flake(self, message):
         """
diff --git a/pyflakes/test/test_api.py b/pyflakes/test/test_api.py
index 856121c..2fac879 100644
--- a/pyflakes/test/test_api.py
+++ b/pyflakes/test/test_api.py
@@ -163,7 +163,7 @@ class TestReporter(TestCase):
         self.assertEqual(
             ("foo.py:3:7: a problem\n"
              "bad line of source\n"
-             "       ^\n"),
+             "    ^\n"),
             err.getvalue())
 
     def test_syntaxErrorNoOffset(self):
@@ -197,7 +197,7 @@ class TestReporter(TestCase):
         self.assertEqual(
             ("foo.py:3:6: a problem\n" +
              lines[-1] + "\n" +
-             "      ^\n"),
+             "    ^\n"),
             err.getvalue())
 
     def test_unexpectedError(self):
@@ -339,6 +339,20 @@ def foo(
         ^
 """ % (sourcePath,)])
 
+    def test_eofSyntaxErrorWithTab(self):
+        """
+        The error reported for source files which end prematurely causing a
+        syntax error reflects the cause for the syntax error.
+        """
+        sourcePath = self.makeTempFile("if True:\n\tfoo =")
+        self.assertHasErrors(
+            sourcePath,
+            ["""\
+%s:2: invalid syntax
+\tfoo =
+\t     ^
+""" % (sourcePath,)])
+
     def test_nonDefaultFollowsDefaultSyntaxError(self):
         """
         Source which has a non-default argument following a default argument
diff --git a/pyflakes/test/test_doctests.py b/pyflakes/test/test_doctests.py
index b466375..d7df779 100644
--- a/pyflakes/test/test_doctests.py
+++ b/pyflakes/test/test_doctests.py
@@ -207,3 +207,22 @@ class Test(TestOther, TestImports, TestUndefinedNames):
                     >>> Foo
                 '''
         """)
+
+    def test_noOffsetSyntaxErrorInDoctest(self):
+        exceptions = super(Test, self).flakes(
+            '''
+            def buildurl(base, *args, **kwargs):
+                """
+                >>> buildurl('/blah.php', ('a', '&'), ('b', '=')
+                '/blah.php?a=%26&b=%3D'
+                >>> buildurl('/blah.php', a='&', 'b'='=')
+                '/blah.php?b=%3D&a=%26'
+                """
+                pass
+            ''',
+            m.DoctestSyntaxError,
+            m.DoctestSyntaxError).messages
+        exc = exceptions[0]
+        self.assertEqual(exc.lineno, 4)
+        exc = exceptions[1]
+        self.assertEqual(exc.lineno, 6)
diff --git a/tox.ini b/tox.ini
index 79d8aab..270c952 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,6 +1,6 @@
 [tox]
 envlist =
-    py25,py26,py27,py32,py33
+    py25,py26,py27,py32,py33,py34
 # pypy is not ready yet; run manually with tox -e pypy if you want to see
 # the failures