merge in trunk

2 files changed, 11 insertions, 3 deletions

diff --git a/sphinx/pycode/__init__.py b/sphinx/pycode/__init__.py
index 17dc6afb..c2086da5 100644
--- a/sphinx/pycode/__init__.py
+++ b/sphinx/pycode/__init__.py
@@ -210,7 +210,10 @@ class ModuleAnalyzer(object):
         if self.parsetree is not None:
             return
         self.tokenize()
-        self.parsetree = pydriver.parse_tokens(self.tokens)
+        try:
+            self.parsetree = pydriver.parse_tokens(self.tokens)
+        except parse.ParseError, err:
+            raise PycodeError('parsing failed', err)
         # find the source code encoding
         encoding = sys.getdefaultencoding()
         comments = self.parsetree.get_prefix()
diff --git a/sphinx/pycode/pgen2/tokenize.py b/sphinx/pycode/pgen2/tokenize.py
index 46ee7842..4489db89 100644
--- a/sphinx/pycode/pgen2/tokenize.py
+++ b/sphinx/pycode/pgen2/tokenize.py
@@ -274,12 +274,17 @@ def generate_tokens(readline):
             line = readline()
         except StopIteration:
             line = ''
+        # if we are not at the end of the file make sure the
+        # line ends with a newline because the parser depends
+        # on that.
+        if line:
+            line = line.rstrip() + '\n'
         lnum = lnum + 1
         pos, max = 0, len(line)
 
         if contstr:                            # continued string
             if not line:
-                raise TokenError, ("EOF in multi-line string", strstart)
+                raise TokenError("EOF in multi-line string", strstart)
             endmatch = endprog.match(line)
             if endmatch:
                 pos = end = endmatch.end(0)
@@ -335,7 +340,7 @@ def generate_tokens(readline):
 
         else:                                  # continued statement
             if not line:
-                raise TokenError, ("EOF in multi-line statement", (lnum, 0))
+                raise TokenError("EOF in multi-line statement", (lnum, 0))
             continued = 0
 
         while pos < max: