diff options
| author | mitsuhiko <devnull@localhost> | 2009-01-09 18:55:42 +0100 |
|---|---|---|
| committer | mitsuhiko <devnull@localhost> | 2009-01-09 18:55:42 +0100 |
| commit | 673bcf9c68066f8acb9b8e1ac0458e8b127ba735 (patch) | |
| tree | 297c263d1897c373b347677739c21103dafc0232 /sphinx/pycode | |
| parent | 06c95a8a4afd6542bb73999405b853d2919ca360 (diff) | |
| download | sphinx-673bcf9c68066f8acb9b8e1ac0458e8b127ba735.tar.gz | |
Changed the tokenizer in pgen2 to add a newline to lines even if it's missing. This makes sure that a file that does not end with a newline is properly parsed.
Diffstat (limited to 'sphinx/pycode')
| -rw-r--r-- | sphinx/pycode/pgen2/tokenize.py | 5 |
1 files changed, 5 insertions, 0 deletions
diff --git a/sphinx/pycode/pgen2/tokenize.py b/sphinx/pycode/pgen2/tokenize.py index 46ee7842..84a9ddcf 100644 --- a/sphinx/pycode/pgen2/tokenize.py +++ b/sphinx/pycode/pgen2/tokenize.py @@ -274,6 +274,11 @@ def generate_tokens(readline): line = readline() except StopIteration: line = '' + # if we are not at the end of the file make sure the + # line ends with a newline because the parser depends + # on that. + if line: + line = line.rstrip() + '\n' lnum = lnum + 1 pos, max = 0, len(line) |