Merge with 3.3

1 files changed, 6 insertions, 0 deletions

diff --git a/Lib/tokenize.py b/Lib/tokenize.py
index 74bd15c623..e320d39936 100644
--- a/Lib/tokenize.py
+++ b/Lib/tokenize.py
@@ -234,6 +234,10 @@ class Untokenizer:
         if row < self.prev_row or row == self.prev_row and col < self.prev_col:
             raise ValueError("start ({},{}) precedes previous end ({},{})"
                              .format(row, col, self.prev_row, self.prev_col))
+        row_offset = row - self.prev_row
+        if row_offset:  
+            self.tokens.append("\\\n" * row_offset)
+            self.prev_col = 0
         col_offset = col - self.prev_col
         if col_offset:
             self.tokens.append(" " * col_offset)
@@ -248,6 +252,8 @@ class Untokenizer:
             if tok_type == ENCODING:
                 self.encoding = token
                 continue
+            if tok_type == ENDMARKER:
+                break
             self.add_whitespace(start)
             self.tokens.append(token)
             self.prev_row, self.prev_col = end