From d268eb9986f7fef9c5f9276c54e313c5313a50c5 Mon Sep 17 00:00:00 2001
From: Ned Batchelder <ned@nedbatchelder.com>
Date: Sat, 13 Mar 2010 08:00:40 -0500
Subject: Deal properly with dos-ended source files while tokenizing.  Fixes
 issue #53.

---
 coverage/phystokens.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

(limited to 'coverage/phystokens.py')

diff --git a/coverage/phystokens.py b/coverage/phystokens.py
index 5824b9b9..60b87932 100644
--- a/coverage/phystokens.py
+++ b/coverage/phystokens.py
@@ -77,7 +77,8 @@ def source_token_lines(source):
     ws_tokens = [token.INDENT, token.DEDENT, token.NEWLINE, tokenize.NL]
     line = []
     col = 0
-    tokgen = tokenize.generate_tokens(StringIO(source.expandtabs(8)).readline)
+    source = source.expandtabs(8).replace('\r\n', '\n')
+    tokgen = tokenize.generate_tokens(StringIO(source).readline)
     for ttype, ttext, (_, scol), (_, ecol), _ in phys_tokens(tokgen):
         mark_start = True
         for part in re.split('(\n)', ttext):
-- 
cgit v1.2.1