From 5f4ddfceb7243e142ba0211672c7a1f34a50fa0c Mon Sep 17 00:00:00 2001
From: Ned Batchelder <ned@nedbatchelder.com>
Date: Sat, 13 Mar 2010 07:35:56 -0500
Subject: Check that our tokenizer works properly for dos-line-ended files.

---
 test/stress_phystoken.tok     | 52 +++++++++++++++++++++++++++++++++++++++++++
 test/stress_phystoken.txt     | 52 -------------------------------------------
 test/stress_phystoken_dos.tok | 52 +++++++++++++++++++++++++++++++++++++++++++
 test/test_phystokens.py       |  4 +++-
 4 files changed, 107 insertions(+), 53 deletions(-)
 create mode 100644 test/stress_phystoken.tok
 delete mode 100644 test/stress_phystoken.txt
 create mode 100644 test/stress_phystoken_dos.tok

diff --git a/test/stress_phystoken.tok b/test/stress_phystoken.tok
new file mode 100644
index 0000000..8d1b6be
--- /dev/null
+++ b/test/stress_phystoken.tok
@@ -0,0 +1,52 @@
+# Here's some random Python so that test_tokenize_myself will have some
+# stressful stuff to try.  This file is .tok instead of .py so pylint won't
+# complain about it, check_eol won't look at it, etc.
+
+first_back = """\
+hey there!
+"""
+
+other_back = """
+hey \
+there
+"""
+
+lots_of_back = """\
+hey \
+there
+"""
+# This next line is supposed to have trailing whitespace:
+fake_back = """\ 
+ouch
+"""
+
+# Lots of difficulty happens with code like:
+#
+# fake_back = """\
+# ouch
+# """
+#
+# Ugh, the edge cases...
+
+# What about a comment like this\
+"what's this string doing here?"
+
+class C(object):
+        def there():
+                this = 5 + \
+                    7
+                that = \
+                    "a continued line"
+
+cont1 = "one line of text" + \
+    "another line of text"
+
+a_long_string =              \
+    "part 1"                 \
+    "2"                      \
+    "3 is longer"
+
+def hello():
+        print("Hello world!")
+
+hello()
diff --git a/test/stress_phystoken.txt b/test/stress_phystoken.txt
deleted file mode 100644
index a1e4149..0000000
--- a/test/stress_phystoken.txt
+++ /dev/null
@@ -1,52 +0,0 @@
-# Here's some random Python so that test_tokenize_myself will have some
-# stressful stuff to try.  This file is .txt instead of .py so pylint won't
-# complain about it.
-
-first_back = """\
-hey there!
-"""
-
-other_back = """
-hey \
-there
-"""
-
-lots_of_back = """\
-hey \
-there
-"""
-# This next line is supposed to have trailing whitespace:
-fake_back = """\ 
-ouch
-"""
-
-# Lots of difficulty happens with code like:
-#
-# fake_back = """\
-# ouch
-# """
-#
-# Ugh, the edge cases...
-
-# What about a comment like this\
-"what's this string doing here?"
-
-class C(object):
-        def there():
-                this = 5 + \
-                    7
-                that = \
-                    "a continued line"
-
-cont1 = "one line of text" + \
-    "another line of text"
-
-a_long_string =              \
-    "part 1"                 \
-    "2"                      \
-    "3 is longer"
-
-def hello():
-        print("Hello world!")
-
-hello()
diff --git a/test/stress_phystoken_dos.tok b/test/stress_phystoken_dos.tok
new file mode 100644
index 0000000..b08fd70
--- /dev/null
+++ b/test/stress_phystoken_dos.tok
@@ -0,0 +1,52 @@
+# Here's some random Python so that test_tokenize_myself will have some
+# stressful stuff to try.  This file is .tok instead of .py so pylint won't
+# complain about it, check_eol won't look at it, etc.
+
+first_back = """\
+hey there!
+"""
+
+other_back = """
+hey \
+there
+"""
+
+lots_of_back = """\
+hey \
+there
+"""
+# This next line is supposed to have trailing whitespace:
+fake_back = """\ 
+ouch
+"""
+
+# Lots of difficulty happens with code like:
+#
+# fake_back = """\
+# ouch
+# """
+#
+# Ugh, the edge cases...
+
+# What about a comment like this\
+"what's this string doing here?"
+
+class C(object):
+        def there():
+                this = 5 + \
+                    7
+                that = \
+                    "a continued line"
+
+cont1 = "one line of text" + \
+    "another line of text"
+
+a_long_string =              \
+    "part 1"                 \
+    "2"                      \
+    "3 is longer"
+
+def hello():
+        print("Hello world!")
+
+hello()
diff --git a/test/test_phystokens.py b/test/test_phystokens.py
index 6b16d68..6b533cf 100644
--- a/test/test_phystokens.py
+++ b/test/test_phystokens.py
@@ -75,5 +75,7 @@ class PhysTokensTest(CoverageTest):
 
     def test_stress(self):
         # Check the tokenization of a stress-test file.
-        stress = os.path.join(HERE, "stress_phystoken.txt")
+        stress = os.path.join(HERE, "stress_phystoken.tok")
+        self.check_file_tokenization(stress)
+        stress = os.path.join(HERE, "stress_phystoken_dos.tok")
         self.check_file_tokenization(stress)
-- 
cgit v1.2.1