Fix the primitive tokenizer to handle HTML comments that contain tags, otherwise this generates completely broken HTML by turning the end-comment into an en-dash.

2 files changed, 8 insertions, 1 deletions

diff --git a/smartypants.py b/smartypants.py
index aec885b..f8d56f3 100755
--- a/smartypants.py
+++ b/smartypants.py
@@ -709,7 +709,7 @@ def _tokenize(text):
 
     tokens = []
 
-    tag_soup = re.compile('([^<]*)(<[^>]*>)')
+    tag_soup = re.compile(r"""(?s)([^<]*)(<!--.*?--\s*>|<[^>]*>)""")
 
     token_match = tag_soup.search(text)
 
diff --git a/tests/test.py b/tests/test.py
index ffd20ac..d0622de 100644
--- a/tests/test.py
+++ b/tests/test.py
@@ -84,6 +84,13 @@ document.write('<a href="' + href + '">' + linktext + "</a>");
              "is python code.</p>")
         self.assertEqual(T, E)
 
+    def test_comments(self):
+
+        self.assertEqual(sp("--"), "&#8212;")
+        self.assertEqual(sp("-->"), "&#8212;>")
+        self.assertEqual(sp("<!-- comment -->"), "<!-- comment -->")
+        self.assertEqual(sp("<!-- <li>Fee-fi-of-fum</li> -->"), "<!-- <li>Fee-fi-of-fum</li> -->")
+
     def test_ordinal_numbers(self):
 
         self.assertEqual(sp("21st century"), "21st century")  # no effect.