From 45db01becdc17108bc1c504503ee7b997059970d Mon Sep 17 00:00:00 2001 From: Andi Albrecht Date: Sun, 3 May 2009 20:54:57 +0200 Subject: Added unittest for issue1 (linebreaks). --- tests/test_tokenize.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/test_tokenize.py b/tests/test_tokenize.py index 74cf1fc..127b4db 100644 --- a/tests/test_tokenize.py +++ b/tests/test_tokenize.py @@ -24,3 +24,17 @@ class TestTokenize(unittest.TestCase): tokens = list(lexer.tokenize(sql)) self.assertEqual(len(tokens), 3) self.assertEqual(tokens[0], (Name, u'`foo`')) + + def test_linebreaks(self): # issue1 + sql = 'foo\nbar\n' + tokens = lexer.tokenize(sql) + self.assertEqual(''.join(str(x[1]) for x in tokens), sql) + sql = 'foo\rbar\r' + tokens = lexer.tokenize(sql) + self.assertEqual(''.join(str(x[1]) for x in tokens), sql) + sql = 'foo\r\nbar\r\n' + tokens = lexer.tokenize(sql) + self.assertEqual(''.join(str(x[1]) for x in tokens), sql) + sql = 'foo\r\nbar\n' + tokens = lexer.tokenize(sql) + self.assertEqual(''.join(str(x[1]) for x in tokens), sql) -- cgit v1.2.1