Allow whitespaces in certain keywords (fixes #97).

3 files changed, 33 insertions, 3 deletions

diff --git a/CHANGES b/CHANGES
index 120c8bb..39beee4 100644
--- a/CHANGES
+++ b/CHANGES
@@ -1,6 +1,10 @@
 Development Version
 -------------------
 
+Bug Fixes
+* Whitespaces within certain keywords are now allowed (issue97, patch proposed
+  by xcombelle).
+
 Enhancements
 * Improve parsing of assignments in UPDATE statements (issue90).
 * Add STRAIGHT_JOIN statement (by Yago Riveiro).
diff --git a/sqlparse/lexer.py b/sqlparse/lexer.py
index b981c0e..8b61fc1 100644
--- a/sqlparse/lexer.py
+++ b/sqlparse/lexer.py
@@ -196,10 +196,10 @@ class Lexer(object):
             # not a real string literal in ANSI SQL:
             (r'(""|".*?[^\\]")', tokens.String.Symbol),
             (r'(\[.*[^\]]\])', tokens.Name),
-            (r'(LEFT |RIGHT )?(INNER |OUTER |STRAIGHT)?JOIN\b', tokens.Keyword),
-            (r'END( IF| LOOP)?\b', tokens.Keyword),
+            (r'(LEFT\s+|RIGHT\s+)?(INNER\s+|OUTER\s+|STRAIGHT\s+)?JOIN\b', tokens.Keyword),
+            (r'END(\s+IF|\s+LOOP)?\b', tokens.Keyword),
             (r'NOT NULL\b', tokens.Keyword),
-            (r'CREATE( OR REPLACE)?\b', tokens.Keyword.DDL),
+            (r'CREATE(\s+OR\s+REPLACE)?\b', tokens.Keyword.DDL),
             (r'(?<=\.)[^\W\d_]\w*', tokens.Name),
             (r'[^\W\d_]\w*', is_keyword),
             (r'[;:()\[\],\.]', tokens.Punctuation),
diff --git a/tests/test_tokenize.py b/tests/test_tokenize.py
index 02af141..3b529d9 100644
--- a/tests/test_tokenize.py
+++ b/tests/test_tokenize.py
@@ -161,3 +161,29 @@ class TestStream(unittest.TestCase):
         tokens = list(lex.get_tokens(stream))
         self.assertEqual(len(tokens), 2)
         self.assertEqual(tokens[1][0], Error)
+
+
+def test_parse_join():
+    p = sqlparse.parse('LEFT JOIN foo')[0]
+    assert len(p.tokens) == 3
+    assert p.tokens[0].ttype is Keyword
+    p = sqlparse.parse('LEFT  OUTER  JOIN foo')[0]
+    assert len(p.tokens) == 3
+    assert p.tokens[0].ttype is Keyword
+
+
+def test_parse_endifloop():
+    p = sqlparse.parse('END IF')[0]
+    assert len(p.tokens) == 1
+    assert p.tokens[0].ttype is Keyword
+    p = sqlparse.parse('END   IF')[0]
+    assert len(p.tokens) == 1
+    p = sqlparse.parse('END\t\nIF')[0]
+    assert len(p.tokens) == 1
+    assert p.tokens[0].ttype is Keyword
+    p = sqlparse.parse('END LOOP')[0]
+    assert len(p.tokens) == 1
+    assert p.tokens[0].ttype is Keyword
+    p = sqlparse.parse('END  LOOP')[0]
+    assert len(p.tokens) == 1
+    assert p.tokens[0].ttype is Keyword