Merge branch 'master' into master

8 files changed, 57 insertions, 17 deletions

diff --git a/AUTHORS b/AUTHORS
index 9019cb4..15debce 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -16,6 +16,7 @@ Alphabetical list of contributors:
 * Cristian Orellana <cristiano@groupon.com>
 * Dag Wieers <dag@wieers.com>
 * Darik Gamble <darik.gamble@gmail.com>
+* Demetrio92 <Demetrio.Rodriguez.T@gmail.com>
 * Dennis Taylor <dennis.taylor@clio.com>
 * Florian Bauer <florian.bauer@zmdi.com>
 * Gavin Wahl <gwahl@fusionbox.com>
@@ -27,6 +28,7 @@ Alphabetical list of contributors:
 * Michael Schuller <chick@mschuller.net>
 * Mike Amy <cocoade@googlemail.com>
 * mulos <daniel.strackbein@gmail.com>
+* Oleg Broytman <phd@phdru.name>
 * Piet Delport <pjdelport@gmail.com>
 * Prudhvi Vatala <pvatala@gmail.com>
 * quest <quest@wonky.windwards.net>
diff --git a/CHANGELOG b/CHANGELOG
index ecd075a..5e49022 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -1,9 +1,17 @@
 Development Version
 -------------------
 
+Enhancements
+
+* Add more keywords for MySQL table options (pr328, pr333, by phdru).
+* Add more PL/pgSQL keywords (pr357, by Demetrio92).
+
 Bug Fixes
 
+* Fix parsing of MySQL table names starting with digits (issue337).
 * Fix detection of identifiers using comparisons (issue327).
+* Fix parsing of UNION ALL after WHERE (issue349).
+
 
 
 Release 0.2.3 (Mar 02, 2017)
diff --git a/setup.cfg b/setup.cfg
index 6595a6c..2bcf8a9 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -1,4 +1,4 @@
-[wheel]
+[bdist_wheel]
 universal = 1
 
 [tool:pytest]
diff --git a/sqlparse/keywords.py b/sqlparse/keywords.py
index fbc77d5..a1242ab 100644
--- a/sqlparse/keywords.py
+++ b/sqlparse/keywords.py
@@ -14,6 +14,7 @@ def is_keyword(value):
     val = value.upper()
     return (KEYWORDS_COMMON.get(val) or
             KEYWORDS_ORACLE.get(val) or
+            KEYWORDS_PLPGSQL.get(val) or
             KEYWORDS.get(val, tokens.Name)), value
 
 
@@ -35,7 +36,7 @@ SQL_REGEX = {
 
         (r"`(``|[^`])*`", tokens.Name),
         (r"´(´´|[^´])*´", tokens.Name),
-        (r'(\$(?:[_A-Z]\w*)?\$)[\s\S]*?\1', tokens.Literal),
+        (r'(\$(?:[_A-ZÀ-Ü]\w*)?\$)[\s\S]*?\1', tokens.Literal),
 
         (r'\?', tokens.Name.Placeholder),
         (r'%(\(\w+\))?s', tokens.Name.Placeholder),
@@ -47,22 +48,20 @@ SQL_REGEX = {
         # is never a functino, see issue183
         (r'(CASE|IN|VALUES|USING)\b', tokens.Keyword),
 
-        (r'(@|##|#)[A-Z]\w+', tokens.Name),
+        (r'(@|##|#)[A-ZÀ-Ü]\w+', tokens.Name),
 
         # see issue #39
         # Spaces around period `schema . name` are valid identifier
         # TODO: Spaces before period not implemented
-        (r'[A-Z]\w*(?=\s*\.)', tokens.Name),  # 'Name'   .
+        (r'[A-ZÀ-Ü]\w*(?=\s*\.)', tokens.Name),  # 'Name'   .
         # FIXME(atronah): never match,
         # because `re.match` doesn't work with lookbehind regexp feature
-        (r'(?<=\.)[A-Z]\w*', tokens.Name),  # .'Name'
-        (r'[A-Z]\w*(?=\()', tokens.Name),  # side effect: change kw to func
-
+        (r'(?<=\.)[A-ZÀ-Ü]\w*', tokens.Name),  # .'Name'
+        (r'[A-ZÀ-Ü]\w*(?=\()', tokens.Name),  # side effect: change kw to func
         (r'-?0x[\dA-F]+', tokens.Number.Hexadecimal),
         (r'-?\d*(\.\d+)?E-?\d+', tokens.Number.Float),
         (r'-?(\d+(\.\d*)|\.\d+)', tokens.Number.Float),
-        (r'-?\d+', tokens.Number.Integer),
-
+        (r'-?\d+(?![_A-ZÀ-Ü])', tokens.Number.Integer),
         (r"'(''|\\\\|\\'|[^'])*'", tokens.String.Single),
         # not a real string literal in ANSI SQL:
         (r'(""|".*?[^\\]")', tokens.String.Symbol),
@@ -78,7 +77,7 @@ SQL_REGEX = {
         (r'CREATE(\s+OR\s+REPLACE)?\b', tokens.Keyword.DDL),
         (r'DOUBLE\s+PRECISION\b', tokens.Name.Builtin),
 
-        (r'[_A-Z][_$#\w]*', is_keyword),
+        (r'[0-9_A-ZÀ-Ü][_$#\w]*', is_keyword),
 
         (r'[;:()\[\],\.]', tokens.Punctuation),
         (r'[<>=~!]+', tokens.Operator.Comparison),
@@ -115,6 +114,7 @@ KEYWORDS = {
     'ATOMIC': tokens.Keyword,
     'AUDIT': tokens.Keyword,
     'AUTHORIZATION': tokens.Keyword,
+    'AUTO_INCREMENT': tokens.Keyword,
     'AVG': tokens.Keyword,
 
     'BACKWARD': tokens.Keyword,
@@ -143,6 +143,7 @@ KEYWORDS = {
     'CHARACTER_SET_NAME': tokens.Keyword,
     'CHARACTER_SET_SCHEMA': tokens.Keyword,
     'CHAR_LENGTH': tokens.Keyword,
+    'CHARSET': tokens.Keyword,
     'CHECK': tokens.Keyword,
     'CHECKED': tokens.Keyword,
     'CHECKPOINT': tokens.Keyword,
@@ -239,6 +240,7 @@ KEYWORDS = {
     'ENCODING': tokens.Keyword,
     'ENCRYPTED': tokens.Keyword,
     'END-EXEC': tokens.Keyword,
+    'ENGINE': tokens.Keyword,
     'EQUALS': tokens.Keyword,
     'ESCAPE': tokens.Keyword,
     'EVERY': tokens.Keyword,
@@ -637,7 +639,7 @@ KEYWORDS = {
     'SERIAL8': tokens.Name.Builtin,
     'SIGNED': tokens.Name.Builtin,
     'SMALLINT': tokens.Name.Builtin,
-    'SYSDATE': tokens.Name.Builtin,
+    'SYSDATE': tokens.Name,
     'TEXT': tokens.Name.Builtin,
     'TINYINT': tokens.Name.Builtin,
     'UNSIGNED': tokens.Name.Builtin,
@@ -798,3 +800,18 @@ KEYWORDS_ORACLE = {
     'UNLIMITED': tokens.Keyword,
     'UNLOCK': tokens.Keyword,
 }
+
+# PostgreSQL Syntax
+KEYWORDS_PLPGSQL = {
+    'PARTITION': tokens.Keyword,
+    'OVER': tokens.Keyword,
+    'PERFORM': tokens.Keyword,
+    'NOTICE': tokens.Keyword,
+    'PLPGSQL': tokens.Keyword,
+    'INHERIT': tokens.Keyword,
+    'INDEXES': tokens.Keyword,
+
+    'FOR': tokens.Keyword,
+    'IN': tokens.Keyword,
+    'LOOP': tokens.Keyword,
+}
diff --git a/sqlparse/sql.py b/sqlparse/sql.py
index f190de8..f698782 100644
--- a/sqlparse/sql.py
+++ b/sqlparse/sql.py
@@ -526,8 +526,9 @@ class Comment(TokenList):
 class Where(TokenList):
     """A WHERE clause."""
     M_OPEN = T.Keyword, 'WHERE'
-    M_CLOSE = T.Keyword, ('ORDER', 'GROUP', 'LIMIT', 'UNION', 'EXCEPT',
-                          'HAVING', 'RETURNING', 'INTO')
+    M_CLOSE = T.Keyword, (
+        'ORDER', 'GROUP', 'LIMIT', 'UNION', 'UNION ALL', 'EXCEPT',
+        'HAVING', 'RETURNING', 'INTO')
 
 
 class Case(TokenList):
diff --git a/sqlparse/utils.py b/sqlparse/utils.py
index ee3e982..ffa029b 100644
--- a/sqlparse/utils.py
+++ b/sqlparse/utils.py
@@ -95,9 +95,9 @@ def imt(token, i=None, m=None, t=None):
         return False
     elif clss and isinstance(token, clss):
         return True
-    elif mpatterns and any((token.match(*pattern) for pattern in mpatterns)):
+    elif mpatterns and any(token.match(*pattern) for pattern in mpatterns):
         return True
-    elif types and any([token.ttype in ttype for ttype in types]):
+    elif types and any(token.ttype in ttype for ttype in types):
         return True
     else:
         return False
diff --git a/tests/test_grouping.py b/tests/test_grouping.py
index 76ba651..1f3a19e 100644
--- a/tests/test_grouping.py
+++ b/tests/test_grouping.py
@@ -216,6 +216,15 @@ def test_grouping_where():
     assert isinstance(p.tokens[-1].tokens[0].tokens[-2], sql.Where)
 
 
+@pytest.mark.parametrize('s', (
+    'select 1 where 1 = 2 union select 2',
+    'select 1 where 1 = 2 union all select 2',
+))
+def test_grouping_where_union(s):
+    p = sqlparse.parse(s)[0]
+    assert p.tokens[5].value.startswith('union')
+
+
 def test_returning_kw_ends_where_clause():
     s = 'delete from foo where x > y returning z'
     p = sqlparse.parse(s)[0]
diff --git a/tests/test_parse.py b/tests/test_parse.py
index 0632889..4ba6ca1 100644
--- a/tests/test_parse.py
+++ b/tests/test_parse.py
@@ -135,11 +135,14 @@ def test_quoted_identifier():
     assert t[2].get_real_name() == 'y'
 
 
-@pytest.mark.parametrize('name', ['foo', '_foo'])
+@pytest.mark.parametrize('name', [
+    'foo', '_foo',  # issue175
+    '1_data',  # valid MySQL table name, see issue337
+])
 def test_valid_identifier_names(name):
-    # issue175
     t = sqlparse.parse(name)[0].tokens
     assert isinstance(t[0], sql.Identifier)
+    assert t[0].get_name() == name
 
 
 def test_psql_quotation_marks():