From 493d04a082adc0ffd02547ef373dd3156d46f8c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Legan=C3=A9s=20Combarro=20=22Piranna=22?= Date: Sat, 19 May 2012 15:42:09 +0200 Subject: Improved ColumnsSelect --- sqlparse/filters.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/sqlparse/filters.py b/sqlparse/filters.py index 92d7bd5..bc2af95 100644 --- a/sqlparse/filters.py +++ b/sqlparse/filters.py @@ -458,17 +458,21 @@ class ColumnsSelect: mode = 1 # We have detected a SELECT statement - elif mode == 1: - if value == 'FROM': + elif mode in (1, 3): + if value in ('FROM', 'WHERE', 'GROUP'): if oldValue: yield oldValue + oldValue = "" - mode = 3 # Columns have been checked + break # Columns have been checked elif value == 'AS': oldValue = "" mode = 2 + elif token_type in Whitespace: + mode = 3 + elif (token_type == Punctuation and value == ',' and not parenthesis): if oldValue: @@ -481,7 +485,11 @@ class ColumnsSelect: elif value == ')': parenthesis -= 1 - oldValue += value + if mode == 3: + oldValue = value + mode = 1 + else: + oldValue += value # We are processing an AS keyword elif mode == 2: @@ -490,6 +498,9 @@ class ColumnsSelect: yield value mode = 1 + if oldValue: + yield oldValue + # --------------------------- # postprocess -- cgit v1.2.1 From 42f9f77e07efc1c4f248aa9e48401e11c0ae4601 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Legan=C3=A9s=20Combarro=20=22Piranna=22?= Date: Sat, 19 May 2012 17:36:56 +0200 Subject: Cleaned group_identifier_list() --- sqlparse/engine/grouping.py | 37 +++++++++++++++++++++---------------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/sqlparse/engine/grouping.py b/sqlparse/engine/grouping.py index 1487c24..8f5ccd9 100644 --- a/sqlparse/engine/grouping.py +++ b/sqlparse/engine/grouping.py @@ -185,9 +185,10 @@ def group_identifier(tlist): def group_identifier_list(tlist): - [group_identifier_list(sgroup) for sgroup in tlist.get_sublists() - if not isinstance(sgroup, sql.IdentifierList)] - idx = 0 + for sgroup in tlist.get_sublists(): + if not isinstance(sgroup, sql.IdentifierList): + group_identifier_list(sgroup) + # Allowed list items fend1_funcs = [lambda t: isinstance(t, (sql.Identifier, sql.Function, sql.Case)), @@ -202,36 +203,40 @@ def group_identifier_list(tlist): lambda t: isinstance(t, sql.Comparison), lambda t: isinstance(t, sql.Comment), ] - tcomma = tlist.token_next_match(idx, T.Punctuation, ',') + + tcomma = tlist.token_next_match(0, T.Punctuation, ',') start = None - while tcomma is not None: + while tcomma: before = tlist.token_prev(tcomma) after = tlist.token_next(tcomma) + # Check if the tokens around tcomma belong to a list bpassed = apassed = False for func in fend1_funcs: - if before is not None and func(before): + if before and func(before): bpassed = True - if after is not None and func(after): + if after and func(after): apassed = True - if not bpassed or not apassed: - # Something's wrong here, skip ahead to next "," - start = None - tcomma = tlist.token_next_match(tlist.token_index(tcomma) + 1, - T.Punctuation, ',') - else: + + if bpassed and apassed: if start is None: start = before + next_ = tlist.token_next(after) - if next_ is None or not next_.match(T.Punctuation, ','): + if next_ and next_.match(T.Punctuation, ','): + tcomma = next_ + else: # Reached the end of the list tokens = tlist.tokens_between(start, after) group = tlist.group_tokens(sql.IdentifierList, tokens) start = None tcomma = tlist.token_next_match(tlist.token_index(group) + 1, T.Punctuation, ',') - else: - tcomma = next_ + else: + # Something's wrong here, skip ahead to next "," + start = None + tcomma = tlist.token_next_match(tlist.token_index(tcomma) + 1, + T.Punctuation, ',') def group_parenthesis(tlist): -- cgit v1.2.1 From 1b1810e4eaa5af17fe8d4f40717ae8f0a30554d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Legan=C3=A9s=20Combarro=20=22Piranna=22?= Date: Sat, 19 May 2012 18:00:43 +0200 Subject: Commented group_identifier_list() --- sqlparse/engine/grouping.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/sqlparse/engine/grouping.py b/sqlparse/engine/grouping.py index 8f5ccd9..37e029b 100644 --- a/sqlparse/engine/grouping.py +++ b/sqlparse/engine/grouping.py @@ -185,6 +185,7 @@ def group_identifier(tlist): def group_identifier_list(tlist): + # First group the `tlist` sublists for sgroup in tlist.get_sublists(): if not isinstance(sgroup, sql.IdentifierList): group_identifier_list(sgroup) @@ -204,13 +205,14 @@ def group_identifier_list(tlist): lambda t: isinstance(t, sql.Comment), ] - tcomma = tlist.token_next_match(0, T.Punctuation, ',') start = None + + tcomma = tlist.token_next_match(0, T.Punctuation, ',') while tcomma: before = tlist.token_prev(tcomma) after = tlist.token_next(tcomma) - # Check if the tokens around tcomma belong to a list + # Check if the tokens around tcomma belong to an identifier list bpassed = apassed = False for func in fend1_funcs: if before and func(before): @@ -218,22 +220,31 @@ def group_identifier_list(tlist): if after and func(after): apassed = True + # Both tokens around tcomma belong to a list if bpassed and apassed: - if start is None: + # Set the start of the identifier list if not defined before + if start == None: start = before + # Look if the next token is another comma next_ = tlist.token_next(after) if next_ and next_.match(T.Punctuation, ','): tcomma = next_ + + # Reached the end of the list else: - # Reached the end of the list + # Create and group the identifiers list tokens = tlist.tokens_between(start, after) group = tlist.group_tokens(sql.IdentifierList, tokens) + + # Skip ahead to next "," start = None tcomma = tlist.token_next_match(tlist.token_index(group) + 1, T.Punctuation, ',') + + # At least one of the tokens around tcomma don't belong to an + # identifier list. Something's wrong here, skip ahead to next "," else: - # Something's wrong here, skip ahead to next "," start = None tcomma = tlist.token_next_match(tlist.token_index(tcomma) + 1, T.Punctuation, ',') -- cgit v1.2.1 From 9255cbc838b1fada24b2360ec9a12e311b9ced63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Legan=C3=A9s=20Combarro=20=22Piranna=22?= Date: Sat, 19 May 2012 18:36:36 +0200 Subject: Improvements on identifierlist --- sqlparse/engine/grouping.py | 32 ++++++++++++++++++++++---------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/sqlparse/engine/grouping.py b/sqlparse/engine/grouping.py index 37e029b..095827c 100644 --- a/sqlparse/engine/grouping.py +++ b/sqlparse/engine/grouping.py @@ -228,19 +228,25 @@ def group_identifier_list(tlist): # Look if the next token is another comma next_ = tlist.token_next(after) - if next_ and next_.match(T.Punctuation, ','): - tcomma = next_ + if next_: + if next_.match(T.Punctuation, ','): + tcomma = next_ + continue + + elif(next_.ttype == T.Keyword + and next_.value not in ('FROM', 'WHERE', 'GROUP')): + tcomma = next_ + continue # Reached the end of the list - else: - # Create and group the identifiers list - tokens = tlist.tokens_between(start, after) - group = tlist.group_tokens(sql.IdentifierList, tokens) + # Create and group the identifiers list + tokens = tlist.tokens_between(start, after) + group = tlist.group_tokens(sql.IdentifierList, tokens) - # Skip ahead to next "," - start = None - tcomma = tlist.token_next_match(tlist.token_index(group) + 1, - T.Punctuation, ',') + # Skip ahead to next "," + start = None + tcomma = tlist.token_next_match(tlist.token_index(group) + 1, + T.Punctuation, ',') # At least one of the tokens around tcomma don't belong to an # identifier list. Something's wrong here, skip ahead to next "," @@ -249,6 +255,12 @@ def group_identifier_list(tlist): tcomma = tlist.token_next_match(tlist.token_index(tcomma) + 1, T.Punctuation, ',') + # There's an open identifier list + if start: + # Create and group the identifiers list + tokens = tlist.tokens_between(start, after) + group = tlist.group_tokens(sql.IdentifierList, tokens) + def group_parenthesis(tlist): _group_matching(tlist, T.Punctuation, '(', T.Punctuation, ')', -- cgit v1.2.1 From cd42b68188970376bc1e77473837083a4cd72226 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Legan=C3=A9s=20Combarro=20=22Piranna=22?= Date: Fri, 1 Jun 2012 22:32:58 +0200 Subject: Added comments --- sqlparse/filters.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/sqlparse/filters.py b/sqlparse/filters.py index bc2af95..5919045 100644 --- a/sqlparse/filters.py +++ b/sqlparse/filters.py @@ -336,17 +336,32 @@ class ReindentFilter: self.offset -= num_offset def _process_identifierlist(self, tlist): + # Get identifiers from the tlist identifiers = list(tlist.get_identifiers()) + + # Split the identifier list if we have more than one identifier + # and its not from a function if len(identifiers) > 1 and not tlist.within(sql.Function): + # Get first token first = list(identifiers[0].flatten())[0] + + # Increase offset the size of the first token num_offset = self._get_offset(first) - len(first.value) self.offset += num_offset + + # Insert a new line between the tokens for token in identifiers[1:]: tlist.insert_before(token, self.nl()) + + # Imsert another new line after comment tokens for token in tlist.tokens: if isinstance(token, sql.Comment): tlist.insert_after(token, self.nl()) + + # Decrease offset the size of the first token self.offset -= num_offset + + # Process default tokens over tlist as usual self._process_default(tlist) def _process_case(self, tlist): @@ -383,7 +398,9 @@ class ReindentFilter: self._split_statements(tlist) if kwds: self._split_kwds(tlist) - [self._process(sgroup) for sgroup in tlist.get_sublists()] + + for sgroup in tlist.get_sublists(): + self._process(sgroup) def process(self, stack, stmt): if isinstance(stmt, sql.Statement): -- cgit v1.2.1 From 46014daed27b6cb0f042b6820937d391a2b2d424 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Legan=C3=A9s=20Combarro=20=22Piranna=22?= Date: Fri, 1 Jun 2012 22:59:54 +0200 Subject: Little clean-up --- sqlparse/filters.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/sqlparse/filters.py b/sqlparse/filters.py index 5919045..55af970 100644 --- a/sqlparse/filters.py +++ b/sqlparse/filters.py @@ -405,17 +405,19 @@ class ReindentFilter: def process(self, stack, stmt): if isinstance(stmt, sql.Statement): self._curr_stmt = stmt + self._process(stmt) + if isinstance(stmt, sql.Statement): - if self._last_stmt is not None: + if self._last_stmt: if unicode(self._last_stmt).endswith('\n'): nl = '\n' else: nl = '\n\n' - stmt.tokens.insert(0, - sql.Token(T.Whitespace, nl)) - if self._last_stmt != stmt: - self._last_stmt = stmt + + stmt.tokens.insert(0, sql.Token(T.Whitespace, nl)) + + self._last_stmt = stmt # FIXME: Doesn't work ;) -- cgit v1.2.1 From 7768a1a1bd67578ef2a591ac779782098716825d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Legan=C3=A9s=20Combarro=20=22Piranna=22?= Date: Fri, 1 Jun 2012 23:05:56 +0200 Subject: Fixed bug on identifiers --- sqlparse/engine/grouping.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqlparse/engine/grouping.py b/sqlparse/engine/grouping.py index 095827c..4120fc3 100644 --- a/sqlparse/engine/grouping.py +++ b/sqlparse/engine/grouping.py @@ -234,7 +234,7 @@ def group_identifier_list(tlist): continue elif(next_.ttype == T.Keyword - and next_.value not in ('FROM', 'WHERE', 'GROUP')): + and next_.value.upper() not in ('FROM', 'WHERE', 'GROUP')): tcomma = next_ continue -- cgit v1.2.1