From 36413e8eba08fcadca60f286122b19e9424d55a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Legan=C3=A9s=20Combarro=20=22Piranna=22?= Date: Thu, 11 Aug 2011 20:40:17 +0200 Subject: Added Get_Comments, StripComments, IncludeStatement, ColumnsSelect and Limit filters. --- sqlparse/filters.py | 158 +++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 155 insertions(+), 3 deletions(-) (limited to 'sqlparse') diff --git a/sqlparse/filters.py b/sqlparse/filters.py index 813be99..ce2fb80 100644 --- a/sqlparse/filters.py +++ b/sqlparse/filters.py @@ -2,8 +2,12 @@ import re -from sqlparse import tokens as T +from os.path import abspath, join + from sqlparse import sql +from sqlparse import tokens as T +from sqlparse.engine import FilterStack +from sqlparse.tokens import Comment, Keyword, Name, Punctuation, String, Whitespace class Filter(object): @@ -52,6 +56,83 @@ class IdentifierCaseFilter(_CaseFilter): yield ttype, value +class Get_Comments(Filter): + """Get the comments from a stack""" + def process(self, stack, stream): + for token_type, value in stream: + if token_type in Comment: + yield token_type, value + + +class StripComments(Filter): + """Strip the comments from a stack""" + def process(self, stack, stream): + for token_type, value in stream: + if token_type not in Comment: + yield token_type, value + + +class IncludeStatement(Filter): + """Filter that enable a INCLUDE statement""" + + def __init__(self, dirpath=".", maxRecursive=10): + self.dirpath = abspath(dirpath) + self.maxRecursive = maxRecursive + + self.detected = False + + + def process(self, stack, stream): + # Run over all tokens in the stream + for token_type, value in stream: + # INCLUDE statement found, set detected mode + if token_type in Name and value.upper() == 'INCLUDE': + self.detected = True + continue + + # INCLUDE statement was found, parse it + elif self.detected: + # Omit whitespaces + if token_type in Whitespace: + pass + + # Get path of file to include + path = None + + if token_type in String.Symbol: +# if token_type in tokens.String.Symbol: + path = join(self.dirpath, value[1:-1]) + + # Include file if path was found + if path: + try: + with open(path) as f: + sql = f.read() + + except IOError, err: + logging.error(err) + yield Comment, u'-- IOError: %s\n' % err + + else: + # Create new FilterStack to parse readed file + # and add all its tokens to the main stack recursively + # [ToDo] Add maximum recursive iteration value + stack = FilterStack() + stack.preprocess.append(IncludeStatement(self.dirpath)) + + for tv in stack.run(sql): + yield tv + + # Set normal mode + self.detected = False + + # Don't include any token while in detected mode + continue + + # Normal token + yield token_type, value + + # ---------------------- # statement process @@ -150,9 +231,9 @@ class ReindentFilter(Filter): t = tlist.token_next_match(i, T.Keyword, split_words, regex=True) if t and t.value.upper() == 'BETWEEN': - t = _next_token(tlist.token_index(t)+1) + t = _next_token(tlist.token_index(t) + 1) if t and t.value.upper() == 'AND': - t = _next_token(tlist.token_index(t)+1) + t = _next_token(tlist.token_index(t) + 1) return t idx = 0 @@ -316,6 +397,56 @@ class RightMarginFilter(Filter): group.tokens = self._process(stack, group, group.tokens) +class ColumnsSelect(Filter): + """Get the columns names of a SELECT query""" + def process(self, stack, stream): + mode = 0 + oldValue = "" + parenthesis = 0 + + for token_type, value in stream: + # Ignore comments + if token_type in Comment: + continue + + # We have not detected a SELECT statement + if mode == 0: + if token_type in Keyword and value == 'SELECT': + mode = 1 + + # We have detected a SELECT statement + elif mode == 1: + if value == 'FROM': + if oldValue: + yield Name, oldValue + + mode = 3 # Columns have been checked + + elif value == 'AS': + oldValue = "" + mode = 2 + + elif token_type == Punctuation and value == ',' and not parenthesis: + if oldValue: + yield Name, oldValue + oldValue = "" + + elif token_type not in Whitespace: + if value == '(': + parenthesis += 1 + elif value == ')': + parenthesis -= 1 + + oldValue += value + + # We are processing an AS keyword + elif mode == 2: + # We check also for Keywords because a bug in SQLParse + if token_type == Name or token_type == Keyword: + yield Name, value + mode = 1 + + # --------------------------- # postprocess @@ -422,3 +553,24 @@ class OutputPHPFilter(Filter): varname = self.varname stmt.tokens = tuple(self._process(stmt.tokens, varname)) return stmt + + +class Limit(Filter): + """Get the LIMIT of a query. + + If not defined, return -1 (SQL specification for no LIMIT query) + """ + def process(self, stack, stream): + index = 7 + stream = list(stream) + stream.reverse() + + # Run over all tokens in the stream from the end + for token_type, value in stream: + index -= 1 + +# if index and token_type in Keyword: + if index and token_type in Keyword and value == 'LIMIT': + return stream[4 - index][1] + + return -1 \ No newline at end of file -- cgit v1.2.1 From 99af50cf179539f8d82e57ea9a0530adad238a96 Mon Sep 17 00:00:00 2001 From: Andi Albrecht Date: Fri, 12 Aug 2011 14:34:58 +0200 Subject: Code cleanup. --- sqlparse/filters.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) (limited to 'sqlparse') diff --git a/sqlparse/filters.py b/sqlparse/filters.py index ce2fb80..2d715a8 100644 --- a/sqlparse/filters.py +++ b/sqlparse/filters.py @@ -7,7 +7,10 @@ from os.path import abspath, join from sqlparse import sql from sqlparse import tokens as T from sqlparse.engine import FilterStack -from sqlparse.tokens import Comment, Keyword, Name, Punctuation, String, Whitespace +from sqlparse.tokens import ( + Comment, Keyword, Name, + Punctuation, String, Whitespace, +) class Filter(object): @@ -56,7 +59,7 @@ class IdentifierCaseFilter(_CaseFilter): yield ttype, value -class Get_Comments(Filter): +class GetComments(Filter): """Get the comments from a stack""" def process(self, stack, stream): for token_type, value in stream: @@ -81,7 +84,6 @@ class IncludeStatement(Filter): self.detected = False - def process(self, stack, stream): # Run over all tokens in the stream for token_type, value in stream: @@ -107,10 +109,9 @@ class IncludeStatement(Filter): if path: try: with open(path) as f: - sql = f.read() + raw_sql = f.read() except IOError, err: - logging.error(err) yield Comment, u'-- IOError: %s\n' % err else: @@ -120,7 +121,7 @@ class IncludeStatement(Filter): stack = FilterStack() stack.preprocess.append(IncludeStatement(self.dirpath)) - for tv in stack.run(sql): + for tv in stack.run(raw_sql): yield tv # Set normal mode @@ -227,6 +228,7 @@ class ReindentFilter(Filter): split_words = ('FROM', 'JOIN$', 'AND', 'OR', 'GROUP', 'ORDER', 'UNION', 'VALUES', 'SET', 'BETWEEN') + def _next_token(i): t = tlist.token_next_match(i, T.Keyword, split_words, regex=True) @@ -426,7 +428,8 @@ class ColumnsSelect(Filter): oldValue = "" mode = 2 - elif token_type == Punctuation and value == ',' and not parenthesis: + elif (token_type == Punctuation + and value == ',' and not parenthesis): if oldValue: yield Name, oldValue oldValue = "" @@ -573,4 +576,4 @@ class Limit(Filter): if index and token_type in Keyword and value == 'LIMIT': return stream[4 - index][1] - return -1 \ No newline at end of file + return -1 -- cgit v1.2.1 From 5a1d58a6fc69ddc04abb8750e1cf2e156caa04ab Mon Sep 17 00:00:00 2001 From: Andi Albrecht Date: Fri, 12 Aug 2011 14:56:21 +0200 Subject: Remove with statement to stay as compatible with older Python versions as possible. For now, sqlparse is still compatible with Python 2.4 :) --- sqlparse/filters.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'sqlparse') diff --git a/sqlparse/filters.py b/sqlparse/filters.py index 2d715a8..cba7b8f 100644 --- a/sqlparse/filters.py +++ b/sqlparse/filters.py @@ -108,9 +108,9 @@ class IncludeStatement(Filter): # Include file if path was found if path: try: - with open(path) as f: - raw_sql = f.read() - + f = open(path) + raw_sql = f.read() + f.close() except IOError, err: yield Comment, u'-- IOError: %s\n' % err -- cgit v1.2.1