diff options
author | Simon Heisterkamp <simon@heisterkamp.dk> | 2022-11-30 22:34:52 +0000 |
---|---|---|
committer | Andi Albrecht <albrecht.andi@gmail.com> | 2023-01-02 08:54:47 +0100 |
commit | f9a73a62cfc23b10c38f22a10bd1d4c3edbb286f (patch) | |
tree | c96c5fcd8e93308456377b1926807fbdf9b912a6 | |
parent | 8515d2edd70fc16d69aa7b1094f9b3534dfa74d9 (diff) | |
download | sqlparse-f9a73a62cfc23b10c38f22a10bd1d4c3edbb286f.tar.gz |
test for changing the regex
-rw-r--r-- | sqlparse/lexer.py | 10 | ||||
-rw-r--r-- | tests/test_parse.py | 34 |
2 files changed, 35 insertions, 9 deletions
diff --git a/sqlparse/lexer.py b/sqlparse/lexer.py index 7408e01..aafb55f 100644 --- a/sqlparse/lexer.py +++ b/sqlparse/lexer.py @@ -99,14 +99,12 @@ class Lexer(metaclass=_LexerSingletonMetaclass): text = text.decode(encoding) else: try: - text = text.decode("utf-8") + text = text.decode('utf-8') except UnicodeDecodeError: - text = text.decode("unicode-escape") + text = text.decode('unicode-escape') else: - raise TypeError( - "Expected text or file-like object, got {!r}" - .format(type(text)) - ) + raise TypeError("Expected text or file-like object, got {!r}". + format(type(text))) iterable = enumerate(text) for pos, char in iterable: diff --git a/tests/test_parse.py b/tests/test_parse.py index 3018d9a..3ac6500 100644 --- a/tests/test_parse.py +++ b/tests/test_parse.py @@ -1,10 +1,11 @@ """Tests sqlparse.parse().""" +import re from io import StringIO import pytest import sqlparse -from sqlparse import sql, tokens as T +from sqlparse import sql, tokens as T, keywords from sqlparse.lexer import Lexer @@ -491,8 +492,7 @@ def test_parenthesis(): T.Newline, T.Punctuation] - -def test_configurable_syntax(): +def test_configurable_keywords(): sql = """select * from foo BACON SPAM EGGS;""" tokens = sqlparse.parse(sql)[0] @@ -517,6 +517,9 @@ def test_configurable_syntax(): tokens = sqlparse.parse(sql)[0] + # reset the syntax for later tests. + Lexer().default_initialization() + assert list( (t.ttype, t.value) for t in tokens if t.ttype not in sqlparse.tokens.Whitespace ) == [ @@ -529,5 +532,30 @@ def test_configurable_syntax(): (sqlparse.tokens.Keyword, "EGGS"), (sqlparse.tokens.Punctuation, ";"), ] + + +def test_configurable_regex(): + lex = Lexer() + lex.clear() + + my_regex = ( + re.compile(r"ZORDER\s+BY\b", keywords.FLAGS).match, + sqlparse.tokens.Keyword, + ) + + lex.set_SQL_REGEX(keywords.SQL_REGEX[:38] + [my_regex] + keywords.SQL_REGEX[38:]) + lex.add_keywords(keywords.KEYWORDS_COMMON) + lex.add_keywords(keywords.KEYWORDS_ORACLE) + lex.add_keywords(keywords.KEYWORDS_PLPGSQL) + lex.add_keywords(keywords.KEYWORDS_HQL) + lex.add_keywords(keywords.KEYWORDS_MSACCESS) + lex.add_keywords(keywords.KEYWORDS) + + tokens = sqlparse.parse("select * from foo zorder by bar;")[0] + # reset the syntax for later tests. Lexer().default_initialization() + + assert list( + (t.ttype, t.value) for t in tokens if t.ttype not in sqlparse.tokens.Whitespace + )[4] == (sqlparse.tokens.Keyword, "zorder by") |