# -*- coding: utf-8 -*- import unittest import types import sqlparse from sqlparse import lexer from sqlparse import sql from sqlparse.tokens import * class TestTokenize(unittest.TestCase): def test_simple(self): sql = 'select * from foo;' stream = lexer.tokenize(sql) self.assert_(type(stream) is types.GeneratorType) tokens = list(stream) self.assertEqual(len(tokens), 8) self.assertEqual(len(tokens[0]), 2) self.assertEqual(tokens[0], (Keyword.DML, u'select')) self.assertEqual(tokens[-1], (Punctuation, u';')) def test_backticks(self): sql = '`foo`.`bar`' tokens = list(lexer.tokenize(sql)) self.assertEqual(len(tokens), 3) self.assertEqual(tokens[0], (Name, u'`foo`')) def test_linebreaks(self): # issue1 sql = 'foo\nbar\n' tokens = lexer.tokenize(sql) self.assertEqual(''.join(str(x[1]) for x in tokens), sql) sql = 'foo\rbar\r' tokens = lexer.tokenize(sql) self.assertEqual(''.join(str(x[1]) for x in tokens), sql) sql = 'foo\r\nbar\r\n' tokens = lexer.tokenize(sql) self.assertEqual(''.join(str(x[1]) for x in tokens), sql) sql = 'foo\r\nbar\n' tokens = lexer.tokenize(sql) self.assertEqual(''.join(str(x[1]) for x in tokens), sql) def test_inline_keywords(self): # issue 7 sql = "create created_foo" tokens = list(lexer.tokenize(sql)) self.assertEqual(len(tokens), 3) self.assertEqual(tokens[0][0], Keyword.DDL) self.assertEqual(tokens[2][0], Name) self.assertEqual(tokens[2][1], u'created_foo') sql = "enddate" tokens = list(lexer.tokenize(sql)) self.assertEqual(len(tokens), 1) self.assertEqual(tokens[0][0], Name) sql = "join_col" tokens = list(lexer.tokenize(sql)) self.assertEqual(len(tokens), 1) self.assertEqual(tokens[0][0], Name) sql = "left join_col" tokens = list(lexer.tokenize(sql)) self.assertEqual(len(tokens), 3) self.assertEqual(tokens[2][0], Name) self.assertEqual(tokens[2][1], 'join_col') def test_negative_numbers(self): sql = "values(-1)" tokens = list(lexer.tokenize(sql)) self.assertEqual(len(tokens), 4) self.assertEqual(tokens[2][0], Number.Integer) self.assertEqual(tokens[2][1], '-1') def test_tab_expansion(self): sql = "\t" lex = lexer.Lexer() lex.tabsize = 5 tokens = list(lex.get_tokens(sql)) self.assertEqual(tokens[0][1], " " * 5) class TestToken(unittest.TestCase): def test_str(self): token = sql.Token(None, 'FoO') self.assertEqual(str(token), 'FoO') def test_repr(self): token = sql.Token(Keyword, 'foo') tst = "