Merge pull request #255 from vmuriart/console-script-examples-sqlOperation

Add Console-script, sql.Operation, fix examples

14 files changed, 293 insertions, 227 deletions

diff --git a/bin/sqlformat b/bin/sqlformat
deleted file mode 100755
index 3f61064..0000000
--- a/bin/sqlformat
+++ /dev/null
@@ -1,115 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-#
-# Copyright (C) 2016 Andi Albrecht, albrecht.andi@gmail.com
-#
-# This module is part of python-sqlparse and is released under
-# the BSD License: http://www.opensource.org/licenses/bsd-license.php
-
-import optparse
-import sys
-
-import sqlparse
-from sqlparse.exceptions import SQLParseError
-
-_CASE_CHOICES = ['upper', 'lower', 'capitalize']
-
-parser = optparse.OptionParser(usage='%prog [OPTIONS] FILE, ...',
-                               version='%%prog %s' % sqlparse.__version__)
-parser.set_description(('Format FILE according to OPTIONS. Use "-" as FILE '
-                        'to read from stdin.'))
-parser.add_option('-v', '--verbose', dest='verbose', action='store_true')
-parser.add_option('-o', '--outfile', dest='outfile', metavar='FILE',
-                  help='write output to FILE (defaults to stdout)')
-group = parser.add_option_group('Formatting Options')
-group.add_option('-k', '--keywords', metavar='CHOICE',
-                 dest='keyword_case', choices=_CASE_CHOICES,
-                 help=('change case of keywords, CHOICE is one of %s'
-                       % ', '.join('"%s"' % x for x in _CASE_CHOICES)))
-group.add_option('-i', '--identifiers', metavar='CHOICE',
-                 dest='identifier_case', choices=_CASE_CHOICES,
-                 help=('change case of identifiers, CHOICE is one of %s'
-                       % ', '.join('"%s"' % x for x in _CASE_CHOICES)))
-group.add_option('-l', '--language', metavar='LANG',
-                 dest='output_format', choices=['python', 'php'],
-                 help=('output a snippet in programming language LANG, '
-                       'choices are "python", "php"'))
-group.add_option('--strip-comments', dest='strip_comments',
-                 action='store_true', default=False,
-                 help='remove comments')
-group.add_option('-r', '--reindent', dest='reindent',
-                 action='store_true', default=False,
-                 help='reindent statements')
-group.add_option('--indent_width', dest='indent_width', default=2,
-                 help='indentation width (defaults to 2 spaces)')
-group.add_option('-a', '--reindent_aligned',
-                 action='store_true', default=False,
-                 help='reindent statements to aligned format')
-group.add_option('-s', '--use_space_around_operators',
-                 action='store_true', default=False,
-                 help='place spaces around mathematical operators')
-group.add_option('--wrap_after', dest='wrap_after', default=0,
-                 help='Column after which lists should be wrapped')
-
-_FORMATTING_GROUP = group
-
-
-def _error(msg, exit_=None):
-    """Print msg and optionally exit with return code exit_."""
-    sys.stderr.write('[ERROR] %s\n' % msg)
-    if exit_ is not None:
-        sys.exit(exit_)
-
-
-def _build_formatter_opts(options):
-    """Convert command line options to dictionary."""
-    d = {}
-    for option in _FORMATTING_GROUP.option_list:
-        d[option.dest] = getattr(options, option.dest)
-    return d
-
-
-def main():
-    options, args = parser.parse_args()
-    if options.verbose:
-        sys.stderr.write('Verbose mode\n')
-
-    if len(args) != 1:
-        _error('No input data.')
-        parser.print_usage()
-        sys.exit(1)
-
-    if '-' in args:  # read from stdin
-        data = sys.stdin.read()
-    else:
-        try:
-            data = ''.join(open(args[0]).readlines())
-        except OSError:
-            err = sys.exc_info()[1]  # Python 2.5 compatibility
-            _error('Failed to read %s: %s' % (args[0], err), exit_=1)
-
-    if options.outfile:
-        try:
-            stream = open(options.outfile, 'w')
-        except OSError:
-            err = sys.exc_info()[1]  # Python 2.5 compatibility
-            _error('Failed to open %s: %s' % (options.outfile, err), exit_=1)
-    else:
-        stream = sys.stdout
-
-    formatter_opts = _build_formatter_opts(options)
-    try:
-        formatter_opts = sqlparse.formatter.validate_options(formatter_opts)
-    except SQLParseError:
-        err = sys.exc_info()[1]  # Python 2.5 compatibility
-        _error('Invalid options: %s' % err, exit_=1)
-
-    s = sqlparse.format(data, **formatter_opts)
-    if sys.version_info < (3,):
-        s = s.encode('utf-8', 'replace')
-    stream.write(s)
-    stream.flush()
-
-
-if __name__ == '__main__':
-    main()
diff --git a/examples/column_defs_lowlevel.py b/examples/column_defs_lowlevel.py
index 5acbdec..5e98be3 100644
--- a/examples/column_defs_lowlevel.py
+++ b/examples/column_defs_lowlevel.py
@@ -11,43 +11,39 @@
 
 import sqlparse
 
-SQL = """CREATE TABLE foo (
-  id integer primary key,
-  title varchar(200) not null,
-  description text
-);"""
-
-
-parsed = sqlparse.parse(SQL)[0]
-
-# extract the parenthesis which holds column definitions
-par = parsed.token_next_by(i=sqlparse.sql.Parenthesis)
-
 
 def extract_definitions(token_list):
     # assumes that token_list is a parenthesis
     definitions = []
     tmp = []
-    # grab the first token, ignoring whitespace
-    token = token_list.token_next(0)
+    # grab the first token, ignoring whitespace. idx=1 to skip open (
+    token = token_list.token_next(1)
     while token and not token.match(sqlparse.tokens.Punctuation, ')'):
         tmp.append(token)
-        idx = token_list.token_index(token)
         # grab the next token, this times including whitespace
-        token = token_list.token_next(idx, skip_ws=False)
+        token = token_list.token_next(token, skip_ws=False)
         # split on ",", except when on end of statement
         if token and token.match(sqlparse.tokens.Punctuation, ','):
             definitions.append(tmp)
             tmp = []
-            idx = token_list.token_index(token)
-            token = token_list.token_next(idx)
+            token = token_list.token_next(token)
     if tmp and isinstance(tmp[0], sqlparse.sql.Identifier):
         definitions.append(tmp)
     return definitions
 
 
-columns = extract_definitions(par)
+if __name__ == '__main__':
+    SQL = """CREATE TABLE foo (
+             id integer primary key,
+             title varchar(200) not null,
+             description text);"""
+
+    parsed = sqlparse.parse(SQL)[0]
+
+    # extract the parenthesis which holds column definitions
+    par = parsed.token_next_by(i=sqlparse.sql.Parenthesis)
+    columns = extract_definitions(par)
 
-for column in columns:
-    print('NAME: {name:10} DEFINITION: {definition}'.format(
-        name=column[0], definition=''.join(str(t) for t in column[1:])))
+    for column in columns:
+        print('NAME: {name:10} DEFINITION: {definition}'.format(
+            name=column[0], definition=''.join(str(t) for t in column[1:])))
diff --git a/setup.py b/setup.py
index 45d560f..9fc1c80 100644
--- a/setup.py
+++ b/setup.py
@@ -1,3 +1,4 @@
+#!/usr/bin/env python
 # -*- coding: utf-8 -*-
 #
 # Copyright (C) 2016 Andi Albrecht, albrecht.andi@gmail.com
@@ -6,16 +7,8 @@
 # the BSD License: http://www.opensource.org/licenses/bsd-license.php
 
 import re
-import sys
 
-try:
-    from setuptools import setup, find_packages
-    packages = find_packages(exclude=('tests',))
-except ImportError:
-    if sys.version_info[0] == 3:
-        raise RuntimeError('distribute is required to install this package.')
-    from distutils.core import setup
-    packages = ['sqlparse', 'sqlparse.engine']
+from setuptools import setup, find_packages
 
 
 def get_version():
@@ -81,19 +74,15 @@ Parsing::
 
 """
 
-VERSION = get_version()
-
-
 setup(
     name='sqlparse',
-    version=VERSION,
-    packages=packages,
-    description='Non-validating SQL parser',
+    version=get_version(),
     author='Andi Albrecht',
     author_email='albrecht.andi@gmail.com',
+    url='https://github.com/andialbrecht/sqlparse',
+    description='Non-validating SQL parser',
     long_description=LONG_DESCRIPTION,
     license='BSD',
-    url='https://github.com/andialbrecht/sqlparse',
     classifiers=[
         'Development Status :: 4 - Beta',
         'Intended Audience :: Developers',
@@ -107,7 +96,12 @@ setup(
         'Programming Language :: Python :: 3.4',
         'Programming Language :: Python :: 3.5',
         'Topic :: Database',
-        'Topic :: Software Development'
+        'Topic :: Software Development',
     ],
-    scripts=['bin/sqlformat'],
+    packages=find_packages(exclude=('tests',)),
+    entry_points={
+        'console_scripts': [
+            'sqlparse = sqlparse.__main__:main',
+        ]
+    },
 )
diff --git a/sqlparse/__main__.py b/sqlparse/__main__.py
new file mode 100644
index 0000000..28abb6c
--- /dev/null
+++ b/sqlparse/__main__.py
@@ -0,0 +1,144 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2016 Andi Albrecht, albrecht.andi@gmail.com
+#
+# This module is part of python-sqlparse and is released under
+# the BSD License: http://www.opensource.org/licenses/bsd-license.php
+
+import argparse
+import sys
+
+import sqlparse
+from sqlparse.compat import PY2
+from sqlparse.exceptions import SQLParseError
+
+_CASE_CHOICES = ['upper', 'lower', 'capitalize']
+
+# TODO: Add CLI Tests
+# TODO: Simplify formatter by using argparse `type` arguments
+parser = argparse.ArgumentParser(
+    prog='sqlparse',
+    description='Format FILE according to OPTIONS. Use "-" as FILE '
+                'to read from stdin.',
+    usage='%(prog)s  [OPTIONS] FILE, ...',
+    version=sqlparse.__version__,)
+
+parser.add_argument('filename')
+
+parser.add_argument(
+    '-o', '--outfile',
+    dest='outfile',
+    metavar='FILE',
+    help='write output to FILE (defaults to stdout)')
+
+group = parser.add_argument_group('Formatting Options')
+
+group.add_argument(
+    '-k', '--keywords',
+    metavar='CHOICE',
+    dest='keyword_case',
+    choices=_CASE_CHOICES,
+    help='change case of keywords, CHOICE is one of {0}'.format(
+        ', '.join('"{0}"'.format(x) for x in _CASE_CHOICES)))
+
+group.add_argument(
+    '-i', '--identifiers',
+    metavar='CHOICE',
+    dest='identifier_case',
+    choices=_CASE_CHOICES,
+    help='change case of identifiers, CHOICE is one of {0}'.format(
+        ', '.join('"{0}"'.format(x) for x in _CASE_CHOICES)))
+
+group.add_argument(
+    '-l', '--language',
+    metavar='LANG',
+    dest='output_format',
+    choices=['python', 'php'],
+    help='output a snippet in programming language LANG, '
+         'choices are "python", "php"')
+
+group.add_argument(
+    '--strip-comments',
+    dest='strip_comments',
+    action='store_true',
+    default=False,
+    help='remove comments')
+
+group.add_argument(
+    '-r', '--reindent',
+    dest='reindent',
+    action='store_true',
+    default=False,
+    help='reindent statements')
+
+group.add_argument(
+    '--indent_width',
+    dest='indent_width',
+    default=2,
+    type=int,
+    help='indentation width (defaults to 2 spaces)')
+
+group.add_argument(
+    '-a', '--reindent_aligned',
+    action='store_true',
+    default=False,
+    help='reindent statements to aligned format')
+
+group.add_argument(
+    '-s', '--use_space_around_operators',
+    action='store_true',
+    default=False,
+    help='place spaces around mathematical operators')
+
+group.add_argument(
+    '--wrap_after',
+    dest='wrap_after',
+    default=0,
+    type=int,
+    help='Column after which lists should be wrapped')
+
+
+def _error(msg):
+    """Print msg and optionally exit with return code exit_."""
+    sys.stderr.write('[ERROR] %s\n' % msg)
+
+
+def main(args=None):
+    args = parser.parse_args(args)
+
+    if args.filename == '-':  # read from stdin
+        data = sys.stdin.read()
+    else:
+        try:
+            data = ''.join(open(args.filename).readlines())
+        except IOError as e:
+            _error('Failed to read %s: %s' % (args.filename, e))
+            return 1
+
+    if args.outfile:
+        try:
+            stream = open(args.outfile, 'w')
+        except IOError as e:
+            _error('Failed to open %s: %s' % (args.outfile, e))
+            return 1
+    else:
+        stream = sys.stdout
+
+    formatter_opts = vars(args)
+    try:
+        formatter_opts = sqlparse.formatter.validate_options(formatter_opts)
+    except SQLParseError as e:
+        _error('Invalid options: %s' % e)
+        return 1
+
+    s = sqlparse.format(data, **formatter_opts)
+    if PY2:
+        s = s.encode('utf-8', 'replace')
+    stream.write(s)
+    stream.flush()
+    return 0
+
+
+if __name__ == '__main__':
+    sys.exit(main())
diff --git a/sqlparse/engine/grouping.py b/sqlparse/engine/grouping.py
index 91bb3d9..6e414b8 100644
--- a/sqlparse/engine/grouping.py
+++ b/sqlparse/engine/grouping.py
@@ -23,11 +23,15 @@ def _group_left_right(tlist, m, cls,
                       valid_right=lambda t: t is not None,
                       semicolon=False):
     """Groups together tokens that are joined by a middle token. ie. x < y"""
-    [_group_left_right(sgroup, m, cls, valid_left, valid_right, semicolon)
-     for sgroup in tlist.get_sublists() if not isinstance(sgroup, cls)]
 
-    token = tlist.token_next_by(m=m)
-    while token:
+    for token in list(tlist):
+        if token.is_group() and not isinstance(token, cls):
+            _group_left_right(token, m, cls, valid_left, valid_right,
+                              semicolon)
+
+        if not token.match(*m):
+            continue
+
         left, right = tlist.token_prev(token), tlist.token_next(token)
 
         if valid_left(left) and valid_right(right):
@@ -36,8 +40,7 @@ def _group_left_right(tlist, m, cls,
                 sright = tlist.token_next_by(m=M_SEMICOLON, idx=right)
                 right = sright or right
             tokens = tlist.tokens_between(left, right)
-            token = tlist.group_tokens(cls, tokens, extend=True)
-        token = tlist.token_next_by(m=m, idx=token)
+            tlist.group_tokens(cls, tokens, extend=True)
 
 
 def _group_matching(tlist, cls):
@@ -85,11 +88,12 @@ def group_assignment(tlist):
 
 
 def group_comparison(tlist):
-    I_COMPERABLE = (sql.Parenthesis, sql.Function, sql.Identifier)
+    I_COMPERABLE = (sql.Parenthesis, sql.Function, sql.Identifier,
+                    sql.Operation)
     T_COMPERABLE = T_NUMERICAL + T_STRING + T_NAME
 
-    func = lambda tk: imt(tk, t=T_COMPERABLE, i=I_COMPERABLE) or (
-        imt(tk, t=T.Keyword) and tk.value.upper() == 'NULL')
+    func = lambda tk: (imt(tk, t=T_COMPERABLE, i=I_COMPERABLE) or
+                       (tk and tk.is_keyword and tk.normalized == 'NULL'))
 
     _group_left_right(tlist, (T.Operator.Comparison, None), sql.Comparison,
                       valid_left=func, valid_right=func)
@@ -134,9 +138,9 @@ def group_arrays(tlist):
 @recurse(sql.Identifier)
 def group_operator(tlist):
     I_CYCLE = (sql.SquareBrackets, sql.Parenthesis, sql.Function,
-               sql.Identifier,)  # sql.Operation)
+               sql.Identifier, sql.Operation)
     # wilcards wouldn't have operations next to them
-    T_CYCLE = T_NUMERICAL + T_STRING + T_NAME  # + T.Wildcard
+    T_CYCLE = T_NUMERICAL + T_STRING + T_NAME
     func = lambda tk: imt(tk, i=I_CYCLE, t=T_CYCLE)
 
     token = tlist.token_next_by(t=(T.Operator, T.Wildcard))
@@ -146,8 +150,7 @@ def group_operator(tlist):
         if func(left) and func(right):
             token.ttype = T.Operator
             tokens = tlist.tokens_between(left, right)
-            # token = tlist.group_tokens(sql.Operation, tokens)
-            token = tlist.group_tokens(sql.Identifier, tokens)
+            token = tlist.group_tokens(sql.Operation, tokens)
 
         token = tlist.token_next_by(t=(T.Operator, T.Wildcard), idx=token)
 
@@ -155,7 +158,7 @@ def group_operator(tlist):
 @recurse(sql.IdentifierList)
 def group_identifier_list(tlist):
     I_IDENT_LIST = (sql.Function, sql.Case, sql.Identifier, sql.Comparison,
-                    sql.IdentifierList)  # sql.Operation
+                    sql.IdentifierList, sql.Operation)
     T_IDENT_LIST = (T_NUMERICAL + T_STRING + T_NAME +
                     (T.Keyword, T.Comment, T.Wildcard))
 
@@ -212,7 +215,7 @@ def group_where(tlist):
 @recurse()
 def group_aliased(tlist):
     I_ALIAS = (sql.Parenthesis, sql.Function, sql.Case, sql.Identifier,
-               )  # sql.Operation)
+               sql.Operation)
 
     token = tlist.token_next_by(i=I_ALIAS, t=T.Number)
     while token:
diff --git a/sqlparse/filters/others.py b/sqlparse/filters/others.py
index 6951c74..71b1f8e 100644
--- a/sqlparse/filters/others.py
+++ b/sqlparse/filters/others.py
@@ -6,7 +6,6 @@
 # the BSD License: http://www.opensource.org/licenses/bsd-license.php
 
 from sqlparse import sql, tokens as T
-from sqlparse.compat import text_type
 from sqlparse.utils import split_unquoted_newlines
 
 
@@ -114,6 +113,5 @@ class SpacesAroundOperatorsFilter(object):
 class SerializerUnicode(object):
     @staticmethod
     def process(stmt):
-        raw = text_type(stmt)
-        lines = split_unquoted_newlines(raw)
+        lines = split_unquoted_newlines(stmt)
         return '\n'.join(line.rstrip() for line in lines)
diff --git a/sqlparse/sql.py b/sqlparse/sql.py
index 43a89e7..cee6af5 100644
--- a/sqlparse/sql.py
+++ b/sqlparse/sql.py
@@ -37,6 +37,10 @@ class Token(object):
     def __str__(self):
         return self.value
 
+    # Pending tokenlist __len__ bug fix
+    # def __len__(self):
+    #     return len(self.value)
+
     def __repr__(self):
         cls = self._get_repr_name()
         value = self._get_repr_value()
@@ -141,6 +145,10 @@ class TokenList(Token):
     def __str__(self):
         return ''.join(token.value for token in self.flatten())
 
+    # weird bug
+    # def __len__(self):
+    #     return len(self.tokens)
+
     def __iter__(self):
         return iter(self.tokens)
 
@@ -152,12 +160,12 @@ class TokenList(Token):
 
     def _pprint_tree(self, max_depth=None, depth=0, f=None):
         """Pretty-print the object tree."""
-        ind = ' ' * (depth * 2)
+        indent = ' | ' * depth
         for idx, token in enumerate(self.tokens):
-            pre = ' +-' if token.is_group() else ' | '
             cls = token._get_repr_name()
             value = token._get_repr_value()
-            print("{ind}{pre}{idx} {cls} '{value}'".format(**locals()), file=f)
+            print("{indent}{idx:2d} {cls} '{value}'"
+                  .format(**locals()), file=f)
 
             if token.is_group() and (max_depth is None or depth < max_depth):
                 token._pprint_tree(max_depth, depth + 1, f)
@@ -216,20 +224,6 @@ class TokenList(Token):
                 if func(token):
                     return token
 
-    def token_first(self, skip_ws=True, skip_cm=False):
-        """Returns the first child token.
-
-        If *ignore_whitespace* is ``True`` (the default), whitespace
-        tokens are ignored.
-
-        if *ignore_comments* is ``True`` (default: ``False``), comments are
-        ignored too.
-        """
-        # this on is inconsistent, using Comment instead of T.Comment...
-        funcs = lambda tk: not ((skip_ws and tk.is_whitespace()) or
-                                (skip_cm and imt(tk, i=Comment)))
-        return self._token_matching(funcs)
-
     def token_next_by(self, i=None, m=None, t=None, idx=0, end=None):
         funcs = lambda tk: imt(tk, i, m, t)
         return self._token_matching(funcs, idx, end)
@@ -242,24 +236,26 @@ class TokenList(Token):
     def token_matching(self, idx, funcs):
         return self._token_matching(funcs, idx)
 
-    def token_prev(self, idx, skip_ws=True, skip_cm=False):
+    def token_prev(self, idx=0, skip_ws=True, skip_cm=False):
         """Returns the previous token relative to *idx*.
 
         If *skip_ws* is ``True`` (the default) whitespace tokens are ignored.
         ``None`` is returned if there's no previous token.
         """
         funcs = lambda tk: not ((skip_ws and tk.is_whitespace()) or
-                                (skip_cm and imt(tk, t=T.Comment)))
+                                (skip_cm and imt(tk, t=T.Comment, i=Comment)))
         return self._token_matching(funcs, idx, reverse=True)
 
-    def token_next(self, idx, skip_ws=True, skip_cm=False):
+    def token_next(self, idx=0, skip_ws=True, skip_cm=False):
         """Returns the next token relative to *idx*.
 
+        If called with idx = 0. Returns the first child token.
         If *skip_ws* is ``True`` (the default) whitespace tokens are ignored.
+        If *skip_cm* is ``True`` (default: ``False``), comments are ignored.
         ``None`` is returned if there's no next token.
         """
         funcs = lambda tk: not ((skip_ws and tk.is_whitespace()) or
-                                (skip_cm and imt(tk, t=T.Comment)))
+                                (skip_cm and imt(tk, t=T.Comment, i=Comment)))
         return self._token_matching(funcs, idx)
 
     def token_index(self, token, start=0):
@@ -387,7 +383,7 @@ class Statement(TokenList):
         Whitespaces and comments at the beginning of the statement
         are ignored.
         """
-        first_token = self.token_first(skip_cm=True)
+        first_token = self.token_next(skip_cm=True)
         if first_token is None:
             # An "empty" statement that either has not tokens at all
             # or only whitespace tokens.
@@ -425,7 +421,7 @@ class Identifier(TokenList):
     def get_typecast(self):
         """Returns the typecast or ``None`` of this object as a string."""
         marker = self.token_next_by(m=(T.Punctuation, '::'))
-        next_ = self.token_next(marker, False)
+        next_ = self.token_next(marker, skip_ws=False)
         return next_.value if next_ else None
 
     def get_ordering(self):
@@ -588,3 +584,7 @@ class Begin(TokenList):
     """A BEGIN/END block."""
     M_OPEN = T.Keyword, 'BEGIN'
     M_CLOSE = T.Keyword, 'END'
+
+
+class Operation(TokenList):
+    """Grouping of operations"""
diff --git a/sqlparse/utils.py b/sqlparse/utils.py
index 8253e0b..4a8646d 100644
--- a/sqlparse/utils.py
+++ b/sqlparse/utils.py
@@ -9,6 +9,7 @@ import itertools
 import re
 from collections import deque
 from contextlib import contextmanager
+from sqlparse.compat import text_type
 
 # This regular expression replaces the home-cooked parser that was here before.
 # It is much faster, but requires an extra post-processing step to get the
@@ -33,11 +34,12 @@ SPLIT_REGEX = re.compile(r"""
 LINE_MATCH = re.compile(r'(\r\n|\r|\n)')
 
 
-def split_unquoted_newlines(text):
+def split_unquoted_newlines(stmt):
     """Split a string on all unquoted newlines.
 
     Unlike str.splitlines(), this will ignore CR/LF/CR+LF if the requisite
     character is inside of a string."""
+    text = text_type(stmt)
     lines = SPLIT_REGEX.split(text)
     outputlines = ['']
     for line in lines:
diff --git a/tests/test_grouping.py b/tests/test_grouping.py
index 7ea1c75..147162f 100644
--- a/tests/test_grouping.py
+++ b/tests/test_grouping.py
@@ -106,15 +106,16 @@ class TestGrouping(TestCaseBase):
         self.assert_(isinstance(p.tokens[0].tokens[0], sql.Function))
         p = sqlparse.parse('foo()||col2 bar')[0]
         self.assert_(isinstance(p.tokens[0], sql.Identifier))
-        self.assert_(isinstance(p.tokens[0].tokens[0], sql.Function))
+        self.assert_(isinstance(p.tokens[0].tokens[0], sql.Operation))
+        self.assert_(isinstance(p.tokens[0].tokens[0].tokens[0], sql.Function))
 
     def test_identifier_extended(self):  # issue 15
         p = sqlparse.parse('foo+100')[0]
-        self.assert_(isinstance(p.tokens[0], sql.Identifier))
+        self.assert_(isinstance(p.tokens[0], sql.Operation))
         p = sqlparse.parse('foo + 100')[0]
-        self.assert_(isinstance(p.tokens[0], sql.Identifier))
+        self.assert_(isinstance(p.tokens[0], sql.Operation))
         p = sqlparse.parse('foo*100')[0]
-        self.assert_(isinstance(p.tokens[0], sql.Identifier))
+        self.assert_(isinstance(p.tokens[0], sql.Operation))
 
     def test_identifier_list(self):
         p = sqlparse.parse('a, b, c')[0]
@@ -267,25 +268,25 @@ class TestStatement(TestCaseBase):
 def test_identifier_with_operators():  # issue 53
     p = sqlparse.parse('foo||bar')[0]
     assert len(p.tokens) == 1
-    assert isinstance(p.tokens[0], sql.Identifier)
+    assert isinstance(p.tokens[0], sql.Operation)
     # again with whitespaces
     p = sqlparse.parse('foo || bar')[0]
     assert len(p.tokens) == 1
-    assert isinstance(p.tokens[0], sql.Identifier)
+    assert isinstance(p.tokens[0], sql.Operation)
 
 
 def test_identifier_with_op_trailing_ws():
     # make sure trailing whitespace isn't grouped with identifier
     p = sqlparse.parse('foo || bar ')[0]
     assert len(p.tokens) == 2
-    assert isinstance(p.tokens[0], sql.Identifier)
+    assert isinstance(p.tokens[0], sql.Operation)
     assert p.tokens[1].ttype is T.Whitespace
 
 
 def test_identifier_with_string_literals():
     p = sqlparse.parse('foo + \'bar\'')[0]
     assert len(p.tokens) == 1
-    assert isinstance(p.tokens[0], sql.Identifier)
+    assert isinstance(p.tokens[0], sql.Operation)
 
 
 # This test seems to be wrong. It was introduced when fixing #53, but #111
diff --git a/tests/test_parse.py b/tests/test_parse.py
index 8654ec4..0f5af04 100644
--- a/tests/test_parse.py
+++ b/tests/test_parse.py
@@ -83,6 +83,7 @@ class SQLParseTest(TestCaseBase):
     def test_placeholder(self):
         def _get_tokens(sql):
             return sqlparse.parse(sql)[0].tokens[-1].tokens
+
         t = _get_tokens('select * from foo where user = ?')
         self.assert_(t[-1].ttype is sqlparse.tokens.Name.Placeholder)
         self.assertEqual(t[-1].value, '?')
@@ -218,7 +219,7 @@ def test_single_quotes_with_linebreaks():  # issue118
 def test_sqlite_identifiers():
     # Make sure we still parse sqlite style escapes
     p = sqlparse.parse('[col1],[col2]')[0].tokens
-    id_names = [id.get_name() for id in p[0].get_identifiers()]
+    id_names = [id_.get_name() for id_ in p[0].get_identifiers()]
     assert len(p) == 1
     assert isinstance(p[0], sqlparse.sql.IdentifierList)
     assert id_names == ['[col1]', '[col2]']
@@ -318,21 +319,65 @@ def test_get_token_at_offset():
 
 
 def test_pprint():
-    p = sqlparse.parse('select * from dual')[0]
+    p = sqlparse.parse('select a0, b0, c0, d0, e0 from '
+                       '(select * from dual) q0 where 1=1 and 2=2')[0]
     output = StringIO()
 
     p._pprint_tree(f=output)
-    pprint = u'\n'.join([
-        " | 0 DML 'select'",
-        " | 1 Whitespace ' '",
-        " | 2 Wildcard '*'",
-        " | 3 Whitespace ' '",
-        " | 4 Keyword 'from'",
-        " | 5 Whitespace ' '",
-        " +-6 Identifier 'dual'",
-        "   | 0 Name 'dual'",
-        "",
-    ])
+    pprint = u'\n'.join([" 0 DML 'select'",
+                         " 1 Whitespace ' '",
+                         " 2 IdentifierList 'a0, b0...'",
+                         " |  0 Identifier 'a0'",
+                         " |  |  0 Name 'a0'",
+                         " |  1 Punctuation ','",
+                         " |  2 Whitespace ' '",
+                         " |  3 Identifier 'b0'",
+                         " |  |  0 Name 'b0'",
+                         " |  4 Punctuation ','",
+                         " |  5 Whitespace ' '",
+                         " |  6 Identifier 'c0'",
+                         " |  |  0 Name 'c0'",
+                         " |  7 Punctuation ','",
+                         " |  8 Whitespace ' '",
+                         " |  9 Identifier 'd0'",
+                         " |  |  0 Name 'd0'",
+                         " | 10 Punctuation ','",
+                         " | 11 Whitespace ' '",
+                         " | 12 Float 'e0'",
+                         " 3 Whitespace ' '",
+                         " 4 Keyword 'from'",
+                         " 5 Whitespace ' '",
+                         " 6 Identifier '(selec...'",
+                         " |  0 Parenthesis '(selec...'",
+                         " |  |  0 Punctuation '('",
+                         " |  |  1 DML 'select'",
+                         " |  |  2 Whitespace ' '",
+                         " |  |  3 Wildcard '*'",
+                         " |  |  4 Whitespace ' '",
+                         " |  |  5 Keyword 'from'",
+                         " |  |  6 Whitespace ' '",
+                         " |  |  7 Identifier 'dual'",
+                         " |  |  |  0 Name 'dual'",
+                         " |  |  8 Punctuation ')'",
+                         " |  1 Whitespace ' '",
+                         " |  2 Identifier 'q0'",
+                         " |  |  0 Name 'q0'",
+                         " 7 Whitespace ' '",
+                         " 8 Where 'where ...'",
+                         " |  0 Keyword 'where'",
+                         " |  1 Whitespace ' '",
+                         " |  2 Comparison '1=1'",
+                         " |  |  0 Integer '1'",
+                         " |  |  1 Comparison '='",
+                         " |  |  2 Integer '1'",
+                         " |  3 Whitespace ' '",
+                         " |  4 Keyword 'and'",
+                         " |  5 Whitespace ' '",
+                         " |  6 Comparison '2=2'",
+                         " |  |  0 Integer '2'",
+                         " |  |  1 Comparison '='",
+                         " |  |  2 Integer '2'",
+                         "", ])
     assert output.getvalue() == pprint
 
 
diff --git a/tests/test_regressions.py b/tests/test_regressions.py
index 4eb1621..616c321 100644
--- a/tests/test_regressions.py
+++ b/tests/test_regressions.py
@@ -48,7 +48,7 @@ class RegressionTests(TestCaseBase):
         self.assert_(p.tokens[0].ttype is T.Comment.Single)
 
     def test_issue34(self):
-        t = sqlparse.parse("create")[0].token_first()
+        t = sqlparse.parse("create")[0].token_next()
         self.assertEqual(t.match(T.Keyword.DDL, "create"), True)
         self.assertEqual(t.match(T.Keyword.DDL, "CREATE"), True)
 
diff --git a/tests/test_tokenize.py b/tests/test_tokenize.py
index adfd1ea..7200682 100644
--- a/tests/test_tokenize.py
+++ b/tests/test_tokenize.py
@@ -104,10 +104,10 @@ class TestTokenList(unittest.TestCase):
 
     def test_token_first(self):
         p = sqlparse.parse(' select foo')[0]
-        first = p.token_first()
+        first = p.token_next()
         self.assertEqual(first.value, 'select')
-        self.assertEqual(p.token_first(skip_ws=False).value, ' ')
-        self.assertEqual(sql.TokenList([]).token_first(), None)
+        self.assertEqual(p.token_next(skip_ws=False).value, ' ')
+        self.assertEqual(sql.TokenList([]).token_next(), None)
 
     def test_token_matching(self):
         t1 = sql.Token(T.Keyword, 'foo')
diff --git a/tests/utils.py b/tests/utils.py
index b255035..8000ae5 100644
--- a/tests/utils.py
+++ b/tests/utils.py
@@ -8,7 +8,7 @@ import os
 import unittest
 
 from sqlparse.utils import split_unquoted_newlines
-from sqlparse.compat import u, StringIO
+from sqlparse.compat import StringIO
 
 DIR_PATH = os.path.dirname(__file__)
 FILES_DIR = os.path.join(DIR_PATH, 'files')
@@ -30,8 +30,8 @@ class TestCaseBase(unittest.TestCase):
             # Using the built-in .splitlines() method here will cause incorrect
             # results when splitting statements that have quoted CR/CR+LF
             # characters.
-            sfirst = split_unquoted_newlines(u(first))
-            ssecond = split_unquoted_newlines(u(second))
+            sfirst = split_unquoted_newlines(first)
+            ssecond = split_unquoted_newlines(second)
             diff = difflib.ndiff(sfirst, ssecond)
 
             fp = StringIO()
diff --git a/tox.ini b/tox.ini
index 6c27e70..87795ab 100644
--- a/tox.ini
+++ b/tox.ini
@@ -17,8 +17,6 @@ deps =
 passenv =
     TRAVIS
 commands =
-    python --version
-    sqlformat --version
     py.test --cov=sqlparse
 
 [testenv:flake8]