An overhaul of expression handling for special symbols particularly

with conjunctions, e.g.
``None`` :func:`.expression.null` :func:`.expression.true`
:func:`.expression.false`, including consistency in rendering NULL
in conjunctions, "short-circuiting" of :func:`.and_` and :func:`.or_`
expressions which contain boolean constants, and rendering of
boolean constants and expressions as compared to "1" or "0" for backends
that don't feature ``true``/``false`` constants. [ticket:2804]

7 files changed, 277 insertions, 87 deletions

diff --git a/lib/sqlalchemy/sql/__init__.py b/lib/sqlalchemy/sql/__init__.py
index e1497e9fa..c6ecb8afd 100644
--- a/lib/sqlalchemy/sql/__init__.py
+++ b/lib/sqlalchemy/sql/__init__.py
@@ -35,6 +35,7 @@ from .expression import (
     exists,
     extract,
     false,
+    False_,
     func,
     insert,
     intersect,
@@ -55,6 +56,7 @@ from .expression import (
     table,
     text,
     true,
+    True_,
     tuple_,
     type_coerce,
     union,
diff --git a/lib/sqlalchemy/sql/compiler.py b/lib/sqlalchemy/sql/compiler.py
index f526203ac..2bf7d3f4a 100644
--- a/lib/sqlalchemy/sql/compiler.py
+++ b/lib/sqlalchemy/sql/compiler.py
@@ -113,6 +113,7 @@ OPERATORS = {
     operators.asc_op: ' ASC',
     operators.nullsfirst_op: ' NULLS FIRST',
     operators.nullslast_op: ' NULLS LAST',
+
 }
 
 FUNCTIONS = {
@@ -608,10 +609,16 @@ class SQLCompiler(Compiled):
         return 'NULL'
 
     def visit_true(self, expr, **kw):
-        return 'true'
+        if self.dialect.supports_native_boolean:
+            return 'true'
+        else:
+            return "1"
 
     def visit_false(self, expr, **kw):
-        return 'false'
+        if self.dialect.supports_native_boolean:
+            return 'false'
+        else:
+            return "0"
 
     def visit_clauselist(self, clauselist, order_by_select=None, **kw):
         if order_by_select is not None:
@@ -783,6 +790,18 @@ class SQLCompiler(Compiled):
             raise exc.CompileError(
                             "Unary expression has no operator or modifier")
 
+    def visit_istrue_unary_operator(self, element, operator, **kw):
+        if self.dialect.supports_native_boolean:
+            return self.process(element.element, **kw)
+        else:
+            return "%s = 1" % self.process(element.element, **kw)
+
+    def visit_isfalse_unary_operator(self, element, operator, **kw):
+        if self.dialect.supports_native_boolean:
+            return "NOT %s" % self.process(element.element, **kw)
+        else:
+            return "%s = 0" % self.process(element.element, **kw)
+
     def visit_binary(self, binary, **kw):
         # don't allow "? = ?" to render
         if self.ansi_bind_rules and \
diff --git a/lib/sqlalchemy/sql/default_comparator.py b/lib/sqlalchemy/sql/default_comparator.py
index 7c803ac4c..423a5b56d 100644
--- a/lib/sqlalchemy/sql/default_comparator.py
+++ b/lib/sqlalchemy/sql/default_comparator.py
@@ -13,7 +13,7 @@ from . import type_api
 from .elements import BindParameter, True_, False_, BinaryExpression, \
         Null, _const_expr, _clause_element_as_expr, \
         ClauseList, ColumnElement, TextClause, UnaryExpression, \
-        collate, _is_literal
+        collate, _is_literal, _literal_as_text
 from .selectable import SelectBase, Alias, Selectable, ScalarSelect
 
 class _DefaultColumnComparator(operators.ColumnOperators):
@@ -75,7 +75,7 @@ class _DefaultColumnComparator(operators.ColumnOperators):
             if op in (operators.eq, operators.ne) and \
                     isinstance(obj, (bool, True_, False_)):
                 return BinaryExpression(expr,
-                                obj,
+                                _literal_as_text(obj),
                                 op,
                                 type_=type_api.BOOLEANTYPE,
                                 negate=negate, modifiers=kwargs)
@@ -209,7 +209,7 @@ class _DefaultColumnComparator(operators.ColumnOperators):
                     self._check_literal(expr, operators.and_, cleft),
                     self._check_literal(expr, operators.and_, cright),
                     operator=operators.and_,
-                    group=False),
+                    group=False, group_contents=False),
                 operators.between_op)
 
     def _collate_impl(self, expr, op, other, **kw):
diff --git a/lib/sqlalchemy/sql/elements.py b/lib/sqlalchemy/sql/elements.py
index 251102d59..e9b995eaa 100644
--- a/lib/sqlalchemy/sql/elements.py
+++ b/lib/sqlalchemy/sql/elements.py
@@ -149,30 +149,6 @@ def outparam(key, type_=None):
                 key, None, type_=type_, unique=False, isoutparam=True)
 
 
-def and_(*clauses):
-    """Join a list of clauses together using the ``AND`` operator.
-
-    The ``&`` operator is also overloaded on all :class:`.ColumnElement`
-    subclasses to produce the
-    same result.
-
-    """
-    if len(clauses) == 1:
-        return clauses[0]
-    return BooleanClauseList(operator=operators.and_, *clauses)
-
-
-def or_(*clauses):
-    """Join a list of clauses together using the ``OR`` operator.
-
-    The ``|`` operator is also overloaded on all
-    :class:`.ColumnElement` subclasses to produce the
-    same result.
-
-    """
-    if len(clauses) == 1:
-        return clauses[0]
-    return BooleanClauseList(operator=operators.or_, *clauses)
 
 
 def not_(clause):
@@ -465,7 +441,10 @@ class ClauseElement(Visitable):
         return or_(self, other)
 
     def __invert__(self):
-        return self._negate()
+        if hasattr(self, 'negation_clause'):
+            return self.negation_clause
+        else:
+            return self._negate()
 
     def __bool__(self):
         raise TypeError("Boolean value of this clause is not defined")
@@ -473,13 +452,10 @@ class ClauseElement(Visitable):
     __nonzero__ = __bool__
 
     def _negate(self):
-        if hasattr(self, 'negation_clause'):
-            return self.negation_clause
-        else:
-            return UnaryExpression(
-                        self.self_group(against=operators.inv),
-                        operator=operators.inv,
-                        negate=None)
+        return UnaryExpression(
+                    self.self_group(against=operators.inv),
+                    operator=operators.inv,
+                    negate=None)
 
     def __repr__(self):
         friendly = getattr(self, 'description', None)
@@ -537,6 +513,19 @@ class ColumnElement(ClauseElement, operators.ColumnOperators):
     _key_label = None
     _alt_names = ()
 
+    def self_group(self, against=None):
+        if against in (operators.and_, operators.or_, operators._asbool) and \
+            self.type._type_affinity is type_api.BOOLEANTYPE._type_affinity:
+            return AsBoolean(self, operators.istrue, operators.isfalse)
+        else:
+            return self
+
+    def _negate(self):
+        if self.type._type_affinity is type_api.BOOLEANTYPE._type_affinity:
+            return AsBoolean(self, operators.isfalse, operators.istrue)
+        else:
+            return super(ColumnElement, self)._negate()
+
     @util.memoized_property
     def type(self):
         return type_api.NULLTYPE
@@ -1062,52 +1051,153 @@ class TextClause(Executable, ClauseElement):
 class Null(ColumnElement):
     """Represent the NULL keyword in a SQL statement.
 
+    :class:`.Null` is accessed as a constant via the
+    :func:`.null` function.
+
     """
 
     __visit_name__ = 'null'
 
-    def __init__(self):
-        """Return a :class:`Null` object, which compiles to ``NULL``.
+    @util.memoized_property
+    def type(self):
+        return type_api.NULLTYPE
 
-        """
-        self.type = type_api.NULLTYPE
+    @classmethod
+    def _singleton(cls):
+        """Return a constant :class:`.Null` construct."""
+
+        return NULL
 
     def compare(self, other):
         return isinstance(other, Null)
 
 
 class False_(ColumnElement):
-    """Represent the ``false`` keyword in a SQL statement.
+    """Represent the ``false`` keyword, or equivalent, in a SQL statement.
+
+    :class:`.False_` is accessed as a constant via the
+    :func:`.false` function.
 
     """
 
     __visit_name__ = 'false'
 
-    def __init__(self):
-        """Return a :class:`False_` object.
+    @util.memoized_property
+    def type(self):
+        return type_api.BOOLEANTYPE
+
+    def _negate(self):
+        return TRUE
+
+    @classmethod
+    def _singleton(cls):
+        """Return a constant :class:`.False_` construct.
+
+        E.g.::
+
+            >>> from sqlalchemy import false
+            >>> print select([t.c.x]).where(false())
+            SELECT x FROM t WHERE false
+
+        A backend which does not support true/false constants will render as
+        an expression against 1 or 0::
+
+            >>> print select([t.c.x]).where(false())
+            SELECT x FROM t WHERE 0 = 1
+
+        The :func:`.true` and :func:`.false` constants also feature
+        "short circuit" operation within an :func:`.and_` or :func:`.or_`
+        conjunction::
+
+            >>> print select([t.c.x]).where(or_(t.c.x > 5, true()))
+            SELECT x FROM t WHERE true
+
+            >>> print select([t.c.x]).where(and_(t.c.x > 5, false()))
+            SELECT x FROM t WHERE false
+
+        .. versionchanged:: 0.9 :func:`.true` and :func:`.false` feature
+           better integrated behavior within conjunctions and on dialects
+           that don't support true/false constants.
+
+        .. seealso::
+
+            :func:`.true`
 
         """
-        self.type = type_api.BOOLEANTYPE
+
+        return FALSE
 
     def compare(self, other):
         return isinstance(other, False_)
 
 class True_(ColumnElement):
-    """Represent the ``true`` keyword in a SQL statement.
+    """Represent the ``true`` keyword, or equivalent, in a SQL statement.
+
+    :class:`.True_` is accessed as a constant via the
+    :func:`.true` function.
 
     """
 
     __visit_name__ = 'true'
 
-    def __init__(self):
-        """Return a :class:`True_` object.
+    @util.memoized_property
+    def type(self):
+        return type_api.BOOLEANTYPE
+
+    def _negate(self):
+        return FALSE
+
+    @classmethod
+    def _ifnone(cls, other):
+        if other is None:
+            return cls._singleton()
+        else:
+            return other
+
+    @classmethod
+    def _singleton(cls):
+        """Return a constant :class:`.True_` construct.
+
+        E.g.::
+
+            >>> from sqlalchemy import true
+            >>> print select([t.c.x]).where(true())
+            SELECT x FROM t WHERE true
+
+        A backend which does not support true/false constants will render as
+        an expression against 1 or 0::
+
+            >>> print select([t.c.x]).where(true())
+            SELECT x FROM t WHERE 1 = 1
+
+        The :func:`.true` and :func:`.false` constants also feature
+        "short circuit" operation within an :func:`.and_` or :func:`.or_`
+        conjunction::
+
+            >>> print select([t.c.x]).where(or_(t.c.x > 5, true()))
+            SELECT x FROM t WHERE true
+
+            >>> print select([t.c.x]).where(and_(t.c.x > 5, false()))
+            SELECT x FROM t WHERE false
+
+        .. versionchanged:: 0.9 :func:`.true` and :func:`.false` feature
+           better integrated behavior within conjunctions and on dialects
+           that don't support true/false constants.
+
+        .. seealso::
+
+            :func:`.false`
 
         """
-        self.type = type_api.BOOLEANTYPE
+
+        return TRUE
 
     def compare(self, other):
         return isinstance(other, True_)
 
+NULL = Null()
+FALSE = False_()
+TRUE = True_()
 
 class ClauseList(ClauseElement):
     """Describe a list of clauses, separated by an operator.
@@ -1124,11 +1214,11 @@ class ClauseList(ClauseElement):
         if self.group_contents:
             self.clauses = [
                 _literal_as_text(clause).self_group(against=self.operator)
-                for clause in clauses if clause is not None]
+                for clause in clauses]
         else:
             self.clauses = [
                 _literal_as_text(clause)
-                for clause in clauses if clause is not None]
+                for clause in clauses]
 
     def __iter__(self):
         return iter(self.clauses)
@@ -1141,10 +1231,6 @@ class ClauseList(ClauseElement):
         return iter(self)
 
     def append(self, clause):
-        # TODO: not sure if i like the 'group_contents' flag.  need to
-        # define the difference between a ClauseList of ClauseLists,
-        # and a "flattened" ClauseList of ClauseLists.  flatten()
-        # method ?
         if self.group_contents:
             self.clauses.append(_literal_as_text(clause).\
                                 self_group(against=self.operator))
@@ -1185,13 +1271,65 @@ class ClauseList(ClauseElement):
             return False
 
 
+
 class BooleanClauseList(ClauseList, ColumnElement):
     __visit_name__ = 'clauselist'
 
-    def __init__(self, *clauses, **kwargs):
-        super(BooleanClauseList, self).__init__(*clauses, **kwargs)
-        self.type = type_api.to_instance(kwargs.get('type_',
-                type_api.BOOLEANTYPE))
+    def __init__(self, *arg, **kw):
+        raise NotImplementedError(
+                "BooleanClauseList has a private constructor")
+
+    @classmethod
+    def _construct(cls, operator, continue_on, skip_on, *clauses, **kw):
+        convert_clauses = []
+
+        for clause in clauses:
+            clause = _literal_as_text(clause)
+
+            if isinstance(clause, continue_on):
+                continue
+            elif isinstance(clause, skip_on):
+                return clause.self_group(against=operators._asbool)
+
+            convert_clauses.append(clause)
+
+        if len(convert_clauses) == 1:
+            return convert_clauses[0].self_group(against=operators._asbool)
+        elif not convert_clauses and clauses:
+            return clauses[0].self_group(against=operators._asbool)
+
+        convert_clauses = [c.self_group(against=operator)
+                                for c in convert_clauses]
+
+        self = cls.__new__(cls)
+        self.clauses = convert_clauses
+        self.group = True
+        self.operator = operator
+        self.group_contents = True
+        self.type = type_api.BOOLEANTYPE
+        return self
+
+    @classmethod
+    def and_(cls, *clauses):
+        """Join a list of clauses together using the ``AND`` operator.
+
+        The ``&`` operator is also overloaded on all :class:`.ColumnElement`
+        subclasses to produce the
+        same result.
+
+        """
+        return cls._construct(operators.and_, True_, False_, *clauses)
+
+    @classmethod
+    def or_(cls, *clauses):
+        """Join a list of clauses together using the ``OR`` operator.
+
+        The ``|`` operator is also overloaded on all
+        :class:`.ColumnElement` subclasses to produce the
+        same result.
+
+        """
+        return cls._construct(operators.or_, False_, True_, *clauses)
 
     @property
     def _select_iterable(self):
@@ -1203,6 +1341,12 @@ class BooleanClauseList(ClauseList, ColumnElement):
         else:
             return super(BooleanClauseList, self).self_group(against=against)
 
+    def _negate(self):
+        return ClauseList._negate(self)
+
+
+and_ = BooleanClauseList.and_
+or_ = BooleanClauseList.or_
 
 class Tuple(ClauseList, ColumnElement):
     """Represent a SQL tuple."""
@@ -1465,9 +1609,7 @@ class UnaryExpression(ColumnElement):
                             type_=None, negate=None):
         self.operator = operator
         self.modifier = modifier
-
-        self.element = _literal_as_text(element).\
-                    self_group(against=self.operator or self.modifier)
+        self.element = element.self_group(against=self.operator or self.modifier)
         self.type = type_api.to_instance(type_)
         self.negate = negate
 
@@ -1484,7 +1626,8 @@ class UnaryExpression(ColumnElement):
           ORDER BY mycol DESC NULLS FIRST
 
         """
-        return UnaryExpression(column, modifier=operators.nullsfirst_op)
+        return UnaryExpression(
+                _literal_as_text(column), modifier=operators.nullsfirst_op)
 
 
     @classmethod
@@ -1500,7 +1643,8 @@ class UnaryExpression(ColumnElement):
             ORDER BY mycol DESC NULLS LAST
 
         """
-        return UnaryExpression(column, modifier=operators.nullslast_op)
+        return UnaryExpression(
+            _literal_as_text(column), modifier=operators.nullslast_op)
 
 
     @classmethod
@@ -1516,7 +1660,8 @@ class UnaryExpression(ColumnElement):
             ORDER BY mycol DESC
 
         """
-        return UnaryExpression(column, modifier=operators.desc_op)
+        return UnaryExpression(
+            _literal_as_text(column), modifier=operators.desc_op)
 
     @classmethod
     def _create_asc(cls, column):
@@ -1531,7 +1676,8 @@ class UnaryExpression(ColumnElement):
           ORDER BY mycol ASC
 
         """
-        return UnaryExpression(column, modifier=operators.asc_op)
+        return UnaryExpression(
+            _literal_as_text(column), modifier=operators.asc_op)
 
     @classmethod
     def _create_distinct(cls, expr):
@@ -1587,16 +1733,31 @@ class UnaryExpression(ColumnElement):
                 modifier=self.modifier,
                 type_=self.type)
         else:
-            return super(UnaryExpression, self)._negate()
+            return ClauseElement._negate(self)
 
     def self_group(self, against=None):
-        if self.operator and operators.is_precedent(self.operator,
-                against):
+        if self.operator and operators.is_precedent(self.operator, against):
             return Grouping(self)
         else:
             return self
 
 
+class AsBoolean(UnaryExpression):
+
+    def __init__(self, element, operator, negate):
+        self.element = element
+        self.type = type_api.BOOLEANTYPE
+        self.operator = operator
+        self.negate = negate
+        self.modifier = None
+
+    def self_group(self, against=None):
+        return self
+
+    def _negate(self):
+        return self.element._negate()
+
+
 class BinaryExpression(ColumnElement):
     """Represent an expression that is ``LEFT <operator> RIGHT``.
 
@@ -1620,8 +1781,8 @@ class BinaryExpression(ColumnElement):
         if isinstance(operator, util.string_types):
             operator = operators.custom_op(operator)
         self._orig = (left, right)
-        self.left = _literal_as_text(left).self_group(against=operator)
-        self.right = _literal_as_text(right).self_group(against=operator)
+        self.left = left.self_group(against=operator)
+        self.right = right.self_group(against=operator)
         self.operator = operator
         self.type = type_api.to_instance(type_)
         self.negate = negate
@@ -1702,6 +1863,9 @@ class Grouping(ColumnElement):
         self.element = element
         self.type = getattr(element, 'type', type_api.NULLTYPE)
 
+    def self_group(self, against=None):
+        return self
+
     @property
     def _label(self):
         return getattr(self.element, '_label', None) or self.anon_label
diff --git a/lib/sqlalchemy/sql/expression.py b/lib/sqlalchemy/sql/expression.py
index 01091bc0a..6be32f454 100644
--- a/lib/sqlalchemy/sql/expression.py
+++ b/lib/sqlalchemy/sql/expression.py
@@ -33,7 +33,7 @@ from .elements import ClauseElement, ColumnElement,\
   BindParameter, UnaryExpression, BooleanClauseList, \
   Label, Cast, Case, ColumnClause, TextClause, Over, Null, \
   True_, False_, BinaryExpression, Tuple, TypeClause, Extract, \
-  Grouping, and_, or_, not_, \
+  Grouping, not_, \
   collate, literal_column, between,\
   literal, outparam, type_coerce, ClauseList
 
@@ -56,6 +56,8 @@ from .dml import Insert, Update, Delete, UpdateBase, ValuesBase
 # the functions to be available in the sqlalchemy.sql.* namespace and
 # to be auto-cross-documenting from the function to the class itself.
 
+and_ = public_factory(BooleanClauseList.and_, ".expression.and_")
+or_ = public_factory(BooleanClauseList.or_, ".expression.or_")
 bindparam = public_factory(BindParameter, ".expression.bindparam")
 select = public_factory(Select, ".expression.select")
 text = public_factory(TextClause, ".expression.tet")
@@ -79,9 +81,9 @@ nullslast = public_factory(UnaryExpression._create_nullslast, ".expression.nulls
 asc = public_factory(UnaryExpression._create_asc, ".expression.asc")
 desc = public_factory(UnaryExpression._create_desc, ".expression.desc")
 distinct = public_factory(UnaryExpression._create_distinct, ".expression.distinct")
-true = public_factory(True_, ".expression.true")
-false = public_factory(False_, ".expression.false")
-null = public_factory(Null, ".expression.null")
+true = public_factory(True_._singleton, ".expression.true")
+false = public_factory(False_._singleton, ".expression.false")
+null = public_factory(Null._singleton, ".expression.null")
 join = public_factory(Join._create_join, ".expression.join")
 outerjoin = public_factory(Join._create_outerjoin, ".expression.outerjoin")
 insert = public_factory(Insert, ".expression.insert")
@@ -89,7 +91,6 @@ update = public_factory(Update, ".expression.update")
 delete = public_factory(Delete, ".expression.delete")
 
 
-
 # internal functions still being called from tests and the ORM,
 # these might be better off in some other namespace
 from .base import _from_objects
diff --git a/lib/sqlalchemy/sql/operators.py b/lib/sqlalchemy/sql/operators.py
index 128442158..e9b904d7c 100644
--- a/lib/sqlalchemy/sql/operators.py
+++ b/lib/sqlalchemy/sql/operators.py
@@ -654,6 +654,12 @@ def exists():
     raise NotImplementedError()
 
 
+def istrue(a):
+    raise NotImplementedError()
+
+def isfalse(a):
+    raise NotImplementedError()
+
 def is_(a, b):
     return a.is_(b)
 
@@ -779,6 +785,7 @@ parenthesize (a op b).
 
 """
 
+_asbool = util.symbol('_asbool', canonical=-10)
 _smallest = util.symbol('_smallest', canonical=-100)
 _largest = util.symbol('_largest', canonical=100)
 
@@ -816,12 +823,15 @@ _PRECEDENCE = {
     between_op: 5,
     distinct_op: 5,
     inv: 5,
+    istrue: 5,
+    isfalse: 5,
     and_: 3,
     or_: 2,
     comma_op: -1,
     collate: 7,
     as_: -1,
     exists: 0,
+    _asbool: -10,
     _smallest: _smallest,
     _largest: _largest
 }
diff --git a/lib/sqlalchemy/sql/selectable.py b/lib/sqlalchemy/sql/selectable.py
index 43d5a084c..550e250f1 100644
--- a/lib/sqlalchemy/sql/selectable.py
+++ b/lib/sqlalchemy/sql/selectable.py
@@ -14,7 +14,7 @@ from .elements import ClauseElement, TextClause, ClauseList, \
 from .elements import _clone, \
         _literal_as_text, _interpret_as_column_or_from, _expand_cloned,\
         _select_iterables, _anonymous_label, _clause_element_as_expr,\
-        _cloned_intersection, _cloned_difference
+        _cloned_intersection, _cloned_difference, True_
 from .base import Immutable, Executable, _generative, \
             ColumnCollection, ColumnSet, _from_objects, Generative
 from . import type_api
@@ -2519,13 +2519,9 @@ class Select(HasPrefixes, SelectBase):
         :term:`method chaining`.
 
         """
-        self._reset_exported()
-        whereclause = _literal_as_text(whereclause)
 
-        if self._whereclause is not None:
-            self._whereclause = and_(self._whereclause, whereclause)
-        else:
-            self._whereclause = whereclause
+        self._reset_exported()
+        self._whereclause = and_(True_._ifnone(self._whereclause), whereclause)
 
     def append_having(self, having):
         """append the given expression to this select() construct's HAVING
@@ -2538,10 +2534,8 @@ class Select(HasPrefixes, SelectBase):
         :term:`method chaining`.
 
         """
-        if self._having is not None:
-            self._having = and_(self._having, _literal_as_text(having))
-        else:
-            self._having = _literal_as_text(having)
+        self._reset_exported()
+        self._having = and_(True_._ifnone(self._having), having)
 
     def append_from(self, fromclause):
         """append the given FromClause expression to this select() construct's