Merge branch 'master' of bitbucket.org:zzzeek/sqlalchemy

8 files changed, 202 insertions, 66 deletions

diff --git a/lib/sqlalchemy/sql/compiler.py b/lib/sqlalchemy/sql/compiler.py
index 755193552..e9c3d0efa 100644
--- a/lib/sqlalchemy/sql/compiler.py
+++ b/lib/sqlalchemy/sql/compiler.py
@@ -1133,7 +1133,7 @@ class SQLCompiler(Compiled):
 
         anonname = name.apply_map(self.anon_map)
 
-        if len(anonname) > self.label_length:
+        if len(anonname) > self.label_length - 6:
             counter = self.truncated_names.get(ident_class, 1)
             truncname = anonname[0:max(self.label_length - 6, 0)] + \
                 "_" + hex(counter)[2:]
@@ -1324,10 +1324,17 @@ class SQLCompiler(Compiled):
             result_expr = _CompileLabel(col_expr,
                                         elements._as_truncated(column.name),
                                         alt_names=(column.key,))
-        elif not isinstance(column,
-                            (elements.UnaryExpression, elements.TextClause)) \
-                and (not hasattr(column, 'name') or
-                     isinstance(column, functions.Function)):
+        elif (
+            not isinstance(column, elements.TextClause) and
+            (
+                not isinstance(column, elements.UnaryExpression) or
+                column.wraps_column_expression
+            ) and
+            (
+                not hasattr(column, 'name') or
+                isinstance(column, functions.Function)
+            )
+        ):
             result_expr = _CompileLabel(col_expr, column.anon_label)
         elif col_expr is not column:
             # TODO: are we sure "column" has a .name and .key here ?
@@ -1528,6 +1535,12 @@ class SQLCompiler(Compiled):
                     'need_result_map_for_compound', False)
             ) or entry.get('need_result_map_for_nested', False)
 
+        # this was first proposed as part of #3372; however, it is not
+        # reached in current tests and could possibly be an assertion
+        # instead.
+        if not populate_result_map and 'add_to_result_map' in kwargs:
+            del kwargs['add_to_result_map']
+
         if needs_nested_translation:
             if populate_result_map:
                 self._transform_result_map_for_nested_joins(
@@ -1555,7 +1568,7 @@ class SQLCompiler(Compiled):
             text += self._generate_prefixes(
                 select, select._prefixes, **kwargs)
 
-        text += self.get_select_precolumns(select)
+        text += self.get_select_precolumns(select, **kwargs)
 
         # the actual list of columns to print in the SELECT column list.
         inner_columns = [
@@ -1600,7 +1613,7 @@ class SQLCompiler(Compiled):
             if per_dialect:
                 text += " " + self.get_statement_hint_text(per_dialect)
 
-        if self.ctes and toplevel:
+        if self.ctes and self._is_toplevel_select(select):
             text = self._render_cte_clause() + text
 
         if select._suffixes:
@@ -1614,6 +1627,20 @@ class SQLCompiler(Compiled):
         else:
             return text
 
+    def _is_toplevel_select(self, select):
+        """Return True if the stack is placed at the given select, and
+        is also the outermost SELECT, meaning there is either no stack
+        before this one, or the enclosing stack is a topmost INSERT.
+
+        """
+        return (
+            self.stack[-1]['selectable'] is select and
+            (
+                len(self.stack) == 1 or self.isinsert and len(self.stack) == 2
+                and self.statement is self.stack[0]['selectable']
+            )
+        )
+
     def _setup_select_hints(self, select):
         byfrom = dict([
             (from_, hinttext % {
@@ -1729,7 +1756,7 @@ class SQLCompiler(Compiled):
         else:
             return "WITH"
 
-    def get_select_precolumns(self, select):
+    def get_select_precolumns(self, select, **kw):
         """Called when building a ``SELECT`` statement, position is just
         before column list.
 
diff --git a/lib/sqlalchemy/sql/ddl.py b/lib/sqlalchemy/sql/ddl.py
index 3834f25f4..71018f132 100644
--- a/lib/sqlalchemy/sql/ddl.py
+++ b/lib/sqlalchemy/sql/ddl.py
@@ -711,8 +711,11 @@ class SchemaGenerator(DDLBase):
         seq_coll = [s for s in metadata._sequences.values()
                     if s.column is None and self._can_create_sequence(s)]
 
+        event_collection = [
+            t for (t, fks) in collection if t is not None
+        ]
         metadata.dispatch.before_create(metadata, self.connection,
-                                        tables=collection,
+                                        tables=event_collection,
                                         checkfirst=self.checkfirst,
                                         _ddl_runner=self)
 
@@ -730,7 +733,7 @@ class SchemaGenerator(DDLBase):
                     self.traverse_single(fkc)
 
         metadata.dispatch.after_create(metadata, self.connection,
-                                       tables=collection,
+                                       tables=event_collection,
                                        checkfirst=self.checkfirst,
                                        _ddl_runner=self)
 
@@ -803,32 +806,50 @@ class SchemaDropper(DDLBase):
             tables = list(metadata.tables.values())
 
         try:
-            collection = reversed(
+            unsorted_tables = [t for t in tables if self._can_drop_table(t)]
+            collection = list(reversed(
                 sort_tables_and_constraints(
-                    [t for t in tables if self._can_drop_table(t)],
-                    filter_fn=
-                    lambda constraint: True if not self.dialect.supports_alter
-                    else False if constraint.name is None
+                    unsorted_tables,
+                    filter_fn=lambda constraint: False
+                    if not self.dialect.supports_alter
+                    or constraint.name is None
                     else None
                 )
-            )
+            ))
         except exc.CircularDependencyError as err2:
-            util.raise_from_cause(
-                exc.CircularDependencyError(
-                    err2.args[0],
-                    err2.cycles, err2.edges,
-                    msg="Can't sort tables for DROP; an "
+            if not self.dialect.supports_alter:
+                util.warn(
+                    "Can't sort tables for DROP; an "
                     "unresolvable foreign key "
-                    "dependency exists between tables: %s.  Please ensure "
-                    "that the ForeignKey and ForeignKeyConstraint objects "
-                    "involved in the cycle have "
-                    "names so that they can be dropped using DROP CONSTRAINT."
+                    "dependency exists between tables: %s, and backend does "
+                    "not support ALTER.  To restore at least a partial sort, "
+                    "apply use_alter=True to ForeignKey and "
+                    "ForeignKeyConstraint "
+                    "objects involved in the cycle to mark these as known "
+                    "cycles that will be ignored."
                     % (
                         ", ".join(sorted([t.fullname for t in err2.cycles]))
                     )
+                )
+                collection = [(t, ()) for t in unsorted_tables]
+            else:
+                util.raise_from_cause(
+                    exc.CircularDependencyError(
+                        err2.args[0],
+                        err2.cycles, err2.edges,
+                        msg="Can't sort tables for DROP; an "
+                        "unresolvable foreign key "
+                        "dependency exists between tables: %s.  Please ensure "
+                        "that the ForeignKey and ForeignKeyConstraint objects "
+                        "involved in the cycle have "
+                        "names so that they can be dropped using "
+                        "DROP CONSTRAINT."
+                        % (
+                            ", ".join(sorted([t.fullname for t in err2.cycles]))
+                        )
 
+                    )
                 )
-            )
 
         seq_coll = [
             s
@@ -836,8 +857,12 @@ class SchemaDropper(DDLBase):
             if s.column is None and self._can_drop_sequence(s)
         ]
 
+        event_collection = [
+            t for (t, fks) in collection if t is not None
+        ]
+
         metadata.dispatch.before_drop(
-            metadata, self.connection, tables=collection,
+            metadata, self.connection, tables=event_collection,
             checkfirst=self.checkfirst, _ddl_runner=self)
 
         for table, fkcs in collection:
@@ -852,7 +877,7 @@ class SchemaDropper(DDLBase):
             self.traverse_single(seq, drop_ok=True)
 
         metadata.dispatch.after_drop(
-            metadata, self.connection, tables=collection,
+            metadata, self.connection, tables=event_collection,
             checkfirst=self.checkfirst, _ddl_runner=self)
 
     def _can_drop_table(self, table):
@@ -1041,7 +1066,8 @@ def sort_tables_and_constraints(
     try:
         candidate_sort = list(
             topological.sort(
-                fixed_dependencies.union(mutable_dependencies), tables
+                fixed_dependencies.union(mutable_dependencies), tables,
+                deterministic_order=True
             )
         )
     except exc.CircularDependencyError as err:
@@ -1058,7 +1084,8 @@ def sort_tables_and_constraints(
                         mutable_dependencies.discard((dependent_on, table))
         candidate_sort = list(
             topological.sort(
-                fixed_dependencies.union(mutable_dependencies), tables
+                fixed_dependencies.union(mutable_dependencies), tables,
+                deterministic_order=True
             )
         )
 
diff --git a/lib/sqlalchemy/sql/dml.py b/lib/sqlalchemy/sql/dml.py
index 6a4768fa1..6756f1554 100644
--- a/lib/sqlalchemy/sql/dml.py
+++ b/lib/sqlalchemy/sql/dml.py
@@ -10,7 +10,8 @@ Provide :class:`.Insert`, :class:`.Update` and :class:`.Delete`.
 """
 
 from .base import Executable, _generative, _from_objects, DialectKWArgs
-from .elements import ClauseElement, _literal_as_text, Null, and_, _clone
+from .elements import ClauseElement, _literal_as_text, Null, and_, _clone, \
+    _column_as_key
 from .selectable import _interpret_as_from, _interpret_as_select, HasPrefixes
 from .. import util
 from .. import exc
@@ -261,10 +262,14 @@ class ValuesBase(UpdateBase):
              has the effect of using the DBAPI
              `executemany() <http://www.python.org/dev/peps/pep-0249/#id18>`_
              method, which provides a high-performance system of invoking
-             a single-row INSERT statement many times against a series
+             a single-row INSERT or single-criteria UPDATE or DELETE statement
+             many times against a series
              of parameter sets.   The "executemany" style is supported by
-             all database backends, as it does not depend on a special SQL
-             syntax.
+             all database backends, and works equally well for INSERT,
+             UPDATE, and DELETE, as it does not depend on a special SQL
+             syntax.  See :ref:`execute_multiple` for an introduction to
+             the traditional Core method of multiple parameter set invocation
+             using this system.
 
          .. versionadded:: 0.8
              Support for multiple-VALUES INSERT statements.
@@ -544,7 +549,8 @@ class Insert(ValuesBase):
                 "This construct already inserts value expressions")
 
         self.parameters, self._has_multi_parameters = \
-            self._process_colparams(dict((n, Null()) for n in names))
+            self._process_colparams(
+                dict((_column_as_key(n), Null()) for n in names))
 
         self.select_names = names
         self.inline = True
diff --git a/lib/sqlalchemy/sql/elements.py b/lib/sqlalchemy/sql/elements.py
index ca8ec1f55..27ecce2b0 100644
--- a/lib/sqlalchemy/sql/elements.py
+++ b/lib/sqlalchemy/sql/elements.py
@@ -2407,13 +2407,14 @@ class UnaryExpression(ColumnElement):
     __visit_name__ = 'unary'
 
     def __init__(self, element, operator=None, modifier=None,
-                 type_=None, negate=None):
+                 type_=None, negate=None, wraps_column_expression=False):
         self.operator = operator
         self.modifier = modifier
         self.element = element.self_group(
             against=self.operator or self.modifier)
         self.type = type_api.to_instance(type_)
         self.negate = negate
+        self.wraps_column_expression = wraps_column_expression
 
     @classmethod
     def _create_nullsfirst(cls, column):
@@ -2455,7 +2456,8 @@ class UnaryExpression(ColumnElement):
         """
         return UnaryExpression(
             _literal_as_label_reference(column),
-            modifier=operators.nullsfirst_op)
+            modifier=operators.nullsfirst_op,
+            wraps_column_expression=False)
 
     @classmethod
     def _create_nullslast(cls, column):
@@ -2496,7 +2498,8 @@ class UnaryExpression(ColumnElement):
         """
         return UnaryExpression(
             _literal_as_label_reference(column),
-            modifier=operators.nullslast_op)
+            modifier=operators.nullslast_op,
+            wraps_column_expression=False)
 
     @classmethod
     def _create_desc(cls, column):
@@ -2534,7 +2537,9 @@ class UnaryExpression(ColumnElement):
 
         """
         return UnaryExpression(
-            _literal_as_label_reference(column), modifier=operators.desc_op)
+            _literal_as_label_reference(column),
+            modifier=operators.desc_op,
+            wraps_column_expression=False)
 
     @classmethod
     def _create_asc(cls, column):
@@ -2571,7 +2576,9 @@ class UnaryExpression(ColumnElement):
 
         """
         return UnaryExpression(
-            _literal_as_label_reference(column), modifier=operators.asc_op)
+            _literal_as_label_reference(column),
+            modifier=operators.asc_op,
+            wraps_column_expression=False)
 
     @classmethod
     def _create_distinct(cls, expr):
@@ -2611,7 +2618,8 @@ class UnaryExpression(ColumnElement):
         """
         expr = _literal_as_binds(expr)
         return UnaryExpression(
-            expr, operator=operators.distinct_op, type_=expr.type)
+            expr, operator=operators.distinct_op,
+            type_=expr.type, wraps_column_expression=False)
 
     @property
     def _order_by_label_element(self):
@@ -2648,7 +2656,8 @@ class UnaryExpression(ColumnElement):
                 operator=self.negate,
                 negate=self.operator,
                 modifier=self.modifier,
-                type_=self.type)
+                type_=self.type,
+                wraps_column_expression=self.wraps_column_expression)
         else:
             return ClauseElement._negate(self)
 
@@ -2667,6 +2676,7 @@ class AsBoolean(UnaryExpression):
         self.operator = operator
         self.negate = negate
         self.modifier = None
+        self.wraps_column_expression = True
 
     def self_group(self, against=None):
         return self
@@ -3093,7 +3103,8 @@ class Label(ColumnElement):
         return self.element,
 
     def _copy_internals(self, clone=_clone, anonymize_labels=False, **kw):
-        self.element = clone(self.element, **kw)
+        self._element = clone(self._element, **kw)
+        self.__dict__.pop('element', None)
         self.__dict__.pop('_allow_label_resolve', None)
         if anonymize_labels:
             self.name = self._resolve_label = _anonymous_label(
@@ -3714,6 +3725,16 @@ def _literal_as_label_reference(element):
     elif hasattr(element, '__clause_element__'):
         element = element.__clause_element__()
 
+    return _literal_as_text(element)
+
+
+def _literal_and_labels_as_label_reference(element):
+    if isinstance(element, util.string_types):
+        return _textual_label_reference(element)
+
+    elif hasattr(element, '__clause_element__'):
+        element = element.__clause_element__()
+
     if isinstance(element, ColumnElement) and \
             element._order_by_label_element is not None:
         return _label_reference(element)
diff --git a/lib/sqlalchemy/sql/schema.py b/lib/sqlalchemy/sql/schema.py
index 3aeba9804..a8989627d 100644
--- a/lib/sqlalchemy/sql/schema.py
+++ b/lib/sqlalchemy/sql/schema.py
@@ -2392,27 +2392,51 @@ class ColumnCollectionMixin(object):
         if _autoattach and self._pending_colargs:
             self._check_attach()
 
+    @classmethod
+    def _extract_col_expression_collection(cls, expressions):
+        for expr in expressions:
+            strname = None
+            column = None
+            if not isinstance(expr, ClauseElement):
+                # this assumes a string
+                strname = expr
+            else:
+                cols = []
+                visitors.traverse(expr, {}, {'column': cols.append})
+                if cols:
+                    column = cols[0]
+            add_element = column if column is not None else strname
+            yield expr, column, strname, add_element
+
     def _check_attach(self, evt=False):
         col_objs = [
             c for c in self._pending_colargs
             if isinstance(c, Column)
         ]
+
         cols_w_table = [
             c for c in col_objs if isinstance(c.table, Table)
         ]
+
         cols_wo_table = set(col_objs).difference(cols_w_table)
 
         if cols_wo_table:
+            # feature #3341 - place event listeners for Column objects
+            # such that when all those cols are attached, we autoattach.
             assert not evt, "Should not reach here on event call"
 
-            def _col_attached(column, table):
-                cols_wo_table.discard(column)
-                if not cols_wo_table:
-                    self._check_attach(evt=True)
-            self._cols_wo_table = cols_wo_table
-            for col in cols_wo_table:
-                col._on_table_attach(_col_attached)
-            return
+            # issue #3411 - don't do the per-column auto-attach if some of the
+            # columns are specified as strings.
+            has_string_cols = set(self._pending_colargs).difference(col_objs)
+            if not has_string_cols:
+                def _col_attached(column, table):
+                    cols_wo_table.discard(column)
+                    if not cols_wo_table:
+                        self._check_attach(evt=True)
+                self._cols_wo_table = cols_wo_table
+                for col in cols_wo_table:
+                    col._on_table_attach(_col_attached)
+                return
 
         columns = cols_w_table
 
@@ -3078,14 +3102,10 @@ class Index(DialectKWArgs, ColumnCollectionMixin, SchemaItem):
         self.table = None
 
         columns = []
-        for expr in expressions:
-            if not isinstance(expr, ClauseElement):
-                columns.append(expr)
-            else:
-                cols = []
-                visitors.traverse(expr, {}, {'column': cols.append})
-                if cols:
-                    columns.append(cols[0])
+        for expr, column, strname, add_element in self.\
+                _extract_col_expression_collection(expressions):
+            if add_element is not None:
+                columns.append(add_element)
 
         self.expressions = expressions
         self.name = quoted_name(name, kw.pop("quote", None))
@@ -3359,11 +3379,14 @@ class MetaData(SchemaItem):
                 'schema': self.schema,
                 'schemas': self._schemas,
                 'sequences': self._sequences,
-                'fk_memos': self._fk_memos}
+                'fk_memos': self._fk_memos,
+                'naming_convention': self.naming_convention
+                }
 
     def __setstate__(self, state):
         self.tables = state['tables']
         self.schema = state['schema']
+        self.naming_convention = state['naming_convention']
         self._bind = None
         self._sequences = state['sequences']
         self._schemas = state['schemas']
@@ -3450,7 +3473,7 @@ class MetaData(SchemaItem):
 
 
         """
-        return ddl.sort_tables(self.tables.values())
+        return ddl.sort_tables(sorted(self.tables.values(), key=lambda t: t.key))
 
     def reflect(self, bind=None, schema=None, views=False, only=None,
                 extend_existing=False,
diff --git a/lib/sqlalchemy/sql/selectable.py b/lib/sqlalchemy/sql/selectable.py
index f848ef6db..245c54817 100644
--- a/lib/sqlalchemy/sql/selectable.py
+++ b/lib/sqlalchemy/sql/selectable.py
@@ -16,7 +16,7 @@ from .elements import _clone, \
     _literal_as_text, _interpret_as_column_or_from, _expand_cloned,\
     _select_iterables, _anonymous_label, _clause_element_as_expr,\
     _cloned_intersection, _cloned_difference, True_, \
-    _literal_as_label_reference
+    _literal_as_label_reference, _literal_and_labels_as_label_reference
 from .base import Immutable, Executable, _generative, \
     ColumnCollection, ColumnSet, _from_objects, Generative
 from . import type_api
@@ -1723,7 +1723,7 @@ class GenerativeSelect(SelectBase):
         if order_by is not None:
             self._order_by_clause = ClauseList(
                 *util.to_list(order_by),
-                _literal_as_text=_literal_as_label_reference)
+                _literal_as_text=_literal_and_labels_as_label_reference)
         if group_by is not None:
             self._group_by_clause = ClauseList(
                 *util.to_list(group_by),
@@ -1912,7 +1912,8 @@ class GenerativeSelect(SelectBase):
             if getattr(self, '_order_by_clause', None) is not None:
                 clauses = list(self._order_by_clause) + list(clauses)
             self._order_by_clause = ClauseList(
-                *clauses, _literal_as_text=_literal_as_label_reference)
+                *clauses,
+                _literal_as_text=_literal_and_labels_as_label_reference)
 
     def append_group_by(self, *clauses):
         """Append the given GROUP BY criterion applied to this selectable.
@@ -3343,7 +3344,8 @@ class Exists(UnaryExpression):
             s = Select(*args, **kwargs).as_scalar().self_group()
 
         UnaryExpression.__init__(self, s, operator=operators.exists,
-                                 type_=type_api.BOOLEANTYPE)
+                                 type_=type_api.BOOLEANTYPE,
+                                 wraps_column_expression=True)
 
     def select(self, whereclause=None, **params):
         return Select([self], whereclause, **params)
diff --git a/lib/sqlalchemy/sql/type_api.py b/lib/sqlalchemy/sql/type_api.py
index 4660850bd..a55eed981 100644
--- a/lib/sqlalchemy/sql/type_api.py
+++ b/lib/sqlalchemy/sql/type_api.py
@@ -128,6 +128,33 @@ class TypeEngine(Visitable):
 
     """
 
+    def compare_against_backend(self, dialect, conn_type):
+        """Compare this type against the given backend type.
+
+        This function is currently not implemented for SQLAlchemy
+        types, and for all built in types will return ``None``.  However,
+        it can be implemented by a user-defined type
+        where it can be consumed by schema comparison tools such as
+        Alembic autogenerate.
+
+        A future release of SQLAlchemy will potentially impement this method
+        for builtin types as well.
+
+        The function should return True if this type is equivalent to the
+        given type; the type is typically reflected from the database
+        so should be database specific.  The dialect in use is also
+        passed.   It can also return False to assert that the type is
+        not equivalent.
+
+        :param dialect: a :class:`.Dialect` that is involved in the comparison.
+
+        :param conn_type: the type object reflected from the backend.
+
+        .. versionadded:: 1.0.3
+
+        """
+        return None
+
     def copy_value(self, value):
         return value
 
diff --git a/lib/sqlalchemy/sql/util.py b/lib/sqlalchemy/sql/util.py
index bec5b5824..8f502fc86 100644
--- a/lib/sqlalchemy/sql/util.py
+++ b/lib/sqlalchemy/sql/util.py
@@ -16,7 +16,8 @@ from itertools import chain
 from collections import deque
 
 from .elements import BindParameter, ColumnClause, ColumnElement, \
-    Null, UnaryExpression, literal_column, Label, _label_reference
+    Null, UnaryExpression, literal_column, Label, _label_reference, \
+    _textual_label_reference
 from .selectable import ScalarSelect, Join, FromClause, FromGrouping
 from .schema import Column
 
@@ -163,6 +164,8 @@ def unwrap_order_by(clause):
         ):
             if isinstance(t, _label_reference):
                 t = t.element
+            if isinstance(t, (_textual_label_reference)):
+                continue
             cols.add(t)
         else:
             for c in t.get_children():