Convert remaining ORM APIs to support 2.0 style

This is kind of a mixed bag of all kinds to help get us
to 1.4 betas.    The documentation stuff is a work in
progress.    Lots of other relatively small changes to
APIs and things.    More commits will follow to continue
improving the documentation and transitioning to the
1.4/2.0 hybrid documentation.  In particular some refinements
to Session usage models so that it can match Engine's
scoping / transactional patterns, and a decision to
start moving away from "subtransactions" completely.

* add select().from_statement() to produce FromStatement in an
  ORM context

* begin referring to select() that has "plugins" for the few edge
  cases where select() will have ORM-only behaviors

* convert dynamic.AppenderQuery to its own object that can use
  select(), though at the moment it uses Query to support legacy
  join calling forms.

* custom query classes for AppenderQuery are replaced by
  do_orm_execute() hooks for custom actions, a separate gerrit
  will document this

* add Session.get() to replace query.get()

* Deprecate session.begin->subtransaction.  propose within the
  test suite a hypothetical recipe for apps that rely on this
  pattern

* introduce Session construction level context manager,
  sessionmaker context manager, rewrite the whole top of the
  session_transaction.rst documentation.   Establish context manager
  patterns for Session that are identical to engine

* ensure same begin_nested() / commit() behavior as engine

* devise all new "join into an external transaction" recipe,
  add test support for it, add rules into Session so it
  just works, write new docs.  need to ensure this doesn't
  break anything

* vastly reduce the verbosity of lots of session docs as
  I dont think people read this stuff and it's difficult
  to keep current in any case

* constructs like case(), with_only_columns() really need
  to move to *columns, add a coercion rule to just change
  these.

* docs need changes everywhere I look.  in_() is not in
  the Core tutorial?  how do people even know about it?
  Remove tons of cruft from Select docs, etc.

* build a system for common ORM options like populate_existing
  and autoflush to populate from execution options.

* others?

Change-Id: Ia4bea0f804250e54d90b3884cf8aab8b66b82ecf

5 files changed, 222 insertions, 124 deletions

diff --git a/lib/sqlalchemy/sql/base.py b/lib/sqlalchemy/sql/base.py
index 4bc6d8280..36a8151d3 100644
--- a/lib/sqlalchemy/sql/base.py
+++ b/lib/sqlalchemy/sql/base.py
@@ -571,6 +571,26 @@ class Options(util.with_metaclass(_MetaOptions)):
         o1.__dict__.update(other)
         return o1
 
+    def __eq__(self, other):
+        # TODO: very inefficient.  This is used only in test suites
+        # right now.
+        for a, b in util.zip_longest(self._cache_attrs, other._cache_attrs):
+            if getattr(self, a) != getattr(other, b):
+                return False
+        return True
+
+    def __repr__(self):
+        # TODO: fairly inefficient, used only in debugging right now.
+
+        return "%s(%s)" % (
+            self.__class__.__name__,
+            ", ".join(
+                "%s=%r" % (k, self.__dict__[k])
+                for k in self._cache_attrs
+                if k in self.__dict__
+            ),
+        )
+
     @hybridmethod
     def add_to_element(self, name, value):
         return self + {name: getattr(self, name) + value}
@@ -610,6 +630,60 @@ class Options(util.with_metaclass(_MetaOptions)):
             )
         return cls + d
 
+    @classmethod
+    def from_execution_options(
+        cls, key, attrs, exec_options, statement_exec_options
+    ):
+        """"process Options argument in terms of execution options.
+
+
+        e.g.::
+
+            (
+                load_options,
+                execution_options,
+            ) = QueryContext.default_load_options.from_execution_options(
+                "_sa_orm_load_options",
+                {
+                    "populate_existing",
+                    "autoflush",
+                    "yield_per"
+                },
+                execution_options,
+                statement._execution_options,
+            )
+
+        get back the Options and refresh "_sa_orm_load_options" in the
+        exec options dict w/ the Options as well
+
+        """
+
+        # common case is that no options we are looking for are
+        # in either dictionary, so cancel for that first
+        check_argnames = attrs.intersection(
+            set(exec_options).union(statement_exec_options)
+        )
+
+        existing_options = exec_options.get(key, cls)
+
+        if check_argnames:
+            result = {}
+            for argname in check_argnames:
+                local = "_" + argname
+                if argname in exec_options:
+                    result[local] = exec_options[argname]
+                elif argname in statement_exec_options:
+                    result[local] = statement_exec_options[argname]
+
+            new_options = existing_options + result
+            exec_options = util.immutabledict().merge_with(
+                exec_options, {key: new_options}
+            )
+            return new_options, exec_options
+
+        else:
+            return existing_options, exec_options
+
 
 class CacheableOptions(Options, HasCacheKey):
     @hybridmethod
diff --git a/lib/sqlalchemy/sql/coercions.py b/lib/sqlalchemy/sql/coercions.py
index be412c770..588c485ae 100644
--- a/lib/sqlalchemy/sql/coercions.py
+++ b/lib/sqlalchemy/sql/coercions.py
@@ -52,6 +52,18 @@ def _document_text_coercion(paramname, meth_rst, param_rst):
     )
 
 
+def _expression_collection_was_a_list(attrname, fnname, args):
+    if args and isinstance(args[0], (list, set)) and len(args) == 1:
+        util.warn_deprecated_20(
+            'The "%s" argument to %s() is now passed as a series of '
+            "positional "
+            "elements, rather than as a list. " % (attrname, fnname)
+        )
+        return args[0]
+    else:
+        return args
+
+
 def expect(role, element, apply_propagate_attrs=None, argname=None, **kw):
     if (
         role.allows_lambda
diff --git a/lib/sqlalchemy/sql/elements.py b/lib/sqlalchemy/sql/elements.py
index 6ce505412..c7e5aabcc 100644
--- a/lib/sqlalchemy/sql/elements.py
+++ b/lib/sqlalchemy/sql/elements.py
@@ -2573,10 +2573,8 @@ class Case(ColumnElement):
         stmt = select([users_table]).\
                     where(
                         case(
-                            [
-                                (users_table.c.name == 'wendy', 'W'),
-                                (users_table.c.name == 'jack', 'J')
-                            ],
+                            (users_table.c.name == 'wendy', 'W'),
+                            (users_table.c.name == 'jack', 'J'),
                             else_='E'
                         )
                     )
@@ -2597,7 +2595,10 @@ class Case(ColumnElement):
         ("else_", InternalTraversal.dp_clauseelement),
     ]
 
-    def __init__(self, whens, value=None, else_=None):
+    # TODO: for Py2k removal, this will be:
+    # def __init__(self, *whens, value=None, else_=None):
+
+    def __init__(self, *whens, **kw):
         r"""Produce a ``CASE`` expression.
 
         The ``CASE`` construct in SQL is a conditional object that
@@ -2612,10 +2613,8 @@ class Case(ColumnElement):
             stmt = select([users_table]).\
                         where(
                             case(
-                                [
-                                    (users_table.c.name == 'wendy', 'W'),
-                                    (users_table.c.name == 'jack', 'J')
-                                ],
+                                (users_table.c.name == 'wendy', 'W'),
+                                (users_table.c.name == 'jack', 'J'),
                                 else_='E'
                             )
                         )
@@ -2660,16 +2659,14 @@ class Case(ColumnElement):
             from sqlalchemy import case, literal_column
 
             case(
-                [
-                    (
-                        orderline.c.qty > 100,
-                        literal_column("'greaterthan100'")
-                    ),
-                    (
-                        orderline.c.qty > 10,
-                        literal_column("'greaterthan10'")
-                    )
-                ],
+                (
+                    orderline.c.qty > 100,
+                    literal_column("'greaterthan100'")
+                ),
+                (
+                    orderline.c.qty > 10,
+                    literal_column("'greaterthan10'")
+                ),
                 else_=literal_column("'lessthan10'")
             )
 
@@ -2683,19 +2680,23 @@ class Case(ColumnElement):
                 ELSE 'lessthan10'
             END
 
-        :param whens: The criteria to be compared against,
+        :param \*whens: The criteria to be compared against,
          :paramref:`.case.whens` accepts two different forms, based on
          whether or not :paramref:`.case.value` is used.
 
+         .. versionchanged:: 1.4 the :func:`_sql.case`
+            function now accepts the series of WHEN conditions positionally;
+            passing the expressions within a list is deprecated.
+
          In the first form, it accepts a list of 2-tuples; each 2-tuple
          consists of ``(<sql expression>, <value>)``, where the SQL
          expression is a boolean expression and "value" is a resulting value,
          e.g.::
 
-            case([
+            case(
                 (users_table.c.name == 'wendy', 'W'),
                 (users_table.c.name == 'jack', 'J')
-            ])
+            )
 
          In the second form, it accepts a Python dictionary of comparison
          values mapped to a resulting value; this form requires
@@ -2720,11 +2721,23 @@ class Case(ColumnElement):
 
         """
 
+        if "whens" in kw:
+            util.warn_deprecated_20(
+                'The "whens" argument to case() is now passed as a series of '
+                "positional "
+                "elements, rather than as a list. "
+            )
+            whens = kw.pop("whens")
+        else:
+            whens = coercions._expression_collection_was_a_list(
+                "whens", "case", whens
+            )
         try:
             whens = util.dictlike_iteritems(whens)
         except TypeError:
             pass
 
+        value = kw.pop("value", None)
         if value is not None:
             whenlist = [
                 (
@@ -2760,11 +2773,16 @@ class Case(ColumnElement):
 
         self.type = type_
         self.whens = whenlist
+
+        else_ = kw.pop("else_", None)
         if else_ is not None:
             self.else_ = coercions.expect(roles.ExpressionElementRole, else_)
         else:
             self.else_ = None
 
+        if kw:
+            raise TypeError("unknown arguments: %s" % (", ".join(sorted(kw))))
+
     @property
     def _from_objects(self):
         return list(
diff --git a/lib/sqlalchemy/sql/operators.py b/lib/sqlalchemy/sql/operators.py
index 85db88345..2d369cdf8 100644
--- a/lib/sqlalchemy/sql/operators.py
+++ b/lib/sqlalchemy/sql/operators.py
@@ -169,9 +169,6 @@ class Operators(object):
           :class:`.Boolean`, and those that do not will be of the same
           type as the left-hand operand.
 
-          .. versionadded:: 1.2.0b3 - added the
-             :paramref:`.Operators.op.return_type` argument.
-
         .. seealso::
 
             :ref:`types_operators`
@@ -194,8 +191,6 @@ class Operators(object):
         :paramref:`.Operators.op.is_comparison`
         flag with True.
 
-        .. versionadded:: 1.2.0b3
-
         .. seealso::
 
             :meth:`.Operators.op`
@@ -723,15 +718,6 @@ class ColumnOperators(Operators):
 
           With the value of ``:param`` as ``"foo/%bar"``.
 
-          .. versionadded:: 1.2
-
-          .. versionchanged:: 1.2.0 The
-            :paramref:`.ColumnOperators.startswith.autoescape` parameter is
-             now a simple boolean rather than a character; the escape
-             character itself is also escaped, and defaults to a forwards
-             slash, which itself can be customized using the
-             :paramref:`.ColumnOperators.startswith.escape` parameter.
-
         :param escape: a character which when given will render with the
           ``ESCAPE`` keyword to establish that character as the escape
           character.  This character can then be placed preceding occurrences
@@ -811,15 +797,6 @@ class ColumnOperators(Operators):
 
           With the value of ``:param`` as ``"foo/%bar"``.
 
-          .. versionadded:: 1.2
-
-          .. versionchanged:: 1.2.0 The
-            :paramref:`.ColumnOperators.endswith.autoescape` parameter is
-             now a simple boolean rather than a character; the escape
-             character itself is also escaped, and defaults to a forwards
-             slash, which itself can be customized using the
-             :paramref:`.ColumnOperators.endswith.escape` parameter.
-
         :param escape: a character which when given will render with the
           ``ESCAPE`` keyword to establish that character as the escape
           character.  This character can then be placed preceding occurrences
@@ -899,15 +876,6 @@ class ColumnOperators(Operators):
 
           With the value of ``:param`` as ``"foo/%bar"``.
 
-          .. versionadded:: 1.2
-
-          .. versionchanged:: 1.2.0 The
-            :paramref:`.ColumnOperators.contains.autoescape` parameter is
-             now a simple boolean rather than a character; the escape
-             character itself is also escaped, and defaults to a forwards
-             slash, which itself can be customized using the
-             :paramref:`.ColumnOperators.contains.escape` parameter.
-
         :param escape: a character which when given will render with the
           ``ESCAPE`` keyword to establish that character as the escape
           character.  This character can then be placed preceding occurrences
diff --git a/lib/sqlalchemy/sql/selectable.py b/lib/sqlalchemy/sql/selectable.py
index 12fcc00c3..1155c273b 100644
--- a/lib/sqlalchemy/sql/selectable.py
+++ b/lib/sqlalchemy/sql/selectable.py
@@ -2225,6 +2225,17 @@ class ForUpdateArg(ClauseElement):
         ("skip_locked", InternalTraversal.dp_boolean),
     ]
 
+    @classmethod
+    def _from_argument(cls, with_for_update):
+        if isinstance(with_for_update, ForUpdateArg):
+            return with_for_update
+        elif with_for_update in (None, False):
+            return None
+        elif with_for_update is True:
+            return ForUpdateArg()
+        else:
+            return ForUpdateArg(**with_for_update)
+
     def __eq__(self, other):
         return (
             isinstance(other, ForUpdateArg)
@@ -2699,6 +2710,12 @@ class SelectStatementGrouping(GroupedElement, SelectBase):
 
 
 class DeprecatedSelectBaseGenerations(object):
+    """A collection of methods available on :class:`_sql.Select` and
+    :class:`_sql.CompoundSelect`, these are all **deprecated** methods as they
+    modify the object in-place.
+
+    """
+
     @util.deprecated(
         "1.4",
         "The :meth:`_expression.GenerativeSelect.append_order_by` "
@@ -2740,9 +2757,6 @@ class DeprecatedSelectBaseGenerations(object):
         as it
         provides standard :term:`method chaining`.
 
-        .. seealso::
-
-            :meth:`_expression.GenerativeSelect.group_by`
 
         """
         self.group_by.non_generative(self, *clauses)
@@ -3353,6 +3367,12 @@ class CompoundSelect(HasCompileState, GenerativeSelect):
 
 
 class DeprecatedSelectGenerations(object):
+    """A collection of methods available on :class:`_sql.Select`, these
+    are all **deprecated** methods as they modify the :class:`_sql.Select`
+    object in -place.
+
+    """
+
     @util.deprecated(
         "1.4",
         "The :meth:`_expression.Select.append_correlation` "
@@ -3377,7 +3397,7 @@ class DeprecatedSelectGenerations(object):
         "1.4",
         "The :meth:`_expression.Select.append_column` method is deprecated "
         "and will be removed in a future release.  Use the generative "
-        "method :meth:`_expression.Select.column`.",
+        "method :meth:`_expression.Select.add_columns`.",
     )
     def append_column(self, column):
         """Append the given column expression to the columns clause of this
@@ -3388,14 +3408,10 @@ class DeprecatedSelectGenerations(object):
             my_select.append_column(some_table.c.new_column)
 
         This is an **in-place** mutation method; the
-        :meth:`_expression.Select.column` method is preferred,
+        :meth:`_expression.Select.add_columns` method is preferred,
         as it provides standard
         :term:`method chaining`.
 
-        See the documentation for :meth:`_expression.Select.with_only_columns`
-        for guidelines on adding /replacing the columns of a
-        :class:`_expression.Select` object.
-
         """
         self.add_columns.non_generative(self, column)
 
@@ -3501,6 +3517,21 @@ class SelectState(util.MemoizedSlots, CompileState):
 
         self.columns_plus_names = statement._generate_columns_plus_names(True)
 
+    @classmethod
+    def _plugin_not_implemented(cls):
+        raise NotImplementedError(
+            "The default SELECT construct without plugins does not "
+            "implement this method."
+        )
+
+    @classmethod
+    def get_column_descriptions(cls, statement):
+        cls._plugin_not_implemented()
+
+    @classmethod
+    def from_statement(cls, statement, from_statement):
+        cls._plugin_not_implemented()
+
     def _get_froms(self, statement):
         seen = set()
         froms = []
@@ -3805,6 +3836,15 @@ class Select(
 ):
     """Represents a ``SELECT`` statement.
 
+    The :class:`_sql.Select` object is normally constructed using the
+    :func:`_sql.select` function.  See that function for details.
+
+    .. seealso::
+
+        :func:`_sql.select`
+
+        :ref:`coretutorial_selecting` - in the Core tutorial
+
     """
 
     __visit_name__ = "select"
@@ -3821,7 +3861,7 @@ class Select(
     _from_obj = ()
     _auto_correlate = True
 
-    compile_options = SelectState.default_select_compile_options
+    _compile_options = SelectState.default_select_compile_options
 
     _traverse_internals = (
         [
@@ -3851,7 +3891,7 @@ class Select(
     )
 
     _cache_key_traversal = _traverse_internals + [
-        ("compile_options", InternalTraversal.dp_has_cache_key)
+        ("_compile_options", InternalTraversal.dp_has_cache_key)
     ]
 
     @classmethod
@@ -4274,12 +4314,35 @@ class Select(
     @property
     def column_descriptions(self):
         """Return a 'column descriptions' structure which may be
-        plugin-specific.
+        :term:`plugin-specific`.
 
         """
         meth = SelectState.get_plugin_class(self).get_column_descriptions
         return meth(self)
 
+    def from_statement(self, statement):
+        """Apply the columns which this :class:`.Select` would select
+        onto another statement.
+
+        This operation is :term:`plugin-specific` and will raise a not
+        supported exception if this :class:`_sql.Select` does not select from
+        plugin-enabled entities.
+
+
+        The statement is typically either a :func:`_expression.text` or
+        :func:`_expression.select` construct, and should return the set of
+        columns appropriate to the entities represented by this
+        :class:`.Select`.
+
+        .. seealso::
+
+            :ref:`orm_tutorial_literal_sql` - usage examples in the
+            ORM tutorial
+
+        """
+        meth = SelectState.get_plugin_class(self).from_statement
+        return meth(self, statement)
+
     @_generative
     def join(self, target, onclause=None, isouter=False, full=False):
         r"""Create a SQL JOIN against this :class:`_expresson.Select`
@@ -4550,7 +4613,7 @@ class Select(
         )
 
     @_generative
-    def with_only_columns(self, columns):
+    def with_only_columns(self, *columns):
         r"""Return a new :func:`_expression.select` construct with its columns
         clause replaced with the given columns.
 
@@ -4558,65 +4621,26 @@ class Select(
         :func:`_expression.select` had been called with the given columns
         clause.   I.e. a statement::
 
-            s = select([table1.c.a, table1.c.b])
-            s = s.with_only_columns([table1.c.b])
+            s = select(table1.c.a, table1.c.b)
+            s = s.with_only_columns(table1.c.b)
 
         should be exactly equivalent to::
 
-            s = select([table1.c.b])
-
-        This means that FROM clauses which are only derived
-        from the column list will be discarded if the new column
-        list no longer contains that FROM::
-
-            >>> table1 = table('t1', column('a'), column('b'))
-            >>> table2 = table('t2', column('a'), column('b'))
-            >>> s1 = select([table1.c.a, table2.c.b])
-            >>> print(s1)
-            SELECT t1.a, t2.b FROM t1, t2
-            >>> s2 = s1.with_only_columns([table2.c.b])
-            >>> print(s2)
-            SELECT t2.b FROM t1
-
-        The preferred way to maintain a specific FROM clause
-        in the construct, assuming it won't be represented anywhere
-        else (i.e. not in the WHERE clause, etc.) is to set it using
-        :meth:`_expression.Select.select_from`::
-
-            >>> s1 = select([table1.c.a, table2.c.b]).\
-            ...         select_from(table1.join(table2,
-            ...                 table1.c.a==table2.c.a))
-            >>> s2 = s1.with_only_columns([table2.c.b])
-            >>> print(s2)
-            SELECT t2.b FROM t1 JOIN t2 ON t1.a=t2.a
-
-        Care should also be taken to use the correct set of column objects
-        passed to :meth:`_expression.Select.with_only_columns`.
-        Since the method is
-        essentially equivalent to calling the :func:`_expression.select`
-        construct in the first place with the given columns, the columns passed
-        to :meth:`_expression.Select.with_only_columns`
-        should usually be a subset of
-        those which were passed to the :func:`_expression.select`
-        construct, not those which are available from the ``.c`` collection of
-        that :func:`_expression.select`.  That is::
-
-            s = select([table1.c.a, table1.c.b]).select_from(table1)
-            s = s.with_only_columns([table1.c.b])
-
-        and **not**::
-
-            # usually incorrect
-            s = s.with_only_columns([s.c.b])
-
-        The latter would produce the SQL::
-
-            SELECT b
-            FROM (SELECT t1.a AS a, t1.b AS b
-            FROM t1), t1
-
-        Since the :func:`_expression.select` construct is essentially
-        being asked to select both from ``table1`` as well as itself.
+            s = select(table1.c.b)
+
+        Note that this will also dynamically alter the FROM clause of the
+        statement if it is not explicitly stated.  To maintain the FROM
+        clause, ensure the :meth:`_sql.Select.select_from` method is
+        used appropriately::
+
+            s = select(table1.c.a, table2.c.b)
+            s = s.select_from(table2.c.b).with_only_columns(table1.c.a)
+
+        :param \*columns: column expressions to be used.
+
+         .. versionchanged:: 1.4 the :meth:`_sql.Select.with_only_columns`
+            method accepts the list of column expressions positionally;
+            passing the expressions as a list is deprecateed.
 
         """
 
@@ -4626,7 +4650,9 @@ class Select(
         self._assert_no_memoizations()
 
         rc = []
-        for c in columns:
+        for c in coercions._expression_collection_was_a_list(
+            "columns", "Select.with_only_columns", columns
+        ):
             c = coercions.expect(roles.ColumnsClauseRole, c,)
             # TODO: why are we doing this here?
             if isinstance(c, ScalarSelect):