Document visitors module

As we are going to be adding a lot of new features to the visitors
module which may impact end-user custom constructs, start documenting
the package for now.

Change-Id: Ibae471c2cb861f6280f601b7b2382d61968825cd

1 files changed, 148 insertions, 20 deletions

diff --git a/lib/sqlalchemy/sql/visitors.py b/lib/sqlalchemy/sql/visitors.py
index 0a3cb8c1c..7b2ac285a 100644
--- a/lib/sqlalchemy/sql/visitors.py
+++ b/lib/sqlalchemy/sql/visitors.py
@@ -13,14 +13,13 @@ they apply functionality.  The most common use of this pattern
 is statement compilation, where individual expression classes match
 up to rendering methods that produce a string result.   Beyond this,
 the visitor system is also used to inspect expressions for various
-information and patterns, as well as for usage in
-some kinds of expression transformation.  Other kinds of transformation
-use a non-visitor traversal system.
+information and patterns, as well as for the purposes of applying
+transformations to expressions.
 
-For many examples of how the visit system is used, see the
-sqlalchemy.sql.util and the sqlalchemy.sql.compiler modules.
-For an introduction to clause adaption, see
-http://techspot.zzzeek.org/2008/01/23/expression-transformations/
+Examples of how the visit system is used can be seen in the source code
+of for example the ``sqlalchemy.sql.util`` and the ``sqlalchemy.sql.compiler``
+modules.  Some background on clause adaption is also at
+http://techspot.zzzeek.org/2008/01/23/expression-transformations/ .
 
 """
 
@@ -48,10 +47,10 @@ __all__ = [
 
 
 class VisitableType(type):
-    """Metaclass which assigns a `_compiler_dispatch` method to classes
-    having a `__visit_name__` attribute.
+    """Metaclass which assigns a ``_compiler_dispatch`` method to classes
+    having a ``__visit_name__`` attribute.
 
-    The _compiler_dispatch attribute becomes an instance method which
+    The ``_compiler_dispatch`` attribute becomes an instance method which
     looks approximately like the following::
 
         def _compiler_dispatch (self, visitor, **kw):
@@ -60,7 +59,8 @@ class VisitableType(type):
             visit_attr = 'visit_%s' % self.__visit_name__
             return getattr(visitor, visit_attr)(self, **kw)
 
-    Classes having no __visit_name__ attribute will remain unaffected.
+    Classes having no ``__visit_name__`` attribute will remain unaffected.
+
     """
 
     def __init__(cls, clsname, bases, clsdict):
@@ -112,14 +112,20 @@ def _generate_dispatch(cls):
 
 class Visitable(util.with_metaclass(VisitableType, object)):
     """Base class for visitable objects, applies the
-    ``VisitableType`` metaclass.
+    :class:`.visitors.VisitableType` metaclass.
+
+    The :class:`.Visitable` class is essentially at the base of the
+    :class:`.ClauseElement` hierarchy.
 
     """
 
 
 class ClauseVisitor(object):
     """Base class for visitor objects which can traverse using
-    the traverse() function.
+    the :func:`.visitors.traverse` function.
+
+    Direct usage of the :func:`.visitors.traverse` function is usually
+    preferred.
 
     """
 
@@ -174,7 +180,11 @@ class ClauseVisitor(object):
 
 class CloningVisitor(ClauseVisitor):
     """Base class for visitor objects which can traverse using
-    the cloned_traverse() function.
+    the :func:`.visitors.cloned_traverse` function.
+
+    Direct usage of the :func:`.visitors.cloned_traverse` function is usually
+    preferred.
+
 
     """
 
@@ -195,7 +205,10 @@ class CloningVisitor(ClauseVisitor):
 
 class ReplacingCloningVisitor(CloningVisitor):
     """Base class for visitor objects which can traverse using
-    the replacement_traverse() function.
+    the :func:`.visitors.replacement_traverse` function.
+
+    Direct usage of the :func:`.visitors.replacement_traverse` function is
+    usually preferred.
 
     """
 
@@ -221,10 +234,24 @@ class ReplacingCloningVisitor(CloningVisitor):
 
 
 def iterate(obj, opts):
-    """traverse the given expression structure, returning an iterator.
+    r"""traverse the given expression structure, returning an iterator.
 
     traversal is configured to be breadth-first.
 
+    The central API feature used by the :func:`.visitors.iterate` and
+    :func:`.visitors.iterate_depthfirst` functions is the
+    :meth:`.ClauseElement.get_children` method of :class:`.ClauseElement`
+    objects.  This method should return all the :class:`.ClauseElement` objects
+    which are associated with a particular :class:`.ClauseElement` object.
+    For example, a :class:`.Case` structure will refer to a series of
+    :class:`.ColumnElement` objects within its "whens" and "else\_" member
+    variables.
+
+    :param obj: :class:`.ClauseElement` structure to be traversed
+
+    :param opts: dictionary of iteration options.   This dictionary is usually
+     empty in modern usage.
+
     """
     # fasttrack for atomic elements like columns
     children = obj.get_children(**opts)
@@ -246,6 +273,15 @@ def iterate_depthfirst(obj, opts):
 
     traversal is configured to be depth-first.
 
+    :param obj: :class:`.ClauseElement` structure to be traversed
+
+    :param opts: dictionary of iteration options.   This dictionary is usually
+     empty in modern usage.
+
+    .. seealso::
+
+        :func:`.visitors.iterate` - includes a general overview of iteration.
+
     """
     # fasttrack for atomic elements like columns
     children = obj.get_children(**opts)
@@ -266,6 +302,27 @@ def traverse_using(iterator, obj, visitors):
     """visit the given expression structure using the given iterator of
     objects.
 
+    :func:`.visitors.traverse_using` is usually called internally as the result
+    of the :func:`.visitors.traverse` or :func:`.visitors.traverse_depthfirst`
+    functions.
+
+    :param iterator: an iterable or sequence which will yield
+     :class:`.ClauseElement` structures; the iterator is assumed to be the
+     product of the :func:`.visitors.iterate` or
+     :func:`.visitors.iterate_depthfirst` functions.
+
+    :param obj: the :class:`.ClauseElement` that was used as the target of the
+     :func:`.iterate` or :func:`.iterate_depthfirst` function.
+
+    :param visitors: dictionary of visit functions.  See :func:`.traverse`
+     for details on this dictionary.
+
+    .. seealso::
+
+        :func:`.traverse`
+
+        :func:`.traverse_depthfirst`
+
     """
     for target in iterator:
         meth = visitors.get(target.__visit_name__, None)
@@ -276,7 +333,32 @@ def traverse_using(iterator, obj, visitors):
 
 def traverse(obj, opts, visitors):
     """traverse and visit the given expression structure using the default
-     iterator.
+    iterator.
+
+     e.g.::
+
+        from sqlalchemy.sql import visitors
+
+        stmt = select([some_table]).where(some_table.c.foo == 'bar')
+
+        def visit_bindparam(bind_param):
+            print("found bound value: %s" % bind_param.value)
+
+        visitors.traverse(stmt, {}, {"bindparam": visit_bindparam})
+
+    The iteration of objects uses the :func:`.visitors.iterate` function,
+    which does a breadth-first traversal using a stack.
+
+    :param obj: :class:`.ClauseElement` structure to be traversed
+
+    :param opts: dictionary of iteration options.   This dictionary is usually
+     empty in modern usage.
+
+    :param visitors: dictionary of visit functions.   The dictionary should
+     have strings as keys, each of which would correspond to the
+     ``__visit_name__`` of a particular kind of SQL expression object, and
+     callable functions  as values, each of which represents a visitor function
+     for that kind of object.
 
     """
     return traverse_using(iterate(obj, opts), obj, visitors)
@@ -286,13 +368,39 @@ def traverse_depthfirst(obj, opts, visitors):
     """traverse and visit the given expression structure using the
     depth-first iterator.
 
+    The iteration of objects uses the :func:`.visitors.iterate_depthfirst`
+    function, which does a depth-first traversal using a stack.
+
+    Usage is the same as that of :func:`.visitors.traverse` function.
+
+
     """
     return traverse_using(iterate_depthfirst(obj, opts), obj, visitors)
 
 
 def cloned_traverse(obj, opts, visitors):
-    """clone the given expression structure, allowing
-    modifications by visitors."""
+    """clone the given expression structure, allowing modifications by
+    visitors.
+
+    Traversal usage is the same as that of :func:`.visitors.traverse`.
+    The visitor functions present in the ``visitors`` dictionary may also
+    modify the internals of the given structure as the traversal proceeds.
+
+    The central API feature used by the :func:`.visitors.cloned_traverse`
+    and :func:`.visitors.replacement_traverse` functions, in addition to the
+    :meth:`.ClauseElement.get_children` function that is used to achieve
+    the iteration, is the :meth:`.ClauseElement._copy_internals` method.
+    For a :class:`.ClauseElement` structure to support cloning and replacement
+    traversals correctly, it needs to be able to pass a cloning function into
+    its internal members in order to make copies of them.
+
+    .. seealso::
+
+        :func:`.visitors.traverse`
+
+        :func:`.visitors.replacement_traverse`
+
+    """
 
     cloned = {}
     stop_on = set(opts.get("stop_on", []))
@@ -316,7 +424,27 @@ def cloned_traverse(obj, opts, visitors):
 
 def replacement_traverse(obj, opts, replace):
     """clone the given expression structure, allowing element
-    replacement by a given replacement function."""
+    replacement by a given replacement function.
+
+    This function is very similar to the :func:`.visitors.cloned_traverse`
+    function, except instead of being passed a dictionary of visitors, all
+    elements are unconditionally passed into the given replace function.
+    The replace function then has the option to return an entirely new object
+    which will replace the one given.  if it returns ``None``, then the object
+    is kept in place.
+
+    The difference in usage between :func:`.visitors.cloned_traverse` and
+    :func:`.visitors.replacement_traverse` is that in the former case, an
+    already-cloned object is passed to the visitor function, and the visitor
+    function can then manipulate the internal state of the object.
+    In the case of the latter, the visitor function should only return an
+    entirely different object, or do nothing.
+
+    The use case for :func:`.visitors.replacement_traverse` is that of
+    replacing a FROM clause inside of a SQL structure with a different one,
+    as is a common use case within the ORM.
+
+    """
 
     cloned = {}
     stop_on = {id(x) for x in opts.get("stop_on", [])}