- mutable examples now move into sqlalchemy.ext.mutable

- streamline interfaces, get Mutable/MutableComposite to be as minimal
in usage as possible
- docs for mutable, warnings regrarding mapper events being global
- move MutableType/mutable=True outwards, move orm tests to its
own module, note in all documentation
- still need more events/tests for correct pickling support of
composites, mutables.  in the case of composites its needed
even without mutation.  see [ticket:2009]

5 files changed, 332 insertions, 49 deletions

diff --git a/lib/sqlalchemy/dialects/postgresql/base.py b/lib/sqlalchemy/dialects/postgresql/base.py
index ee0277b67..1d83d4a91 100644
--- a/lib/sqlalchemy/dialects/postgresql/base.py
+++ b/lib/sqlalchemy/dialects/postgresql/base.py
@@ -247,7 +247,11 @@ class ARRAY(sqltypes.MutableType, sqltypes.Concatenable, sqltypes.TypeEngine):
           "mutable types" mode in the ORM.  Be sure to read the 
           notes for :class:`.MutableType` regarding ORM 
           performance implications (default changed from ``True`` in 
-          0.7.0).   
+          0.7.0).
+          
+          .. note:: This functionality is now superceded by the
+             ``sqlalchemy.ext.mutable`` extension described in 
+             :ref:`mutable_toplevel`.
         
         :param as_tuple=False: Specify whether return results
           should be converted to tuples from lists. DBAPIs such
diff --git a/lib/sqlalchemy/ext/mutable.py b/lib/sqlalchemy/ext/mutable.py
new file mode 100644
index 000000000..7dcbfd996
--- /dev/null
+++ b/lib/sqlalchemy/ext/mutable.py
@@ -0,0 +1,281 @@
+"""Provide support for tracking of in-place changes to scalar values,
+which are propagated to owning parent objects.
+
+The ``mutable`` extension is a replacement for the :class:`.types.MutableType`
+class as well as the ``mutable=True`` flag available on types which subclass
+it.
+
+
+"""
+from sqlalchemy.orm.attributes import flag_modified
+from sqlalchemy import event, types
+from sqlalchemy.orm import mapper, object_mapper
+from sqlalchemy.util import memoized_property
+import weakref
+
+class Mutable(object):
+    """Mixin that defines transparent propagation of change
+    events to a parent object.
+    
+    """
+    
+    @memoized_property
+    def _parents(self):
+        """Dictionary of parent object->attribute name on the parent."""
+        
+        return weakref.WeakKeyDictionary()
+        
+    def on_change(self):
+        """Subclasses should call this method whenever change events occur."""
+        
+        for parent, key in self._parents.items():
+            flag_modified(parent, key)
+    
+    @classmethod
+    def coerce(cls, key, value):
+        """Given a value, coerce it into this type.
+        
+        By default raises ValueError.
+        """
+        if value is None:
+            return None
+        raise ValueError("Attribute '%s' accepts objects of type %s" % (key, cls))
+        
+        
+    @classmethod
+    def associate_with_attribute(cls, attribute):
+        """Establish this type as a mutation listener for the given 
+        mapped descriptor.
+        
+        """
+        key = attribute.key
+        parent_cls = attribute.class_
+        
+        def on_load(state):
+            """Listen for objects loaded or refreshed.   
+            
+            Wrap the target data member's value with 
+            ``Mutable``.
+            
+            """
+            val = state.dict.get(key, None)
+            if val is not None:
+                val = cls.coerce(key, val)
+                state.dict[key] = val
+                val._parents[state.obj()] = key
+
+        def on_set(target, value, oldvalue, initiator):
+            """Listen for set/replace events on the target
+            data member.
+            
+            Establish a weak reference to the parent object
+            on the incoming value, remove it for the one 
+            outgoing.
+            
+            """
+            
+            if not isinstance(value, cls):
+                value = cls.coerce(key, value) 
+            value._parents[target.obj()] = key
+            if isinstance(oldvalue, cls):
+                oldvalue._parents.pop(state.obj(), None)
+            return value
+        
+        event.listen(parent_cls, 'on_load', on_load, raw=True)
+        event.listen(parent_cls, 'on_refresh', on_load, raw=True)
+        event.listen(attribute, 'on_set', on_set, raw=True, retval=True)
+
+        # TODO: need a deserialize hook here
+
+    @classmethod
+    def associate_with(cls, sqltype):
+        """Associate this wrapper with all future mapped columns 
+        of the given type.
+        
+        This is a convenience method that calls ``associate_with_attribute`` automatically.
+
+        .. warning:: The listeners established by this method are *global*
+           to all mappers, and are *not* garbage collected.   Only use 
+           :meth:`.associate_with` for types that are permanent to an application,
+           not with ad-hoc types else this will cause unbounded growth
+           in memory usage.
+        
+        """
+
+        def listen_for_type(mapper, class_):
+            for prop in mapper.iterate_properties:
+                if hasattr(prop, 'columns'):
+                    if isinstance(prop.columns[0].type, sqltype):
+                        cls.associate_with_attribute(getattr(class_, prop.key))
+                        break
+                    
+        event.listen(mapper, 'on_mapper_configured', listen_for_type)
+    
+    @classmethod
+    def as_mutable(cls, sqltype):
+        """Associate a SQL type with this mutable Python type.
+    
+        This establishes listeners that will detect ORM mappings against
+        the given type, adding mutation event trackers to those mappings.
+    
+        The type is returned, unconditionally as an instance, so that 
+        :meth:`.as_mutable` can be used inline::
+    
+            Table('mytable', metadata,
+                Column('id', Integer, primary_key=True),
+                Column('data', MyMutableType.as_mutable(PickleType))
+            )
+        
+        Note that the returned type is always an instance, even if a class
+        is given, and that only columns which are declared specifically with that
+        type instance receive additional instrumentation.
+    
+        To associate a particular mutable type with all occurences of a 
+        particular type, use the :meth:`.Mutable.associate_with` classmethod
+        of the particular :meth:`.Mutable` subclass to establish a global
+        assoiation.
+    
+        .. warning:: The listeners established by this method are *global*
+           to all mappers, and are *not* garbage collected.   Only use 
+           :meth:`.as_mutable` for types that are permanent to an application,
+           not with ad-hoc types else this will cause unbounded growth
+           in memory usage.
+    
+        """
+        sqltype = types.to_instance(sqltype)
+
+        def listen_for_type(mapper, class_):
+            for prop in mapper.iterate_properties:
+                if hasattr(prop, 'columns'):
+                    if prop.columns[0].type is sqltype:
+                        cls.associate_with_attribute(getattr(class_, prop.key))
+                        break
+                
+        event.listen(mapper, 'on_mapper_configured', listen_for_type)
+        
+        return sqltype
+
+
+class _MutableCompositeMeta(type):
+    def __init__(cls, classname, bases, dict_):
+        cls._setup_listeners()
+        return type.__init__(cls, classname, bases, dict_)
+
+class MutableComposite(object):
+    """Mixin that defines transparent propagation of change
+    events on a SQLAlchemy "composite" object to its
+    owning parent or parents.
+    
+    Composite classes, in addition to meeting the usage contract
+    defined in :ref:`mapper_composite`, also define some system
+    of relaying change events to the given :meth:`.on_change` 
+    method, which will notify all parents of the change.  Below
+    the special Python method ``__setattr__`` is used to intercept
+    all changes::
+    
+        class Point(MutableComposite):
+            def __init__(self, x, y):
+                self.x = x
+                self.y = y
+
+            def __setattr__(self, key, value):
+                object.__setattr__(self, key, value)
+                self.on_change()
+        
+            def __composite_values__(self):
+                return self.x, self.y
+            
+            def __eq__(self, other):
+                return isinstance(other, Point) and \
+                    other.x == self.x and \
+                    other.y == self.y
+
+    :class:`.MutableComposite` defines a metaclass which augments
+    the creation of :class:`.MutableComposite` subclasses with an event
+    that will listen for any :func:`~.orm.composite` mappings against the 
+    new type, establishing listeners that will track parent associations.
+
+    .. warning:: The listeners established by the :class:`.MutableComposite`
+       class are *global* to all mappers, and are *not* garbage collected.   Only use 
+       :class:`.MutableComposite` for types that are permanent to an application,
+       not with ad-hoc types else this will cause unbounded growth
+       in memory usage.
+    
+    """
+    __metaclass__ = _MutableCompositeMeta
+
+    @memoized_property
+    def _parents(self):
+        """Dictionary of parent object->attribute name on the parent."""
+        
+        return weakref.WeakKeyDictionary()
+
+    def on_change(self):
+        """Subclasses should call this method whenever change events occur."""
+        
+        for parent, key in self._parents.items():
+            
+            prop = object_mapper(parent).get_property(key)
+            for value, attr_name in zip(
+                                    self.__composite_values__(), 
+                                    prop._attribute_keys):
+                setattr(parent, attr_name, value)
+    
+    @classmethod
+    def _listen_on_attribute(cls, attribute):
+        """Establish this type as a mutation listener for the given 
+        mapped descriptor.
+        
+        """
+        key = attribute.key
+        parent_cls = attribute.class_
+        
+        def on_load(state):
+            """Listen for objects loaded or refreshed.   
+            
+            Wrap the target data member's value with 
+            ``Mutable``.
+            
+            """
+            
+            val = state.dict.get(key, None)
+            if val is not None:
+                val._parents[state.obj()] = key
+
+        def on_set(target, value, oldvalue, initiator):
+            """Listen for set/replace events on the target
+            data member.
+            
+            Establish a weak reference to the parent object
+            on the incoming value, remove it for the one 
+            outgoing.
+            
+            """
+            
+            value._parents[target.obj()] = key
+            if isinstance(oldvalue, cls):
+                oldvalue._parents.pop(state.obj(), None)
+            return value
+        
+        event.listen(parent_cls, 'on_load', on_load, raw=True)
+        event.listen(parent_cls, 'on_refresh', on_load, raw=True)
+        event.listen(attribute, 'on_set', on_set, raw=True, retval=True)
+
+        # TODO: need a deserialize hook here
+    
+    @classmethod
+    def _setup_listeners(cls):
+        """Associate this wrapper with all future mapped compoistes
+        of the given type.
+        
+        This is a convenience method that calls ``associate_with_attribute`` automatically.
+        
+        """
+        
+        def listen_for_type(mapper, class_):
+            for prop in mapper.iterate_properties:
+                if hasattr(prop, 'composite_class') and issubclass(prop.composite_class, cls):
+                    cls._listen_on_attribute(getattr(class_, prop.key))
+                    
+        event.listen(mapper, 'on_mapper_configured', listen_for_type)
+
diff --git a/lib/sqlalchemy/orm/descriptor_props.py b/lib/sqlalchemy/orm/descriptor_props.py
index 5f974e260..d0f871664 100644
--- a/lib/sqlalchemy/orm/descriptor_props.py
+++ b/lib/sqlalchemy/orm/descriptor_props.py
@@ -191,6 +191,7 @@ class CompositeProperty(DescriptorProperty):
         event.listen(self.parent, 'on_refresh', load_handler, raw=True)
         event.listen(self.parent, "on_expire", expire_handler, raw=True)
         
+        # TODO: need a deserialize hook here
         
     @util.memoized_property
     def _attribute_keys(self):
diff --git a/lib/sqlalchemy/orm/state.py b/lib/sqlalchemy/orm/state.py
index 22be5f58f..89a84e898 100644
--- a/lib/sqlalchemy/orm/state.py
+++ b/lib/sqlalchemy/orm/state.py
@@ -175,7 +175,9 @@ class InstanceState(object):
 
         if 'load_path' in state:
             self.load_path = interfaces.deserialize_path(state['load_path'])
-
+        
+        # TODO: need an event here, link to composite, mutable
+        
     def initialize(self, key):
         """Set this attribute to an empty value or collection, 
            based on the AttributeImpl in use."""
diff --git a/lib/sqlalchemy/types.py b/lib/sqlalchemy/types.py
index f5df02367..1756cf6ff 100644
--- a/lib/sqlalchemy/types.py
+++ b/lib/sqlalchemy/types.py
@@ -94,6 +94,10 @@ class TypeEngine(AbstractType):
         are serialized into strings are examples of "mutable" 
         column structures.
         
+        .. note:: This functionality is now superceded by the
+          ``sqlalchemy.ext.mutable`` extension described in 
+          :ref:`mutable_toplevel`.
+        
         When this method is overridden, :meth:`copy_value` should
         also be supplied.   The :class:`.MutableType` mixin
         is recommended as a helper.
@@ -511,10 +515,10 @@ class TypeDecorator(TypeEngine):
         objects alone.  Values such as dicts, lists which
         are serialized into strings are examples of "mutable" 
         column structures.
-        
-        When this method is overridden, :meth:`copy_value` should
-        also be supplied.   The :class:`.MutableType` mixin
-        is recommended as a helper.
+
+        .. note:: This functionality is now superceded by the
+          ``sqlalchemy.ext.mutable`` extension described in 
+          :ref:`mutable_toplevel`.
         
         """
         return self.impl.is_mutable()
@@ -528,8 +532,16 @@ class TypeDecorator(TypeEngine):
 
 class MutableType(object):
     """A mixin that marks a :class:`TypeEngine` as representing
-    a mutable Python object type.
-
+    a mutable Python object type.   This functionality is used
+    only by the ORM.
+    
+    .. note:: :class:`.MutableType` is superceded as of SQLAlchemy 0.7
+       by the ``sqlalchemy.ext.mutable`` extension described in
+       :ref:`mutable_toplevel`.   This extension provides an event
+       driven approach to in-place mutation detection that does not
+       incur the severe performance penalty of the :class:`.MutableType`
+       approach.
+       
     "mutable" means that changes can occur in place to a value 
     of this type.   Examples includes Python lists, dictionaries,
     and sets, as well as user-defined objects.  The primary
@@ -550,49 +562,28 @@ class MutableType(object):
     represent a copy and compare function for values of this
     type - implementing subclasses should override these
     appropriately.
-
-    The usage of mutable types has significant performance
-    implications when using the ORM. In order to detect changes, the
-    ORM must create a copy of the value when it is first
-    accessed, so that changes to the current value can be compared
-    against the "clean" database-loaded value. Additionally, when the
-    ORM checks to see if any data requires flushing, it must scan
-    through all instances in the session which are known to have
-    "mutable" attributes and compare the current value of each
-    one to its "clean"
-    value. So for example, if the Session contains 6000 objects (a
-    fairly large amount) and autoflush is enabled, every individual
-    execution of :class:`Query` will require a full scan of that subset of
-    the 6000 objects that have mutable attributes, possibly resulting
-    in tens of thousands of additional method calls for every query.
     
-    Note that for small numbers (< 100 in the Session at a time)
-    of objects with "mutable" values, the performance degradation is 
-    negligible.  
+    .. warning:: The usage of mutable types has significant performance
+        implications when using the ORM. In order to detect changes, the
+        ORM must create a copy of the value when it is first
+        accessed, so that changes to the current value can be compared
+        against the "clean" database-loaded value. Additionally, when the
+        ORM checks to see if any data requires flushing, it must scan
+        through all instances in the session which are known to have
+        "mutable" attributes and compare the current value of each
+        one to its "clean"
+        value. So for example, if the Session contains 6000 objects (a
+        fairly large amount) and autoflush is enabled, every individual
+        execution of :class:`Query` will require a full scan of that subset of
+        the 6000 objects that have mutable attributes, possibly resulting
+        in tens of thousands of additional method calls for every query.
     
-    It is perfectly fine to represent "mutable" data types with the
-    "mutable" flag set to False, which eliminates any performance
-    issues. It means that the ORM will only reliably detect changes
-    for values of this type if a newly modified value is of a different 
-    identity (i.e., ``id(value)``) than what was present before - 
-    i.e., instead of operations like these::
-    
-        myobject.somedict['foo'] = 'bar'
-        myobject.someset.add('bar')
-        myobject.somelist.append('bar')
-        
-    You'd instead say::
+        As of SQLAlchemy 0.7, the ``sqlalchemy.ext.mutable`` is provided which
+        allows an event driven approach to in-place mutation detection. This
+        approach should now be favored over the usage of :class:`.MutableType`
+        with ``mutable=True``. ``sqlalchemy.ext.mutable`` is described in
+        :ref:`mutable_toplevel`.
     
-        myobject.somevalue = {'foo':'bar'}
-        myobject.someset = myobject.someset.union(['bar'])
-        myobject.somelist = myobject.somelist + ['bar']
-        
-    A future release of SQLAlchemy will include instrumented
-    collection support for mutable types, such that at least usage of
-    plain Python datastructures will be able to emit events for
-    in-place changes, removing the need for pessimistic scanning for
-    changes.
-
     """
 
     def is_mutable(self):
@@ -1594,7 +1585,11 @@ class PickleType(MutableType, TypeDecorator):
           ``comparator`` argument is present.   See
           :class:`.MutableType` for details on "mutable" type
           behavior. (default changed from ``True`` in 
-          0.7.0).   
+          0.7.0).
+
+          .. note:: This functionality is now superceded by the
+             ``sqlalchemy.ext.mutable`` extension described in 
+             :ref:`mutable_toplevel`.
 
         :param comparator: a 2-arg callable predicate used
           to compare values of this type.  If left as ``None``,