- The :class:`.SessionEvents` suite now includes events to allowticket_2677

unambiguous tracking of all object lifecycle state transitions
in terms of the :class:`.Session` itself, e.g. pending,
transient,  persistent, detached.   The state of the object
within each event is also defined.
fixes #2677
- Added a new session lifecycle state :term:`deleted`.  This new state
represents an object that has been deleted from the :term:`persistent`
state and will move to the :term:`detached` state once the transaction
is committed.  This resolves the long-standing issue that objects
which were deleted existed in a gray area between persistent and
detached.   The :attr:`.InstanceState.persistent` accessor will
**no longer** report on a deleted object as persistent; the
:attr:`.InstanceState.deleted` accessor will instead be True for
these objects, until they become detached.
- The :paramref:`.Session.weak_identity_map` parameter is deprecated.
See the new recipe at :ref:`session_referencing_behavior` for
an event-based approach to maintaining strong identity map behavior.
references #3517

5 files changed, 597 insertions, 23 deletions

diff --git a/test/orm/test_events.py b/test/orm/test_events.py
index b9fafb105..ab61077ae 100644
--- a/test/orm/test_events.py
+++ b/test/orm/test_events.py
@@ -1617,6 +1617,506 @@ class SessionEventsTest(_RemoveListeners, _fixtures.FixtureTest):
         )
 
 
+class SessionLifecycleEventsTest(_RemoveListeners, _fixtures.FixtureTest):
+    run_inserts = None
+
+    def _fixture(self, include_address=False):
+        users, User = self.tables.users, self.classes.User
+
+        if include_address:
+            addresses, Address = self.tables.addresses, self.classes.Address
+            mapper(User, users, properties={
+                "addresses": relationship(
+                    Address, cascade="all, delete-orphan")
+            })
+            mapper(Address, addresses)
+        else:
+            mapper(User, users)
+
+        listener = Mock()
+
+        sess = Session()
+
+        def start_events():
+            event.listen(
+                sess, "transient_to_pending", listener.transient_to_pending)
+            event.listen(
+                sess, "pending_to_transient", listener.pending_to_transient)
+            event.listen(
+                sess, "persistent_to_transient",
+                listener.persistent_to_transient)
+            event.listen(
+                sess, "pending_to_persistent", listener.pending_to_persistent)
+            event.listen(
+                sess, "detached_to_persistent",
+                listener.detached_to_persistent)
+            event.listen(
+                sess, "loaded_as_persistent", listener.loaded_as_persistent)
+
+            event.listen(
+                sess, "persistent_to_detached",
+                listener.persistent_to_detached)
+            event.listen(
+                sess, "deleted_to_detached", listener.deleted_to_detached)
+
+            event.listen(
+                sess, "persistent_to_deleted", listener.persistent_to_deleted)
+            event.listen(
+                sess, "deleted_to_persistent", listener.deleted_to_persistent)
+            return listener
+
+        if include_address:
+            return sess, User, Address, start_events
+        else:
+            return sess, User, start_events
+
+    def test_transient_to_pending(self):
+        sess, User, start_events = self._fixture()
+
+        listener = start_events()
+
+        @event.listens_for(sess, "transient_to_pending")
+        def trans_to_pending(session, instance):
+            assert instance in session
+            listener.flag_checked(instance)
+
+        u1 = User(name='u1')
+        sess.add(u1)
+
+        eq_(
+            listener.mock_calls,
+            [
+                call.transient_to_pending(sess, u1),
+                call.flag_checked(u1)
+            ]
+        )
+
+    def test_pending_to_transient_via_rollback(self):
+        sess, User, start_events = self._fixture()
+
+        u1 = User(name='u1')
+        sess.add(u1)
+
+        listener = start_events()
+
+        @event.listens_for(sess, "pending_to_transient")
+        def test_deleted_flag(session, instance):
+            assert instance not in session
+            listener.flag_checked(instance)
+
+        sess.rollback()
+        assert u1 not in sess
+
+        eq_(
+            listener.mock_calls,
+            [
+                call.pending_to_transient(sess, u1),
+                call.flag_checked(u1)
+            ]
+        )
+
+    def test_pending_to_transient_via_expunge(self):
+        sess, User, start_events = self._fixture()
+
+        u1 = User(name='u1')
+        sess.add(u1)
+
+        listener = start_events()
+
+        @event.listens_for(sess, "pending_to_transient")
+        def test_deleted_flag(session, instance):
+            assert instance not in session
+            listener.flag_checked(instance)
+
+        sess.expunge(u1)
+        assert u1 not in sess
+
+        eq_(
+            listener.mock_calls,
+            [
+                call.pending_to_transient(sess, u1),
+                call.flag_checked(u1)
+            ]
+        )
+
+    def test_pending_to_persistent(self):
+        sess, User, start_events = self._fixture()
+
+        u1 = User(name='u1')
+        sess.add(u1)
+
+        listener = start_events()
+
+        @event.listens_for(sess, "pending_to_persistent")
+        def test_flag(session, instance):
+            assert instance in session
+            assert instance._sa_instance_state.persistent
+            assert instance._sa_instance_state.key in session.identity_map
+            listener.flag_checked(instance)
+
+        sess.flush()
+
+        eq_(
+            listener.mock_calls,
+            [
+                call.pending_to_persistent(sess, u1),
+                call.flag_checked(u1)
+            ]
+        )
+
+    def test_detached_to_persistent(self):
+        sess, User, start_events = self._fixture()
+
+        u1 = User(name='u1')
+        sess.add(u1)
+        sess.flush()
+
+        sess.expunge(u1)
+
+        listener = start_events()
+
+        @event.listens_for(sess, "detached_to_persistent")
+        def test_deleted_flag(session, instance):
+            assert instance not in session.deleted
+            assert instance in session
+            listener.flag_checked()
+
+        sess.add(u1)
+
+        eq_(
+            listener.mock_calls,
+            [
+                call.detached_to_persistent(sess, u1),
+                call.flag_checked()
+            ]
+        )
+
+    def test_loaded_as_persistent(self):
+        sess, User, start_events = self._fixture()
+
+        u1 = User(name='u1')
+        sess.add(u1)
+        sess.commit()
+        sess.close()
+
+        listener = start_events()
+
+        @event.listens_for(sess, "loaded_as_persistent")
+        def test_identity_flag(session, instance):
+            assert instance in session
+            assert instance._sa_instance_state.persistent
+            assert instance._sa_instance_state.key in session.identity_map
+            assert not instance._sa_instance_state.deleted
+            assert not instance._sa_instance_state.detached
+            assert instance._sa_instance_state.persistent
+            listener.flag_checked(instance)
+
+        u1 = sess.query(User).filter_by(name='u1').one()
+
+        eq_(
+            listener.mock_calls,
+            [
+                call.loaded_as_persistent(sess, u1),
+                call.flag_checked(u1)
+            ]
+        )
+
+    def test_detached_to_persistent_via_deleted(self):
+        sess, User, start_events = self._fixture()
+
+        u1 = User(name='u1')
+        sess.add(u1)
+        sess.commit()
+        sess.close()
+
+        listener = start_events()
+
+        @event.listens_for(sess, "detached_to_persistent")
+        def test_deleted_flag_persistent(session, instance):
+            assert instance not in session.deleted
+            assert instance in session
+            assert not instance._sa_instance_state.deleted
+            assert not instance._sa_instance_state.detached
+            assert instance._sa_instance_state.persistent
+            listener.dtp_flag_checked(instance)
+
+        @event.listens_for(sess, "persistent_to_deleted")
+        def test_deleted_flag_detached(session, instance):
+            assert instance not in session.deleted
+            assert instance not in session
+            assert not instance._sa_instance_state.persistent
+            assert instance._sa_instance_state.deleted
+            assert not instance._sa_instance_state.detached
+            listener.ptd_flag_checked(instance)
+
+        sess.delete(u1)
+        assert u1 in sess.deleted
+
+        eq_(
+            listener.mock_calls,
+            [
+                call.detached_to_persistent(sess, u1),
+                call.dtp_flag_checked(u1)
+            ]
+        )
+
+        sess.flush()
+
+        eq_(
+            listener.mock_calls,
+            [
+                call.detached_to_persistent(sess, u1),
+                call.dtp_flag_checked(u1),
+                call.persistent_to_deleted(sess, u1),
+                call.ptd_flag_checked(u1),
+            ]
+        )
+
+    def test_detached_to_persistent_via_cascaded_delete(self):
+        sess, User, Address, start_events = self._fixture(include_address=True)
+
+        u1 = User(name='u1')
+        sess.add(u1)
+        a1 = Address(email_address='e1')
+        u1.addresses.append(a1)
+        sess.commit()
+        u1.addresses  # ensure u1.addresses refers to a1 before detachment
+        sess.close()
+
+        listener = start_events()
+
+        @event.listens_for(sess, "detached_to_persistent")
+        def test_deleted_flag(session, instance):
+            assert instance not in session.deleted
+            assert instance in session
+            assert not instance._sa_instance_state.deleted
+            assert not instance._sa_instance_state.detached
+            assert instance._sa_instance_state.persistent
+            listener.flag_checked(instance)
+
+        sess.delete(u1)
+        assert u1 in sess.deleted
+        assert a1 in sess.deleted
+
+        eq_(
+            listener.mock_calls,
+            [
+                call.detached_to_persistent(sess, u1),
+                call.flag_checked(u1),
+                call.detached_to_persistent(sess, a1),
+                call.flag_checked(a1),
+            ]
+        )
+
+        sess.flush()
+
+    def test_persistent_to_deleted(self):
+        sess, User, start_events = self._fixture()
+
+        u1 = User(name='u1')
+        sess.add(u1)
+        sess.commit()
+
+        listener = start_events()
+
+        @event.listens_for(sess, "persistent_to_deleted")
+        def test_deleted_flag(session, instance):
+            assert instance not in session.deleted
+            assert instance not in session
+            assert instance._sa_instance_state.deleted
+            assert not instance._sa_instance_state.detached
+            assert not instance._sa_instance_state.persistent
+            listener.flag_checked(instance)
+
+        sess.delete(u1)
+        assert u1 in sess.deleted
+
+        eq_(
+            listener.mock_calls,
+            []
+        )
+
+        sess.flush()
+        assert u1 not in sess
+
+        eq_(
+            listener.mock_calls,
+            [
+                call.persistent_to_deleted(sess, u1),
+                call.flag_checked(u1)
+            ]
+        )
+
+    def test_persistent_to_detached_via_expunge(self):
+        sess, User, start_events = self._fixture()
+
+        u1 = User(name='u1')
+        sess.add(u1)
+        sess.flush()
+
+        listener = start_events()
+
+        @event.listens_for(sess, "persistent_to_detached")
+        def test_deleted_flag(session, instance):
+            assert instance not in session.deleted
+            assert instance not in session
+            assert not instance._sa_instance_state.deleted
+            assert instance._sa_instance_state.detached
+            assert not instance._sa_instance_state.persistent
+            listener.flag_checked(instance)
+
+        assert u1 in sess
+        sess.expunge(u1)
+        assert u1 not in sess
+
+        eq_(
+            listener.mock_calls,
+            [
+                call.persistent_to_detached(sess, u1),
+                call.flag_checked(u1)
+            ]
+        )
+
+    def test_persistent_to_detached_via_expunge_all(self):
+        sess, User, start_events = self._fixture()
+
+        u1 = User(name='u1')
+        sess.add(u1)
+        sess.flush()
+
+        listener = start_events()
+
+        @event.listens_for(sess, "persistent_to_detached")
+        def test_deleted_flag(session, instance):
+            assert instance not in session.deleted
+            assert instance not in session
+            assert not instance._sa_instance_state.deleted
+            assert instance._sa_instance_state.detached
+            assert not instance._sa_instance_state.persistent
+            listener.flag_checked(instance)
+
+        assert u1 in sess
+        sess.expunge_all()
+        assert u1 not in sess
+
+        eq_(
+            listener.mock_calls,
+            [
+                call.persistent_to_detached(sess, u1),
+                call.flag_checked(u1)
+            ]
+        )
+
+    def test_persistent_to_transient_via_rollback(self):
+        sess, User, start_events = self._fixture()
+
+        u1 = User(name='u1')
+        sess.add(u1)
+        sess.flush()
+
+        listener = start_events()
+
+        @event.listens_for(sess, "persistent_to_transient")
+        def test_deleted_flag(session, instance):
+            assert instance not in session.deleted
+            assert instance not in session
+            assert not instance._sa_instance_state.deleted
+            assert not instance._sa_instance_state.detached
+            assert not instance._sa_instance_state.persistent
+            assert instance._sa_instance_state.transient
+            listener.flag_checked(instance)
+
+        sess.rollback()
+
+        eq_(
+            listener.mock_calls,
+            [
+                call.persistent_to_transient(sess, u1),
+                call.flag_checked(u1)
+            ]
+        )
+
+    def test_deleted_to_persistent_via_rollback(self):
+        sess, User, start_events = self._fixture()
+
+        u1 = User(name='u1')
+        sess.add(u1)
+        sess.commit()
+
+        sess.delete(u1)
+        sess.flush()
+
+        listener = start_events()
+
+        @event.listens_for(sess, "deleted_to_persistent")
+        def test_deleted_flag(session, instance):
+            assert instance not in session.deleted
+            assert instance in session
+            assert not instance._sa_instance_state.deleted
+            assert not instance._sa_instance_state.detached
+            assert instance._sa_instance_state.persistent
+            listener.flag_checked(instance)
+
+        assert u1 not in sess
+        assert u1._sa_instance_state.deleted
+        assert not u1._sa_instance_state.persistent
+        assert not u1._sa_instance_state.detached
+
+        sess.rollback()
+
+        assert u1 in sess
+        assert u1._sa_instance_state.persistent
+        assert not u1._sa_instance_state.deleted
+        assert not u1._sa_instance_state.detached
+
+        eq_(
+            listener.mock_calls,
+            [
+                call.deleted_to_persistent(sess, u1),
+                call.flag_checked(u1)
+            ]
+        )
+
+    def test_deleted_to_detached_via_commit(self):
+        sess, User, start_events = self._fixture()
+
+        u1 = User(name='u1')
+        sess.add(u1)
+        sess.commit()
+
+        sess.delete(u1)
+        sess.flush()
+
+        listener = start_events()
+
+        @event.listens_for(sess, "deleted_to_detached")
+        def test_detached_flag(session, instance):
+            assert instance not in session.deleted
+            assert instance not in session
+            assert not instance._sa_instance_state.deleted
+            assert instance._sa_instance_state.detached
+            listener.flag_checked(instance)
+
+        assert u1 not in sess
+        assert u1._sa_instance_state.deleted
+        assert not u1._sa_instance_state.persistent
+        assert not u1._sa_instance_state.detached
+
+        sess.commit()
+
+        assert u1 not in sess
+        assert not u1._sa_instance_state.deleted
+        assert u1._sa_instance_state.detached
+
+        eq_(
+            listener.mock_calls,
+            [
+                call.deleted_to_detached(sess, u1),
+                call.flag_checked(u1)
+            ]
+        )
+
+
 class MapperExtensionTest(_fixtures.FixtureTest):
 
     """Superseded by MapperEventsTest - test backwards
diff --git a/test/orm/test_hasparent.py b/test/orm/test_hasparent.py
index fd246b527..df4b05980 100644
--- a/test/orm/test_hasparent.py
+++ b/test/orm/test_hasparent.py
@@ -116,7 +116,7 @@ class ParentRemovalTest(fixtures.MappedTest):
         User = self.classes.User
         s, u1, a1 = self._fixture()
 
-        s._expunge_state(attributes.instance_state(u1))
+        s._expunge_states([attributes.instance_state(u1)])
         del u1
         gc_collect()
 
@@ -178,7 +178,7 @@ class ParentRemovalTest(fixtures.MappedTest):
         u2 = User(addresses=[a1])
         s.add(u2)
         s.flush()
-        s._expunge_state(attributes.instance_state(u2))
+        s._expunge_states([attributes.instance_state(u2)])
         del u2
         gc_collect()
 
diff --git a/test/orm/test_load_on_fks.py b/test/orm/test_load_on_fks.py
index 813d8d17a..471c8665a 100644
--- a/test/orm/test_load_on_fks.py
+++ b/test/orm/test_load_on_fks.py
@@ -301,7 +301,8 @@ class LoadOnFKsTest(AssertsExecutionResults, fixtures.TestBase):
                             c2 = Child()
 
                             if attach:
-                                sess._attach(instance_state(c2))
+                                state = instance_state(c2)
+                                state.session_id = sess.hash_key
 
                             if enable_relationship_rel:
                                 sess.enable_relationship_loading(c2)
diff --git a/test/orm/test_session.py b/test/orm/test_session.py
index 58551d763..f6ddcb566 100644
--- a/test/orm/test_session.py
+++ b/test/orm/test_session.py
@@ -493,8 +493,10 @@ class SessionStateTest(_fixtures.FixtureTest):
                               'is already attached to session',
                               s2.delete, user)
         u2 = s2.query(User).get(user.id)
-        assert_raises_message(sa.exc.InvalidRequestError,
-                              'another instance with key', s.delete, u2)
+        s2.expunge(u2)
+        assert_raises_message(
+            sa.exc.InvalidRequestError,
+            'another instance .* is already present', s.delete, u2)
         s.expire(user)
         s.expunge(user)
         assert user not in s
@@ -543,8 +545,14 @@ class SessionStateTest(_fixtures.FixtureTest):
             s.expunge(u2)
             s.identity_map.add(sa.orm.attributes.instance_state(u1))
 
-            assert_raises(AssertionError, s.identity_map.add,
-                          sa.orm.attributes.instance_state(u2))
+            assert_raises_message(
+                sa.exc.InvalidRequestError,
+                "Can't attach instance <User.*?>; another instance "
+                "with key .*? is already "
+                "present in this session.",
+                s.identity_map.add,
+                sa.orm.attributes.instance_state(u2)
+            )
 
     def test_pickled_update(self):
         users, User = self.tables.users, pickleable.User
@@ -581,7 +589,13 @@ class SessionStateTest(_fixtures.FixtureTest):
         assert u2 is not None and u2 is not u1
         assert u2 in sess
 
-        assert_raises(AssertionError, lambda: sess.add(u1))
+        assert_raises_message(
+            sa.exc.InvalidRequestError,
+            "Can't attach instance <User.*?>; another instance "
+            "with key .*? is already "
+            "present in this session.",
+            sess.add, u1
+        )
 
         sess.expunge(u2)
         assert u2 not in sess
@@ -1124,11 +1138,56 @@ class WeakIdentityMapTest(_fixtures.FixtureTest):
 class StrongIdentityMapTest(_fixtures.FixtureTest):
     run_inserts = None
 
+    def _strong_ident_fixture(self):
+        sess = create_session(weak_identity_map=False)
+        return sess, sess.prune
+
+    def _event_fixture(self):
+        session = create_session()
+
+        @event.listens_for(session, "pending_to_persistent")
+        @event.listens_for(session, "deleted_to_persistent")
+        @event.listens_for(session, "detached_to_persistent")
+        @event.listens_for(session, "loaded_as_persistent")
+        def strong_ref_object(sess, instance):
+            if 'refs' not in sess.info:
+                sess.info['refs'] = refs = set()
+            else:
+                refs = sess.info['refs']
+
+            refs.add(instance)
+
+        @event.listens_for(session, "persistent_to_detached")
+        @event.listens_for(session, "persistent_to_deleted")
+        @event.listens_for(session, "persistent_to_transient")
+        def deref_object(sess, instance):
+            sess.info['refs'].discard(instance)
+
+        def prune():
+            if 'refs' not in session.info:
+                return 0
+
+            sess_size = len(session.identity_map)
+            session.info['refs'].clear()
+            gc_collect()
+            session.info['refs'] = set(
+                s.obj() for s in session.identity_map.all_states())
+            return sess_size - len(session.identity_map)
+
+        return session, prune
+
     @testing.uses_deprecated()
-    def test_strong_ref(self):
+    def test_strong_ref_imap(self):
+        self._test_strong_ref(self._strong_ident_fixture)
+
+    def test_strong_ref_events(self):
+        self._test_strong_ref(self._event_fixture)
+
+    def _test_strong_ref(self, fixture):
+        s, prune = fixture()
+
         users, User = self.tables.users, self.classes.User
 
-        s = create_session(weak_identity_map=False)
         mapper(User, users)
 
         # save user
@@ -1148,12 +1207,19 @@ class StrongIdentityMapTest(_fixtures.FixtureTest):
         eq_(users.select().execute().fetchall(), [(user.id, 'u2')])
 
     @testing.uses_deprecated()
+    def test_prune_imap(self):
+        self._test_prune(self._strong_ident_fixture)
+
+    def test_prune_events(self):
+        self._test_prune(self._event_fixture)
+
     @testing.fails_if(lambda: pypy, "pypy has a real GC")
     @testing.fails_on('+zxjdbc', 'http://www.sqlalchemy.org/trac/ticket/1473')
-    def test_prune(self):
+    def _test_prune(self, fixture):
+        s, prune = fixture()
+
         users, User = self.tables.users, self.classes.User
 
-        s = create_session(weak_identity_map=False)
         mapper(User, users)
 
         for o in [User(name='u%s' % x) for x in range(10)]:
@@ -1161,43 +1227,44 @@ class StrongIdentityMapTest(_fixtures.FixtureTest):
         # o is still live after this loop...
 
         self.assert_(len(s.identity_map) == 0)
-        self.assert_(s.prune() == 0)
+        eq_(prune(), 0)
         s.flush()
         gc_collect()
-        self.assert_(s.prune() == 9)
+        eq_(prune(), 9)
+        # o is still in local scope here, so still present
         self.assert_(len(s.identity_map) == 1)
 
         id = o.id
         del o
-        self.assert_(s.prune() == 1)
+        eq_(prune(), 1)
         self.assert_(len(s.identity_map) == 0)
 
         u = s.query(User).get(id)
-        self.assert_(s.prune() == 0)
+        eq_(prune(), 0)
         self.assert_(len(s.identity_map) == 1)
         u.name = 'squiznart'
         del u
-        self.assert_(s.prune() == 0)
+        eq_(prune(), 0)
         self.assert_(len(s.identity_map) == 1)
         s.flush()
-        self.assert_(s.prune() == 1)
+        eq_(prune(), 1)
         self.assert_(len(s.identity_map) == 0)
 
         s.add(User(name='x'))
-        self.assert_(s.prune() == 0)
+        eq_(prune(), 0)
         self.assert_(len(s.identity_map) == 0)
         s.flush()
         self.assert_(len(s.identity_map) == 1)
-        self.assert_(s.prune() == 1)
+        eq_(prune(), 1)
         self.assert_(len(s.identity_map) == 0)
 
         u = s.query(User).get(id)
         s.delete(u)
         del u
-        self.assert_(s.prune() == 0)
+        eq_(prune(), 0)
         self.assert_(len(s.identity_map) == 1)
         s.flush()
-        self.assert_(s.prune() == 0)
+        eq_(prune(), 0)
         self.assert_(len(s.identity_map) == 0)
 
 
diff --git a/test/orm/test_transaction.py b/test/orm/test_transaction.py
index 91846a67e..73c6b977a 100644
--- a/test/orm/test_transaction.py
+++ b/test/orm/test_transaction.py
@@ -895,7 +895,13 @@ class AutoExpireTest(_LocalFixture):
         assert u1_state.obj() is None
 
         s.rollback()
-        assert u1_state in s.identity_map.all_states()
+        # new in 1.1, not in identity map if the object was
+        # gc'ed and we restore snapshot; we've changed update_impl
+        # to just skip this object
+        assert u1_state not in s.identity_map.all_states()
+
+        # in any version, the state is replaced by the query
+        # because the identity map would switch it
         u1 = s.query(User).filter_by(name='ed').one()
         assert u1_state not in s.identity_map.all_states()
         assert s.scalar(users.count()) == 1