A performance fix related to the usage of the :func:`.defer` option

when loading mapped entities.   The function overhead of applying
a per-object deferred callable to an instance at load time was
significantly higher than that of just loading the data from the row
(note that ``defer()`` is meant to reduce DB/network overhead, not
necessarily function call count); the function call overhead is now
less than that of loading data from the column in all cases.  There
is also a reduction in the number of "lazy callable" objects created
per load from N (total deferred values in the result) to 1 (total
number of deferred cols).
[ticket:2778]

3 files changed, 93 insertions, 14 deletions

diff --git a/test/aaa_profiling/test_orm.py b/test/aaa_profiling/test_orm.py
index 199b96e5f..6d71468b7 100644
--- a/test/aaa_profiling/test_orm.py
+++ b/test/aaa_profiling/test_orm.py
@@ -2,7 +2,7 @@ from sqlalchemy.testing import eq_, assert_raises, \
     assert_raises_message
 from sqlalchemy import exc as sa_exc, util, Integer, String, ForeignKey
 from sqlalchemy.orm import exc as orm_exc, mapper, relationship, \
-    sessionmaker, Session
+    sessionmaker, Session, defer
 from sqlalchemy import testing
 from sqlalchemy.testing import profiling
 from sqlalchemy.testing import fixtures
@@ -257,4 +257,56 @@ class MergeBackrefsTest(fixtures.MappedTest):
         ]:
             s.merge(a)
 
+class DeferOptionsTest(fixtures.MappedTest):
+    @classmethod
+    def define_tables(cls, metadata):
+        Table('a', metadata,
+            Column('id', Integer, primary_key=True),
+            Column('x', String(5)),
+            Column('y', String(5)),
+            Column('z', String(5)),
+            Column('q', String(5)),
+            Column('p', String(5)),
+            Column('r', String(5)),
+        )
+
+    @classmethod
+    def setup_classes(cls):
+        class A(cls.Basic):
+            pass
+
+    @classmethod
+    def setup_mappers(cls):
+        A = cls.classes.A
+        a = cls.tables.a
+        mapper(A, a)
+
+    @classmethod
+    def insert_data(cls):
+        A = cls.classes.A
+        s = Session()
+        s.add_all([
+            A(id=i,
+                **dict((letter, "%s%d" % (letter, i)) for letter in
+                        ['x', 'y', 'z', 'p', 'q', 'r'])
+            ) for i in range(1, 1001)
+        ])
+        s.commit()
+
+    @profiling.function_call_count(variance=.10)
+    def test_baseline(self):
+        # as of [ticket:2778], this is at 39025
+        A = self.classes.A
+        s = Session()
+        s.query(A).all()
+
+    @profiling.function_call_count(variance=.10)
+    def test_defer_many_cols(self):
+        # with [ticket:2778], this goes from 50805 to 32817,
+        # as it should be fewer function calls than the baseline
+        A = self.classes.A
+        s = Session()
+        s.query(A).options(
+            *[defer(letter) for letter in ['x', 'y', 'z', 'p', 'q', 'r']]).\
+            all()
 
diff --git a/test/orm/test_attributes.py b/test/orm/test_attributes.py
index 0d253e4e3..4bcecb71b 100644
--- a/test/orm/test_attributes.py
+++ b/test/orm/test_attributes.py
@@ -11,12 +11,19 @@ from sqlalchemy.util import jython
 from sqlalchemy import event
 from sqlalchemy import testing
 from sqlalchemy.testing.mock import Mock, call
+from sqlalchemy.orm.state import InstanceState
 
 # global for pickling tests
 MyTest = None
 MyTest2 = None
 
 
+
+def _set_callable(state, dict_, key, callable_):
+    fn = InstanceState._row_processor(state.manager, callable_, key)
+    fn(state, dict_, None)
+
+
 class AttributeImplAPITest(fixtures.MappedTest):
     def _scalar_obj_fixture(self):
         class A(object):
@@ -602,8 +609,10 @@ class AttributesTest(fixtures.ORMTest):
 
         """
 
-        class Post(object):pass
-        class Blog(object):pass
+        class Post(object):
+            pass
+        class Blog(object):
+            pass
         instrumentation.register_class(Post)
         instrumentation.register_class(Blog)
 
@@ -618,10 +627,10 @@ class AttributesTest(fixtures.ORMTest):
         # create objects as if they'd been freshly loaded from the database (without history)
         b = Blog()
         p1 = Post()
-        attributes.instance_state(b)._set_callable(attributes.instance_dict(b),
-                                                    'posts', lambda passive:[p1])
-        attributes.instance_state(p1)._set_callable(attributes.instance_dict(p1),
-                                                    'blog', lambda passive:b)
+        _set_callable(attributes.instance_state(b), attributes.instance_dict(b),
+                                                    'posts', lambda state, passive:[p1])
+        _set_callable(attributes.instance_state(p1), attributes.instance_dict(p1),
+                                                    'blog', lambda state, passive:b)
         p1, attributes.instance_state(b)._commit_all(attributes.instance_dict(b))
 
         # no orphans (called before the lazy loaders fire off)
@@ -2628,7 +2637,7 @@ class TestUnlink(fixtures.TestBase):
         coll = a1.bs
         a1.bs.append(B())
         state = attributes.instance_state(a1)
-        state._set_callable(state.dict, "bs", lambda: B())
+        _set_callable(state, state.dict, "bs", lambda: B())
         assert_raises(
             Warning,
             coll.append, B()
diff --git a/test/profiles.txt b/test/profiles.txt
index 95813f4f7..f5e480d0f 100644
--- a/test/profiles.txt
+++ b/test/profiles.txt
@@ -1,15 +1,15 @@
 # /Users/classic/dev/sqlalchemy/test/profiles.txt
 # This file is written out on a per-environment basis.
-# For each test in aaa_profiling, the corresponding function and
+# For each test in aaa_profiling, the corresponding function and 
 # environment is located within this file.  If it doesn't exist,
 # the test is skipped.
-# If a callcount does exist, it is compared to what we received.
+# If a callcount does exist, it is compared to what we received. 
 # assertions are raised if the counts do not match.
-#
-# To add a new callcount test, apply the function_call_count
-# decorator and re-run the tests using the --write-profiles
+# 
+# To add a new callcount test, apply the function_call_count 
+# decorator and re-run the tests using the --write-profiles 
 # option - this file will be rewritten including the new count.
-#
+# 
 
 # TEST: test.aaa_profiling.test_compiler.CompileTest.test_insert
 
@@ -91,6 +91,24 @@ test.aaa_profiling.test_compiler.CompileTest.test_update_whereclause 3.3_oracle_
 test.aaa_profiling.test_compiler.CompileTest.test_update_whereclause 3.3_postgresql_psycopg2_nocextensions 136
 test.aaa_profiling.test_compiler.CompileTest.test_update_whereclause 3.3_sqlite_pysqlite_nocextensions 136
 
+# TEST: test.aaa_profiling.test_orm.DeferOptionsTest.test_baseline
+
+test.aaa_profiling.test_orm.DeferOptionsTest.test_baseline 2.7_mysql_mysqldb_cextensions 30052
+test.aaa_profiling.test_orm.DeferOptionsTest.test_baseline 2.7_mysql_mysqldb_nocextensions 39069
+test.aaa_profiling.test_orm.DeferOptionsTest.test_baseline 2.7_postgresql_psycopg2_cextensions 42032
+test.aaa_profiling.test_orm.DeferOptionsTest.test_baseline 2.7_postgresql_psycopg2_nocextensions 51049
+test.aaa_profiling.test_orm.DeferOptionsTest.test_baseline 2.7_sqlite_pysqlite_cextensions 30008
+test.aaa_profiling.test_orm.DeferOptionsTest.test_baseline 2.7_sqlite_pysqlite_nocextensions 39025
+
+# TEST: test.aaa_profiling.test_orm.DeferOptionsTest.test_defer_many_cols
+
+test.aaa_profiling.test_orm.DeferOptionsTest.test_defer_many_cols 2.7_mysql_mysqldb_cextensions 29994
+test.aaa_profiling.test_orm.DeferOptionsTest.test_defer_many_cols 2.7_mysql_mysqldb_nocextensions 32867
+test.aaa_profiling.test_orm.DeferOptionsTest.test_defer_many_cols 2.7_postgresql_psycopg2_cextensions 29830
+test.aaa_profiling.test_orm.DeferOptionsTest.test_defer_many_cols 2.7_postgresql_psycopg2_nocextensions 32835
+test.aaa_profiling.test_orm.DeferOptionsTest.test_defer_many_cols 2.7_sqlite_pysqlite_cextensions 29812
+test.aaa_profiling.test_orm.DeferOptionsTest.test_defer_many_cols 2.7_sqlite_pysqlite_nocextensions 32817
+
 # TEST: test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_identity
 
 test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_identity 2.6_sqlite_pysqlite_nocextensions 17987