Run eager loaders on unexpire

Eager loaders, such as joined loading, SELECT IN loading, etc., when
configured on a mapper or via query options will now be invoked during
the refresh on an expired object; in the case of selectinload and
subqueryload, since the additional load is for a single object only,
the "immediateload" scheme is used in these cases which resembles the
single-parent query emitted by lazy loading.

Change-Id: I7ca2c77bff58dc21015d60093a88c387937376b2
Fixes: #1763

6 files changed, 59 insertions, 8 deletions

diff --git a/lib/sqlalchemy/orm/attributes.py b/lib/sqlalchemy/orm/attributes.py
index 2f54fcd32..d50eab03e 100644
--- a/lib/sqlalchemy/orm/attributes.py
+++ b/lib/sqlalchemy/orm/attributes.py
@@ -702,7 +702,8 @@ class AttributeImpl(object):
                 if not passive & CALLABLES_OK:
                     return PASSIVE_NO_RESULT
 
-                if key in state.expired_attributes:
+                if self.accepts_scalar_loader and \
+                        key in state.expired_attributes:
                     value = state._load_expired(state, passive)
                 elif key in state.callables:
                     callable_ = state.callables[key]
diff --git a/lib/sqlalchemy/orm/instrumentation.py b/lib/sqlalchemy/orm/instrumentation.py
index ee0cc0600..61184ee0a 100644
--- a/lib/sqlalchemy/orm/instrumentation.py
+++ b/lib/sqlalchemy/orm/instrumentation.py
@@ -123,6 +123,10 @@ class ClassManager(dict):
             ]
         )
 
+    @_memoized_key_collection
+    def _loader_impls(self):
+        return frozenset([attr.impl for attr in self.values()])
+
     @util.memoized_property
     def mapper(self):
         # raises unless self.mapper has been assigned
diff --git a/lib/sqlalchemy/orm/loading.py b/lib/sqlalchemy/orm/loading.py
index fd283f432..f07067d17 100644
--- a/lib/sqlalchemy/orm/loading.py
+++ b/lib/sqlalchemy/orm/loading.py
@@ -269,6 +269,10 @@ def load_on_pk_identity(
     else:
         version_check = False
 
+    if refresh_state and refresh_state.load_options:
+        q = q._with_current_path(refresh_state.load_path.parent)
+        q = q._conditional_options(refresh_state.load_options)
+
     q._get_options(
         populate_existing=bool(refresh_state),
         version_check=version_check,
diff --git a/lib/sqlalchemy/orm/mapper.py b/lib/sqlalchemy/orm/mapper.py
index 5e8d25647..0c4e1543b 100644
--- a/lib/sqlalchemy/orm/mapper.py
+++ b/lib/sqlalchemy/orm/mapper.py
@@ -2812,11 +2812,14 @@ class Mapper(InspectionAttr):
         """
         props = self._props
 
+        col_attribute_names = set(attribute_names).intersection(
+            state.mapper.column_attrs.keys()
+        )
         tables = set(
             chain(
                 *[
                     sql_util.find_tables(c, check_columns=True)
-                    for key in attribute_names
+                    for key in col_attribute_names
                     for c in props[key].columns
                 ]
             )
@@ -2884,7 +2887,7 @@ class Mapper(InspectionAttr):
         cond = sql.and_(*allconds)
 
         cols = []
-        for key in attribute_names:
+        for key in col_attribute_names:
             cols.extend(props[key].columns)
         return sql.select(cols, cond, use_labels=True)
 
diff --git a/lib/sqlalchemy/orm/state.py b/lib/sqlalchemy/orm/state.py
index f6c06acc8..ead9bf2bb 100644
--- a/lib/sqlalchemy/orm/state.py
+++ b/lib/sqlalchemy/orm/state.py
@@ -595,12 +595,23 @@ class InstanceState(interfaces.InspectionAttrInfo):
         self.expired_attributes.update(
             [
                 impl.key
-                for impl in self.manager._scalar_loader_impls
+                for impl in self.manager._loader_impls
                 if impl.expire_missing or impl.key in dict_
             ]
         )
 
         if self.callables:
+            # the per state loader callables we can remove here are
+            # LoadDeferredColumns, which undefers a column at the instance
+            # level that is mapped with deferred, and LoadLazyAttribute,
+            # which lazy loads a relationship at the instance level that
+            # is mapped with "noload" or perhaps "immediateload".
+            # Before 1.4, only column-based
+            # attributes could be considered to be "expired", so here they
+            # were the only ones "unexpired", which means to make them deferred
+            # again.   For the moment, as of 1.4 we also apply the same
+            # treatment relationships now, that is, an instance level lazy
+            # loader is reset in the same way as a column loader.
             for k in self.expired_attributes.intersection(self.callables):
                 del self.callables[k]
 
diff --git a/lib/sqlalchemy/orm/strategies.py b/lib/sqlalchemy/orm/strategies.py
index fc86076b1..49b5b4f64 100644
--- a/lib/sqlalchemy/orm/strategies.py
+++ b/lib/sqlalchemy/orm/strategies.py
@@ -702,6 +702,9 @@ class LazyLoader(AbstractRelationshipLoader, util.MemoizedSlots):
             if _none_set.issuperset(primary_key_identity):
                 return None
 
+            if self.key in state.dict:
+                return attributes.ATTR_WAS_SET
+
             # look for this identity in the identity map.  Delegate to the
             # Query class in use, as it may have special rules for how it
             # does this, including how it decides what the correct
@@ -841,6 +844,8 @@ class LazyLoader(AbstractRelationshipLoader, util.MemoizedSlots):
                 )
 
         lazy_clause, params = self._generate_lazy_clause(state, passive)
+        if self.key in state.dict:
+            return attributes.ATTR_WAS_SET
 
         if pending:
             if util.has_intersection(orm_util._none_set, params.values()):
@@ -934,8 +939,21 @@ class LoadLazyAttribute(object):
         return strategy._load_for_state(state, passive)
 
 
+class PostLoader(AbstractRelationshipLoader):
+    """A relationship loader that emits a second SELECT statement."""
+
+    def _immediateload_create_row_processor(
+        self, context, path, loadopt, mapper, result, adapter, populators
+    ):
+        return self.parent_property._get_strategy(
+            (("lazy", "immediate"),)
+        ).create_row_processor(
+            context, path, loadopt, mapper, result, adapter, populators
+        )
+
+
 @properties.RelationshipProperty.strategy_for(lazy="immediate")
-class ImmediateLoader(AbstractRelationshipLoader):
+class ImmediateLoader(PostLoader):
     __slots__ = ()
 
     def init_class_attribute(self, mapper):
@@ -967,7 +985,7 @@ class ImmediateLoader(AbstractRelationshipLoader):
 
 @log.class_logger
 @properties.RelationshipProperty.strategy_for(lazy="subquery")
-class SubqueryLoader(AbstractRelationshipLoader):
+class SubqueryLoader(PostLoader):
     __slots__ = ("join_depth",)
 
     def __init__(self, parent, strategy_key):
@@ -991,7 +1009,7 @@ class SubqueryLoader(AbstractRelationshipLoader):
         **kwargs
     ):
 
-        if not context.query._enable_eagerloads:
+        if not context.query._enable_eagerloads or context.refresh_state:
             return
         elif context.query._yield_per:
             context.query._no_yield_per("subquery")
@@ -1320,6 +1338,11 @@ class SubqueryLoader(AbstractRelationshipLoader):
     def create_row_processor(
         self, context, path, loadopt, mapper, result, adapter, populators
     ):
+        if context.refresh_state:
+            return self._immediateload_create_row_processor(
+                context, path, loadopt, mapper, result, adapter, populators
+            )
+
         if not self.parent.class_manager[self.key].impl.supports_population:
             raise sa_exc.InvalidRequestError(
                 "'%s' does not support object "
@@ -2066,7 +2089,7 @@ class JoinedLoader(AbstractRelationshipLoader):
 
 @log.class_logger
 @properties.RelationshipProperty.strategy_for(lazy="selectin")
-class SelectInLoader(AbstractRelationshipLoader, util.MemoizedSlots):
+class SelectInLoader(PostLoader, util.MemoizedSlots):
     __slots__ = (
         "join_depth",
         "omit_join",
@@ -2182,6 +2205,11 @@ class SelectInLoader(AbstractRelationshipLoader, util.MemoizedSlots):
     def create_row_processor(
         self, context, path, loadopt, mapper, result, adapter, populators
     ):
+        if context.refresh_state:
+            return self._immediateload_create_row_processor(
+                context, path, loadopt, mapper, result, adapter, populators
+            )
+
         if not self.parent.class_manager[self.key].impl.supports_population:
             raise sa_exc.InvalidRequestError(
                 "'%s' does not support object "