Memoize load_path in all cases, run quick populators for path change

Adds a new variant to the "isnew" state within entity loading
for isnew=False, but the load path is new.  This is to address
the use case of an entity appearing in multiple places in
the row in a more generalized way than the fixes in [ticket:3431],
[ticket:3811] in that loading.py will be able to tell the
populator that this row is not "isnew" but is a "new" path
for the entity.   For the moment, the new information is only
being applied to the use of "quick" populators so that
simple column loads can take place on top of a deferred loader
from elsewhere in the row.

As part of this change, state.load_path() will now always
be populated with the "path" that was in effect when this state
was originally loaded, which for multi-path loads of the
same entity is still non-deterministic.  Ideally there'd be some
kind of "here's all the paths that loaded this state and how"
type of data structure though it's not clear if that could be
done while maintaining performance.

Fixes: #3822
Change-Id: Ib915365353dfcca09e15c24001a8581113b97d5e

2 files changed, 36 insertions, 14 deletions

diff --git a/lib/sqlalchemy/ext/baked.py b/lib/sqlalchemy/ext/baked.py
index 2f658edf3..a6191f5cb 100644
--- a/lib/sqlalchemy/ext/baked.py
+++ b/lib/sqlalchemy/ext/baked.py
@@ -441,14 +441,12 @@ class BakedLazyLoader(strategies.LazyLoader):
         if pending or passive & attributes.NO_AUTOFLUSH:
             q.add_criteria(lambda q: q.autoflush(False))
 
-        if state.load_path:
+        if state.load_options:
             q.spoil()
+            args = state.load_path[self.parent_property]
             q.add_criteria(
                 lambda q:
-                q._with_current_path(state.load_path[self.parent_property]))
-
-        if state.load_options:
-            q.spoil()
+                q._with_current_path(args), args)
             q.add_criteria(
                 lambda q: q._conditional_options(*state.load_options))
 
diff --git a/lib/sqlalchemy/orm/loading.py b/lib/sqlalchemy/orm/loading.py
index d457f3c63..5ee8f6abe 100644
--- a/lib/sqlalchemy/orm/loading.py
+++ b/lib/sqlalchemy/orm/loading.py
@@ -330,9 +330,8 @@ def _instance_processor(
                 context, path, mapper, result, adapter, populators)
 
     propagate_options = context.propagate_options
-    if propagate_options:
-        load_path = context.query._current_path + path \
-            if context.query._current_path.path else path
+    load_path = context.query._current_path + path \
+        if context.query._current_path.path else path
 
     session_identity_map = context.session.identity_map
 
@@ -426,12 +425,15 @@ def _instance_processor(
             # full population routines.  Objects here are either
             # just created, or we are doing a populate_existing
 
-            if isnew and propagate_options:
+            # be conservative about setting load_path when populate_existing
+            # is in effect; want to maintain options from the original
+            # load.  see test_expire->test_refresh_maintains_deferred_options
+            if isnew and (propagate_options or not populate_existing):
                 state.load_options = propagate_options
                 state.load_path = load_path
 
             _populate_full(
-                context, row, state, dict_, isnew,
+                context, row, state, dict_, isnew, load_path,
                 loaded_instance, populate_existing, populators)
 
             if isnew:
@@ -463,7 +465,7 @@ def _instance_processor(
                 # and add to the "context.partials" collection.
 
                 to_load = _populate_partial(
-                    context, row, state, dict_, isnew,
+                    context, row, state, dict_, isnew, load_path,
                     unloaded, populators)
 
                 if isnew:
@@ -486,7 +488,7 @@ def _instance_processor(
 
 
 def _populate_full(
-        context, row, state, dict_, isnew,
+        context, row, state, dict_, isnew, load_path,
         loaded_instance, populate_existing, populators):
     if isnew:
         # first time we are seeing a row with this identity.
@@ -507,15 +509,37 @@ def _populate_full(
             populator(state, dict_, row)
         for key, populator in populators["delayed"]:
             populator(state, dict_, row)
+    elif load_path != state.load_path:
+        # new load path, e.g. object is present in more than one
+        # column position in a series of rows
+        state.load_path = load_path
+
+        # if we have data, and the data isn't in the dict, OK, let's put
+        # it in.
+        for key, getter in populators["quick"]:
+            if key not in dict_:
+                dict_[key] = getter(row)
+
+        # otherwise treat like an "already seen" row
+        for key, populator in populators["existing"]:
+            populator(state, dict_, row)
+            # TODO:  allow "existing" populator to know this is
+            # a new path for the state:
+            # populator(state, dict_, row, new_path=True)
+
     else:
-        # have already seen rows with this identity.
+        # have already seen rows with this identity in this same path.
         for key, populator in populators["existing"]:
             populator(state, dict_, row)
 
+            # TODO: same path
+            # populator(state, dict_, row, new_path=False)
+
 
 def _populate_partial(
-        context, row, state, dict_, isnew,
+        context, row, state, dict_, isnew, load_path,
         unloaded, populators):
+
     if not isnew:
         to_load = context.partials[state]
         for key, populator in populators["existing"]: