Merge branch 'sverker/22/dist-lock-bug/ERL-1337/OTP-16869' into maint-22

* sverker/22/dist-lock-bug/ERL-1337/OTP-16869:
  erts: Fix race on ErtsMonLnkDist in erts_net_message

5 files changed, 63 insertions, 29 deletions

diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index f1cd97c3bf..494d3b38a9 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -1601,16 +1601,22 @@ int erts_net_message(Port *prt,
                                                  from, to);
             ASSERT(ldp->a.other.item == to);
             ASSERT(eq(ldp->b.other.item, from));
-            code = erts_link_dist_insert(&ldp->a, dep->mld);
-            ASSERT(code);
 
-            if (erts_proc_sig_send_link(NULL, to, &ldp->b))
-                break; /* done */
+            code = erts_link_dist_insert(&ldp->a, ede.mld);
+            if (erts_proc_sig_send_link(NULL, to, &ldp->b)) {
+                if (!code) {
+                    /* Race: connection already down => send link exit */
+                    erts_proc_sig_send_link_exit(NULL, THE_NON_VALUE, &ldp->a,
+                                                 am_noconnection, NIL);
+                }
+                break; /* Done */
+            }
 
             /* Failed to send signal; cleanup and reply noproc... */
-
-            code = erts_link_dist_delete(&ldp->a);
-            ASSERT(code);
+            if (code) {
+                code = erts_link_dist_delete(&ldp->a);
+                ASSERT(code);
+            }
             erts_link_release_both(ldp);
         }
 
@@ -1690,8 +1696,11 @@ int erts_net_message(Port *prt,
             mdp = erts_monitor_create(ERTS_MON_TYPE_DIST_PROC,
                                       ref, watcher, pid, name);
 
-            code = erts_monitor_dist_insert(&mdp->origin, dep->mld);
-            ASSERT(code); (void)code;
+            if (!erts_monitor_dist_insert(&mdp->origin, ede.mld)) {
+                /* Race: connection down => do nothing */
+                erts_monitor_release_both(mdp);
+                break;
+            }
 
             if (erts_proc_sig_send_monitor(&mdp->target, pid))
                 break; /* done */
@@ -1741,16 +1750,17 @@ int erts_net_message(Port *prt,
             ;
         }
         else if (is_atom(watched)) {
-            ErtsMonLnkDist *mld = dep->mld;
             ErtsMonitor *mon;
 
-            erts_mtx_lock(&mld->mtx);
-
-            mon = erts_monitor_tree_lookup(mld->orig_name_monitors, ref);
-            if (mon)
-                erts_monitor_tree_delete(&mld->orig_name_monitors, mon);
-
-            erts_mtx_unlock(&mld->mtx);
+            erts_mtx_lock(&ede.mld->mtx);
+            if (ede.mld->alive) {
+                mon = erts_monitor_tree_lookup(ede.mld->orig_name_monitors, ref);
+                if (mon)
+                    erts_monitor_tree_delete(&ede.mld->orig_name_monitors, mon);
+            }
+            else
+                mon = NULL;
+            erts_mtx_unlock(&ede.mld->mtx);
 
             if (mon)
                 erts_proc_sig_send_demonitor(mon);
diff --git a/erts/emulator/beam/erl_monitor_link.c b/erts/emulator/beam/erl_monitor_link.c
index afee54a69a..b44723f203 100644
--- a/erts/emulator/beam/erl_monitor_link.c
+++ b/erts/emulator/beam/erl_monitor_link.c
@@ -543,9 +543,10 @@ erts_mon_link_dist_create(Eterm nodename)
     return mld;
 }
 
-void
-erts_mon_link_dist_destroy__(ErtsMonLnkDist *mld)
+static void
+mon_link_dist_destroy(void* vmld)
 {
+    ErtsMonLnkDist *mld = (ErtsMonLnkDist*)vmld;
     ERTS_ML_ASSERT(erts_atomic_read_nob(&mld->refc) == 0);
     ERTS_ML_ASSERT(!mld->alive);
     ERTS_ML_ASSERT(!mld->links);
@@ -556,6 +557,21 @@ erts_mon_link_dist_destroy__(ErtsMonLnkDist *mld)
     erts_free(ERTS_ALC_T_ML_DIST, mld);
 }
 
+void
+erts_schedule_mon_link_dist_destruction__(ErtsMonLnkDist *mld)
+{
+    ERTS_ML_ASSERT(erts_atomic_read_nob(&mld->refc) == 0);
+    ERTS_ML_ASSERT(!mld->alive);
+    ERTS_ML_ASSERT(!mld->links);
+    ERTS_ML_ASSERT(!mld->monitors);
+    ERTS_ML_ASSERT(!mld->orig_name_monitors);
+
+    erts_schedule_thr_prgr_later_cleanup_op(mon_link_dist_destroy,
+                                            mld,
+                                            &mld->cleanup_lop,
+                                            sizeof(ErtsMonLnkDist));
+}
+
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\
  * Monitor Operations                                                        *
 \*                                                                           */
diff --git a/erts/emulator/beam/erl_monitor_link.h b/erts/emulator/beam/erl_monitor_link.h
index 678025cb05..df3afa1970 100644
--- a/erts/emulator/beam/erl_monitor_link.h
+++ b/erts/emulator/beam/erl_monitor_link.h
@@ -396,6 +396,11 @@
 #include "erl_proc_sig_queue.h"
 #undef ERTS_PROC_SIG_QUEUE_TYPE_ONLY
 
+#define ERL_THR_PROGRESS_TSD_TYPE_ONLY
+#include "erl_thr_progress.h"
+#undef ERL_THR_PROGRESS_TSD_TYPE_ONLY
+
+
 #if defined(DEBUG) || 0
 #  define ERTS_ML_DEBUG
 #else
@@ -468,7 +473,7 @@ struct ErtsMonLnkNode__ {
     Uint16 type;
 };
 
-typedef struct {
+typedef struct ErtsMonLnkDist__ {
     Eterm nodename;
     Uint32 connection_id;
     erts_atomic_t refc;
@@ -478,6 +483,7 @@ typedef struct {
     ErtsMonLnkNode *monitors; /* Monitor double linked circular list */
     ErtsMonLnkNode *orig_name_monitors; /* Origin named monitors
                                            read-black tree */
+    ErtsThrPrgrLaterOp cleanup_lop;
 } ErtsMonLnkDist;
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\
@@ -530,7 +536,7 @@ ERTS_GLB_INLINE void erts_ml_dl_list_delete__(ErtsMonLnkNode **list,
                                               ErtsMonLnkNode *ml);
 ERTS_GLB_INLINE ErtsMonLnkNode *erts_ml_dl_list_first__(ErtsMonLnkNode *list);
 ERTS_GLB_INLINE ErtsMonLnkNode *erts_ml_dl_list_last__(ErtsMonLnkNode *list);
-void erts_mon_link_dist_destroy__(ErtsMonLnkDist *mld);
+void erts_schedule_mon_link_dist_destruction__(ErtsMonLnkDist *mld);
 ERTS_GLB_INLINE void *erts_ml_node_to_main_struct__(ErtsMonLnkNode *mln);
 
 /* implementations for globally inlined misc functions... */
@@ -548,7 +554,7 @@ erts_mon_link_dist_dec_refc(ErtsMonLnkDist *mld)
 {
     ERTS_ML_ASSERT(erts_atomic_read_nob(&mld->refc) > 0);
     if (erts_atomic_dec_read_nob(&mld->refc) == 0)
-        erts_mon_link_dist_destroy__(mld);
+        erts_schedule_mon_link_dist_destruction__(mld);
 }
 
 ERTS_GLB_INLINE void *
@@ -1434,14 +1440,14 @@ erts_monitor_dist_insert(ErtsMonitor *mon, ErtsMonLnkDist *dist)
 
     ERTS_ML_ASSERT(!mdep->dist);
     ERTS_ML_ASSERT(dist);
-    mdep->dist = dist;
-
-    erts_mon_link_dist_inc_refc(dist);
 
     erts_mtx_lock(&dist->mtx);
 
     insert = dist->alive;
     if (insert) {
+        mdep->dist = dist;
+        erts_mon_link_dist_inc_refc(dist);
+
         if ((mon->flags & (ERTS_ML_FLG_NAME
                            | ERTS_ML_FLG_TARGET)) == ERTS_ML_FLG_NAME)
             erts_monitor_tree_insert(&dist->orig_name_monitors, mon);
@@ -2332,15 +2338,15 @@ erts_link_dist_insert(ErtsLink *lnk, ErtsMonLnkDist *dist)
 
     ERTS_ML_ASSERT(!ldep->dist);
     ERTS_ML_ASSERT(dist);
-    ldep->dist = dist;
-
-    erts_mon_link_dist_inc_refc(dist);
 
     erts_mtx_lock(&dist->mtx);
 
     insert = dist->alive;
-    if (insert)
+    if (insert) {
+        ldep->dist = dist;
+        erts_mon_link_dist_inc_refc(dist);
         erts_link_list_insert(&dist->links, lnk);
+    }
 
     erts_mtx_unlock(&dist->mtx);
 
diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index c8d0bf7f40..15dcb069d2 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -820,6 +820,7 @@ erts_prepare_dist_ext(ErtsDistExternal *edep,
     edep->heap_size = -1;
     edep->flags = 0;
     edep->dep = dep;
+    edep->mld = dep->mld;
     edep->connection_id = conn_id;
     edep->data->ext_endp = ext+size;
     edep->data->binp = binp;
diff --git a/erts/emulator/beam/external.h b/erts/emulator/beam/external.h
index ea2b8598ff..0bc64f30fb 100644
--- a/erts/emulator/beam/external.h
+++ b/erts/emulator/beam/external.h
@@ -141,6 +141,7 @@ typedef struct erl_dist_external {
     Uint32 flags;
     Uint32 connection_id;
     ErtsDistExternalData *data;
+    struct ErtsMonLnkDist__ *mld;   /* copied from DistEntry.mld */
     ErtsAtomTranslationTable attab;
 } ErtsDistExternal;