Merge PR-5149 from sverker/writable-bin-gc-optim OTP-17602

Optimize off-heap list traversal during GC

2 files changed, 25 insertions, 22 deletions

diff --git a/erts/emulator/beam/erl_binary.h b/erts/emulator/beam/erl_binary.h
index bd3669d896..bd3a0632d2 100644
--- a/erts/emulator/beam/erl_binary.h
+++ b/erts/emulator/beam/erl_binary.h
@@ -358,12 +358,8 @@ erts_free_aligned_binary_bytes(byte* buf)
  *
  * This check also ensures, indirectly, that there won't be an overflow when
  * the size is bumped by CHICKEN_PAD and the binary struct itself. */
-#define BINARY_OVERFLOW_CHECK(BYTE_SIZE) \
-    do { \
-         if (ERTS_UNLIKELY(BYTE_SIZE > ERTS_UWORD_MAX / CHAR_BIT)) { \
-             return NULL; \
-         } \
-    } while(0)
+#define IS_BINARY_SIZE_OK(BYTE_SIZE) \
+    ERTS_LIKELY(BYTE_SIZE <= ERTS_UWORD_MAX / CHAR_BIT)
 
 /* Explicit extra bytes allocated to counter buggy drivers.
 ** These extra bytes where earlier (< R13B04) added by an alignment-bug
@@ -381,7 +377,8 @@ erts_bin_drv_alloc_fnf(Uint size)
     Binary *res;
     Uint bsize;
 
-    BINARY_OVERFLOW_CHECK(size);
+    if (!IS_BINARY_SIZE_OK(size))
+        return NULL;
     bsize = ERTS_SIZEOF_Binary(size) + CHICKEN_PAD;
 
     res = (Binary *)erts_alloc_fnf(ERTS_ALC_T_DRV_BINARY, bsize);
@@ -414,8 +411,9 @@ erts_bin_nrml_alloc_fnf(Uint size)
     Binary *res;
     Uint bsize;
 
-    BINARY_OVERFLOW_CHECK(size);
-    bsize = ERTS_SIZEOF_Binary(size) + CHICKEN_PAD;
+    if (!IS_BINARY_SIZE_OK(size))
+        return NULL;
+    bsize = ERTS_SIZEOF_Binary(size);
 
     res = (Binary *)erts_alloc_fnf(ERTS_ALC_T_BINARY, bsize);
     ERTS_CHK_BIN_ALIGNMENT(res);
@@ -432,7 +430,7 @@ erts_bin_nrml_alloc_fnf(Uint size)
 ERTS_GLB_INLINE Binary *
 erts_bin_nrml_alloc(Uint size)
 {
-    Binary *res = erts_bin_drv_alloc_fnf(size);
+    Binary *res = erts_bin_nrml_alloc_fnf(size);
 
     if (res) {
         return res;
@@ -448,12 +446,18 @@ erts_bin_realloc_fnf(Binary *bp, Uint size)
     Binary *nbp;
     Uint bsize;
     
-    type = (bp->intern.flags & BIN_FLAG_DRV) ? ERTS_ALC_T_DRV_BINARY
-                                             : ERTS_ALC_T_BINARY;
     ASSERT((bp->intern.flags & BIN_FLAG_MAGIC) == 0);
+    if (!IS_BINARY_SIZE_OK(size))
+        return NULL;
+    bsize = ERTS_SIZEOF_Binary(size);
 
-    BINARY_OVERFLOW_CHECK(size);
-    bsize = ERTS_SIZEOF_Binary(size) + CHICKEN_PAD;
+    if (bp->intern.flags & BIN_FLAG_DRV) {
+        type = ERTS_ALC_T_DRV_BINARY;
+        bsize += CHICKEN_PAD;
+    }
+    else {
+        type = ERTS_ALC_T_BINARY;
+    }
 
     nbp = (Binary *)erts_realloc_fnf(type, (void *) bp, bsize);
     ERTS_CHK_BIN_ALIGNMENT(nbp);
diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index d3d79959df..07a7b6c9e8 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -456,17 +456,19 @@ erts_gc_after_bif_call(Process* p, Eterm result, Eterm* regs, Uint arity)
                                       result, regs, arity);
 }
 
-static ERTS_INLINE void reset_active_writer(Process *p)
+static ERTS_INLINE void assert_no_active_writers(Process *p)
 {
+#ifdef DEBUG
     struct erl_off_heap_header* ptr;
     ptr = MSO(p).first;
     while (ptr) {
 	if (ptr->thing_word == HEADER_PROC_BIN) {	
 	    ProcBin *pbp = (ProcBin*) ptr;
-	    pbp->flags &= ~PB_ACTIVE_WRITER;
+	    ERTS_ASSERT(!(pbp->flags & PB_ACTIVE_WRITER));
 	}
 	ptr = ptr->next;
     }
+#endif
 }
 
 #define ERTS_DELAY_GC_EXTRA_FREE 40
@@ -771,7 +773,7 @@ do_major_collection:
         ERTS_MSACC_SET_STATE_CACHED_X(ERTS_MSACC_STATE_GC);
     }
 
-    reset_active_writer(p);
+    assert_no_active_writers(p);
 
     /*
      * Finish.
@@ -2809,13 +2811,10 @@ link_live_proc_bin(struct shrink_cand_data *shrink,
 
     *currpp = pbp->next;
     if (pbp->flags & (PB_ACTIVE_WRITER|PB_IS_WRITABLE)) {
-	ASSERT(((pbp->flags & (PB_ACTIVE_WRITER|PB_IS_WRITABLE))
-		== (PB_ACTIVE_WRITER|PB_IS_WRITABLE))
-	       || ((pbp->flags & (PB_ACTIVE_WRITER|PB_IS_WRITABLE))
-		   == PB_IS_WRITABLE));
-
+	ASSERT(pbp->flags & PB_IS_WRITABLE);
 
 	if (pbp->flags & PB_ACTIVE_WRITER) {
+            pbp->flags &= ~PB_ACTIVE_WRITER;
 	    shrink->no_of_active++;
 	}
 	else { /* inactive */