nonmoving: Rework mark queue representation

The previous representation needlessly limited the array length to
16-bits on 32-bit platforms.

2 files changed, 18 insertions, 23 deletions

diff --git a/rts/sm/NonMovingMark.c b/rts/sm/NonMovingMark.c
index e236056092..638be30c20 100644
--- a/rts/sm/NonMovingMark.c
+++ b/rts/sm/NonMovingMark.c
@@ -461,7 +461,7 @@ markQueuePushClosureGC (MarkQueue *q, StgClosure *p)
 
     MarkQueueEnt ent = {
         .mark_closure = {
-            .p = UNTAG_CLOSURE(p),
+            .p = TAG_CLOSURE(MARK_CLOSURE, UNTAG_CLOSURE(p)),
             .origin = NULL,
         }
     };
@@ -492,7 +492,7 @@ void push_closure (MarkQueue *q,
 
     MarkQueueEnt ent = {
         .mark_closure = {
-            .p = p,
+            .p = TAG_CLOSURE(MARK_CLOSURE, UNTAG_CLOSURE(p)),
             .origin = origin,
         }
     };
@@ -510,8 +510,8 @@ void push_array (MarkQueue *q,
 
     MarkQueueEnt ent = {
         .mark_array = {
-            .array = array,
-            .start_index = (start_index << 16) | 0x3,
+            .array = (const StgMutArrPtrs *) TAG_CLOSURE(MARK_ARRAY, UNTAG_CLOSURE((StgClosure *) array)),
+            .start_index = start_index,
         }
     };
     push(q, &ent);
@@ -1160,6 +1160,7 @@ bump_static_flag(StgClosure **link_field, StgClosure *q STG_UNUSED)
     }
 }
 
+/* N.B. p0 may be tagged */
 static GNUC_ATTR_HOT void
 mark_closure (MarkQueue *queue, const StgClosure *p0, StgClosure **origin)
 {
@@ -1679,11 +1680,13 @@ nonmovingMark (MarkQueue *queue)
             mark_closure(queue, ent.mark_closure.p, ent.mark_closure.origin);
             break;
         case MARK_ARRAY: {
-            const StgMutArrPtrs *arr = ent.mark_array.array;
-            StgWord start = ent.mark_array.start_index >> 16;
+            const StgMutArrPtrs *arr = (const StgMutArrPtrs *)
+                UNTAG_CLOSURE((StgClosure *) ent.mark_array.array);
+            StgWord start = ent.mark_array.start_index;
             StgWord end = start + MARK_ARRAY_CHUNK_LENGTH;
             if (end < arr->ptrs) {
-                markQueuePushArray(queue, ent.mark_array.array, end);
+                // There is more to be marked after this chunk.
+                markQueuePushArray(queue, arr, end);
             } else {
                 end = arr->ptrs;
             }
diff --git a/rts/sm/NonMovingMark.h b/rts/sm/NonMovingMark.h
index 36158ec917..1e3fb01bd2 100644
--- a/rts/sm/NonMovingMark.h
+++ b/rts/sm/NonMovingMark.h
@@ -18,8 +18,8 @@
 
 enum EntryType {
     NULL_ENTRY = 0,
-    MARK_CLOSURE,
-    MARK_ARRAY
+    MARK_CLOSURE = 1,
+    MARK_ARRAY = 2
 };
 
 /* Note [Origin references in the nonmoving collector]
@@ -43,13 +43,10 @@ enum EntryType {
  */
 
 typedef struct {
-    // Which kind of mark queue entry we have is determined by the low bits of
-    // the second word: they must be zero in the case of a mark_closure entry
-    // (since the second word of a mark_closure entry points to a pointer and
-    // pointers must be word-aligned).  In the case of a mark_array we set them
-    // to 0x3 (the value of start_index is shifted to the left to accomodate
-    // this). null_entry where p==NULL is used to indicate the end of the queue.
+    // Which kind of mark queue entry we have is determined by the tag bits of
+    // the first word (using the tags defined by the EntryType enum).
     union {
+        // A null_entry indicates the end of the queue.
         struct {
             void *p;              // must be NULL
         } null_entry;
@@ -67,14 +64,9 @@ typedef struct {
 
 INLINE_HEADER enum EntryType nonmovingMarkQueueEntryType(MarkQueueEnt *ent)
 {
-    if (ent->null_entry.p == NULL) {
-        return NULL_ENTRY;
-    } else if (((uintptr_t) ent->mark_closure.origin & TAG_MASK) == 0) {
-        return MARK_CLOSURE;
-    } else {
-        ASSERT((ent->mark_array.start_index & TAG_MASK) == 0x3);
-        return MARK_ARRAY;
-    }
+    uintptr_t tag = (uintptr_t) ent->null_entry.p & TAG_MASK;
+    ASSERT(tag <= MARK_ARRAY);
+    return tag;
 }
 
 typedef struct {