Add atomic_inc()/atomic_dec(), and use them to replace gc_running_mutex

This also fixes a memory leak on Windows with -threaded, because we
were calling initMutex(&gc_running_mutex) for each GC, which allocates
memory.

2 files changed, 83 insertions, 22 deletions

diff --git a/includes/SMP.h b/includes/SMP.h
index 873bbbbfc2..7592abb261 100644
--- a/includes/SMP.h
+++ b/includes/SMP.h
@@ -52,6 +52,24 @@ EXTERN_INLINE StgWord xchg(StgPtr p, StgWord w);
  */
 EXTERN_INLINE StgWord cas(StgVolatilePtr p, StgWord o, StgWord n);
 
+/*
+ * Atomic increment
+ *
+ * atomic_inc(p) {
+ *   return ++(*p);
+ * }
+ */
+EXTERN_INLINE StgWord atomic_inc(StgVolatilePtr p);
+
+/*
+ * Atomic decrement
+ *
+ * atomic_dec(p) {
+ *   return --(*p);
+ * }
+ */
+EXTERN_INLINE StgWord atomic_dec(StgVolatilePtr p);
+
 #endif // !IN_STG_CODE
 
 /*
@@ -164,6 +182,48 @@ cas(StgVolatilePtr p, StgWord o, StgWord n)
 #endif
 }
 
+EXTERN_INLINE StgWord
+atomic_inc(StgVolatilePtr p)
+{
+#if 0 // defined(i386_HOST_ARCH) || defined(x86_64_HOST_ARCH)
+    StgWord r;
+    r = 1;
+    __asm__ __volatile__ (
+        "lock\nxadd %0,%1":
+            "+r" (r), "+m" (*p):
+    );
+    return r;
+#else
+    StgWord old, new;
+    do {
+        old = *p;
+        new = old + 1;
+    } while (cas(p, old, new) != old);
+    return new;
+#endif
+}
+
+EXTERN_INLINE StgWord
+atomic_dec(StgVolatilePtr p)
+{
+#if 0 //defined(i386_HOST_ARCH) || defined(x86_64_HOST_ARCH)
+    StgWord r;
+    r = (StgWord)-1;
+    __asm__ __volatile__ (
+        "lock\nxadd %0,%1":
+            "+r" (r), "+m" (*p):
+    );
+    return r;
+#else
+    StgWord old, new;
+    do {
+        old = *p;
+        new = old - 1;
+    } while (cas(p, old, new) != old);
+    return new;
+#endif
+}
+
 #endif // !IN_STG_CODE
 
 /*
@@ -249,6 +309,18 @@ cas(StgVolatilePtr p, StgWord o, StgWord n)
     return result;
 }
 
+INLINE_HEADER StgWord
+atomic_inc(StgVolatilePtr p)
+{
+    return ++(*p);
+}
+
+INLINE_HEADER StgWord
+atomic_dec(StgVolatilePtr p)
+{
+    return --(*p);
+}
+
 #endif /* !THREADED_RTS */
 
 #endif /* SMP_H */
diff --git a/rts/sm/GC.c b/rts/sm/GC.c
index cfe4c6bdb5..74880064e4 100644
--- a/rts/sm/GC.c
+++ b/rts/sm/GC.c
@@ -147,8 +147,8 @@ static void resize_generations      (void);
 static void resize_nursery          (void);
 static void start_gc_threads        (void);
 static void scavenge_until_all_done (void);
-static nat  inc_running             (void);
-static nat  dec_running             (void);
+static StgWord inc_running          (void);
+static StgWord dec_running          (void);
 static void wakeup_gc_threads       (nat n_threads, nat me);
 static void shutdown_gc_threads     (nat n_threads, nat me);
 
@@ -941,32 +941,22 @@ initGcThreads (void)
    Start GC threads
    ------------------------------------------------------------------------- */
 
-static nat gc_running_threads;
+static volatile StgWord gc_running_threads;
 
-#if defined(THREADED_RTS)
-static Mutex gc_running_mutex;
-#endif
-
-static nat
+static StgWord
 inc_running (void)
 {
-    nat n_running;
-    ACQUIRE_LOCK(&gc_running_mutex);
-    n_running = ++gc_running_threads;
-    RELEASE_LOCK(&gc_running_mutex);
-    ASSERT(n_running <= n_gc_threads);
-    return n_running;
+    StgWord new;
+    new = atomic_inc(&gc_running_threads);
+    ASSERT(new <= n_gc_threads);
+    return new;
 }
 
-static nat
+static StgWord
 dec_running (void)
 {
-    nat n_running;
-    ACQUIRE_LOCK(&gc_running_mutex);
-    ASSERT(n_gc_threads != 0);
-    n_running = --gc_running_threads;
-    RELEASE_LOCK(&gc_running_mutex);
-    return n_running;
+    ASSERT(gc_running_threads != 0);
+    return atomic_dec(&gc_running_threads);
 }
 
 static rtsBool
@@ -1142,7 +1132,6 @@ start_gc_threads (void)
 {
 #if defined(THREADED_RTS)
     gc_running_threads = 0;
-    initMutex(&gc_running_mutex);
 #endif
 }