Disallow decay during reentrancy.

Decay should not be triggered during reentrant calls (may cause lock order
reversal / deadlocks).  Added a delay_trigger flag to the tickers to bypass
decay when rentrancy_level is not zero.

3 files changed, 76 insertions, 21 deletions

diff --git a/include/jemalloc/internal/arena_inlines_b.h b/include/jemalloc/internal/arena_inlines_b.h
index c9d7db86..609e73d3 100644
--- a/include/jemalloc/internal/arena_inlines_b.h
+++ b/include/jemalloc/internal/arena_inlines_b.h
@@ -131,7 +131,8 @@ arena_decay_ticks(tsdn_t *tsdn, arena_t *arena, unsigned nticks) {
 	 */
 	ticker_geom_t *decay_ticker = tsd_arena_decay_tickerp_get(tsd);
 	uint64_t *prng_state = tsd_prng_statep_get(tsd);
-	if (unlikely(ticker_geom_ticks(decay_ticker, prng_state, nticks))) {
+	if (unlikely(ticker_geom_ticks(decay_ticker, prng_state, nticks,
+	    tsd_reentrancy_level_get(tsd) > 0))) {
 		arena_decay(tsdn, arena, false, false);
 	}
 }
diff --git a/include/jemalloc/internal/ticker.h b/include/jemalloc/internal/ticker.h
index 6b51ddec..de034995 100644
--- a/include/jemalloc/internal/ticker.h
+++ b/include/jemalloc/internal/ticker.h
@@ -57,23 +57,27 @@ ticker_read(const ticker_t *ticker) {
 JEMALLOC_NOINLINE
 #endif
 static bool
-ticker_fixup(ticker_t *ticker) {
+ticker_fixup(ticker_t *ticker, bool delay_trigger) {
+	if (delay_trigger) {
+		ticker->tick = 0;
+		return false;
+	}
 	ticker->tick = ticker->nticks;
 	return true;
 }
 
 static inline bool
-ticker_ticks(ticker_t *ticker, int32_t nticks) {
+ticker_ticks(ticker_t *ticker, int32_t nticks, bool delay_trigger) {
 	ticker->tick -= nticks;
 	if (unlikely(ticker->tick < 0)) {
-		return ticker_fixup(ticker);
+		return ticker_fixup(ticker, delay_trigger);
 	}
 	return false;
 }
 
 static inline bool
-ticker_tick(ticker_t *ticker) {
-	return ticker_ticks(ticker, 1);
+ticker_tick(ticker_t *ticker, bool delay_trigger) {
+	return ticker_ticks(ticker, 1, delay_trigger);
 }
 
 /*
@@ -150,26 +154,35 @@ ticker_geom_read(const ticker_geom_t *ticker) {
 JEMALLOC_NOINLINE
 #endif
 static bool
-ticker_geom_fixup(ticker_geom_t *ticker, uint64_t *prng_state) {
+ticker_geom_fixup(ticker_geom_t *ticker, uint64_t *prng_state,
+    bool delay_trigger) {
+	if (delay_trigger) {
+		ticker->tick = 0;
+		return false;
+	}
+
 	uint64_t idx = prng_lg_range_u64(prng_state, TICKER_GEOM_NBITS);
 	ticker->tick = (uint32_t)(
 	    (uint64_t)ticker->nticks * (uint64_t)ticker_geom_table[idx]
 	    / (uint64_t)TICKER_GEOM_MUL);
+
 	return true;
 }
 
 static inline bool
-ticker_geom_ticks(ticker_geom_t *ticker, uint64_t *prng_state, int32_t nticks) {
+ticker_geom_ticks(ticker_geom_t *ticker, uint64_t *prng_state, int32_t nticks,
+    bool delay_trigger) {
 	ticker->tick -= nticks;
 	if (unlikely(ticker->tick < 0)) {
-		return ticker_geom_fixup(ticker, prng_state);
+		return ticker_geom_fixup(ticker, prng_state, delay_trigger);
 	}
 	return false;
 }
 
 static inline bool
-ticker_geom_tick(ticker_geom_t *ticker, uint64_t *prng_state) {
-	return ticker_geom_ticks(ticker, prng_state, 1);
+ticker_geom_tick(ticker_geom_t *ticker, uint64_t *prng_state,
+    bool delay_trigger) {
+	return ticker_geom_ticks(ticker, prng_state, 1, delay_trigger);
 }
 
 #endif /* JEMALLOC_INTERNAL_TICKER_H */
diff --git a/test/unit/ticker.c b/test/unit/ticker.c
index 0dd77861..c4147a0c 100644
--- a/test/unit/ticker.c
+++ b/test/unit/ticker.c
@@ -13,12 +13,12 @@ TEST_BEGIN(test_ticker_tick) {
 		for (j = 0; j < NTICKS; j++) {
 			expect_u_eq(ticker_read(&ticker), NTICKS - j,
 			    "Unexpected ticker value (i=%d, j=%d)", i, j);
-			expect_false(ticker_tick(&ticker),
+			expect_false(ticker_tick(&ticker, false),
 			    "Unexpected ticker fire (i=%d, j=%d)", i, j);
 		}
 		expect_u32_eq(ticker_read(&ticker), 0,
 		    "Expected ticker depletion");
-		expect_true(ticker_tick(&ticker),
+		expect_true(ticker_tick(&ticker, false),
 		    "Expected ticker fire (i=%d)", i);
 		expect_u32_eq(ticker_read(&ticker), NTICKS,
 		    "Expected ticker reset");
@@ -34,12 +34,15 @@ TEST_BEGIN(test_ticker_ticks) {
 	ticker_init(&ticker, NTICKS);
 
 	expect_u_eq(ticker_read(&ticker), NTICKS, "Unexpected ticker value");
-	expect_false(ticker_ticks(&ticker, NTICKS), "Unexpected ticker fire");
+	expect_false(ticker_ticks(&ticker, NTICKS, false),
+	    "Unexpected ticker fire");
 	expect_u_eq(ticker_read(&ticker), 0, "Unexpected ticker value");
-	expect_true(ticker_ticks(&ticker, NTICKS), "Expected ticker fire");
+	expect_true(ticker_ticks(&ticker, NTICKS, false),
+	    "Expected ticker fire");
 	expect_u_eq(ticker_read(&ticker), NTICKS, "Unexpected ticker value");
 
-	expect_true(ticker_ticks(&ticker, NTICKS + 1), "Expected ticker fire");
+	expect_true(ticker_ticks(&ticker, NTICKS + 1, false),
+	    "Expected ticker fire");
 	expect_u_eq(ticker_read(&ticker), NTICKS, "Unexpected ticker value");
 #undef NTICKS
 }
@@ -52,13 +55,14 @@ TEST_BEGIN(test_ticker_copy) {
 	ticker_init(&ta, NTICKS);
 	ticker_copy(&tb, &ta);
 	expect_u_eq(ticker_read(&tb), NTICKS, "Unexpected ticker value");
-	expect_true(ticker_ticks(&tb, NTICKS + 1), "Expected ticker fire");
+	expect_true(ticker_ticks(&tb, NTICKS + 1, false),
+	    "Expected ticker fire");
 	expect_u_eq(ticker_read(&tb), NTICKS, "Unexpected ticker value");
 
-	ticker_tick(&ta);
+	ticker_tick(&ta, false);
 	ticker_copy(&tb, &ta);
 	expect_u_eq(ticker_read(&tb), NTICKS - 1, "Unexpected ticker value");
-	expect_true(ticker_ticks(&tb, NTICKS), "Expected ticker fire");
+	expect_true(ticker_ticks(&tb, NTICKS, false), "Expected ticker fire");
 	expect_u_eq(ticker_read(&tb), NTICKS, "Unexpected ticker value");
 #undef NTICKS
 }
@@ -74,7 +78,7 @@ TEST_BEGIN(test_ticker_geom) {
 	/* Just some random constant. */
 	uint64_t prng_state = 0x343219f93496db9fULL;
 	for (uint64_t i = 0; i < niters; i++) {
-		while(!ticker_geom_tick(&ticker, &prng_state)) {
+		while(!ticker_geom_tick(&ticker, &prng_state, false)) {
 			total_ticks++;
 		}
 	}
@@ -90,11 +94,48 @@ TEST_BEGIN(test_ticker_geom) {
 }
 TEST_END
 
+TEST_BEGIN(test_ticker_delay) {
+	const int32_t ticks = 1000;
+	const uint64_t niters = 10000;
+
+	ticker_t t1;
+	ticker_init(&t1, ticks);
+
+	ticker_geom_t t2;
+	/* Just some random constant. */
+	uint64_t prng_state = 0x43219f93496db9f3ULL;
+	ticker_geom_init(&t2, ticks);
+
+	bool delay = false;
+	expect_false(ticker_ticks(&t1, ticks, delay), "Unexpected ticker fire");
+	expect_false(ticker_geom_ticks(&t2, &prng_state, ticks, delay),
+	    "Unexpected ticker fire");
+	expect_d_eq(ticker_read(&t1), 0, "Unexpected ticker value");
+	expect_d_eq(ticker_geom_read(&t2), 0, "Unexpected ticker value");
+
+	delay = true;
+	/* Not allowed to fire when delay is set to true. */
+	for (unsigned i = 0; i < niters; i++) {
+		expect_false(ticker_tick(&t1, delay), "Unexpected ticker fire");
+		expect_false(ticker_geom_tick(&t2, &prng_state, delay),
+		    "Unexpected ticker fire");
+		expect_d_eq(ticker_read(&t1), 0, "Unexpected ticker value");
+		expect_d_eq(ticker_geom_read(&t2), 0, "Unexpected ticker value");
+	}
+
+	delay = false;
+	expect_true(ticker_tick(&t1, delay), "Expected ticker fire");
+	expect_true(ticker_geom_tick(&t2, &prng_state, delay),
+	    "Expected ticker fire");
+}
+TEST_END
+
 int
 main(void) {
 	return test(
 	    test_ticker_tick,
 	    test_ticker_ticks,
 	    test_ticker_copy,
-	    test_ticker_geom);
+	    test_ticker_geom,
+	    test_ticker_delay);
 }