San: Implement bump alloc

The new allocator will be used to allocate guarded extents used as slabs
for guarded small allocations.

4 files changed, 127 insertions, 16 deletions

diff --git a/test/include/test/arena_decay.h b/test/include/test/arena_decay.h
index da659212..524ee218 100644
--- a/test/include/test/arena_decay.h
+++ b/test/include/test/arena_decay.h
@@ -1,4 +1,4 @@
-static unsigned
+static inline unsigned
 do_arena_create(ssize_t dirty_decay_ms, ssize_t muzzy_decay_ms) {
 	unsigned arena_ind;
 	size_t sz = sizeof(unsigned);
@@ -24,7 +24,7 @@ do_arena_create(ssize_t dirty_decay_ms, ssize_t muzzy_decay_ms) {
 	return arena_ind;
 }
 
-static void
+static inline void
 do_arena_destroy(unsigned arena_ind) {
 	size_t mib[3];
 	size_t miblen = sizeof(mib)/sizeof(size_t);
@@ -35,14 +35,14 @@ do_arena_destroy(unsigned arena_ind) {
 	    "Unexpected mallctlbymib() failure");
 }
 
-static void
+static inline void
 do_epoch(void) {
 	uint64_t epoch = 1;
 	expect_d_eq(mallctl("epoch", NULL, NULL, (void *)&epoch, sizeof(epoch)),
 	    0, "Unexpected mallctl() failure");
 }
 
-static void
+static inline void
 do_purge(unsigned arena_ind) {
 	size_t mib[3];
 	size_t miblen = sizeof(mib)/sizeof(size_t);
@@ -53,7 +53,7 @@ do_purge(unsigned arena_ind) {
 	    "Unexpected mallctlbymib() failure");
 }
 
-static void
+static inline void
 do_decay(unsigned arena_ind) {
 	size_t mib[3];
 	size_t miblen = sizeof(mib)/sizeof(size_t);
@@ -64,7 +64,7 @@ do_decay(unsigned arena_ind) {
 	    "Unexpected mallctlbymib() failure");
 }
 
-static uint64_t
+static inline uint64_t
 get_arena_npurge_impl(const char *mibname, unsigned arena_ind) {
 	size_t mib[4];
 	size_t miblen = sizeof(mib)/sizeof(size_t);
@@ -78,32 +78,32 @@ get_arena_npurge_impl(const char *mibname, unsigned arena_ind) {
 	return npurge;
 }
 
-static uint64_t
+static inline uint64_t
 get_arena_dirty_npurge(unsigned arena_ind) {
 	do_epoch();
 	return get_arena_npurge_impl("stats.arenas.0.dirty_npurge", arena_ind);
 }
 
-static uint64_t
+static inline uint64_t
 get_arena_dirty_purged(unsigned arena_ind) {
 	do_epoch();
 	return get_arena_npurge_impl("stats.arenas.0.dirty_purged", arena_ind);
 }
 
-static uint64_t
+static inline uint64_t
 get_arena_muzzy_npurge(unsigned arena_ind) {
 	do_epoch();
 	return get_arena_npurge_impl("stats.arenas.0.muzzy_npurge", arena_ind);
 }
 
-static uint64_t
+static inline uint64_t
 get_arena_npurge(unsigned arena_ind) {
 	do_epoch();
 	return get_arena_npurge_impl("stats.arenas.0.dirty_npurge", arena_ind) +
 	    get_arena_npurge_impl("stats.arenas.0.muzzy_npurge", arena_ind);
 }
 
-static size_t
+static inline size_t
 get_arena_pdirty(unsigned arena_ind) {
 	do_epoch();
 	size_t mib[4];
@@ -118,7 +118,7 @@ get_arena_pdirty(unsigned arena_ind) {
 	return pdirty;
 }
 
-static size_t
+static inline size_t
 get_arena_pmuzzy(unsigned arena_ind) {
 	do_epoch();
 	size_t mib[4];
@@ -133,14 +133,14 @@ get_arena_pmuzzy(unsigned arena_ind) {
 	return pmuzzy;
 }
 
-static void *
+static inline void *
 do_mallocx(size_t size, int flags) {
 	void *p = mallocx(size, flags);
 	expect_ptr_not_null(p, "Unexpected mallocx() failure");
 	return p;
 }
 
-static void
+static inline void
 generate_dirty(unsigned arena_ind, size_t size) {
 	int flags = MALLOCX_ARENA(arena_ind) | MALLOCX_TCACHE_NONE;
 	void *p = do_mallocx(size, flags);
diff --git a/test/unit/retained.c b/test/unit/retained.c
index 76bda50f..53c90f24 100644
--- a/test/unit/retained.c
+++ b/test/unit/retained.c
@@ -104,7 +104,7 @@ TEST_BEGIN(test_retained) {
 
 	arena_ind = do_arena_create(NULL);
 	sz = nallocx(HUGEPAGE, 0);
-	size_t guard_sz = san_enabled() ? PAGE_GUARDS_SIZE : 0;
+	size_t guard_sz = san_enabled() ? SAN_PAGE_GUARDS_SIZE : 0;
 	esz = sz + sz_large_pad + guard_sz;
 
 	atomic_store_u(&epoch, 0, ATOMIC_RELAXED);
diff --git a/test/unit/san.c b/test/unit/san.c
index 93e292f6..eb9ff517 100644
--- a/test/unit/san.c
+++ b/test/unit/san.c
@@ -122,7 +122,7 @@ TEST_BEGIN(test_guarded_decay) {
 	/* Verify that guarded extents as dirty. */
 	size_t sz1 = PAGE, sz2 = PAGE * 2;
 	/* W/o maps_coalesce, guarded extents are unguarded eagerly. */
-	size_t add_guard_size = maps_coalesce ? 0 : PAGE_GUARDS_SIZE;
+	size_t add_guard_size = maps_coalesce ? 0 : SAN_PAGE_GUARDS_SIZE;
 	generate_dirty(arena_ind, sz1);
 	verify_pdirty(arena_ind, sz1 + add_guard_size);
 	verify_pmuzzy(arena_ind, 0);
diff --git a/test/unit/san_bump.c b/test/unit/san_bump.c
new file mode 100644
index 00000000..fbee53e5
--- /dev/null
+++ b/test/unit/san_bump.c
@@ -0,0 +1,111 @@
+#include "test/jemalloc_test.h"
+#include "test/arena_decay.h"
+
+#include "jemalloc/internal/arena_structs.h"
+#include "jemalloc/internal/san_bump.h"
+
+TEST_BEGIN(test_san_bump_alloc) {
+	test_skip_if(!maps_coalesce || !opt_retain);
+
+	tsdn_t *tsdn = tsdn_fetch();
+
+	san_bump_alloc_t sba;
+	san_bump_alloc_init(&sba);
+
+	unsigned arena_ind = do_arena_create(0, 0);
+	assert_u_ne(arena_ind, UINT_MAX, "Failed to create an arena");
+
+	arena_t *arena = arena_get(tsdn, arena_ind, false);
+	pac_t *pac = &arena->pa_shard.pac;
+
+	size_t alloc_size = PAGE * 16;
+	size_t alloc_n = alloc_size / sizeof(unsigned);
+	edata_t* edata = san_bump_alloc(tsdn, &sba, pac, pac_ehooks_get(pac),
+	    alloc_size, /* zero */ false);
+
+	expect_ptr_not_null(edata, "Failed to allocate edata");
+	expect_u_eq(edata_arena_ind_get(edata), arena_ind,
+	    "Edata was assigned an incorrect arena id");
+	expect_zu_eq(edata_size_get(edata), alloc_size,
+	    "Allocated edata of incorrect size");
+	expect_false(edata_slab_get(edata),
+	    "Bump allocator incorrectly assigned 'slab' to true");
+	expect_true(edata_committed_get(edata), "Edata is not committed");
+
+	void *ptr = edata_addr_get(edata);
+	expect_ptr_not_null(ptr, "Edata was assigned an invalid address");
+	/* Test that memory is allocated; no guard pages are misplaced */
+	for (unsigned i = 0; i < alloc_n; ++i) {
+		((unsigned *)ptr)[i] = 1;
+	}
+
+	size_t alloc_size2 = PAGE * 28;
+	size_t alloc_n2 = alloc_size / sizeof(unsigned);
+	edata_t *edata2 = san_bump_alloc(tsdn, &sba, pac, pac_ehooks_get(pac),
+	    alloc_size2, /* zero */ true);
+
+	expect_ptr_not_null(edata2, "Failed to allocate edata");
+	expect_u_eq(edata_arena_ind_get(edata2), arena_ind,
+	    "Edata was assigned an incorrect arena id");
+	expect_zu_eq(edata_size_get(edata2), alloc_size2,
+	    "Allocated edata of incorrect size");
+	expect_false(edata_slab_get(edata2),
+	    "Bump allocator incorrectly assigned 'slab' to true");
+	expect_true(edata_committed_get(edata2), "Edata is not committed");
+
+	void *ptr2 = edata_addr_get(edata2);
+	expect_ptr_not_null(ptr, "Edata was assigned an invalid address");
+
+	uintptr_t ptrdiff = ptr2 > ptr ? (uintptr_t)ptr2 - (uintptr_t)ptr
+	    : (uintptr_t)ptr - (uintptr_t)ptr2;
+	size_t between_allocs = (size_t)ptrdiff - alloc_size;
+
+	expect_zu_ge(between_allocs, PAGE,
+	    "Guard page between allocs is missing");
+
+	for (unsigned i = 0; i < alloc_n2; ++i) {
+		expect_u_eq(((unsigned *)ptr2)[i], 0, "Memory is not zeroed");
+	}
+}
+TEST_END
+
+TEST_BEGIN(test_large_alloc_size) {
+	test_skip_if(!maps_coalesce || !opt_retain);
+
+	tsdn_t *tsdn = tsdn_fetch();
+
+	san_bump_alloc_t sba;
+	san_bump_alloc_init(&sba);
+
+	unsigned arena_ind = do_arena_create(0, 0);
+	assert_u_ne(arena_ind, UINT_MAX, "Failed to create an arena");
+
+	arena_t *arena = arena_get(tsdn, arena_ind, false);
+	pac_t *pac = &arena->pa_shard.pac;
+
+	size_t alloc_size = SBA_RETAINED_ALLOC_SIZE * 2;
+	edata_t* edata = san_bump_alloc(tsdn, &sba, pac, pac_ehooks_get(pac),
+	    alloc_size, /* zero */ false);
+	expect_u_eq(edata_arena_ind_get(edata), arena_ind,
+	    "Edata was assigned an incorrect arena id");
+	expect_zu_eq(edata_size_get(edata), alloc_size,
+	    "Allocated edata of incorrect size");
+	expect_false(edata_slab_get(edata),
+	    "Bump allocator incorrectly assigned 'slab' to true");
+	expect_true(edata_committed_get(edata), "Edata is not committed");
+
+	void *ptr = edata_addr_get(edata);
+	expect_ptr_not_null(ptr, "Edata was assigned an invalid address");
+	/* Test that memory is allocated; no guard pages are misplaced */
+	for (unsigned i = 0; i < alloc_size / PAGE; ++i) {
+		*((char *)ptr + PAGE * i) = 1;
+	}
+}
+TEST_END
+
+int
+main(void) {
+	return test(
+	    test_san_bump_alloc,
+	    test_large_alloc_size);
+}