diff options
Diffstat (limited to 'libgomp/allocator.c')
| -rw-r--r-- | libgomp/allocator.c | 354 |
1 files changed, 354 insertions, 0 deletions
diff --git a/libgomp/allocator.c b/libgomp/allocator.c new file mode 100644 index 00000000000..8592de6e2b5 --- /dev/null +++ b/libgomp/allocator.c @@ -0,0 +1,354 @@ +/* Copyright (C) 2020 Free Software Foundation, Inc. + Contributed by Jakub Jelinek <jakub@redhat.com>. + + This file is part of the GNU Offloading and Multi Processing Library + (libgomp). + + Libgomp is free software; you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY + WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + FOR A PARTICULAR PURPOSE. See the GNU General Public License for + more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + <http://www.gnu.org/licenses/>. */ + +/* This file contains wrappers for the system allocation routines. Most + places in the OpenMP API do not make any provision for failure, so in + general we cannot allow memory allocation to fail. */ + +#define _GNU_SOURCE +#include "libgomp.h" +#include <stdlib.h> + +#define omp_max_predefined_alloc omp_thread_mem_alloc + +struct omp_allocator_data +{ + omp_memspace_handle_t memspace; + omp_uintptr_t alignment; + omp_uintptr_t pool_size; + omp_uintptr_t used_pool_size; + omp_allocator_handle_t fb_data; + unsigned int sync_hint : 8; + unsigned int access : 8; + unsigned int fallback : 8; + unsigned int pinned : 1; + unsigned int partition : 7; +#ifndef HAVE_SYNC_BUILTINS + gomp_mutex_t lock; +#endif +}; + +struct omp_mem_header +{ + void *ptr; + size_t size; + omp_allocator_handle_t allocator; + void *pad; +}; + +omp_allocator_handle_t +omp_init_allocator (omp_memspace_handle_t memspace, int ntraits, + const omp_alloctrait_t traits[]) +{ + struct omp_allocator_data data + = { memspace, 1, ~(uintptr_t) 0, 0, 0, omp_atv_contended, omp_atv_all, + omp_atv_default_mem_fb, omp_atv_false, omp_atv_environment }; + struct omp_allocator_data *ret; + int i; + + if (memspace > omp_low_lat_mem_space) + return omp_null_allocator; + for (i = 0; i < ntraits; i++) + switch (traits[i].key) + { + case omp_atk_sync_hint: + switch (traits[i].value) + { + case omp_atv_default: + data.sync_hint = omp_atv_contended; + break; + case omp_atv_contended: + case omp_atv_uncontended: + case omp_atv_sequential: + case omp_atv_private: + data.sync_hint = traits[i].value; + break; + default: + return omp_null_allocator; + } + break; + case omp_atk_alignment: + if ((traits[i].value & (traits[i].value - 1)) != 0 + || !traits[i].value) + return omp_null_allocator; + data.alignment = traits[i].value; + break; + case omp_atk_access: + switch (traits[i].value) + { + case omp_atv_default: + data.access = omp_atv_all; + break; + case omp_atv_all: + case omp_atv_cgroup: + case omp_atv_pteam: + case omp_atv_thread: + data.access = traits[i].value; + break; + default: + return omp_null_allocator; + } + break; + case omp_atk_pool_size: + data.pool_size = traits[i].value; + break; + case omp_atk_fallback: + switch (traits[i].value) + { + case omp_atv_default: + data.fallback = omp_atv_default_mem_fb; + break; + case omp_atv_default_mem_fb: + case omp_atv_null_fb: + case omp_atv_abort_fb: + case omp_atv_allocator_fb: + data.fallback = traits[i].value; + break; + default: + return omp_null_allocator; + } + break; + case omp_atk_fb_data: + data.fb_data = traits[i].value; + break; + case omp_atk_pinned: + switch (traits[i].value) + { + case omp_atv_default: + case omp_atv_false: + data.pinned = omp_atv_false; + break; + case omp_atv_true: + data.pinned = omp_atv_true; + break; + default: + return omp_null_allocator; + } + break; + case omp_atk_partition: + switch (traits[i].value) + { + case omp_atv_default: + data.partition = omp_atv_environment; + break; + case omp_atv_environment: + case omp_atv_nearest: + case omp_atv_blocked: + case omp_atv_interleaved: + data.partition = traits[i].value; + break; + default: + return omp_null_allocator; + } + break; + default: + return omp_null_allocator; + } + + if (data.alignment < sizeof (void *)) + data.alignment = sizeof (void *); + + /* No support for these so far (for hbw will use memkind). */ + if (data.pinned || data.memspace == omp_high_bw_mem_space) + return omp_null_allocator; + + ret = gomp_malloc (sizeof (struct omp_allocator_data)); + *ret = data; +#ifndef HAVE_SYNC_BUILTINS + gomp_mutex_init (&ret->lock); +#endif + return (omp_allocator_handle_t) ret; +} + +void +omp_destroy_allocator (omp_allocator_handle_t allocator) +{ + if (allocator != omp_null_allocator) + { +#ifndef HAVE_SYNC_BUILTINS + gomp_mutex_destroy (&((struct omp_allocator_data *) allocator)->lock); +#endif + free ((void *) allocator); + } +} + +void * +omp_alloc (size_t size, omp_allocator_handle_t allocator) +{ + struct omp_allocator_data *allocator_data; + size_t alignment, new_size; + void *ptr, *ret; + +retry: + if (allocator == omp_null_allocator) + { + struct gomp_thread *thr = gomp_thread (); + if (thr->ts.def_allocator == omp_null_allocator) + thr->ts.def_allocator = gomp_def_allocator; + allocator = (omp_allocator_handle_t) thr->ts.def_allocator; + } + + if (allocator > omp_max_predefined_alloc) + { + allocator_data = (struct omp_allocator_data *) allocator; + alignment = allocator_data->alignment; + } + else + { + allocator_data = NULL; + alignment = sizeof (void *); + } + + new_size = sizeof (struct omp_mem_header); + if (alignment > sizeof (void *)) + new_size += alignment - sizeof (void *); + if (__builtin_add_overflow (size, new_size, &new_size)) + goto fail; + + if (__builtin_expect (allocator_data + && allocator_data->pool_size < ~(uintptr_t) 0, 0)) + { + uintptr_t used_pool_size; + if (new_size > allocator_data->pool_size) + goto fail; +#ifdef HAVE_SYNC_BUILTINS + used_pool_size = __atomic_load_n (&allocator_data->used_pool_size, + MEMMODEL_RELAXED); + do + { + uintptr_t new_pool_size; + if (__builtin_add_overflow (used_pool_size, new_size, + &new_pool_size) + || new_pool_size > allocator_data->pool_size) + goto fail; + if (__atomic_compare_exchange_n (&allocator_data->used_pool_size, + &used_pool_size, new_pool_size, + true, MEMMODEL_RELAXED, + MEMMODEL_RELAXED)) + break; + } + while (1); +#else + gomp_mutex_lock (&allocator_data->lock); + if (__builtin_add_overflow (allocator_data->used_pool_size, new_size, + &used_pool_size) + || used_pool_size > allocator_data->pool_size) + { + gomp_mutex_unlock (&allocator_data->lock); + goto fail; + } + allocator_data->used_pool_size = used_pool_size; + gomp_mutex_unlock (&allocator_data->lock); +#endif + ptr = malloc (new_size); + if (ptr == NULL) + { +#ifdef HAVE_SYNC_BUILTINS + __atomic_add_fetch (&allocator_data->used_pool_size, -new_size, + MEMMODEL_RELAXED); +#else + gomp_mutex_lock (&allocator_data->lock); + allocator_data->used_pool_size -= new_size; + gomp_mutex_unlock (&allocator_data->lock); +#endif + goto fail; + } + } + else + { + ptr = malloc (new_size); + if (ptr == NULL) + goto fail; + } + + if (alignment > sizeof (void *)) + ret = (void *) (((uintptr_t) ptr + + sizeof (struct omp_mem_header) + + alignment - sizeof (void *)) & ~(alignment - 1)); + else + ret = (char *) ptr + sizeof (struct omp_mem_header); + ((struct omp_mem_header *) ret)[-1].ptr = ptr; + ((struct omp_mem_header *) ret)[-1].size = new_size; + ((struct omp_mem_header *) ret)[-1].allocator = allocator; + return ret; + +fail: + if (allocator_data) + { + switch (allocator_data->fallback) + { + case omp_atv_default_mem_fb: + if (alignment > sizeof (void *) + || (allocator_data + && allocator_data->pool_size < ~(uintptr_t) 0)) + { + allocator = omp_default_mem_alloc; + goto retry; + } + /* Otherwise, we've already performed default mem allocation + and if that failed, it won't succeed again (unless it was + intermitent. Return NULL then, as that is the fallback. */ + break; + case omp_atv_null_fb: + break; + default: + case omp_atv_abort_fb: + gomp_fatal ("Out of memory allocating %lu bytes", + (unsigned long) size); + case omp_atv_allocator_fb: + allocator = allocator_data->fb_data; + goto retry; + } + } + return NULL; +} + +void +omp_free (void *ptr, omp_allocator_handle_t allocator) +{ + struct omp_mem_header *data; + + if (ptr == NULL) + return; + (void) allocator; + data = &((struct omp_mem_header *) ptr)[-1]; + if (data->allocator > omp_max_predefined_alloc) + { + struct omp_allocator_data *allocator_data + = (struct omp_allocator_data *) (data->allocator); + if (allocator_data->pool_size < ~(uintptr_t) 0) + { +#ifdef HAVE_SYNC_BUILTINS + __atomic_add_fetch (&allocator_data->used_pool_size, -data->size, + MEMMODEL_RELAXED); +#else + gomp_mutex_lock (&allocator_data->lock); + allocator_data->used_pool_size -= data->new_size; + gomp_mutex_unlock (&allocator_data->lock); +#endif + } + } + free (data->ptr); +} |