mirror/glibc
2
0
Fork 0
mirror of git://sourceware.org/git/glibc.git synced 2024-11-21 01:12:26 +08:00
Code Issues Packages Projects Releases Wiki Activity

malloc: Use C11 atomics on memusage

Browse Source 
Checked on x86_64-linux-gnu.

Reviewed-by: DJ Delorie <dj@redhat.com>
This commit is contained in:
Adhemerval Zanella 2022-03-11 13:53:11 -03:00
parent 33237fe83d
commit 998fba01b2
1 changed files with 111 additions and 82 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

193
malloc/memusage.c
View File

@ -17,21 +17,16 @@
<https://www.gnu.org/licenses/>. */ <https://www.gnu.org/licenses/>. */
#include <assert.h> #include <assert.h>
#include <atomic.h>
#include <dlfcn.h> #include <dlfcn.h>
#include <errno.h>
#include <fcntl.h> #include <fcntl.h>
#include <inttypes.h> #include <stdatomic.h>
#include <signal.h>
#include <stdarg.h>
#include <stdbool.h> #include <stdbool.h>
#include <stdio.h> #include <stdio.h>
#include <stdlib.h> #include <stdlib.h>
#include <string.h> #include <stdarg.h>
#include <unistd.h>
#include <stdint.h>
#include <sys/mman.h> #include <sys/mman.h>
#include <sys/time.h> #include <sys/time.h>
#include <unistd.h>
#include <hp-timing.h> #include <hp-timing.h>
#include <machine-sp.h> #include <machine-sp.h>
@ -73,20 +68,20 @@ struct header
#define MAGIC 0xfeedbeaf #define MAGIC 0xfeedbeaf
static unsigned long int calls[idx_last]; static _Atomic unsigned long int calls[idx_last];
static unsigned long int failed[idx_last]; static _Atomic unsigned long int failed[idx_last];
static size_t total[idx_last]; static _Atomic size_t total[idx_last];
static size_t grand_total; static _Atomic size_t grand_total;
static unsigned long int histogram[65536 / 16]; static _Atomic unsigned long int histogram[65536 / 16];
static unsigned long int large; static _Atomic unsigned long int large;
static unsigned long int calls_total; static _Atomic unsigned long int calls_total;
static unsigned long int inplace; static _Atomic unsigned long int inplace;
static unsigned long int decreasing; static _Atomic unsigned long int decreasing;
static unsigned long int realloc_free; static _Atomic unsigned long int realloc_free;
static unsigned long int inplace_mremap; static _Atomic unsigned long int inplace_mremap;
static unsigned long int decreasing_mremap; static _Atomic unsigned long int decreasing_mremap;
static size_t current_heap; static _Atomic size_t current_heap;
static size_t peak_use[3]; static _Atomic size_t peak_use[3];
static __thread uintptr_t start_sp; static __thread uintptr_t start_sp;
/* A few macros to make the source more readable. */ /* A few macros to make the source more readable. */
@ -113,7 +108,7 @@ struct entry
}; };
static struct entry buffer[2 * DEFAULT_BUFFER_SIZE]; static struct entry buffer[2 * DEFAULT_BUFFER_SIZE];
static uint32_t buffer_cnt; static _Atomic uint32_t buffer_cnt;
static struct entry first; static struct entry first;
static void static void
@ -134,6 +129,19 @@ gettime (struct entry *e)
#endif #endif
} }
static inline void
peak_atomic_max (_Atomic size_t *peak, size_t val)
{
size_t v;
do
{
v = atomic_load_explicit (peak, memory_order_relaxed);
if (v >= val)
break;
}
while (! atomic_compare_exchange_weak (peak, &v, val));
}
/* Update the global data after a successful function call. */ /* Update the global data after a successful function call. */
static void static void
update_data (struct header *result, size_t len, size_t old_len) update_data (struct header *result, size_t len, size_t old_len)
@ -148,8 +156,9 @@ update_data (struct header *result, size_t len, size_t old_len)
/* Compute current heap usage and compare it with the maximum value. */ /* Compute current heap usage and compare it with the maximum value. */
size_t heap size_t heap
= catomic_exchange_and_add (&current_heap, len - old_len) + len - old_len; = atomic_fetch_add_explicit (&current_heap, len - old_len,
catomic_max (&peak_heap, heap); memory_order_relaxed) + len - old_len;
peak_atomic_max (&peak_heap, heap);
/* Compute current stack usage and compare it with the maximum /* Compute current stack usage and compare it with the maximum
value. The base stack pointer might not be set if this is not value. The base stack pointer might not be set if this is not
@ -172,15 +181,16 @@ update_data (struct header *result, size_t len, size_t old_len)
start_sp = sp; start_sp = sp;
size_t current_stack = start_sp - sp; size_t current_stack = start_sp - sp;
#endif #endif
catomic_max (&peak_stack, current_stack); peak_atomic_max (&peak_stack, current_stack);
/* Add up heap and stack usage and compare it with the maximum value. */ /* Add up heap and stack usage and compare it with the maximum value. */
catomic_max (&peak_total, heap + current_stack); peak_atomic_max (&peak_total, heap + current_stack);
/* Store the value only if we are writing to a file. */ /* Store the value only if we are writing to a file. */
if (fd != -1) if (fd != -1)
{ {
uint32_t idx = catomic_exchange_and_add (&buffer_cnt, 1); uint32_t idx = atomic_fetch_add_explicit (&buffer_cnt, 1,
memory_order_relaxed);
if (idx + 1 >= 2 * buffer_size) if (idx + 1 >= 2 * buffer_size)
{ {
/* We try to reset the counter to the correct range. If /* We try to reset the counter to the correct range. If
@ -188,7 +198,8 @@ update_data (struct header *result, size_t len, size_t old_len)
counter it does not matter since that thread will take counter it does not matter since that thread will take
care of the correction. */ care of the correction. */
uint32_t reset = (idx + 1) % (2 * buffer_size); uint32_t reset = (idx + 1) % (2 * buffer_size);
catomic_compare_and_exchange_val_acq (&buffer_cnt, reset, idx + 1); uint32_t expected = idx + 1;
atomic_compare_exchange_weak (&buffer_cnt, &expected, reset);
if (idx >= 2 * buffer_size) if (idx >= 2 * buffer_size)
idx = reset - 1; idx = reset - 1;
} }
@ -362,24 +373,25 @@ malloc (size_t len)
return (*mallocp)(len); return (*mallocp)(len);
/* Keep track of number of calls. */ /* Keep track of number of calls. */
catomic_increment (&calls[idx_malloc]); atomic_fetch_add_explicit (&calls[idx_malloc], 1, memory_order_relaxed);
/* Keep track of total memory consumption for `malloc'. */ /* Keep track of total memory consumption for `malloc'. */
catomic_add (&total[idx_malloc], len); atomic_fetch_add_explicit (&total[idx_malloc], len, memory_order_relaxed);
/* Keep track of total memory requirement. */ /* Keep track of total memory requirement. */
catomic_add (&grand_total, len); atomic_fetch_add_explicit (&grand_total, len, memory_order_relaxed);
/* Remember the size of the request. */ /* Remember the size of the request. */
if (len < 65536) if (len < 65536)
catomic_increment (&histogram[len / 16]); atomic_fetch_add_explicit (&histogram[len / 16], 1, memory_order_relaxed);
else else
catomic_increment (&large); atomic_fetch_add_explicit (&large, 1, memory_order_relaxed);
/* Total number of calls of any of the functions. */ /* Total number of calls of any of the functions. */
catomic_increment (&calls_total); atomic_fetch_add_explicit (&calls_total, 1, memory_order_relaxed);
/* Do the real work. */ /* Do the real work. */
result = (struct header *) (*mallocp)(len + sizeof (struct header)); result = (struct header *) (*mallocp)(len + sizeof (struct header));
if (result == NULL) if (result == NULL)
{ {
catomic_increment (&failed[idx_malloc]); atomic_fetch_add_explicit (&failed[idx_malloc], 1,
memory_order_relaxed);
return NULL; return NULL;
} }
@ -430,21 +442,24 @@ realloc (void *old, size_t len)
} }
/* Keep track of number of calls. */ /* Keep track of number of calls. */
catomic_increment (&calls[idx_realloc]); atomic_fetch_add_explicit (&calls[idx_realloc], 1, memory_order_relaxed);
if (len > old_len) if (len > old_len)
{ {
/* Keep track of total memory consumption for `realloc'. */ /* Keep track of total memory consumption for `realloc'. */
catomic_add (&total[idx_realloc], len - old_len); atomic_fetch_add_explicit (&total[idx_realloc], len - old_len,
memory_order_relaxed);
/* Keep track of total memory requirement. */ /* Keep track of total memory requirement. */
catomic_add (&grand_total, len - old_len); atomic_fetch_add_explicit (&grand_total, len - old_len,
memory_order_relaxed);
} }
if (len == 0 && old != NULL) if (len == 0 && old != NULL)
{ {
/* Special case. */ /* Special case. */
catomic_increment (&realloc_free); atomic_fetch_add_explicit (&realloc_free, 1, memory_order_relaxed);
/* Keep track of total memory freed using `free'. */ /* Keep track of total memory freed using `free'. */
catomic_add (&total[idx_free], real->length); atomic_fetch_add_explicit (&total[idx_free], real->length,
memory_order_relaxed);
/* Update the allocation data and write out the records if necessary. */ /* Update the allocation data and write out the records if necessary. */
update_data (NULL, 0, old_len); update_data (NULL, 0, old_len);
@ -457,26 +472,27 @@ realloc (void *old, size_t len)
/* Remember the size of the request. */ /* Remember the size of the request. */
if (len < 65536) if (len < 65536)
catomic_increment (&histogram[len / 16]); atomic_fetch_add_explicit (&histogram[len / 16], 1, memory_order_relaxed);
else else
catomic_increment (&large); atomic_fetch_add_explicit (&large, 1, memory_order_relaxed);
/* Total number of calls of any of the functions. */ /* Total number of calls of any of the functions. */
catomic_increment (&calls_total); atomic_fetch_add_explicit (&calls_total, 1, memory_order_relaxed);
/* Do the real work. */ /* Do the real work. */
result = (struct header *) (*reallocp)(real, len + sizeof (struct header)); result = (struct header *) (*reallocp)(real, len + sizeof (struct header));
if (result == NULL) if (result == NULL)
{ {
catomic_increment (&failed[idx_realloc]); atomic_fetch_add_explicit (&failed[idx_realloc], 1,
memory_order_relaxed);
return NULL; return NULL;
} }
/* Record whether the reduction/increase happened in place. */ /* Record whether the reduction/increase happened in place. */
if (real == result) if (real == result)
catomic_increment (&inplace); atomic_fetch_add_explicit (&inplace, 1, memory_order_relaxed);
/* Was the buffer increased? */ /* Was the buffer increased? */
if (old_len > len) if (old_len > len)
catomic_increment (&decreasing); atomic_fetch_add_explicit (&decreasing, 1, memory_order_relaxed);
/* Update the allocation data and write out the records if necessary. */ /* Update the allocation data and write out the records if necessary. */
update_data (result, len, old_len); update_data (result, len, old_len);
@ -508,16 +524,17 @@ calloc (size_t n, size_t len)
return (*callocp)(n, len); return (*callocp)(n, len);
/* Keep track of number of calls. */ /* Keep track of number of calls. */
catomic_increment (&calls[idx_calloc]); atomic_fetch_add_explicit (&calls[idx_calloc], 1, memory_order_relaxed);
/* Keep track of total memory consumption for `calloc'. */ /* Keep track of total memory consumption for `calloc'. */
catomic_add (&total[idx_calloc], size); atomic_fetch_add_explicit (&total[idx_calloc], size, memory_order_relaxed);
/* Keep track of total memory requirement. */ /* Keep track of total memory requirement. */
catomic_add (&grand_total, size); atomic_fetch_add_explicit (&grand_total, size, memory_order_relaxed);
/* Remember the size of the request. */ /* Remember the size of the request. */
if (size < 65536) if (size < 65536)
catomic_increment (&histogram[size / 16]); atomic_fetch_add_explicit (&histogram[size / 16], 1,
memory_order_relaxed);
else else
catomic_increment (&large); atomic_fetch_add_explicit (&large, 1, memory_order_relaxed);
/* Total number of calls of any of the functions. */ /* Total number of calls of any of the functions. */
++calls_total; ++calls_total;
@ -525,7 +542,8 @@ calloc (size_t n, size_t len)
result = (struct header *) (*mallocp)(size + sizeof (struct header)); result = (struct header *) (*mallocp)(size + sizeof (struct header));
if (result == NULL) if (result == NULL)
{ {
catomic_increment (&failed[idx_calloc]); atomic_fetch_add_explicit (&failed[idx_calloc], 1,
memory_order_relaxed);
return NULL; return NULL;
} }
@ -563,7 +581,7 @@ free (void *ptr)
/* `free (NULL)' has no effect. */ /* `free (NULL)' has no effect. */
if (ptr == NULL) if (ptr == NULL)
{ {
catomic_increment (&calls[idx_free]); atomic_fetch_add_explicit (&calls[idx_free], 1, memory_order_relaxed);
return; return;
} }
@ -577,9 +595,10 @@ free (void *ptr)
} }
/* Keep track of number of calls. */ /* Keep track of number of calls. */
catomic_increment (&calls[idx_free]); atomic_fetch_add_explicit (&calls[idx_free], 1, memory_order_relaxed);
/* Keep track of total memory freed using `free'. */ /* Keep track of total memory freed using `free'. */
catomic_add (&total[idx_free], real->length); atomic_fetch_add_explicit (&total[idx_free], real->length,
memory_order_relaxed);
/* Update the allocation data and write out the records if necessary. */ /* Update the allocation data and write out the records if necessary. */
update_data (NULL, 0, real->length); update_data (NULL, 0, real->length);
@ -614,22 +633,23 @@ mmap (void *start, size_t len, int prot, int flags, int fd, off_t offset)
? idx_mmap_a : prot & PROT_WRITE ? idx_mmap_w : idx_mmap_r); ? idx_mmap_a : prot & PROT_WRITE ? idx_mmap_w : idx_mmap_r);
/* Keep track of number of calls. */ /* Keep track of number of calls. */
catomic_increment (&calls[idx]); atomic_fetch_add_explicit (&calls[idx], 1, memory_order_relaxed);
/* Keep track of total memory consumption for `malloc'. */ /* Keep track of total memory consumption for `malloc'. */
catomic_add (&total[idx], len); atomic_fetch_add_explicit (&total[idx], len, memory_order_relaxed);
/* Keep track of total memory requirement. */ /* Keep track of total memory requirement. */
catomic_add (&grand_total, len); atomic_fetch_add_explicit (&grand_total, len, memory_order_relaxed);
/* Remember the size of the request. */ /* Remember the size of the request. */
if (len < 65536) if (len < 65536)
catomic_increment (&histogram[len / 16]); atomic_fetch_add_explicit (&histogram[len / 16], 1,
memory_order_relaxed);
else else
catomic_increment (&large); atomic_fetch_add_explicit (&large, 1, memory_order_relaxed);
/* Total number of calls of any of the functions. */ /* Total number of calls of any of the functions. */
catomic_increment (&calls_total); atomic_fetch_add_explicit (&calls_total, 1, memory_order_relaxed);
/* Check for failures. */ /* Check for failures. */
if (result == NULL) if (result == NULL)
catomic_increment (&failed[idx]); atomic_fetch_add_explicit (&failed[idx], 1, memory_order_relaxed);
else if (idx == idx_mmap_w) else if (idx == idx_mmap_w)
/* Update the allocation data and write out the records if /* Update the allocation data and write out the records if
necessary. Note the first parameter is NULL which means necessary. Note the first parameter is NULL which means
@ -667,22 +687,23 @@ mmap64 (void *start, size_t len, int prot, int flags, int fd, off64_t offset)
? idx_mmap_a : prot & PROT_WRITE ? idx_mmap_w : idx_mmap_r); ? idx_mmap_a : prot & PROT_WRITE ? idx_mmap_w : idx_mmap_r);
/* Keep track of number of calls. */ /* Keep track of number of calls. */
catomic_increment (&calls[idx]); atomic_fetch_add_explicit (&calls[idx], 1, memory_order_relaxed);
/* Keep track of total memory consumption for `malloc'. */ /* Keep track of total memory consumption for `malloc'. */
catomic_add (&total[idx], len); atomic_fetch_add_explicit (&total[idx], len, memory_order_relaxed);
/* Keep track of total memory requirement. */ /* Keep track of total memory requirement. */
catomic_add (&grand_total, len); atomic_fetch_add_explicit (&grand_total, len, memory_order_relaxed);
/* Remember the size of the request. */ /* Remember the size of the request. */
if (len < 65536) if (len < 65536)
catomic_increment (&histogram[len / 16]); atomic_fetch_add_explicit (&histogram[len / 16], 1,
memory_order_relaxed);
else else
catomic_increment (&large); atomic_fetch_add_explicit (&large, 1, memory_order_relaxed);
/* Total number of calls of any of the functions. */ /* Total number of calls of any of the functions. */
catomic_increment (&calls_total); atomic_fetch_add_explicit (&calls_total, 1, memory_order_relaxed);
/* Check for failures. */ /* Check for failures. */
if (result == NULL) if (result == NULL)
catomic_increment (&failed[idx]); atomic_fetch_add_explicit (&failed[idx], 1, memory_order_relaxed);
else if (idx == idx_mmap_w) else if (idx == idx_mmap_w)
/* Update the allocation data and write out the records if /* Update the allocation data and write out the records if
necessary. Note the first parameter is NULL which means necessary. Note the first parameter is NULL which means
@ -722,33 +743,39 @@ mremap (void *start, size_t old_len, size_t len, int flags, ...)
if (!not_me && trace_mmap) if (!not_me && trace_mmap)
{ {
/* Keep track of number of calls. */ /* Keep track of number of calls. */
catomic_increment (&calls[idx_mremap]); atomic_fetch_add_explicit (&calls[idx_mremap], 1, memory_order_relaxed);
if (len > old_len) if (len > old_len)
{ {
/* Keep track of total memory consumption for `malloc'. */ /* Keep track of total memory consumption for `malloc'. */
catomic_add (&total[idx_mremap], len - old_len); atomic_fetch_add_explicit (&total[idx_mremap], len - old_len,
memory_order_relaxed);
/* Keep track of total memory requirement. */ /* Keep track of total memory requirement. */
catomic_add (&grand_total, len - old_len); atomic_fetch_add_explicit (&grand_total, len - old_len,
memory_order_relaxed);
} }
/* Remember the size of the request. */ /* Remember the size of the request. */
if (len < 65536) if (len < 65536)
catomic_increment (&histogram[len / 16]); atomic_fetch_add_explicit (&histogram[len / 16], 1,
memory_order_relaxed);
else else
catomic_increment (&large); atomic_fetch_add_explicit (&large, 1, memory_order_relaxed);
/* Total number of calls of any of the functions. */ /* Total number of calls of any of the functions. */
catomic_increment (&calls_total); atomic_fetch_add_explicit (&calls_total, 1, memory_order_relaxed);
/* Check for failures. */ /* Check for failures. */
if (result == NULL) if (result == NULL)
catomic_increment (&failed[idx_mremap]); atomic_fetch_add_explicit (&failed[idx_mremap], 1,
memory_order_relaxed);
else else
{ {
/* Record whether the reduction/increase happened in place. */ /* Record whether the reduction/increase happened in place. */
if (start == result) if (start == result)
catomic_increment (&inplace_mremap); atomic_fetch_add_explicit (&inplace_mremap, 1,
memory_order_relaxed);
/* Was the buffer increased? */ /* Was the buffer increased? */
if (old_len > len) if (old_len > len)
catomic_increment (&decreasing_mremap); atomic_fetch_add_explicit (&decreasing_mremap, 1,
memory_order_relaxed);
/* Update the allocation data and write out the records if /* Update the allocation data and write out the records if
necessary. Note the first parameter is NULL which means necessary. Note the first parameter is NULL which means
@ -783,19 +810,21 @@ munmap (void *start, size_t len)
if (!not_me && trace_mmap) if (!not_me && trace_mmap)
{ {
/* Keep track of number of calls. */ /* Keep track of number of calls. */
catomic_increment (&calls[idx_munmap]); atomic_fetch_add_explicit (&calls[idx_munmap], 1, memory_order_relaxed);
if (__glibc_likely (result == 0)) if (__glibc_likely (result == 0))
{ {
/* Keep track of total memory freed using `free'. */ /* Keep track of total memory freed using `free'. */
catomic_add (&total[idx_munmap], len); atomic_fetch_add_explicit (&total[idx_munmap], len,
memory_order_relaxed);
/* Update the allocation data and write out the records if /* Update the allocation data and write out the records if
necessary. */ necessary. */
update_data (NULL, 0, len); update_data (NULL, 0, len);
} }
else else
catomic_increment (&failed[idx_munmap]); atomic_fetch_add_explicit (&failed[idx_munmap], 1,
memory_order_relaxed);
} }
return result; return result;