From b9b8d9f1885175ba4a59e4c22748b29158e35c1d Mon Sep 17 00:00:00 2001 From: Radkesvat <134321679+radkesvat@users.noreply.github.com> Date: Tue, 9 Jul 2024 01:20:08 +0000 Subject: [PATCH] use wireshark memory manager --- tunnels/shared/openssl/openssl_globals.c | 4 +- ww/CMakeLists.txt | 1 + ww/managers/alloc-engine/wof_allocator.c | 909 +++++++++++++++ ww/managers/alloc-engine/wof_allocator.h | 41 + ww/managers/memory_manager.c | 1312 +--------------------- ww/managers/memory_manager.h | 221 +--- ww/ww.c | 18 +- ww/ww.h | 4 +- 8 files changed, 1048 insertions(+), 1462 deletions(-) create mode 100644 ww/managers/alloc-engine/wof_allocator.c create mode 100644 ww/managers/alloc-engine/wof_allocator.h diff --git a/tunnels/shared/openssl/openssl_globals.c b/tunnels/shared/openssl/openssl_globals.c index 8fba3c89..a8aea8ec 100644 --- a/tunnels/shared/openssl/openssl_globals.c +++ b/tunnels/shared/openssl/openssl_globals.c @@ -11,7 +11,7 @@ static int openssl_lib_initialized = false; -static struct ww_dedictaed_mem_s *openssl_dedicated_memory_manager; +static struct dedicated_memory_s *openssl_dedicated_memory_manager; static void *opennsl_dedicated_malloc(size_t num, const char *file, int line) { @@ -44,7 +44,7 @@ void opensslGlobalInit(void) // #else // OPENSSL_init_ssl(OPENSSL_INIT_SSL_DEFAULT, NULL); // #endif - openssl_dedicated_memory_manager = wwmDedicatedCreateDefault(); + openssl_dedicated_memory_manager = createWWDedicatedMemory(); if (0 == CRYPTO_set_mem_functions(opennsl_dedicated_malloc, opennsl_dedicated_realloc, opennsl_dedicated_free)) { LOGF("OpenSSl Global: could not swap openssl allocators (almost always because allocations have already " diff --git a/ww/CMakeLists.txt b/ww/CMakeLists.txt index 8164d6d5..ca9a6d30 100644 --- a/ww/CMakeLists.txt +++ b/ww/CMakeLists.txt @@ -20,6 +20,7 @@ add_library(ww STATIC utils/btree.c managers/socket_manager.c managers/node_manager.c + managers/alloc-engine/wof_allocator.c managers/memory_manager.c managers/data/iprange_mci.c managers/data/iprange_irancell.c diff --git a/ww/managers/alloc-engine/wof_allocator.c b/ww/managers/alloc-engine/wof_allocator.c new file mode 100644 index 00000000..10d26a85 --- /dev/null +++ b/ww/managers/alloc-engine/wof_allocator.c @@ -0,0 +1,909 @@ +/* Wheel-of-Fortune Memory Allocator + * Copyright 2013, Evan Huus + */ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Woverflow" + +#include +#include + +#include "wof_allocator.h" + + + +#define BOOL int +#define TRUE 1 +#define FALSE 0 + +/* https://mail.gnome.org/archives/gtk-devel-list/2004-December/msg00091.html + * The 2*sizeof(size_t) alignment here is borrowed from GNU libc, so it should + * be good most everywhere. It is more conservative than is needed on some + * 64-bit platforms, but ia64 does require a 16-byte alignment. The SIMD + * extensions for x86 and ppc32 would want a larger alignment than this, but + * we don't need to do better than malloc. + */ +#define WOF_ALIGN_AMOUNT (2 * sizeof (size_t)) +#define WOF_ALIGN_SIZE(SIZE) ((~(WOF_ALIGN_AMOUNT-1)) & \ + ((SIZE) + (WOF_ALIGN_AMOUNT-1))) + +/* When required, allocate more memory from the OS in chunks of this size. + * 8MB is a pretty arbitrary value - it's big enough that it should last a while + * and small enough that a mostly-unused one doesn't waste *too* much. It's + * also a nice power of two, of course. */ +#define WOF_BLOCK_SIZE (8 * 1024 * 1024) + +/* The header for an entire OS-level 'block' of memory */ +typedef struct _wof_block_hdr_t { + struct _wof_block_hdr_t *prev, *next; +} wof_block_hdr_t; + +/* The header for a single 'chunk' of memory as returned from alloc/realloc. + * The 'jumbo' flag indicates an allocation larger than a normal-sized block + * would be capable of serving. If this is set, it is the only chunk in the + * block and the other chunk header fields are irrelevant. + */ +typedef struct _wof_chunk_hdr_t { + int prev; + + /* flags */ + int last:1; + int used:1; + int jumbo:1; + + int len:29; +} wof_chunk_hdr_t; + +/* Handy macros for navigating the chunks in a block as if they were a + * doubly-linked list. */ +#define WOF_CHUNK_PREV(CHUNK) ((CHUNK)->prev \ + ? ((wof_chunk_hdr_t*)(((unsigned char*)(CHUNK)) - (CHUNK)->prev)) \ + : NULL) + +#define WOF_CHUNK_NEXT(CHUNK) ((CHUNK)->last \ + ? NULL \ + : ((wof_chunk_hdr_t*)(((unsigned char*)(CHUNK)) + (CHUNK)->len))) + +#define WOF_CHUNK_HEADER_SIZE WOF_ALIGN_SIZE(sizeof(wof_chunk_hdr_t)) + +/* other handy chunk macros */ +#define WOF_CHUNK_TO_DATA(CHUNK) ((void*)((unsigned char*)(CHUNK) + WOF_CHUNK_HEADER_SIZE)) +#define WOF_DATA_TO_CHUNK(DATA) ((wof_chunk_hdr_t*)((unsigned char*)(DATA) - WOF_CHUNK_HEADER_SIZE)) +#define WOF_CHUNK_DATA_LEN(CHUNK) ((CHUNK)->len - WOF_CHUNK_HEADER_SIZE) + +/* some handy block macros */ +#define WOF_BLOCK_HEADER_SIZE WOF_ALIGN_SIZE(sizeof(wof_block_hdr_t)) +#define WOF_BLOCK_TO_CHUNK(BLOCK) ((wof_chunk_hdr_t*)((unsigned char*)(BLOCK) + WOF_BLOCK_HEADER_SIZE)) +#define WOF_CHUNK_TO_BLOCK(CHUNK) ((wof_block_hdr_t*)((unsigned char*)(CHUNK) - WOF_BLOCK_HEADER_SIZE)) + +#define WOF_BLOCK_MAX_ALLOC_SIZE (WOF_BLOCK_SIZE - \ + (WOF_BLOCK_HEADER_SIZE + WOF_CHUNK_HEADER_SIZE)) + +/* This is what the 'data' section of a chunk contains if it is free. */ +typedef struct _wof_free_hdr_t { + wof_chunk_hdr_t *prev, *next; +} wof_free_hdr_t; + +/* Handy macro for accessing the free-header of a chunk */ +#define WOF_GET_FREE(CHUNK) ((wof_free_hdr_t*)WOF_CHUNK_TO_DATA(CHUNK)) + +/* The size of the free header does not need to be aligned, as it is never used + * in any way that would affect the alignment of the memory returned from + * wof_alloc. This macro is defined just for consistency with all the other + * WOF_*_SIZE macros (which do need to be aligned). */ +#define WOF_FREE_HEADER_SIZE sizeof(wof_free_hdr_t) + +struct _wof_allocator_t { + wof_block_hdr_t *block_list; + wof_chunk_hdr_t *master_head; + wof_chunk_hdr_t *recycler_head; +}; + +/* MASTER/RECYCLER HELPERS */ + +/* Cycles the recycler. See the design notes in the readme for more details. */ +static void +wof_cycle_recycler(wof_allocator_t *allocator) +{ + wof_chunk_hdr_t *chunk; + wof_free_hdr_t *free_chunk; + + chunk = allocator->recycler_head; + + if (chunk == NULL) { + return; + } + + free_chunk = WOF_GET_FREE(chunk); + + if (free_chunk->next->len < chunk->len) { + /* Hold the current head fixed during rotation. */ + WOF_GET_FREE(free_chunk->next)->prev = free_chunk->prev; + WOF_GET_FREE(free_chunk->prev)->next = free_chunk->next; + + free_chunk->prev = free_chunk->next; + free_chunk->next = WOF_GET_FREE(free_chunk->next)->next; + + WOF_GET_FREE(free_chunk->next)->prev = chunk; + WOF_GET_FREE(free_chunk->prev)->next = chunk; + } + else { + /* Just rotate everything. */ + allocator->recycler_head = free_chunk->next; + } +} + +/* Adds a chunk from the recycler. */ +static void +wof_add_to_recycler(wof_allocator_t *allocator, + wof_chunk_hdr_t *chunk) +{ + wof_free_hdr_t *free_chunk; + + if (WOF_CHUNK_DATA_LEN(chunk) < WOF_FREE_HEADER_SIZE) { + return; + } + + free_chunk = WOF_GET_FREE(chunk); + + if (! allocator->recycler_head) { + /* First one */ + free_chunk->next = chunk; + free_chunk->prev = chunk; + allocator->recycler_head = chunk; + } + else { + free_chunk->next = allocator->recycler_head; + free_chunk->prev = WOF_GET_FREE(allocator->recycler_head)->prev; + + WOF_GET_FREE(free_chunk->next)->prev = chunk; + WOF_GET_FREE(free_chunk->prev)->next = chunk; + + if (chunk->len > allocator->recycler_head->len) { + allocator->recycler_head = chunk; + } + } +} + +/* Removes a chunk from the recycler. */ +static void +wof_remove_from_recycler(wof_allocator_t *allocator, + wof_chunk_hdr_t *chunk) +{ + wof_free_hdr_t *free_chunk; + + free_chunk = WOF_GET_FREE(chunk); + + if (free_chunk->prev == chunk && free_chunk->next == chunk) { + /* Only one item in recycler, just empty it. */ + allocator->recycler_head = NULL; + } + else { + /* Two or more items, usual doubly-linked-list removal. It's circular + * so we don't need to worry about null-checking anything, which is + * nice. */ + WOF_GET_FREE(free_chunk->prev)->next = free_chunk->next; + WOF_GET_FREE(free_chunk->next)->prev = free_chunk->prev; + if (allocator->recycler_head == chunk) { + allocator->recycler_head = free_chunk->next; + } + } +} + +/* Pushes a chunk onto the master stack. */ +static void +wof_push_master(wof_allocator_t *allocator, + wof_chunk_hdr_t *chunk) +{ + wof_free_hdr_t *free_chunk; + + free_chunk = WOF_GET_FREE(chunk); + free_chunk->prev = NULL; + free_chunk->next = allocator->master_head; + if (free_chunk->next) { + WOF_GET_FREE(free_chunk->next)->prev = chunk; + } + allocator->master_head = chunk; +} + +/* Removes the top chunk from the master stack. */ +static void +wof_pop_master(wof_allocator_t *allocator) +{ + wof_chunk_hdr_t *chunk; + wof_free_hdr_t *free_chunk; + + chunk = allocator->master_head; + + free_chunk = WOF_GET_FREE(chunk); + + allocator->master_head = free_chunk->next; + if (free_chunk->next) { + WOF_GET_FREE(free_chunk->next)->prev = NULL; + } +} + +/* CHUNK HELPERS */ + +/* Takes a free chunk and checks the chunks to its immediate right and left in + * the block. If they are also free, the contigous free chunks are merged into + * a single free chunk. The resulting chunk ends up in either the master list or + * the recycler, depending on where the merged chunks were originally. + */ +static void +wof_merge_free(wof_allocator_t *allocator, + wof_chunk_hdr_t *chunk) +{ + wof_chunk_hdr_t *tmp; + wof_chunk_hdr_t *left_free = NULL; + wof_chunk_hdr_t *right_free = NULL; + + /* Check the chunk to our right. If it is free, merge it into our current + * chunk. If it is big enough to hold a free-header, save it for later (we + * need to know about the left chunk before we decide what goes where). */ + tmp = WOF_CHUNK_NEXT(chunk); + if (tmp && !tmp->used) { + if (WOF_CHUNK_DATA_LEN(tmp) >= WOF_FREE_HEADER_SIZE) { + right_free = tmp; + } + chunk->len += tmp->len; + chunk->last = tmp->last; + } + + /* Check the chunk to our left. If it is free, merge our current chunk into + * it (thus chunk = tmp). As before, save it if it has enough space to + * hold a free-header. */ + tmp = WOF_CHUNK_PREV(chunk); + if (tmp && !tmp->used) { + if (WOF_CHUNK_DATA_LEN(tmp) >= WOF_FREE_HEADER_SIZE) { + left_free = tmp; + } + tmp->len += chunk->len; + tmp->last = chunk->last; + chunk = tmp; + } + + /* The length of our chunk may have changed. If we have a chunk following, + * update its 'prev' count. */ + if (!chunk->last) { + WOF_CHUNK_NEXT(chunk)->prev = chunk->len; + } + + /* Now that the chunk headers are merged and consistent, we need to figure + * out what goes where in which free list. */ + if (right_free && right_free == allocator->master_head) { + /* If we merged right, and that chunk was the head of the master list, + * then we leave the resulting chunk at the head of the master list. */ + wof_free_hdr_t *moved; + if (left_free) { + wof_remove_from_recycler(allocator, left_free); + } + moved = WOF_GET_FREE(chunk); + moved->prev = NULL; + moved->next = WOF_GET_FREE(right_free)->next; + allocator->master_head = chunk; + if (moved->next) { + WOF_GET_FREE(moved->next)->prev = chunk; + } + } + else { + /* Otherwise, we remove the right-merged chunk (if there was one) from + * the recycler. Then, if we merged left we have nothing to do, since + * that recycler entry is still valid. If not, we add the chunk. */ + if (right_free) { + wof_remove_from_recycler(allocator, right_free); + } + if (!left_free) { + wof_add_to_recycler(allocator, chunk); + } + } +} + +/* Takes an unused chunk and a size, and splits it into two chunks if possible. + * The first chunk (at the same address as the input chunk) is guaranteed to + * hold at least `size` bytes of data, and to not be in either the master or + * recycler lists. + * + * The second chunk gets whatever data is left over. It is marked unused and + * replaces the input chunk in whichever list it originally inhabited. */ +static void +wof_split_free_chunk(wof_allocator_t *allocator, + wof_chunk_hdr_t *chunk, + const size_t size) +{ + wof_chunk_hdr_t *extra; + wof_free_hdr_t *old_blk, *new_blk; + size_t aligned_size, available; + BOOL last; + + aligned_size = WOF_ALIGN_SIZE(size) + WOF_CHUNK_HEADER_SIZE; + + if (WOF_CHUNK_DATA_LEN(chunk) < aligned_size + WOF_FREE_HEADER_SIZE) { + /* If the available space is not enought to store all of + * (hdr + requested size + alignment padding + hdr + free-header) then + * just remove the current chunk from the free list and return, since we + * can't usefully split it. */ + if (chunk == allocator->master_head) { + wof_pop_master(allocator); + } + else if (WOF_CHUNK_DATA_LEN(chunk) >= WOF_FREE_HEADER_SIZE) { + wof_remove_from_recycler(allocator, chunk); + } + return; + } + + /* preserve a few values from chunk that we'll need to manipulate */ + last = chunk->last; + available = chunk->len - aligned_size; + + /* set new values for chunk */ + chunk->len = (int) aligned_size; + chunk->last = FALSE; + + /* with chunk's values set, we can use the standard macro to calculate + * the location and size of the new free chunk */ + extra = WOF_CHUNK_NEXT(chunk); + + /* Now we move the free chunk's address without changing its location + * in whichever list it is in. + * + * Note that the new chunk header 'extra' may overlap the old free header, + * so we have to copy the free header before we write anything to extra. + */ + old_blk = WOF_GET_FREE(chunk); + new_blk = WOF_GET_FREE(extra); + + if (allocator->master_head == chunk) { + new_blk->prev = old_blk->prev; + new_blk->next = old_blk->next; + + if (old_blk->next) { + WOF_GET_FREE(old_blk->next)->prev = extra; + } + + allocator->master_head = extra; + } + else { + if (old_blk->prev == chunk) { + new_blk->prev = extra; + new_blk->next = extra; + } + else { + new_blk->prev = old_blk->prev; + new_blk->next = old_blk->next; + + WOF_GET_FREE(old_blk->prev)->next = extra; + WOF_GET_FREE(old_blk->next)->prev = extra; + } + + if (allocator->recycler_head == chunk) { + allocator->recycler_head = extra; + } + } + + /* Now that we've copied over the free-list stuff (which may have overlapped + * with our new chunk header) we can safely write our new chunk header. */ + extra->len = (int) available; + extra->last = last; + extra->prev = chunk->len; + extra->used = FALSE; + extra->jumbo = FALSE; + + /* Correctly update the following chunk's back-pointer */ + if (!last) { + WOF_CHUNK_NEXT(extra)->prev = extra->len; + } +} + +/* Takes a used chunk and a size, and splits it into two chunks if possible. + * The first chunk can hold at least `size` bytes of data, while the second gets + * whatever's left over. The second is marked as unused and is added to the + * recycler. */ +static void +wof_split_used_chunk(wof_allocator_t *allocator, + wof_chunk_hdr_t *chunk, + const size_t size) +{ + wof_chunk_hdr_t *extra; + size_t aligned_size, available; + BOOL last; + + aligned_size = WOF_ALIGN_SIZE(size) + WOF_CHUNK_HEADER_SIZE; + + if (aligned_size > WOF_CHUNK_DATA_LEN(chunk)) { + /* in this case we don't have enough space to really split it, so + * it's basically a no-op */ + return; + } + /* otherwise, we have room to split it, though the remaining free chunk + * may still not be usefully large */ + + /* preserve a few values from chunk that we'll need to manipulate */ + last = chunk->last; + available = chunk->len - aligned_size; + + /* set new values for chunk */ + chunk->len = (int) aligned_size; + chunk->last = FALSE; + + /* with chunk's values set, we can use the standard macro to calculate + * the location and size of the new free chunk */ + extra = WOF_CHUNK_NEXT(chunk); + + /* set the new values for the chunk */ + extra->len = (int) available; + extra->last = last; + extra->prev = chunk->len; + extra->used = FALSE; + extra->jumbo = FALSE; + + /* Correctly update the following chunk's back-pointer */ + if (!last) { + WOF_CHUNK_NEXT(extra)->prev = extra->len; + } + + /* Merge it to its right if possible (it can't be merged left, obviously). + * This also adds it to the recycler. */ + wof_merge_free(allocator, extra); +} + +/* BLOCK HELPERS */ + +/* Add a block to the allocator's embedded doubly-linked list of OS-level blocks + * that it owns. */ +static void +wof_add_to_block_list(wof_allocator_t *allocator, + wof_block_hdr_t *block) +{ + block->prev = NULL; + block->next = allocator->block_list; + if (block->next) { + block->next->prev = block; + } + allocator->block_list = block; +} + +/* Remove a block from the allocator's embedded doubly-linked list of OS-level + * blocks that it owns. */ +static void +wof_remove_from_block_list(wof_allocator_t *allocator, + wof_block_hdr_t *block) +{ + if (block->prev) { + block->prev->next = block->next; + } + else { + allocator->block_list = block->next; + } + + if (block->next) { + block->next->prev = block->prev; + } +} + +/* Initializes a single unused chunk at the beginning of the block, and + * adds that chunk to the free list. */ +static void +wof_init_block(wof_allocator_t *allocator, + wof_block_hdr_t *block) +{ + wof_chunk_hdr_t *chunk; + + /* a new block contains one chunk, right at the beginning */ + chunk = WOF_BLOCK_TO_CHUNK(block); + + chunk->used = FALSE; + chunk->jumbo = FALSE; + chunk->last = TRUE; + chunk->prev = 0; + chunk->len = WOF_BLOCK_SIZE - WOF_BLOCK_HEADER_SIZE; + + /* now push that chunk onto the master list */ + wof_push_master(allocator, chunk); +} + +/* Creates a new block, and initializes it. */ +static void +wof_new_block(wof_allocator_t *allocator) +{ + wof_block_hdr_t *block; + + /* allocate the new block and add it to the block list */ + block = (wof_block_hdr_t *)malloc(WOF_BLOCK_SIZE); + + if (block == NULL) { + return; + } + + wof_add_to_block_list(allocator, block); + + /* initialize it */ + wof_init_block(allocator, block); +} + +/* JUMBO ALLOCATIONS */ + +/* Allocates special 'jumbo' blocks for sizes that won't fit normally. */ +static void * +wof_alloc_jumbo(wof_allocator_t *allocator, const size_t size) +{ + wof_block_hdr_t *block; + wof_chunk_hdr_t *chunk; + + /* allocate a new block of exactly the right size */ + block = (wof_block_hdr_t *) malloc(size + + WOF_BLOCK_HEADER_SIZE + + WOF_CHUNK_HEADER_SIZE); + + if (block == NULL) { + return NULL; + } + + /* add it to the block list */ + wof_add_to_block_list(allocator, block); + + /* the new block contains a single jumbo chunk */ + chunk = WOF_BLOCK_TO_CHUNK(block); + chunk->last = TRUE; + chunk->used = TRUE; + chunk->jumbo = TRUE; + chunk->len = 0; + chunk->prev = 0; + + /* and return the data pointer */ + return WOF_CHUNK_TO_DATA(chunk); +} + +/* Frees special 'jumbo' blocks of sizes that won't fit normally. */ +static void +wof_free_jumbo(wof_allocator_t *allocator, + wof_chunk_hdr_t *chunk) +{ + wof_block_hdr_t *block; + + block = WOF_CHUNK_TO_BLOCK(chunk); + + wof_remove_from_block_list(allocator, block); + + free(block); +} + +/* Reallocs special 'jumbo' blocks of sizes that won't fit normally. */ +static void * +wof_realloc_jumbo(wof_allocator_t *allocator, + wof_chunk_hdr_t *chunk, + const size_t size) +{ + wof_block_hdr_t *block; + + block = WOF_CHUNK_TO_BLOCK(chunk); + + block = (wof_block_hdr_t *) realloc(block, size + + WOF_BLOCK_HEADER_SIZE + + WOF_CHUNK_HEADER_SIZE); + + if (block == NULL) { + return NULL; + } + + if (block->next) { + block->next->prev = block; + } + + if (block->prev) { + block->prev->next = block; + } + else { + allocator->block_list = block; + } + + return WOF_CHUNK_TO_DATA(WOF_BLOCK_TO_CHUNK(block)); +} + +/* API */ + +#ifdef __cplusplus +extern "C" { +#endif /* __cplusplus */ + +void * +wof_alloc(wof_allocator_t *allocator, const size_t size) +{ + wof_chunk_hdr_t *chunk; + + if (size == 0) { + return NULL; + } + else if (size > WOF_BLOCK_MAX_ALLOC_SIZE) { + return wof_alloc_jumbo(allocator, size); + } + + if (allocator->recycler_head && + WOF_CHUNK_DATA_LEN(allocator->recycler_head) >= size) { + + /* If we can serve it from the recycler, do so. */ + chunk = allocator->recycler_head; + } + else { + if (allocator->master_head && + WOF_CHUNK_DATA_LEN(allocator->master_head) < size) { + + /* Recycle the head of the master list if necessary. */ + chunk = allocator->master_head; + wof_pop_master(allocator); + wof_add_to_recycler(allocator, chunk); + } + + if (!allocator->master_head) { + /* Allocate a new block if necessary. */ + wof_new_block(allocator); + } + + chunk = allocator->master_head; + } + + if (!chunk) { + /* We don't have enough, and the OS wouldn't give us more. */ + return NULL; + } + + /* Split our chunk into two to preserve any trailing free space */ + wof_split_free_chunk(allocator, chunk, size); + + /* Now cycle the recycler */ + wof_cycle_recycler(allocator); + + /* mark it as used */ + chunk->used = TRUE; + + /* and return the user's pointer */ + return WOF_CHUNK_TO_DATA(chunk); +} + +void +wof_free(wof_allocator_t *allocator, void *ptr) +{ + wof_chunk_hdr_t *chunk; + + if (ptr == NULL) { + return; + } + + chunk = WOF_DATA_TO_CHUNK(ptr); + + if (chunk->jumbo) { + wof_free_jumbo(allocator, chunk); + return; + } + + /* mark it as unused */ + chunk->used = FALSE; + + /* merge it with any other free chunks adjacent to it, so that contiguous + * free space doesn't get fragmented */ + wof_merge_free(allocator, chunk); + + /* Now cycle the recycler */ + wof_cycle_recycler(allocator); +} + +void * +wof_realloc(wof_allocator_t *allocator, void *ptr, const size_t size) +{ + wof_chunk_hdr_t *chunk; + + if (ptr == NULL) { + return wof_alloc(allocator, size); + } + + if (size == 0) { + wof_free(allocator, ptr); + return NULL; + } + + chunk = WOF_DATA_TO_CHUNK(ptr); + + if (chunk->jumbo) { + return wof_realloc_jumbo(allocator, chunk, size); + } + + if (size > WOF_CHUNK_DATA_LEN(chunk)) { + /* grow */ + wof_chunk_hdr_t *tmp; + + tmp = WOF_CHUNK_NEXT(chunk); + + if (tmp && (!tmp->used) && + (size < WOF_CHUNK_DATA_LEN(chunk) + tmp->len)) { + /* the next chunk is free and has enough extra, so just grab + * from that */ + size_t split_size; + + /* we ask for the next chunk to be split, but we don't end up + * using the split chunk header (it just gets merged into this one), + * so we want the split to be of (size - curdatalen - header_size). + * However, this can underflow by header_size, so we do a quick + * check here and floor the value to 0. */ + split_size = size - WOF_CHUNK_DATA_LEN(chunk); + + if (split_size < WOF_CHUNK_HEADER_SIZE) { + split_size = 0; + } + else { + split_size -= WOF_CHUNK_HEADER_SIZE; + } + + wof_split_free_chunk(allocator, tmp, split_size); + + /* Now do a 'quickie' merge between the current block and the left- + * hand side of the split. Simply calling wof_merge_free + * might confuse things, since we may temporarily have two blocks + * to our right that are both free (and it isn't guaranteed to + * handle that case). Update our 'next' count and last flag, and + * our (new) successor's 'prev' count */ + chunk->len += tmp->len; + chunk->last = tmp->last; + tmp = WOF_CHUNK_NEXT(chunk); + if (tmp) { + tmp->prev = chunk->len; + } + + /* Now cycle the recycler */ + wof_cycle_recycler(allocator); + + /* And return the same old pointer */ + return ptr; + } + else { + /* no room to grow, need to alloc, copy, free */ + void *newptr; + + newptr = wof_alloc(allocator, size); + if (newptr == NULL) { + return NULL; + } + memcpy(newptr, ptr, WOF_CHUNK_DATA_LEN(chunk)); + wof_free(allocator, ptr); + + /* No need to cycle the recycler, alloc and free both did that + * already */ + + return newptr; + } + } + else if (size < WOF_CHUNK_DATA_LEN(chunk)) { + /* shrink */ + wof_split_used_chunk(allocator, chunk, size); + + /* Now cycle the recycler */ + wof_cycle_recycler(allocator); + + return ptr; + } + + /* no-op */ + return ptr; +} + +void +wof_free_all(wof_allocator_t *allocator) +{ + wof_block_hdr_t *cur; + wof_chunk_hdr_t *chunk; + + /* the existing free lists are entirely irrelevant */ + allocator->master_head = NULL; + allocator->recycler_head = NULL; + + /* iterate through the blocks, reinitializing each one */ + cur = allocator->block_list; + + while (cur) { + chunk = WOF_BLOCK_TO_CHUNK(cur); + if (chunk->jumbo) { + wof_remove_from_block_list(allocator, cur); + cur = cur->next; + free(WOF_CHUNK_TO_BLOCK(chunk)); + } + else { + wof_init_block(allocator, cur); + cur = cur->next; + } + } +} + +void +wof_gc(wof_allocator_t *allocator) +{ + wof_block_hdr_t *cur, *next; + wof_chunk_hdr_t *chunk; + wof_free_hdr_t *free_chunk; + + /* Walk through the blocks, adding used blocks to the new list and + * completely destroying unused blocks. */ + cur = allocator->block_list; + allocator->block_list = NULL; + + while (cur) { + chunk = WOF_BLOCK_TO_CHUNK(cur); + next = cur->next; + + if (!chunk->jumbo && !chunk->used && chunk->last) { + /* If the first chunk is also the last, and is unused, then + * the block as a whole is entirely unused, so return it to + * the OS and remove it from whatever lists it is in. */ + free_chunk = WOF_GET_FREE(chunk); + if (free_chunk->next) { + WOF_GET_FREE(free_chunk->next)->prev = free_chunk->prev; + } + if (free_chunk->prev) { + WOF_GET_FREE(free_chunk->prev)->next = free_chunk->next; + } + if (allocator->recycler_head == chunk) { + if (free_chunk->next == chunk) { + allocator->recycler_head = NULL; + } + else { + allocator->recycler_head = free_chunk->next; + } + } + else if (allocator->master_head == chunk) { + allocator->master_head = free_chunk->next; + } + free(cur); + } + else { + /* part of this block is used, so add it to the new block list */ + wof_add_to_block_list(allocator, cur); + } + + cur = next; + } +} + +void +wof_allocator_destroy(wof_allocator_t *allocator) +{ + /* The combination of free_all and gc returns all our memory to the OS + * except for the struct itself */ + wof_free_all(allocator); + wof_gc(allocator); + + /* then just free the struct */ + free(allocator); +} + +wof_allocator_t * +wof_allocator_new(void) +{ + wof_allocator_t *allocator; + + allocator = (wof_allocator_t *)malloc(sizeof(wof_allocator_t)); + + if (allocator == NULL) { + return NULL; + } + + allocator->block_list = NULL; + allocator->master_head = NULL; + allocator->recycler_head = NULL; + + return allocator; +} + +#ifdef __cplusplus +} +#endif /* __cplusplus */ + +/* + * Editor modelines - http://www.wireshark.org/tools/modelines.html + * + * Local variables: + * c-basic-offset: 4 + * tab-width: 8 + * indent-tabs-mode: nil + * End: + * + * vi: set shiftwidth=4 tabstop=8 expandtab: + * :indentSize=4:tabSize=8:noTabs=true: + */ +#pragma GCC diagnostic pop diff --git a/ww/managers/alloc-engine/wof_allocator.h b/ww/managers/alloc-engine/wof_allocator.h new file mode 100644 index 00000000..f05ae7bd --- /dev/null +++ b/ww/managers/alloc-engine/wof_allocator.h @@ -0,0 +1,41 @@ +/* Wheel-of-Fortune Memory Allocator + * Copyright 2013, Evan Huus + */ + +#ifndef __WOF_ALLOCATOR_H__ +#define __WOF_ALLOCATOR_H__ + +#include + +#ifdef __cplusplus +extern "C" { +#endif /* __cplusplus */ + +typedef struct _wof_allocator_t wof_allocator_t; + +void * +wof_alloc(wof_allocator_t *allocator, const size_t size); + +void +wof_free(wof_allocator_t *allocator, void *ptr); + +void * +wof_realloc(wof_allocator_t *allocator, void *ptr, const size_t size); + +void +wof_free_all(wof_allocator_t *allocator); + +void +wof_gc(wof_allocator_t *allocator); + +void +wof_allocator_destroy(wof_allocator_t *allocator); + +wof_allocator_t * +wof_allocator_new(void); + +#ifdef __cplusplus +} +#endif /* __cplusplus */ + +#endif /* __WOF_ALLOCATOR_H__ */ diff --git a/ww/managers/memory_manager.c b/ww/managers/memory_manager.c index 4d7aca0c..fbde32c2 100644 --- a/ww/managers/memory_manager.c +++ b/ww/managers/memory_manager.c @@ -1,1314 +1,104 @@ -/** -Author & Copyright (C) 2017 Johannes Bernhard Steffens. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -*/ - -/* - - This started from tbman library at https://github.com/johsteffens/tbman/tree/master - - Changed many prats and names of this file, and it is now WW memory manager - -*/ - - #include "memory_manager.h" -#include "utils/btree.h" +#include "alloc-engine/wof_allocator.h" #include "hmutex.h" -#include -#include -#include #include +#include +#include +#include #include -#ifdef OS_WIN -#include -#endif -/**********************************************************************************************************************/ -// default parameters - -static const size_t default_pool_size = 0x10000; -static const size_t default_min_block_size = 8; -static const size_t default_max_block_size = 1024 * 16; -static const size_t default_stepping_method = 1; -static const bool default_full_align = true; - -/// Minimum alignment of memory blocks -#define wwmGlobalALIGN 0x100 - -/**********************************************************************************************************************/ -/// error messages - -static void ext_err( const char* func, const char* file, int line, const char* format, ... ) -{ - fprintf( stderr, "error in function %s (%s:%i):\n", func, file, line ); - va_list args; - va_start( args, format ); - vfprintf( stderr, format, args ); - va_end( args ); - fprintf( stderr, "\n" ); - abort(); -} - -#define ERR( ... ) ext_err( __func__, __FILE__, __LINE__, __VA_ARGS__ ) - - -#define ASSERT_GLOBAL_INITIALIZED() \ - if( wwmGlobalState == NULL ) ERR( "Manager was not initialized. Call createWWMemoryManager() at the beginning of your program." ) - -/**********************************************************************************************************************/ - -static inline void* stdlib_alloc( void* current_ptr, size_t requested_size ) -{ - if( requested_size == 0 ) - { - if( current_ptr ) free( current_ptr ); - current_ptr = NULL; - } - else - { - if( current_ptr ) - { - current_ptr = realloc( current_ptr, requested_size ); - } - else - { - current_ptr = malloc( requested_size ); - } - if( !current_ptr ) ERR( "Failed allocating %zu bytes", requested_size ); - } - return current_ptr; -} - -#ifdef OS_UNIX - -static inline void* posix_aligned_malloc(size_t alignment, size_t size) { - void* aligned_memory = NULL; - int result = posix_memalign(&aligned_memory, alignment, size); - if (result != 0) { - return NULL; - } - return aligned_memory; -} - -#endif - -/**********************************************************************************************************************/ -/**********************************************************************************************************************/ -/** Token-Manager - * - * Fragmentation-free and fast (O(1)) pool based dynamic management using fixed sized blocks. - * A free block is identified by a token representing its address. Tokens are managed in a stack. - * An alloc-request consumes the top token from stack. A free-request pushes the token back onto the stack. - * - * The instance token_manager_s occupies the memory-pool; being its header. This supports efficient (O(log(n)) - * determination of the correct token-manager by the memory-manager (s. algorithm below). - * - * Token managers can be run in full-alignment-mode in which they are aligned to pool_size, which is - * a power of two. This allows O(1) lookup of the pool manager from any of its managed allocations. - * - */ -typedef struct token_manager_s -{ - size_t pool_size; - size_t block_size; - uint16_t stack_size; // size of token-stack - uint16_t stack_index; // index into token-stack - - /** aligned - * The memory-pool is considered aligned when the integer-evaluation of its address - * is a multiple of pool_size, which means that the pool address can be obtained - * from any pointer inside the pool by a mere integer division. - */ - bool aligned; - - struct block_manager_s* parent; - size_t parent_index; - uint16_t token_stack[]; // stack of block-tokens (part of pool) -} token_manager_s; - -// --------------------------------------------------------------------------------------------------------------------- - -static void token_manager_s_init( token_manager_s* o ) -{ - memset( o, 0, sizeof( *o ) ); -} - -// --------------------------------------------------------------------------------------------------------------------- - -static void token_manager_s_down( token_manager_s* o ) -{ - (void) o; -} - -// --------------------------------------------------------------------------------------------------------------------- - -static token_manager_s* token_manager_s_create( size_t pool_size, size_t block_size, bool align ) -{ - if( ( pool_size & ( pool_size - 1 ) ) != 0 ) ERR( "pool_size %zu is not a power of two", pool_size ); - size_t stack_size = pool_size / block_size; - if( stack_size > 0x10000 ) ERR( "stack_size %zu exceeds 0x10000", stack_size ); - size_t reserved_size = sizeof( token_manager_s ) + sizeof( uint16_t ) * stack_size; - size_t reserved_blocks = reserved_size / block_size + ( ( reserved_size % block_size ) > 0 ); - if( stack_size < ( reserved_blocks + 1 ) ) ERR( "pool_size %zu is too small", pool_size ); - - token_manager_s* o; - if( align ) - { -#ifdef OS_WIN - o = _aligned_malloc( pool_size, pool_size ); -#elif defined (OS_ANDROID) - o = posix_aligned_malloc( pool_size, pool_size ); -#else - o = aligned_alloc( pool_size, pool_size ); -#endif - if( !o ) ERR( "Failed aligned allocating %zu bytes", pool_size ); - } - else - { -#ifdef OS_WIN - o = _aligned_malloc( wwmGlobalALIGN, pool_size ); -#elif defined (OS_ANDROID) - o = posix_aligned_malloc( wwmGlobalALIGN, pool_size ); -#else - o = aligned_alloc( wwmGlobalALIGN, pool_size ); -#endif - if( !o ) ERR( "Failed allocating %zu bytes", pool_size ); - } - - token_manager_s_init( o ); - o->aligned = ( ( intptr_t )o & ( intptr_t )( pool_size - 1 ) ) == 0; - o->pool_size = pool_size; - o->block_size = block_size; - o->stack_size = stack_size; - o->stack_index = 0; - for( size_t i = 0; i < o->stack_size; i++ ) o->token_stack[ i ] = ( i + reserved_blocks ) < stack_size ? ( i + reserved_blocks ) : 0; - return o; -} - -// --------------------------------------------------------------------------------------------------------------------- - -static void token_manager_s_discard( token_manager_s* o ) -{ - if( !o ) return; - token_manager_s_down( o ); -#ifdef OS_WIN - _aligned_free( o ); -#else - free( o ); -#endif -} - -// --------------------------------------------------------------------------------------------------------------------- - -static bool token_manager_s_is_full( token_manager_s* o ) -{ - return o->token_stack[ o->stack_index ] == 0; -} - -// --------------------------------------------------------------------------------------------------------------------- - -static bool token_manager_s_is_empty( token_manager_s* o ) -{ - return o->stack_index == 0; -} - -// --------------------------------------------------------------------------------------------------------------------- - -static void* token_manager_s_alloc( token_manager_s* o ) -{ - assert( !token_manager_s_is_full( o ) ); - void* ret = ( uint8_t* )o + o->token_stack[ o->stack_index ] * o->block_size; - assert( ( uint8_t* )ret >= ( uint8_t* )o + sizeof( token_manager_s ) ); - o->stack_index++; - return ret; -} - -// --------------------------------------------------------------------------------------------------------------------- - -// forward declarations (implementation below) -static void block_manager_s_full_to_free( struct block_manager_s* o, token_manager_s* child ); -static void block_manager_s_free_to_empty( struct block_manager_s* o, token_manager_s* child ); - -static void token_manager_s_free( token_manager_s* o, void* ptr ) -{ - #ifdef RTCHECKS - if( o->stack_index == 0 ) ERR( "Block manager is empty." ); - if( ( size_t )( ( ptrdiff_t )( ( uint8_t* )ptr - ( uint8_t* )o ) ) > o->pool_size ) ERR( "Attempt to free memory outside pool." ); - #endif - - uint16_t token = ( ( ptrdiff_t )( ( uint8_t* )ptr - ( uint8_t* )o ) ) / o->block_size; - - #ifdef RTCHECKS - if( token * o->block_size < sizeof( token_manager_s ) ) ERR( "Attempt to free reserved memory." ); - for( size_t i = o->stack_index; i < o->stack_size; i++ ) if( o->token_stack[ i ] == token ) ERR( "Attempt to free memory that is declared free." ); - #endif // RTCHECKS - - if( o->token_stack[ o->stack_index ] == 0 ) block_manager_s_full_to_free( o->parent, o ); - - o->stack_index--; - o->token_stack[ o->stack_index ] = token; - - if( o->stack_index == 0 ) block_manager_s_free_to_empty( o->parent, o ); -} - -// --------------------------------------------------------------------------------------------------------------------- - -static size_t token_manager_s_total_alloc( const token_manager_s* o ) -{ - return o->block_size * o->stack_index; -} - -// --------------------------------------------------------------------------------------------------------------------- - -static size_t token_manager_s_total_instances( const token_manager_s* o ) -{ - return o->stack_index; -} - -// --------------------------------------------------------------------------------------------------------------------- - -static size_t token_manager_s_total_space( const token_manager_s* o ) -{ - return o->pool_size + o->stack_size * sizeof( uint16_t ); -} - -// --------------------------------------------------------------------------------------------------------------------- - -static void token_manager_s_for_each_instance( token_manager_s* o, void (*cb)( void* arg, void* ptr, size_t space ), void* arg ) -{ - if( !cb ) return; - for( size_t i = 0; i < o->stack_index; i++ ) - { - size_t token = o->token_stack[ i ]; - cb( arg, ( uint8_t* )o + token * o->block_size, o->block_size ); - } -} - -// --------------------------------------------------------------------------------------------------------------------- - -static void print_token_manager_s_status( const token_manager_s* o, int detail_level ) -{ - if( detail_level <= 0 ) return; - printf( " pool_size: %zu\n", o->pool_size ); - printf( " block_size: %zu\n", o->block_size ); - printf( " stack_size: %u\n", o->stack_size ); - printf( " aligned: %s\n", o->aligned ? "true" : "false" ); - printf( " stack_index: %zu\n", ( size_t )o->stack_index ); - printf( " total alloc: %zu\n", token_manager_s_total_alloc( o ) ); - printf( " total space: %zu\n", token_manager_s_total_space( o ) ); -} - -// --------------------------------------------------------------------------------------------------------------------- - -/**********************************************************************************************************************/ -/**********************************************************************************************************************/ -/** Block-Manager - * - * Contains an array of token-managers, each of the same block-size. - * A token-manager has one of three stats: 'full', 'free' and 'empty'. - * A 'full' token-manager has no space left for allocation - * A 'free' token-manager has (some) space available for allocation. - * An 'empty' token-manager has all space available for allocation. - * Token managers are linearly arranged by state in the order: full, free, empty. - * An index (free_index) points to the full-free border. - * - * Alloc request: O(1) - * - redirected to the free_indexe(d) token-manager. - * - if that token-manager becomes 'full', free_index is incremented - * - if all token-managers are full, a new token-manager is appended at the next alloc request - * - * Free request: - * - block_manager_s does not directly receive free requests. Instead the parent-manager directly invokes the - * the corresponding token manager. - * - If a token-manager turns from full to free, it reports to the block manager, which swaps its position - * with the last full token_manager and decrements free_index. - * - If a token-manager turns from free to empty, it reports to the block manager, which swaps its position - * with the last free token_manager. When enough empty token-managers accumulated (sweep_hysteresis), they - * are discarded (memory returned to the system). - * - */ -typedef struct block_manager_s -{ - size_t pool_size; // pool size of all token-managers - size_t block_size; // block size of all token-managers - bool align; // attempt to align token_managers to pool_size - token_manager_s** data; - size_t size, space; - size_t free_index; // entries equal or above free_index have space for allocation - double sweep_hysteresis; // if ( empty token-managers ) / ( used token-managers ) < sweep_hysteresis, empty token-managers are discarded - bool aligned; // all token managers are aligned to pool_size - struct ww_dedictaed_mem_s* parent; - btree_vd_s* internal_btree; -} block_manager_s; - -// --------------------------------------------------------------------------------------------------------------------- - -static void block_manager_s_init( block_manager_s* o ) -{ - memset( o, 0, sizeof( *o ) ); - o->aligned = true; - o->sweep_hysteresis = 0.125; -} - -// --------------------------------------------------------------------------------------------------------------------- - -static void block_manager_s_down( block_manager_s* o ) -{ - if( o->data ) - { - for( size_t i = 0; i < o->size; i++ ) token_manager_s_discard( o->data[ i ] ); - free( o->data ); - o->data = NULL; - o->size = o->space = 0; - } -} - -// --------------------------------------------------------------------------------------------------------------------- - -static block_manager_s* block_manager_s_create( size_t pool_size, size_t block_size, bool align ) -{ - block_manager_s* o = malloc( sizeof( block_manager_s ) ); - if( !o ) ERR( "Failed allocating %zu bytes", sizeof( block_manager_s ) ); - block_manager_s_init( o ); - o->pool_size = pool_size; - o->block_size = block_size; - o->align = align; - return o; -} - -// --------------------------------------------------------------------------------------------------------------------- - -static void block_manager_s_discard( block_manager_s* o ) -{ - if( !o ) return; - block_manager_s_down( o ); - free( o ); -} - -// --------------------------------------------------------------------------------------------------------------------- - -static void wwmDedicatedlost_alignment( struct ww_dedictaed_mem_s* o, const block_manager_s* child ); - -static void* block_manager_s_alloc( block_manager_s* o ) -{ - if( o->free_index == o->size ) - { - if( o->size == o->space ) - { - o->space = ( o->space > 0 ) ? o->space * 2 : 1; - - if( o->data ) - { - o->data = realloc( o->data, sizeof( token_manager_s* ) * o->space ); - } - else - { - o->data = malloc( sizeof( token_manager_s* ) * o->space ); - } - - if( !o->data ) ERR( "Failed allocating %zu bytes", sizeof( token_manager_s* ) * o->space ); - } - o->data[ o->size ] = token_manager_s_create( o->pool_size, o->block_size, o->align ); - o->data[ o->size ]->parent_index = o->size; - o->data[ o->size ]->parent = o; - if( o->aligned && !o->data[ o->size ]->aligned ) - { - o->aligned = false; - wwmDedicatedlost_alignment( o->parent, o ); - } - if( btree_vd_s_set( o->internal_btree, o->data[ o->size ] ) != 1 ) ERR( "Failed registering block address." ); - o->size++; - } - token_manager_s* child = o->data[ o->free_index ]; - void* ret = token_manager_s_alloc( child ); - if( token_manager_s_is_full( child ) ) o->free_index++; - return ret; -} - -// --------------------------------------------------------------------------------------------------------------------- - -// A child reports turning full --> free -static void block_manager_s_full_to_free( block_manager_s* o, token_manager_s* child ) -{ - assert( o->free_index > 0 ); - - o->free_index--; - - // swap child with current free position - size_t child_index = child->parent_index; - size_t swapc_index = o->free_index; - - token_manager_s* swapc = o->data[ swapc_index ]; - o->data[ swapc_index ] = child; - o->data[ child_index ] = swapc; - child->parent_index = swapc_index; - swapc->parent_index = child_index; -} - -// --------------------------------------------------------------------------------------------------------------------- - -static size_t block_manager_s_empty_tail( const block_manager_s* o ) -{ - if( o->size == 0 ) return 0; - size_t empty_index = o->size; - while( empty_index > 0 && token_manager_s_is_empty( o->data[ empty_index - 1 ] ) ) empty_index--; - return o->size - empty_index; -} - -// --------------------------------------------------------------------------------------------------------------------- - -// A child reports turning free --> empty -static void block_manager_s_free_to_empty( block_manager_s* o, token_manager_s* child ) -{ - // move empty manager to tail (if not already there) - size_t child_index = child->parent_index; - size_t empty_tail = block_manager_s_empty_tail( o ); - if( empty_tail < o->size ) - { - size_t swapc_index = o->size - empty_tail - 1; - if( child_index < swapc_index ) - { - token_manager_s* swapc = o->data[ swapc_index ]; - o->data[ child_index ] = swapc; - o->data[ swapc_index ] = child; - child->parent_index = swapc_index; - swapc->parent_index = child_index; - empty_tail++; - } - } - - if( empty_tail > ( o->size - empty_tail ) * o->sweep_hysteresis ) // discard empty managers when enough accumulated - { - while( o->size > 0 && token_manager_s_is_empty( o->data[ o->size - 1 ] ) ) - { - o->size--; - - if( btree_vd_s_remove( o->internal_btree, o->data[ o->size ] ) != 1 ) ERR( "Failed removing block address." ); - - #ifdef RTCHECKS - if( btree_vd_s_exists( o->internal_btree, o->data[ o->size ] ) ) ERR( "Removed block address still exists" ); - #endif - - token_manager_s_discard( o->data[ o->size ] ); - o->data[ o->size ] = NULL; - } - } -} - -// --------------------------------------------------------------------------------------------------------------------- - -static size_t block_manager_s_total_alloc( const block_manager_s* o ) -{ - size_t sum = 0; - for( size_t i = 0; i < o->size; i++ ) - { - sum += token_manager_s_total_alloc( o->data[ i ] ); - } - return sum; -} - -// --------------------------------------------------------------------------------------------------------------------- - -static size_t block_manager_s_total_instances( const block_manager_s* o ) -{ - size_t sum = 0; - for( size_t i = 0; i < o->size; i++ ) - { - sum += token_manager_s_total_instances( o->data[ i ] ); - } - return sum; -} - -// --------------------------------------------------------------------------------------------------------------------- - -static size_t block_manager_s_total_space( const block_manager_s* o ) -{ - size_t sum = 0; - for( size_t i = 0; i < o->size; i++ ) - { - sum += token_manager_s_total_space( o->data[ i ] ); - } - return sum; -} - -// --------------------------------------------------------------------------------------------------------------------- - -static void block_manager_s_for_each_instance( block_manager_s* o, void (*cb)( void* arg, void* ptr, size_t space ), void* arg ) -{ - for( size_t i = 0; i < o->size; i++ ) token_manager_s_for_each_instance( o->data[ i ], cb, arg ); -} - -// --------------------------------------------------------------------------------------------------------------------- -static void print_block_manager_s_status( const block_manager_s* o, int detail_level ) +struct dedicated_memory_s { - if( detail_level <= 0 ) return; - printf( " pool_size: %zu\n", o->pool_size ); - printf( " block_size: %zu\n", o->block_size ); - printf( " sweep_hysteresis: %g\n", o->sweep_hysteresis ); - printf( " aligned: %s\n", o->aligned ? "true" : "false" ); - printf( " token_managers: %zu\n", o->size ); - printf( " full: %zu\n", o->free_index ); - printf( " empty: %zu\n", block_manager_s_empty_tail( o ) ); - printf( " total alloc: %zu\n", block_manager_s_total_alloc( o ) ); - printf( " total space: %zu\n", block_manager_s_total_space( o ) ); - if( detail_level > 1 ) - { - for( size_t i = 0; i < o->size; i++ ) - { - printf( "\nblock manager %zu:\n", i ); - print_token_manager_s_status( o->data[ i ], detail_level - 1 ); - } - } -} - -// --------------------------------------------------------------------------------------------------------------------- - -/**********************************************************************************************************************/ -/**********************************************************************************************************************/ -/** Memory-Manager - * - * Contains a fixed-size array of block-managers with exponentially increasing block_size. - * (E.g. via size-doubling, but other arrangements are also possible) - * - * Alloc request: - * - directed to the block-manager with the smallest fitting bock-size - * - if the largest block size is yet too small, the request is passed on to the OS (-->aligned_alloc) - * --> O(1) for size requests equal or below largest block size assuming alloc and free requests are statistically - * balanced such the overall memory in use is not dramatically varying. - * - * Free request: - * - If the previously allocated size is available and all token managers are aligned - * the address of the token manager is directly calculated from the allocated address. (O(1)) - * - Otherwise: The corresponding token-manager is determined via internal_btree from the memory-address - * (O(log(n)) - where 'n' is the current amount of token managers.) - * - */ -typedef struct ww_dedictaed_mem_s -{ - block_manager_s** data; // block managers are sorted by increasing block size - size_t size; - size_t pool_size; // pool size for all token managers - size_t min_block_size; - size_t max_block_size; - bool aligned; // all token managers are aligned - size_t* block_size_array; // copy of block size values (for fast access) - btree_vd_s* internal_btree; - btree_ps_s* external_btree; - hhybridmutex_t mutex; -} ww_dedictaed_mem_t; - -// --------------------------------------------------------------------------------------------------------------------- - -static void wwmDedicatedinit( ww_dedictaed_mem_t* o, size_t pool_size, size_t min_block_size, size_t max_block_size, size_t stepping_method, bool full_align ) -{ - memset( o, 0, sizeof( *o ) ); - hhybridmutex_init(&o->mutex); - - o->internal_btree = btree_vd_s_create( stdlib_alloc ); - o->external_btree = btree_ps_s_create( stdlib_alloc ); - - /// The following three values are configurable parameters of memory manager - o->pool_size = pool_size; - o->min_block_size = min_block_size; - o->max_block_size = max_block_size; - - size_t mask_bxp = stepping_method; - size_t size_mask = ( 1 << mask_bxp ) - 1; - size_t size_inc = o->min_block_size; - while( ( size_mask < o->min_block_size ) || ( ( size_mask << 1 ) & o->min_block_size ) != 0 ) size_mask <<= 1; - - size_t space = 0; - - for( size_t block_size = o->min_block_size; block_size <= o->max_block_size; block_size += size_inc ) - { - if( o->size == space ) - { - space = space > 0 ? space * 2 : 16; - - if( o->data ) - { - o->data = realloc( o->data, sizeof( block_manager_s* ) * space ); - } - else - { - o->data = malloc( sizeof( block_manager_s* ) * space ); - } - - if( !o->data ) ERR( "Failed allocating %zu bytes", sizeof( block_manager_s* ) * space ); - } - o->data[ o->size ] = block_manager_s_create( o->pool_size, block_size, full_align ); - o->data[ o->size ]->internal_btree = o->internal_btree; - o->data[ o->size ]->parent = o; - o->size++; - - if( block_size > size_mask ) - { - size_mask <<= 1; - size_inc <<= 1; - } - } - - o->block_size_array = malloc( o->size * sizeof( size_t ) ); - if( !o->block_size_array ) ERR( "Failed allocating %zu bytes", o->size * sizeof( size_t ) ); - - o->aligned = true; - for( size_t i = 0; i < o->size; i++ ) - { - o->aligned = o->aligned && o->data[ i ]->aligned; - o->block_size_array[ i ] = o->data[ i ]->block_size; - } -} + hhybridmutex_t mut; + wof_allocator_t *wof_state; +}; -// --------------------------------------------------------------------------------------------------------------------- +dedicated_memory_t *state; -static void wwmDedicateddown( ww_dedictaed_mem_t* o ) -{ - size_t leaking_bytes = wwmDedicatedtotalGrantedSpace( o ); - - if( leaking_bytes > 0 ) - { - size_t leaking_instances = wwmDedicatedtotalInstances( o ); - fprintf - ( - stderr, - "TBMAN WARNING: Detected %zu instances with a total of %zu bytes leaking space.\n", - leaking_instances, - leaking_bytes - ); - } - - hhybridmutex_lock( &o->mutex ); - if( o->data ) - { - for( size_t i = 0; i < o->size; i++ ) block_manager_s_discard( o->data[ i ] ); - free( o->data ); - } - - btree_vd_s_discard( o->internal_btree ); - btree_ps_s_discard( o->external_btree ); - - if( o->block_size_array ) free( o->block_size_array ); - - hhybridmutex_unlock( &o->mutex ); - hhybridmutex_destroy( &o->mutex ); -} - -// --------------------------------------------------------------------------------------------------------------------- - -ww_dedictaed_mem_t* wwmDedicatedCreate - ( - size_t pool_size, - size_t min_block_size, - size_t max_block_size, - size_t stepping_method, - bool full_align - ) -{ - ww_dedictaed_mem_t* o = malloc( sizeof( ww_dedictaed_mem_t ) ); - if( !o ) ERR( "Failed allocating %zu bytes", sizeof( ww_dedictaed_mem_t ) ); - wwmDedicatedinit( o, pool_size, min_block_size, max_block_size, stepping_method, full_align ); - return o; -} - -// --------------------------------------------------------------------------------------------------------------------- - -ww_dedictaed_mem_t* wwmDedicatedCreateDefault( void ) -{ #ifdef ALLOCATOR_BYPASS - return NULL; -#endif - return wwmDedicatedCreate - ( - default_pool_size, - default_min_block_size, - default_max_block_size, - default_stepping_method, - default_full_align - ); -} - -// --------------------------------------------------------------------------------------------------------------------- -void wwmDedicatedDiscard( ww_dedictaed_mem_t* o ) +dedicated_memory_t *createWWMemoryManager() { - if( !o ) return; - wwmDedicateddown( o ); - free( o ); + return NULL; } -// --------------------------------------------------------------------------------------------------------------------- - -static void wwmDedicatedlost_alignment( struct ww_dedictaed_mem_s* o, const block_manager_s* child ) +dedicated_memory_t *getWWMemoryManager(void) { - (void) child; - o->aligned = false; + return NULL; } -// --------------------------------------------------------------------------------------------------------------------- - -static void* wwmDedicatedmem_alloc( ww_dedictaed_mem_t* o, size_t requested_size, size_t* granted_size ) +void setWWMemoryManager(dedicated_memory_t *new_state) { - block_manager_s* block_manager = NULL; - for( size_t i = 0; i < o->size; i++ ) - { - if( requested_size <= o->block_size_array[ i ] ) - { - block_manager = o->data[ i ]; - break; - } - } - - void* reserved_ptr = NULL; - if( block_manager ) - { - reserved_ptr = block_manager_s_alloc( block_manager ); - if( granted_size ) *granted_size = block_manager->block_size; - } - else - { -#ifdef OS_WIN - reserved_ptr = _aligned_malloc( wwmGlobalALIGN, requested_size ); -#elif defined (OS_ANDROID) - reserved_ptr = posix_aligned_malloc( wwmGlobalALIGN, requested_size ); -#else - reserved_ptr = aligned_alloc( wwmGlobalALIGN, requested_size ); -#endif - if( !reserved_ptr ) ERR( "Failed allocating %zu bytes.", requested_size ); - if( granted_size ) *granted_size = requested_size; - if( btree_ps_s_set( o->external_btree, reserved_ptr, requested_size ) != 1 ) ERR( "Registering new address failed" ); - } - - return reserved_ptr; + (void) new_state; } -// --------------------------------------------------------------------------------------------------------------------- - -static void wwmDedicatedmem_free( ww_dedictaed_mem_t* o, void* current_ptr, const size_t* current_size ) +dedicated_memory_t *createWWDedicatedMemory() { - if( current_size && *current_size <= o->max_block_size && o->aligned ) - { - token_manager_s* token_manager = ( token_manager_s* )( ( intptr_t )current_ptr & ~( intptr_t )( o->pool_size - 1 ) ); - token_manager_s_free( token_manager, current_ptr ); - } - else - { - void* block_ptr = btree_vd_s_largest_below_equal( o->internal_btree, current_ptr ); - if( block_ptr && ( ( (uint8_t*)current_ptr - (uint8_t*)block_ptr ) < (long) o->pool_size ) ) - { - token_manager_s_free( block_ptr, current_ptr ); - } - else - { - if( btree_ps_s_remove( o->external_btree, current_ptr ) != 1 ) ERR( "Attempt to free invalid memory" ); - #ifdef OS_WIN - _aligned_free( current_ptr ); - #else - free( current_ptr ); - #endif - - } - } + return NULL; } -// --------------------------------------------------------------------------------------------------------------------- - -static void* wwmDedicatedmem_realloc( ww_dedictaed_mem_t* o, void* current_ptr, const size_t* current_size, size_t requested_size, size_t* granted_size ) -{ - token_manager_s* token_manager = NULL; - if( current_size && *current_size <= o->max_block_size && o->aligned ) - { - token_manager = ( token_manager_s* )( ( intptr_t )current_ptr & ~( intptr_t )( o->pool_size - 1 ) ); - } - else - { - void* block_ptr = btree_vd_s_largest_below_equal( o->internal_btree, current_ptr ); - if( block_ptr && ( ( (uint8_t*)current_ptr - (uint8_t*)block_ptr ) < (long) o->pool_size ) ) token_manager = block_ptr; - } - - if( token_manager ) - { - if( requested_size > token_manager->block_size ) - { - void* reserved_ptr = wwmDedicatedmem_alloc( o, requested_size, granted_size ); - memcpy( reserved_ptr, current_ptr, token_manager->block_size ); - token_manager_s_free( token_manager, current_ptr ); - return reserved_ptr; - } - else // size reduction - { - block_manager_s* block_manager = NULL; - for( size_t i = 0; i < o->size; i++ ) - { - if( requested_size <= o->block_size_array[ i ] ) - { - block_manager = o->data[ i ]; - break; - } - } - - if( block_manager->block_size != token_manager->block_size ) - { - void* reserved_ptr = block_manager_s_alloc( block_manager ); - memcpy( reserved_ptr, current_ptr, requested_size ); - token_manager_s_free( token_manager, current_ptr ); - if( granted_size ) *granted_size = block_manager->block_size; - return reserved_ptr; - } - else - { - // same block-size: keep current location - if( granted_size ) *granted_size = token_manager->block_size; - return current_ptr; - } - } - } - else - { - if( requested_size <= o->max_block_size ) // new size fits into manager, old size was outside manager - { - void* reserved_ptr = wwmDedicatedmem_alloc( o, requested_size, granted_size ); - memcpy( reserved_ptr, current_ptr, requested_size ); - if( btree_ps_s_remove( o->external_btree, current_ptr ) != 1 ) ERR( "Attempt to free invalid memory" ); - #ifdef OS_WIN - _aligned_free( current_ptr ); - #else - free( current_ptr ); - #endif - return reserved_ptr; - } - else // neither old nor new size handled by this manager - { - size_t* p_current_size = btree_ps_s_val( o->external_btree, current_ptr ); - if( !p_current_size ) ERR( "Could not retrieve current external memory" ); - size_t current_ext_bytes = *p_current_size; - - // is requested bytes is less but not significantly less than current bytes, keep current memory - if( ( requested_size < current_ext_bytes ) && ( requested_size >= ( current_ext_bytes >> 1 ) ) ) - { - if( granted_size ) *granted_size = current_ext_bytes; - return current_ptr; - } -#ifdef OS_WIN - void* reserved_ptr = _aligned_malloc( wwmGlobalALIGN, requested_size ); -#elif defined (OS_ANDROID) - void* reserved_ptr = posix_aligned_malloc( wwmGlobalALIGN, requested_size ); #else - void* reserved_ptr = aligned_alloc( wwmGlobalALIGN, requested_size ); -#endif - if( !reserved_ptr ) ERR( "Failed allocating %zu bytes.", requested_size ); - if( granted_size ) *granted_size = requested_size; - if( btree_ps_s_set( o->external_btree, reserved_ptr, requested_size ) != 1 ) ERR( "Registering new address failed" ); - - size_t copy_bytes = ( requested_size < current_ext_bytes ) ? requested_size : current_ext_bytes; - memcpy( reserved_ptr, current_ptr, copy_bytes ); - - if( btree_ps_s_remove( o->external_btree, current_ptr ) != 1 ) ERR( "Attempt to free invalid memory" ); - #ifdef OS_WIN - _aligned_free( current_ptr ); - #else - free( current_ptr ); - #endif - return reserved_ptr; - } - } -} - -// --------------------------------------------------------------------------------------------------------------------- -void* wwmDedicatedAlloc( ww_dedictaed_mem_t* o, void* current_ptr, size_t requested_size, size_t* granted_size ) +dedicated_memory_t *createWWMemoryManager(void) { - hhybridmutex_lock( &o->mutex ); - void* ret = NULL; - if( requested_size == 0 ) - { - if( current_ptr ) - { - wwmDedicatedmem_free( o, current_ptr, NULL ); - } - if( granted_size ) *granted_size = 0; - } - else - { - if( current_ptr ) - { - ret = wwmDedicatedmem_realloc( o, current_ptr, NULL, requested_size, granted_size ); - } - else - { - ret = wwmDedicatedmem_alloc( o, requested_size, granted_size ); - } - } - hhybridmutex_unlock( &o->mutex ); - return ret; + assert(state == NULL); + state = createWWDedicatedMemory(); + return state; } -// --------------------------------------------------------------------------------------------------------------------- - -void* wwmDedicatedNalloc( ww_dedictaed_mem_t* o, void* current_ptr, size_t current_size, size_t requested_size, size_t* granted_size ) +dedicated_memory_t *getWWMemoryManager(void) { - hhybridmutex_lock( &o->mutex ); - void* ret = NULL; - if( requested_size == 0 ) - { - if( current_size ) // 0 means current_ptr may not be used for free or realloc - { - wwmDedicatedmem_free( o, current_ptr, ¤t_size ); - } - if( granted_size ) *granted_size = 0; - } - else - { - if( current_size ) // 0 means current_ptr may not be used for free or realloc - { - ret = wwmDedicatedmem_realloc( o, current_ptr, ¤t_size, requested_size, granted_size ); - } - else - { - ret = wwmDedicatedmem_alloc( o, requested_size, granted_size ); - } - } - hhybridmutex_unlock( &o->mutex ); - return ret; + return state; } -// --------------------------------------------------------------------------------------------------------------------- - -static size_t wwmDedicatedexternal_total_alloc( const ww_dedictaed_mem_t* o ) +void setWWMemoryManager(dedicated_memory_t *new_state) { - return btree_ps_s_sum( o->external_btree, NULL, NULL ); + assert(state == NULL); + state = new_state; } -// --------------------------------------------------------------------------------------------------------------------- - -static void ext_count( void* arg, btree_ps_key_t key, btree_ps_val_t val ) { - (void) key; - (void) val; - *(size_t*)arg += 1; - -} - -static size_t wwmDedicatedexternal_total_instances( const ww_dedictaed_mem_t* o ) -{ - size_t size = 0; - btree_ps_s_run( o->external_btree, ext_count, &size ); - return size; -} - -// --------------------------------------------------------------------------------------------------------------------- - -typedef struct ext_for_instance_arg +dedicated_memory_t *createWWDedicatedMemory(void) { - void (*cb)( void* arg, void* ptr, size_t space ); - void* arg; -} ext_for_instance_arg; + dedicated_memory_t *dm = malloc(sizeof(dedicated_memory_t)); + hhybridmutex_init(&dm->mut); + dm->wof_state = wof_allocator_new(); -static void ext_for_instance( void* arg, btree_ps_key_t key, btree_ps_val_t val ) -{ - ext_for_instance_arg* iarg = arg; - iarg->cb( iarg->arg, key, val ); + return dm; } -static void wwmDedicatedexternal_for_each_instance( ww_dedictaed_mem_t* o, void (*cb)( void* arg, void* ptr, size_t space ), void* arg ) +void *wwmGlobalMalloc(size_t size) { - ext_for_instance_arg iarg = { .cb = cb, .arg = arg }; - btree_ps_s_run( o->external_btree, ext_for_instance, &iarg ); + return wwmDedicatedMalloc(state, size); } - -// --------------------------------------------------------------------------------------------------------------------- - -static size_t wwmDedicatedinternal_total_alloc( const ww_dedictaed_mem_t* o ) +void *wwmGlobalRealloc(void *ptr, size_t size) { - size_t sum = 0; - for( size_t i = 0; i < o->size; i++ ) - { - sum += block_manager_s_total_alloc( o->data[ i ] ); - } - return sum; + return wwmDedicatedRealloc(state, ptr, size); } - -// --------------------------------------------------------------------------------------------------------------------- - -static size_t wwmDedicatedinternal_total_instances( const ww_dedictaed_mem_t* o ) +void wwmGlobalFree(void *ptr) { - size_t sum = 0; - for( size_t i = 0; i < o->size; i++ ) - { - sum += block_manager_s_total_instances( o->data[ i ] ); - } - return sum; + wwmDedicatedFree(state, ptr); } -// --------------------------------------------------------------------------------------------------------------------- - -static void wwmDedicatedinternal_for_each_instance( ww_dedictaed_mem_t* o, void (*cb)( void* arg, void* ptr, size_t space ), void* arg ) +void *wwmDedicatedMalloc(dedicated_memory_t *dm, size_t size) { - for( size_t i = 0; i < o->size; i++ ) - { - block_manager_s_for_each_instance( o->data[ i ], cb, arg ); - } + hhybridmutex_lock(&dm->mut); + void *ptr = wof_alloc(dm->wof_state, size); + hhybridmutex_unlock(&dm->mut); + return ptr; } - -// --------------------------------------------------------------------------------------------------------------------- - -static size_t wwmDedicatedtotal_alloc( const ww_dedictaed_mem_t* o ) +void *wwmDedicatedRealloc(dedicated_memory_t *dm, void *ptr, size_t size) { - return wwmDedicatedexternal_total_alloc( o ) - + wwmDedicatedinternal_total_alloc( o ); + hhybridmutex_lock(&dm->mut); + void *newptr = wof_realloc(dm->wof_state, ptr, size); + hhybridmutex_unlock(&dm->mut); + return newptr; } - -// --------------------------------------------------------------------------------------------------------------------- - -static size_t wwmDedicatedtotal_space( const ww_dedictaed_mem_t* o ) +void wwmDedicatedFree(dedicated_memory_t *dm, void *ptr) { - size_t sum = 0; - for( size_t i = 0; i < o->size; i++ ) - { - sum += block_manager_s_total_space( o->data[ i ] ); - } - return sum; + hhybridmutex_lock(&dm->mut); + wof_free(dm->wof_state, ptr); + hhybridmutex_unlock(&dm->mut); } -// --------------------------------------------------------------------------------------------------------------------- - -/**********************************************************************************************************************/ -// Interface - -static ww_dedictaed_mem_t* wwmGlobalState = NULL; - -// --------------------------------------------------------------------------------------------------------------------- - -static void createWWGlobalMemory(void) -{ - wwmGlobalState = wwmDedicatedCreate - ( - default_pool_size, - default_min_block_size, - default_max_block_size, - default_stepping_method, - default_full_align - ); -} - -// --------------------------------------------------------------------------------------------------------------------- - -static void discard_tbman(void) -{ - wwmDedicatedDiscard( wwmGlobalState ); - wwmGlobalState = NULL; -} - -// --------------------------------------------------------------------------------------------------------------------- - -ww_dedictaed_mem_t* createWWMemoryManager( void ) -{ -#ifdef ALLOCATOR_BYPASS - return NULL; #endif - assert(wwmGlobalState == NULL); - static honce_t flag = HONCE_INIT; - honce( &flag, createWWGlobalMemory ); - return wwmGlobalState; -} - -ww_dedictaed_mem_t *getWWMemoryManager(void) -{ - return wwmGlobalState; -} - -void setWWMemoryManager(ww_dedictaed_mem_t *new_state) -{ - assert(wwmGlobalState == NULL); - wwmGlobalState = new_state; -} - -// --------------------------------------------------------------------------------------------------------------------- - -void wwmGlobalClose( void ) -{ - discard_tbman(); -} - -// --------------------------------------------------------------------------------------------------------------------- - -void* wwmGlobalAlloc( void* current_ptr, size_t requested_size, size_t* granted_size ) -{ - ASSERT_GLOBAL_INITIALIZED(); - return wwmDedicatedAlloc( wwmGlobalState, current_ptr, requested_size, granted_size ); -} - -// --------------------------------------------------------------------------------------------------------------------- - -void* wwmGlobalNalloc( void* current_ptr, size_t current_size, size_t requested_size, size_t* granted_size ) -{ - ASSERT_GLOBAL_INITIALIZED(); - return wwmDedicatedNalloc( wwmGlobalState, current_ptr, current_size, requested_size, granted_size ); -} - -// --------------------------------------------------------------------------------------------------------------------- - -static size_t wwmGlobalStaterantedSpace( ww_dedictaed_mem_t* o, const void* current_ptr ) -{ - token_manager_s* token_manager = NULL; - { - void* block_ptr = btree_vd_s_largest_below_equal( o->internal_btree, ( void* )current_ptr ); - if( block_ptr && ( ( ( uint8_t* )current_ptr - ( uint8_t* )block_ptr ) < (long) o->pool_size ) ) token_manager = block_ptr; - } - - if( token_manager ) - { - return token_manager->block_size; - } - else - { - size_t* p_current_size = btree_ps_s_val( o->external_btree, ( void* )current_ptr ); - if( !p_current_size ) return 0; - return *p_current_size; - } -} - -// --------------------------------------------------------------------------------------------------------------------- - -size_t wwmGlobalGrantedSpace( const void* current_ptr ) -{ - ASSERT_GLOBAL_INITIALIZED(); - return wwmGlobalStaterantedSpace( wwmGlobalState, current_ptr ); -} - -// --------------------------------------------------------------------------------------------------------------------- - -size_t wwmDedicatedtotalGrantedSpace( ww_dedictaed_mem_t* o ) -{ - hhybridmutex_lock( &o->mutex ); - size_t space = wwmDedicatedtotal_alloc( o ); - hhybridmutex_unlock( &o->mutex ); - return space; -} - -// --------------------------------------------------------------------------------------------------------------------- - -size_t wwmGlobaltotalGrantedSpace( void ) -{ - ASSERT_GLOBAL_INITIALIZED(); - return wwmDedicatedtotalGrantedSpace( wwmGlobalState ); -} - -// --------------------------------------------------------------------------------------------------------------------- - -size_t wwmGlobaltotalInstances( void ) -{ - ASSERT_GLOBAL_INITIALIZED(); - return wwmDedicatedtotalInstances( wwmGlobalState ); -} - -// --------------------------------------------------------------------------------------------------------------------- - -size_t wwmDedicatedtotalInstances( ww_dedictaed_mem_t* o ) -{ - hhybridmutex_lock( &o->mutex ); - size_t count = 0; - count += wwmDedicatedexternal_total_instances( o ); - count += wwmDedicatedinternal_total_instances( o ); - hhybridmutex_unlock( &o->mutex ); - return count; -} - -// --------------------------------------------------------------------------------------------------------------------- - -typedef struct wwmGlobalmnode { void* p; size_t s; } wwmGlobalmnode; -typedef struct wwmGlobalmnode_arr { wwmGlobalmnode* data; size_t size; size_t space; } wwmGlobalmnode_arr; - -static void for_each_instance_collect_callback( void* arg, void* ptr, size_t space ) -{ - assert( arg ); - wwmGlobalmnode_arr* arr = arg; - assert( arr->size < arr->space ); - arr->data[ arr->size ] = ( wwmGlobalmnode ){ .p = ptr, .s = space }; - arr->size++; -} - -void wwmDedicatedForEachInstance( ww_dedictaed_mem_t* o, void (*cb)( void* arg, void* ptr, size_t space ), void* arg ) -{ - if( !cb ) return; - size_t size = wwmDedicatedtotalInstances( o ); - if( !size ) return; - - wwmGlobalmnode_arr arr; - arr.data = malloc( sizeof( wwmGlobalmnode ) * size ); - arr.space = size; - arr.size = 0; - - hhybridmutex_lock( &o->mutex ); - wwmDedicatedexternal_for_each_instance( o, for_each_instance_collect_callback, &arr ); - wwmDedicatedinternal_for_each_instance( o, for_each_instance_collect_callback, &arr ); - hhybridmutex_unlock( &o->mutex ); - - assert( arr.size == arr.space ); - - for( size_t i = 0; i < size; i++ ) cb( arg, arr.data[ i ].p, arr.data[ i ].s ); - - free( arr.data ); -} - -// --------------------------------------------------------------------------------------------------------------------- - -void wwmGlobalForEachInstance( void (*cb)( void* arg, void* ptr, size_t space ), void* arg ) -{ - ASSERT_GLOBAL_INITIALIZED(); - wwmDedicatedForEachInstance( wwmGlobalState, cb, arg ); -} - -// --------------------------------------------------------------------------------------------------------------------- - -// not thread-safe -void printWWMDedicatedstatus( ww_dedictaed_mem_t* o, int detail_level ) -{ - if( detail_level <= 0 ) return; - printf( "pool_size: %zu\n", o->pool_size ); - printf( "block managers: %zu\n", o->size ); - printf( "token managers: %zu\n", btree_vd_s_count( o->internal_btree, NULL, NULL ) ); - printf( "external allocs: %zu\n", btree_ps_s_count( o->external_btree, NULL, NULL ) ); - printf( "internal_btree depth: %zu\n", btree_vd_s_depth( o->internal_btree ) ); - printf( "external_btree depth: %zu\n", btree_ps_s_depth( o->external_btree ) ); - printf( "min_block_size: %zu\n", o->size > 0 ? o->data[ 0 ]->block_size : 0 ); - printf( "max_block_size: %zu\n", o->size > 0 ? o->data[ o->size - 1 ]->block_size : 0 ); - printf( "aligned: %s\n", o->aligned ? "true" : "false" ); - printf( "total external granted: %zu\n", wwmDedicatedexternal_total_alloc( o ) ); - printf( "total internal granted: %zu\n", wwmDedicatedinternal_total_alloc( o ) ); - printf( "total internal used: %zu\n", wwmDedicatedtotal_space( o ) ); - if( detail_level > 1 ) - { - for( size_t i = 0; i < o->size; i++ ) - { - printf( "\nblock manager %zu:\n", i ); - print_block_manager_s_status( o->data[ i ], detail_level - 1 ); - } - } -} - -// --------------------------------------------------------------------------------------------------------------------- - -void printWWMGlobalstatus( int detail_level ) -{ - printWWMDedicatedstatus( wwmGlobalState, detail_level ); -} - -/**********************************************************************************************************************/ diff --git a/ww/managers/memory_manager.h b/ww/managers/memory_manager.h index 84fa8be6..b7591701 100644 --- a/ww/managers/memory_manager.h +++ b/ww/managers/memory_manager.h @@ -1,221 +1,68 @@ -/** - -Author & Copyright (C) 2017 Johannes Bernhard Steffens. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - -------------------------------------------------------------------------------- -*/ - -/* - - This started from tbman library at https://github.com/johsteffens/tbman/tree/master - - Changed many prats and names of this file, and it is now WW memory manager - - Memory Manager Evaluation: (tested on my machine) - - malloc, free, realloc (stdlib) ... - speed test alloc-free (general): 457ns per call - speed test realloc (general) : 1482ns per call - speed test alloc-free (local) : 44ns per call - - tbman_malloc, tbman_free, tbman_realloc ... - speed test alloc-free (general): 143ns per call - speed test realloc (general) : 540ns per call - speed test alloc-free (local) : 40ns per call - - tbman_malloc, tbman_nfree, tbman_nrealloc ... - speed test alloc-free (general): 126ns per call - speed test realloc (general) : 418ns per call - speed test alloc-free (local) : 32ns per call - - - -*/ - #pragma once -// memory manager is not well tested on windows yet... - // #define ALLOCATOR_BYPASS // switch to stdlib allocators -#include #include +#include +#include -struct ww_dedictaed_mem_s; - -typedef struct ww_dedictaed_mem_s ww_dedictaed_mem_t; - -/// Creates a dedicated manager with default parameters -ww_dedictaed_mem_t* wwmDedicatedCreateDefault( void ); +struct dedicated_memory_s; +typedef struct dedicated_memory_s dedicated_memory_t; -/// Creates a dedicated manager with specified parameters (consider using wwmDedicatedCreateDefault) -ww_dedictaed_mem_t* wwmDedicatedCreate - ( - size_t pool_size, // size of a memory pool in a token manager - size_t min_block_size, // minimal block size - size_t max_block_size, // maximal block size - size_t stepping_method, // 1: uses power-2 block size stepping; > 1 uses more fine grained stepping - bool full_align // true: uses full memory alignment (fastest) - ); -/// Discards a dedicated manager -void wwmDedicatedDiscard( ww_dedictaed_mem_t* o ); /// opens global memory manager (call this once before first usage of global ww_mem functions below) -ww_dedictaed_mem_t* createWWMemoryManager( void ); +dedicated_memory_t *createWWMemoryManager(void); /// get the memory manager global state -ww_dedictaed_mem_t *getWWMemoryManager(void); +dedicated_memory_t *getWWMemoryManager(void); /// set the memory manager global state -void setWWMemoryManager(ww_dedictaed_mem_t *new_state); - -/// closes global memory manager (call this once at the end of your program) -void wwmGlobalClose( void ); - -/// creates a dedicated memory manager instance ( close with wwmDedicatedClose ) -static inline ww_dedictaed_mem_t* wwmDedicatedOpen( void ) { return wwmDedicatedCreateDefault(); } - -/// closes dedicated memory manager instance -static inline void wwmDedicatedClose( ww_dedictaed_mem_t* o ) { wwmDedicatedDiscard( o ); } - -/**********************************************************************************************************************/ -/** Advanced memory management using the internal manager (thread-safe). - * This function provides allocation, re-allocation and freeing of memory - * with advanced controls to improve memory efficiency. - * In this context, a free request is represented as re-allocation with requested_size == 0 - * - * Arguments - * current_ptr: - * Pointer to current memory location: - * ==NULL for pure-allocation - * !=NULL for re-allocation or freeing - * - * current_size: - * Optional information to memory manager about last reserved or requested amount. - * Allowed values: 0 or previously requested or reserved amount. - * 0 makes the function ignore current_ptr (assumes it is NULL) - * - * requested_size: - * > 0 for pure-allocation or re-allocation - * == 0 for freeing - * - * granted_size - * Memory granted to requester. - * The memory manager grants at least the requested amount of bytes. But it may grant more memory. - * The requester may use granted memory without re-allocation. (E.g. for dynamic arrays.) - * Retrieving the granted amount is optional. Use NULL when not desired. - * - * Return - * Allocated memory address. NULL in case all memory was freed. - * - * Alignment: (default behavior) - * A request of size of n*m bytes, where n,m are positive integers and m is (largest possible) - * integer power of 2, returns an address aligned to the lesser of m and wwmGlobalALIGN - * (wwmGlobalALIGN is defined in memory_manager.c). - * This provides correct alignment of standard data types but also for composite types - * (e.g. int32x4_t) for use with a SIMD extension of the CPU (e.g. Intel's SSE or ARM's Neon). - * - */ -void* wwmGlobalAlloc( void* current_ptr, size_t requested_size, size_t* granted_size ); -void* wwmGlobalNalloc( void* current_ptr, size_t current_size, size_t requested_size, size_t* granted_size ); -void* wwmDedicatedAlloc( ww_dedictaed_mem_t* o, void* current_ptr, size_t requested_size, size_t* granted_size ); -void* wwmDedicatedNalloc( ww_dedictaed_mem_t* o, void* current_ptr, size_t current_size, size_t requested_size, size_t* granted_size ); +void setWWMemoryManager(dedicated_memory_t *new_state); + +dedicated_memory_t *createWWDedicatedMemory(void); /// malloc, free and realloc (thread-safe). #ifdef ALLOCATOR_BYPASS -#include -static inline void* wwmGlobalMalloc( size_t size ) { return malloc(size); } -static inline void* wwmGlobalRealloc( void* ptr, size_t size ) { return realloc( ptr, size ); } -static inline void wwmGlobalFree( void* ptr ) { free( ptr); } - -static inline void* wwmDedicatedMalloc( ww_dedictaed_mem_t* o, size_t size ) {(void)o; return malloc(size); } -static inline void* wwmDedicatedRealloc( ww_dedictaed_mem_t* o, void* ptr, size_t size ) {(void)o; return realloc( ptr, size ); } -static inline void wwmDedicatedFree( ww_dedictaed_mem_t* o, void* ptr ) {(void)o; free( ptr); } - -#else - -static inline void* wwmGlobalMalloc( size_t size ) { return wwmGlobalAlloc( NULL, size, NULL ); } -static inline void* wwmGlobalRealloc( void* ptr, size_t size ) { return wwmGlobalAlloc( ptr, size, NULL ); } -static inline void wwmGlobalFree( void* ptr ) { wwmGlobalAlloc( ptr, 0, NULL ); } - -static inline void* wwmDedicatedMalloc( ww_dedictaed_mem_t* o, size_t size ) { return wwmDedicatedAlloc( o, NULL, size, NULL ); } -static inline void* wwmDedicatedRealloc( ww_dedictaed_mem_t* o, void* ptr, size_t size ) { return wwmDedicatedAlloc( o, ptr, size, NULL ); } -static inline void wwmDedicatedFree( ww_dedictaed_mem_t* o, void* ptr ) { wwmDedicatedAlloc( o, ptr, 0, NULL ); } - - -#endif - - - - -/// realloc, specifying current size (thread-safe). -static inline void* wwmGlobalNRealloc( void* current_ptr, size_t current_size, size_t new_size ) +static inline void *wwmGlobalMalloc(size_t size) { - return wwmGlobalNalloc( current_ptr, current_size, new_size, NULL ); + return malloc(size); } - -static inline void* wwmDedicatedNRealloc( ww_dedictaed_mem_t* o, void* current_ptr, size_t current_size, size_t new_size ) +static inline void *wwmGlobalRealloc(void *ptr, size_t size) { - return wwmDedicatedNalloc( o, current_ptr, current_size, new_size, NULL ); + return realloc(ptr, size); } - -/// free, specifying current size (thread-safe). -static inline void wwmGlobalNFree( void* current_ptr, size_t current_size ) +static inline void wwmGlobalFree(void *ptr) { - wwmGlobalNalloc( current_ptr, current_size, 0, NULL ); + free(ptr); } -static inline void wwmDedicatedNFree( ww_dedictaed_mem_t* o, void* current_ptr, size_t current_size ) +static inline void *wwmDedicatedMalloc(dedicated_memory_t *dm, size_t size) { - wwmDedicatedNalloc( o, current_ptr, current_size, 0, NULL ); + return malloc(size); +} +static inline void *wwmDedicatedRealloc(dedicated_memory_t *dm, void *ptr, size_t size) +{ + return realloc(ptr, size); +} +static inline void wwmDedicatedFree(dedicated_memory_t *dm, void *ptr) +{ + free(ptr); } -/**********************************************************************************************************************/ -/// Diagnostics - -/// Returns currently granted space for a specified memory instance (thread-safe) -size_t wwmGlobalGrantedSpace( const void* current_ptr ); -size_t wwmDedicatedGrantedSpace( ww_dedictaed_mem_t* o, const void* current_ptr ); -/// Returns total of currently granted space (thread-safe) -size_t wwmGlobaltotalGrantedSpace( void ); -size_t wwmDedicatedtotalGrantedSpace( ww_dedictaed_mem_t* o ); +} -/// Returns number of open allocation instances (thread-safe) -size_t wwmGlobaltotalInstances( void ); -size_t wwmDedicatedtotalInstances( ww_dedictaed_mem_t* o ); +#else -/** Iterates through all open instances and calls 'callback' per instance (thread-safe) - * The callback function may change the manager's state. - * Only instances which where open at the moment of entering 'bcore_wwmDedicatedForEachInstance' are iterated. - * While 'bcore_wwmDedicatedForEachInstance' executes, any instance closed or newly opened will not change the iteration. - */ -void wwmGlobalForEachInstance( void (*cb)( void* arg, void* ptr, size_t space ), void* arg ); -void wwmDedicatedForEachInstance( ww_dedictaed_mem_t* o, void (*cb)( void* arg, void* ptr, size_t space ), void* arg ); +void *wwmGlobalMalloc(size_t size); +void *wwmGlobalRealloc(void *ptr, size_t size); +void wwmGlobalFree(void *ptr); -/// prints internal status to stdout (use only for debugging/testing - not thread-safe) -void printWWMGlobalstatus( int detail_level ); -void printWWMDedicatedstatus( ww_dedictaed_mem_t* o, int detail_level ); +void *wwmDedicatedMalloc(dedicated_memory_t *dm, size_t size); +void *wwmDedicatedRealloc(dedicated_memory_t *dm, void *ptr, size_t size); +void wwmDedicatedFree(dedicated_memory_t *dm, void *ptr); -/**********************************************************************************************************************/ +#endif diff --git a/ww/ww.c b/ww/ww.c index ad759ceb..70549f9d 100644 --- a/ww/ww.c +++ b/ww/ww.c @@ -37,8 +37,8 @@ struct generic_pool_s **context_pools = NULL; struct generic_pool_s **line_pools = NULL; struct generic_pool_s **pipeline_msg_pools = NULL; struct generic_pool_s **libhv_hio_pools = NULL; -struct ww_dedictaed_mem_s **dedicated_memory_managers = NULL; -struct ww_dedictaed_mem_s *memory_manager = NULL; +struct dedicated_memory_s **dedicated_memory_managers = NULL; +struct dedicated_memory_s *memory_manager = NULL; struct socket_manager_s *socekt_manager = NULL; struct node_manager_s *node_manager = NULL; logger_t *core_logger = NULL; @@ -57,8 +57,8 @@ struct ww_runtime_state_s struct generic_pool_s **line_pools; struct generic_pool_s **pipeline_msg_pools; struct generic_pool_s **libhv_hio_pools; - struct ww_dedictaed_mem_s **dedicated_memory_managers; - struct ww_dedictaed_mem_s *memory_manager; + struct dedicated_memory_s **dedicated_memory_managers; + struct dedicated_memory_s *memory_manager; struct socket_manager_s *socekt_manager; struct node_manager_s *node_manager; logger_t *core_logger; @@ -188,7 +188,7 @@ void createWW(const ww_construction_data_t init_data) workers = (hthread_t *) malloc(sizeof(hthread_t) * (workers_count + kAdditionalReservedWorkers)); loops = (hloop_t **) malloc(sizeof(hloop_t *) * (workers_count + kAdditionalReservedWorkers)); - ram_profile = init_data.ram_profile; + ram_profile = init_data.ram_profile; buffer_pools = (struct buffer_pool_s **) malloc(sizeof(struct buffer_pool_s *) * (workers_count + kAdditionalReservedWorkers)); @@ -204,7 +204,7 @@ void createWW(const ww_construction_data_t init_data) libhv_hio_pools = (struct generic_pool_s **) malloc(sizeof(struct generic_pool_s *) * (workers_count + kAdditionalReservedWorkers)); dedicated_memory_managers = - (struct ww_dedictaed_mem_s **) malloc(sizeof(struct ww_dedictaed_mem_s *) * (workers_count)); + (struct dedicated_memory_s **) malloc(sizeof(struct dedicated_memory_s *) * (workers_count)); for (unsigned int i = 0; i < workers_count; ++i) { @@ -221,7 +221,7 @@ void createWW(const ww_construction_data_t init_data) pipeline_msg_pools[i] = newGenericPoolWithCap((8) + ram_profile, allocPipeLineMsgPoolHandle, destroyPipeLineMsgPoolHandle); - dedicated_memory_managers[i] = wwmDedicatedCreateDefault(); + dedicated_memory_managers[i] = createWWDedicatedMemory(); // todo (half implemented) // libhv_hio_pools[i] = @@ -229,8 +229,6 @@ void createWW(const ww_construction_data_t init_data) } } - - // [Section] initate workers and eventloops { loops[0] = hloop_new(HLOOP_FLAG_AUTO_FREE, buffer_pools[0], 0); @@ -258,7 +256,7 @@ void createWW(const ww_construction_data_t init_data) socekt_manager = createSocketManager(loops[socketmanager_tid], socketmanager_tid); } - + // [Section] setup NodeManager { node_manager = createNodeManager(); diff --git a/ww/ww.h b/ww/ww.h index 75667140..074d759c 100644 --- a/ww/ww.h +++ b/ww/ww.h @@ -99,8 +99,8 @@ extern struct generic_pool_s **context_pools; extern struct generic_pool_s **line_pools; extern struct generic_pool_s **pipeline_msg_pools; extern struct generic_pool_s **libhv_hio_pools; -extern struct ww_dedictaed_mem_s **dedicated_memory_managers; -extern struct ww_dedictaed_mem_s *memory_manager; +extern struct dedicated_memory_s **dedicated_memory_managers; +extern struct dedicated_memory_s *memory_manager; extern struct socket_manager_s *socekt_manager; extern struct node_manager_s *node_manager; extern struct logger_s *core_logger;