|
Packit |
345191 |
/*
|
|
Packit |
345191 |
* Copyright (C) 2017 - 2019 Intel Corporation.
|
|
Packit |
345191 |
* All rights reserved.
|
|
Packit |
345191 |
*
|
|
Packit |
345191 |
* Redistribution and use in source and binary forms, with or without
|
|
Packit |
345191 |
* modification, are permitted provided that the following conditions are met:
|
|
Packit |
345191 |
* 1. Redistributions of source code must retain the above copyright notice(s),
|
|
Packit |
345191 |
* this list of conditions and the following disclaimer.
|
|
Packit |
345191 |
* 2. Redistributions in binary form must reproduce the above copyright notice(s),
|
|
Packit |
345191 |
* this list of conditions and the following disclaimer in the documentation
|
|
Packit |
345191 |
* and/or other materials provided with the distribution.
|
|
Packit |
345191 |
*
|
|
Packit |
345191 |
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY EXPRESS
|
|
Packit |
345191 |
* OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
|
|
Packit |
345191 |
* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
|
|
Packit |
345191 |
* EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY DIRECT, INDIRECT,
|
|
Packit |
345191 |
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
|
Packit |
345191 |
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
|
|
Packit |
345191 |
* PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
|
|
Packit |
345191 |
* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
|
|
Packit |
345191 |
* OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
|
|
Packit |
345191 |
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
Packit |
345191 |
*/
|
|
Packit |
345191 |
|
|
Packit |
345191 |
#include <memkind/internal/memkind_default.h>
|
|
Packit |
345191 |
#include <memkind/internal/memkind_log.h>
|
|
Packit |
345191 |
#include <memkind/internal/memkind_private.h>
|
|
Packit |
345191 |
#include <memkind/internal/tbb_wrapper.h>
|
|
Packit |
345191 |
#include <memkind/internal/tbb_mem_pool_policy.h>
|
|
Packit |
345191 |
#include <limits.h>
|
|
Packit |
345191 |
|
|
Packit |
345191 |
#include <stdint.h>
|
|
Packit |
345191 |
#include <errno.h>
|
|
Packit |
345191 |
#include <stdio.h>
|
|
Packit |
345191 |
#include <sys/mman.h>
|
|
Packit |
345191 |
#include <stdlib.h>
|
|
Packit |
345191 |
#include <dlfcn.h>
|
|
Packit |
345191 |
#include <string.h>
|
|
Packit |
345191 |
|
|
Packit |
345191 |
void *(*pool_malloc)(void *, size_t);
|
|
Packit |
345191 |
void *(*pool_realloc)(void *, void *, size_t);
|
|
Packit |
345191 |
void *(*pool_aligned_malloc)(void *, size_t, size_t);
|
|
Packit |
345191 |
bool (*pool_free)(void *, void *);
|
|
Packit |
345191 |
int (*pool_create_v1)(intptr_t, const struct MemPoolPolicy *, void **);
|
|
Packit |
345191 |
bool (*pool_destroy)(void *);
|
|
Packit |
345191 |
void *(*pool_identify)(void *object);
|
|
Packit |
345191 |
size_t (*pool_msize)(void *, void *);
|
|
Packit |
345191 |
|
|
Packit |
345191 |
static void *tbb_handle = NULL;
|
|
Packit |
345191 |
static bool TBBInitDone = false;
|
|
Packit |
345191 |
|
|
Packit |
345191 |
void load_tbb_symbols(void)
|
|
Packit |
345191 |
{
|
|
Packit |
345191 |
const char so_name[]="libtbbmalloc.so.2";
|
|
Packit |
345191 |
tbb_handle = dlopen(so_name, RTLD_LAZY);
|
|
Packit |
345191 |
if(!tbb_handle) {
|
|
Packit |
345191 |
log_fatal("%s not found.", so_name);
|
|
Packit |
345191 |
abort();
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
|
|
Packit |
345191 |
pool_malloc = dlsym(tbb_handle, "_ZN3rml11pool_mallocEPNS_10MemoryPoolEm");
|
|
Packit |
345191 |
pool_realloc = dlsym(tbb_handle, "_ZN3rml12pool_reallocEPNS_10MemoryPoolEPvm");
|
|
Packit |
345191 |
pool_aligned_malloc = dlsym(tbb_handle,
|
|
Packit |
345191 |
"_ZN3rml19pool_aligned_mallocEPNS_10MemoryPoolEmm");
|
|
Packit |
345191 |
pool_free = dlsym(tbb_handle, "_ZN3rml9pool_freeEPNS_10MemoryPoolEPv");
|
|
Packit |
345191 |
pool_create_v1 = dlsym(tbb_handle,
|
|
Packit |
345191 |
"_ZN3rml14pool_create_v1ElPKNS_13MemPoolPolicyEPPNS_10MemoryPoolE");
|
|
Packit |
345191 |
pool_destroy = dlsym(tbb_handle, "_ZN3rml12pool_destroyEPNS_10MemoryPoolE");
|
|
Packit |
345191 |
pool_identify = dlsym(tbb_handle, "_ZN3rml13pool_identifyEPv");
|
|
Packit |
345191 |
pool_msize = dlsym(tbb_handle, "_ZN3rml10pool_msizeEPNS_10MemoryPoolEPv");
|
|
Packit |
345191 |
|
|
Packit |
345191 |
if(!pool_malloc ||
|
|
Packit |
345191 |
!pool_realloc ||
|
|
Packit |
345191 |
!pool_aligned_malloc ||
|
|
Packit |
345191 |
!pool_free ||
|
|
Packit |
345191 |
!pool_create_v1 ||
|
|
Packit |
345191 |
!pool_destroy ||
|
|
Packit |
345191 |
!pool_identify ) {
|
|
Packit |
345191 |
log_fatal("Could not find symbols in %s.", so_name);
|
|
Packit |
345191 |
dlclose(tbb_handle);
|
|
Packit |
345191 |
abort();
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
TBBInitDone = true;
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
|
|
Packit |
345191 |
//Granularity of raw_alloc allocations
|
|
Packit |
345191 |
#define GRANULARITY 2*1024*1024
|
|
Packit |
345191 |
static void *raw_alloc(intptr_t pool_id, size_t *bytes/*=n*GRANULARITY*/)
|
|
Packit |
345191 |
{
|
|
Packit |
345191 |
void *ptr = kind_mmap((struct memkind *)pool_id, NULL, *bytes);
|
|
Packit |
345191 |
return (ptr==MAP_FAILED) ? NULL : ptr;
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
|
|
Packit |
345191 |
static int raw_free(intptr_t pool_id, void *raw_ptr, size_t raw_bytes)
|
|
Packit |
345191 |
{
|
|
Packit |
345191 |
return munmap(raw_ptr, raw_bytes);
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
|
|
Packit |
345191 |
static void *tbb_pool_malloc(struct memkind *kind, size_t size)
|
|
Packit |
345191 |
{
|
|
Packit |
345191 |
if(size_out_of_bounds(size)) return NULL;
|
|
Packit |
345191 |
void *result = pool_malloc(kind->priv, size);
|
|
Packit |
345191 |
if (!result)
|
|
Packit |
345191 |
errno = ENOMEM;
|
|
Packit |
345191 |
return result;
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
|
|
Packit |
345191 |
static void *tbb_pool_calloc(struct memkind *kind, size_t num, size_t size)
|
|
Packit |
345191 |
{
|
|
Packit |
345191 |
if (size_out_of_bounds(num) || size_out_of_bounds(size)) return NULL;
|
|
Packit |
345191 |
|
|
Packit |
345191 |
const size_t array_size = num*size;
|
|
Packit |
345191 |
if (array_size/num != size) {
|
|
Packit |
345191 |
errno = ENOMEM;
|
|
Packit |
345191 |
return NULL;
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
void *result = pool_malloc(kind->priv, array_size);
|
|
Packit |
345191 |
if (result) {
|
|
Packit |
345191 |
memset(result, 0, array_size);
|
|
Packit |
345191 |
} else {
|
|
Packit |
345191 |
errno = ENOMEM;
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
return result;
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
|
|
Packit |
345191 |
static void *tbb_pool_common_realloc(void *pool, void *ptr, size_t size)
|
|
Packit |
345191 |
{
|
|
Packit |
345191 |
if (size_out_of_bounds(size)) {
|
|
Packit |
345191 |
pool_free(pool, ptr);
|
|
Packit |
345191 |
return NULL;
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
void *result = pool_realloc(pool, ptr, size);
|
|
Packit |
345191 |
if (!result)
|
|
Packit |
345191 |
errno = ENOMEM;
|
|
Packit |
345191 |
return result;
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
|
|
Packit |
345191 |
static void *tbb_pool_realloc(struct memkind *kind, void *ptr, size_t size)
|
|
Packit |
345191 |
{
|
|
Packit |
345191 |
return tbb_pool_common_realloc(kind->priv, ptr, size);
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
|
|
Packit |
345191 |
void *tbb_pool_realloc_with_kind_detect(void *ptr, size_t size)
|
|
Packit |
345191 |
{
|
|
Packit |
345191 |
if (!ptr) {
|
|
Packit |
345191 |
errno = EINVAL;
|
|
Packit |
345191 |
return NULL;
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
return tbb_pool_common_realloc(pool_identify(ptr), ptr, size);
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
|
|
Packit |
345191 |
int tbb_get_global_stat(memkind_stat_type stat, size_t *value)
|
|
Packit |
345191 |
{
|
|
Packit |
345191 |
log_err("Get allocator statistic is not supported by TBB");
|
|
Packit |
345191 |
return MEMKIND_ERROR_OPERATION_FAILED;
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
|
|
Packit |
345191 |
static int tbb_get_kind_stat(memkind_t kind, memkind_stat_type stat,
|
|
Packit |
345191 |
size_t *value)
|
|
Packit |
345191 |
{
|
|
Packit |
345191 |
log_err("Get kind statistic is not supported by TBB");
|
|
Packit |
345191 |
return MEMKIND_ERROR_OPERATION_FAILED;
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
|
|
Packit |
345191 |
int tbb_update_cached_stats(void)
|
|
Packit |
345191 |
{
|
|
Packit |
345191 |
log_err("Update cached statistics is not supported by TBB");
|
|
Packit |
345191 |
return MEMKIND_ERROR_OPERATION_FAILED;
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
|
|
Packit |
345191 |
void *tbb_pool_defrag_reallocate_with_kind_detect(void *ptr)
|
|
Packit |
345191 |
{
|
|
Packit |
345191 |
log_err("Defrag reallocate method is not supported by TBB");
|
|
Packit |
345191 |
return NULL;
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
|
|
Packit |
345191 |
static void *tbb_defrag_reallocate(struct memkind *kind, void *ptr)
|
|
Packit |
345191 |
{
|
|
Packit |
345191 |
log_err("Defrag reallocate method is not supported by TBB");
|
|
Packit |
345191 |
return NULL;
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
|
|
Packit |
345191 |
struct memkind *tbb_detect_kind(void *ptr)
|
|
Packit |
345191 |
{
|
|
Packit |
345191 |
if (!ptr) {
|
|
Packit |
345191 |
return NULL;
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
struct memkind *kind = NULL;
|
|
Packit |
345191 |
unsigned i;
|
|
Packit |
345191 |
void *pool = pool_identify(ptr);
|
|
Packit |
345191 |
for (i = 0; i < MEMKIND_MAX_KIND; ++i) {
|
|
Packit |
345191 |
int err = memkind_get_kind_by_partition(i, &kind);
|
|
Packit |
345191 |
if (!err && kind->priv == pool) {
|
|
Packit |
345191 |
break;
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
|
|
Packit |
345191 |
return kind;
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
|
|
Packit |
345191 |
static int tbb_pool_posix_memalign(struct memkind *kind, void **memptr,
|
|
Packit |
345191 |
size_t alignment, size_t size)
|
|
Packit |
345191 |
{
|
|
Packit |
345191 |
//Check if alignment is "at least as large as sizeof(void *)".
|
|
Packit |
345191 |
if(!alignment && (0 != (alignment & (alignment-sizeof(void *))))) return EINVAL;
|
|
Packit |
345191 |
//Check if alignment is "a power of 2".
|
|
Packit |
345191 |
if(alignment & (alignment-1)) return EINVAL;
|
|
Packit |
345191 |
if(size_out_of_bounds(size)) {
|
|
Packit |
345191 |
*memptr = NULL;
|
|
Packit |
345191 |
return 0;
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
void *result = pool_aligned_malloc(kind->priv, size, alignment);
|
|
Packit |
345191 |
if (!result) {
|
|
Packit |
345191 |
return ENOMEM;
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
*memptr = result;
|
|
Packit |
345191 |
return 0;
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
|
|
Packit |
345191 |
void tbb_pool_free_with_kind_detect(void *ptr)
|
|
Packit |
345191 |
{
|
|
Packit |
345191 |
if (ptr) {
|
|
Packit |
345191 |
pool_free(pool_identify(ptr), ptr);
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
|
|
Packit |
345191 |
static void tbb_pool_free(struct memkind *kind, void *ptr)
|
|
Packit |
345191 |
{
|
|
Packit |
345191 |
pool_free(kind->priv, ptr);
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
|
|
Packit |
345191 |
static size_t tbb_pool_common_malloc_usable_size(void *pool, void *ptr)
|
|
Packit |
345191 |
{
|
|
Packit |
345191 |
if (pool_msize) {
|
|
Packit |
345191 |
return pool_msize(pool, ptr);
|
|
Packit |
345191 |
} else {
|
|
Packit |
345191 |
log_err("memkind_malloc_usable_size() is not supported by this TBB version.");
|
|
Packit |
345191 |
return 0;
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
|
|
Packit |
345191 |
static size_t tbb_pool_malloc_usable_size(struct memkind *kind, void *ptr)
|
|
Packit |
345191 |
{
|
|
Packit |
345191 |
return tbb_pool_common_malloc_usable_size(kind->priv, ptr);
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
|
|
Packit |
345191 |
size_t tbb_pool_malloc_usable_size_with_kind_detect(void *ptr)
|
|
Packit |
345191 |
{
|
|
Packit |
345191 |
size_t size = 0;
|
|
Packit |
345191 |
if (ptr) {
|
|
Packit |
345191 |
size = tbb_pool_common_malloc_usable_size(pool_identify(ptr), ptr);
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
return size;
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
|
|
Packit |
345191 |
static int tbb_update_memory_usage_policy(struct memkind *kind,
|
|
Packit |
345191 |
memkind_mem_usage_policy policy)
|
|
Packit |
345191 |
{
|
|
Packit |
345191 |
log_info("Memkind memory usage policies have no effect in TBB manager.");
|
|
Packit |
345191 |
return MEMKIND_SUCCESS;
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
|
|
Packit |
345191 |
static int tbb_destroy(struct memkind *kind)
|
|
Packit |
345191 |
{
|
|
Packit |
345191 |
bool pool_destroy_ret = pool_destroy(kind->priv);
|
|
Packit |
345191 |
dlclose(tbb_handle);
|
|
Packit |
345191 |
|
|
Packit |
345191 |
if(!pool_destroy_ret) {
|
|
Packit |
345191 |
log_err("TBB pool destroy failure.");
|
|
Packit |
345191 |
return MEMKIND_ERROR_OPERATION_FAILED;
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
return MEMKIND_SUCCESS;
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
|
|
Packit |
345191 |
void tbb_initialize(struct memkind *kind)
|
|
Packit |
345191 |
{
|
|
Packit |
345191 |
if(!kind || !TBBInitDone) {
|
|
Packit |
345191 |
log_fatal("Failed to initialize TBB.");
|
|
Packit |
345191 |
abort();
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
|
|
Packit |
345191 |
struct MemPoolPolicy policy = {
|
|
Packit |
345191 |
.pAlloc = raw_alloc,
|
|
Packit |
345191 |
.pFree = raw_free,
|
|
Packit |
345191 |
.granularity = GRANULARITY,
|
|
Packit |
345191 |
.version = 1,
|
|
Packit |
345191 |
.fixedPool = false,
|
|
Packit |
345191 |
.keepAllMemory = false,
|
|
Packit |
345191 |
.reserved = 0
|
|
Packit |
345191 |
};
|
|
Packit |
345191 |
|
|
Packit |
345191 |
pool_create_v1((intptr_t)kind, &policy, &kind->priv);
|
|
Packit |
345191 |
if (!kind->priv) {
|
|
Packit |
345191 |
log_fatal("Unable to create TBB memory pool.");
|
|
Packit |
345191 |
abort();
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
|
|
Packit |
345191 |
kind->ops->malloc = tbb_pool_malloc;
|
|
Packit |
345191 |
kind->ops->calloc = tbb_pool_calloc;
|
|
Packit |
345191 |
kind->ops->posix_memalign = tbb_pool_posix_memalign;
|
|
Packit |
345191 |
kind->ops->realloc = tbb_pool_realloc;
|
|
Packit |
345191 |
kind->ops->free = tbb_pool_free;
|
|
Packit |
345191 |
kind->ops->finalize = tbb_destroy;
|
|
Packit |
345191 |
kind->ops->malloc_usable_size = tbb_pool_malloc_usable_size;
|
|
Packit |
345191 |
kind->ops->update_memory_usage_policy = tbb_update_memory_usage_policy;
|
|
Packit |
345191 |
kind->ops->get_stat = tbb_get_kind_stat;
|
|
Packit |
345191 |
kind->ops->defrag_reallocate = tbb_defrag_reallocate;
|
|
Packit |
345191 |
}
|