/* * Copyright (C) 2014 - 2019 Intel Corporation. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * 1. Redistributions of source code must retain the above copyright notice(s), * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright notice(s), * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY EXPRESS * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO * EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #define MEMKIND_VERSION_MAJOR 1 #define MEMKIND_VERSION_MINOR 10 #define MEMKIND_VERSION_PATCH 0 #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "config.h" #include #include #include #include #include #include #include #include #include #include #include #include #include /* Clear bits in x, but only this specified in mask. */ #define CLEAR_BIT(x, mask) ((x) &= (~(mask))) extern struct memkind_ops MEMKIND_HBW_GBTLB_OPS; extern struct memkind_ops MEMKIND_HBW_PREFERRED_GBTLB_OPS; extern struct memkind_ops MEMKIND_GBTLB_OPS; static struct memkind MEMKIND_DEFAULT_STATIC = { .ops = &MEMKIND_DEFAULT_OPS, .partition = MEMKIND_PARTITION_DEFAULT, .name = "memkind_default", .init_once = PTHREAD_ONCE_INIT, .arena_zero = 0, .arena_map_len = ARENA_LIMIT_DEFAULT_KIND - 1, }; static struct memkind MEMKIND_HUGETLB_STATIC = { .ops = &MEMKIND_HUGETLB_OPS, .partition = MEMKIND_PARTITION_HUGETLB, .name = "memkind_hugetlb", .init_once = PTHREAD_ONCE_INIT, }; static struct memkind MEMKIND_INTERLEAVE_STATIC = { .ops = &MEMKIND_INTERLEAVE_OPS, .partition = MEMKIND_PARTITION_INTERLEAVE, .name = "memkind_interleave", .init_once = PTHREAD_ONCE_INIT, }; static struct memkind MEMKIND_HBW_STATIC = { .ops = &MEMKIND_HBW_OPS, .partition = MEMKIND_PARTITION_HBW, .name = "memkind_hbw", .init_once = PTHREAD_ONCE_INIT, }; static struct memkind MEMKIND_HBW_ALL_STATIC = { .ops = &MEMKIND_HBW_ALL_OPS, .partition = MEMKIND_PARTITION_HBW_ALL, .name = "memkind_hbw_all", .init_once = PTHREAD_ONCE_INIT, }; static struct memkind MEMKIND_HBW_PREFERRED_STATIC = { .ops = &MEMKIND_HBW_PREFERRED_OPS, .partition = MEMKIND_PARTITION_HBW_PREFERRED, .name = "memkind_hbw_preferred", .init_once = PTHREAD_ONCE_INIT, }; static struct memkind MEMKIND_HBW_HUGETLB_STATIC = { .ops = &MEMKIND_HBW_HUGETLB_OPS, .partition = MEMKIND_PARTITION_HBW_HUGETLB, .name = "memkind_hbw_hugetlb", .init_once = PTHREAD_ONCE_INIT, }; static struct memkind MEMKIND_HBW_ALL_HUGETLB_STATIC = { .ops = &MEMKIND_HBW_ALL_HUGETLB_OPS, .partition = MEMKIND_PARTITION_HBW_ALL_HUGETLB, .name = "memkind_hbw_all_hugetlb", .init_once = PTHREAD_ONCE_INIT, }; static struct memkind MEMKIND_HBW_PREFERRED_HUGETLB_STATIC = { .ops = &MEMKIND_HBW_PREFERRED_HUGETLB_OPS, .partition = MEMKIND_PARTITION_HBW_PREFERRED_HUGETLB, .name = "memkind_hbw_preferred_hugetlb", .init_once = PTHREAD_ONCE_INIT, }; static struct memkind MEMKIND_HBW_GBTLB_STATIC = { .ops = &MEMKIND_HBW_GBTLB_OPS, .partition = MEMKIND_PARTITION_HBW_GBTLB, .name = "memkind_hbw_gbtlb", .init_once = PTHREAD_ONCE_INIT, }; static struct memkind MEMKIND_HBW_PREFERRED_GBTLB_STATIC = { .ops = &MEMKIND_HBW_PREFERRED_GBTLB_OPS, .partition = MEMKIND_PARTITION_HBW_PREFERRED_GBTLB, .name = "memkind_hbw_preferred_gbtlb", .init_once = PTHREAD_ONCE_INIT, }; static struct memkind MEMKIND_GBTLB_STATIC = { .ops = &MEMKIND_GBTLB_OPS, .partition = MEMKIND_PARTITION_GBTLB, .name = "memkind_gbtlb", .init_once = PTHREAD_ONCE_INIT, }; static struct memkind MEMKIND_HBW_INTERLEAVE_STATIC = { .ops = &MEMKIND_HBW_INTERLEAVE_OPS, .partition = MEMKIND_PARTITION_HBW_INTERLEAVE, .name = "memkind_hbw_interleave", .init_once = PTHREAD_ONCE_INIT, }; static struct memkind MEMKIND_REGULAR_STATIC = { .ops = &MEMKIND_REGULAR_OPS, .partition = MEMKIND_PARTITION_REGULAR, .name = "memkind_regular", .init_once = PTHREAD_ONCE_INIT, }; static struct memkind MEMKIND_DAX_KMEM_STATIC = { .ops = &MEMKIND_DAX_KMEM_OPS, .partition = MEMKIND_PARTITION_DAX_KMEM, .name = "memkind_dax_kmem", .init_once = PTHREAD_ONCE_INIT, }; static struct memkind MEMKIND_DAX_KMEM_ALL_STATIC = { .ops = &MEMKIND_DAX_KMEM_ALL_OPS, .partition = MEMKIND_PARTITION_DAX_KMEM_ALL, .name = "memkind_dax_kmem_all", .init_once = PTHREAD_ONCE_INIT, }; static struct memkind MEMKIND_DAX_KMEM_PREFERRED_STATIC = { .ops = &MEMKIND_DAX_KMEM_PREFERRED_OPS, .partition = MEMKIND_PARTITION_DAX_KMEM_PREFERRED, .name = "memkind_dax_kmem_preferred", .init_once = PTHREAD_ONCE_INIT, }; MEMKIND_EXPORT struct memkind *MEMKIND_DEFAULT = &MEMKIND_DEFAULT_STATIC; MEMKIND_EXPORT struct memkind *MEMKIND_HUGETLB = &MEMKIND_HUGETLB_STATIC; MEMKIND_EXPORT struct memkind *MEMKIND_INTERLEAVE = &MEMKIND_INTERLEAVE_STATIC; MEMKIND_EXPORT struct memkind *MEMKIND_HBW = &MEMKIND_HBW_STATIC; MEMKIND_EXPORT struct memkind *MEMKIND_HBW_ALL = &MEMKIND_HBW_ALL_STATIC; MEMKIND_EXPORT struct memkind *MEMKIND_HBW_PREFERRED = &MEMKIND_HBW_PREFERRED_STATIC; MEMKIND_EXPORT struct memkind *MEMKIND_HBW_HUGETLB = &MEMKIND_HBW_HUGETLB_STATIC; MEMKIND_EXPORT struct memkind *MEMKIND_HBW_ALL_HUGETLB = &MEMKIND_HBW_ALL_HUGETLB_STATIC; MEMKIND_EXPORT struct memkind *MEMKIND_HBW_PREFERRED_HUGETLB = &MEMKIND_HBW_PREFERRED_HUGETLB_STATIC; MEMKIND_EXPORT struct memkind *MEMKIND_HBW_GBTLB = &MEMKIND_HBW_GBTLB_STATIC; MEMKIND_EXPORT struct memkind *MEMKIND_HBW_PREFERRED_GBTLB = &MEMKIND_HBW_PREFERRED_GBTLB_STATIC; MEMKIND_EXPORT struct memkind *MEMKIND_GBTLB = &MEMKIND_GBTLB_STATIC; MEMKIND_EXPORT struct memkind *MEMKIND_HBW_INTERLEAVE = &MEMKIND_HBW_INTERLEAVE_STATIC; MEMKIND_EXPORT struct memkind *MEMKIND_REGULAR = &MEMKIND_REGULAR_STATIC; MEMKIND_EXPORT struct memkind *MEMKIND_DAX_KMEM = &MEMKIND_DAX_KMEM_STATIC; MEMKIND_EXPORT struct memkind *MEMKIND_DAX_KMEM_ALL = &MEMKIND_DAX_KMEM_ALL_STATIC; MEMKIND_EXPORT struct memkind *MEMKIND_DAX_KMEM_PREFERRED = &MEMKIND_DAX_KMEM_PREFERRED_STATIC; struct memkind_registry { struct memkind *partition_map[MEMKIND_MAX_KIND]; int num_kind; pthread_mutex_t lock; }; static struct memkind_registry memkind_registry_g = { { [MEMKIND_PARTITION_DEFAULT] = &MEMKIND_DEFAULT_STATIC, [MEMKIND_PARTITION_HBW] = &MEMKIND_HBW_STATIC, [MEMKIND_PARTITION_HBW_PREFERRED] = &MEMKIND_HBW_PREFERRED_STATIC, [MEMKIND_PARTITION_HBW_HUGETLB] = &MEMKIND_HBW_HUGETLB_STATIC, [MEMKIND_PARTITION_HBW_PREFERRED_HUGETLB] = &MEMKIND_HBW_PREFERRED_HUGETLB_STATIC, [MEMKIND_PARTITION_HUGETLB] = &MEMKIND_HUGETLB_STATIC, [MEMKIND_PARTITION_HBW_GBTLB] = &MEMKIND_HBW_GBTLB_STATIC, [MEMKIND_PARTITION_HBW_PREFERRED_GBTLB] = &MEMKIND_HBW_PREFERRED_GBTLB_STATIC, [MEMKIND_PARTITION_GBTLB] = &MEMKIND_GBTLB_STATIC, [MEMKIND_PARTITION_HBW_INTERLEAVE] = &MEMKIND_HBW_INTERLEAVE_STATIC, [MEMKIND_PARTITION_INTERLEAVE] = &MEMKIND_INTERLEAVE_STATIC, [MEMKIND_PARTITION_REGULAR] = &MEMKIND_REGULAR_STATIC, [MEMKIND_PARTITION_HBW_ALL] = &MEMKIND_HBW_ALL_STATIC, [MEMKIND_PARTITION_HBW_ALL_HUGETLB] = &MEMKIND_HBW_ALL_HUGETLB_STATIC, [MEMKIND_PARTITION_DAX_KMEM] = &MEMKIND_DAX_KMEM_STATIC, [MEMKIND_PARTITION_DAX_KMEM_ALL] = &MEMKIND_DAX_KMEM_ALL_STATIC, [MEMKIND_PARTITION_DAX_KMEM_PREFERRED] = &MEMKIND_DAX_KMEM_PREFERRED_STATIC, }, MEMKIND_NUM_BASE_KIND, PTHREAD_MUTEX_INITIALIZER }; void *kind_mmap(struct memkind *kind, void *addr, size_t size) { if (MEMKIND_LIKELY(kind->ops->mmap == NULL)) { return memkind_default_mmap(kind, addr, size); } else { return kind->ops->mmap(kind, addr, size); } } static int validate_memtype_bits(memkind_memtype_t memtype) { if(memtype == 0) return -1; CLEAR_BIT(memtype, MEMKIND_MEMTYPE_DEFAULT); CLEAR_BIT(memtype, MEMKIND_MEMTYPE_HIGH_BANDWIDTH); if(memtype != 0) return -1; return 0; } static int validate_flags_bits(memkind_bits_t flags) { CLEAR_BIT(flags, MEMKIND_MASK_PAGE_SIZE_2MB); if(flags != 0) return -1; return 0; } static int validate_policy(memkind_policy_t policy) { if((policy >= 0) && (policy < MEMKIND_POLICY_MAX_VALUE)) return 0; return -1; } struct create_args { memkind_t kind; memkind_policy_t policy; memkind_bits_t flags; memkind_memtype_t memtype_flags; }; static struct create_args supported_args[] = { {&MEMKIND_HBW_STATIC, MEMKIND_POLICY_BIND_LOCAL, 0, MEMKIND_MEMTYPE_HIGH_BANDWIDTH}, {&MEMKIND_HBW_HUGETLB_STATIC, MEMKIND_POLICY_BIND_LOCAL, MEMKIND_MASK_PAGE_SIZE_2MB, MEMKIND_MEMTYPE_HIGH_BANDWIDTH}, {&MEMKIND_HBW_ALL_STATIC, MEMKIND_POLICY_BIND_ALL, 0, MEMKIND_MEMTYPE_HIGH_BANDWIDTH}, {&MEMKIND_HBW_ALL_HUGETLB_STATIC, MEMKIND_POLICY_BIND_ALL, MEMKIND_MASK_PAGE_SIZE_2MB, MEMKIND_MEMTYPE_HIGH_BANDWIDTH}, {&MEMKIND_HBW_PREFERRED_STATIC, MEMKIND_POLICY_PREFERRED_LOCAL, 0, MEMKIND_MEMTYPE_HIGH_BANDWIDTH}, {&MEMKIND_HBW_PREFERRED_HUGETLB_STATIC, MEMKIND_POLICY_PREFERRED_LOCAL, MEMKIND_MASK_PAGE_SIZE_2MB, MEMKIND_MEMTYPE_HIGH_BANDWIDTH}, {&MEMKIND_HBW_INTERLEAVE_STATIC, MEMKIND_POLICY_INTERLEAVE_ALL, 0, MEMKIND_MEMTYPE_HIGH_BANDWIDTH}, {&MEMKIND_DEFAULT_STATIC, MEMKIND_POLICY_PREFERRED_LOCAL, 0, MEMKIND_MEMTYPE_DEFAULT}, {&MEMKIND_HUGETLB_STATIC, MEMKIND_POLICY_PREFERRED_LOCAL, MEMKIND_MASK_PAGE_SIZE_2MB, MEMKIND_MEMTYPE_DEFAULT}, {&MEMKIND_INTERLEAVE_STATIC, MEMKIND_POLICY_INTERLEAVE_ALL, 0, MEMKIND_MEMTYPE_HIGH_BANDWIDTH | MEMKIND_MEMTYPE_DEFAULT}, }; /* Kind creation */ MEMKIND_EXPORT int memkind_create_kind(memkind_memtype_t memtype_flags, memkind_policy_t policy, memkind_bits_t flags, memkind_t *kind) { if(validate_memtype_bits(memtype_flags) != 0) { log_err("Cannot create kind: incorrect memtype_flags."); return MEMKIND_ERROR_INVALID; } if(validate_flags_bits(flags) != 0) { log_err("Cannot create kind: incorrect flags."); return MEMKIND_ERROR_INVALID; } if(validate_policy(policy) != 0) { log_err("Cannot create kind: incorrect policy."); return MEMKIND_ERROR_INVALID; } if(kind == NULL) { log_err("Cannot create kind: 'kind' is NULL pointer."); return MEMKIND_ERROR_INVALID; } int i, num_supported_args = sizeof(supported_args) / sizeof(struct create_args); for(i = 0; i < num_supported_args; i++) { if((supported_args[i].memtype_flags == memtype_flags) && (supported_args[i].policy == policy) && (supported_args[i].flags == flags)) { if(memkind_check_available(supported_args[i].kind) == 0) { *kind = supported_args[i].kind; return MEMKIND_SUCCESS; } else if(policy == MEMKIND_POLICY_PREFERRED_LOCAL) { *kind = MEMKIND_DEFAULT; return MEMKIND_SUCCESS; } log_err("Cannot create kind: requested memory type is not available."); return MEMKIND_ERROR_MEMTYPE_NOT_AVAILABLE; } } log_err("Cannot create kind: unsupported set of capabilities."); return MEMKIND_ERROR_INVALID; } static void memkind_destroy_dynamic_kind_from_register(unsigned int i, memkind_t kind) { if (i >= MEMKIND_NUM_BASE_KIND) { memkind_registry_g.partition_map[i] = NULL; --memkind_registry_g.num_kind; free(kind); } } /* Kind destruction. */ MEMKIND_EXPORT int memkind_destroy_kind(memkind_t kind) { if (pthread_mutex_lock(&memkind_registry_g.lock) != 0) assert(0 && "failed to acquire mutex"); unsigned i; int err = kind->ops->destroy(kind); for (i = MEMKIND_NUM_BASE_KIND; i < MEMKIND_MAX_KIND; ++i) { if (memkind_registry_g.partition_map[i] && strcmp(kind->name, memkind_registry_g.partition_map[i]->name) == 0) { memkind_destroy_dynamic_kind_from_register(i, kind); break; } } if (pthread_mutex_unlock(&memkind_registry_g.lock) != 0) assert(0 && "failed to release mutex"); return err; } MEMKIND_EXPORT memkind_t memkind_detect_kind(void *ptr) { return heap_manager_detect_kind(ptr); } /* Declare weak symbols for allocator decorators */ extern void memkind_malloc_pre(struct memkind **, size_t *) __attribute__((weak)); extern void memkind_malloc_post(struct memkind *, size_t, void **) __attribute__((weak)); extern void memkind_calloc_pre(struct memkind **, size_t *, size_t *) __attribute__((weak)); extern void memkind_calloc_post(struct memkind *, size_t, size_t, void **) __attribute__((weak)); extern void memkind_posix_memalign_pre(struct memkind **, void **, size_t *, size_t *) __attribute__((weak)); extern void memkind_posix_memalign_post(struct memkind *, void **, size_t, size_t, int *) __attribute__((weak)); extern void memkind_realloc_pre(struct memkind **, void **, size_t *) __attribute__((weak)); extern void memkind_realloc_post(struct memkind *, void *, size_t, void **) __attribute__((weak)); extern void memkind_free_pre(struct memkind **, void **) __attribute__((weak)); extern void memkind_free_post(struct memkind *, void *) __attribute__((weak)); MEMKIND_EXPORT int memkind_get_version() { return MEMKIND_VERSION_MAJOR * 1000000 + MEMKIND_VERSION_MINOR * 1000 + MEMKIND_VERSION_PATCH; } MEMKIND_EXPORT void memkind_error_message(int err, char *msg, size_t size) { switch (err) { case MEMKIND_ERROR_UNAVAILABLE: strncpy(msg, " Requested memory kind is not available", size); break; case MEMKIND_ERROR_MBIND: strncpy(msg, " Call to mbind() failed", size); break; case MEMKIND_ERROR_MMAP: strncpy(msg, " Call to mmap() failed", size); break; case MEMKIND_ERROR_MALLOC: strncpy(msg, " Call to malloc() failed", size); break; case MEMKIND_ERROR_ENVIRON: strncpy(msg, " Error parsing environment variable (MEMKIND_*)", size); break; case MEMKIND_ERROR_INVALID: strncpy(msg, " Invalid input arguments to memkind routine", size); break; case MEMKIND_ERROR_TOOMANY: snprintf(msg, size, " Attempted to initialize more than maximum (%i) number of kinds", MEMKIND_MAX_KIND); break; case MEMKIND_ERROR_RUNTIME: strncpy(msg, " Unspecified run-time error", size); break; case EINVAL: strncpy(msg, " Alignment must be a power of two and larger than sizeof(void *)", size); break; case ENOMEM: strncpy(msg, " Call to jemk_mallocx() failed", size); break; case MEMKIND_ERROR_HUGETLB: strncpy(msg, " unable to allocate huge pages", size); break; case MEMKIND_ERROR_BADOPS: strncpy(msg, " memkind_ops structure is poorly formed (missing or incorrect functions)", size); break; case MEMKIND_ERROR_MEMTYPE_NOT_AVAILABLE: strncpy(msg, " Requested memory type is not available", size); break; case MEMKIND_ERROR_OPERATION_FAILED: strncpy(msg, " Operation failed", size); break; case MEMKIND_ERROR_ARENAS_CREATE: strncpy(msg, " Call to jemalloc's arenas.create () failed", size); break; default: snprintf(msg, size, " Undefined error number: %i", err); break; } if (size > 0) { msg[size-1] = '\0'; } } void memkind_init(memkind_t kind, bool check_numa) { log_info("Initializing kind %s.", kind->name); heap_manager_init(kind); if (check_numa) { int err = numa_available(); if (err) { log_fatal("[%s] NUMA not available (error code:%d).", kind->name, err); abort(); } } } static void nop(void) {} static int memkind_create(struct memkind_ops *ops, const char *name, struct memkind **kind) { int err; unsigned i; unsigned id_kind = 0; *kind = NULL; if (pthread_mutex_lock(&memkind_registry_g.lock) != 0) assert(0 && "failed to acquire mutex"); if (memkind_registry_g.num_kind == MEMKIND_MAX_KIND) { log_err("Attempted to initialize more than maximum (%i) number of kinds.", MEMKIND_MAX_KIND); err = MEMKIND_ERROR_TOOMANY; goto exit; } if (ops->create == NULL || ops->destroy == NULL || ops->malloc == NULL || ops->calloc == NULL || ops->realloc == NULL || ops->posix_memalign == NULL || ops->free == NULL || ops->init_once != NULL) { err = MEMKIND_ERROR_BADOPS; goto exit; } for (i = 0; i < MEMKIND_MAX_KIND; ++i) { if (memkind_registry_g.partition_map[i] == NULL) { id_kind = i; break; } else if (strcmp(name, memkind_registry_g.partition_map[i]->name) == 0) { log_err("Kind with the name %s already exists", name); err = MEMKIND_ERROR_INVALID; goto exit; } } *kind = (struct memkind *)calloc(1, sizeof(struct memkind)); if (!*kind) { err = MEMKIND_ERROR_MALLOC; log_err("calloc() failed."); goto exit; } (*kind)->partition = memkind_registry_g.num_kind; err = ops->create(*kind, ops, name); if (err) { free(*kind); goto exit; } memkind_registry_g.partition_map[id_kind] = *kind; ++memkind_registry_g.num_kind; (*kind)->init_once = PTHREAD_ONCE_INIT; pthread_once(&(*kind)->init_once, nop); //this is done to avoid init_once for dynamic kinds exit: if (pthread_mutex_unlock(&memkind_registry_g.lock) != 0) assert(0 && "failed to release mutex"); return err; } #ifdef __GNUC__ __attribute__((constructor)) #endif static void memkind_construct(void) { const char *env = secure_getenv("MEMKIND_HEAP_MANAGER"); if (env && strcmp(env, "TBB") == 0) { load_tbb_symbols(); } else { env = secure_getenv("MEMKIND_BACKGROUND_THREAD_LIMIT"); if (env) { char *end; errno = 0; size_t thread_limit = strtoull(env, &end, 10); if (*end != '\0' || errno != 0 ) { log_err("Error on parsing value MEMKIND_BACKGROUND_THREAD_LIMIT"); return; } memkind_arena_enable_background_threads(thread_limit); } } } #ifdef __GNUC__ __attribute__((destructor)) #endif static int memkind_finalize(void) { struct memkind *kind; unsigned i; int err = MEMKIND_SUCCESS; if (pthread_mutex_lock(&memkind_registry_g.lock) != 0) assert(0 && "failed to acquire mutex"); for (i = 0; i < MEMKIND_MAX_KIND; ++i) { kind = memkind_registry_g.partition_map[i]; if (kind && kind->ops->finalize) { err = kind->ops->finalize(kind); if (err) { goto exit; } memkind_destroy_dynamic_kind_from_register(i, kind); } } assert(memkind_registry_g.num_kind == MEMKIND_NUM_BASE_KIND); exit: if (pthread_mutex_unlock(&memkind_registry_g.lock) != 0) assert(0 && "failed to release mutex"); return err; } MEMKIND_EXPORT int memkind_check_available(struct memkind *kind) { int err = MEMKIND_SUCCESS; if (MEMKIND_LIKELY(kind->ops->check_available)) { err = kind->ops->check_available(kind); } return err; } MEMKIND_EXPORT size_t memkind_malloc_usable_size(struct memkind *kind, void *ptr) { if (!kind) { return heap_manager_malloc_usable_size(ptr); } else { return kind->ops->malloc_usable_size(kind, ptr); } } MEMKIND_EXPORT void *memkind_malloc(struct memkind *kind, size_t size) { void *result; pthread_once(&kind->init_once, kind->ops->init_once); #ifdef MEMKIND_DECORATION_ENABLED if (memkind_malloc_pre) { memkind_malloc_pre(&kind, &size); } #endif result = kind->ops->malloc(kind, size); #ifdef MEMKIND_DECORATION_ENABLED if (memkind_malloc_post) { memkind_malloc_post(kind, size, &result); } #endif return result; } MEMKIND_EXPORT void *memkind_calloc(struct memkind *kind, size_t num, size_t size) { void *result; pthread_once(&kind->init_once, kind->ops->init_once); #ifdef MEMKIND_DECORATION_ENABLED if (memkind_calloc_pre) { memkind_calloc_pre(&kind, &num, &size); } #endif result = kind->ops->calloc(kind, num, size); #ifdef MEMKIND_DECORATION_ENABLED if (memkind_calloc_post) { memkind_calloc_post(kind, num, size, &result); } #endif return result; } MEMKIND_EXPORT int memkind_posix_memalign(struct memkind *kind, void **memptr, size_t alignment, size_t size) { int err; pthread_once(&kind->init_once, kind->ops->init_once); #ifdef MEMKIND_DECORATION_ENABLED if (memkind_posix_memalign_pre) { memkind_posix_memalign_pre(&kind, memptr, &alignment, &size); } #endif err = kind->ops->posix_memalign(kind, memptr, alignment, size); #ifdef MEMKIND_DECORATION_ENABLED if (memkind_posix_memalign_post) { memkind_posix_memalign_post(kind, memptr, alignment, size, &err); } #endif return err; } MEMKIND_EXPORT void *memkind_realloc(struct memkind *kind, void *ptr, size_t size) { void *result; #ifdef MEMKIND_DECORATION_ENABLED if (memkind_realloc_pre) { memkind_realloc_pre(&kind, &ptr, &size); } #endif if (!kind) { result = heap_manager_realloc(ptr, size); } else { pthread_once(&kind->init_once, kind->ops->init_once); result = kind->ops->realloc(kind, ptr, size); } #ifdef MEMKIND_DECORATION_ENABLED if (memkind_realloc_post) { memkind_realloc_post(kind, ptr, size, &result); } #endif return result; } MEMKIND_EXPORT void memkind_free(struct memkind *kind, void *ptr) { #ifdef MEMKIND_DECORATION_ENABLED if (memkind_free_pre) { memkind_free_pre(&kind, &ptr); } #endif if (!kind) { heap_manager_free(ptr); } else { pthread_once(&kind->init_once, kind->ops->init_once); kind->ops->free(kind, ptr); } #ifdef MEMKIND_DECORATION_ENABLED if (memkind_free_post) { memkind_free_post(kind, ptr); } #endif } static int memkind_tmpfile(const char *dir, int *fd) { static char template[] = "/memkind.XXXXXX"; int err = MEMKIND_SUCCESS; int oerrno; int dir_len = strlen(dir); if (dir_len > PATH_MAX) { log_err("Could not create temporary file: too long path."); return MEMKIND_ERROR_INVALID; } char fullname[dir_len + sizeof (template)]; (void) strcpy(fullname, dir); (void) strcat(fullname, template); sigset_t set, oldset; sigfillset(&set); (void) sigprocmask(SIG_BLOCK, &set, &oldset); if ((*fd = mkstemp(fullname)) < 0) { log_err("Could not create temporary file: errno=%d.", errno); err = MEMKIND_ERROR_INVALID; goto exit; } (void) unlink(fullname); (void) sigprocmask(SIG_SETMASK, &oldset, NULL); return err; exit: oerrno = errno; (void) sigprocmask(SIG_SETMASK, &oldset, NULL); if (*fd != -1) { (void) close(*fd); } *fd = -1; errno = oerrno; return err; } MEMKIND_EXPORT struct memkind_config *memkind_config_new(void) { struct memkind_config *cfg = (struct memkind_config *)malloc(sizeof( struct memkind_config)); return cfg; } MEMKIND_EXPORT void memkind_config_delete(struct memkind_config *cfg) { free(cfg); } MEMKIND_EXPORT void memkind_config_set_path(struct memkind_config *cfg, const char *pmem_dir) { cfg->pmem_dir = pmem_dir; } MEMKIND_EXPORT void memkind_config_set_size(struct memkind_config *cfg, size_t pmem_size) { cfg->pmem_size = pmem_size; } MEMKIND_EXPORT void memkind_config_set_memory_usage_policy( struct memkind_config *cfg, memkind_mem_usage_policy policy) { cfg->policy = policy; } MEMKIND_EXPORT int memkind_create_pmem(const char *dir, size_t max_size, struct memkind **kind) { int oerrno; if (max_size && max_size < MEMKIND_PMEM_MIN_SIZE) { log_err("Cannot create pmem: invalid size."); return MEMKIND_ERROR_INVALID; } if (max_size) { /* round up to a multiple of jemalloc chunk size */ max_size = roundup(max_size, MEMKIND_PMEM_CHUNK_SIZE); } int fd = -1; char name[16]; int err = memkind_tmpfile(dir, &fd); if (err) { goto exit; } snprintf(name, sizeof (name), "pmem%08x", fd); err = memkind_create(&MEMKIND_PMEM_OPS, name, kind); if (err) { goto exit; } struct memkind_pmem *priv = (*kind)->priv; priv->fd = fd; priv->offset = 0; priv->current_size = 0; priv->max_size = max_size; return err; exit: oerrno = errno; if (fd != -1) { (void) close(fd); } errno = oerrno; return err; } MEMKIND_EXPORT int memkind_create_pmem_with_config(struct memkind_config *cfg, struct memkind **kind) { int status = memkind_create_pmem(cfg->pmem_dir, cfg->pmem_size, kind); if (MEMKIND_LIKELY(!status)) { status = (*kind)->ops->update_memory_usage_policy(*kind, cfg->policy); } return status; } static int memkind_get_kind_by_partition_internal(int partition, struct memkind **kind) { int err = MEMKIND_SUCCESS; if (MEMKIND_LIKELY(partition >= 0 && partition < MEMKIND_MAX_KIND && memkind_registry_g.partition_map[partition] != NULL)) { *kind = memkind_registry_g.partition_map[partition]; } else { *kind = NULL; err = MEMKIND_ERROR_UNAVAILABLE; } return err; } MEMKIND_EXPORT int memkind_get_kind_by_partition(int partition, struct memkind **kind) { return memkind_get_kind_by_partition_internal(partition, kind); } MEMKIND_EXPORT int memkind_update_cached_stats(void) { return heap_manager_update_cached_stats(); } MEMKIND_EXPORT void *memkind_defrag_reallocate(memkind_t kind, void *ptr) { if (!kind) { return heap_manager_defrag_reallocate(ptr); } else { return kind->ops->defrag_reallocate(kind, ptr); } } MEMKIND_EXPORT int memkind_get_stat(memkind_t kind, memkind_stat_type stat, size_t *value) { if (MEMKIND_UNLIKELY(stat >= MEMKIND_STAT_TYPE_MAX_VALUE)) { log_err("Unrecognized type of memory statistic %d", stat); return MEMKIND_ERROR_INVALID; } if (!kind) { return heap_manager_get_stat(stat, value); } else { return kind->ops->get_stat(kind, stat, value); } }