|
Packit |
345191 |
/*
|
|
Packit |
345191 |
* Copyright (C) 2014 - 2019 Intel Corporation.
|
|
Packit |
345191 |
* All rights reserved.
|
|
Packit |
345191 |
*
|
|
Packit |
345191 |
* Redistribution and use in source and binary forms, with or without
|
|
Packit |
345191 |
* modification, are permitted provided that the following conditions are met:
|
|
Packit |
345191 |
* 1. Redistributions of source code must retain the above copyright notice(s),
|
|
Packit |
345191 |
* this list of conditions and the following disclaimer.
|
|
Packit |
345191 |
* 2. Redistributions in binary form must reproduce the above copyright notice(s),
|
|
Packit |
345191 |
* this list of conditions and the following disclaimer in the documentation
|
|
Packit |
345191 |
* and/or other materials provided with the distribution.
|
|
Packit |
345191 |
*
|
|
Packit |
345191 |
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY EXPRESS
|
|
Packit |
345191 |
* OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
|
|
Packit |
345191 |
* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
|
|
Packit |
345191 |
* EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY DIRECT, INDIRECT,
|
|
Packit |
345191 |
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
|
Packit |
345191 |
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
|
|
Packit |
345191 |
* PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
|
|
Packit |
345191 |
* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
|
|
Packit |
345191 |
* OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
|
|
Packit |
345191 |
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
Packit |
345191 |
*/
|
|
Packit |
345191 |
|
|
Packit |
345191 |
#include <memkind/internal/memkind_arena.h>
|
|
Packit |
345191 |
#include <memkind/internal/memkind_default.h>
|
|
Packit |
345191 |
#include <memkind/internal/memkind_private.h>
|
|
Packit |
345191 |
#include <memkind/internal/memkind_log.h>
|
|
Packit |
345191 |
#include <memkind/internal/tbb_wrapper.h>
|
|
Packit |
345191 |
#include <memkind/internal/heap_manager.h>
|
|
Packit |
345191 |
|
|
Packit |
345191 |
#include <numa.h>
|
|
Packit |
345191 |
#include <numaif.h>
|
|
Packit |
345191 |
#include <sys/mman.h>
|
|
Packit |
345191 |
#include <errno.h>
|
|
Packit |
345191 |
#include <jemalloc/jemalloc.h>
|
|
Packit |
345191 |
#include <stdint.h>
|
|
Packit |
345191 |
|
|
Packit |
345191 |
#include "config.h"
|
|
Packit |
345191 |
|
|
Packit |
345191 |
#ifndef MADV_NOHUGEPAGE
|
|
Packit |
345191 |
#define MADV_NOHUGEPAGE 15
|
|
Packit |
345191 |
#endif
|
|
Packit |
345191 |
|
|
Packit |
345191 |
static int memkind_default_get_kind_stat(struct memkind *kind,
|
|
Packit |
345191 |
memkind_stat_type stat, size_t *value)
|
|
Packit |
345191 |
{
|
|
Packit |
345191 |
return memkind_arena_get_stat_with_check_init(kind, stat, true, value);
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
|
|
Packit |
345191 |
MEMKIND_EXPORT struct memkind_ops MEMKIND_DEFAULT_OPS = {
|
|
Packit |
345191 |
.create = memkind_default_create,
|
|
Packit |
345191 |
.destroy = memkind_default_destroy,
|
|
Packit |
345191 |
.malloc = memkind_default_malloc,
|
|
Packit |
345191 |
.calloc = memkind_default_calloc,
|
|
Packit |
345191 |
.posix_memalign = memkind_default_posix_memalign,
|
|
Packit |
345191 |
.realloc = memkind_default_realloc,
|
|
Packit |
345191 |
.free = memkind_default_free,
|
|
Packit |
345191 |
.init_once = memkind_default_init_once,
|
|
Packit |
345191 |
.malloc_usable_size = memkind_default_malloc_usable_size,
|
|
Packit |
345191 |
.finalize = memkind_default_destroy,
|
|
Packit |
345191 |
.get_stat = memkind_default_get_kind_stat,
|
|
Packit |
345191 |
.defrag_reallocate = memkind_arena_defrag_reallocate
|
|
Packit |
345191 |
};
|
|
Packit |
345191 |
|
|
Packit |
345191 |
MEMKIND_EXPORT int memkind_default_create(struct memkind *kind,
|
|
Packit |
345191 |
struct memkind_ops *ops, const char *name)
|
|
Packit |
345191 |
{
|
|
Packit |
345191 |
int err = 0;
|
|
Packit |
345191 |
|
|
Packit |
345191 |
kind->ops = ops;
|
|
Packit |
345191 |
if (strlen(name) >= MEMKIND_NAME_LENGTH_PRIV) {
|
|
Packit |
345191 |
kind->name[0] = '\0';
|
|
Packit |
345191 |
err = MEMKIND_ERROR_INVALID;
|
|
Packit |
345191 |
} else {
|
|
Packit |
345191 |
strcpy(kind->name, name);
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
return err;
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
|
|
Packit |
345191 |
MEMKIND_EXPORT int memkind_default_destroy(struct memkind *kind)
|
|
Packit |
345191 |
{
|
|
Packit |
345191 |
return 0;
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
|
|
Packit |
345191 |
MEMKIND_EXPORT void *memkind_default_malloc(struct memkind *kind, size_t size)
|
|
Packit |
345191 |
{
|
|
Packit |
345191 |
if(MEMKIND_UNLIKELY(size_out_of_bounds(size))) {
|
|
Packit |
345191 |
return NULL;
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
return jemk_malloc(size);
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
|
|
Packit |
345191 |
MEMKIND_EXPORT void *memkind_default_calloc(struct memkind *kind, size_t num,
|
|
Packit |
345191 |
size_t size)
|
|
Packit |
345191 |
{
|
|
Packit |
345191 |
if(MEMKIND_UNLIKELY(size_out_of_bounds(num) || size_out_of_bounds(size))) {
|
|
Packit |
345191 |
return NULL;
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
return jemk_calloc(num, size);
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
|
|
Packit |
345191 |
MEMKIND_EXPORT int memkind_default_posix_memalign(struct memkind *kind,
|
|
Packit |
345191 |
void **memptr, size_t alignment, size_t size)
|
|
Packit |
345191 |
{
|
|
Packit |
345191 |
if(MEMKIND_UNLIKELY(size_out_of_bounds(size))) {
|
|
Packit |
345191 |
*memptr = NULL;
|
|
Packit |
345191 |
return 0;
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
return jemk_posix_memalign(memptr, alignment, size);
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
|
|
Packit |
345191 |
MEMKIND_EXPORT void *memkind_default_realloc(struct memkind *kind, void *ptr,
|
|
Packit |
345191 |
size_t size)
|
|
Packit |
345191 |
{
|
|
Packit |
345191 |
if(MEMKIND_UNLIKELY(size_out_of_bounds(size))) {
|
|
Packit |
345191 |
jemk_free(ptr);
|
|
Packit |
345191 |
return NULL;
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
return jemk_realloc(ptr, size);
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
|
|
Packit |
345191 |
MEMKIND_EXPORT void memkind_default_free(struct memkind *kind, void *ptr)
|
|
Packit |
345191 |
{
|
|
Packit |
345191 |
jemk_free(ptr);
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
|
|
Packit |
345191 |
MEMKIND_EXPORT size_t memkind_default_malloc_usable_size(struct memkind *kind,
|
|
Packit |
345191 |
void *ptr)
|
|
Packit |
345191 |
{
|
|
Packit |
345191 |
return jemk_malloc_usable_size(ptr);
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
|
|
Packit |
345191 |
MEMKIND_EXPORT void *memkind_default_mmap(struct memkind *kind, void *addr,
|
|
Packit |
345191 |
size_t size)
|
|
Packit |
345191 |
{
|
|
Packit |
345191 |
void *result = MAP_FAILED;
|
|
Packit |
345191 |
int err = 0;
|
|
Packit |
345191 |
int flags;
|
|
Packit |
345191 |
|
|
Packit |
345191 |
if (kind->ops->get_mmap_flags) {
|
|
Packit |
345191 |
err = kind->ops->get_mmap_flags(kind, &flags);
|
|
Packit |
345191 |
} else {
|
|
Packit |
345191 |
err = memkind_default_get_mmap_flags(kind, &flags);
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
if (MEMKIND_LIKELY(!err)) {
|
|
Packit |
345191 |
result = mmap(addr, size, PROT_READ | PROT_WRITE, flags, -1, 0);
|
|
Packit |
345191 |
if (result == MAP_FAILED) {
|
|
Packit |
345191 |
log_err("syscall mmap() returned: %p", result);
|
|
Packit |
345191 |
return result;
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
if (kind->ops->mbind) {
|
|
Packit |
345191 |
err = kind->ops->mbind(kind, result, size);
|
|
Packit |
345191 |
if (err) {
|
|
Packit |
345191 |
munmap(result, size);
|
|
Packit |
345191 |
result = MAP_FAILED;
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
if (kind->ops->madvise) {
|
|
Packit |
345191 |
err = kind->ops->madvise(kind, result, size);
|
|
Packit |
345191 |
if (err) {
|
|
Packit |
345191 |
munmap(result, size);
|
|
Packit |
345191 |
result = MAP_FAILED;
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
return result;
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
|
|
Packit |
345191 |
MEMKIND_EXPORT int memkind_nohugepage_madvise(struct memkind *kind, void *addr,
|
|
Packit |
345191 |
size_t size)
|
|
Packit |
345191 |
{
|
|
Packit |
345191 |
int err = madvise(addr, size, MADV_NOHUGEPAGE);
|
|
Packit |
345191 |
|
|
Packit |
345191 |
//checking if EINVAL was returned due to lack of THP support in kernel
|
|
Packit |
345191 |
if ((err == EINVAL) && (((uintptr_t) addr & (uintptr_t) 0xfff) == 0) &&
|
|
Packit |
345191 |
(size > 0)) {
|
|
Packit |
345191 |
return 0;
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
if (MEMKIND_UNLIKELY(err)) {
|
|
Packit |
345191 |
log_err("syscall madvise() returned: %d", err);
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
return err;
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
|
|
Packit |
345191 |
MEMKIND_EXPORT int memkind_default_mbind(struct memkind *kind, void *ptr,
|
|
Packit |
345191 |
size_t size)
|
|
Packit |
345191 |
{
|
|
Packit |
345191 |
nodemask_t nodemask;
|
|
Packit |
345191 |
int err = 0;
|
|
Packit |
345191 |
int mode;
|
|
Packit |
345191 |
|
|
Packit |
345191 |
if (MEMKIND_UNLIKELY(kind->ops->get_mbind_nodemask == NULL ||
|
|
Packit |
345191 |
kind->ops->get_mbind_mode == NULL)) {
|
|
Packit |
345191 |
log_err("memkind_ops->mbind_mode or memkind_ops->bind_nodemask is NULL.");
|
|
Packit |
345191 |
return MEMKIND_ERROR_BADOPS;
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
err = kind->ops->get_mbind_nodemask(kind, nodemask.n, NUMA_NUM_NODES);
|
|
Packit |
345191 |
if (MEMKIND_UNLIKELY(err)) {
|
|
Packit |
345191 |
return err;
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
err = kind->ops->get_mbind_mode(kind, &mode);
|
|
Packit |
345191 |
if (MEMKIND_UNLIKELY(err)) {
|
|
Packit |
345191 |
return err;
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
err = mbind(ptr, size, mode, nodemask.n, NUMA_NUM_NODES, 0);
|
|
Packit |
345191 |
if (MEMKIND_UNLIKELY(err)) {
|
|
Packit |
345191 |
log_err("syscall mbind() returned: %d", err);
|
|
Packit |
345191 |
return MEMKIND_ERROR_MBIND;
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
return err;
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
|
|
Packit |
345191 |
MEMKIND_EXPORT int memkind_default_get_mmap_flags(struct memkind *kind,
|
|
Packit |
345191 |
int *flags)
|
|
Packit |
345191 |
{
|
|
Packit |
345191 |
*flags = MAP_PRIVATE | MAP_ANONYMOUS;
|
|
Packit |
345191 |
return 0;
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
|
|
Packit |
345191 |
MEMKIND_EXPORT int memkind_default_get_mbind_nodemask(struct memkind *kind,
|
|
Packit |
345191 |
unsigned long *nodemask,
|
|
Packit |
345191 |
unsigned long maxnode)
|
|
Packit |
345191 |
{
|
|
Packit |
345191 |
struct bitmask nodemask_bm = {maxnode, nodemask};
|
|
Packit |
345191 |
copy_bitmask_to_bitmask(numa_all_nodes_ptr, &nodemask_bm);
|
|
Packit |
345191 |
return 0;
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
|
|
Packit |
345191 |
MEMKIND_EXPORT int memkind_default_get_mbind_mode(struct memkind *kind,
|
|
Packit |
345191 |
int *mode)
|
|
Packit |
345191 |
{
|
|
Packit |
345191 |
*mode = MPOL_BIND;
|
|
Packit |
345191 |
return 0;
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
|
|
Packit |
345191 |
MEMKIND_EXPORT int memkind_preferred_get_mbind_mode(struct memkind *kind,
|
|
Packit |
345191 |
int *mode)
|
|
Packit |
345191 |
{
|
|
Packit |
345191 |
*mode = MPOL_PREFERRED;
|
|
Packit |
345191 |
return 0;
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
|
|
Packit |
345191 |
MEMKIND_EXPORT int memkind_interleave_get_mbind_mode(struct memkind *kind,
|
|
Packit |
345191 |
int *mode)
|
|
Packit |
345191 |
{
|
|
Packit |
345191 |
*mode = MPOL_INTERLEAVE;
|
|
Packit |
345191 |
return 0;
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
|
|
Packit |
345191 |
MEMKIND_EXPORT int memkind_posix_check_alignment(struct memkind *kind,
|
|
Packit |
345191 |
size_t alignment)
|
|
Packit |
345191 |
{
|
|
Packit |
345191 |
int err = 0;
|
|
Packit |
345191 |
if ((alignment < sizeof(void *)) ||
|
|
Packit |
345191 |
(((alignment - 1) & alignment) != 0)) {
|
|
Packit |
345191 |
err = EINVAL;
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
return err;
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
|
|
Packit |
345191 |
MEMKIND_EXPORT void memkind_default_init_once(void)
|
|
Packit |
345191 |
{
|
|
Packit |
345191 |
heap_manager_init(MEMKIND_DEFAULT);
|
|
Packit |
345191 |
}
|