|
Packit |
345191 |
/*
|
|
Packit |
345191 |
* Copyright (C) 2014 - 2019 Intel Corporation.
|
|
Packit |
345191 |
* All rights reserved.
|
|
Packit |
345191 |
*
|
|
Packit |
345191 |
* Redistribution and use in source and binary forms, with or without
|
|
Packit |
345191 |
* modification, are permitted provided that the following conditions are met:
|
|
Packit |
345191 |
* 1. Redistributions of source code must retain the above copyright notice(s),
|
|
Packit |
345191 |
* this list of conditions and the following disclaimer.
|
|
Packit |
345191 |
* 2. Redistributions in binary form must reproduce the above copyright notice(s),
|
|
Packit |
345191 |
* this list of conditions and the following disclaimer in the documentation
|
|
Packit |
345191 |
* and/or other materials provided with the distribution.
|
|
Packit |
345191 |
*
|
|
Packit |
345191 |
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY EXPRESS
|
|
Packit |
345191 |
* OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
|
|
Packit |
345191 |
* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
|
|
Packit |
345191 |
* EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY DIRECT, INDIRECT,
|
|
Packit |
345191 |
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
|
Packit |
345191 |
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
|
|
Packit |
345191 |
* PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
|
|
Packit |
345191 |
* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
|
|
Packit |
345191 |
* OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
|
|
Packit |
345191 |
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
Packit |
345191 |
*/
|
|
Packit |
345191 |
|
|
Packit |
345191 |
#include <hbwmalloc.h>
|
|
Packit |
345191 |
#include <memkind.h>
|
|
Packit |
345191 |
#include <memkind/internal/memkind_private.h>
|
|
Packit |
345191 |
#include <memkind/internal/memkind_hbw.h>
|
|
Packit |
345191 |
#include <memkind/internal/memkind_log.h>
|
|
Packit |
345191 |
|
|
Packit |
345191 |
#include <stdlib.h>
|
|
Packit |
345191 |
#include <stdio.h>
|
|
Packit |
345191 |
#include <pthread.h>
|
|
Packit |
345191 |
#include <errno.h>
|
|
Packit |
345191 |
#include <numa.h>
|
|
Packit |
345191 |
#include <numaif.h>
|
|
Packit |
345191 |
#include <unistd.h>
|
|
Packit |
345191 |
#include <stdint.h>
|
|
Packit |
345191 |
|
|
Packit |
345191 |
static hbw_policy_t hbw_policy_g = HBW_POLICY_PREFERRED;
|
|
Packit |
345191 |
static pthread_once_t hbw_policy_once_g = PTHREAD_ONCE_INIT;
|
|
Packit |
345191 |
|
|
Packit |
345191 |
static void hbw_policy_bind_init(void)
|
|
Packit |
345191 |
{
|
|
Packit |
345191 |
hbw_policy_g = HBW_POLICY_BIND;
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
|
|
Packit |
345191 |
static void hbw_policy_bind_all_init(void)
|
|
Packit |
345191 |
{
|
|
Packit |
345191 |
hbw_policy_g = HBW_POLICY_BIND_ALL;
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
|
|
Packit |
345191 |
static void hbw_policy_preferred_init(void)
|
|
Packit |
345191 |
{
|
|
Packit |
345191 |
hbw_policy_g = HBW_POLICY_PREFERRED;
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
|
|
Packit |
345191 |
static void hbw_policy_interleave_init(void)
|
|
Packit |
345191 |
{
|
|
Packit |
345191 |
hbw_policy_g = HBW_POLICY_INTERLEAVE;
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
|
|
Packit |
345191 |
// This function is intended to be called once per pagesize
|
|
Packit |
345191 |
// Getting kind should be done using hbw_get_kind() defined below
|
|
Packit |
345191 |
static memkind_t hbw_choose_kind(hbw_pagesize_t pagesize)
|
|
Packit |
345191 |
{
|
|
Packit |
345191 |
memkind_t result = NULL;
|
|
Packit |
345191 |
|
|
Packit |
345191 |
hbw_set_policy(hbw_policy_g);
|
|
Packit |
345191 |
|
|
Packit |
345191 |
int policy = hbw_get_policy();
|
|
Packit |
345191 |
|
|
Packit |
345191 |
// PREFERRED policy have separate handling cause it can fallback
|
|
Packit |
345191 |
// to non-HBW kinds in case of HBW absence
|
|
Packit |
345191 |
if (policy != HBW_POLICY_PREFERRED ) {
|
|
Packit |
345191 |
switch (pagesize) {
|
|
Packit |
345191 |
case HBW_PAGESIZE_2MB:
|
|
Packit |
345191 |
if(policy == HBW_POLICY_BIND_ALL) {
|
|
Packit |
345191 |
result = MEMKIND_HBW_ALL_HUGETLB;
|
|
Packit |
345191 |
} else {
|
|
Packit |
345191 |
result = MEMKIND_HBW_HUGETLB;
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
break;
|
|
Packit |
345191 |
case HBW_PAGESIZE_1GB:
|
|
Packit |
345191 |
case HBW_PAGESIZE_1GB_STRICT:
|
|
Packit |
345191 |
result = MEMKIND_HBW_GBTLB;
|
|
Packit |
345191 |
break;
|
|
Packit |
345191 |
default:
|
|
Packit |
345191 |
if (policy == HBW_POLICY_BIND) {
|
|
Packit |
345191 |
result = MEMKIND_HBW;
|
|
Packit |
345191 |
} else if (policy == HBW_POLICY_BIND_ALL) {
|
|
Packit |
345191 |
result = MEMKIND_HBW_ALL;
|
|
Packit |
345191 |
} else {
|
|
Packit |
345191 |
result = MEMKIND_HBW_INTERLEAVE;
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
break;
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
} else if (memkind_check_available(MEMKIND_HBW) == 0) {
|
|
Packit |
345191 |
switch (pagesize) {
|
|
Packit |
345191 |
case HBW_PAGESIZE_2MB:
|
|
Packit |
345191 |
result = MEMKIND_HBW_PREFERRED_HUGETLB;
|
|
Packit |
345191 |
break;
|
|
Packit |
345191 |
case HBW_PAGESIZE_1GB:
|
|
Packit |
345191 |
case HBW_PAGESIZE_1GB_STRICT:
|
|
Packit |
345191 |
result = MEMKIND_HBW_PREFERRED_GBTLB;
|
|
Packit |
345191 |
break;
|
|
Packit |
345191 |
default:
|
|
Packit |
345191 |
result = MEMKIND_HBW_PREFERRED;
|
|
Packit |
345191 |
break;
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
} else {
|
|
Packit |
345191 |
switch (pagesize) {
|
|
Packit |
345191 |
case HBW_PAGESIZE_2MB:
|
|
Packit |
345191 |
result = MEMKIND_HUGETLB;
|
|
Packit |
345191 |
break;
|
|
Packit |
345191 |
case HBW_PAGESIZE_1GB:
|
|
Packit |
345191 |
case HBW_PAGESIZE_1GB_STRICT:
|
|
Packit |
345191 |
result = MEMKIND_GBTLB;
|
|
Packit |
345191 |
break;
|
|
Packit |
345191 |
default:
|
|
Packit |
345191 |
result = MEMKIND_DEFAULT;
|
|
Packit |
345191 |
break;
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
return result;
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
|
|
Packit |
345191 |
static memkind_t pagesize_kind[HBW_PAGESIZE_MAX_VALUE];
|
|
Packit |
345191 |
static inline memkind_t hbw_get_kind(hbw_pagesize_t pagesize)
|
|
Packit |
345191 |
{
|
|
Packit |
345191 |
if(pagesize_kind[pagesize] == NULL) {
|
|
Packit |
345191 |
pagesize_kind[pagesize] = hbw_choose_kind(pagesize);
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
return pagesize_kind[pagesize];
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
|
|
Packit |
345191 |
|
|
Packit |
345191 |
MEMKIND_EXPORT hbw_policy_t hbw_get_policy(void)
|
|
Packit |
345191 |
{
|
|
Packit |
345191 |
return hbw_policy_g;
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
|
|
Packit |
345191 |
MEMKIND_EXPORT int hbw_set_policy(hbw_policy_t mode)
|
|
Packit |
345191 |
{
|
|
Packit |
345191 |
switch(mode) {
|
|
Packit |
345191 |
case HBW_POLICY_PREFERRED:
|
|
Packit |
345191 |
pthread_once(&hbw_policy_once_g, hbw_policy_preferred_init);
|
|
Packit |
345191 |
break;
|
|
Packit |
345191 |
case HBW_POLICY_BIND:
|
|
Packit |
345191 |
pthread_once(&hbw_policy_once_g, hbw_policy_bind_init);
|
|
Packit |
345191 |
break;
|
|
Packit |
345191 |
case HBW_POLICY_BIND_ALL:
|
|
Packit |
345191 |
pthread_once(&hbw_policy_once_g, hbw_policy_bind_all_init);
|
|
Packit |
345191 |
break;
|
|
Packit |
345191 |
case HBW_POLICY_INTERLEAVE:
|
|
Packit |
345191 |
pthread_once(&hbw_policy_once_g, hbw_policy_interleave_init);
|
|
Packit |
345191 |
break;
|
|
Packit |
345191 |
default:
|
|
Packit |
345191 |
return EINVAL;
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
|
|
Packit |
345191 |
if (mode != hbw_policy_g) {
|
|
Packit |
345191 |
return EPERM;
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
|
|
Packit |
345191 |
return 0;
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
|
|
Packit |
345191 |
MEMKIND_EXPORT int hbw_check_available(void)
|
|
Packit |
345191 |
{
|
|
Packit |
345191 |
return (memkind_check_available(MEMKIND_HBW) == 0) ? 0 : ENODEV;
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
|
|
Packit |
345191 |
static inline void hbw_touch_page(void *addr)
|
|
Packit |
345191 |
{
|
|
Packit |
345191 |
volatile char *temp_ptr = (volatile char *) addr;
|
|
Packit |
345191 |
char value = temp_ptr[0];
|
|
Packit |
345191 |
temp_ptr[0] = value;
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
|
|
Packit |
345191 |
MEMKIND_EXPORT int hbw_verify_memory_region(void *addr, size_t size, int flags)
|
|
Packit |
345191 |
{
|
|
Packit |
345191 |
/*
|
|
Packit |
345191 |
* if size is invalid, flags have unsupported bit set or if addr is NULL.
|
|
Packit |
345191 |
*/
|
|
Packit |
345191 |
if (addr == NULL || size == 0 || flags & ~HBW_TOUCH_PAGES) {
|
|
Packit |
345191 |
return EINVAL;
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
|
|
Packit |
345191 |
/*
|
|
Packit |
345191 |
* 4KB is the smallest pagesize. When pagesize is bigger, pages are verified more than once
|
|
Packit |
345191 |
*/
|
|
Packit |
345191 |
const size_t page_size = sysconf(_SC_PAGESIZE);
|
|
Packit |
345191 |
const size_t page_mask = ~(page_size-1);
|
|
Packit |
345191 |
|
|
Packit |
345191 |
/*
|
|
Packit |
345191 |
* block size should be power of two to enable compiler optimizations
|
|
Packit |
345191 |
*/
|
|
Packit |
345191 |
const unsigned block_size = 64;
|
|
Packit |
345191 |
|
|
Packit |
345191 |
char *end = addr + size;
|
|
Packit |
345191 |
char *aligned_beg = (char *)((uintptr_t)addr & page_mask);
|
|
Packit |
345191 |
nodemask_t nodemask;
|
|
Packit |
345191 |
struct bitmask expected_nodemask = {NUMA_NUM_NODES, nodemask.n};
|
|
Packit |
345191 |
|
|
Packit |
345191 |
memkind_hbw_all_get_mbind_nodemask(NULL, expected_nodemask.maskp,
|
|
Packit |
345191 |
expected_nodemask.size);
|
|
Packit |
345191 |
|
|
Packit |
345191 |
while(aligned_beg < end) {
|
|
Packit |
345191 |
int nodes[block_size];
|
|
Packit |
345191 |
void *pages[block_size];
|
|
Packit |
345191 |
int i = 0, page_count = 0;
|
|
Packit |
345191 |
char *iter_end = aligned_beg + block_size*page_size;
|
|
Packit |
345191 |
|
|
Packit |
345191 |
if (iter_end > end) {
|
|
Packit |
345191 |
iter_end = end;
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
|
|
Packit |
345191 |
while (aligned_beg < iter_end) {
|
|
Packit |
345191 |
if (flags & HBW_TOUCH_PAGES) {
|
|
Packit |
345191 |
hbw_touch_page(aligned_beg);
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
pages[page_count++] = aligned_beg;
|
|
Packit |
345191 |
aligned_beg += page_size;
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
|
|
Packit |
345191 |
if (move_pages(0, page_count, pages, NULL, nodes, MPOL_MF_MOVE)) {
|
|
Packit |
345191 |
return EFAULT;
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
|
|
Packit |
345191 |
for (i = 0; i < page_count; i++) {
|
|
Packit |
345191 |
/*
|
|
Packit |
345191 |
* negative value of nodes[i] indicates that move_pages could not establish
|
|
Packit |
345191 |
* page location, e.g. addr is not pointing to valid virtual mapping
|
|
Packit |
345191 |
*/
|
|
Packit |
345191 |
if(nodes[i] < 0) {
|
|
Packit |
345191 |
return -1;
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
/*
|
|
Packit |
345191 |
* if nodes[i] is not present in expected_nodemask then
|
|
Packit |
345191 |
* physical memory backing page is not hbw
|
|
Packit |
345191 |
*/
|
|
Packit |
345191 |
if (!numa_bitmask_isbitset(&expected_nodemask, nodes[i])) {
|
|
Packit |
345191 |
return -1;
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
|
|
Packit |
345191 |
return 0;
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
|
|
Packit |
345191 |
MEMKIND_EXPORT void *hbw_malloc(size_t size)
|
|
Packit |
345191 |
{
|
|
Packit |
345191 |
return memkind_malloc(hbw_get_kind(HBW_PAGESIZE_4KB), size);
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
|
|
Packit |
345191 |
MEMKIND_EXPORT void *hbw_calloc(size_t num, size_t size)
|
|
Packit |
345191 |
{
|
|
Packit |
345191 |
return memkind_calloc(hbw_get_kind(HBW_PAGESIZE_4KB), num, size);
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
|
|
Packit |
345191 |
MEMKIND_EXPORT int hbw_posix_memalign(void **memptr, size_t alignment,
|
|
Packit |
345191 |
size_t size)
|
|
Packit |
345191 |
{
|
|
Packit |
345191 |
return memkind_posix_memalign(hbw_get_kind(HBW_PAGESIZE_4KB), memptr, alignment,
|
|
Packit |
345191 |
size);
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
|
|
Packit |
345191 |
MEMKIND_EXPORT int hbw_posix_memalign_psize(void **memptr, size_t alignment,
|
|
Packit |
345191 |
size_t size,
|
|
Packit |
345191 |
hbw_pagesize_t pagesize)
|
|
Packit |
345191 |
{
|
|
Packit |
345191 |
if (pagesize == HBW_PAGESIZE_1GB_STRICT && size % (1 << 30)) {
|
|
Packit |
345191 |
return EINVAL;
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
|
|
Packit |
345191 |
if((pagesize == HBW_PAGESIZE_2MB ||
|
|
Packit |
345191 |
pagesize == HBW_PAGESIZE_1GB_STRICT ||
|
|
Packit |
345191 |
pagesize == HBW_PAGESIZE_1GB) &&
|
|
Packit |
345191 |
hbw_get_policy() == HBW_POLICY_INTERLEAVE) {
|
|
Packit |
345191 |
|
|
Packit |
345191 |
log_err("HBW_POLICY_INTERLEAVE is unsupported with used page size!");
|
|
Packit |
345191 |
return EINVAL;
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
|
|
Packit |
345191 |
return memkind_posix_memalign(hbw_get_kind(pagesize), memptr, alignment, size);
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
|
|
Packit |
345191 |
MEMKIND_EXPORT void *hbw_realloc(void *ptr, size_t size)
|
|
Packit |
345191 |
{
|
|
Packit |
345191 |
return memkind_realloc(hbw_get_kind(HBW_PAGESIZE_4KB), ptr, size);
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
|
|
Packit |
345191 |
MEMKIND_EXPORT void hbw_free(void *ptr)
|
|
Packit |
345191 |
{
|
|
Packit |
345191 |
memkind_free(0, ptr);
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
|
|
Packit |
345191 |
MEMKIND_EXPORT size_t hbw_malloc_usable_size(void *ptr)
|
|
Packit |
345191 |
{
|
|
Packit |
345191 |
return memkind_malloc_usable_size(0, ptr);
|
|
Packit |
345191 |
}
|