Blame hugeutils.c

Packit Service b439df
/*
Packit Service b439df
 * libhugetlbfs - Easy use of Linux hugepages
Packit Service b439df
 * Copyright (C) 2005-2006 David Gibson & Adam Litke, IBM Corporation.
Packit Service b439df
 *
Packit Service b439df
 * This library is free software; you can redistribute it and/or
Packit Service b439df
 * modify it under the terms of the GNU Lesser General Public License
Packit Service b439df
 * as published by the Free Software Foundation; either version 2.1 of
Packit Service b439df
 * the License, or (at your option) any later version.
Packit Service b439df
 *
Packit Service b439df
 * This library is distributed in the hope that it will be useful, but
Packit Service b439df
 * WITHOUT ANY WARRANTY; without even the implied warranty of
Packit Service b439df
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
Packit Service b439df
 * Lesser General Public License for more details.
Packit Service b439df
 *
Packit Service b439df
 * You should have received a copy of the GNU Lesser General Public
Packit Service b439df
 * License along with this library; if not, write to the Free Software
Packit Service b439df
 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
Packit Service b439df
 */
Packit Service b439df
Packit Service b439df
#define _LARGEFILE64_SOURCE /* Need this for statfs64 */
Packit Service b439df
#define _GNU_SOURCE
Packit Service b439df
#include <dlfcn.h>
Packit Service b439df
#include <features.h>
Packit Service b439df
Packit Service b439df
#include <stdio.h>
Packit Service b439df
#include <stdlib.h>
Packit Service b439df
#include <stdint.h>
Packit Service b439df
#include <errno.h>
Packit Service b439df
#include <limits.h>
Packit Service b439df
#include <string.h>
Packit Service b439df
#include <ctype.h>
Packit Service b439df
#include <signal.h>
Packit Service b439df
#include <dirent.h>
Packit Service b439df
Packit Service b439df
#include <unistd.h>
Packit Service b439df
#include <fcntl.h>
Packit Service b439df
#include <sys/vfs.h>
Packit Service b439df
#include <sys/statfs.h>
Packit Service b439df
#include <sys/types.h>
Packit Service b439df
#include <sys/mman.h>
Packit Service b439df
#include <sys/file.h>
Packit Service b439df
#include <sys/uio.h>
Packit Service b439df
#include <sys/syscall.h>
Packit Service b439df
#include <linux/types.h>
Packit Service b439df
#include <linux/unistd.h>
Packit Service b439df
#include <dirent.h>
Packit Service b439df
Packit Service b439df
#include "libhugetlbfs_internal.h"
Packit Service b439df
#include "hugetlbfs.h"
Packit Service b439df
Packit Service b439df
struct libhugeopts_t __hugetlb_opts;
Packit Service b439df
Packit Service b439df
static int hugepagesize_errno; /* = 0 */
Packit Service b439df
Packit Service b439df
#define MAX_HPAGE_SIZES 10
Packit Service b439df
static struct hpage_size hpage_sizes[MAX_HPAGE_SIZES];
Packit Service b439df
static int nr_hpage_sizes;
Packit Service b439df
static int hpage_sizes_default_idx = -1;
Packit Service b439df
Packit Service b439df
static long default_size;
Packit Service b439df
Packit Service b439df
/********************************************************************/
Packit Service b439df
/* Internal functions                                               */
Packit Service b439df
/********************************************************************/
Packit Service b439df
Packit Service b439df
/*
Packit Service b439df
 * Lookup the kernel default page size.
Packit Service b439df
 */
Packit Service b439df
long kernel_default_hugepage_size()
Packit Service b439df
{
Packit Service b439df
	if (default_size == 0) {
Packit Service b439df
		default_size = file_read_ulong(MEMINFO, "Hugepagesize:");
Packit Service b439df
		default_size = size_to_smaller_unit(default_size); /* kB to B */	}
Packit Service b439df
	return default_size;
Packit Service b439df
}
Packit Service b439df
void kernel_default_hugepage_size_reset(void)
Packit Service b439df
{
Packit Service b439df
	default_size = 0;
Packit Service b439df
}
Packit Service b439df
Packit Service b439df
#define BUF_SZ 256
Packit Service b439df
#define MEMINFO_SIZE	2048
Packit Service b439df
Packit Service b439df
/*
Packit Service b439df
 * Convert a quantity in a given unit to the next smallest unit by
Packit Service b439df
 * multiplying the quantity by 1024 (eg. convert 1MB to 1024kB).
Packit Service b439df
 * If the conversion would overflow the variable, return ULONGLONG_MAX to
Packit Service b439df
 * signify the error.
Packit Service b439df
 */
Packit Service b439df
unsigned long long size_to_smaller_unit(unsigned long long size)
Packit Service b439df
{
Packit Service b439df
	if (size * 1024 < size)
Packit Service b439df
		return -1;
Packit Service b439df
	else
Packit Service b439df
		return size * 1024;
Packit Service b439df
}
Packit Service b439df
Packit Service b439df
/*
Packit Service b439df
 * Convert a page size string with an optional unit suffix into a page size
Packit Service b439df
 * in bytes.
Packit Service b439df
 *
Packit Service b439df
 * On error, -1 is returned and errno is set appropriately:
Packit Service b439df
 * 	EINVAL		- str could not be parsed or was not greater than zero
Packit Service b439df
 *	EOVERFLOW	- Overflow when converting from the specified units
Packit Service b439df
 */
Packit Service b439df
long parse_page_size(const char *str)
Packit Service b439df
{
Packit Service b439df
	char *pos;
Packit Service b439df
	long size;
Packit Service b439df
Packit Service b439df
	errno = 0;
Packit Service b439df
	size = strtol(str, &pos, 0);
Packit Service b439df
	/* Catch strtoul errors and sizes that overflow the native word size */
Packit Service b439df
	if (errno || str == pos || size <= 0) {
Packit Service b439df
		if (errno == ERANGE)
Packit Service b439df
			errno = EOVERFLOW;
Packit Service b439df
		else
Packit Service b439df
			errno = EINVAL;
Packit Service b439df
		return -1;
Packit Service b439df
	}
Packit Service b439df
Packit Service b439df
	switch (*pos) {
Packit Service b439df
	case 'G':
Packit Service b439df
	case 'g':
Packit Service b439df
		size = size_to_smaller_unit(size);
Packit Service b439df
	case 'M':
Packit Service b439df
	case 'm':
Packit Service b439df
		size = size_to_smaller_unit(size);
Packit Service b439df
	case 'K':
Packit Service b439df
	case 'k':
Packit Service b439df
		size = size_to_smaller_unit(size);
Packit Service b439df
	}
Packit Service b439df
Packit Service b439df
	if (size < 0)
Packit Service b439df
		errno = EOVERFLOW;
Packit Service b439df
	return size;
Packit Service b439df
}
Packit Service b439df
Packit Service b439df
struct hugetlb_pool_counter_info_t {
Packit Service b439df
	char *meminfo_key;
Packit Service b439df
	char *sysfs_file;
Packit Service b439df
};
Packit Service b439df
Packit Service b439df
static struct hugetlb_pool_counter_info_t hugetlb_counter_info[] = {
Packit Service b439df
	[HUGEPAGES_TOTAL] = {
Packit Service b439df
		.meminfo_key	= "HugePages_Total:",
Packit Service b439df
		.sysfs_file	= "nr_hugepages",
Packit Service b439df
	},
Packit Service b439df
	[HUGEPAGES_TOTAL_MEMPOL] = {
Packit Service b439df
		.meminfo_key	= "HugePages_Total:",
Packit Service b439df
		.sysfs_file	= "nr_hugepages_mempolicy",
Packit Service b439df
	},
Packit Service b439df
	[HUGEPAGES_FREE] = {
Packit Service b439df
		.meminfo_key	= "HugePages_Free:",
Packit Service b439df
		.sysfs_file	= "free_hugepages",
Packit Service b439df
	},
Packit Service b439df
	[HUGEPAGES_RSVD] = {
Packit Service b439df
		.meminfo_key	= "HugePages_Rsvd:",
Packit Service b439df
		.sysfs_file	= "resv_hugepages",
Packit Service b439df
	},
Packit Service b439df
	[HUGEPAGES_SURP] = {
Packit Service b439df
		.meminfo_key	= "HugePages_Surp:",
Packit Service b439df
		.sysfs_file	= "surplus_hugepages",
Packit Service b439df
	},
Packit Service b439df
	[HUGEPAGES_OC] = {
Packit Service b439df
		.meminfo_key	= NULL,
Packit Service b439df
		.sysfs_file	= "nr_overcommit_hugepages"
Packit Service b439df
	},
Packit Service b439df
};
Packit Service b439df
Packit Service b439df
/*
Packit Service b439df
 * Read numeric data from raw and tagged kernel status files.  Used to read
Packit Service b439df
 * /proc and /sys data (without a tag) and from /proc/meminfo (with a tag).
Packit Service b439df
 */
Packit Service b439df
long file_read_ulong(char *file, const char *tag)
Packit Service b439df
{
Packit Service b439df
	int fd;
Packit Service b439df
	char buf[MEMINFO_SIZE];
Packit Service b439df
	int len, readerr;
Packit Service b439df
	char *p, *q;
Packit Service b439df
	long val;
Packit Service b439df
Packit Service b439df
	fd = open(file, O_RDONLY);
Packit Service b439df
	if (fd < 0) {
Packit Service b439df
		ERROR("Couldn't open %s: %s\n", file, strerror(errno));
Packit Service b439df
		return -1;
Packit Service b439df
	}
Packit Service b439df
Packit Service b439df
	len = read(fd, buf, sizeof(buf));
Packit Service b439df
	readerr = errno;
Packit Service b439df
	close(fd);
Packit Service b439df
	if (len < 0) {
Packit Service b439df
		ERROR("Error reading %s: %s\n", file, strerror(readerr));
Packit Service b439df
		return -1;
Packit Service b439df
	}
Packit Service b439df
	if (len == sizeof(buf)) {
Packit Service b439df
		ERROR("%s is too large\n", file);
Packit Service b439df
		return -1;
Packit Service b439df
	}
Packit Service b439df
	buf[len] = '\0';
Packit Service b439df
Packit Service b439df
	/* Search for a tag if provided */
Packit Service b439df
	if (tag) {
Packit Service b439df
		p = strstr(buf, tag);
Packit Service b439df
		if (!p)
Packit Service b439df
			return -1; /* looks like the line we want isn't there */
Packit Service b439df
		p += strlen(tag);
Packit Service b439df
	} else
Packit Service b439df
		p = buf;
Packit Service b439df
Packit Service b439df
	val = strtol(p, &q, 0);
Packit Service b439df
	if (! isspace(*q)) {
Packit Service b439df
		ERROR("Couldn't parse %s value\n", file);
Packit Service b439df
		return -1;
Packit Service b439df
	}
Packit Service b439df
Packit Service b439df
	return val;
Packit Service b439df
}
Packit Service b439df
Packit Service b439df
int file_write_ulong(char *file, unsigned long val)
Packit Service b439df
{
Packit Service 724880
	int fd, ret, buflen;
Packit Service 724880
	char buf[20];
Packit Service b439df
Packit Service 724880
	fd = open(file, O_WRONLY);
Packit Service 724880
	if (fd < 0) {
Packit Service b439df
		ERROR("Couldn't open %s: %s\n", file, strerror(errno));
Packit Service b439df
		return -1;
Packit Service b439df
	}
Packit Service b439df
Packit Service 724880
	buflen = sprintf(buf, "%lu", val);
Packit Service 724880
	ret = write(fd, buf, buflen);
Packit Service 724880
	close(fd);
Packit Service b439df
	return ret > 0 ? 0 : -1;
Packit Service b439df
}
Packit Service b439df
Packit Service b439df
Packit Service b439df
/*
Packit Service b439df
 * Return the name of this executable, using buf as temporary space.
Packit Service b439df
 */
Packit Service b439df
#define MAX_EXE 4096
Packit Service b439df
static char *get_exe_name(char *buf, int size)
Packit Service b439df
{
Packit Service b439df
	char *p;
Packit Service b439df
	int fd;
Packit Service b439df
	ssize_t nread;
Packit Service b439df
Packit Service b439df
	buf[0] = 0;
Packit Service b439df
	fd = open("/proc/self/cmdline", O_RDONLY);
Packit Service b439df
	if (fd < 0) {
Packit Service b439df
		WARNING("Unable to open cmdline, no exe name\n");
Packit Service b439df
		return buf;
Packit Service b439df
	}
Packit Service b439df
	nread = read(fd, buf, size-1);
Packit Service b439df
	close(fd);
Packit Service b439df
Packit Service b439df
	if (nread < 0) {
Packit Service b439df
		WARNING("Error %d reading cmdline, no exe name\n", errno);
Packit Service b439df
		return buf;
Packit Service b439df
	}
Packit Service b439df
	if (nread == 0) {
Packit Service b439df
		WARNING("Read zero bytes from cmdline, no exe name\n");
Packit Service b439df
		return buf;
Packit Service b439df
	}
Packit Service b439df
Packit Service b439df
	buf[nread] = 0; /* make sure we're null terminated */
Packit Service b439df
	/*
Packit Service b439df
	 * Take advantage of cmdline being a series of null-terminated
Packit Service b439df
	 * strings.  The first string is the path to the executable in
Packit Service b439df
	 * the form:
Packit Service b439df
	 *
Packit Service b439df
	 *      /path/to/exe
Packit Service b439df
	 *
Packit Service b439df
	 * The exe name starts one character after the last '/'.
Packit Service b439df
	 */
Packit Service b439df
	p = strrchr(buf, '/');
Packit Service b439df
	if (!p)
Packit Service b439df
		return buf;
Packit Service b439df
	return p + 1;           /* skip over "/" */
Packit Service b439df
}
Packit Service b439df
Packit Service b439df
Packit Service b439df
/*
Packit Service b439df
 * Reads the contents of hugetlb environment variables and save their
Packit Service b439df
 * values for later use.
Packit Service b439df
 */
Packit Service b439df
void hugetlbfs_setup_env()
Packit Service b439df
{
Packit Service b439df
	char *env;
Packit Service b439df
Packit Service b439df
	__hugetlb_opts.min_copy = true;
Packit Service b439df
Packit Service b439df
	env = getenv("HUGETLB_VERBOSE");
Packit Service b439df
	if (env)
Packit Service b439df
		__hugetlbfs_verbose = atoi(env);
Packit Service b439df
Packit Service b439df
	env = getenv("HUGETLB_DEBUG");
Packit Service b439df
	if (env) {
Packit Service b439df
		__hugetlbfs_debug = true;
Packit Service b439df
		__hugetlbfs_verbose = VERBOSE_DEBUG;
Packit Service b439df
	}
Packit Service b439df
Packit Service b439df
	env = getenv("HUGETLB_RESTRICT_EXE");
Packit Service b439df
	if (env) {
Packit Service b439df
		char *p, *tok, *exe, buf[MAX_EXE+1], restriction[MAX_EXE];
Packit Service b439df
		int found = 0;
Packit Service b439df
Packit Service b439df
		exe = get_exe_name(buf, sizeof buf);
Packit Service b439df
		DEBUG("Found HUGETLB_RESTRICT_EXE, this exe is \"%s\"\n", exe);
Packit Service b439df
		strncpy(restriction, env, sizeof restriction);
Packit Service b439df
		restriction[sizeof(restriction)-1] = 0;
Packit Service b439df
		for (p = restriction; (tok = strtok(p, ":")) != NULL; p = NULL) {
Packit Service b439df
			DEBUG("  ...check exe match for \"%s\"\n",  tok);
Packit Service b439df
			if (strcmp(tok, exe) == 0) {
Packit Service b439df
				found = 1;
Packit Service b439df
				DEBUG("exe match - libhugetlbfs is active for this exe\n");
Packit Service b439df
				break;
Packit Service b439df
			}
Packit Service b439df
		}
Packit Service b439df
		if (!found) {
Packit Service b439df
			DEBUG("No exe match - libhugetlbfs is inactive for this exe\n");
Packit Service b439df
			return;
Packit Service b439df
		}
Packit Service b439df
	}
Packit Service b439df
Packit Service b439df
	env = getenv("HUGETLB_NO_PREFAULT");
Packit Service b439df
	if (env)
Packit Service b439df
		__hugetlbfs_prefault = false;
Packit Service b439df
Packit Service b439df
	__hugetlb_opts.share_path = getenv("HUGETLB_SHARE_PATH");
Packit Service b439df
	__hugetlb_opts.elfmap = getenv("HUGETLB_ELFMAP");
Packit Service b439df
	__hugetlb_opts.ld_preload = getenv("LD_PRELOAD");
Packit Service b439df
	__hugetlb_opts.def_page_size = getenv("HUGETLB_DEFAULT_PAGE_SIZE");
Packit Service b439df
	__hugetlb_opts.path = getenv("HUGETLB_PATH");
Packit Service b439df
	__hugetlb_opts.features = getenv("HUGETLB_FEATURES");
Packit Service b439df
	__hugetlb_opts.morecore = getenv("HUGETLB_MORECORE");
Packit Service b439df
	__hugetlb_opts.heapbase = getenv("HUGETLB_MORECORE_HEAPBASE");
Packit Service b439df
Packit Service b439df
	if (__hugetlb_opts.morecore)
Packit Service b439df
		__hugetlb_opts.thp_morecore =
Packit Service b439df
			(strcasecmp(__hugetlb_opts.morecore, "thp") == 0);
Packit Service b439df
Packit Service b439df
	if (__hugetlb_opts.thp_morecore && __hugetlb_opts.heapbase) {
Packit Service b439df
		DEBUG("Heapbase specified with THP for morecore, ignoring heapbase\n");
Packit Service b439df
		__hugetlb_opts.heapbase = NULL;
Packit Service b439df
	}
Packit Service b439df
Packit Service b439df
	env = getenv("HUGETLB_FORCE_ELFMAP");
Packit Service b439df
	if (env && (strcasecmp(env, "yes") == 0))
Packit Service b439df
		__hugetlb_opts.force_elfmap = 1;
Packit Service b439df
Packit Service b439df
	env = getenv("HUGETLB_MINIMAL_COPY");
Packit Service b439df
	if (__hugetlb_opts.min_copy && env && (strcasecmp(env, "no") == 0)) {
Packit Service b439df
		INFO("HUGETLB_MINIMAL_COPY=%s, disabling filesz copy "
Packit Service b439df
			"optimization\n", env);
Packit Service b439df
		__hugetlb_opts.min_copy = false;
Packit Service b439df
	}
Packit Service b439df
Packit Service b439df
	env = getenv("HUGETLB_SHARE");
Packit Service b439df
	if (env)
Packit Service b439df
		__hugetlb_opts.sharing = atoi(env);
Packit Service b439df
Packit Service b439df
	/*
Packit Service b439df
	 * We have been seeing some unexpected behavior from malloc when
Packit Service b439df
	 * heap shrinking is enabled, so heap shrinking is disabled by
Packit Service b439df
	 * default.
Packit Service b439df
	 *
Packit Service b439df
	 * If malloc has been called successfully before setup_morecore,
Packit Service b439df
	 * glibc will notice a gap between the previous top-of-heap and
Packit Service b439df
	 * the new top-of-heap when it calls hugetlbfs_morecore.  It treats
Packit Service b439df
	 * this as a "foreign sbrk."  Unfortunately, the "foreign sbrk"
Packit Service b439df
	 * handling code will then immediately try to free the memory
Packit Service b439df
	 * allocated by hugetlbfs_morecore!
Packit Service b439df
	 *
Packit Service b439df
	 * This behavior has been reported to the ptmalloc2 maintainer,
Packit Service b439df
	 * along with a patch to correct the behavior.
Packit Service b439df
	 */
Packit Service b439df
	env = getenv("HUGETLB_MORECORE_SHRINK");
Packit Service b439df
	if (env && strcasecmp(env, "yes") == 0)
Packit Service b439df
		__hugetlb_opts.shrink_ok = true;
Packit Service b439df
Packit Service b439df
	/* Determine if shmget() calls should be overridden */
Packit Service b439df
	env = getenv("HUGETLB_SHM");
Packit Service b439df
	if (env && !strcasecmp(env, "yes"))
Packit Service b439df
		__hugetlb_opts.shm_enabled = true;
Packit Service b439df
Packit Service b439df
	/* Determine if all reservations should be avoided */
Packit Service b439df
	env = getenv("HUGETLB_NO_RESERVE");
Packit Service b439df
	if (env && !strcasecmp(env, "yes"))
Packit Service b439df
		__hugetlb_opts.no_reserve = true;
Packit Service b439df
}
Packit Service b439df
Packit Service b439df
void hugetlbfs_setup_kernel_page_size()
Packit Service b439df
{
Packit Service b439df
	long page_size = kernel_default_hugepage_size();
Packit Service b439df
Packit Service b439df
	if (page_size <= 0) {
Packit Service b439df
		WARNING("Unable to find default kernel huge page size\n");
Packit Service b439df
		return;
Packit Service b439df
	}
Packit Service b439df
Packit Service b439df
	INFO("Found pagesize %ld kB\n", page_size / 1024);
Packit Service b439df
	hpage_sizes[0].pagesize = page_size;
Packit Service b439df
Packit Service b439df
	nr_hpage_sizes = 1;
Packit Service b439df
}
Packit Service b439df
Packit Service b439df
void hugetlbfs_check_priv_resv()
Packit Service b439df
{
Packit Service b439df
	/*
Packit Service b439df
	 * If the kernel supports MAP_PRIVATE reservations, we can skip
Packit Service b439df
	 * prefaulting the huge pages we allocate since the kernel
Packit Service b439df
	 * guarantees them.  This can help NUMA performance quite a bit.
Packit Service b439df
	 */
Packit Service b439df
	if (hugetlbfs_test_feature(HUGETLB_FEATURE_PRIVATE_RESV) > 0) {
Packit Service b439df
		INFO("Kernel has MAP_PRIVATE reservations.  Disabling "
Packit Service b439df
			"heap prefaulting.\n");
Packit Service b439df
		__hugetlbfs_prefault = false;
Packit Service b439df
	}
Packit Service b439df
}
Packit Service b439df
Packit Service b439df
void hugetlbfs_check_safe_noreserve()
Packit Service b439df
{
Packit Service b439df
	/*
Packit Service b439df
	 * Some kernels will trigger an OOM if MAP_NORESERVE is used and
Packit Service b439df
	 * a huge page allocation fails. This is unfortunate so limit
Packit Service b439df
	 * the user of NORESERVE where necessary
Packit Service b439df
	 */
Packit Service b439df
	if (__hugetlb_opts.no_reserve &&
Packit Service b439df
		hugetlbfs_test_feature(HUGETLB_FEATURE_SAFE_NORESERVE) <= 0) {
Packit Service b439df
		INFO("Kernel is not safe for MAP_NORESERVE. Forcing "
Packit Service b439df
			"use of reservations.\n");
Packit Service b439df
		__hugetlb_opts.no_reserve = false;
Packit Service b439df
	}
Packit Service b439df
}
Packit Service b439df
Packit Service b439df
void hugetlbfs_check_map_hugetlb()
Packit Service b439df
{
Packit Service b439df
/*
Packit Service b439df
 * FIXME: MAP_HUGETLB has not been picked up by glibc so even though the
Packit Service b439df
 * kernel may support it, without the userspace mmap flag it cannot be
Packit Service b439df
 * used.  This ifdef should be removed when the MAP_HUGETLB flag makes it
Packit Service b439df
 * into glibc.
Packit Service b439df
 */
Packit Service b439df
#ifdef MAP_HUGETLB
Packit Service b439df
	/*
Packit Service b439df
	 * Kernels after 2.6.32 support mmaping pseudo-anonymous regions
Packit Service b439df
	 * backed by huge pages, use this feature for huge pages we
Packit Service b439df
	 * don't intend to share.
Packit Service b439df
	 */
Packit Service b439df
	if (hugetlbfs_test_feature(HUGETLB_FEATURE_MAP_HUGETLB) > 0) {
Packit Service b439df
		INFO("Kernel supports MAP_HUGETLB\n");
Packit Service b439df
		__hugetlb_opts.map_hugetlb = true;
Packit Service b439df
	}
Packit Service b439df
#endif
Packit Service b439df
}
Packit Service b439df
Packit Service b439df
/*
Packit Service b439df
 * Pool counters are typically exposed in sysfs in modern kernels, the
Packit Service b439df
 * counters for the default page size are exposed in procfs in all kernels
Packit Service b439df
 * supporting hugepages.  Given a specific counter (e.g. HUGEPAGES_RSVD)
Packit Service b439df
 * and a page size return both a filename and an optional tag to locate
Packit Service b439df
 * and extract this counter.
Packit Service b439df
 */
Packit Service b439df
static int select_pool_counter(unsigned int counter, unsigned long pagesize,
Packit Service b439df
				char *filename, char **key)
Packit Service b439df
{
Packit Service b439df
	long default_size;
Packit Service b439df
	char *meminfo_key;
Packit Service b439df
	char *sysfs_file;
Packit Service b439df
Packit Service b439df
	if (counter >= HUGEPAGES_MAX_COUNTERS) {
Packit Service b439df
		ERROR("Invalid counter specified\n");
Packit Service b439df
		return -1;
Packit Service b439df
	}
Packit Service b439df
Packit Service b439df
	meminfo_key = hugetlb_counter_info[counter].meminfo_key;
Packit Service b439df
	sysfs_file = hugetlb_counter_info[counter].sysfs_file;
Packit Service b439df
	if (key)
Packit Service b439df
		*key = NULL;
Packit Service b439df
Packit Service b439df
	/*
Packit Service b439df
	 * Get the meminfo page size.
Packit Service b439df
	 * This could be made more efficient if utility functions were shared
Packit Service b439df
	 * between libhugetlbfs and the test suite.  For now we will just
Packit Service b439df
	 * read /proc/meminfo.
Packit Service b439df
	 */
Packit Service b439df
	default_size = kernel_default_hugepage_size();
Packit Service b439df
	if (default_size < 0) {
Packit Service b439df
		ERROR("Cannot determine the default page size\n");
Packit Service b439df
		return -1;
Packit Service b439df
	}
Packit Service b439df
Packit Service b439df
	/* If the user is dealing in the default page size, we can use /proc */
Packit Service b439df
	if (pagesize == default_size) {
Packit Service b439df
		if (meminfo_key && key) {
Packit Service b439df
			strcpy(filename, MEMINFO);
Packit Service b439df
			*key = meminfo_key;
Packit Service b439df
		} else
Packit Service b439df
			sprintf(filename, PROC_HUGEPAGES_DIR "%s", sysfs_file);
Packit Service b439df
	} else /* Use the sysfs interface */
Packit Service b439df
		sprintf(filename, SYSFS_HUGEPAGES_DIR "hugepages-%lukB/%s",
Packit Service b439df
			pagesize / 1024, sysfs_file);
Packit Service b439df
	return 0;
Packit Service b439df
}
Packit Service b439df
Packit Service b439df
static int hpage_size_to_index(unsigned long size)
Packit Service b439df
{
Packit Service b439df
	int i;
Packit Service b439df
Packit Service b439df
	for (i = 0; i < nr_hpage_sizes; i++)
Packit Service b439df
		if (hpage_sizes[i].pagesize == size)
Packit Service b439df
			return i;
Packit Service b439df
	return -1;
Packit Service b439df
}
Packit Service b439df
Packit Service b439df
void probe_default_hpage_size(void)
Packit Service b439df
{
Packit Service b439df
	long size;
Packit Service b439df
	int index;
Packit Service b439df
	int default_overrided;
Packit Service b439df
Packit Service b439df
	if (nr_hpage_sizes == 0) {
Packit Service b439df
		INFO("No configured huge page sizes\n");
Packit Service b439df
		hpage_sizes_default_idx = -1;
Packit Service b439df
		return;
Packit Service b439df
	}
Packit Service b439df
Packit Service b439df
	/*
Packit Service b439df
	 * Check if the user specified a default size, otherwise use the
Packit Service b439df
	 * system default size as reported by /proc/meminfo.
Packit Service b439df
	 */
Packit Service b439df
	default_overrided = (__hugetlb_opts.def_page_size &&
Packit Service b439df
				strlen(__hugetlb_opts.def_page_size) > 0);
Packit Service b439df
	if (default_overrided)
Packit Service b439df
		size = parse_page_size(__hugetlb_opts.def_page_size);
Packit Service b439df
	else {
Packit Service b439df
		size = kernel_default_hugepage_size();
Packit Service b439df
	}
Packit Service b439df
Packit Service b439df
	if (size >= 0) {
Packit Service b439df
		index = hpage_size_to_index(size);
Packit Service b439df
		if (index >= 0)
Packit Service b439df
			hpage_sizes_default_idx = index;
Packit Service b439df
		else {
Packit Service b439df
			/*
Packit Service b439df
			 * If the user specified HUGETLB_DEFAULT_PAGE_SIZE,
Packit Service b439df
			 * then this situation will alter semantics and they
Packit Service b439df
			 * should receive a WARNING.  Otherwise, this detail
Packit Service b439df
			 * is purely informational in nature.
Packit Service b439df
			 */
Packit Service b439df
			char msg[] = "No mount point found for default huge " \
Packit Service b439df
				"page size. Using first available mount "
Packit Service b439df
				"point.\n";
Packit Service b439df
			if (default_overrided)
Packit Service b439df
				WARNING("%s", msg);
Packit Service b439df
			else
Packit Service b439df
				INFO("%s", msg);
Packit Service b439df
			hpage_sizes_default_idx = 0;
Packit Service b439df
		}
Packit Service b439df
	} else {
Packit Service b439df
		ERROR("Unable to determine default huge page size\n");
Packit Service b439df
		hpage_sizes_default_idx = -1;
Packit Service b439df
	}
Packit Service b439df
}
Packit Service b439df
Packit Service b439df
static void add_hugetlbfs_mount(char *path, int user_mount)
Packit Service b439df
{
Packit Service b439df
	int idx;
Packit Service b439df
	long size;
Packit Service b439df
Packit Service b439df
	if (strlen(path) > PATH_MAX)
Packit Service b439df
		return;
Packit Service b439df
Packit Service b439df
	if (!hugetlbfs_test_path(path)) {
Packit Service b439df
		WARNING("%s is not a hugetlbfs mount point, ignoring\n", path);
Packit Service b439df
		return;
Packit Service b439df
	}
Packit Service b439df
Packit Service b439df
	size = hugetlbfs_test_pagesize(path);
Packit Service b439df
	if (size < 0) {
Packit Service b439df
		WARNING("Unable to detect page size for path %s\n", path);
Packit Service b439df
		return;
Packit Service b439df
	}
Packit Service b439df
Packit Service b439df
	idx = hpage_size_to_index(size);
Packit Service b439df
	if (idx < 0) {
Packit Service b439df
		if (nr_hpage_sizes >= MAX_HPAGE_SIZES) {
Packit Service b439df
			WARNING("Maximum number of huge page sizes exceeded, "
Packit Service b439df
				"ignoring %lukB page size\n", size);
Packit Service b439df
			return;
Packit Service b439df
		}
Packit Service b439df
Packit Service b439df
		idx = nr_hpage_sizes;
Packit Service b439df
		hpage_sizes[nr_hpage_sizes++].pagesize = size;
Packit Service b439df
	}
Packit Service b439df
Packit Service b439df
	if (strlen(hpage_sizes[idx].mount)) {
Packit Service b439df
		if (user_mount)
Packit Service b439df
			WARNING("Mount point already defined for size %li, "
Packit Service b439df
				"ignoring %s\n", size, path);
Packit Service b439df
		return;
Packit Service b439df
	}
Packit Service b439df
Packit Service b439df
	strcpy(hpage_sizes[idx].mount, path);
Packit Service b439df
}
Packit Service b439df
Packit Service b439df
void debug_show_page_sizes(void)
Packit Service b439df
{
Packit Service b439df
	int i;
Packit Service b439df
Packit Service b439df
	INFO("Detected page sizes:\n");
Packit Service b439df
	for (i = 0; i < nr_hpage_sizes; i++)
Packit Service b439df
		INFO("   Size: %li kB %s  Mount: %s\n",
Packit Service b439df
			hpage_sizes[i].pagesize / 1024,
Packit Service b439df
			i == hpage_sizes_default_idx ? "(default)" : "",
Packit Service b439df
			hpage_sizes[i].mount);
Packit Service b439df
}
Packit Service b439df
Packit Service b439df
#define LINE_MAXLEN	2048
Packit Service b439df
static void find_mounts(void)
Packit Service b439df
{
Packit Service b439df
	int fd;
Packit Service b439df
	char path[PATH_MAX+1];
Packit Service b439df
	char line[LINE_MAXLEN + 1];
Packit Service b439df
	char *eol;
Packit Service b439df
	char *match;
Packit Service b439df
	char *end;
Packit Service b439df
	int bytes;
Packit Service b439df
	off_t offset;
Packit Service b439df
Packit Service b439df
	fd = open("/proc/mounts", O_RDONLY);
Packit Service b439df
	if (fd < 0) {
Packit Service b439df
		fd = open("/etc/mtab", O_RDONLY);
Packit Service b439df
		if (fd < 0) {
Packit Service b439df
			ERROR("Couldn't open /proc/mounts or /etc/mtab (%s)\n",
Packit Service b439df
				strerror(errno));
Packit Service b439df
			return;
Packit Service b439df
		}
Packit Service b439df
	}
Packit Service b439df
Packit Service b439df
	while ((bytes = read(fd, line, LINE_MAXLEN)) > 0) {
Packit Service b439df
		line[LINE_MAXLEN] = '\0';
Packit Service b439df
		eol = strchr(line, '\n');
Packit Service b439df
		if (!eol) {
Packit Service b439df
			ERROR("Line too long when parsing mounts\n");
Packit Service b439df
			break;
Packit Service b439df
		}
Packit Service b439df
Packit Service b439df
		/*
Packit Service b439df
		 * Truncate the string to just one line and reset the file
Packit Service b439df
		 * to begin reading at the start of the next line.
Packit Service b439df
		 */
Packit Service b439df
		*eol = '\0';
Packit Service b439df
		offset = bytes - (eol + 1 - line);
Packit Service b439df
		lseek(fd, -offset, SEEK_CUR);
Packit Service b439df
Packit Service b439df
		/* Match only hugetlbfs filesystems. */
Packit Service b439df
		match = strstr(line, " hugetlbfs ");
Packit Service b439df
		if (match) {
Packit Service b439df
			match = strchr(line, '/');
Packit Service b439df
			if (!match)
Packit Service b439df
				continue;
Packit Service b439df
			end = strchr(match, ' ');
Packit Service b439df
			if (!end)
Packit Service b439df
				continue;
Packit Service b439df
Packit Service b439df
			strncpy(path, match, end - match);
Packit Service b439df
			path[end - match] = '\0';
Packit Service b439df
			if ((hugetlbfs_test_path(path) == 1) &&
Packit Service b439df
			    !(access(path, R_OK | W_OK | X_OK)))
Packit Service b439df
				add_hugetlbfs_mount(path, 0);
Packit Service b439df
		}
Packit Service b439df
	}
Packit Service b439df
	close(fd);
Packit Service b439df
}
Packit Service b439df
Packit Service b439df
void setup_mounts(void)
Packit Service b439df
{
Packit Service b439df
	int do_scan = 1;
Packit Service b439df
Packit Service b439df
	/* If HUGETLB_PATH is set, only add mounts specified there */
Packit Service b439df
	while (__hugetlb_opts.path) {
Packit Service b439df
		char path[PATH_MAX + 1];
Packit Service b439df
		char *next = strchrnul(__hugetlb_opts.path, ':');
Packit Service b439df
Packit Service b439df
		do_scan = 0;
Packit Service b439df
		if (next - __hugetlb_opts.path > PATH_MAX) {
Packit Service b439df
			ERROR("Path too long in HUGETLB_PATH -- "
Packit Service b439df
				"ignoring environment\n");
Packit Service b439df
			break;
Packit Service b439df
		}
Packit Service b439df
Packit Service b439df
		strncpy(path, __hugetlb_opts.path, next - __hugetlb_opts.path);
Packit Service b439df
		path[next - __hugetlb_opts.path] = '\0';
Packit Service b439df
		add_hugetlbfs_mount(path, 1);
Packit Service b439df
Packit Service b439df
		/* skip the ':' token */
Packit Service b439df
		__hugetlb_opts.path = *next == '\0' ? NULL : next + 1;
Packit Service b439df
	}
Packit Service b439df
Packit Service b439df
	/* Then probe all mounted filesystems */
Packit Service b439df
	if (do_scan)
Packit Service b439df
		find_mounts();
Packit Service b439df
}
Packit Service b439df
Packit Service b439df
int get_pool_size(long size, struct hpage_pool *pool)
Packit Service b439df
{
Packit Service b439df
	long nr_over = 0;
Packit Service b439df
	long nr_used = 0;
Packit Service b439df
	long nr_surp = 0;
Packit Service b439df
	long nr_resv = 0;
Packit Service b439df
	long nr_static = 0;
Packit Service b439df
Packit Service b439df
	long it_used = -1;
Packit Service b439df
	long it_surp = -1;
Packit Service b439df
	long it_resv = -1;
Packit Service b439df
Packit Service b439df
	/*
Packit Service b439df
	 * Pick up those values which are basically stable with respect to
Packit Service b439df
	 * the admin; ie. only changed by them.
Packit Service b439df
	 *
Packit Service b439df
	 * nr_over may be negative if this kernel does not support overcommit
Packit Service b439df
	 * in that case we will consider it always 0 and max will track min
Packit Service b439df
	 * always.
Packit Service b439df
	 */
Packit Service b439df
	nr_over = get_huge_page_counter(size, HUGEPAGES_OC);
Packit Service b439df
	if (nr_over < 0)
Packit Service b439df
		nr_over = 0;
Packit Service b439df
Packit Service b439df
	/* Sample the volatile values until they are stable. */
Packit Service b439df
	while (nr_used != it_used || nr_surp != it_surp || nr_resv != it_resv) {
Packit Service b439df
		nr_used = it_used;
Packit Service b439df
		nr_surp = it_surp;
Packit Service b439df
		nr_resv = it_resv;
Packit Service b439df
Packit Service b439df
		it_used = get_huge_page_counter(size, HUGEPAGES_TOTAL);
Packit Service b439df
		it_surp = get_huge_page_counter(size, HUGEPAGES_SURP);
Packit Service b439df
		it_resv = get_huge_page_counter(size, HUGEPAGES_RSVD);
Packit Service b439df
	}
Packit Service b439df
	if (nr_surp < 0)
Packit Service b439df
		nr_surp = 0;
Packit Service b439df
	if (nr_resv < 0)
Packit Service b439df
		nr_resv = 0;
Packit Service b439df
Packit Service b439df
	nr_static = nr_used - nr_surp;
Packit Service b439df
Packit Service b439df
	if (nr_static >= 0) {
Packit Service b439df
		DEBUG("pagesize<%ld> min<%ld> max<%ld> "
Packit Service b439df
			"in-use<%ld>\n",
Packit Service b439df
			size, nr_static, nr_static + nr_over,
Packit Service b439df
			nr_used);
Packit Service b439df
		pool->pagesize = size;
Packit Service b439df
		pool->minimum = nr_static;
Packit Service b439df
		pool->maximum = nr_static + nr_over;
Packit Service b439df
		pool->size = nr_used;
Packit Service b439df
		pool->is_default = 0;
Packit Service b439df
Packit Service b439df
		return 1;
Packit Service b439df
	}
Packit Service b439df
Packit Service b439df
	return 0;
Packit Service b439df
}
Packit Service b439df
Packit Service b439df
int hpool_sizes(struct hpage_pool *pools, int pcnt)
Packit Service b439df
{
Packit Service b439df
	long default_size;
Packit Service b439df
	int which = 0;
Packit Service b439df
	DIR *dir;
Packit Service b439df
	struct dirent *entry;
Packit Service b439df
Packit Service b439df
	default_size = kernel_default_hugepage_size();
Packit Service b439df
	if (default_size >= 0 && which < pcnt)
Packit Service b439df
		if (get_pool_size(default_size, &pools[which])) {
Packit Service b439df
			pools[which].is_default = 1;
Packit Service b439df
			which++;
Packit Service b439df
		}
Packit Service b439df
Packit Service b439df
	dir = opendir(SYSFS_HUGEPAGES_DIR);
Packit Service b439df
	if (dir) {
Packit Service b439df
		while ((entry = readdir(dir))) {
Packit Service b439df
			char *name = entry->d_name;
Packit Service b439df
			long size;
Packit Service b439df
Packit Service b439df
			DEBUG("parsing<%s>\n", name);
Packit Service b439df
			if (strncmp(name, "hugepages-", 10) != 0)
Packit Service b439df
				continue;
Packit Service b439df
			name += 10;
Packit Service b439df
Packit Service b439df
			size = size_to_smaller_unit(atol(name));
Packit Service b439df
			if (size < 0 || size == default_size)
Packit Service b439df
				continue;
Packit Service b439df
Packit Service b439df
			if (get_pool_size(size, &pools[which]))
Packit Service b439df
				which++;
Packit Service b439df
		}
Packit Service b439df
		closedir(dir);
Packit Service b439df
	}
Packit Service b439df
Packit Service b439df
	return (which < pcnt) ? which : -1;
Packit Service b439df
}
Packit Service b439df
Packit Service 529abc
int arch_has_slice_support(void)
Packit Service 529abc
{
Packit Service 529abc
#ifdef __powerpc64__
Packit Service 529abc
	char mmu_type[16];
Packit Service 529abc
	FILE *fp;
Packit Service 529abc
Packit Service 529abc
	fp = popen("cat /proc/cpuinfo | grep MMU | awk '{ print $3}'", "r");
Packit Service 529abc
	if (!fp || fscanf(fp, "%s", mmu_type) < 0) {
Packit Service 529abc
		ERROR("Failed to determine MMU type\n");
Packit Service 529abc
		abort();
Packit Service 529abc
	}
Packit Service 529abc
Packit Service 529abc
	pclose(fp);
Packit Service 529abc
	return strcmp(mmu_type, "Hash") == 0;
Packit Service 529abc
#elif defined(__powerpc__) && !defined(PPC_NO_SEGMENTS)
Packit Service 529abc
	return 1;
Packit Service 529abc
#else
Packit Service 529abc
	return 0;
Packit Service 529abc
#endif
Packit Service 529abc
}
Packit Service 529abc
Packit Service b439df
/*
Packit Service b439df
 * If we have a default page size then we support hugepages.
Packit Service b439df
 */
Packit Service b439df
int kernel_has_hugepages(void)
Packit Service b439df
{
Packit Service b439df
	long default_size = kernel_default_hugepage_size();
Packit Service b439df
	if (default_size < 0)
Packit Service b439df
		return 0;
Packit Service b439df
Packit Service b439df
	return 1;
Packit Service b439df
}
Packit Service b439df
Packit Service b439df
/*
Packit Service b439df
 * If we can find the default page size, and if we can find an overcommit
Packit Service b439df
 * control for it then the kernel must support overcommit.
Packit Service b439df
 */
Packit Service b439df
int kernel_has_overcommit(void)
Packit Service b439df
{
Packit Service b439df
	long default_size = kernel_default_hugepage_size();
Packit Service b439df
	if (default_size < 0)
Packit Service b439df
		return 0;
Packit Service b439df
Packit Service b439df
	if (get_huge_page_counter(default_size, HUGEPAGES_OC) < 0)
Packit Service b439df
		return 0;
Packit Service b439df
Packit Service b439df
	return 1;
Packit Service b439df
}
Packit Service b439df
Packit Service b439df
/********************************************************************/
Packit Service b439df
/* Library user visible functions                                   */
Packit Service b439df
/********************************************************************/
Packit Service b439df
Packit Service b439df
/*
Packit Service b439df
 * NOTE: This function uses data that is initialized by
Packit Service b439df
 * setup_mounts() which is called during libhugetlbfs initialization.
Packit Service b439df
 *
Packit Service b439df
 * returns:
Packit Service b439df
 *   on success, size of a huge page in number of bytes
Packit Service b439df
 *   on failure, -1
Packit Service b439df
 *	errno set to ENOSYS if huge pages are not supported
Packit Service b439df
 *	errno set to EOVERFLOW if huge page size would overflow return type
Packit Service b439df
 */
Packit Service b439df
long gethugepagesize(void)
Packit Service b439df
{
Packit Service b439df
	long hpage_size;
Packit Service b439df
Packit Service b439df
	/* Are huge pages available and have they been initialized? */
Packit Service b439df
	if (hpage_sizes_default_idx == -1) {
Packit Service b439df
		errno = hugepagesize_errno = ENOSYS;
Packit Service b439df
		return -1;
Packit Service b439df
	}
Packit Service b439df
Packit Service b439df
	errno = 0;
Packit Service b439df
	hpage_size = hpage_sizes[hpage_sizes_default_idx].pagesize;
Packit Service b439df
	return hpage_size;
Packit Service b439df
}
Packit Service b439df
Packit Service b439df
int gethugepagesizes(long pagesizes[], int n_elem)
Packit Service b439df
{
Packit Service b439df
	long default_size;
Packit Service b439df
	DIR *sysfs;
Packit Service b439df
	struct dirent *ent;
Packit Service b439df
	int nr_sizes = 0;
Packit Service b439df
Packit Service b439df
	if (n_elem < 0) {
Packit Service b439df
		errno = EINVAL;
Packit Service b439df
		return -1;
Packit Service b439df
	}
Packit Service b439df
Packit Service b439df
	if (n_elem > 0 && pagesizes == NULL) {
Packit Service b439df
		errno = EINVAL;
Packit Service b439df
		return -1;
Packit Service b439df
	}
Packit Service b439df
Packit Service b439df
	errno = 0;
Packit Service b439df
Packit Service b439df
	/* Get the system default size. */
Packit Service b439df
	default_size = kernel_default_hugepage_size();
Packit Service b439df
	if (default_size < 0)
Packit Service b439df
		return 0;
Packit Service b439df
Packit Service b439df
	if (pagesizes && (nr_sizes == n_elem))
Packit Service b439df
		return nr_sizes;
Packit Service b439df
	if (pagesizes)
Packit Service b439df
		pagesizes[nr_sizes] = default_size;
Packit Service b439df
	nr_sizes++;
Packit Service b439df
Packit Service b439df
	/*
Packit Service b439df
	 * Scan sysfs to look for other sizes.
Packit Service b439df
	 * Non-existing dir is not an error, we got one size from /proc/meminfo.
Packit Service b439df
	 */
Packit Service b439df
	sysfs = opendir(SYSFS_HUGEPAGES_DIR);
Packit Service b439df
	if (!sysfs) {
Packit Service b439df
		if (errno == ENOENT) {
Packit Service b439df
			errno = 0;
Packit Service b439df
			return nr_sizes;
Packit Service b439df
		} else
Packit Service b439df
			return -1;
Packit Service b439df
	}
Packit Service b439df
	while ((ent = readdir(sysfs))) {
Packit Service b439df
		long size;
Packit Service b439df
Packit Service b439df
		if (strncmp(ent->d_name, "hugepages-", 10))
Packit Service b439df
			continue;
Packit Service b439df
Packit Service b439df
		size = strtol(ent->d_name + 10, NULL, 10);
Packit Service b439df
		if (size == LONG_MIN || size == LONG_MAX)
Packit Service b439df
			continue;
Packit Service b439df
		size = size_to_smaller_unit(size);
Packit Service b439df
Packit Service b439df
		if (size < 0 || size == default_size)
Packit Service b439df
			continue;
Packit Service b439df
		if (pagesizes && (nr_sizes == n_elem))
Packit Service b439df
			return nr_sizes;
Packit Service b439df
		if (pagesizes)
Packit Service b439df
			pagesizes[nr_sizes] = size;
Packit Service b439df
		nr_sizes++;
Packit Service b439df
	}
Packit Service b439df
	closedir(sysfs);
Packit Service b439df
Packit Service b439df
	return nr_sizes;
Packit Service b439df
}
Packit Service b439df
Packit Service b439df
int getpagesizes(long pagesizes[], int n_elem)
Packit Service b439df
{
Packit Service b439df
	int ret;
Packit Service b439df
Packit Service b439df
	if (n_elem < 0 || (n_elem > 0 && pagesizes == NULL)) {
Packit Service b439df
		errno = EINVAL;
Packit Service b439df
		return -1;
Packit Service b439df
	}
Packit Service b439df
Packit Service b439df
	/* Requests for sizing, we need one more slot than gethugepagesizes. */
Packit Service b439df
	if (pagesizes == NULL && n_elem == 0) {
Packit Service b439df
		ret = gethugepagesizes(pagesizes, n_elem);
Packit Service b439df
	} else {
Packit Service b439df
		/* Install the base page size. */
Packit Service b439df
		if (pagesizes && n_elem == 0)
Packit Service b439df
			return 0;
Packit Service b439df
		if (pagesizes)
Packit Service b439df
			pagesizes[0] = sysconf(_SC_PAGESIZE);
Packit Service b439df
Packit Service b439df
		ret = gethugepagesizes(pagesizes + 1, n_elem - 1);
Packit Service b439df
	}
Packit Service b439df
	if (ret < 0)
Packit Service b439df
		return ret;
Packit Service b439df
	return ret + 1;
Packit Service b439df
}
Packit Service b439df
Packit Service b439df
int hugetlbfs_test_path(const char *mount)
Packit Service b439df
{
Packit Service b439df
	struct statfs64 sb;
Packit Service b439df
	int err;
Packit Service b439df
Packit Service b439df
	/* Bugs in the 32<->64 translation code in pre-2.6.15 kernels
Packit Service b439df
	 * mean that plain statfs() returns bogus errors on hugetlbfs
Packit Service b439df
	 * filesystems.  Use statfs64() to work around. */
Packit Service b439df
	err = statfs64(mount, &sb);
Packit Service b439df
	if (err)
Packit Service b439df
		return -1;
Packit Service b439df
Packit Service b439df
	return (sb.f_type == HUGETLBFS_MAGIC);
Packit Service b439df
}
Packit Service b439df
Packit Service b439df
/* Return the page size for the given mount point in bytes */
Packit Service b439df
long hugetlbfs_test_pagesize(const char *mount)
Packit Service b439df
{
Packit Service b439df
	struct statfs64 sb;
Packit Service b439df
	int err;
Packit Service b439df
Packit Service b439df
	err = statfs64(mount, &sb);
Packit Service b439df
	if (err)
Packit Service b439df
		return -1;
Packit Service b439df
Packit Service b439df
	if ((sb.f_bsize <= 0) || (sb.f_bsize > LONG_MAX))
Packit Service b439df
		return -1;
Packit Service b439df
Packit Service b439df
	return sb.f_bsize;
Packit Service b439df
}
Packit Service b439df
Packit Service b439df
const char *hugetlbfs_find_path_for_size(long page_size)
Packit Service b439df
{
Packit Service b439df
	char *path;
Packit Service b439df
	int idx;
Packit Service b439df
Packit Service b439df
	idx = hpage_size_to_index(page_size);
Packit Service b439df
	if (idx >= 0) {
Packit Service b439df
		path = hpage_sizes[idx].mount;
Packit Service b439df
		if (strlen(path))
Packit Service b439df
			return path;
Packit Service b439df
	}
Packit Service b439df
	return NULL;
Packit Service b439df
}
Packit Service b439df
Packit Service b439df
const char *hugetlbfs_find_path(void)
Packit Service b439df
{
Packit Service b439df
	long hpage_size = gethugepagesize();
Packit Service b439df
	if (hpage_size > 0)
Packit Service b439df
		return hugetlbfs_find_path_for_size(hpage_size);
Packit Service b439df
	else
Packit Service b439df
		return NULL;
Packit Service b439df
}
Packit Service b439df
Packit Service b439df
int hugetlbfs_unlinked_fd_for_size(long page_size)
Packit Service b439df
{
Packit Service b439df
	const char *path;
Packit Service b439df
	char name[PATH_MAX+1];
Packit Service b439df
	int fd;
Packit Service b439df
Packit Service b439df
	path = hugetlbfs_find_path_for_size(page_size);
Packit Service b439df
	if (!path)
Packit Service b439df
		return -1;
Packit Service b439df
Packit Service b439df
	name[sizeof(name)-1] = '\0';
Packit Service b439df
Packit Service b439df
	strcpy(name, path);
Packit Service b439df
	strncat(name, "/libhugetlbfs.tmp.XXXXXX", sizeof(name)-1);
Packit Service b439df
	/* FIXME: deal with overflows */
Packit Service b439df
Packit Service b439df
	fd = mkstemp64(name);
Packit Service b439df
Packit Service b439df
	if (fd < 0) {
Packit Service b439df
		ERROR("mkstemp() failed: %s\n", strerror(errno));
Packit Service b439df
		return -1;
Packit Service b439df
	}
Packit Service b439df
Packit Service b439df
	unlink(name);
Packit Service b439df
Packit Service b439df
	return fd;
Packit Service b439df
}
Packit Service b439df
Packit Service b439df
int hugetlbfs_unlinked_fd(void)
Packit Service b439df
{
Packit Service b439df
	long hpage_size = gethugepagesize();
Packit Service b439df
	if (hpage_size > 0)
Packit Service b439df
		return hugetlbfs_unlinked_fd_for_size(hpage_size);
Packit Service b439df
	else
Packit Service b439df
		return -1;
Packit Service b439df
}
Packit Service b439df
Packit Service b439df
#define IOV_LEN 64
Packit Service b439df
int hugetlbfs_prefault(void *addr, size_t length)
Packit Service b439df
{
Packit Service b439df
	size_t offset;
Packit Service b439df
	struct iovec iov[IOV_LEN];
Packit Service b439df
	int ret;
Packit Service b439df
	int i;
Packit Service b439df
	int fd;
Packit Service b439df
Packit Service b439df
	if (!__hugetlbfs_prefault)
Packit Service b439df
		return 0;
Packit Service b439df
Packit Service b439df
	/*
Packit Service b439df
	 * The NUMA users of libhugetlbfs' malloc feature are
Packit Service b439df
	 * expected to use the numactl program to specify an
Packit Service b439df
	 * appropriate policy for hugepage allocation
Packit Service b439df
	 *
Packit Service b439df
	 * Use readv(2) to instantiate the hugepages unless HUGETLB_NO_PREFAULT
Packit Service b439df
	 * is set. If we instead returned a hugepage mapping with insufficient
Packit Service b439df
	 * hugepages, the VM system would kill the process when the
Packit Service b439df
	 * process tried to access the missing memory.
Packit Service b439df
	 *
Packit Service b439df
	 * The value of this environment variable is read during library
Packit Service b439df
	 * initialisation and sets __hugetlbfs_prefault accordingly. If
Packit Service b439df
	 * prefaulting is enabled and we can't get all that were requested,
Packit Service b439df
	 * -ENOMEM is returned. The caller is expected to release the entire
Packit Service b439df
	 * mapping and optionally it may recover by mapping base pages instead.
Packit Service b439df
	 */
Packit Service b439df
Packit Service b439df
	fd = open("/dev/zero", O_RDONLY);
Packit Service b439df
	if (fd < 0) {
Packit Service b439df
		ERROR("Failed to open /dev/zero for reading\n");
Packit Service b439df
		return -ENOMEM;
Packit Service b439df
	}
Packit Service b439df
Packit Service b439df
	for (offset = 0; offset < length; ) {
Packit Service b439df
		for (i = 0; i < IOV_LEN && offset < length; i++) {
Packit Service b439df
			iov[i].iov_base = addr + offset;
Packit Service b439df
			iov[i].iov_len = 1;
Packit Service b439df
			offset += gethugepagesize();
Packit Service b439df
		}
Packit Service b439df
		ret = readv(fd, iov, i);
Packit Service b439df
		if (ret != i) {
Packit Service b439df
			DEBUG("Got %d of %d requested; err=%d\n", ret,
Packit Service b439df
					i, ret < 0 ? errno : 0);
Packit Service b439df
			WARNING("Failed to reserve %ld huge pages "
Packit Service b439df
					"for new region\n",
Packit Service b439df
					length / gethugepagesize());
Packit Service b439df
			close(fd);
Packit Service b439df
			return -ENOMEM;
Packit Service b439df
		}
Packit Service b439df
	}
Packit Service b439df
Packit Service b439df
	close(fd);
Packit Service b439df
	return 0;
Packit Service b439df
}
Packit Service b439df
Packit Service b439df
long get_huge_page_counter(long pagesize, unsigned int counter)
Packit Service b439df
{
Packit Service b439df
	char file[PATH_MAX+1];
Packit Service b439df
	char *key;
Packit Service b439df
Packit Service b439df
	if (select_pool_counter(counter, pagesize, file, &key))
Packit Service b439df
		return -1;
Packit Service b439df
Packit Service b439df
	if (access(file, O_RDONLY))
Packit Service b439df
		return -1;
Packit Service b439df
Packit Service b439df
	return file_read_ulong(file, key);
Packit Service b439df
}
Packit Service b439df
Packit Service b439df
int set_huge_page_counter(long pagesize, unsigned int counter,
Packit Service b439df
			unsigned long val)
Packit Service b439df
{
Packit Service b439df
	char file[PATH_MAX+1];
Packit Service b439df
Packit Service b439df
	if (select_pool_counter(counter, pagesize, file, NULL))
Packit Service b439df
		return -1;
Packit Service b439df
Packit Service b439df
	return file_write_ulong(file, val);
Packit Service b439df
}
Packit Service b439df
Packit Service b439df
int set_nr_hugepages(long pagesize, unsigned long val)
Packit Service b439df
{
Packit Service b439df
	return set_huge_page_counter(pagesize, HUGEPAGES_TOTAL, val);
Packit Service b439df
}
Packit Service b439df
Packit Service b439df
int set_nr_overcommit_hugepages(long pagesize, unsigned long val)
Packit Service b439df
{
Packit Service b439df
	DEBUG("setting HUGEPAGES_OC to %ld\n", val);
Packit Service b439df
	return set_huge_page_counter(pagesize, HUGEPAGES_OC, val);
Packit Service b439df
}
Packit Service b439df
Packit Service b439df
long read_nr_overcommit(long page_size)
Packit Service b439df
{
Packit Service b439df
	if (!kernel_has_overcommit())
Packit Service b439df
		return -1;
Packit Service b439df
Packit Service b439df
	return get_huge_page_counter(page_size, HUGEPAGES_OC);
Packit Service b439df
}
Packit Service b439df
Packit Service b439df
void restore_overcommit_pages(long page_size, long oc_pool)
Packit Service b439df
{
Packit Service b439df
	if (!kernel_has_overcommit())
Packit Service b439df
		return;
Packit Service b439df
Packit Service b439df
	set_nr_overcommit_hugepages(page_size, oc_pool);
Packit Service b439df
}
Packit Service b439df
Packit Service b439df
/********************************************************************/
Packit Service b439df
/* Library user visible DIAGNOSES/DEBUGGING ONLY functions          */
Packit Service b439df
/********************************************************************/
Packit Service b439df
Packit Service b439df
#define MAPS_BUF_SZ 4096
Packit Service b439df
long dump_proc_pid_maps()
Packit Service b439df
{
Packit Service b439df
	FILE *f;
Packit Service b439df
	char line[MAPS_BUF_SZ];
Packit Service b439df
	size_t ret;
Packit Service b439df
Packit Service b439df
	f = fopen("/proc/self/maps", "r");
Packit Service b439df
	if (!f) {
Packit Service b439df
		ERROR("Failed to open /proc/self/maps\n");
Packit Service b439df
		return -1;
Packit Service b439df
	}
Packit Service b439df
Packit Service b439df
	while (1) {
Packit Service b439df
		ret = fread(line, sizeof(char), MAPS_BUF_SZ, f);
Packit Service b439df
		if (ret < 0) {
Packit Service b439df
			ERROR("Failed to read /proc/self/maps\n");
Packit Service b439df
			return -1;
Packit Service b439df
		}
Packit Service b439df
		if (ret == 0)
Packit Service b439df
			break;
Packit Service b439df
		ret = fwrite(line, sizeof(char), ret, stderr);
Packit Service b439df
		if (ret < 0) {
Packit Service b439df
			ERROR("Failed to write /proc/self/maps to stderr\n");
Packit Service b439df
			return -1;
Packit Service b439df
		}
Packit Service b439df
	}
Packit Service b439df
Packit Service b439df
	fclose(f);
Packit Service b439df
	return 0;
Packit Service b439df
}
Packit Service b439df
Packit Service b439df
long read_meminfo(const char *tag)
Packit Service b439df
{
Packit Service b439df
	return file_read_ulong(MEMINFO, tag);
Packit Service b439df
}