/*
* libhugetlbfs - Easy use of Linux hugepages
* Copyright (C) 2005-2006 David Gibson & Adam Litke, IBM Corporation.
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public License
* as published by the Free Software Foundation; either version 2.1 of
* the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful, but
* WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
*/
#define _GNU_SOURCE
#include <stdarg.h>
#include <stdio.h>
#include <stdlib.h>
#include <link.h>
#include <malloc.h>
#include <string.h>
#include <unistd.h>
#include <fcntl.h>
#include <signal.h>
#include <sys/syscall.h>
#include <sys/file.h>
#include <linux/unistd.h>
#include <sys/mman.h>
#include <sys/wait.h>
#include <sys/stat.h>
#include <errno.h>
#include <limits.h>
#include <elf.h>
#include <dlfcn.h>
#include "version.h"
#include "hugetlbfs.h"
#include "libhugetlbfs_internal.h"
#ifdef __LP64__
#define Elf_Ehdr Elf64_Ehdr
#define Elf_Phdr Elf64_Phdr
#define Elf_Dyn Elf64_Dyn
#define Elf_Sym Elf64_Sym
#define ELF_ST_BIND(x) ELF64_ST_BIND(x)
#define ELF_ST_TYPE(x) ELF64_ST_TYPE(x)
#else
#define Elf_Ehdr Elf32_Ehdr
#define Elf_Phdr Elf32_Phdr
#define Elf_Dyn Elf32_Dyn
#define Elf_Sym Elf32_Sym
#define ELF_ST_BIND(x) ELF64_ST_BIND(x)
#define ELF_ST_TYPE(x) ELF64_ST_TYPE(x)
#endif
/*
* SHARED_TIMEOUT is used by find_or_prepare_shared_file for when it
* should timeout while waiting for other users to finish preparing
* the file it wants. The value is the number of tries before giving
* up with a 1 second wait between tries
*/
#define SHARED_TIMEOUT 10
/* This function prints an error message to stderr, then aborts. It
* is safe to call, even if the executable segments are presently
* unmapped.
*
* Arguments are printf() like, but at present supports only %d and %p
* with no modifiers
*
* FIXME: This works in practice, but I suspect it
* is not guaranteed safe: the library functions we call could in
* theory call other functions via the PLT which will blow up. */
static void write_err(const char *start, int len)
{
direct_syscall(__NR_write, 2 /*stderr*/, start, len);
}
static void sys_abort(void)
{
pid_t pid = direct_syscall(__NR_getpid);
direct_syscall(__NR_kill, pid, SIGABRT);
}
static void write_err_base(unsigned long val, int base)
{
const char digit[] = "0123456789abcdef";
char str1[sizeof(val)*8];
char str2[sizeof(val)*8];
int len = 0;
int i;
str1[0] = '0';
while (val) {
str1[len++] = digit[val % base];
val /= base;
}
if (len == 0)
len = 1;
/* Reverse digits */
for (i = 0; i < len; i++)
str2[i] = str1[len-i-1];
write_err(str2, len);
}
static void unmapped_abort(const char *fmt, ...)
{
const char *p, *q;
int done = 0;
unsigned long val;
va_list ap;
/* World's worst printf()... */
va_start(ap, fmt);
p = q = fmt;
while (! done) {
switch (*p) {
case '\0':
write_err(q, p-q);
done = 1;
break;
case '%':
write_err(q, p-q);
p++;
switch (*p) {
case 'u':
val = va_arg(ap, unsigned);
write_err_base(val, 10);
p++;
break;
case 'p':
val = (unsigned long)va_arg(ap, void *);
write_err_base(val, 16);
p++;
break;
}
q = p;
break;
default:
p++;
}
}
va_end(ap);
sys_abort();
}
/* The directory to use for sharing readonly segments */
static char share_readonly_path[PATH_MAX+1];
#define MAX_HTLB_SEGS 3
#define MAX_SEGS 10
struct seg_info {
void *vaddr;
unsigned long filesz, memsz, extrasz;
int prot;
int fd;
int index;
long page_size;
};
struct seg_layout {
unsigned long start, end;
long page_size;
};
static struct seg_info htlb_seg_table[MAX_HTLB_SEGS];
static int htlb_num_segs;
static unsigned long force_remap; /* =0 */
static long hpage_readonly_size, hpage_writable_size;
/**
* assemble_path - handy wrapper around snprintf() for building paths
* @dst: buffer of size PATH_MAX+1 to assemble string into
* @fmt: format string for path
* @...: printf() style parameters for path
*
* assemble_path() builds a path in the target buffer (which must have
* PATH_MAX+1 available bytes), similar to sprintf(). However, f the
* assembled path would exceed PATH_MAX characters in length,
* assemble_path() prints an error and abort()s, so there is no need
* to check the return value and backout.
*/
static void assemble_path(char *dst, const char *fmt, ...)
{
va_list ap;
int len;
va_start(ap, fmt);
len = vsnprintf(dst, PATH_MAX+1, fmt, ap);
va_end(ap);
if (len < 0) {
ERROR("vsnprintf() error\n");
abort();
}
if (len > PATH_MAX) {
ERROR("Overflow assembling path\n");
abort();
}
}
static void check_memsz()
{
int i;
unsigned long memsz_total = 0, memsz_max = 0;
if (htlb_num_segs == 0)
return;
/*
* rough heuristic to see if we'll run out of address
* space
*/
for (i = 0; i < htlb_num_segs; i++) {
memsz_total += htlb_seg_table[i].memsz;
if (htlb_seg_table[i].memsz > memsz_max)
memsz_max = htlb_seg_table[i].memsz;
}
/* avoid overflow checking by using two checks */
DEBUG("Total memsz = %#0lx, memsz of largest segment = %#0lx\n",
memsz_total, memsz_max);
}
/**
* find_or_create_share_path - obtain a directory to store the shared
* hugetlbfs files
*
* Checks environment and filesystem to locate a suitable directory
* for shared hugetlbfs files, creating a new directory if necessary.
* The determined path is stored in global variable share_readonly_path.
*
* returns:
* -1, on error
* 0, on success
*/
static int find_or_create_share_path(long page_size)
{
const char *base_path;
struct stat sb;
int ret;
/* If no remaping is planned for the read-only segments we are done */
if (!page_size)
return 0;
if (__hugetlb_opts.share_path) {
/* Given an explicit path */
if (hugetlbfs_test_path(__hugetlb_opts.share_path) != 1) {
WARNING("HUGETLB_SHARE_PATH %s is not on a hugetlbfs"
" filesystem\n", __hugetlb_opts.share_path);
return -1;
}
/* Make sure the page size matches */
if (page_size !=
hugetlbfs_test_pagesize(__hugetlb_opts.share_path)) {
WARNING("HUGETLB_SHARE_PATH %s is not valid for a %li "
"kB page size\n", __hugetlb_opts.share_path,
page_size / 1024);
return -1;
}
assemble_path(share_readonly_path, "%s",
__hugetlb_opts.share_path);
return 0;
}
base_path = hugetlbfs_find_path_for_size(page_size);
if (!base_path)
return -1;
assemble_path(share_readonly_path, "%s/elflink-uid-%d",
base_path, getuid());
ret = mkdir(share_readonly_path, 0700);
if ((ret != 0) && (errno != EEXIST)) {
WARNING("Error creating share directory %s\n",
share_readonly_path);
return -1;
}
/* Check the share directory is sane */
ret = lstat(share_readonly_path, &sb);
if (ret != 0) {
WARNING("Couldn't stat() %s: %s\n", share_readonly_path,
strerror(errno));
return -1;
}
if (! S_ISDIR(sb.st_mode)) {
WARNING("%s is not a directory\n", share_readonly_path);
return -1;
}
if (sb.st_uid != getuid()) {
WARNING("%s has wrong owner (uid=%d instead of %d)\n",
share_readonly_path, sb.st_uid, getuid());
return -1;
}
if (sb.st_mode & (S_IWGRP | S_IWOTH)) {
WARNING("%s has bad permissions 0%03o\n",
share_readonly_path, sb.st_mode);
return -1;
}
return 0;
}
/*
* Look for non-zero BSS data inside a range and print out any matches
*/
static void check_bss(unsigned long *start, unsigned long *end)
{
unsigned long *addr;
for (addr = start; addr < end; addr++) {
if (*addr != 0)
DEBUG("Non-zero BSS data @ %p: %lx\n", addr, *addr);
}
}
/**
* get_shared_file_name - create a shared file name from program name,
* segment number and current word size
* @htlb_seg_info: pointer to program's segment data
* @file_path: pointer to a PATH_MAX+1 array to store filename in
*
* The file name created is *not* intended to be unique, except when
* the name, gid or phdr number differ. The goal here is to have a
* standard means of accessing particular segments of particular
* executables.
*
* returns:
* -1, on failure
* 0, on success
*/
static int get_shared_file_name(struct seg_info *htlb_seg_info, char *file_path)
{
int ret;
char binary[PATH_MAX+1];
char *binary2;
memset(binary, 0, sizeof(binary));
ret = readlink("/proc/self/exe", binary, PATH_MAX);
if (ret < 0) {
WARNING("shared_file: readlink() on /proc/self/exe "
"failed: %s\n", strerror(errno));
return -1;
}
binary2 = basename(binary);
if (!binary2) {
WARNING("shared_file: basename() on %s failed: %s\n",
binary, strerror(errno));
return -1;
}
assemble_path(file_path, "%s/%s_%zd_%d", share_readonly_path, binary2,
sizeof(unsigned long) * 8, htlb_seg_info->index);
return 0;
}
/* Find the .dynamic program header */
static int find_dynamic(Elf_Dyn **dyntab, const ElfW(Addr) addr,
const Elf_Phdr *phdr, int phnum)
{
int i = 1;
while ((phdr[i].p_type != PT_DYNAMIC) && (i < phnum)) {
++i;
}
if (phdr[i].p_type == PT_DYNAMIC) {
*dyntab = (Elf_Dyn *)(addr + phdr[i].p_vaddr);
return 0;
} else {
DEBUG("No dynamic segment found\n");
return -1;
}
}
/* Find the dynamic string and symbol tables */
static int find_tables(Elf_Dyn *dyntab, Elf_Sym **symtab, char **strtab)
{
int i = 1;
while ((dyntab[i].d_tag != DT_NULL)) {
if (dyntab[i].d_tag == DT_SYMTAB)
*symtab = (Elf_Sym *)dyntab[i].d_un.d_ptr;
else if (dyntab[i].d_tag == DT_STRTAB)
*strtab = (char *)dyntab[i].d_un.d_ptr;
i++;
}
if (!*symtab) {
DEBUG("No symbol table found\n");
return -1;
}
if (!*strtab) {
DEBUG("No string table found\n");
return -1;
}
return 0;
}
/* Find the number of symbol table entries */
static int find_numsyms(Elf_Sym *symtab, char *strtab)
{
/*
* WARNING - The symbol table size calculation does not follow the ELF
* standard, but rather exploits an assumption we enforce in
* our linker scripts that the string table follows
* immediately after the symbol table. The linker scripts
* must maintain this assumption or this code will break.
*/
if ((void *)strtab <= (void *)symtab) {
DEBUG("Could not calculate dynamic symbol table size\n");
return -1;
}
return ((void *)strtab - (void *)symtab) / sizeof(Elf_Sym);
}
/*
* To reduce the size of the extra copy window, we can eliminate certain
* symbols based on information in the dynamic section. The following
* characteristics apply to symbols which may require copying:
* - Within the BSS
* - Global or Weak binding
* - Object type (variable)
* - Non-zero size (zero size means the symbol is just a marker with no data)
*/
static inline int keep_symbol(char *strtab, Elf_Sym *s, void *start, void *end)
{
if ((void *)s->st_value < start)
return 0;
if ((void *)s->st_value > end)
return 0;
if ((ELF_ST_BIND(s->st_info) != STB_GLOBAL) &&
(ELF_ST_BIND(s->st_info) != STB_WEAK))
return 0;
if (ELF_ST_TYPE(s->st_info) != STT_OBJECT)
return 0;
if (s->st_size == 0)
return 0;
if (__hugetlbfs_debug)
DEBUG("symbol to copy at %p: %s\n", (void *)s->st_value,
strtab + s->st_name);
return 1;
}
/* If unspecified by the architecture, no extra copying of the plt is needed */
ElfW(Word) __attribute__ ((weak)) plt_extrasz(ElfW(Dyn) *dyntab)
{
return 0;
}
/*
* Subtle: Since libhugetlbfs depends on glibc, we allow it
* it to be loaded before us. As part of its init functions, it
* initializes stdin, stdout, and stderr in the bss. We need to
* include these initialized variables in our copy.
*/
static void get_extracopy(struct seg_info *seg, const ElfW(Addr) addr,
const Elf_Phdr *phdr, int phnum)
{
Elf_Dyn *dyntab; /* dynamic segment table */
Elf_Sym *symtab = NULL; /* dynamic symbol table */
Elf_Sym *sym; /* a symbol */
char *strtab = NULL; /* string table for dynamic symbols */
int ret, numsyms, found_sym = 0;
void *start, *end, *end_orig;
void *sym_end;
void *plt_end;
end_orig = seg->vaddr + seg->memsz;
start = seg->vaddr + seg->filesz;
if (seg->filesz == seg->memsz)
return;
if (!__hugetlb_opts.min_copy)
goto bail2;
/* Find dynamic program header */
ret = find_dynamic(&dyntab, addr, phdr, phnum);
if (ret < 0)
goto bail;
/* Find symbol and string tables */
ret = find_tables(dyntab, &symtab, &strtab);
if (ret < 0)
goto bail;
numsyms = find_numsyms(symtab, strtab);
if (numsyms < 0)
goto bail;
/*
* We must ensure any returns done hereafter have sane start and end
* values, as the criss-cross apple sauce algorithm is beginning
*/
end = start;
for (sym = symtab; sym < symtab + numsyms; sym++) {
if (!keep_symbol(strtab, sym, start, end_orig))
continue;
/* These are the droids we are looking for */
found_sym = 1;
sym_end = (void *)(sym->st_value + sym->st_size);
if (sym_end > end)
end = sym_end;
}
/*
* Some platforms (PowerPC 64bit ELF) place their PLT beyond the filesz
* part of the data segment. When this is the case, we must extend the
* copy window to include this data which has been initialized by the
* run-time linker.
*/
plt_end = start + plt_extrasz(dyntab);
if (plt_end > end) {
end = plt_end;
found_sym = 1;
}
if (__hugetlbfs_debug)
check_bss(end, end_orig);
if (found_sym) {
seg->extrasz = end - start;
}
/*
* else no need to copy anything, so leave seg->extrasz as zero
*/
return;
bail:
DEBUG("Unable to perform minimal copy\n");
bail2:
seg->extrasz = end_orig - start;
}
#if defined(__powerpc64__) || \
(defined(__powerpc__) && !defined(PPC_NO_SEGMENTS))
#define SLICE_LOW_TOP (0x100000000UL)
#define SLICE_LOW_SIZE (1UL << SLICE_LOW_SHIFT)
#define SLICE_HIGH_SIZE (1UL << SLICE_HIGH_SHIFT)
#endif
/*
* Return the address of the start and end of the hugetlb slice
* containing @addr. A slice is a range of addresses, start inclusive
* and end exclusive.
* Note, that since relinking is not supported on ia64, we can leave it
* out here.
*/
static unsigned long hugetlb_slice_start(unsigned long addr)
{
if (arch_has_slice_support()) {
#if defined(__powerpc64__)
if (addr < SLICE_LOW_TOP)
return ALIGN_DOWN(addr, SLICE_LOW_SIZE);
else if (addr < SLICE_HIGH_SIZE)
return SLICE_LOW_TOP;
else
return ALIGN_DOWN(addr, SLICE_HIGH_SIZE);
#elif defined(__powerpc__) && !defined(PPC_NO_SEGMENTS)
return ALIGN_DOWN(addr, SLICE_LOW_SIZE);
#endif
}
return ALIGN_DOWN(addr, gethugepagesize());
}
static unsigned long hugetlb_slice_end(unsigned long addr)
{
if (arch_has_slice_support()) {
#if defined(__powerpc64__)
if (addr < SLICE_LOW_TOP)
return ALIGN_UP(addr, SLICE_LOW_SIZE) - 1;
else
return ALIGN_UP(addr, SLICE_HIGH_SIZE) - 1;
#elif defined(__powerpc__) && !defined(PPC_NO_SEGMENTS)
return ALIGN_UP(addr, SLICE_LOW_SIZE) - 1;
#endif
}
return ALIGN_UP(addr, gethugepagesize()) - 1;
}
static unsigned long hugetlb_next_slice_start(unsigned long addr)
{
return hugetlb_slice_end(addr) + 1;
}
static unsigned long hugetlb_prev_slice_end(unsigned long addr)
{
return hugetlb_slice_start(addr) - 1;
}
/*
* Store a copy of the given program header
*/
static int save_phdr(int table_idx, int phnum, const ElfW(Addr) addr,
const ElfW(Phdr) *phdr)
{
int prot = 0;
if (table_idx >= MAX_HTLB_SEGS) {
WARNING("Executable has too many segments (max %d)\n",
MAX_HTLB_SEGS);
htlb_num_segs = 0;
return -1;
}
if (phdr->p_flags & PF_R)
prot |= PROT_READ;
if (phdr->p_flags & PF_W)
prot |= PROT_WRITE;
if (phdr->p_flags & PF_X)
prot |= PROT_EXEC;
htlb_seg_table[table_idx].vaddr = (void *)(addr + phdr->p_vaddr);
htlb_seg_table[table_idx].filesz = phdr->p_filesz;
htlb_seg_table[table_idx].memsz = phdr->p_memsz;
htlb_seg_table[table_idx].prot = prot;
htlb_seg_table[table_idx].index = phnum;
INFO("Segment %d (phdr %d): %#0lx-%#0lx (filesz=%#0lx) "
"(prot = %#0x)\n", table_idx, phnum,
(unsigned long) addr + phdr->p_vaddr,
(unsigned long) addr + phdr->p_vaddr + phdr->p_memsz,
(unsigned long) phdr->p_filesz, (unsigned int) prot);
return 0;
}
static int verify_segment_layout(struct seg_layout *segs, int num_segs)
{
int i;
long base_size = getpagesize();
for (i = 1; i < num_segs; i++) {
unsigned long prev_end = segs[i - 1].end;
unsigned long start = segs[i].start;
/*
* Do not worry about the boundary between segments that will
* not be remapped.
*/
if (segs[i - 1].page_size == base_size &&
segs[i].page_size == base_size)
continue;
/* Make sure alignment hasn't caused segments to overlap */
if (prev_end > start) {
WARNING("Layout problem with segments %i and %i:\n\t"
"Segments would overlap\n", i - 1, i);
return 1;
}
/* Make sure page size transitions occur on slice boundaries */
if ((segs[i - 1].page_size != segs[i].page_size) &&
hugetlb_slice_end(prev_end) >
hugetlb_slice_start(start)) {
WARNING("Layout problem with segments %i and %i:\n\t"
"Only one page size per slice\n", i - 1, i);
return 1;
}
}
return 0;
}
static long segment_requested_page_size(const ElfW(Phdr) *phdr)
{
int writable = phdr->p_flags & PF_W;
/* Check if a page size was requested by the user */
if (writable && hpage_writable_size)
return hpage_writable_size;
if (!writable && hpage_readonly_size)
return hpage_readonly_size;
/* Check if this segment requests remapping by default */
if (!hpage_readonly_size && !hpage_writable_size &&
(phdr->p_flags & PF_LINUX_HUGETLB))
return gethugepagesize();
/* No remapping selected, return the base page size */
return getpagesize();
}
static
int parse_elf_normal(struct dl_phdr_info *info, size_t size, void *data)
{
int i, num_segs;
unsigned long page_size, seg_psize, start, end;
struct seg_layout segments[MAX_SEGS];
page_size = getpagesize();
num_segs = 0;
for (i = 0; i < info->dlpi_phnum; i++) {
if (info->dlpi_phdr[i].p_type != PT_LOAD)
continue;
if (i >= MAX_SEGS) {
WARNING("Maximum number of PT_LOAD segments"
"exceeded\n");
return 1;
}
seg_psize = segment_requested_page_size(&info->dlpi_phdr[i]);
if (seg_psize != page_size) {
if (save_phdr(htlb_num_segs, i, info->dlpi_addr,
&info->dlpi_phdr[i]))
return 1;
get_extracopy(&htlb_seg_table[htlb_num_segs],
info->dlpi_addr, info->dlpi_phdr,
info->dlpi_phnum);
htlb_seg_table[htlb_num_segs].page_size = seg_psize;
htlb_num_segs++;
}
start = ALIGN_DOWN(info->dlpi_addr +
info->dlpi_phdr[i].p_vaddr, seg_psize);
end = ALIGN(info->dlpi_addr + info->dlpi_phdr[i].p_vaddr +
info->dlpi_phdr[i].p_memsz, seg_psize);
segments[num_segs].page_size = seg_psize;
segments[num_segs].start = start;
segments[num_segs].end = end;
num_segs++;
}
if (verify_segment_layout(segments, num_segs))
htlb_num_segs = 0;
if (__hugetlbfs_debug)
check_memsz();
return 1;
}
/*
* Parse the phdrs of a normal program to attempt partial segment remapping
*/
static
int parse_elf_partial(struct dl_phdr_info *info, size_t size, void *data)
{
unsigned long vaddr, memsz, gap;
unsigned long slice_end;
int i;
/* This should never actually be called more than once in an
* iteration: we assume that dl_iterate_phdrs() always gives
* us the main program's phdrs on the first iteration, and
* always return 1 to cease iteration at that point. */
for (i = 0; i < info->dlpi_phnum; i++) {
if (info->dlpi_phdr[i].p_type != PT_LOAD)
continue;
/*
* Partial segment remapping only makes sense if the
* memory size of the segment is larger than the
* granularity at which hugepages can be used. This
* mostly affects ppc, where the segment must be larger
* than 256M. This guarantees that remapping the binary
* in this forced way won't violate any contiguity
* constraints.
*/
vaddr = hugetlb_next_slice_start(info->dlpi_addr +
info->dlpi_phdr[i].p_vaddr);
gap = vaddr - (info->dlpi_addr + info->dlpi_phdr[i].p_vaddr);
slice_end = hugetlb_slice_end(vaddr);
/*
* we should stop remapping just before the slice
* containing the end of the memsz portion (taking away
* the gap of the memsz)
*/
memsz = info->dlpi_phdr[i].p_memsz;
if (memsz < gap) {
INFO("Segment %d's unaligned memsz is too small: "
"%#0lx < %#0lx\n",
i, memsz, gap);
continue;
}
memsz -= gap;
if (memsz < (slice_end - vaddr)) {
INFO("Segment %d's aligned memsz is too small: "
"%#0lx < %#0lx\n",
i, memsz, slice_end - vaddr);
continue;
}
memsz = hugetlb_prev_slice_end(vaddr + memsz) - vaddr;
if (save_phdr(htlb_num_segs, i, info->dlpi_addr,
&info->dlpi_phdr[i]))
return 1;
/*
* When remapping partial segments, we create a sub-segment
* that is based on the original. For this reason, we must
* make some changes to the phdr captured by save_phdr():
* vaddr is aligned upwards to a slice boundary
* memsz is aligned downwards to a slice boundary
* filesz is set to memsz to force all memory to be copied
*/
htlb_seg_table[htlb_num_segs].vaddr = (void *)vaddr;
htlb_seg_table[htlb_num_segs].filesz = memsz;
htlb_seg_table[htlb_num_segs].memsz = memsz;
htlb_num_segs++;
}
return 1;
}
/*
* Verify that a range of memory is unoccupied and usable
*/
static void check_range_empty(void *addr, unsigned long len)
{
void *p;
p = mmap(addr, len, PROT_READ, MAP_PRIVATE|MAP_ANON, 0, 0);
if (p != addr) {
WARNING("Unable to verify address range %p - %p. Not empty?\n",
addr, addr + len);
if (__hugetlbfs_debug)
dump_proc_pid_maps();
}
if (p != MAP_FAILED)
munmap(p, len);
}
/*
* Copy a program segment into a huge page. If possible, try to copy the
* smallest amount of data possible, unless the user disables this
* optimization via the HUGETLB_ELFMAP environment variable.
*/
static int prepare_segment(struct seg_info *seg)
{
void *start, *p, *end, *new_end;
unsigned long size, offset;
long page_size = getpagesize();
long hpage_size;
int mmap_reserve = __hugetlb_opts.no_reserve ? MAP_NORESERVE : 0;
hpage_size = seg->page_size;
/*
* mmaps must begin at an address aligned to the page size. If the
* vaddr of this segment is not hpage_size aligned, align it downward
* and begin the mmap there. Note the offset so we can copy data to
* the correct starting address within the temporary mmap.
*/
start = (void *) ALIGN_DOWN((unsigned long)seg->vaddr, hpage_size);
offset = seg->vaddr - start;
/*
* Calculate the size of the temporary mapping we must create.
* This includes the offset (described above) and the filesz and
* extrasz portions of the segment (described below). We must align
* this total to the huge page size so it will be valid for mmap.
*/
size = ALIGN(offset + seg->filesz + seg->extrasz, hpage_size);
/*
* If the segment's start or end addresses have been adjusted to align
* them to the hpage_size, check to make sure nothing is mapped in the
* padding before and after the segment.
*/
end = (void *) ALIGN((unsigned long)seg->vaddr + seg->memsz, page_size);
new_end = (void *) ALIGN((unsigned long)end, hpage_size);
if (ALIGN_DOWN(offset, page_size))
check_range_empty(start, ALIGN_DOWN(offset, page_size));
if (end != new_end)
check_range_empty(end, new_end - end);
/* Create the temporary huge page mmap */
p = mmap(NULL, size, PROT_READ|PROT_WRITE,
MAP_SHARED|mmap_reserve, seg->fd, 0);
if (p == MAP_FAILED) {
WARNING("Couldn't map hugepage segment to copy data: %s\n",
strerror(errno));
return -1;
}
/*
* Minimizing the amount of data copied will maximize performance.
* By definition, the filesz portion of the segment contains
* initialized data and must be copied. If part of the memsz portion
* is known to be initialized already, extrasz will be non-zero and
* that many addtional bytes will be copied from the beginning of the
* memsz region. The rest of the memsz is understood to be zeroes and
* need not be copied.
*/
INFO("Mapped hugeseg at %p. Copying %#0lx bytes and %#0lx extra bytes"
" from %p...", p, seg->filesz, seg->extrasz, seg->vaddr);
memcpy(p + offset, seg->vaddr, seg->filesz + seg->extrasz);
INFO_CONT("done\n");
munmap(p, size);
return 0;
}
/*
* [PPC] Prior to 2.6.22 (which added slices), our temporary hugepage
* mappings are placed in the segment before the stack. This 'taints' that
* segment for be hugepage-only for the lifetime of the process, resulting
* in a maximum stack size of 256MB. If we instead create our hugepage
* mappings in a child process, we can avoid this problem.
*
* This does not adversely affect non-PPC platforms so do it everywhere.
*/
static int fork_and_prepare_segment(struct seg_info *htlb_seg_info)
{
int pid, ret, status;
if ((pid = fork()) < 0) {
WARNING("fork failed");
return -1;
}
if (pid == 0) {
ret = prepare_segment(htlb_seg_info);
if (ret < 0) {
WARNING("Failed to prepare segment\n");
exit(1);
}
else
exit(0);
}
ret = waitpid(pid, &status, 0);
if (ret == -1) {
WARNING("waitpid failed");
return -1;
}
if (WEXITSTATUS(status) != 0)
return -1;
INFO("Prepare succeeded\n");
return 0;
}
/**
* find_or_prepare_shared_file - get one shareable file
* @htlb_seg_info: pointer to program's segment data
*
* This function either locates a hugetlbfs file already containing
* data for a given program segment, or creates one if it doesn't
* already exist.
*
* We use the following algorithm to ensure that when processes race
* to instantiate the hugepage file, we will never obtain an
* incompletely prepared file or have multiple processes prepar
* separate copies of the file.
* - first open 'filename.tmp' with O_EXCL (this acts as a lockfile)
* - second open 'filename' with O_RDONLY (even if the first open
* succeeded).
* Then:
* - If both opens succeed, close the O_EXCL open, unlink
* filename.tmp and use the O_RDONLY fd. (Somebody else has prepared
* the file already)
* - If only the O_RDONLY open suceeds, and the O_EXCL open
* fails with EEXIST, just used the O_RDONLY fd. (Somebody else has
* prepared the file already, but we raced with their rename()).
* - If only the O_EXCL open suceeds, and the O_RDONLY fails with
* ENOENT, prepare the the O_EXCL open, then rename() filename.tmp to
* filename. (We're the first in, we have to prepare the file).
* - If both opens fail, with EEXIST and ENOENT, respectively,
* wait for a little while, then try again from the beginning
* (Somebody else is preparing the file, but hasn't finished yet)
*
* returns:
* -1, on failure
* 0, on success
*/
static int find_or_prepare_shared_file(struct seg_info *htlb_seg_info)
{
int fdx = -1, fds;
int errnox, errnos;
int ret;
int i;
char final_path[PATH_MAX+1];
char tmp_path[PATH_MAX+1];
ret = get_shared_file_name(htlb_seg_info, final_path);
if (ret < 0)
return -1;
assemble_path(tmp_path, "%s.tmp", final_path);
for (i = 0; i < SHARED_TIMEOUT; i++) {
/* NB: mode is modified by umask */
fdx = open(tmp_path, O_CREAT | O_EXCL | O_RDWR, 0666);
errnox = errno;
fds = open(final_path, O_RDONLY);
errnos = errno;
if (fds >= 0) {
/* Got an already-prepared file -> use it */
if (fdx > 0) {
/* Also got an exclusive file -> clean up */
ret = unlink(tmp_path);
if (ret != 0)
WARNING("shared_file: unable to clean "
"up unneeded file %s: %s\n",
tmp_path, strerror(errno));
close(fdx);
} else if (errnox != EEXIST) {
WARNING("shared_file: Unexpected failure on exclusive"
" open of %s: %s\n", tmp_path,
strerror(errnox));
}
htlb_seg_info->fd = fds;
return 0;
}
if (fdx >= 0) {
/* It's our job to prepare */
if (errnos != ENOENT)
WARNING("shared_file: Unexpected failure on"
" shared open of %s: %s\n", final_path,
strerror(errnos));
htlb_seg_info->fd = fdx;
INFO("Got unpopulated shared fd -- Preparing\n");
ret = fork_and_prepare_segment(htlb_seg_info);
if (ret < 0)
goto fail;
INFO("Prepare succeeded\n");
/* move to permanent location */
ret = rename(tmp_path, final_path);
if (ret != 0) {
WARNING("shared_file: unable to rename %s"
" to %s: %s\n", tmp_path, final_path,
strerror(errno));
goto fail;
}
return 0;
}
/* Both opens failed, somebody else is still preparing */
/* Wait and try again */
sleep(1);
}
fail:
if (fdx > 0) {
ret = unlink(tmp_path);
if (ret != 0)
WARNING("shared_file: Unable to clean up temp file %s "
"on failure: %s\n", tmp_path, strerror(errno));
close(fdx);
}
return -1;
}
/**
* obtain_prepared_file - multiplex callers depending on if
* sharing or not
* @htlb_seg_info: pointer to program's segment data
*
* returns:
* -1, on error
* 0, on success
*/
static int obtain_prepared_file(struct seg_info *htlb_seg_info)
{
int fd = -1;
int ret;
long hpage_size = htlb_seg_info->page_size;
/* Share only read-only segments */
if (__hugetlb_opts.sharing && !(htlb_seg_info->prot & PROT_WRITE)) {
/* first, try to share */
ret = find_or_prepare_shared_file(htlb_seg_info);
if (ret == 0)
return 0;
/* but, fall through to unlinked files, if sharing fails */
WARNING("Falling back to unlinked files\n");
}
fd = hugetlbfs_unlinked_fd_for_size(hpage_size);
if (fd < 0)
return -1;
htlb_seg_info->fd = fd;
return fork_and_prepare_segment(htlb_seg_info);
}
static void remap_segments(struct seg_info *seg, int num)
{
int i;
void *p;
unsigned long start, offset, mapsize;
long page_size = getpagesize();
long hpage_size;
int mmap_flags;
/*
* XXX: The bogus call to mmap below forces ld.so to resolve the
* mmap symbol before we unmap the plt in the data segment
* below. This might only be needed in the case where sharing
* is enabled and the hugetlbfs files have already been prepared
* by another process.
*/
p = mmap(0, 0, 0, 0, 0, 0);
/* This is the hairy bit, between unmap and remap we enter a
* black hole. We can't call anything which uses static data
* (ie. essentially any library function...)
*/
for (i = 0; i < num; i++) {
start = ALIGN_DOWN((unsigned long)seg[i].vaddr, page_size);
offset = (unsigned long)(seg[i].vaddr - start);
mapsize = ALIGN(offset + seg[i].memsz, page_size);
munmap((void *) start, mapsize);
}
/* Step 4. Rebuild the address space with hugetlb mappings */
/* NB: we can't do the remap as hugepages within the main loop
* because of PowerPC: we may need to unmap all the normal
* segments before the MMU segment is ok for hugepages */
for (i = 0; i < num; i++) {
hpage_size = seg[i].page_size;
start = ALIGN_DOWN((unsigned long)seg[i].vaddr, hpage_size);
offset = (unsigned long)(seg[i].vaddr - start);
mapsize = ALIGN(offset + seg[i].memsz, hpage_size);
mmap_flags = MAP_PRIVATE|MAP_FIXED;
/* If requested, make no reservations */
if (__hugetlb_opts.no_reserve)
mmap_flags |= MAP_NORESERVE;
/*
* If this is a read-only mapping whose contents are
* entirely contained within the file, then use MAP_NORESERVE.
* The assumption is that the pages already exist in the
* page cache for the hugetlbfs file since it was prepared
* earlier and that mprotect() will not be called which would
* require a COW
*/
if (!(seg[i].prot & PROT_WRITE) &&
seg[i].filesz == seg[i].memsz)
mmap_flags |= MAP_NORESERVE;
p = mmap((void *) start, mapsize, seg[i].prot,
mmap_flags, seg[i].fd, 0);
if (p == MAP_FAILED)
unmapped_abort("Failed to map hugepage segment %u: "
"%p-%p (errno=%u)\n", i, start,
start + mapsize, errno);
if (p != (void *) start)
unmapped_abort("Mapped hugepage segment %u (%p-%p) at "
"wrong address %p\n", i, seg[i].vaddr,
seg[i].vaddr+mapsize, p);
}
/* The segments are all back at this point.
* and it should be safe to reference static data
*/
}
static int set_hpage_sizes(const char *env)
{
char *pos;
long size;
char *key;
char keys[5] = { "R\0" "W\0" "\0" };
/* For each key in R,W */
for (key = keys; *key != '\0'; key += 2) {
pos = strcasestr(env, key);
if (!pos)
continue;
if (*(++pos) == '=') {
size = parse_page_size(pos + 1);
if (size == -1)
return size;
} else
size = gethugepagesize();
if (size <= 0) {
if (errno == ENOSYS)
WARNING("Hugepages unavailable\n");
else if (errno == EOVERFLOW)
WARNING("Hugepage size too large\n");
else
WARNING("Hugepage size (%s)\n",
strerror(errno));
size = 0;
} else if (!hugetlbfs_find_path_for_size(size)) {
WARNING("Hugepage size %li unavailable", size);
size = 0;
}
if (*key == 'R')
hpage_readonly_size = size;
else
hpage_writable_size = size;
}
return 0;
}
static int check_env(void)
{
extern Elf_Ehdr __executable_start __attribute__((weak));
if (__hugetlb_opts.elfmap &&
(strcasecmp(__hugetlb_opts.elfmap, "no") == 0)) {
INFO("HUGETLB_ELFMAP=%s, not attempting to remap program "
"segments\n", __hugetlb_opts.elfmap);
return -1;
}
if (__hugetlb_opts.elfmap && set_hpage_sizes(__hugetlb_opts.elfmap)) {
WARNING("Cannot set elfmap page sizes: %s", strerror(errno));
return -1;
}
if (__hugetlb_opts.ld_preload &&
strstr(__hugetlb_opts.ld_preload, "libhugetlbfs")) {
if (__hugetlb_opts.force_elfmap) {
force_remap = 1;
INFO("HUGETLB_FORCE_ELFMAP=yes, "
"enabling partial segment "
"remapping for non-relinked "
"binaries\n");
INFO("Disabling filesz copy optimization\n");
__hugetlb_opts.min_copy = false;
} else {
if (&__executable_start) {
WARNING("LD_PRELOAD is incompatible with "
"segment remapping\n");
WARNING("Segment remapping has been "
"DISABLED\n");
return -1;
}
}
}
if (__hugetlb_opts.sharing == 2) {
WARNING("HUGETLB_SHARE=%d, however sharing of writable\n"
"segments has been deprecated and is now disabled\n",
__hugetlb_opts.sharing);
__hugetlb_opts.sharing = 0;
} else {
INFO("HUGETLB_SHARE=%d, sharing ", __hugetlb_opts.sharing);
if (__hugetlb_opts.sharing == 1) {
INFO_CONT("enabled for only read-only segments\n");
} else {
INFO_CONT("disabled\n");
__hugetlb_opts.sharing = 0;
}
}
INFO("HUGETLB_NO_RESERVE=%s, reservations %s\n",
__hugetlb_opts.no_reserve ? "yes" : "no",
__hugetlb_opts.no_reserve ? "disabled" : "enabled");
return 0;
}
/*
* Parse an ELF header and record segment information for any segments
* which contain hugetlb information.
*/
static int parse_elf()
{
if (force_remap)
dl_iterate_phdr(parse_elf_partial, NULL);
else
dl_iterate_phdr(parse_elf_normal, NULL);
if (htlb_num_segs == 0) {
INFO("No segments were appropriate for remapping\n");
return -1;
}
return 0;
}
void hugetlbfs_setup_elflink(void)
{
int i, ret;
if (check_env())
return;
if (parse_elf())
return;
INFO("libhugetlbfs version: %s\n", VERSION);
/* Do we need to find a share directory */
if (__hugetlb_opts.sharing) {
/*
* If HUGETLB_ELFMAP is undefined but a shareable segment has
* PF_LINUX_HUGETLB set, segment remapping will occur using the
* default huge page size.
*/
long page_size = hpage_readonly_size ?
hpage_readonly_size : gethugepagesize();
ret = find_or_create_share_path(page_size);
if (ret != 0) {
WARNING("Segment remapping is disabled");
return;
}
}
/* Step 1. Obtain hugepage files with our program data */
for (i = 0; i < htlb_num_segs; i++) {
ret = obtain_prepared_file(&htlb_seg_table[i]);
if (ret < 0) {
WARNING("Failed to setup hugetlbfs file for segment "
"%d\n", i);
/* Close files we have already prepared */
for (i--; i >= 0; i--)
close(htlb_seg_table[i].fd);
return;
}
}
/* Step 3. Unmap the old segments, map in the new ones */
remap_segments(htlb_seg_table, htlb_num_segs);
}