/*

 * libhugetlbfs - Easy use of Linux hugepages

 * Copyright (C) 2005-2006 David Gibson & Adam Litke, IBM Corporation.

 *

 * This library is free software; you can redistribute it and/or

 * modify it under the terms of the GNU Lesser General Public License

 * as published by the Free Software Foundation; either version 2.1 of

 * the License, or (at your option) any later version.

 *

 * This library is distributed in the hope that it will be useful, but

 * WITHOUT ANY WARRANTY; without even the implied warranty of

 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU

 * Lesser General Public License for more details.

 *

 * You should have received a copy of the GNU Lesser General Public

 * License along with this library; if not, write to the Free Software

 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA

 */

#define _GNU_SOURCE

#include <stdarg.h>

#include <stdio.h>

#include <stdlib.h>

#include <link.h>

#include <malloc.h>

#include <string.h>

#include <unistd.h>

#include <fcntl.h>

#include <signal.h>

#include <sys/syscall.h>

#include <sys/file.h>

#include <linux/unistd.h>

#include <sys/mman.h>

#include <sys/wait.h>

#include <sys/stat.h>

#include <errno.h>

#include <limits.h>

#include <elf.h>

#include <dlfcn.h>

#include "version.h"

#include "hugetlbfs.h"

#include "libhugetlbfs_internal.h"

#ifdef __LP64__

#define Elf_Ehdr	Elf64_Ehdr

#define Elf_Phdr	Elf64_Phdr

#define Elf_Dyn		Elf64_Dyn

#define Elf_Sym		Elf64_Sym

#define ELF_ST_BIND(x)  ELF64_ST_BIND(x)

#define ELF_ST_TYPE(x)  ELF64_ST_TYPE(x)

#else

#define Elf_Ehdr	Elf32_Ehdr

#define Elf_Phdr	Elf32_Phdr

#define Elf_Dyn		Elf32_Dyn

#define Elf_Sym		Elf32_Sym

#define ELF_ST_BIND(x)  ELF64_ST_BIND(x)

#define ELF_ST_TYPE(x)  ELF64_ST_TYPE(x)

#endif

/*

 * SHARED_TIMEOUT is used by find_or_prepare_shared_file for when it

 * should timeout while waiting for other users to finish preparing

 * the file it wants.  The value is the number of tries before giving

 * up with a 1 second wait between tries

 */

#define SHARED_TIMEOUT 10

/* This function prints an error message to stderr, then aborts.  It

 * is safe to call, even if the executable segments are presently

 * unmapped.

 *

 * Arguments are printf() like, but at present supports only %d and %p

 * with no modifiers

 *

 * FIXME: This works in practice, but I suspect it

 * is not guaranteed safe: the library functions we call could in

 * theory call other functions via the PLT which will blow up. */

static void write_err(const char *start, int len)

{

	direct_syscall(__NR_write, 2 /*stderr*/, start, len);

}

static void sys_abort(void)

{

	pid_t pid = direct_syscall(__NR_getpid);

	direct_syscall(__NR_kill, pid, SIGABRT);

}

static void write_err_base(unsigned long val, int base)

{

	const char digit[] = "0123456789abcdef";

	char str1[sizeof(val)*8];

	char str2[sizeof(val)*8];

	int len = 0;

	int i;

	str1[0] = '0';

	while (val) {

		str1[len++] = digit[val % base];

		val /= base;

	}

	if (len == 0)

		len = 1;

	/* Reverse digits */

	for (i = 0; i < len; i++)

		str2[i] = str1[len-i-1];

	write_err(str2, len);

}

static void unmapped_abort(const char *fmt, ...)

{

	const char *p, *q;

	int done = 0;

	unsigned long val;

	va_list ap;

	/* World's worst printf()... */

	va_start(ap, fmt);

	p = q = fmt;

	while (! done) {

		switch (*p) {

		case '\0':

			write_err(q, p-q);

			done = 1;

			break;

		case '%':

			write_err(q, p-q);

			p++;

			switch (*p) {

			case 'u':

				val = va_arg(ap, unsigned);

				write_err_base(val, 10);

				p++;

				break;

			case 'p':

				val = (unsigned long)va_arg(ap, void *);

				write_err_base(val, 16);

				p++;

				break;

			}

			q = p;

			break;

		default:

			p++;

		}

	}

	va_end(ap);

	sys_abort();

}

/* The directory to use for sharing readonly segments */

static char share_readonly_path[PATH_MAX+1];

#define MAX_HTLB_SEGS	3

#define MAX_SEGS	10

struct seg_info {

	void *vaddr;

	unsigned long filesz, memsz, extrasz;

	int prot;

	int fd;

	int index;

	long page_size;

};

struct seg_layout {

	unsigned long start, end;

	long page_size;

};

static struct seg_info htlb_seg_table[MAX_HTLB_SEGS];

static int htlb_num_segs;

static unsigned long force_remap; /* =0 */

static long hpage_readonly_size, hpage_writable_size;

/**

 * assemble_path - handy wrapper around snprintf() for building paths

 * @dst: buffer of size PATH_MAX+1 to assemble string into

 * @fmt: format string for path

 * @...: printf() style parameters for path

 *

 * assemble_path() builds a path in the target buffer (which must have

 * PATH_MAX+1 available bytes), similar to sprintf().  However, f the

 * assembled path would exceed PATH_MAX characters in length,

 * assemble_path() prints an error and abort()s, so there is no need

 * to check the return value and backout.

 */

static void assemble_path(char *dst, const char *fmt, ...)

{

	va_list ap;

	int len;

	va_start(ap, fmt);

	len = vsnprintf(dst, PATH_MAX+1, fmt, ap);

	va_end(ap);

	if (len < 0) {

		ERROR("vsnprintf() error\n");

		abort();

	}

	if (len > PATH_MAX) {

		ERROR("Overflow assembling path\n");

		abort();

	}

}

static void check_memsz()

{

	int i;

	unsigned long memsz_total = 0, memsz_max = 0;

	if (htlb_num_segs == 0)

		return;

	/*

	 * rough heuristic to see if we'll run out of address

	 * space

	 */

	for (i = 0; i < htlb_num_segs; i++) {

		memsz_total += htlb_seg_table[i].memsz;

		if (htlb_seg_table[i].memsz > memsz_max)

			memsz_max = htlb_seg_table[i].memsz;

	}

	/* avoid overflow checking by using two checks */

	DEBUG("Total memsz = %#0lx, memsz of largest segment = %#0lx\n",

			memsz_total, memsz_max);

}

/**

 * find_or_create_share_path - obtain a directory to store the shared

 * hugetlbfs files

 *

 * Checks environment and filesystem to locate a suitable directory

 * for shared hugetlbfs files, creating a new directory if necessary.

 * The determined path is stored in global variable share_readonly_path.

 *

 * returns:

 *  -1, on error

 *  0, on success

 */

static int find_or_create_share_path(long page_size)

{

	const char *base_path;

	struct stat sb;

	int ret;

	/* If no remaping is planned for the read-only segments we are done */

	if (!page_size)

		return 0;

	if (__hugetlb_opts.share_path) {

		/* Given an explicit path */

		if (hugetlbfs_test_path(__hugetlb_opts.share_path) != 1) {

			WARNING("HUGETLB_SHARE_PATH %s is not on a hugetlbfs"

			      " filesystem\n", __hugetlb_opts.share_path);

			return -1;

		}

		/* Make sure the page size matches */

		if (page_size !=

			hugetlbfs_test_pagesize(__hugetlb_opts.share_path)) {

			WARNING("HUGETLB_SHARE_PATH %s is not valid for a %li "

			      "kB page size\n", __hugetlb_opts.share_path,

				page_size / 1024);

			return -1;

		}

		assemble_path(share_readonly_path, "%s",

				__hugetlb_opts.share_path);

		return 0;

	}

	base_path = hugetlbfs_find_path_for_size(page_size);

	if (!base_path)

		return -1;

	assemble_path(share_readonly_path, "%s/elflink-uid-%d",

			base_path, getuid());

	ret = mkdir(share_readonly_path, 0700);

	if ((ret != 0) && (errno != EEXIST)) {

		WARNING("Error creating share directory %s\n",

			share_readonly_path);

		return -1;

	}

	/* Check the share directory is sane */

	ret = lstat(share_readonly_path, &sb);

	if (ret != 0) {

		WARNING("Couldn't stat() %s: %s\n", share_readonly_path,

			strerror(errno));

		return -1;

	}

	if (! S_ISDIR(sb.st_mode)) {

		WARNING("%s is not a directory\n", share_readonly_path);

		return -1;

	}

	if (sb.st_uid != getuid()) {

		WARNING("%s has wrong owner (uid=%d instead of %d)\n",

		      share_readonly_path, sb.st_uid, getuid());

		return -1;

	}

	if (sb.st_mode & (S_IWGRP | S_IWOTH)) {

		WARNING("%s has bad permissions 0%03o\n",

		      share_readonly_path, sb.st_mode);

		return -1;

	}

	return 0;

}

/*

 * Look for non-zero BSS data inside a range and print out any matches

 */

static void check_bss(unsigned long *start, unsigned long *end)

{

	unsigned long *addr;

	for (addr = start; addr < end; addr++) {

		if (*addr != 0)

			DEBUG("Non-zero BSS data @ %p: %lx\n", addr, *addr);

	}

}

/**

 * get_shared_file_name - create a shared file name from program name,

 * segment number and current word size

 * @htlb_seg_info: pointer to program's segment data

 * @file_path: pointer to a PATH_MAX+1 array to store filename in

 *

 * The file name created is *not* intended to be unique, except when

 * the name, gid or phdr number differ. The goal here is to have a

 * standard means of accessing particular segments of particular

 * executables.

 *

 * returns:

 *   -1, on failure

 *   0, on success

 */

static int get_shared_file_name(struct seg_info *htlb_seg_info, char *file_path)

{

	int ret;

	char binary[PATH_MAX+1];

	char *binary2;

	memset(binary, 0, sizeof(binary));

	ret = readlink("/proc/self/exe", binary, PATH_MAX);

	if (ret < 0) {

		WARNING("shared_file: readlink() on /proc/self/exe "

		      "failed: %s\n", strerror(errno));

		return -1;

	}

	binary2 = basename(binary);

	if (!binary2) {

		WARNING("shared_file: basename() on %s failed: %s\n",

		      binary, strerror(errno));

		return -1;

	}

	assemble_path(file_path, "%s/%s_%zd_%d", share_readonly_path, binary2,

		      sizeof(unsigned long) * 8, htlb_seg_info->index);

	return 0;

}

/* Find the .dynamic program header */

static int find_dynamic(Elf_Dyn **dyntab, const ElfW(Addr) addr,

			const Elf_Phdr *phdr, int phnum)

{

	int i = 1;

	while ((phdr[i].p_type != PT_DYNAMIC) && (i < phnum)) {

		++i;

	}

	if (phdr[i].p_type == PT_DYNAMIC) {

		*dyntab = (Elf_Dyn *)(addr + phdr[i].p_vaddr);

		return 0;

	} else {

		DEBUG("No dynamic segment found\n");

		return -1;

	}

}

/* Find the dynamic string and symbol tables */

static int find_tables(Elf_Dyn *dyntab, Elf_Sym **symtab, char **strtab)

{

	int i = 1;

	while ((dyntab[i].d_tag != DT_NULL)) {

		if (dyntab[i].d_tag == DT_SYMTAB)

			*symtab = (Elf_Sym *)dyntab[i].d_un.d_ptr;

		else if (dyntab[i].d_tag == DT_STRTAB)

			*strtab = (char *)dyntab[i].d_un.d_ptr;

		i++;

	}

	if (!*symtab) {

		DEBUG("No symbol table found\n");

		return -1;

	}

	if (!*strtab) {

		DEBUG("No string table found\n");

		return -1;

	}

	return 0;

}

/* Find the number of symbol table entries */

static int find_numsyms(Elf_Sym *symtab, char *strtab)

{

	/*

	 * WARNING - The symbol table size calculation does not follow the ELF

	 *           standard, but rather exploits an assumption we enforce in

	 *           our linker scripts that the string table follows

	 *           immediately after the symbol table. The linker scripts

	 *           must maintain this assumption or this code will break.

	 */

	if ((void *)strtab <= (void *)symtab) {

		DEBUG("Could not calculate dynamic symbol table size\n");

		return -1;

	}

	return ((void *)strtab - (void *)symtab) / sizeof(Elf_Sym);

}

/*

 * To reduce the size of the extra copy window, we can eliminate certain

 * symbols based on information in the dynamic section. The following

 * characteristics apply to symbols which may require copying:

 * - Within the BSS

 * - Global or Weak binding

 * - Object type (variable)

 * - Non-zero size (zero size means the symbol is just a marker with no data)

 */

static inline int keep_symbol(char *strtab, Elf_Sym *s, void *start, void *end)

{

	if ((void *)s->st_value < start)

		return 0;

	if ((void *)s->st_value > end)

		return 0;

	if ((ELF_ST_BIND(s->st_info) != STB_GLOBAL) &&

	    (ELF_ST_BIND(s->st_info) != STB_WEAK))

		return 0;

	if (ELF_ST_TYPE(s->st_info) != STT_OBJECT)

		return 0;

	if (s->st_size == 0)

		return 0;

	if (__hugetlbfs_debug)

		DEBUG("symbol to copy at %p: %s\n", (void *)s->st_value,

						strtab + s->st_name);

	return 1;

}

/* If unspecified by the architecture, no extra copying of the plt is needed */

ElfW(Word) __attribute__ ((weak)) plt_extrasz(ElfW(Dyn) *dyntab)

{

	return 0;

}

/*

 * Subtle:  Since libhugetlbfs depends on glibc, we allow it

 * it to be loaded before us.  As part of its init functions, it

 * initializes stdin, stdout, and stderr in the bss.  We need to

 * include these initialized variables in our copy.

 */

static void get_extracopy(struct seg_info *seg, const ElfW(Addr) addr,

			  const Elf_Phdr *phdr, int phnum)

{

	Elf_Dyn *dyntab;        /* dynamic segment table */

	Elf_Sym *symtab = NULL; /* dynamic symbol table */

	Elf_Sym *sym;           /* a symbol */

	char *strtab = NULL;    /* string table for dynamic symbols */

	int ret, numsyms, found_sym = 0;

	void *start, *end, *end_orig;

	void *sym_end;

	void *plt_end;

	end_orig = seg->vaddr + seg->memsz;

	start = seg->vaddr + seg->filesz;

	if (seg->filesz == seg->memsz)

		return;

	if (!__hugetlb_opts.min_copy)

		goto bail2;

	/* Find dynamic program header */

	ret = find_dynamic(&dyntab, addr, phdr, phnum);

	if (ret < 0)

		goto bail;

	/* Find symbol and string tables */

	ret = find_tables(dyntab, &symtab, &strtab);

	if (ret < 0)

		goto bail;

	numsyms = find_numsyms(symtab, strtab);

	if (numsyms < 0)

		goto bail;

	/*

	 * We must ensure any returns done hereafter have sane start and end

	 * values, as the criss-cross apple sauce algorithm is beginning

	 */

	end = start;

	for (sym = symtab; sym < symtab + numsyms; sym++) {

		if (!keep_symbol(strtab, sym, start, end_orig))

			continue;

		/* These are the droids we are looking for */

		found_sym = 1;

		sym_end = (void *)(sym->st_value + sym->st_size);

		if (sym_end > end)

			end = sym_end;

	}

	/*

	 * Some platforms (PowerPC 64bit ELF) place their PLT beyond the filesz

	 * part of the data segment.  When this is the case, we must extend the

	 * copy window to include this data which has been initialized by the

	 * run-time linker.

	 */

	plt_end = start + plt_extrasz(dyntab);

	if (plt_end > end) {

		end = plt_end;

		found_sym = 1;

	}

	if (__hugetlbfs_debug)

		check_bss(end, end_orig);

	if (found_sym) {

		seg->extrasz = end - start;

	}

	/*

	 * else no need to copy anything, so leave seg->extrasz as zero

	 */

	return;

bail:

	DEBUG("Unable to perform minimal copy\n");

bail2:

	seg->extrasz = end_orig - start;

}

#if defined(__powerpc64__) || \

	(defined(__powerpc__) && !defined(PPC_NO_SEGMENTS))

#define SLICE_LOW_TOP		(0x100000000UL)

#define SLICE_LOW_SIZE		(1UL << SLICE_LOW_SHIFT)

#define SLICE_HIGH_SIZE		(1UL << SLICE_HIGH_SHIFT)

#endif

/*

 * Return the address of the start and end of the hugetlb slice

 * containing @addr. A slice is a range of addresses, start inclusive

 * and end exclusive.

 * Note, that since relinking is not supported on ia64, we can leave it

 * out here.

 */

static unsigned long hugetlb_slice_start(unsigned long addr)

{

	if (!arch_has_slice_support()) {

		return ALIGN_DOWN(addr, gethugepagesize());

	}

#if defined(__powerpc64__)

	if (addr < SLICE_LOW_TOP)

		return ALIGN_DOWN(addr, SLICE_LOW_SIZE);

	else if (addr < SLICE_HIGH_SIZE)

		return SLICE_LOW_TOP;

	else

		return ALIGN_DOWN(addr, SLICE_HIGH_SIZE);

#elif defined(__powerpc__) && !defined(PPC_NO_SEGMENTS)

	return ALIGN_DOWN(addr, SLICE_LOW_SIZE);

#endif

}

static unsigned long hugetlb_slice_end(unsigned long addr)

{

	if (!arch_has_slice_support()) {

		return ALIGN_UP(addr, gethugepagesize()) - 1;

	}

#if defined(__powerpc64__)

	if (addr < SLICE_LOW_TOP)

		return ALIGN_UP(addr, SLICE_LOW_SIZE) - 1;

	else

		return ALIGN_UP(addr, SLICE_HIGH_SIZE) - 1;

#elif defined(__powerpc__) && !defined(PPC_NO_SEGMENTS)

	return ALIGN_UP(addr, SLICE_LOW_SIZE) - 1;

#endif

}

static unsigned long hugetlb_next_slice_start(unsigned long addr)

{

	return hugetlb_slice_end(addr) + 1;

}

static unsigned long hugetlb_prev_slice_end(unsigned long addr)

{

	return hugetlb_slice_start(addr) - 1;

}

/*

 * Store a copy of the given program header

 */

static int save_phdr(int table_idx, int phnum, const ElfW(Addr) addr,

		     const ElfW(Phdr) *phdr)

{

	int prot = 0;

	if (table_idx >= MAX_HTLB_SEGS) {

		WARNING("Executable has too many segments (max %d)\n",

			MAX_HTLB_SEGS);

		htlb_num_segs = 0;

		return -1;

	}

	if (phdr->p_flags & PF_R)

		prot |= PROT_READ;

	if (phdr->p_flags & PF_W)

		prot |= PROT_WRITE;

	if (phdr->p_flags & PF_X)

		prot |= PROT_EXEC;

	htlb_seg_table[table_idx].vaddr = (void *)(addr + phdr->p_vaddr);

	htlb_seg_table[table_idx].filesz = phdr->p_filesz;

	htlb_seg_table[table_idx].memsz = phdr->p_memsz;

	htlb_seg_table[table_idx].prot = prot;

	htlb_seg_table[table_idx].index = phnum;

	INFO("Segment %d (phdr %d): %#0lx-%#0lx  (filesz=%#0lx) "

		"(prot = %#0x)\n", table_idx, phnum,

		(unsigned long) addr + phdr->p_vaddr,

		(unsigned long) addr + phdr->p_vaddr + phdr->p_memsz,

		(unsigned long) phdr->p_filesz, (unsigned int) prot);

	return 0;

}

static int verify_segment_layout(struct seg_layout *segs, int num_segs)

{

	int i;

	long base_size = getpagesize();

	for (i = 1; i < num_segs; i++) {

		unsigned long prev_end = segs[i - 1].end;

		unsigned long start = segs[i].start;

		/*

		 * Do not worry about the boundary between segments that will

		 * not be remapped.

		 */

		if (segs[i - 1].page_size == base_size &&

				segs[i].page_size == base_size)

			continue;

		/* Make sure alignment hasn't caused segments to overlap */

		if (prev_end > start) {

			WARNING("Layout problem with segments %i and %i:\n\t"

				"Segments would overlap\n", i - 1, i);

			return 1;

		}

		/* Make sure page size transitions occur on slice boundaries */

		if ((segs[i - 1].page_size != segs[i].page_size) &&

				hugetlb_slice_end(prev_end) >

				hugetlb_slice_start(start)) {

			WARNING("Layout problem with segments %i and %i:\n\t"

				"Only one page size per slice\n", i - 1, i);

			return 1;

		}

	}

	return 0;

}

static long segment_requested_page_size(const ElfW(Phdr) *phdr)

{

	int writable = phdr->p_flags & PF_W;

	/* Check if a page size was requested by the user */

	if (writable && hpage_writable_size)

		return hpage_writable_size;

	if (!writable && hpage_readonly_size)

		return hpage_readonly_size;

	/* Check if this segment requests remapping by default */

	if (!hpage_readonly_size && !hpage_writable_size &&

			(phdr->p_flags & PF_LINUX_HUGETLB))

		return gethugepagesize();

	/* No remapping selected, return the base page size */

	return getpagesize();

}

static

int parse_elf_normal(struct dl_phdr_info *info, size_t size, void *data)

{

	int i, num_segs;

	unsigned long page_size, seg_psize, start, end;

	struct seg_layout segments[MAX_SEGS];

	page_size = getpagesize();

	num_segs = 0;

	for (i = 0; i < info->dlpi_phnum; i++) {

		if (info->dlpi_phdr[i].p_type != PT_LOAD)

			continue;

		if (i >= MAX_SEGS) {

			WARNING("Maximum number of PT_LOAD segments"

					"exceeded\n");

			return 1;

		}

		seg_psize = segment_requested_page_size(&info->dlpi_phdr[i]);

		if (seg_psize != page_size) {

			if (save_phdr(htlb_num_segs, i, info->dlpi_addr,

				      &info->dlpi_phdr[i]))

				return 1;

			get_extracopy(&htlb_seg_table[htlb_num_segs],

				      info->dlpi_addr, info->dlpi_phdr,

				      info->dlpi_phnum);

			htlb_seg_table[htlb_num_segs].page_size = seg_psize;

			htlb_num_segs++;

		}

		start = ALIGN_DOWN(info->dlpi_addr +

				   info->dlpi_phdr[i].p_vaddr, seg_psize);

		end = ALIGN(info->dlpi_addr + info->dlpi_phdr[i].p_vaddr +

			    info->dlpi_phdr[i].p_memsz, seg_psize);

		segments[num_segs].page_size = seg_psize;

		segments[num_segs].start = start;

		segments[num_segs].end = end;

		num_segs++;

	}

	if (verify_segment_layout(segments, num_segs))

		htlb_num_segs = 0;

	if (__hugetlbfs_debug)

		check_memsz();

	return 1;

}

/*

 * Parse the phdrs of a normal program to attempt partial segment remapping

 */

static

int parse_elf_partial(struct dl_phdr_info *info, size_t size, void *data)

{

	unsigned long vaddr, memsz, gap;

	unsigned long slice_end;

	int i;

	/* This should never actually be called more than once in an

	 * iteration: we assume that dl_iterate_phdrs() always gives

	 * us the main program's phdrs on the first iteration, and

	 * always return 1 to cease iteration at that point. */

	for (i = 0; i < info->dlpi_phnum; i++) {

		if (info->dlpi_phdr[i].p_type != PT_LOAD)

			continue;

		/*

		 * Partial segment remapping only makes sense if the

		 * memory size of the segment is larger than the

		 * granularity at which hugepages can be used. This

		 * mostly affects ppc, where the segment must be larger

		 * than 256M. This guarantees that remapping the binary

		 * in this forced way won't violate any contiguity

		 * constraints.

		 */

		vaddr = hugetlb_next_slice_start(info->dlpi_addr +

						 info->dlpi_phdr[i].p_vaddr);

		gap = vaddr - (info->dlpi_addr + info->dlpi_phdr[i].p_vaddr);

		slice_end = hugetlb_slice_end(vaddr);

		/*

		 * we should stop remapping just before the slice

		 * containing the end of the memsz portion (taking away

		 * the gap of the memsz)

		 */

		memsz = info->dlpi_phdr[i].p_memsz;

		if (memsz < gap) {

			INFO("Segment %d's unaligned memsz is too small: "

					"%#0lx < %#0lx\n",

					i, memsz, gap);

			continue;

		}

		memsz -= gap;

		if (memsz < (slice_end - vaddr)) {

			INFO("Segment %d's aligned memsz is too small: "

					"%#0lx < %#0lx\n",

					i, memsz, slice_end - vaddr);

			continue;

		}

		memsz = hugetlb_prev_slice_end(vaddr + memsz) - vaddr;

		if (save_phdr(htlb_num_segs, i, info->dlpi_addr,

			      &info->dlpi_phdr[i]))

			return 1;

		/*

		 * When remapping partial segments, we create a sub-segment

		 * that is based on the original.  For this reason, we must

		 * make some changes to the phdr captured by save_phdr():

		 * 	vaddr is aligned upwards to a slice boundary

		 * 	memsz is aligned downwards to a slice boundary

		 * 	filesz is set to memsz to force all memory to be copied

		 */

		htlb_seg_table[htlb_num_segs].vaddr = (void *)vaddr;

		htlb_seg_table[htlb_num_segs].filesz = memsz;

		htlb_seg_table[htlb_num_segs].memsz = memsz;

		htlb_num_segs++;

	}

	return 1;

}

/*

 * Verify that a range of memory is unoccupied and usable

 */

static void check_range_empty(void *addr, unsigned long len)

{

	void *p;

	p = mmap(addr, len, PROT_READ, MAP_PRIVATE|MAP_ANON, 0, 0);

	if (p != addr) {

		WARNING("Unable to verify address range %p - %p.  Not empty?\n",

				addr, addr + len);

		if (__hugetlbfs_debug)

			dump_proc_pid_maps();

	}

	if (p != MAP_FAILED)

		munmap(p, len);

}

/*

 * Copy a program segment into a huge page. If possible, try to copy the

 * smallest amount of data possible, unless the user disables this

 * optimization via the HUGETLB_ELFMAP environment variable.

 */

static int prepare_segment(struct seg_info *seg)

{

	void *start, *p, *end, *new_end;

	unsigned long size, offset;

	long page_size = getpagesize();

	long hpage_size;

	int mmap_reserve = __hugetlb_opts.no_reserve ? MAP_NORESERVE : 0;

	hpage_size = seg->page_size;

	/*

	 * mmaps must begin at an address aligned to the page size.  If the

	 * vaddr of this segment is not hpage_size aligned, align it downward

	 * and begin the mmap there.  Note the offset so we can copy data to

	 * the correct starting address within the temporary mmap.

	 */

	start = (void *) ALIGN_DOWN((unsigned long)seg->vaddr, hpage_size);

	offset = seg->vaddr - start;

	/*

	 * Calculate the size of the temporary mapping we must create.

	 * This includes the offset (described above) and the filesz and

	 * extrasz portions of the segment (described below).  We must align

	 * this total to the huge page size so it will be valid for mmap.

	 */

	size = ALIGN(offset + seg->filesz + seg->extrasz, hpage_size);

	/*