|
Packit Service |
eb6d0d |
/* Repeating a memory blob, with alias mapping optimization.
|
|
Packit Service |
eb6d0d |
Copyright (C) 2018 Free Software Foundation, Inc.
|
|
Packit Service |
eb6d0d |
This file is part of the GNU C Library.
|
|
Packit Service |
eb6d0d |
|
|
Packit Service |
eb6d0d |
The GNU C Library is free software; you can redistribute it and/or
|
|
Packit Service |
eb6d0d |
modify it under the terms of the GNU Lesser General Public
|
|
Packit Service |
eb6d0d |
License as published by the Free Software Foundation; either
|
|
Packit Service |
eb6d0d |
version 2.1 of the License, or (at your option) any later version.
|
|
Packit Service |
eb6d0d |
|
|
Packit Service |
eb6d0d |
The GNU C Library is distributed in the hope that it will be useful,
|
|
Packit Service |
eb6d0d |
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
Packit Service |
eb6d0d |
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
Packit Service |
eb6d0d |
Lesser General Public License for more details.
|
|
Packit Service |
eb6d0d |
|
|
Packit Service |
eb6d0d |
You should have received a copy of the GNU Lesser General Public
|
|
Packit Service |
eb6d0d |
License along with the GNU C Library; if not, see
|
|
Packit Service |
eb6d0d |
<http://www.gnu.org/licenses/>. */
|
|
Packit Service |
eb6d0d |
|
|
Packit Service |
eb6d0d |
#include <errno.h>
|
|
Packit Service |
eb6d0d |
#include <fcntl.h>
|
|
Packit Service |
eb6d0d |
#include <stdbool.h>
|
|
Packit Service |
eb6d0d |
#include <stdlib.h>
|
|
Packit Service |
eb6d0d |
#include <string.h>
|
|
Packit Service |
eb6d0d |
#include <support/blob_repeat.h>
|
|
Packit Service |
eb6d0d |
#include <support/check.h>
|
|
Packit Service |
eb6d0d |
#include <support/test-driver.h>
|
|
Packit Service |
eb6d0d |
#include <support/support.h>
|
|
Packit Service |
eb6d0d |
#include <support/xunistd.h>
|
|
Packit Service |
eb6d0d |
#include <sys/mman.h>
|
|
Packit Service |
eb6d0d |
#include <unistd.h>
|
|
Packit Service |
eb6d0d |
#include <wchar.h>
|
|
Packit Service |
eb6d0d |
|
|
Packit Service |
eb6d0d |
/* Small allocations should use malloc directly instead of the mmap
|
|
Packit Service |
eb6d0d |
optimization because mappings carry a lot of overhead. */
|
|
Packit Service |
eb6d0d |
static const size_t maximum_small_size = 4 * 1024 * 1024;
|
|
Packit Service |
eb6d0d |
|
|
Packit Service |
eb6d0d |
/* Internal helper for fill. */
|
|
Packit Service |
eb6d0d |
static void
|
|
Packit Service |
eb6d0d |
fill0 (char *target, const char *element, size_t element_size,
|
|
Packit Service |
eb6d0d |
size_t count)
|
|
Packit Service |
eb6d0d |
{
|
|
Packit Service |
eb6d0d |
while (count > 0)
|
|
Packit Service |
eb6d0d |
{
|
|
Packit Service |
eb6d0d |
memcpy (target, element, element_size);
|
|
Packit Service |
eb6d0d |
target += element_size;
|
|
Packit Service |
eb6d0d |
--count;
|
|
Packit Service |
eb6d0d |
}
|
|
Packit Service |
eb6d0d |
}
|
|
Packit Service |
eb6d0d |
|
|
Packit Service |
eb6d0d |
/* Fill the buffer at TARGET with COUNT copies of the ELEMENT_SIZE
|
|
Packit Service |
eb6d0d |
bytes starting at ELEMENT. */
|
|
Packit Service |
eb6d0d |
static void
|
|
Packit Service |
eb6d0d |
fill (char *target, const char *element, size_t element_size,
|
|
Packit Service |
eb6d0d |
size_t count)
|
|
Packit Service |
eb6d0d |
{
|
|
Packit Service |
eb6d0d |
if (element_size == 0 || count == 0)
|
|
Packit Service |
eb6d0d |
return;
|
|
Packit Service |
eb6d0d |
else if (element_size == 1)
|
|
Packit Service |
eb6d0d |
memset (target, element[0], count);
|
|
Packit Service |
eb6d0d |
else if (element_size == sizeof (wchar_t))
|
|
Packit Service |
eb6d0d |
{
|
|
Packit Service |
eb6d0d |
wchar_t wc;
|
|
Packit Service |
eb6d0d |
memcpy (&wc, element, sizeof (wc));
|
|
Packit Service |
eb6d0d |
wmemset ((wchar_t *) target, wc, count);
|
|
Packit Service |
eb6d0d |
}
|
|
Packit Service |
eb6d0d |
else if (element_size < 1024 && count > 4096)
|
|
Packit Service |
eb6d0d |
{
|
|
Packit Service |
eb6d0d |
/* Use larger copies for really small element sizes. */
|
|
Packit Service |
eb6d0d |
char buffer[8192];
|
|
Packit Service |
eb6d0d |
size_t buffer_count = sizeof (buffer) / element_size;
|
|
Packit Service |
eb6d0d |
fill0 (buffer, element, element_size, buffer_count);
|
|
Packit Service |
eb6d0d |
while (count > 0)
|
|
Packit Service |
eb6d0d |
{
|
|
Packit Service |
eb6d0d |
size_t copy_count = buffer_count;
|
|
Packit Service |
eb6d0d |
if (copy_count > count)
|
|
Packit Service |
eb6d0d |
copy_count = count;
|
|
Packit Service |
eb6d0d |
size_t copy_bytes = copy_count * element_size;
|
|
Packit Service |
eb6d0d |
memcpy (target, buffer, copy_bytes);
|
|
Packit Service |
eb6d0d |
target += copy_bytes;
|
|
Packit Service |
eb6d0d |
count -= copy_count;
|
|
Packit Service |
eb6d0d |
}
|
|
Packit Service |
eb6d0d |
}
|
|
Packit Service |
eb6d0d |
else
|
|
Packit Service |
eb6d0d |
fill0 (target, element, element_size, count);
|
|
Packit Service |
eb6d0d |
}
|
|
Packit Service |
eb6d0d |
|
|
Packit Service |
eb6d0d |
/* Use malloc instead of mmap for small allocations and unusual size
|
|
Packit Service |
eb6d0d |
combinations. */
|
|
Packit Service |
eb6d0d |
static struct support_blob_repeat
|
|
Packit Service |
eb6d0d |
allocate_malloc (size_t total_size, const void *element, size_t element_size,
|
|
Packit Service |
eb6d0d |
size_t count)
|
|
Packit Service |
eb6d0d |
{
|
|
Packit Service |
eb6d0d |
void *buffer = malloc (total_size);
|
|
Packit Service |
eb6d0d |
if (buffer == NULL)
|
|
Packit Service |
eb6d0d |
return (struct support_blob_repeat) { 0 };
|
|
Packit Service |
eb6d0d |
fill (buffer, element, element_size, count);
|
|
Packit Service |
eb6d0d |
return (struct support_blob_repeat)
|
|
Packit Service |
eb6d0d |
{
|
|
Packit Service |
eb6d0d |
.start = buffer,
|
|
Packit Service |
eb6d0d |
.size = total_size,
|
|
Packit Service |
eb6d0d |
.use_malloc = true
|
|
Packit Service |
eb6d0d |
};
|
|
Packit Service |
eb6d0d |
}
|
|
Packit Service |
eb6d0d |
|
|
Packit Service |
eb6d0d |
/* Return the least common multiple of PAGE_SIZE and ELEMENT_SIZE,
|
|
Packit Service |
eb6d0d |
avoiding overflow. This assumes that PAGE_SIZE is a power of
|
|
Packit Service |
eb6d0d |
two. */
|
|
Packit Service |
eb6d0d |
static size_t
|
|
Packit Service |
eb6d0d |
minimum_stride_size (size_t page_size, size_t element_size)
|
|
Packit Service |
eb6d0d |
{
|
|
Packit Service |
eb6d0d |
TEST_VERIFY_EXIT (page_size > 0);
|
|
Packit Service |
eb6d0d |
TEST_VERIFY_EXIT (element_size > 0);
|
|
Packit Service |
eb6d0d |
|
|
Packit Service |
eb6d0d |
/* Compute the number of trailing zeros common to both sizes. */
|
|
Packit Service |
eb6d0d |
unsigned int common_zeros = __builtin_ctzll (page_size | element_size);
|
|
Packit Service |
eb6d0d |
|
|
Packit Service |
eb6d0d |
/* In the product, this power of two appears twice, but in the least
|
|
Packit Service |
eb6d0d |
common multiple, it appears only once. Therefore, shift one
|
|
Packit Service |
eb6d0d |
factor. */
|
|
Packit Service |
eb6d0d |
size_t multiple;
|
|
Packit Service |
eb6d0d |
if (__builtin_mul_overflow (page_size >> common_zeros, element_size,
|
|
Packit Service |
eb6d0d |
&multiple))
|
|
Packit Service |
eb6d0d |
return 0;
|
|
Packit Service |
eb6d0d |
return multiple;
|
|
Packit Service |
eb6d0d |
}
|
|
Packit Service |
eb6d0d |
|
|
Packit Service |
eb6d0d |
/* Allocations larger than maximum_small_size potentially use mmap
|
|
Packit Service |
eb6d0d |
with alias mappings. */
|
|
Packit Service |
eb6d0d |
static struct support_blob_repeat
|
|
Packit Service |
eb6d0d |
allocate_big (size_t total_size, const void *element, size_t element_size,
|
|
Packit Service |
eb6d0d |
size_t count)
|
|
Packit Service |
eb6d0d |
{
|
|
Packit Service |
eb6d0d |
unsigned long page_size = xsysconf (_SC_PAGESIZE);
|
|
Packit Service |
eb6d0d |
size_t stride_size = minimum_stride_size (page_size, element_size);
|
|
Packit Service |
eb6d0d |
if (stride_size == 0)
|
|
Packit Service |
eb6d0d |
{
|
|
Packit Service |
eb6d0d |
errno = EOVERFLOW;
|
|
Packit Service |
eb6d0d |
return (struct support_blob_repeat) { 0 };
|
|
Packit Service |
eb6d0d |
}
|
|
Packit Service |
eb6d0d |
|
|
Packit Service |
eb6d0d |
/* Ensure that the stride size is at least maximum_small_size. This
|
|
Packit Service |
eb6d0d |
is necessary to reduce the number of distinct mappings. */
|
|
Packit Service |
eb6d0d |
if (stride_size < maximum_small_size)
|
|
Packit Service |
eb6d0d |
stride_size
|
|
Packit Service |
eb6d0d |
= ((maximum_small_size + stride_size - 1) / stride_size) * stride_size;
|
|
Packit Service |
eb6d0d |
|
|
Packit Service |
eb6d0d |
if (stride_size > total_size)
|
|
Packit Service |
eb6d0d |
/* The mmap optimization would not save anything. */
|
|
Packit Service |
eb6d0d |
return allocate_malloc (total_size, element, element_size, count);
|
|
Packit Service |
eb6d0d |
|
|
Packit Service |
eb6d0d |
/* Reserve the memory region. If we cannot create the mapping,
|
|
Packit Service |
eb6d0d |
there is no reason to set up the backing file. */
|
|
Packit Service |
eb6d0d |
void *target = mmap (NULL, total_size, PROT_NONE,
|
|
Packit Service |
eb6d0d |
MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
|
|
Packit Service |
eb6d0d |
if (target == MAP_FAILED)
|
|
Packit Service |
eb6d0d |
return (struct support_blob_repeat) { 0 };
|
|
Packit Service |
eb6d0d |
|
|
Packit Service |
eb6d0d |
/* Create the backing file for the repeated mapping. Call mkstemp
|
|
Packit Service |
eb6d0d |
directly to remove the resources backing the temporary file
|
|
Packit Service |
eb6d0d |
immediately, once support_blob_repeat_free is called. Using
|
|
Packit Service |
eb6d0d |
create_temp_file would result in a warning during post-test
|
|
Packit Service |
eb6d0d |
cleanup. */
|
|
Packit Service |
eb6d0d |
int fd;
|
|
Packit Service |
eb6d0d |
{
|
|
Packit Service |
eb6d0d |
char *temppath = xasprintf ("%s/support_blob_repeat-XXXXXX", test_dir);
|
|
Packit Service |
eb6d0d |
fd = mkstemp (temppath);
|
|
Packit Service |
eb6d0d |
if (fd < 0)
|
|
Packit Service |
eb6d0d |
FAIL_EXIT1 ("mkstemp (\"%s\"): %m", temppath);
|
|
Packit Service |
eb6d0d |
xunlink (temppath);
|
|
Packit Service |
eb6d0d |
free (temppath);
|
|
Packit Service |
eb6d0d |
}
|
|
Packit Service |
eb6d0d |
|
|
Packit Service |
eb6d0d |
/* Make sure that there is backing storage, so that the fill
|
|
Packit Service |
eb6d0d |
operation will not fault. */
|
|
Packit Service |
eb6d0d |
if (posix_fallocate (fd, 0, stride_size) != 0)
|
|
Packit Service |
eb6d0d |
FAIL_EXIT1 ("posix_fallocate (%zu): %m", stride_size);
|
|
Packit Service |
eb6d0d |
|
|
Packit Service |
eb6d0d |
/* The stride size must still be a multiple of the page size and
|
|
Packit Service |
eb6d0d |
element size. */
|
|
Packit Service |
eb6d0d |
TEST_VERIFY_EXIT ((stride_size % page_size) == 0);
|
|
Packit Service |
eb6d0d |
TEST_VERIFY_EXIT ((stride_size % element_size) == 0);
|
|
Packit Service |
eb6d0d |
|
|
Packit Service |
eb6d0d |
/* Fill the backing store. */
|
|
Packit Service |
eb6d0d |
{
|
|
Packit Service |
eb6d0d |
void *ptr = mmap (target, stride_size, PROT_READ | PROT_WRITE,
|
|
Packit Service |
eb6d0d |
MAP_FIXED | MAP_FILE | MAP_SHARED, fd, 0);
|
|
Packit Service |
eb6d0d |
if (ptr == MAP_FAILED)
|
|
Packit Service |
eb6d0d |
{
|
|
Packit Service |
eb6d0d |
int saved_errno = errno;
|
|
Packit Service |
eb6d0d |
xmunmap (target, total_size);
|
|
Packit Service |
eb6d0d |
xclose (fd);
|
|
Packit Service |
eb6d0d |
errno = saved_errno;
|
|
Packit Service |
eb6d0d |
return (struct support_blob_repeat) { 0 };
|
|
Packit Service |
eb6d0d |
}
|
|
Packit Service |
eb6d0d |
if (ptr != target)
|
|
Packit Service |
eb6d0d |
FAIL_EXIT1 ("mapping of %zu bytes moved from %p to %p",
|
|
Packit Service |
eb6d0d |
stride_size, target, ptr);
|
|
Packit Service |
eb6d0d |
|
|
Packit Service |
eb6d0d |
/* Write the repeating data. */
|
|
Packit Service |
eb6d0d |
fill (target, element, element_size, stride_size / element_size);
|
|
Packit Service |
eb6d0d |
|
|
Packit Service |
eb6d0d |
/* Return to a PROT_NONE mapping, just to be on the safe side. */
|
|
Packit Service |
eb6d0d |
ptr = mmap (target, stride_size, PROT_NONE,
|
|
Packit Service |
eb6d0d |
MAP_FIXED | MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
|
|
Packit Service |
eb6d0d |
if (ptr == MAP_FAILED)
|
|
Packit Service |
eb6d0d |
FAIL_EXIT1 ("Failed to reinstate PROT_NONE mapping: %m");
|
|
Packit Service |
eb6d0d |
if (ptr != target)
|
|
Packit Service |
eb6d0d |
FAIL_EXIT1 ("PROT_NONE mapping of %zu bytes moved from %p to %p",
|
|
Packit Service |
eb6d0d |
stride_size, target, ptr);
|
|
Packit Service |
eb6d0d |
}
|
|
Packit Service |
eb6d0d |
|
|
Packit Service |
eb6d0d |
/* Create the alias mappings. */
|
|
Packit Service |
eb6d0d |
{
|
|
Packit Service |
eb6d0d |
size_t remaining_size = total_size;
|
|
Packit Service |
eb6d0d |
char *current = target;
|
|
Packit Service |
eb6d0d |
int flags = MAP_FIXED | MAP_FILE | MAP_PRIVATE;
|
|
Packit Service |
eb6d0d |
#ifdef MAP_NORESERVE
|
|
Packit Service |
eb6d0d |
flags |= MAP_NORESERVE;
|
|
Packit Service |
eb6d0d |
#endif
|
|
Packit Service |
eb6d0d |
while (remaining_size > 0)
|
|
Packit Service |
eb6d0d |
{
|
|
Packit Service |
eb6d0d |
size_t to_map = stride_size;
|
|
Packit Service |
eb6d0d |
if (to_map > remaining_size)
|
|
Packit Service |
eb6d0d |
to_map = remaining_size;
|
|
Packit Service |
eb6d0d |
void *ptr = mmap (current, to_map, PROT_READ | PROT_WRITE,
|
|
Packit Service |
eb6d0d |
flags, fd, 0);
|
|
Packit Service |
eb6d0d |
if (ptr == MAP_FAILED)
|
|
Packit Service |
eb6d0d |
{
|
|
Packit Service |
eb6d0d |
int saved_errno = errno;
|
|
Packit Service |
eb6d0d |
xmunmap (target, total_size);
|
|
Packit Service |
eb6d0d |
xclose (fd);
|
|
Packit Service |
eb6d0d |
errno = saved_errno;
|
|
Packit Service |
eb6d0d |
return (struct support_blob_repeat) { 0 };
|
|
Packit Service |
eb6d0d |
}
|
|
Packit Service |
eb6d0d |
if (ptr != current)
|
|
Packit Service |
eb6d0d |
FAIL_EXIT1 ("MAP_PRIVATE mapping of %zu bytes moved from %p to %p",
|
|
Packit Service |
eb6d0d |
to_map, target, ptr);
|
|
Packit Service |
eb6d0d |
remaining_size -= to_map;
|
|
Packit Service |
eb6d0d |
current += to_map;
|
|
Packit Service |
eb6d0d |
}
|
|
Packit Service |
eb6d0d |
}
|
|
Packit Service |
eb6d0d |
|
|
Packit Service |
eb6d0d |
xclose (fd);
|
|
Packit Service |
eb6d0d |
|
|
Packit Service |
eb6d0d |
return (struct support_blob_repeat)
|
|
Packit Service |
eb6d0d |
{
|
|
Packit Service |
eb6d0d |
.start = target,
|
|
Packit Service |
eb6d0d |
.size = total_size,
|
|
Packit Service |
eb6d0d |
.use_malloc = false
|
|
Packit Service |
eb6d0d |
};
|
|
Packit Service |
eb6d0d |
}
|
|
Packit Service |
eb6d0d |
|
|
Packit Service |
eb6d0d |
struct support_blob_repeat
|
|
Packit Service |
eb6d0d |
support_blob_repeat_allocate (const void *element, size_t element_size,
|
|
Packit Service |
eb6d0d |
size_t count)
|
|
Packit Service |
eb6d0d |
{
|
|
Packit Service |
eb6d0d |
size_t total_size;
|
|
Packit Service |
eb6d0d |
if (__builtin_mul_overflow (element_size, count, &total_size))
|
|
Packit Service |
eb6d0d |
{
|
|
Packit Service |
eb6d0d |
errno = EOVERFLOW;
|
|
Packit Service |
eb6d0d |
return (struct support_blob_repeat) { 0 };
|
|
Packit Service |
eb6d0d |
}
|
|
Packit Service |
eb6d0d |
if (total_size <= maximum_small_size)
|
|
Packit Service |
eb6d0d |
return allocate_malloc (total_size, element, element_size, count);
|
|
Packit Service |
eb6d0d |
else
|
|
Packit Service |
eb6d0d |
return allocate_big (total_size, element, element_size, count);
|
|
Packit Service |
eb6d0d |
}
|
|
Packit Service |
eb6d0d |
|
|
Packit Service |
eb6d0d |
void
|
|
Packit Service |
eb6d0d |
support_blob_repeat_free (struct support_blob_repeat *blob)
|
|
Packit Service |
eb6d0d |
{
|
|
Packit Service |
eb6d0d |
if (blob->size > 0)
|
|
Packit Service |
eb6d0d |
{
|
|
Packit Service |
eb6d0d |
int saved_errno = errno;
|
|
Packit Service |
eb6d0d |
if (blob->use_malloc)
|
|
Packit Service |
eb6d0d |
free (blob->start);
|
|
Packit Service |
eb6d0d |
else
|
|
Packit Service |
eb6d0d |
xmunmap (blob->start, blob->size);
|
|
Packit Service |
eb6d0d |
errno = saved_errno;
|
|
Packit Service |
eb6d0d |
}
|
|
Packit Service |
eb6d0d |
*blob = (struct support_blob_repeat) { 0 };
|
|
Packit Service |
eb6d0d |
}
|