|
Packit |
6c4009 |
/* Cache memory handling.
|
|
Packit |
6c4009 |
Copyright (C) 2004-2018 Free Software Foundation, Inc.
|
|
Packit |
6c4009 |
This file is part of the GNU C Library.
|
|
Packit |
6c4009 |
Contributed by Ulrich Drepper <drepper@redhat.com>, 2004.
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
This program is free software; you can redistribute it and/or modify
|
|
Packit |
6c4009 |
it under the terms of the GNU General Public License as published
|
|
Packit |
6c4009 |
by the Free Software Foundation; version 2 of the License, or
|
|
Packit |
6c4009 |
(at your option) any later version.
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
This program is distributed in the hope that it will be useful,
|
|
Packit |
6c4009 |
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
Packit |
6c4009 |
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
Packit |
6c4009 |
GNU General Public License for more details.
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
You should have received a copy of the GNU General Public License
|
|
Packit |
6c4009 |
along with this program; if not, see <http://www.gnu.org/licenses/>. */
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
#include <assert.h>
|
|
Packit |
6c4009 |
#include <errno.h>
|
|
Packit |
6c4009 |
#include <error.h>
|
|
Packit |
6c4009 |
#include <fcntl.h>
|
|
Packit |
6c4009 |
#include <inttypes.h>
|
|
Packit |
6c4009 |
#include <libintl.h>
|
|
Packit |
6c4009 |
#include <limits.h>
|
|
Packit |
6c4009 |
#include <obstack.h>
|
|
Packit |
6c4009 |
#include <stdlib.h>
|
|
Packit |
6c4009 |
#include <string.h>
|
|
Packit |
6c4009 |
#include <unistd.h>
|
|
Packit |
6c4009 |
#include <sys/mman.h>
|
|
Packit |
6c4009 |
#include <sys/param.h>
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
#include "dbg_log.h"
|
|
Packit |
6c4009 |
#include "nscd.h"
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
static int
|
|
Packit |
6c4009 |
sort_he (const void *p1, const void *p2)
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
struct hashentry *h1 = *(struct hashentry **) p1;
|
|
Packit |
6c4009 |
struct hashentry *h2 = *(struct hashentry **) p2;
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
if (h1 < h2)
|
|
Packit |
6c4009 |
return -1;
|
|
Packit |
6c4009 |
if (h1 > h2)
|
|
Packit |
6c4009 |
return 1;
|
|
Packit |
6c4009 |
return 0;
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
static int
|
|
Packit |
6c4009 |
sort_he_data (const void *p1, const void *p2)
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
struct hashentry *h1 = *(struct hashentry **) p1;
|
|
Packit |
6c4009 |
struct hashentry *h2 = *(struct hashentry **) p2;
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
if (h1->packet < h2->packet)
|
|
Packit |
6c4009 |
return -1;
|
|
Packit |
6c4009 |
if (h1->packet > h2->packet)
|
|
Packit |
6c4009 |
return 1;
|
|
Packit |
6c4009 |
return 0;
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
/* Basic definitions for the bitmap implementation. Only BITMAP_T
|
|
Packit |
6c4009 |
needs to be changed to choose a different word size. */
|
|
Packit |
6c4009 |
#define BITMAP_T uint8_t
|
|
Packit |
6c4009 |
#define BITS (CHAR_BIT * sizeof (BITMAP_T))
|
|
Packit |
6c4009 |
#define ALLBITS ((((BITMAP_T) 1) << BITS) - 1)
|
|
Packit |
6c4009 |
#define HIGHBIT (((BITMAP_T) 1) << (BITS - 1))
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
static void
|
|
Packit |
6c4009 |
markrange (BITMAP_T *mark, ref_t start, size_t len)
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
/* Adjust parameters for block alignment. */
|
|
Packit |
6c4009 |
assert ((start & BLOCK_ALIGN_M1) == 0);
|
|
Packit |
6c4009 |
start /= BLOCK_ALIGN;
|
|
Packit |
6c4009 |
len = (len + BLOCK_ALIGN_M1) / BLOCK_ALIGN;
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
size_t elem = start / BITS;
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
if (start % BITS != 0)
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
if (start % BITS + len <= BITS)
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
/* All fits in the partial byte. */
|
|
Packit |
6c4009 |
mark[elem] |= (ALLBITS >> (BITS - len)) << (start % BITS);
|
|
Packit |
6c4009 |
return;
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
mark[elem++] |= ALLBITS << (start % BITS);
|
|
Packit |
6c4009 |
len -= BITS - (start % BITS);
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
while (len >= BITS)
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
mark[elem++] = ALLBITS;
|
|
Packit |
6c4009 |
len -= BITS;
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
if (len > 0)
|
|
Packit |
6c4009 |
mark[elem] |= ALLBITS >> (BITS - len);
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
void
|
|
Packit |
6c4009 |
gc (struct database_dyn *db)
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
/* We need write access. */
|
|
Packit |
6c4009 |
pthread_rwlock_wrlock (&db->lock);
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
/* And the memory handling lock. */
|
|
Packit |
6c4009 |
pthread_mutex_lock (&db->memlock);
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
/* We need an array representing the data area. All memory
|
|
Packit |
6c4009 |
allocation is BLOCK_ALIGN aligned so this is the level at which
|
|
Packit |
6c4009 |
we have to look at the memory. We use a mark and sweep algorithm
|
|
Packit |
6c4009 |
where the marks are placed in this array. */
|
|
Packit |
6c4009 |
assert (db->head->first_free % BLOCK_ALIGN == 0);
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
BITMAP_T *mark;
|
|
Packit |
6c4009 |
bool mark_use_malloc;
|
|
Packit |
6c4009 |
/* In prune_cache we are also using a dynamically allocated array.
|
|
Packit |
6c4009 |
If the array in the caller is too large we have malloc'ed it. */
|
|
Packit |
6c4009 |
size_t stack_used = sizeof (bool) * db->head->module;
|
|
Packit |
6c4009 |
if (__glibc_unlikely (stack_used > MAX_STACK_USE))
|
|
Packit |
6c4009 |
stack_used = 0;
|
|
Packit |
6c4009 |
size_t nmark = (db->head->first_free / BLOCK_ALIGN + BITS - 1) / BITS;
|
|
Packit |
6c4009 |
size_t memory_needed = nmark * sizeof (BITMAP_T);
|
|
Packit |
6c4009 |
if (__glibc_likely (stack_used + memory_needed <= MAX_STACK_USE))
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
mark = (BITMAP_T *) alloca_account (memory_needed, stack_used);
|
|
Packit |
6c4009 |
mark_use_malloc = false;
|
|
Packit |
6c4009 |
memset (mark, '\0', memory_needed);
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
else
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
mark = (BITMAP_T *) xcalloc (1, memory_needed);
|
|
Packit |
6c4009 |
mark_use_malloc = true;
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
/* Create an array which can hold pointer to all the entries in hash
|
|
Packit |
6c4009 |
entries. */
|
|
Packit |
6c4009 |
memory_needed = 2 * db->head->nentries * sizeof (struct hashentry *);
|
|
Packit |
6c4009 |
struct hashentry **he;
|
|
Packit |
6c4009 |
struct hashentry **he_data;
|
|
Packit |
6c4009 |
bool he_use_malloc;
|
|
Packit |
6c4009 |
if (__glibc_likely (stack_used + memory_needed <= MAX_STACK_USE))
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
he = alloca_account (memory_needed, stack_used);
|
|
Packit |
6c4009 |
he_use_malloc = false;
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
else
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
he = xmalloc (memory_needed);
|
|
Packit |
6c4009 |
he_use_malloc = true;
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
he_data = &he[db->head->nentries];
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
size_t cnt = 0;
|
|
Packit |
6c4009 |
for (size_t idx = 0; idx < db->head->module; ++idx)
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
ref_t *prevp = &db->head->array[idx];
|
|
Packit |
6c4009 |
ref_t run = *prevp;
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
while (run != ENDREF)
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
assert (cnt < db->head->nentries);
|
|
Packit |
6c4009 |
he[cnt] = (struct hashentry *) (db->data + run);
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
he[cnt]->prevp = prevp;
|
|
Packit |
6c4009 |
prevp = &he[cnt]->next;
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
/* This is the hash entry itself. */
|
|
Packit |
6c4009 |
markrange (mark, run, sizeof (struct hashentry));
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
/* Add the information for the data itself. We do this
|
|
Packit |
6c4009 |
only for the one special entry marked with FIRST. */
|
|
Packit |
6c4009 |
if (he[cnt]->first)
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
struct datahead *dh
|
|
Packit |
6c4009 |
= (struct datahead *) (db->data + he[cnt]->packet);
|
|
Packit |
6c4009 |
markrange (mark, he[cnt]->packet, dh->allocsize);
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
run = he[cnt]->next;
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
++cnt;
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
assert (cnt == db->head->nentries);
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
/* Sort the entries by the addresses of the referenced data. All
|
|
Packit |
6c4009 |
the entries pointing to the same DATAHEAD object will have the
|
|
Packit |
6c4009 |
same key. Stability of the sorting is unimportant. */
|
|
Packit |
6c4009 |
memcpy (he_data, he, cnt * sizeof (struct hashentry *));
|
|
Packit |
6c4009 |
qsort (he_data, cnt, sizeof (struct hashentry *), sort_he_data);
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
/* Sort the entries by their address. */
|
|
Packit |
6c4009 |
qsort (he, cnt, sizeof (struct hashentry *), sort_he);
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
#define obstack_chunk_alloc xmalloc
|
|
Packit |
6c4009 |
#define obstack_chunk_free free
|
|
Packit |
6c4009 |
struct obstack ob;
|
|
Packit |
6c4009 |
obstack_init (&ob;;
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
/* Determine the highest used address. */
|
|
Packit |
6c4009 |
size_t high = nmark;
|
|
Packit |
6c4009 |
while (high > 0 && mark[high - 1] == 0)
|
|
Packit |
6c4009 |
--high;
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
/* No memory used. */
|
|
Packit |
6c4009 |
if (high == 0)
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
db->head->first_free = 0;
|
|
Packit |
6c4009 |
goto out;
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
/* Determine the highest offset. */
|
|
Packit |
6c4009 |
BITMAP_T mask = HIGHBIT;
|
|
Packit |
6c4009 |
ref_t highref = (high * BITS - 1) * BLOCK_ALIGN;
|
|
Packit |
6c4009 |
while ((mark[high - 1] & mask) == 0)
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
mask >>= 1;
|
|
Packit |
6c4009 |
highref -= BLOCK_ALIGN;
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
/* Now we can iterate over the MARK array and find bits which are not
|
|
Packit |
6c4009 |
set. These represent memory which can be recovered. */
|
|
Packit |
6c4009 |
size_t byte = 0;
|
|
Packit |
6c4009 |
/* Find the first gap. */
|
|
Packit |
6c4009 |
while (byte < high && mark[byte] == ALLBITS)
|
|
Packit |
6c4009 |
++byte;
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
if (byte == high
|
|
Packit |
6c4009 |
|| (byte == high - 1 && (mark[byte] & ~(mask | (mask - 1))) == 0))
|
|
Packit |
6c4009 |
/* No gap. */
|
|
Packit |
6c4009 |
goto out;
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
mask = 1;
|
|
Packit |
6c4009 |
cnt = 0;
|
|
Packit |
6c4009 |
while ((mark[byte] & mask) != 0)
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
++cnt;
|
|
Packit |
6c4009 |
mask <<= 1;
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
ref_t off_free = (byte * BITS + cnt) * BLOCK_ALIGN;
|
|
Packit |
6c4009 |
assert (off_free <= db->head->first_free);
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
struct hashentry **next_hash = he;
|
|
Packit |
6c4009 |
struct hashentry **next_data = he_data;
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
/* Skip over the hash entries in the first block which does not get
|
|
Packit |
6c4009 |
moved. */
|
|
Packit |
6c4009 |
while (next_hash < &he[db->head->nentries]
|
|
Packit |
6c4009 |
&& *next_hash < (struct hashentry *) (db->data + off_free))
|
|
Packit |
6c4009 |
++next_hash;
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
while (next_data < &he_data[db->head->nentries]
|
|
Packit |
6c4009 |
&& (*next_data)->packet < off_free)
|
|
Packit |
6c4009 |
++next_data;
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
/* Now we start modifying the data. Make sure all readers of the
|
|
Packit |
6c4009 |
data are aware of this and temporarily don't use the data. */
|
|
Packit |
6c4009 |
++db->head->gc_cycle;
|
|
Packit |
6c4009 |
assert ((db->head->gc_cycle & 1) == 1);
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
/* We do not perform the move operations right away since the
|
|
Packit |
6c4009 |
he_data array is not sorted by the address of the data. */
|
|
Packit |
6c4009 |
struct moveinfo
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
void *from;
|
|
Packit |
6c4009 |
void *to;
|
|
Packit |
6c4009 |
size_t size;
|
|
Packit |
6c4009 |
struct moveinfo *next;
|
|
Packit |
6c4009 |
} *moves = NULL;
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
while (byte < high)
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
/* Search for the next filled block. BYTE is the index of the
|
|
Packit |
6c4009 |
entry in MARK, MASK is the bit, and CNT is the bit number.
|
|
Packit |
6c4009 |
OFF_FILLED is the corresponding offset. */
|
|
Packit |
6c4009 |
if ((mark[byte] & ~(mask - 1)) == 0)
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
/* No other bit set in the same element of MARK. Search in the
|
|
Packit |
6c4009 |
following memory. */
|
|
Packit |
6c4009 |
do
|
|
Packit |
6c4009 |
++byte;
|
|
Packit |
6c4009 |
while (byte < high && mark[byte] == 0);
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
if (byte == high)
|
|
Packit |
6c4009 |
/* That was it. */
|
|
Packit |
6c4009 |
break;
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
mask = 1;
|
|
Packit |
6c4009 |
cnt = 0;
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
/* Find the exact bit. */
|
|
Packit |
6c4009 |
while ((mark[byte] & mask) == 0)
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
++cnt;
|
|
Packit |
6c4009 |
mask <<= 1;
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
ref_t off_alloc = (byte * BITS + cnt) * BLOCK_ALIGN;
|
|
Packit |
6c4009 |
assert (off_alloc <= db->head->first_free);
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
/* Find the end of the used area. */
|
|
Packit |
6c4009 |
if ((mark[byte] & ~(mask - 1)) == (BITMAP_T) ~(mask - 1))
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
/* All other bits set. Search the next bytes in MARK. */
|
|
Packit |
6c4009 |
do
|
|
Packit |
6c4009 |
++byte;
|
|
Packit |
6c4009 |
while (byte < high && mark[byte] == ALLBITS);
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
mask = 1;
|
|
Packit |
6c4009 |
cnt = 0;
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
if (byte < high)
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
/* Find the exact bit. */
|
|
Packit |
6c4009 |
while ((mark[byte] & mask) != 0)
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
++cnt;
|
|
Packit |
6c4009 |
mask <<= 1;
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
ref_t off_allocend = (byte * BITS + cnt) * BLOCK_ALIGN;
|
|
Packit |
6c4009 |
assert (off_allocend <= db->head->first_free);
|
|
Packit |
6c4009 |
/* Now we know that we can copy the area from OFF_ALLOC to
|
|
Packit |
6c4009 |
OFF_ALLOCEND (not included) to the memory starting at
|
|
Packit |
6c4009 |
OFF_FREE. First fix up all the entries for the
|
|
Packit |
6c4009 |
displacement. */
|
|
Packit |
6c4009 |
ref_t disp = off_alloc - off_free;
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
struct moveinfo *new_move;
|
|
Packit |
6c4009 |
if (__builtin_expect (stack_used + sizeof (*new_move) <= MAX_STACK_USE,
|
|
Packit |
6c4009 |
1))
|
|
Packit |
6c4009 |
new_move = alloca_account (sizeof (*new_move), stack_used);
|
|
Packit |
6c4009 |
else
|
|
Packit |
6c4009 |
new_move = obstack_alloc (&ob, sizeof (*new_move));
|
|
Packit |
6c4009 |
new_move->from = db->data + off_alloc;
|
|
Packit |
6c4009 |
new_move->to = db->data + off_free;
|
|
Packit |
6c4009 |
new_move->size = off_allocend - off_alloc;
|
|
Packit |
6c4009 |
/* Create a circular list to be always able to append at the end. */
|
|
Packit |
6c4009 |
if (moves == NULL)
|
|
Packit |
6c4009 |
moves = new_move->next = new_move;
|
|
Packit |
6c4009 |
else
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
new_move->next = moves->next;
|
|
Packit |
6c4009 |
moves = moves->next = new_move;
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
/* The following loop will prepare to move this much data. */
|
|
Packit |
6c4009 |
off_free += off_allocend - off_alloc;
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
while (off_alloc < off_allocend)
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
/* Determine whether the next entry is for a hash entry or
|
|
Packit |
6c4009 |
the data. */
|
|
Packit |
6c4009 |
if ((struct hashentry *) (db->data + off_alloc) == *next_hash)
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
/* Just correct the forward reference. */
|
|
Packit |
6c4009 |
*(*next_hash++)->prevp -= disp;
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
off_alloc += ((sizeof (struct hashentry) + BLOCK_ALIGN_M1)
|
|
Packit |
6c4009 |
& ~BLOCK_ALIGN_M1);
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
else
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
assert (next_data < &he_data[db->head->nentries]);
|
|
Packit |
6c4009 |
assert ((*next_data)->packet == off_alloc);
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
struct datahead *dh = (struct datahead *) (db->data + off_alloc);
|
|
Packit |
6c4009 |
do
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
assert ((*next_data)->key >= (*next_data)->packet);
|
|
Packit |
6c4009 |
assert ((*next_data)->key + (*next_data)->len
|
|
Packit |
6c4009 |
<= (*next_data)->packet + dh->allocsize);
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
(*next_data)->packet -= disp;
|
|
Packit |
6c4009 |
(*next_data)->key -= disp;
|
|
Packit |
6c4009 |
++next_data;
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
while (next_data < &he_data[db->head->nentries]
|
|
Packit |
6c4009 |
&& (*next_data)->packet == off_alloc);
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
off_alloc += (dh->allocsize + BLOCK_ALIGN_M1) & ~BLOCK_ALIGN_M1;
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
assert (off_alloc == off_allocend);
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
assert (off_alloc <= db->head->first_free);
|
|
Packit |
6c4009 |
if (off_alloc == db->head->first_free)
|
|
Packit |
6c4009 |
/* We are done, that was the last block. */
|
|
Packit |
6c4009 |
break;
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
assert (next_hash == &he[db->head->nentries]);
|
|
Packit |
6c4009 |
assert (next_data == &he_data[db->head->nentries]);
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
/* Now perform the actual moves. */
|
|
Packit |
6c4009 |
if (moves != NULL)
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
struct moveinfo *runp = moves->next;
|
|
Packit |
6c4009 |
do
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
assert ((char *) runp->to >= db->data);
|
|
Packit |
6c4009 |
assert ((char *) runp->to + runp->size
|
|
Packit |
6c4009 |
<= db->data + db->head->first_free);
|
|
Packit |
6c4009 |
assert ((char *) runp->from >= db->data);
|
|
Packit |
6c4009 |
assert ((char *) runp->from + runp->size
|
|
Packit |
6c4009 |
<= db->data + db->head->first_free);
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
/* The regions may overlap. */
|
|
Packit |
6c4009 |
memmove (runp->to, runp->from, runp->size);
|
|
Packit |
6c4009 |
runp = runp->next;
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
while (runp != moves->next);
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
if (__glibc_unlikely (debug_level >= 3))
|
|
Packit |
6c4009 |
dbg_log (_("freed %zu bytes in %s cache"),
|
|
Packit |
6c4009 |
(size_t) (db->head->first_free
|
|
Packit |
6c4009 |
- ((char *) moves->to + moves->size - db->data)),
|
|
Packit |
6c4009 |
dbnames[db - dbs]);
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
/* The byte past the end of the last copied block is the next
|
|
Packit |
6c4009 |
available byte. */
|
|
Packit |
6c4009 |
db->head->first_free = (char *) moves->to + moves->size - db->data;
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
/* Consistency check. */
|
|
Packit |
6c4009 |
if (__glibc_unlikely (debug_level >= 3))
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
for (size_t idx = 0; idx < db->head->module; ++idx)
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
ref_t run = db->head->array[idx];
|
|
Packit |
6c4009 |
size_t cnt = 0;
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
while (run != ENDREF)
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
if (run + sizeof (struct hashentry) > db->head->first_free)
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
dbg_log ("entry %zu in hash bucket %zu out of bounds: "
|
|
Packit |
6c4009 |
"%" PRIu32 "+%zu > %zu\n",
|
|
Packit |
6c4009 |
cnt, idx, run, sizeof (struct hashentry),
|
|
Packit |
6c4009 |
(size_t) db->head->first_free);
|
|
Packit |
6c4009 |
break;
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
struct hashentry *he = (struct hashentry *) (db->data + run);
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
if (he->key + he->len > db->head->first_free)
|
|
Packit |
6c4009 |
dbg_log ("key of entry %zu in hash bucket %zu out of "
|
|
Packit |
6c4009 |
"bounds: %" PRIu32 "+%zu > %zu\n",
|
|
Packit |
6c4009 |
cnt, idx, he->key, (size_t) he->len,
|
|
Packit |
6c4009 |
(size_t) db->head->first_free);
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
if (he->packet + sizeof (struct datahead)
|
|
Packit |
6c4009 |
> db->head->first_free)
|
|
Packit |
6c4009 |
dbg_log ("packet of entry %zu in hash bucket %zu out of "
|
|
Packit |
6c4009 |
"bounds: %" PRIu32 "+%zu > %zu\n",
|
|
Packit |
6c4009 |
cnt, idx, he->packet, sizeof (struct datahead),
|
|
Packit |
6c4009 |
(size_t) db->head->first_free);
|
|
Packit |
6c4009 |
else
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
struct datahead *dh = (struct datahead *) (db->data
|
|
Packit |
6c4009 |
+ he->packet);
|
|
Packit |
6c4009 |
if (he->packet + dh->allocsize
|
|
Packit |
6c4009 |
> db->head->first_free)
|
|
Packit |
6c4009 |
dbg_log ("full key of entry %zu in hash bucket %zu "
|
|
Packit |
6c4009 |
"out of bounds: %" PRIu32 "+%zu > %zu",
|
|
Packit |
6c4009 |
cnt, idx, he->packet, (size_t) dh->allocsize,
|
|
Packit |
6c4009 |
(size_t) db->head->first_free);
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
run = he->next;
|
|
Packit |
6c4009 |
++cnt;
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
/* Make sure the data on disk is updated. */
|
|
Packit |
6c4009 |
if (db->persistent)
|
|
Packit |
6c4009 |
msync (db->head, db->data + db->head->first_free - (char *) db->head,
|
|
Packit |
6c4009 |
MS_ASYNC);
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
/* Now we are done modifying the data. */
|
|
Packit |
6c4009 |
++db->head->gc_cycle;
|
|
Packit |
6c4009 |
assert ((db->head->gc_cycle & 1) == 0);
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
/* We are done. */
|
|
Packit |
6c4009 |
out:
|
|
Packit |
6c4009 |
pthread_mutex_unlock (&db->memlock);
|
|
Packit |
6c4009 |
pthread_rwlock_unlock (&db->lock);
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
if (he_use_malloc)
|
|
Packit |
6c4009 |
free (he);
|
|
Packit |
6c4009 |
if (mark_use_malloc)
|
|
Packit |
6c4009 |
free (mark);
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
obstack_free (&ob, NULL);
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
void *
|
|
Packit |
6c4009 |
mempool_alloc (struct database_dyn *db, size_t len, int data_alloc)
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
/* Make sure LEN is a multiple of our maximum alignment so we can
|
|
Packit |
6c4009 |
keep track of used memory is multiples of this alignment value. */
|
|
Packit |
6c4009 |
if ((len & BLOCK_ALIGN_M1) != 0)
|
|
Packit |
6c4009 |
len += BLOCK_ALIGN - (len & BLOCK_ALIGN_M1);
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
if (data_alloc)
|
|
Packit |
6c4009 |
pthread_rwlock_rdlock (&db->lock);
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
pthread_mutex_lock (&db->memlock);
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
assert ((db->head->first_free & BLOCK_ALIGN_M1) == 0);
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
bool tried_resize = false;
|
|
Packit |
6c4009 |
void *res;
|
|
Packit |
6c4009 |
retry:
|
|
Packit |
6c4009 |
res = db->data + db->head->first_free;
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
if (__glibc_unlikely (db->head->first_free + len > db->head->data_size))
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
if (! tried_resize)
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
/* Try to resize the database. Grow size of 1/8th. */
|
|
Packit |
6c4009 |
size_t oldtotal = (sizeof (struct database_pers_head)
|
|
Packit |
6c4009 |
+ roundup (db->head->module * sizeof (ref_t),
|
|
Packit |
6c4009 |
ALIGN)
|
|
Packit |
6c4009 |
+ db->head->data_size);
|
|
Packit |
6c4009 |
size_t new_data_size = (db->head->data_size
|
|
Packit |
6c4009 |
+ MAX (2 * len, db->head->data_size / 8));
|
|
Packit |
6c4009 |
size_t newtotal = (sizeof (struct database_pers_head)
|
|
Packit |
6c4009 |
+ roundup (db->head->module * sizeof (ref_t), ALIGN)
|
|
Packit |
6c4009 |
+ new_data_size);
|
|
Packit |
6c4009 |
if (newtotal > db->max_db_size)
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
new_data_size -= newtotal - db->max_db_size;
|
|
Packit |
6c4009 |
newtotal = db->max_db_size;
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
if (db->mmap_used && newtotal > oldtotal
|
|
Packit |
6c4009 |
/* We only have to adjust the file size. The new pages
|
|
Packit |
6c4009 |
become magically available. */
|
|
Packit |
6c4009 |
&& TEMP_FAILURE_RETRY_VAL (posix_fallocate (db->wr_fd, oldtotal,
|
|
Packit |
6c4009 |
newtotal
|
|
Packit |
6c4009 |
- oldtotal)) == 0)
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
db->head->data_size = new_data_size;
|
|
Packit |
6c4009 |
tried_resize = true;
|
|
Packit |
6c4009 |
goto retry;
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
if (data_alloc)
|
|
Packit |
6c4009 |
pthread_rwlock_unlock (&db->lock);
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
if (! db->last_alloc_failed)
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
dbg_log (_("no more memory for database '%s'"), dbnames[db - dbs]);
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
db->last_alloc_failed = true;
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
++db->head->addfailed;
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
/* No luck. */
|
|
Packit |
6c4009 |
res = NULL;
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
else
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
db->head->first_free += len;
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
db->last_alloc_failed = false;
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
pthread_mutex_unlock (&db->memlock);
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
return res;
|
|
Packit |
6c4009 |
}
|