|
Packit |
6c4009 |
/* Thread-local storage handling in the ELF dynamic linker. Generic version.
|
|
Packit |
6c4009 |
Copyright (C) 2002-2018 Free Software Foundation, Inc.
|
|
Packit |
6c4009 |
This file is part of the GNU C Library.
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
The GNU C Library is free software; you can redistribute it and/or
|
|
Packit |
6c4009 |
modify it under the terms of the GNU Lesser General Public
|
|
Packit |
6c4009 |
License as published by the Free Software Foundation; either
|
|
Packit |
6c4009 |
version 2.1 of the License, or (at your option) any later version.
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
The GNU C Library is distributed in the hope that it will be useful,
|
|
Packit |
6c4009 |
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
Packit |
6c4009 |
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
Packit |
6c4009 |
Lesser General Public License for more details.
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
You should have received a copy of the GNU Lesser General Public
|
|
Packit |
6c4009 |
License along with the GNU C Library; if not, see
|
|
Packit |
6c4009 |
<http://www.gnu.org/licenses/>. */
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
#include <assert.h>
|
|
Packit |
6c4009 |
#include <errno.h>
|
|
Packit |
6c4009 |
#include <libintl.h>
|
|
Packit |
6c4009 |
#include <signal.h>
|
|
Packit |
6c4009 |
#include <stdlib.h>
|
|
Packit |
6c4009 |
#include <unistd.h>
|
|
Packit |
6c4009 |
#include <sys/param.h>
|
|
Packit |
6c4009 |
#include <atomic.h>
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
#include <tls.h>
|
|
Packit |
6c4009 |
#include <dl-tls.h>
|
|
Packit |
6c4009 |
#include <ldsodefs.h>
|
|
Packit |
6c4009 |
|
|
Packit Service |
541783 |
/* Amount of excess space to allocate in the static TLS area
|
|
Packit Service |
541783 |
to allow dynamic loading of modules defining IE-model TLS data. */
|
|
Packit Service |
541783 |
#define TLS_STATIC_SURPLUS 64 + DL_NNS * 100
|
|
Packit Service |
b7620e |
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
/* Out-of-memory handler. */
|
|
Packit |
6c4009 |
static void
|
|
Packit |
6c4009 |
__attribute__ ((__noreturn__))
|
|
Packit |
6c4009 |
oom (void)
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
_dl_fatal_printf ("cannot allocate memory for thread-local data: ABORT\n");
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
size_t
|
|
Packit |
6c4009 |
_dl_next_tls_modid (void)
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
size_t result;
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
if (__builtin_expect (GL(dl_tls_dtv_gaps), false))
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
size_t disp = 0;
|
|
Packit |
6c4009 |
struct dtv_slotinfo_list *runp = GL(dl_tls_dtv_slotinfo_list);
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
/* Note that this branch will never be executed during program
|
|
Packit |
6c4009 |
start since there are no gaps at that time. Therefore it
|
|
Packit |
6c4009 |
does not matter that the dl_tls_dtv_slotinfo is not allocated
|
|
Packit |
6c4009 |
yet when the function is called for the first times.
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
NB: the offset +1 is due to the fact that DTV[0] is used
|
|
Packit |
6c4009 |
for something else. */
|
|
Packit |
6c4009 |
result = GL(dl_tls_static_nelem) + 1;
|
|
Packit |
6c4009 |
if (result <= GL(dl_tls_max_dtv_idx))
|
|
Packit |
6c4009 |
do
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
while (result - disp < runp->len)
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
if (runp->slotinfo[result - disp].map == NULL)
|
|
Packit |
6c4009 |
break;
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
++result;
|
|
Packit |
6c4009 |
assert (result <= GL(dl_tls_max_dtv_idx) + 1);
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
if (result - disp < runp->len)
|
|
Packit |
6c4009 |
break;
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
disp += runp->len;
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
while ((runp = runp->next) != NULL);
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
if (result > GL(dl_tls_max_dtv_idx))
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
/* The new index must indeed be exactly one higher than the
|
|
Packit |
6c4009 |
previous high. */
|
|
Packit |
6c4009 |
assert (result == GL(dl_tls_max_dtv_idx) + 1);
|
|
Packit |
6c4009 |
/* There is no gap anymore. */
|
|
Packit |
6c4009 |
GL(dl_tls_dtv_gaps) = false;
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
goto nogaps;
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
else
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
/* No gaps, allocate a new entry. */
|
|
Packit |
6c4009 |
nogaps:
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
result = ++GL(dl_tls_max_dtv_idx);
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
return result;
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
size_t
|
|
Packit |
6c4009 |
_dl_count_modids (void)
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
/* It is rare that we have gaps; see elf/dl-open.c (_dl_open) where
|
|
Packit |
6c4009 |
we fail to load a module and unload it leaving a gap. If we don't
|
|
Packit |
6c4009 |
have gaps then the number of modids is the current maximum so
|
|
Packit |
6c4009 |
return that. */
|
|
Packit |
6c4009 |
if (__glibc_likely (!GL(dl_tls_dtv_gaps)))
|
|
Packit |
6c4009 |
return GL(dl_tls_max_dtv_idx);
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
/* We have gaps and are forced to count the non-NULL entries. */
|
|
Packit |
6c4009 |
size_t n = 0;
|
|
Packit |
6c4009 |
struct dtv_slotinfo_list *runp = GL(dl_tls_dtv_slotinfo_list);
|
|
Packit |
6c4009 |
while (runp != NULL)
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
for (size_t i = 0; i < runp->len; ++i)
|
|
Packit |
6c4009 |
if (runp->slotinfo[i].map != NULL)
|
|
Packit |
6c4009 |
++n;
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
runp = runp->next;
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
return n;
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
#ifdef SHARED
|
|
Packit |
6c4009 |
void
|
|
Packit |
6c4009 |
_dl_determine_tlsoffset (void)
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
size_t max_align = TLS_TCB_ALIGN;
|
|
Packit |
6c4009 |
size_t freetop = 0;
|
|
Packit |
6c4009 |
size_t freebottom = 0;
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
/* The first element of the dtv slot info list is allocated. */
|
|
Packit |
6c4009 |
assert (GL(dl_tls_dtv_slotinfo_list) != NULL);
|
|
Packit |
6c4009 |
/* There is at this point only one element in the
|
|
Packit |
6c4009 |
dl_tls_dtv_slotinfo_list list. */
|
|
Packit |
6c4009 |
assert (GL(dl_tls_dtv_slotinfo_list)->next == NULL);
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
struct dtv_slotinfo *slotinfo = GL(dl_tls_dtv_slotinfo_list)->slotinfo;
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
/* Determining the offset of the various parts of the static TLS
|
|
Packit |
6c4009 |
block has several dependencies. In addition we have to work
|
|
Packit |
6c4009 |
around bugs in some toolchains.
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
Each TLS block from the objects available at link time has a size
|
|
Packit |
6c4009 |
and an alignment requirement. The GNU ld computes the alignment
|
|
Packit |
6c4009 |
requirements for the data at the positions *in the file*, though.
|
|
Packit |
6c4009 |
I.e, it is not simply possible to allocate a block with the size
|
|
Packit |
6c4009 |
of the TLS program header entry. The data is layed out assuming
|
|
Packit |
6c4009 |
that the first byte of the TLS block fulfills
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
p_vaddr mod p_align == &TLS_BLOCK mod p_align
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
This means we have to add artificial padding at the beginning of
|
|
Packit |
6c4009 |
the TLS block. These bytes are never used for the TLS data in
|
|
Packit |
6c4009 |
this module but the first byte allocated must be aligned
|
|
Packit |
6c4009 |
according to mod p_align == 0 so that the first byte of the TLS
|
|
Packit |
6c4009 |
block is aligned according to p_vaddr mod p_align. This is ugly
|
|
Packit |
6c4009 |
and the linker can help by computing the offsets in the TLS block
|
|
Packit |
6c4009 |
assuming the first byte of the TLS block is aligned according to
|
|
Packit |
6c4009 |
p_align.
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
The extra space which might be allocated before the first byte of
|
|
Packit |
6c4009 |
the TLS block need not go unused. The code below tries to use
|
|
Packit |
6c4009 |
that memory for the next TLS block. This can work if the total
|
|
Packit |
6c4009 |
memory requirement for the next TLS block is smaller than the
|
|
Packit |
6c4009 |
gap. */
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
#if TLS_TCB_AT_TP
|
|
Packit |
6c4009 |
/* We simply start with zero. */
|
|
Packit |
6c4009 |
size_t offset = 0;
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
for (size_t cnt = 0; slotinfo[cnt].map != NULL; ++cnt)
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
assert (cnt < GL(dl_tls_dtv_slotinfo_list)->len);
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
size_t firstbyte = (-slotinfo[cnt].map->l_tls_firstbyte_offset
|
|
Packit |
6c4009 |
& (slotinfo[cnt].map->l_tls_align - 1));
|
|
Packit |
6c4009 |
size_t off;
|
|
Packit |
6c4009 |
max_align = MAX (max_align, slotinfo[cnt].map->l_tls_align);
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
if (freebottom - freetop >= slotinfo[cnt].map->l_tls_blocksize)
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
off = roundup (freetop + slotinfo[cnt].map->l_tls_blocksize
|
|
Packit |
6c4009 |
- firstbyte, slotinfo[cnt].map->l_tls_align)
|
|
Packit |
6c4009 |
+ firstbyte;
|
|
Packit |
6c4009 |
if (off <= freebottom)
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
freetop = off;
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
/* XXX For some architectures we perhaps should store the
|
|
Packit |
6c4009 |
negative offset. */
|
|
Packit |
6c4009 |
slotinfo[cnt].map->l_tls_offset = off;
|
|
Packit |
6c4009 |
continue;
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
off = roundup (offset + slotinfo[cnt].map->l_tls_blocksize - firstbyte,
|
|
Packit |
6c4009 |
slotinfo[cnt].map->l_tls_align) + firstbyte;
|
|
Packit |
6c4009 |
if (off > offset + slotinfo[cnt].map->l_tls_blocksize
|
|
Packit |
6c4009 |
+ (freebottom - freetop))
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
freetop = offset;
|
|
Packit |
6c4009 |
freebottom = off - slotinfo[cnt].map->l_tls_blocksize;
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
offset = off;
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
/* XXX For some architectures we perhaps should store the
|
|
Packit |
6c4009 |
negative offset. */
|
|
Packit |
6c4009 |
slotinfo[cnt].map->l_tls_offset = off;
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
GL(dl_tls_static_used) = offset;
|
|
Packit Service |
541783 |
GL(dl_tls_static_size) = (roundup (offset + TLS_STATIC_SURPLUS, max_align)
|
|
Packit |
6c4009 |
+ TLS_TCB_SIZE);
|
|
Packit |
6c4009 |
#elif TLS_DTV_AT_TP
|
|
Packit |
6c4009 |
/* The TLS blocks start right after the TCB. */
|
|
Packit |
6c4009 |
size_t offset = TLS_TCB_SIZE;
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
for (size_t cnt = 0; slotinfo[cnt].map != NULL; ++cnt)
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
assert (cnt < GL(dl_tls_dtv_slotinfo_list)->len);
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
size_t firstbyte = (-slotinfo[cnt].map->l_tls_firstbyte_offset
|
|
Packit |
6c4009 |
& (slotinfo[cnt].map->l_tls_align - 1));
|
|
Packit |
6c4009 |
size_t off;
|
|
Packit |
6c4009 |
max_align = MAX (max_align, slotinfo[cnt].map->l_tls_align);
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
if (slotinfo[cnt].map->l_tls_blocksize <= freetop - freebottom)
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
off = roundup (freebottom, slotinfo[cnt].map->l_tls_align);
|
|
Packit |
6c4009 |
if (off - freebottom < firstbyte)
|
|
Packit |
6c4009 |
off += slotinfo[cnt].map->l_tls_align;
|
|
Packit |
6c4009 |
if (off + slotinfo[cnt].map->l_tls_blocksize - firstbyte <= freetop)
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
slotinfo[cnt].map->l_tls_offset = off - firstbyte;
|
|
Packit |
6c4009 |
freebottom = (off + slotinfo[cnt].map->l_tls_blocksize
|
|
Packit |
6c4009 |
- firstbyte);
|
|
Packit |
6c4009 |
continue;
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
off = roundup (offset, slotinfo[cnt].map->l_tls_align);
|
|
Packit |
6c4009 |
if (off - offset < firstbyte)
|
|
Packit |
6c4009 |
off += slotinfo[cnt].map->l_tls_align;
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
slotinfo[cnt].map->l_tls_offset = off - firstbyte;
|
|
Packit |
6c4009 |
if (off - firstbyte - offset > freetop - freebottom)
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
freebottom = offset;
|
|
Packit |
6c4009 |
freetop = off - firstbyte;
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
offset = off + slotinfo[cnt].map->l_tls_blocksize - firstbyte;
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
GL(dl_tls_static_used) = offset;
|
|
Packit Service |
541783 |
GL(dl_tls_static_size) = roundup (offset + TLS_STATIC_SURPLUS,
|
|
Packit |
6c4009 |
TLS_TCB_ALIGN);
|
|
Packit |
6c4009 |
#else
|
|
Packit |
6c4009 |
# error "Either TLS_TCB_AT_TP or TLS_DTV_AT_TP must be defined"
|
|
Packit |
6c4009 |
#endif
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
/* The alignment requirement for the static TLS block. */
|
|
Packit |
6c4009 |
GL(dl_tls_static_align) = max_align;
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
#endif /* SHARED */
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
static void *
|
|
Packit |
6c4009 |
allocate_dtv (void *result)
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
dtv_t *dtv;
|
|
Packit |
6c4009 |
size_t dtv_length;
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
/* We allocate a few more elements in the dtv than are needed for the
|
|
Packit |
6c4009 |
initial set of modules. This should avoid in most cases expansions
|
|
Packit |
6c4009 |
of the dtv. */
|
|
Packit |
6c4009 |
dtv_length = GL(dl_tls_max_dtv_idx) + DTV_SURPLUS;
|
|
Packit |
6c4009 |
dtv = calloc (dtv_length + 2, sizeof (dtv_t));
|
|
Packit |
6c4009 |
if (dtv != NULL)
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
/* This is the initial length of the dtv. */
|
|
Packit |
6c4009 |
dtv[0].counter = dtv_length;
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
/* The rest of the dtv (including the generation counter) is
|
|
Packit |
6c4009 |
Initialize with zero to indicate nothing there. */
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
/* Add the dtv to the thread data structures. */
|
|
Packit |
6c4009 |
INSTALL_DTV (result, dtv);
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
else
|
|
Packit |
6c4009 |
result = NULL;
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
return result;
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
/* Get size and alignment requirements of the static TLS block. */
|
|
Packit |
6c4009 |
void
|
|
Packit |
6c4009 |
_dl_get_tls_static_info (size_t *sizep, size_t *alignp)
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
*sizep = GL(dl_tls_static_size);
|
|
Packit |
6c4009 |
*alignp = GL(dl_tls_static_align);
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
/* Derive the location of the pointer to the start of the original
|
|
Packit |
6c4009 |
allocation (before alignment) from the pointer to the TCB. */
|
|
Packit |
6c4009 |
static inline void **
|
|
Packit |
6c4009 |
tcb_to_pointer_to_free_location (void *tcb)
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
#if TLS_TCB_AT_TP
|
|
Packit |
6c4009 |
/* The TCB follows the TLS blocks, and the pointer to the front
|
|
Packit |
6c4009 |
follows the TCB. */
|
|
Packit |
6c4009 |
void **original_pointer_location = tcb + TLS_TCB_SIZE;
|
|
Packit |
6c4009 |
#elif TLS_DTV_AT_TP
|
|
Packit |
6c4009 |
/* The TCB comes first, preceded by the pre-TCB, and the pointer is
|
|
Packit |
6c4009 |
before that. */
|
|
Packit |
6c4009 |
void **original_pointer_location = tcb - TLS_PRE_TCB_SIZE - sizeof (void *);
|
|
Packit |
6c4009 |
#endif
|
|
Packit |
6c4009 |
return original_pointer_location;
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
void *
|
|
Packit |
6c4009 |
_dl_allocate_tls_storage (void)
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
void *result;
|
|
Packit |
6c4009 |
size_t size = GL(dl_tls_static_size);
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
#if TLS_DTV_AT_TP
|
|
Packit |
6c4009 |
/* Memory layout is:
|
|
Packit |
6c4009 |
[ TLS_PRE_TCB_SIZE ] [ TLS_TCB_SIZE ] [ TLS blocks ]
|
|
Packit |
6c4009 |
^ This should be returned. */
|
|
Packit |
6c4009 |
size += TLS_PRE_TCB_SIZE;
|
|
Packit |
6c4009 |
#endif
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
/* Perform the allocation. Reserve space for the required alignment
|
|
Packit |
6c4009 |
and the pointer to the original allocation. */
|
|
Packit |
6c4009 |
size_t alignment = GL(dl_tls_static_align);
|
|
Packit |
6c4009 |
void *allocated = malloc (size + alignment + sizeof (void *));
|
|
Packit |
6c4009 |
if (__glibc_unlikely (allocated == NULL))
|
|
Packit |
6c4009 |
return NULL;
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
/* Perform alignment and allocate the DTV. */
|
|
Packit |
6c4009 |
#if TLS_TCB_AT_TP
|
|
Packit |
6c4009 |
/* The TCB follows the TLS blocks, which determine the alignment.
|
|
Packit |
6c4009 |
(TCB alignment requirements have been taken into account when
|
|
Packit |
6c4009 |
calculating GL(dl_tls_static_align).) */
|
|
Packit |
6c4009 |
void *aligned = (void *) roundup ((uintptr_t) allocated, alignment);
|
|
Packit |
6c4009 |
result = aligned + size - TLS_TCB_SIZE;
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
/* Clear the TCB data structure. We can't ask the caller (i.e.
|
|
Packit |
6c4009 |
libpthread) to do it, because we will initialize the DTV et al. */
|
|
Packit |
6c4009 |
memset (result, '\0', TLS_TCB_SIZE);
|
|
Packit |
6c4009 |
#elif TLS_DTV_AT_TP
|
|
Packit |
6c4009 |
/* Pre-TCB and TCB come before the TLS blocks. The layout computed
|
|
Packit |
6c4009 |
in _dl_determine_tlsoffset assumes that the TCB is aligned to the
|
|
Packit |
6c4009 |
TLS block alignment, and not just the TLS blocks after it. This
|
|
Packit |
6c4009 |
can leave an unused alignment gap between the TCB and the TLS
|
|
Packit |
6c4009 |
blocks. */
|
|
Packit |
6c4009 |
result = (void *) roundup
|
|
Packit |
6c4009 |
(sizeof (void *) + TLS_PRE_TCB_SIZE + (uintptr_t) allocated,
|
|
Packit |
6c4009 |
alignment);
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
/* Clear the TCB data structure and TLS_PRE_TCB_SIZE bytes before
|
|
Packit |
6c4009 |
it. We can't ask the caller (i.e. libpthread) to do it, because
|
|
Packit |
6c4009 |
we will initialize the DTV et al. */
|
|
Packit |
6c4009 |
memset (result - TLS_PRE_TCB_SIZE, '\0', TLS_PRE_TCB_SIZE + TLS_TCB_SIZE);
|
|
Packit |
6c4009 |
#endif
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
/* Record the value of the original pointer for later
|
|
Packit |
6c4009 |
deallocation. */
|
|
Packit |
6c4009 |
*tcb_to_pointer_to_free_location (result) = allocated;
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
result = allocate_dtv (result);
|
|
Packit |
6c4009 |
if (result == NULL)
|
|
Packit |
6c4009 |
free (allocated);
|
|
Packit |
6c4009 |
return result;
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
#ifndef SHARED
|
|
Packit |
6c4009 |
extern dtv_t _dl_static_dtv[];
|
|
Packit |
6c4009 |
# define _dl_initial_dtv (&_dl_static_dtv[1])
|
|
Packit |
6c4009 |
#endif
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
static dtv_t *
|
|
Packit |
6c4009 |
_dl_resize_dtv (dtv_t *dtv)
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
/* Resize the dtv. */
|
|
Packit |
6c4009 |
dtv_t *newp;
|
|
Packit |
6c4009 |
/* Load GL(dl_tls_max_dtv_idx) atomically since it may be written to by
|
|
Packit |
6c4009 |
other threads concurrently. */
|
|
Packit |
6c4009 |
size_t newsize
|
|
Packit |
6c4009 |
= atomic_load_acquire (&GL(dl_tls_max_dtv_idx)) + DTV_SURPLUS;
|
|
Packit |
6c4009 |
size_t oldsize = dtv[-1].counter;
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
if (dtv == GL(dl_initial_dtv))
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
/* This is the initial dtv that was either statically allocated in
|
|
Packit |
6c4009 |
__libc_setup_tls or allocated during rtld startup using the
|
|
Packit |
6c4009 |
dl-minimal.c malloc instead of the real malloc. We can't free
|
|
Packit |
6c4009 |
it, we have to abandon the old storage. */
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
newp = malloc ((2 + newsize) * sizeof (dtv_t));
|
|
Packit |
6c4009 |
if (newp == NULL)
|
|
Packit |
6c4009 |
oom ();
|
|
Packit |
6c4009 |
memcpy (newp, &dtv[-1], (2 + oldsize) * sizeof (dtv_t));
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
else
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
newp = realloc (&dtv[-1],
|
|
Packit |
6c4009 |
(2 + newsize) * sizeof (dtv_t));
|
|
Packit |
6c4009 |
if (newp == NULL)
|
|
Packit |
6c4009 |
oom ();
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
newp[0].counter = newsize;
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
/* Clear the newly allocated part. */
|
|
Packit |
6c4009 |
memset (newp + 2 + oldsize, '\0',
|
|
Packit |
6c4009 |
(newsize - oldsize) * sizeof (dtv_t));
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
/* Return the generation counter. */
|
|
Packit |
6c4009 |
return &newp[1];
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
void *
|
|
Packit |
6c4009 |
_dl_allocate_tls_init (void *result)
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
if (result == NULL)
|
|
Packit |
6c4009 |
/* The memory allocation failed. */
|
|
Packit |
6c4009 |
return NULL;
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
dtv_t *dtv = GET_DTV (result);
|
|
Packit |
6c4009 |
struct dtv_slotinfo_list *listp;
|
|
Packit |
6c4009 |
size_t total = 0;
|
|
Packit |
6c4009 |
size_t maxgen = 0;
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
/* Check if the current dtv is big enough. */
|
|
Packit |
6c4009 |
if (dtv[-1].counter < GL(dl_tls_max_dtv_idx))
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
/* Resize the dtv. */
|
|
Packit |
6c4009 |
dtv = _dl_resize_dtv (dtv);
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
/* Install this new dtv in the thread data structures. */
|
|
Packit |
6c4009 |
INSTALL_DTV (result, &dtv[-1]);
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
/* We have to prepare the dtv for all currently loaded modules using
|
|
Packit |
6c4009 |
TLS. For those which are dynamically loaded we add the values
|
|
Packit |
6c4009 |
indicating deferred allocation. */
|
|
Packit |
6c4009 |
listp = GL(dl_tls_dtv_slotinfo_list);
|
|
Packit |
6c4009 |
while (1)
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
size_t cnt;
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
for (cnt = total == 0 ? 1 : 0; cnt < listp->len; ++cnt)
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
struct link_map *map;
|
|
Packit |
6c4009 |
void *dest;
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
/* Check for the total number of used slots. */
|
|
Packit |
6c4009 |
if (total + cnt > GL(dl_tls_max_dtv_idx))
|
|
Packit |
6c4009 |
break;
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
map = listp->slotinfo[cnt].map;
|
|
Packit |
6c4009 |
if (map == NULL)
|
|
Packit |
6c4009 |
/* Unused entry. */
|
|
Packit |
6c4009 |
continue;
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
/* Keep track of the maximum generation number. This might
|
|
Packit |
6c4009 |
not be the generation counter. */
|
|
Packit |
6c4009 |
assert (listp->slotinfo[cnt].gen <= GL(dl_tls_generation));
|
|
Packit |
6c4009 |
maxgen = MAX (maxgen, listp->slotinfo[cnt].gen);
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
dtv[map->l_tls_modid].pointer.val = TLS_DTV_UNALLOCATED;
|
|
Packit |
6c4009 |
dtv[map->l_tls_modid].pointer.to_free = NULL;
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
if (map->l_tls_offset == NO_TLS_OFFSET
|
|
Packit |
6c4009 |
|| map->l_tls_offset == FORCED_DYNAMIC_TLS_OFFSET)
|
|
Packit |
6c4009 |
continue;
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
assert (map->l_tls_modid == total + cnt);
|
|
Packit |
6c4009 |
assert (map->l_tls_blocksize >= map->l_tls_initimage_size);
|
|
Packit |
6c4009 |
#if TLS_TCB_AT_TP
|
|
Packit |
6c4009 |
assert ((size_t) map->l_tls_offset >= map->l_tls_blocksize);
|
|
Packit |
6c4009 |
dest = (char *) result - map->l_tls_offset;
|
|
Packit |
6c4009 |
#elif TLS_DTV_AT_TP
|
|
Packit |
6c4009 |
dest = (char *) result + map->l_tls_offset;
|
|
Packit |
6c4009 |
#else
|
|
Packit |
6c4009 |
# error "Either TLS_TCB_AT_TP or TLS_DTV_AT_TP must be defined"
|
|
Packit |
6c4009 |
#endif
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
/* Set up the DTV entry. The simplified __tls_get_addr that
|
|
Packit |
6c4009 |
some platforms use in static programs requires it. */
|
|
Packit |
6c4009 |
dtv[map->l_tls_modid].pointer.val = dest;
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
/* Copy the initialization image and clear the BSS part. */
|
|
Packit |
6c4009 |
memset (__mempcpy (dest, map->l_tls_initimage,
|
|
Packit |
6c4009 |
map->l_tls_initimage_size), '\0',
|
|
Packit |
6c4009 |
map->l_tls_blocksize - map->l_tls_initimage_size);
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
total += cnt;
|
|
Packit |
6c4009 |
if (total >= GL(dl_tls_max_dtv_idx))
|
|
Packit |
6c4009 |
break;
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
listp = listp->next;
|
|
Packit |
6c4009 |
assert (listp != NULL);
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
/* The DTV version is up-to-date now. */
|
|
Packit |
6c4009 |
dtv[0].counter = maxgen;
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
return result;
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
rtld_hidden_def (_dl_allocate_tls_init)
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
void *
|
|
Packit |
6c4009 |
_dl_allocate_tls (void *mem)
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
return _dl_allocate_tls_init (mem == NULL
|
|
Packit |
6c4009 |
? _dl_allocate_tls_storage ()
|
|
Packit |
6c4009 |
: allocate_dtv (mem));
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
rtld_hidden_def (_dl_allocate_tls)
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
void
|
|
Packit |
6c4009 |
_dl_deallocate_tls (void *tcb, bool dealloc_tcb)
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
dtv_t *dtv = GET_DTV (tcb);
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
/* We need to free the memory allocated for non-static TLS. */
|
|
Packit |
6c4009 |
for (size_t cnt = 0; cnt < dtv[-1].counter; ++cnt)
|
|
Packit |
6c4009 |
free (dtv[1 + cnt].pointer.to_free);
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
/* The array starts with dtv[-1]. */
|
|
Packit |
6c4009 |
if (dtv != GL(dl_initial_dtv))
|
|
Packit |
6c4009 |
free (dtv - 1);
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
if (dealloc_tcb)
|
|
Packit |
6c4009 |
free (*tcb_to_pointer_to_free_location (tcb));
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
rtld_hidden_def (_dl_deallocate_tls)
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
#ifdef SHARED
|
|
Packit |
6c4009 |
/* The __tls_get_addr function has two basic forms which differ in the
|
|
Packit |
6c4009 |
arguments. The IA-64 form takes two parameters, the module ID and
|
|
Packit |
6c4009 |
offset. The form used, among others, on IA-32 takes a reference to
|
|
Packit |
6c4009 |
a special structure which contain the same information. The second
|
|
Packit |
6c4009 |
form seems to be more often used (in the moment) so we default to
|
|
Packit |
6c4009 |
it. Users of the IA-64 form have to provide adequate definitions
|
|
Packit |
6c4009 |
of the following macros. */
|
|
Packit |
6c4009 |
# ifndef GET_ADDR_ARGS
|
|
Packit |
6c4009 |
# define GET_ADDR_ARGS tls_index *ti
|
|
Packit |
6c4009 |
# define GET_ADDR_PARAM ti
|
|
Packit |
6c4009 |
# endif
|
|
Packit |
6c4009 |
# ifndef GET_ADDR_MODULE
|
|
Packit |
6c4009 |
# define GET_ADDR_MODULE ti->ti_module
|
|
Packit |
6c4009 |
# endif
|
|
Packit |
6c4009 |
# ifndef GET_ADDR_OFFSET
|
|
Packit |
6c4009 |
# define GET_ADDR_OFFSET ti->ti_offset
|
|
Packit |
6c4009 |
# endif
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
/* Allocate one DTV entry. */
|
|
Packit |
6c4009 |
static struct dtv_pointer
|
|
Packit |
6c4009 |
allocate_dtv_entry (size_t alignment, size_t size)
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
if (powerof2 (alignment) && alignment <= _Alignof (max_align_t))
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
/* The alignment is supported by malloc. */
|
|
Packit |
6c4009 |
void *ptr = malloc (size);
|
|
Packit |
6c4009 |
return (struct dtv_pointer) { ptr, ptr };
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
/* Emulate memalign to by manually aligning a pointer returned by
|
|
Packit |
6c4009 |
malloc. First compute the size with an overflow check. */
|
|
Packit |
6c4009 |
size_t alloc_size = size + alignment;
|
|
Packit |
6c4009 |
if (alloc_size < size)
|
|
Packit |
6c4009 |
return (struct dtv_pointer) {};
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
/* Perform the allocation. This is the pointer we need to free
|
|
Packit |
6c4009 |
later. */
|
|
Packit |
6c4009 |
void *start = malloc (alloc_size);
|
|
Packit |
6c4009 |
if (start == NULL)
|
|
Packit |
6c4009 |
return (struct dtv_pointer) {};
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
/* Find the aligned position within the larger allocation. */
|
|
Packit |
6c4009 |
void *aligned = (void *) roundup ((uintptr_t) start, alignment);
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
return (struct dtv_pointer) { .val = aligned, .to_free = start };
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
static struct dtv_pointer
|
|
Packit |
6c4009 |
allocate_and_init (struct link_map *map)
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
struct dtv_pointer result = allocate_dtv_entry
|
|
Packit |
6c4009 |
(map->l_tls_align, map->l_tls_blocksize);
|
|
Packit |
6c4009 |
if (result.val == NULL)
|
|
Packit |
6c4009 |
oom ();
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
/* Initialize the memory. */
|
|
Packit |
6c4009 |
memset (__mempcpy (result.val, map->l_tls_initimage,
|
|
Packit |
6c4009 |
map->l_tls_initimage_size),
|
|
Packit |
6c4009 |
'\0', map->l_tls_blocksize - map->l_tls_initimage_size);
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
return result;
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
struct link_map *
|
|
Packit |
6c4009 |
_dl_update_slotinfo (unsigned long int req_modid)
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
struct link_map *the_map = NULL;
|
|
Packit |
6c4009 |
dtv_t *dtv = THREAD_DTV ();
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
/* The global dl_tls_dtv_slotinfo array contains for each module
|
|
Packit |
6c4009 |
index the generation counter current when the entry was created.
|
|
Packit |
6c4009 |
This array never shrinks so that all module indices which were
|
|
Packit |
6c4009 |
valid at some time can be used to access it. Before the first
|
|
Packit |
6c4009 |
use of a new module index in this function the array was extended
|
|
Packit |
6c4009 |
appropriately. Access also does not have to be guarded against
|
|
Packit |
6c4009 |
modifications of the array. It is assumed that pointer-size
|
|
Packit |
6c4009 |
values can be read atomically even in SMP environments. It is
|
|
Packit |
6c4009 |
possible that other threads at the same time dynamically load
|
|
Packit |
6c4009 |
code and therefore add to the slotinfo list. This is a problem
|
|
Packit |
6c4009 |
since we must not pick up any information about incomplete work.
|
|
Packit |
6c4009 |
The solution to this is to ignore all dtv slots which were
|
|
Packit |
6c4009 |
created after the one we are currently interested. We know that
|
|
Packit |
6c4009 |
dynamic loading for this module is completed and this is the last
|
|
Packit |
6c4009 |
load operation we know finished. */
|
|
Packit |
6c4009 |
unsigned long int idx = req_modid;
|
|
Packit |
6c4009 |
struct dtv_slotinfo_list *listp = GL(dl_tls_dtv_slotinfo_list);
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
while (idx >= listp->len)
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
idx -= listp->len;
|
|
Packit |
6c4009 |
listp = listp->next;
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
if (dtv[0].counter < listp->slotinfo[idx].gen)
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
/* The generation counter for the slot is higher than what the
|
|
Packit |
6c4009 |
current dtv implements. We have to update the whole dtv but
|
|
Packit |
6c4009 |
only those entries with a generation counter <= the one for
|
|
Packit |
6c4009 |
the entry we need. */
|
|
Packit |
6c4009 |
size_t new_gen = listp->slotinfo[idx].gen;
|
|
Packit |
6c4009 |
size_t total = 0;
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
/* We have to look through the entire dtv slotinfo list. */
|
|
Packit |
6c4009 |
listp = GL(dl_tls_dtv_slotinfo_list);
|
|
Packit |
6c4009 |
do
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
for (size_t cnt = total == 0 ? 1 : 0; cnt < listp->len; ++cnt)
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
size_t gen = listp->slotinfo[cnt].gen;
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
if (gen > new_gen)
|
|
Packit |
6c4009 |
/* This is a slot for a generation younger than the
|
|
Packit |
6c4009 |
one we are handling now. It might be incompletely
|
|
Packit |
6c4009 |
set up so ignore it. */
|
|
Packit |
6c4009 |
continue;
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
/* If the entry is older than the current dtv layout we
|
|
Packit |
6c4009 |
know we don't have to handle it. */
|
|
Packit |
6c4009 |
if (gen <= dtv[0].counter)
|
|
Packit |
6c4009 |
continue;
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
/* If there is no map this means the entry is empty. */
|
|
Packit |
6c4009 |
struct link_map *map = listp->slotinfo[cnt].map;
|
|
Packit |
6c4009 |
if (map == NULL)
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
if (dtv[-1].counter >= total + cnt)
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
/* If this modid was used at some point the memory
|
|
Packit |
6c4009 |
might still be allocated. */
|
|
Packit |
6c4009 |
free (dtv[total + cnt].pointer.to_free);
|
|
Packit |
6c4009 |
dtv[total + cnt].pointer.val = TLS_DTV_UNALLOCATED;
|
|
Packit |
6c4009 |
dtv[total + cnt].pointer.to_free = NULL;
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
continue;
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
/* Check whether the current dtv array is large enough. */
|
|
Packit |
6c4009 |
size_t modid = map->l_tls_modid;
|
|
Packit |
6c4009 |
assert (total + cnt == modid);
|
|
Packit |
6c4009 |
if (dtv[-1].counter < modid)
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
/* Resize the dtv. */
|
|
Packit |
6c4009 |
dtv = _dl_resize_dtv (dtv);
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
assert (modid <= dtv[-1].counter);
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
/* Install this new dtv in the thread data
|
|
Packit |
6c4009 |
structures. */
|
|
Packit |
6c4009 |
INSTALL_NEW_DTV (dtv);
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
/* If there is currently memory allocate for this
|
|
Packit |
6c4009 |
dtv entry free it. */
|
|
Packit |
6c4009 |
/* XXX Ideally we will at some point create a memory
|
|
Packit |
6c4009 |
pool. */
|
|
Packit |
6c4009 |
free (dtv[modid].pointer.to_free);
|
|
Packit |
6c4009 |
dtv[modid].pointer.val = TLS_DTV_UNALLOCATED;
|
|
Packit |
6c4009 |
dtv[modid].pointer.to_free = NULL;
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
if (modid == req_modid)
|
|
Packit |
6c4009 |
the_map = map;
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
total += listp->len;
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
while ((listp = listp->next) != NULL);
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
/* This will be the new maximum generation counter. */
|
|
Packit |
6c4009 |
dtv[0].counter = new_gen;
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
return the_map;
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
static void *
|
|
Packit |
6c4009 |
__attribute_noinline__
|
|
Packit |
6c4009 |
tls_get_addr_tail (GET_ADDR_ARGS, dtv_t *dtv, struct link_map *the_map)
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
/* The allocation was deferred. Do it now. */
|
|
Packit |
6c4009 |
if (the_map == NULL)
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
/* Find the link map for this module. */
|
|
Packit |
6c4009 |
size_t idx = GET_ADDR_MODULE;
|
|
Packit |
6c4009 |
struct dtv_slotinfo_list *listp = GL(dl_tls_dtv_slotinfo_list);
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
while (idx >= listp->len)
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
idx -= listp->len;
|
|
Packit |
6c4009 |
listp = listp->next;
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
the_map = listp->slotinfo[idx].map;
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
/* Make sure that, if a dlopen running in parallel forces the
|
|
Packit |
6c4009 |
variable into static storage, we'll wait until the address in the
|
|
Packit |
6c4009 |
static TLS block is set up, and use that. If we're undecided
|
|
Packit |
6c4009 |
yet, make sure we make the decision holding the lock as well. */
|
|
Packit |
6c4009 |
if (__glibc_unlikely (the_map->l_tls_offset
|
|
Packit |
6c4009 |
!= FORCED_DYNAMIC_TLS_OFFSET))
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
__rtld_lock_lock_recursive (GL(dl_load_lock));
|
|
Packit |
6c4009 |
if (__glibc_likely (the_map->l_tls_offset == NO_TLS_OFFSET))
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
the_map->l_tls_offset = FORCED_DYNAMIC_TLS_OFFSET;
|
|
Packit |
6c4009 |
__rtld_lock_unlock_recursive (GL(dl_load_lock));
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
else if (__glibc_likely (the_map->l_tls_offset
|
|
Packit |
6c4009 |
!= FORCED_DYNAMIC_TLS_OFFSET))
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
#if TLS_TCB_AT_TP
|
|
Packit |
6c4009 |
void *p = (char *) THREAD_SELF - the_map->l_tls_offset;
|
|
Packit |
6c4009 |
#elif TLS_DTV_AT_TP
|
|
Packit |
6c4009 |
void *p = (char *) THREAD_SELF + the_map->l_tls_offset + TLS_PRE_TCB_SIZE;
|
|
Packit |
6c4009 |
#else
|
|
Packit |
6c4009 |
# error "Either TLS_TCB_AT_TP or TLS_DTV_AT_TP must be defined"
|
|
Packit |
6c4009 |
#endif
|
|
Packit |
6c4009 |
__rtld_lock_unlock_recursive (GL(dl_load_lock));
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
dtv[GET_ADDR_MODULE].pointer.to_free = NULL;
|
|
Packit |
6c4009 |
dtv[GET_ADDR_MODULE].pointer.val = p;
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
return (char *) p + GET_ADDR_OFFSET;
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
else
|
|
Packit |
6c4009 |
__rtld_lock_unlock_recursive (GL(dl_load_lock));
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
struct dtv_pointer result = allocate_and_init (the_map);
|
|
Packit |
6c4009 |
dtv[GET_ADDR_MODULE].pointer = result;
|
|
Packit |
6c4009 |
assert (result.to_free != NULL);
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
return (char *) result.val + GET_ADDR_OFFSET;
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
static struct link_map *
|
|
Packit |
6c4009 |
__attribute_noinline__
|
|
Packit |
6c4009 |
update_get_addr (GET_ADDR_ARGS)
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
struct link_map *the_map = _dl_update_slotinfo (GET_ADDR_MODULE);
|
|
Packit |
6c4009 |
dtv_t *dtv = THREAD_DTV ();
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
void *p = dtv[GET_ADDR_MODULE].pointer.val;
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
if (__glibc_unlikely (p == TLS_DTV_UNALLOCATED))
|
|
Packit |
6c4009 |
return tls_get_addr_tail (GET_ADDR_PARAM, dtv, the_map);
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
return (void *) p + GET_ADDR_OFFSET;
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
/* For all machines that have a non-macro version of __tls_get_addr, we
|
|
Packit |
6c4009 |
want to use rtld_hidden_proto/rtld_hidden_def in order to call the
|
|
Packit |
6c4009 |
internal alias for __tls_get_addr from ld.so. This avoids a PLT entry
|
|
Packit |
6c4009 |
in ld.so for __tls_get_addr. */
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
#ifndef __tls_get_addr
|
|
Packit |
6c4009 |
extern void * __tls_get_addr (GET_ADDR_ARGS);
|
|
Packit |
6c4009 |
rtld_hidden_proto (__tls_get_addr)
|
|
Packit |
6c4009 |
rtld_hidden_def (__tls_get_addr)
|
|
Packit |
6c4009 |
#endif
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
/* The generic dynamic and local dynamic model cannot be used in
|
|
Packit |
6c4009 |
statically linked applications. */
|
|
Packit |
6c4009 |
void *
|
|
Packit |
6c4009 |
__tls_get_addr (GET_ADDR_ARGS)
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
dtv_t *dtv = THREAD_DTV ();
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
if (__glibc_unlikely (dtv[0].counter != GL(dl_tls_generation)))
|
|
Packit |
6c4009 |
return update_get_addr (GET_ADDR_PARAM);
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
void *p = dtv[GET_ADDR_MODULE].pointer.val;
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
if (__glibc_unlikely (p == TLS_DTV_UNALLOCATED))
|
|
Packit |
6c4009 |
return tls_get_addr_tail (GET_ADDR_PARAM, dtv, NULL);
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
return (char *) p + GET_ADDR_OFFSET;
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
#endif
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
/* Look up the module's TLS block as for __tls_get_addr,
|
|
Packit |
6c4009 |
but never touch anything. Return null if it's not allocated yet. */
|
|
Packit |
6c4009 |
void *
|
|
Packit |
6c4009 |
_dl_tls_get_addr_soft (struct link_map *l)
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
if (__glibc_unlikely (l->l_tls_modid == 0))
|
|
Packit |
6c4009 |
/* This module has no TLS segment. */
|
|
Packit |
6c4009 |
return NULL;
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
dtv_t *dtv = THREAD_DTV ();
|
|
Packit |
6c4009 |
if (__glibc_unlikely (dtv[0].counter != GL(dl_tls_generation)))
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
/* This thread's DTV is not completely current,
|
|
Packit |
6c4009 |
but it might already cover this module. */
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
if (l->l_tls_modid >= dtv[-1].counter)
|
|
Packit |
6c4009 |
/* Nope. */
|
|
Packit |
6c4009 |
return NULL;
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
size_t idx = l->l_tls_modid;
|
|
Packit |
6c4009 |
struct dtv_slotinfo_list *listp = GL(dl_tls_dtv_slotinfo_list);
|
|
Packit |
6c4009 |
while (idx >= listp->len)
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
idx -= listp->len;
|
|
Packit |
6c4009 |
listp = listp->next;
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
/* We've reached the slot for this module.
|
|
Packit |
6c4009 |
If its generation counter is higher than the DTV's,
|
|
Packit |
6c4009 |
this thread does not know about this module yet. */
|
|
Packit |
6c4009 |
if (dtv[0].counter < listp->slotinfo[idx].gen)
|
|
Packit |
6c4009 |
return NULL;
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
void *data = dtv[l->l_tls_modid].pointer.val;
|
|
Packit |
6c4009 |
if (__glibc_unlikely (data == TLS_DTV_UNALLOCATED))
|
|
Packit |
6c4009 |
/* The DTV is current, but this thread has not yet needed
|
|
Packit |
6c4009 |
to allocate this module's segment. */
|
|
Packit |
6c4009 |
data = NULL;
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
return data;
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
void
|
|
Packit Service |
541783 |
_dl_add_to_slotinfo (struct link_map *l)
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
/* Now that we know the object is loaded successfully add
|
|
Packit |
6c4009 |
modules containing TLS data to the dtv info table. We
|
|
Packit |
6c4009 |
might have to increase its size. */
|
|
Packit |
6c4009 |
struct dtv_slotinfo_list *listp;
|
|
Packit |
6c4009 |
struct dtv_slotinfo_list *prevp;
|
|
Packit |
6c4009 |
size_t idx = l->l_tls_modid;
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
/* Find the place in the dtv slotinfo list. */
|
|
Packit |
6c4009 |
listp = GL(dl_tls_dtv_slotinfo_list);
|
|
Packit |
6c4009 |
prevp = NULL; /* Needed to shut up gcc. */
|
|
Packit |
6c4009 |
do
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
/* Does it fit in the array of this list element? */
|
|
Packit |
6c4009 |
if (idx < listp->len)
|
|
Packit |
6c4009 |
break;
|
|
Packit |
6c4009 |
idx -= listp->len;
|
|
Packit |
6c4009 |
prevp = listp;
|
|
Packit |
6c4009 |
listp = listp->next;
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
while (listp != NULL);
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
if (listp == NULL)
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
/* When we come here it means we have to add a new element
|
|
Packit |
6c4009 |
to the slotinfo list. And the new module must be in
|
|
Packit |
6c4009 |
the first slot. */
|
|
Packit |
6c4009 |
assert (idx == 0);
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
listp = prevp->next = (struct dtv_slotinfo_list *)
|
|
Packit |
6c4009 |
malloc (sizeof (struct dtv_slotinfo_list)
|
|
Packit |
6c4009 |
+ TLS_SLOTINFO_SURPLUS * sizeof (struct dtv_slotinfo));
|
|
Packit |
6c4009 |
if (listp == NULL)
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
/* We ran out of memory. We will simply fail this
|
|
Packit |
6c4009 |
call but don't undo anything we did so far. The
|
|
Packit |
6c4009 |
application will crash or be terminated anyway very
|
|
Packit |
6c4009 |
soon. */
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
/* We have to do this since some entries in the dtv
|
|
Packit |
6c4009 |
slotinfo array might already point to this
|
|
Packit |
6c4009 |
generation. */
|
|
Packit |
6c4009 |
++GL(dl_tls_generation);
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
_dl_signal_error (ENOMEM, "dlopen", NULL, N_("\
|
|
Packit |
6c4009 |
cannot create TLS data structures"));
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
listp->len = TLS_SLOTINFO_SURPLUS;
|
|
Packit |
6c4009 |
listp->next = NULL;
|
|
Packit |
6c4009 |
memset (listp->slotinfo, '\0',
|
|
Packit |
6c4009 |
TLS_SLOTINFO_SURPLUS * sizeof (struct dtv_slotinfo));
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
/* Add the information into the slotinfo data structure. */
|
|
Packit Service |
541783 |
listp->slotinfo[idx].map = l;
|
|
Packit Service |
541783 |
listp->slotinfo[idx].gen = GL(dl_tls_generation) + 1;
|
|
Packit |
6c4009 |
}
|