|
Packit Service |
82fcde |
/* Thread-local storage handling in the ELF dynamic linker. x86_64 version.
|
|
Packit Service |
82fcde |
Copyright (C) 2004-2018 Free Software Foundation, Inc.
|
|
Packit Service |
82fcde |
This file is part of the GNU C Library.
|
|
Packit Service |
82fcde |
|
|
Packit Service |
82fcde |
The GNU C Library is free software; you can redistribute it and/or
|
|
Packit Service |
82fcde |
modify it under the terms of the GNU Lesser General Public
|
|
Packit Service |
82fcde |
License as published by the Free Software Foundation; either
|
|
Packit Service |
82fcde |
version 2.1 of the License, or (at your option) any later version.
|
|
Packit Service |
82fcde |
|
|
Packit Service |
82fcde |
The GNU C Library is distributed in the hope that it will be useful,
|
|
Packit Service |
82fcde |
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
Packit Service |
82fcde |
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
Packit Service |
82fcde |
Lesser General Public License for more details.
|
|
Packit Service |
82fcde |
|
|
Packit Service |
82fcde |
You should have received a copy of the GNU Lesser General Public
|
|
Packit Service |
82fcde |
License along with the GNU C Library; if not, see
|
|
Packit Service |
82fcde |
<http://www.gnu.org/licenses/>. */
|
|
Packit Service |
82fcde |
|
|
Packit Service |
82fcde |
#include <sysdep.h>
|
|
Packit Service |
82fcde |
#include <tls.h>
|
|
Packit Service |
82fcde |
#include "tlsdesc.h"
|
|
Packit Service |
82fcde |
|
|
Packit Service |
82fcde |
.text
|
|
Packit Service |
82fcde |
|
|
Packit Service |
82fcde |
/* This function is used to compute the TP offset for symbols in
|
|
Packit Service |
82fcde |
Static TLS, i.e., whose TP offset is the same for all
|
|
Packit Service |
82fcde |
threads.
|
|
Packit Service |
82fcde |
|
|
Packit Service |
82fcde |
The incoming %rax points to the TLS descriptor, such that
|
|
Packit Service |
82fcde |
0(%rax) points to _dl_tlsdesc_return itself, and 8(%rax) holds
|
|
Packit Service |
82fcde |
the TP offset of the symbol corresponding to the object
|
|
Packit Service |
82fcde |
denoted by the argument. */
|
|
Packit Service |
82fcde |
|
|
Packit Service |
82fcde |
.hidden _dl_tlsdesc_return
|
|
Packit Service |
82fcde |
.global _dl_tlsdesc_return
|
|
Packit Service |
82fcde |
.type _dl_tlsdesc_return,@function
|
|
Packit Service |
82fcde |
cfi_startproc
|
|
Packit Service |
82fcde |
.align 16
|
|
Packit Service |
82fcde |
_dl_tlsdesc_return:
|
|
Packit Service |
82fcde |
_CET_ENDBR
|
|
Packit Service |
82fcde |
movq 8(%rax), %rax
|
|
Packit Service |
82fcde |
ret
|
|
Packit Service |
82fcde |
cfi_endproc
|
|
Packit Service |
82fcde |
.size _dl_tlsdesc_return, .-_dl_tlsdesc_return
|
|
Packit Service |
82fcde |
|
|
Packit Service |
82fcde |
/* This function is used for undefined weak TLS symbols, for
|
|
Packit Service |
82fcde |
which the base address (i.e., disregarding any addend) should
|
|
Packit Service |
82fcde |
resolve to NULL.
|
|
Packit Service |
82fcde |
|
|
Packit Service |
82fcde |
%rax points to the TLS descriptor, such that 0(%rax) points to
|
|
Packit Service |
82fcde |
_dl_tlsdesc_undefweak itself, and 8(%rax) holds the addend.
|
|
Packit Service |
82fcde |
We return the addend minus the TP, such that, when the caller
|
|
Packit Service |
82fcde |
adds TP, it gets the addend back. If that's zero, as usual,
|
|
Packit Service |
82fcde |
that's most likely a NULL pointer. */
|
|
Packit Service |
82fcde |
|
|
Packit Service |
82fcde |
.hidden _dl_tlsdesc_undefweak
|
|
Packit Service |
82fcde |
.global _dl_tlsdesc_undefweak
|
|
Packit Service |
82fcde |
.type _dl_tlsdesc_undefweak,@function
|
|
Packit Service |
82fcde |
cfi_startproc
|
|
Packit Service |
82fcde |
.align 16
|
|
Packit Service |
82fcde |
_dl_tlsdesc_undefweak:
|
|
Packit Service |
82fcde |
_CET_ENDBR
|
|
Packit Service |
82fcde |
movq 8(%rax), %rax
|
|
Packit Service |
82fcde |
subq %fs:0, %rax
|
|
Packit Service |
82fcde |
ret
|
|
Packit Service |
82fcde |
cfi_endproc
|
|
Packit Service |
82fcde |
.size _dl_tlsdesc_undefweak, .-_dl_tlsdesc_undefweak
|
|
Packit Service |
82fcde |
|
|
Packit Service |
82fcde |
#ifdef SHARED
|
|
Packit Service |
82fcde |
.hidden _dl_tlsdesc_dynamic
|
|
Packit Service |
82fcde |
.global _dl_tlsdesc_dynamic
|
|
Packit Service |
82fcde |
.type _dl_tlsdesc_dynamic,@function
|
|
Packit Service |
82fcde |
|
|
Packit Service |
82fcde |
/* %rax points to the TLS descriptor, such that 0(%rax) points to
|
|
Packit Service |
82fcde |
_dl_tlsdesc_dynamic itself, and 8(%rax) points to a struct
|
|
Packit Service |
82fcde |
tlsdesc_dynamic_arg object. It must return in %rax the offset
|
|
Packit Service |
82fcde |
between the thread pointer and the object denoted by the
|
|
Packit Service |
82fcde |
argument, without clobbering any registers.
|
|
Packit Service |
82fcde |
|
|
Packit Service |
82fcde |
The assembly code that follows is a rendition of the following
|
|
Packit Service |
82fcde |
C code, hand-optimized a little bit.
|
|
Packit Service |
82fcde |
|
|
Packit Service |
82fcde |
ptrdiff_t
|
|
Packit Service |
82fcde |
_dl_tlsdesc_dynamic (register struct tlsdesc *tdp asm ("%rax"))
|
|
Packit Service |
82fcde |
{
|
|
Packit Service |
82fcde |
struct tlsdesc_dynamic_arg *td = tdp->arg;
|
|
Packit Service |
82fcde |
dtv_t *dtv = *(dtv_t **)((char *)__thread_pointer + DTV_OFFSET);
|
|
Packit Service |
82fcde |
if (__builtin_expect (td->gen_count <= dtv[0].counter
|
|
Packit Service |
82fcde |
&& (dtv[td->tlsinfo.ti_module].pointer.val
|
|
Packit Service |
82fcde |
!= TLS_DTV_UNALLOCATED),
|
|
Packit Service |
82fcde |
1))
|
|
Packit Service |
82fcde |
return dtv[td->tlsinfo.ti_module].pointer.val + td->tlsinfo.ti_offset
|
|
Packit Service |
82fcde |
- __thread_pointer;
|
|
Packit Service |
82fcde |
|
|
Packit Service |
82fcde |
return __tls_get_addr_internal (&td->tlsinfo) - __thread_pointer;
|
|
Packit Service |
82fcde |
}
|
|
Packit Service |
82fcde |
*/
|
|
Packit Service |
82fcde |
cfi_startproc
|
|
Packit Service |
82fcde |
.align 16
|
|
Packit Service |
82fcde |
_dl_tlsdesc_dynamic:
|
|
Packit Service |
82fcde |
_CET_ENDBR
|
|
Packit Service |
82fcde |
/* Preserve call-clobbered registers that we modify.
|
|
Packit Service |
82fcde |
We need two scratch regs anyway. */
|
|
Packit Service |
82fcde |
movq %rsi, -16(%rsp)
|
|
Packit Service |
82fcde |
movq %fs:DTV_OFFSET, %rsi
|
|
Packit Service |
82fcde |
movq %rdi, -8(%rsp)
|
|
Packit Service |
82fcde |
movq TLSDESC_ARG(%rax), %rdi
|
|
Packit Service |
82fcde |
movq (%rsi), %rax
|
|
Packit Service |
82fcde |
cmpq %rax, TLSDESC_GEN_COUNT(%rdi)
|
|
Packit Service |
82fcde |
ja .Lslow
|
|
Packit Service |
82fcde |
movq TLSDESC_MODID(%rdi), %rax
|
|
Packit Service |
82fcde |
salq $4, %rax
|
|
Packit Service |
82fcde |
movq (%rax,%rsi), %rax
|
|
Packit Service |
82fcde |
cmpq $-1, %rax
|
|
Packit Service |
82fcde |
je .Lslow
|
|
Packit Service |
82fcde |
addq TLSDESC_MODOFF(%rdi), %rax
|
|
Packit Service |
82fcde |
.Lret:
|
|
Packit Service |
82fcde |
movq -16(%rsp), %rsi
|
|
Packit Service |
82fcde |
subq %fs:0, %rax
|
|
Packit Service |
82fcde |
movq -8(%rsp), %rdi
|
|
Packit Service |
82fcde |
ret
|
|
Packit Service |
82fcde |
.Lslow:
|
|
Packit Service |
82fcde |
/* Besides rdi and rsi, saved above, save rdx, rcx, r8, r9,
|
|
Packit Service |
82fcde |
r10 and r11. Also, align the stack, that's off by 8 bytes. */
|
|
Packit Service |
82fcde |
subq $72, %rsp
|
|
Packit Service |
82fcde |
cfi_adjust_cfa_offset (72)
|
|
Packit Service |
82fcde |
movq %rdx, 8(%rsp)
|
|
Packit Service |
82fcde |
movq %rcx, 16(%rsp)
|
|
Packit Service |
82fcde |
movq %r8, 24(%rsp)
|
|
Packit Service |
82fcde |
movq %r9, 32(%rsp)
|
|
Packit Service |
82fcde |
movq %r10, 40(%rsp)
|
|
Packit Service |
82fcde |
movq %r11, 48(%rsp)
|
|
Packit Service |
82fcde |
/* %rdi already points to the tlsinfo data structure. */
|
|
Packit Service |
82fcde |
#ifdef NO_RTLD_HIDDEN
|
|
Packit Service |
82fcde |
call JUMPTARGET (__tls_get_addr)
|
|
Packit Service |
82fcde |
#else
|
|
Packit Service |
82fcde |
call HIDDEN_JUMPTARGET (__tls_get_addr)
|
|
Packit Service |
82fcde |
#endif
|
|
Packit Service |
82fcde |
movq 8(%rsp), %rdx
|
|
Packit Service |
82fcde |
movq 16(%rsp), %rcx
|
|
Packit Service |
82fcde |
movq 24(%rsp), %r8
|
|
Packit Service |
82fcde |
movq 32(%rsp), %r9
|
|
Packit Service |
82fcde |
movq 40(%rsp), %r10
|
|
Packit Service |
82fcde |
movq 48(%rsp), %r11
|
|
Packit Service |
82fcde |
addq $72, %rsp
|
|
Packit Service |
82fcde |
cfi_adjust_cfa_offset (-72)
|
|
Packit Service |
82fcde |
jmp .Lret
|
|
Packit Service |
82fcde |
cfi_endproc
|
|
Packit Service |
82fcde |
.size _dl_tlsdesc_dynamic, .-_dl_tlsdesc_dynamic
|
|
Packit Service |
82fcde |
#endif /* SHARED */
|
|
Packit Service |
82fcde |
|
|
Packit Service |
82fcde |
/* This function is a wrapper for a lazy resolver for TLS_DESC
|
|
Packit Service |
82fcde |
RELA relocations. The incoming 0(%rsp) points to the caller's
|
|
Packit Service |
82fcde |
link map, pushed by the dynamic object's internal lazy TLS
|
|
Packit Service |
82fcde |
resolver front-end before tail-calling us. We need to pop it
|
|
Packit Service |
82fcde |
ourselves. %rax points to a TLS descriptor, such that 0(%rax)
|
|
Packit Service |
82fcde |
holds the address of the internal resolver front-end (unless
|
|
Packit Service |
82fcde |
some other thread beat us to resolving it) and 8(%rax) holds a
|
|
Packit Service |
82fcde |
pointer to the relocation.
|
|
Packit Service |
82fcde |
|
|
Packit Service |
82fcde |
When the actual resolver returns, it will have adjusted the
|
|
Packit Service |
82fcde |
TLS descriptor such that we can tail-call it for it to return
|
|
Packit Service |
82fcde |
the TP offset of the symbol. */
|
|
Packit Service |
82fcde |
|
|
Packit Service |
82fcde |
.hidden _dl_tlsdesc_resolve_rela
|
|
Packit Service |
82fcde |
.global _dl_tlsdesc_resolve_rela
|
|
Packit Service |
82fcde |
.type _dl_tlsdesc_resolve_rela,@function
|
|
Packit Service |
82fcde |
cfi_startproc
|
|
Packit Service |
82fcde |
.align 16
|
|
Packit Service |
82fcde |
/* The PLT entry will have pushed the link_map pointer. */
|
|
Packit Service |
82fcde |
_dl_tlsdesc_resolve_rela:
|
|
Packit Service |
82fcde |
_CET_ENDBR
|
|
Packit Service |
82fcde |
cfi_adjust_cfa_offset (8)
|
|
Packit Service |
82fcde |
/* Save all call-clobbered registers. Add 8 bytes for push in
|
|
Packit Service |
82fcde |
the PLT entry to align the stack. */
|
|
Packit Service |
82fcde |
subq $80, %rsp
|
|
Packit Service |
82fcde |
cfi_adjust_cfa_offset (80)
|
|
Packit Service |
82fcde |
movq %rax, (%rsp)
|
|
Packit Service |
82fcde |
movq %rdi, 8(%rsp)
|
|
Packit Service |
82fcde |
movq %rax, %rdi /* Pass tlsdesc* in %rdi. */
|
|
Packit Service |
82fcde |
movq %rsi, 16(%rsp)
|
|
Packit Service |
82fcde |
movq 80(%rsp), %rsi /* Pass link_map* in %rsi. */
|
|
Packit Service |
82fcde |
movq %r8, 24(%rsp)
|
|
Packit Service |
82fcde |
movq %r9, 32(%rsp)
|
|
Packit Service |
82fcde |
movq %r10, 40(%rsp)
|
|
Packit Service |
82fcde |
movq %r11, 48(%rsp)
|
|
Packit Service |
82fcde |
movq %rdx, 56(%rsp)
|
|
Packit Service |
82fcde |
movq %rcx, 64(%rsp)
|
|
Packit Service |
82fcde |
call _dl_tlsdesc_resolve_rela_fixup
|
|
Packit Service |
82fcde |
movq (%rsp), %rax
|
|
Packit Service |
82fcde |
movq 8(%rsp), %rdi
|
|
Packit Service |
82fcde |
movq 16(%rsp), %rsi
|
|
Packit Service |
82fcde |
movq 24(%rsp), %r8
|
|
Packit Service |
82fcde |
movq 32(%rsp), %r9
|
|
Packit Service |
82fcde |
movq 40(%rsp), %r10
|
|
Packit Service |
82fcde |
movq 48(%rsp), %r11
|
|
Packit Service |
82fcde |
movq 56(%rsp), %rdx
|
|
Packit Service |
82fcde |
movq 64(%rsp), %rcx
|
|
Packit Service |
82fcde |
addq $88, %rsp
|
|
Packit Service |
82fcde |
cfi_adjust_cfa_offset (-88)
|
|
Packit Service |
82fcde |
jmp *(%rax)
|
|
Packit Service |
82fcde |
cfi_endproc
|
|
Packit Service |
82fcde |
.size _dl_tlsdesc_resolve_rela, .-_dl_tlsdesc_resolve_rela
|
|
Packit Service |
82fcde |
|
|
Packit Service |
82fcde |
/* This function is a placeholder for lazy resolving of TLS
|
|
Packit Service |
82fcde |
relocations. Once some thread starts resolving a TLS
|
|
Packit Service |
82fcde |
relocation, it sets up the TLS descriptor to use this
|
|
Packit Service |
82fcde |
resolver, such that other threads that would attempt to
|
|
Packit Service |
82fcde |
resolve it concurrently may skip the call to the original lazy
|
|
Packit Service |
82fcde |
resolver and go straight to a condition wait.
|
|
Packit Service |
82fcde |
|
|
Packit Service |
82fcde |
When the actual resolver returns, it will have adjusted the
|
|
Packit Service |
82fcde |
TLS descriptor such that we can tail-call it for it to return
|
|
Packit Service |
82fcde |
the TP offset of the symbol. */
|
|
Packit Service |
82fcde |
|
|
Packit Service |
82fcde |
.hidden _dl_tlsdesc_resolve_hold
|
|
Packit Service |
82fcde |
.global _dl_tlsdesc_resolve_hold
|
|
Packit Service |
82fcde |
.type _dl_tlsdesc_resolve_hold,@function
|
|
Packit Service |
82fcde |
cfi_startproc
|
|
Packit Service |
82fcde |
.align 16
|
|
Packit Service |
82fcde |
_dl_tlsdesc_resolve_hold:
|
|
Packit Service |
82fcde |
0:
|
|
Packit Service |
82fcde |
_CET_ENDBR
|
|
Packit Service |
82fcde |
/* Save all call-clobbered registers. */
|
|
Packit Service |
82fcde |
subq $72, %rsp
|
|
Packit Service |
82fcde |
cfi_adjust_cfa_offset (72)
|
|
Packit Service |
82fcde |
movq %rax, (%rsp)
|
|
Packit Service |
82fcde |
movq %rdi, 8(%rsp)
|
|
Packit Service |
82fcde |
movq %rax, %rdi /* Pass tlsdesc* in %rdi. */
|
|
Packit Service |
82fcde |
movq %rsi, 16(%rsp)
|
|
Packit Service |
82fcde |
/* Pass _dl_tlsdesc_resolve_hold's address in %rsi. */
|
|
Packit Service |
82fcde |
leaq . - _dl_tlsdesc_resolve_hold(%rip), %rsi
|
|
Packit Service |
82fcde |
movq %r8, 24(%rsp)
|
|
Packit Service |
82fcde |
movq %r9, 32(%rsp)
|
|
Packit Service |
82fcde |
movq %r10, 40(%rsp)
|
|
Packit Service |
82fcde |
movq %r11, 48(%rsp)
|
|
Packit Service |
82fcde |
movq %rdx, 56(%rsp)
|
|
Packit Service |
82fcde |
movq %rcx, 64(%rsp)
|
|
Packit Service |
82fcde |
call _dl_tlsdesc_resolve_hold_fixup
|
|
Packit Service |
82fcde |
1:
|
|
Packit Service |
82fcde |
movq (%rsp), %rax
|
|
Packit Service |
82fcde |
movq 8(%rsp), %rdi
|
|
Packit Service |
82fcde |
movq 16(%rsp), %rsi
|
|
Packit Service |
82fcde |
movq 24(%rsp), %r8
|
|
Packit Service |
82fcde |
movq 32(%rsp), %r9
|
|
Packit Service |
82fcde |
movq 40(%rsp), %r10
|
|
Packit Service |
82fcde |
movq 48(%rsp), %r11
|
|
Packit Service |
82fcde |
movq 56(%rsp), %rdx
|
|
Packit Service |
82fcde |
movq 64(%rsp), %rcx
|
|
Packit Service |
82fcde |
addq $72, %rsp
|
|
Packit Service |
82fcde |
cfi_adjust_cfa_offset (-72)
|
|
Packit Service |
82fcde |
jmp *(%rax)
|
|
Packit Service |
82fcde |
cfi_endproc
|
|
Packit Service |
82fcde |
.size _dl_tlsdesc_resolve_hold, .-_dl_tlsdesc_resolve_hold
|