/* strlen/strnlen/wcslen/wcsnlen optimized with AVX2.
Copyright (C) 2017-2018 Free Software Foundation, Inc.
This file is part of the GNU C Library.
The GNU C Library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
The GNU C Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with the GNU C Library; if not, see
<http://www.gnu.org/licenses/>. */
#if IS_IN (libc)
# include <sysdep.h>
# ifndef STRLEN
# define STRLEN __strlen_avx2
# endif
# ifdef USE_AS_WCSLEN
# define VPCMPEQ vpcmpeqd
# define VPMINU vpminud
# else
# define VPCMPEQ vpcmpeqb
# define VPMINU vpminub
# endif
# ifndef VZEROUPPER
# define VZEROUPPER vzeroupper
# endif
# define VEC_SIZE 32
.section .text.avx,"ax",@progbits
ENTRY (STRLEN)
# ifdef USE_AS_STRNLEN
/* Check for zero length. */
testq %rsi, %rsi
jz L(zero)
# ifdef USE_AS_WCSLEN
shl $2, %rsi
# endif
movq %rsi, %r8
# endif
movl %edi, %ecx
movq %rdi, %rdx
vpxor %xmm0, %xmm0, %xmm0
/* Check if we may cross page boundary with one vector load. */
andl $(2 * VEC_SIZE - 1), %ecx
cmpl $VEC_SIZE, %ecx
ja L(cros_page_boundary)
/* Check the first VEC_SIZE bytes. */
VPCMPEQ (%rdi), %ymm0, %ymm1
vpmovmskb %ymm1, %eax
testl %eax, %eax
# ifdef USE_AS_STRNLEN
jnz L(first_vec_x0_check)
/* Adjust length and check the end of data. */
subq $VEC_SIZE, %rsi
jbe L(max)
# else
jnz L(first_vec_x0)
# endif
/* Align data for aligned loads in the loop. */
addq $VEC_SIZE, %rdi
andl $(VEC_SIZE - 1), %ecx
andq $-VEC_SIZE, %rdi
# ifdef USE_AS_STRNLEN
/* Adjust length. */
addq %rcx, %rsi
subq $(VEC_SIZE * 4), %rsi
jbe L(last_4x_vec_or_less)
# endif
jmp L(more_4x_vec)
.p2align 4
L(cros_page_boundary):
andl $(VEC_SIZE - 1), %ecx
andq $-VEC_SIZE, %rdi
VPCMPEQ (%rdi), %ymm0, %ymm1
vpmovmskb %ymm1, %eax
/* Remove the leading bytes. */
sarl %cl, %eax
testl %eax, %eax
jz L(aligned_more)
tzcntl %eax, %eax
# ifdef USE_AS_STRNLEN
/* Check the end of data. */
cmpq %rax, %rsi
jbe L(max)
# endif
addq %rdi, %rax
addq %rcx, %rax
subq %rdx, %rax
# ifdef USE_AS_WCSLEN
shrq $2, %rax
# endif
VZEROUPPER
ret
.p2align 4
L(aligned_more):
# ifdef USE_AS_STRNLEN
/* "rcx" is less than VEC_SIZE. Calculate "rdx + rcx - VEC_SIZE"
with "rdx - (VEC_SIZE - rcx)" instead of "(rdx + rcx) - VEC_SIZE"
to void possible addition overflow. */
negq %rcx
addq $VEC_SIZE, %rcx
/* Check the end of data. */
subq %rcx, %rsi
jbe L(max)
# endif
addq $VEC_SIZE, %rdi
# ifdef USE_AS_STRNLEN
subq $(VEC_SIZE * 4), %rsi
jbe L(last_4x_vec_or_less)
# endif
L(more_4x_vec):
/* Check the first 4 * VEC_SIZE. Only one VEC_SIZE at a time
since data is only aligned to VEC_SIZE. */
VPCMPEQ (%rdi), %ymm0, %ymm1
vpmovmskb %ymm1, %eax
testl %eax, %eax
jnz L(first_vec_x0)
VPCMPEQ VEC_SIZE(%rdi), %ymm0, %ymm1
vpmovmskb %ymm1, %eax
testl %eax, %eax
jnz L(first_vec_x1)
VPCMPEQ (VEC_SIZE * 2)(%rdi), %ymm0, %ymm1
vpmovmskb %ymm1, %eax
testl %eax, %eax
jnz L(first_vec_x2)
VPCMPEQ (VEC_SIZE * 3)(%rdi), %ymm0, %ymm1
vpmovmskb %ymm1, %eax
testl %eax, %eax
jnz L(first_vec_x3)
addq $(VEC_SIZE * 4), %rdi
# ifdef USE_AS_STRNLEN
subq $(VEC_SIZE * 4), %rsi
jbe L(last_4x_vec_or_less)
# endif
/* Align data to 4 * VEC_SIZE. */
movq %rdi, %rcx
andl $(4 * VEC_SIZE - 1), %ecx
andq $-(4 * VEC_SIZE), %rdi
# ifdef USE_AS_STRNLEN
/* Adjust length. */
addq %rcx, %rsi
# endif
.p2align 4
L(loop_4x_vec):
/* Compare 4 * VEC at a time forward. */
vmovdqa (%rdi), %ymm1
vmovdqa VEC_SIZE(%rdi), %ymm2
vmovdqa (VEC_SIZE * 2)(%rdi), %ymm3
vmovdqa (VEC_SIZE * 3)(%rdi), %ymm4
VPMINU %ymm1, %ymm2, %ymm5
VPMINU %ymm3, %ymm4, %ymm6
VPMINU %ymm5, %ymm6, %ymm5
VPCMPEQ %ymm5, %ymm0, %ymm5
vpmovmskb %ymm5, %eax
testl %eax, %eax
jnz L(4x_vec_end)
addq $(VEC_SIZE * 4), %rdi
# ifndef USE_AS_STRNLEN
jmp L(loop_4x_vec)
# else
subq $(VEC_SIZE * 4), %rsi
ja L(loop_4x_vec)
L(last_4x_vec_or_less):
/* Less than 4 * VEC and aligned to VEC_SIZE. */
addl $(VEC_SIZE * 2), %esi
jle L(last_2x_vec)
VPCMPEQ (%rdi), %ymm0, %ymm1
vpmovmskb %ymm1, %eax
testl %eax, %eax
jnz L(first_vec_x0)
VPCMPEQ VEC_SIZE(%rdi), %ymm0, %ymm1
vpmovmskb %ymm1, %eax
testl %eax, %eax
jnz L(first_vec_x1)
VPCMPEQ (VEC_SIZE * 2)(%rdi), %ymm0, %ymm1
vpmovmskb %ymm1, %eax
testl %eax, %eax
jnz L(first_vec_x2_check)
subl $VEC_SIZE, %esi
jle L(max)
VPCMPEQ (VEC_SIZE * 3)(%rdi), %ymm0, %ymm1
vpmovmskb %ymm1, %eax
testl %eax, %eax
jnz L(first_vec_x3_check)
movq %r8, %rax
# ifdef USE_AS_WCSLEN
shrq $2, %rax
# endif
VZEROUPPER
ret
.p2align 4
L(last_2x_vec):
addl $(VEC_SIZE * 2), %esi
VPCMPEQ (%rdi), %ymm0, %ymm1
vpmovmskb %ymm1, %eax
testl %eax, %eax
jnz L(first_vec_x0_check)
subl $VEC_SIZE, %esi
jle L(max)
VPCMPEQ VEC_SIZE(%rdi), %ymm0, %ymm1
vpmovmskb %ymm1, %eax
testl %eax, %eax
jnz L(first_vec_x1_check)
movq %r8, %rax
# ifdef USE_AS_WCSLEN
shrq $2, %rax
# endif
VZEROUPPER
ret
.p2align 4
L(first_vec_x0_check):
tzcntl %eax, %eax
/* Check the end of data. */
cmpq %rax, %rsi
jbe L(max)
addq %rdi, %rax
subq %rdx, %rax
# ifdef USE_AS_WCSLEN
shrq $2, %rax
# endif
VZEROUPPER
ret
.p2align 4
L(first_vec_x1_check):
tzcntl %eax, %eax
/* Check the end of data. */
cmpq %rax, %rsi
jbe L(max)
addq $VEC_SIZE, %rax
addq %rdi, %rax
subq %rdx, %rax
# ifdef USE_AS_WCSLEN
shrq $2, %rax
# endif
VZEROUPPER
ret
.p2align 4
L(first_vec_x2_check):
tzcntl %eax, %eax
/* Check the end of data. */
cmpq %rax, %rsi
jbe L(max)
addq $(VEC_SIZE * 2), %rax
addq %rdi, %rax
subq %rdx, %rax
# ifdef USE_AS_WCSLEN
shrq $2, %rax
# endif
VZEROUPPER
ret
.p2align 4
L(first_vec_x3_check):
tzcntl %eax, %eax
/* Check the end of data. */
cmpq %rax, %rsi
jbe L(max)
addq $(VEC_SIZE * 3), %rax
addq %rdi, %rax
subq %rdx, %rax
# ifdef USE_AS_WCSLEN
shrq $2, %rax
# endif
VZEROUPPER
ret
.p2align 4
L(max):
movq %r8, %rax
# ifdef USE_AS_WCSLEN
shrq $2, %rax
# endif
VZEROUPPER
ret
.p2align 4
L(zero):
xorl %eax, %eax
ret
# endif
.p2align 4
L(first_vec_x0):
tzcntl %eax, %eax
addq %rdi, %rax
subq %rdx, %rax
# ifdef USE_AS_WCSLEN
shrq $2, %rax
# endif
VZEROUPPER
ret
.p2align 4
L(first_vec_x1):
tzcntl %eax, %eax
addq $VEC_SIZE, %rax
addq %rdi, %rax
subq %rdx, %rax
# ifdef USE_AS_WCSLEN
shrq $2, %rax
# endif
VZEROUPPER
ret
.p2align 4
L(first_vec_x2):
tzcntl %eax, %eax
addq $(VEC_SIZE * 2), %rax
addq %rdi, %rax
subq %rdx, %rax
# ifdef USE_AS_WCSLEN
shrq $2, %rax
# endif
VZEROUPPER
ret
.p2align 4
L(4x_vec_end):
VPCMPEQ %ymm1, %ymm0, %ymm1
vpmovmskb %ymm1, %eax
testl %eax, %eax
jnz L(first_vec_x0)
VPCMPEQ %ymm2, %ymm0, %ymm2
vpmovmskb %ymm2, %eax
testl %eax, %eax
jnz L(first_vec_x1)
VPCMPEQ %ymm3, %ymm0, %ymm3
vpmovmskb %ymm3, %eax
testl %eax, %eax
jnz L(first_vec_x2)
VPCMPEQ %ymm4, %ymm0, %ymm4
vpmovmskb %ymm4, %eax
L(first_vec_x3):
tzcntl %eax, %eax
addq $(VEC_SIZE * 3), %rax
addq %rdi, %rax
subq %rdx, %rax
# ifdef USE_AS_WCSLEN
shrq $2, %rax
# endif
VZEROUPPER
ret
END (STRLEN)
#endif