/* strstr with unaligned loads
Copyright (C) 2009-2018 Free Software Foundation, Inc.
This file is part of the GNU C Library.
The GNU C Library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
The GNU C Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with the GNU C Library; if not, see
<http://www.gnu.org/licenses/>. */
#include <sysdep.h>
ENTRY(__strstr_sse2_unaligned)
movzbl (%rsi), %eax
testb %al, %al
je L(empty)
movzbl 1(%rsi), %edx
testb %dl, %dl
je L(strchr)
movd %eax, %xmm1
movd %edx, %xmm2
movq %rdi, %rax
andl $4095, %eax
punpcklbw %xmm1, %xmm1
cmpq $4031, %rax
punpcklbw %xmm2, %xmm2
punpcklwd %xmm1, %xmm1
punpcklwd %xmm2, %xmm2
pshufd $0, %xmm1, %xmm1
pshufd $0, %xmm2, %xmm2
ja L(cross_page)
movdqu (%rdi), %xmm3
pxor %xmm5, %xmm5
movdqu 1(%rdi), %xmm4
movdqa %xmm3, %xmm6
pcmpeqb %xmm1, %xmm3
pcmpeqb %xmm2, %xmm4
movdqu 16(%rdi), %xmm0
pcmpeqb %xmm5, %xmm6
pminub %xmm4, %xmm3
movdqa %xmm3, %xmm4
movdqu 17(%rdi), %xmm3
pcmpeqb %xmm0, %xmm5
pcmpeqb %xmm2, %xmm3
por %xmm6, %xmm4
pcmpeqb %xmm1, %xmm0
pminub %xmm3, %xmm0
por %xmm5, %xmm0
pmovmskb %xmm4, %r8d
pmovmskb %xmm0, %eax
salq $16, %rax
orq %rax, %r8
je L(next_32_bytes)
L(next_pair_index):
bsf %r8, %rax
addq %rdi, %rax
cmpb $0, (%rax)
je L(zero1)
movzbl 2(%rsi), %edx
testb %dl, %dl
je L(found1)
cmpb 2(%rax), %dl
jne L(next_pair)
xorl %edx, %edx
jmp L(pair_loop_start)
.p2align 4
L(strchr):
movzbl %al, %esi
jmp __strchr_sse2
.p2align 4
L(pair_loop):
addq $1, %rdx
cmpb 2(%rax,%rdx), %cl
jne L(next_pair)
L(pair_loop_start):
movzbl 3(%rsi,%rdx), %ecx
testb %cl, %cl
jne L(pair_loop)
L(found1):
ret
L(zero1):
xorl %eax, %eax
ret
.p2align 4
L(next_pair):
leaq -1(%r8), %rax
andq %rax, %r8
jne L(next_pair_index)
.p2align 4
L(next_32_bytes):
movdqu 32(%rdi), %xmm3
pxor %xmm5, %xmm5
movdqu 33(%rdi), %xmm4
movdqa %xmm3, %xmm6
pcmpeqb %xmm1, %xmm3
pcmpeqb %xmm2, %xmm4
movdqu 48(%rdi), %xmm0
pcmpeqb %xmm5, %xmm6
pminub %xmm4, %xmm3
movdqa %xmm3, %xmm4
movdqu 49(%rdi), %xmm3
pcmpeqb %xmm0, %xmm5
pcmpeqb %xmm2, %xmm3
por %xmm6, %xmm4
pcmpeqb %xmm1, %xmm0
pminub %xmm3, %xmm0
por %xmm5, %xmm0
pmovmskb %xmm4, %eax
salq $32, %rax
pmovmskb %xmm0, %r8d
salq $48, %r8
orq %rax, %r8
je L(loop_header)
L(next_pair2_index):
bsfq %r8, %rax
addq %rdi, %rax
cmpb $0, (%rax)
je L(zero2)
movzbl 2(%rsi), %edx
testb %dl, %dl
je L(found2)
cmpb 2(%rax), %dl
jne L(next_pair2)
xorl %edx, %edx
jmp L(pair_loop2_start)
.p2align 4
L(pair_loop2):
addq $1, %rdx
cmpb 2(%rax,%rdx), %cl
jne L(next_pair2)
L(pair_loop2_start):
movzbl 3(%rsi,%rdx), %ecx
testb %cl, %cl
jne L(pair_loop2)
L(found2):
ret
L(zero2):
xorl %eax, %eax
ret
L(empty):
mov %rdi, %rax
ret
.p2align 4
L(next_pair2):
leaq -1(%r8), %rax
andq %rax, %r8
jne L(next_pair2_index)
L(loop_header):
movq $-512, %r11
movq %rdi, %r9
pxor %xmm7, %xmm7
andq $-64, %rdi
.p2align 4
L(loop):
movdqa 64(%rdi), %xmm3
movdqu 63(%rdi), %xmm6
movdqa %xmm3, %xmm0
pxor %xmm2, %xmm3
pxor %xmm1, %xmm6
movdqa 80(%rdi), %xmm10
por %xmm3, %xmm6
pminub %xmm10, %xmm0
movdqu 79(%rdi), %xmm3
pxor %xmm2, %xmm10
pxor %xmm1, %xmm3
movdqa 96(%rdi), %xmm9
por %xmm10, %xmm3
pminub %xmm9, %xmm0
pxor %xmm2, %xmm9
movdqa 112(%rdi), %xmm8
addq $64, %rdi
pminub %xmm6, %xmm3
movdqu 31(%rdi), %xmm4
pminub %xmm8, %xmm0
pxor %xmm2, %xmm8
pxor %xmm1, %xmm4
por %xmm9, %xmm4
pminub %xmm4, %xmm3
movdqu 47(%rdi), %xmm5
pxor %xmm1, %xmm5
por %xmm8, %xmm5
pminub %xmm5, %xmm3
pminub %xmm3, %xmm0
pcmpeqb %xmm7, %xmm0
pmovmskb %xmm0, %eax
testl %eax, %eax
je L(loop)
pminub (%rdi), %xmm6
pminub 32(%rdi),%xmm4
pminub 48(%rdi),%xmm5
pcmpeqb %xmm7, %xmm6
pcmpeqb %xmm7, %xmm5
pmovmskb %xmm6, %edx
movdqa 16(%rdi), %xmm8
pcmpeqb %xmm7, %xmm4
movdqu 15(%rdi), %xmm0
pmovmskb %xmm5, %r8d
movdqa %xmm8, %xmm3
pmovmskb %xmm4, %ecx
pcmpeqb %xmm1,%xmm0
pcmpeqb %xmm2,%xmm3
salq $32, %rcx
pcmpeqb %xmm7,%xmm8
salq $48, %r8
pminub %xmm0,%xmm3
orq %rcx, %rdx
por %xmm3,%xmm8
orq %rdx, %r8
pmovmskb %xmm8, %eax
salq $16, %rax
orq %rax, %r8
je L(loop)
L(next_pair_index3):
bsfq %r8, %rcx
addq %rdi, %rcx
cmpb $0, (%rcx)
je L(zero)
xorl %eax, %eax
movzbl 2(%rsi), %edx
testb %dl, %dl
je L(success3)
cmpb 1(%rcx), %dl
jne L(next_pair3)
jmp L(pair_loop_start3)
.p2align 4
L(pair_loop3):
addq $1, %rax
cmpb 1(%rcx,%rax), %dl
jne L(next_pair3)
L(pair_loop_start3):
movzbl 3(%rsi,%rax), %edx
testb %dl, %dl
jne L(pair_loop3)
L(success3):
lea -1(%rcx), %rax
ret
.p2align 4
L(next_pair3):
addq %rax, %r11
movq %rdi, %rax
subq %r9, %rax
cmpq %r11, %rax
jl L(switch_strstr)
leaq -1(%r8), %rax
andq %rax, %r8
jne L(next_pair_index3)
jmp L(loop)
.p2align 4
L(switch_strstr):
movq %rdi, %rdi
jmp __strstr_sse2
.p2align 4
L(cross_page):
movq %rdi, %rax
pxor %xmm0, %xmm0
andq $-64, %rax
movdqa (%rax), %xmm3
movdqu -1(%rax), %xmm4
movdqa %xmm3, %xmm8
movdqa 16(%rax), %xmm5
pcmpeqb %xmm1, %xmm4
pcmpeqb %xmm0, %xmm8
pcmpeqb %xmm2, %xmm3
movdqa %xmm5, %xmm7
pminub %xmm4, %xmm3
movdqu 15(%rax), %xmm4
pcmpeqb %xmm0, %xmm7
por %xmm3, %xmm8
movdqa %xmm5, %xmm3
movdqa 32(%rax), %xmm5
pcmpeqb %xmm1, %xmm4
pcmpeqb %xmm2, %xmm3
movdqa %xmm5, %xmm6
pmovmskb %xmm8, %ecx
pminub %xmm4, %xmm3
movdqu 31(%rax), %xmm4
por %xmm3, %xmm7
movdqa %xmm5, %xmm3
pcmpeqb %xmm0, %xmm6
movdqa 48(%rax), %xmm5
pcmpeqb %xmm1, %xmm4
pmovmskb %xmm7, %r8d
pcmpeqb %xmm2, %xmm3
pcmpeqb %xmm5, %xmm0
pminub %xmm4, %xmm3
movdqu 47(%rax), %xmm4
por %xmm3, %xmm6
movdqa %xmm5, %xmm3
salq $16, %r8
pcmpeqb %xmm1, %xmm4
pcmpeqb %xmm2, %xmm3
pmovmskb %xmm6, %r10d
pminub %xmm4, %xmm3
por %xmm3, %xmm0
salq $32, %r10
orq %r10, %r8
orq %rcx, %r8
movl %edi, %ecx
pmovmskb %xmm0, %edx
subl %eax, %ecx
salq $48, %rdx
orq %rdx, %r8
shrq %cl, %r8
je L(loop_header)
L(next_pair_index4):
bsfq %r8, %rax
addq %rdi, %rax
cmpb $0, (%rax)
je L(zero)
cmpq %rax,%rdi
je L(next_pair4)
movzbl 2(%rsi), %edx
testb %dl, %dl
je L(found3)
cmpb 1(%rax), %dl
jne L(next_pair4)
xorl %edx, %edx
jmp L(pair_loop_start4)
.p2align 4
L(pair_loop4):
addq $1, %rdx
cmpb 1(%rax,%rdx), %cl
jne L(next_pair4)
L(pair_loop_start4):
movzbl 3(%rsi,%rdx), %ecx
testb %cl, %cl
jne L(pair_loop4)
L(found3):
subq $1, %rax
ret
.p2align 4
L(next_pair4):
leaq -1(%r8), %rax
andq %rax, %r8
jne L(next_pair_index4)
jmp L(loop_header)
.p2align 4
L(found):
rep
ret
.p2align 4
L(zero):
xorl %eax, %eax
ret
END(__strstr_sse2_unaligned)