/* Optimized memset implementation for PowerPC64/POWER8.
Copyright (C) 2014-2018 Free Software Foundation, Inc.
This file is part of the GNU C Library.
The GNU C Library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
The GNU C Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with the GNU C Library; if not, see
<http://www.gnu.org/licenses/>. */
#include <sysdep.h>
#define MTVSRD_V1_R4 .long 0x7c240166 /* mtvsrd v1,r4 */
/* void * [r3] memset (void *s [r3], int c [r4], size_t n [r5]));
Returns 's'. */
#ifndef MEMSET
# define MEMSET memset
#endif
/* No need to use .machine power8 since mtvsrd is already
handled by the define. It avoid breakage on binutils
that does not support this machine specifier. */
.machine power7
ENTRY_TOCLESS (MEMSET, 5)
CALL_MCOUNT 3
L(_memset):
cmpldi cr7,r5,31
neg r0,r3
mr r10,r3
insrdi r4,r4,8,48
insrdi r4,r4,16,32 /* Replicate byte to word. */
ble cr7,L(write_LT_32)
andi. r11,r10,15 /* Check alignment of DST. */
insrdi r4,r4,32,0 /* Replicate word to double word. */
beq L(big_aligned)
mtocrf 0x01,r0
clrldi r0,r0,60
/* Get DST aligned to 16 bytes. */
1: bf 31,2f
stb r4,0(r10)
addi r10,r10,1
2: bf 30,4f
sth r4,0(r10)
addi r10,r10,2
4: bf 29,8f
stw r4,0(r10)
addi r10,r10,4
8: bf 28,16f
std r4,0(r10)
addi r10,r10,8
16: subf r5,r0,r5
.align 4
L(big_aligned):
/* For sizes larger than 255 two possible paths:
- if constant is '0', zero full cache lines with dcbz
- otherwise uses vector instructions. */
cmpldi cr5,r5,255
dcbtst 0,r10
cmpldi cr6,r4,0
crand 27,26,21
bt 27,L(huge_dcbz)
bge cr5,L(huge_vector)
/* Size between 32 and 255 bytes with constant different than 0, use
doubleword store instruction to achieve best throughput. */
srdi r8,r5,5
clrldi r11,r5,59
cmpldi cr6,r11,0
cmpdi r8,0
beq L(tail_bytes)
mtctr r8
/* Main aligned write loop, writes 32-bytes at a time. */
.align 4
L(big_loop):
std r4,0(r10)
std r4,8(r10)
std r4,16(r10)
std r4,24(r10)
addi r10,r10,32
bdz L(tail_bytes)
std r4,0(r10)
std r4,8(r10)
std r4,16(r10)
std r4,24(r10)
addi r10,10,32
bdnz L(big_loop)
b L(tail_bytes)
/* Write remaining 1~31 bytes. */
.align 4
L(tail_bytes):
beqlr cr6
srdi r7,r11,4
clrldi r8,r11,60
mtocrf 0x01,r7
.align 4
bf 31,8f
std r4,0(r10)
std r4,8(r10)
addi r10,r10,16
.align 4
8: mtocrf 0x1,r8
bf 28,4f
std r4,0(r10)
addi r10,r10,8
.align 4
4: bf 29,2f
stw 4,0(10)
addi 10,10,4
.align 4
2: bf 30,1f
sth 4,0(10)
addi 10,10,2
.align 4
1: bflr 31
stb 4,0(10)
blr
/* Size larger than 255 bytes with constant different than 0, use
vector instruction to achieve best throughput. */
L(huge_vector):
/* Replicate set byte to quadword in VMX register. */
MTVSRD_V1_R4
xxpermdi 32,v0,v1,0
vspltb v2,v0,15
/* Main aligned write loop: 128 bytes at a time. */
li r6,16
li r7,32
li r8,48
mtocrf 0x02,r5
srdi r12,r5,7
cmpdi r12,0
beq L(aligned_tail)
mtctr r12
b L(aligned_128loop)
.align 4
L(aligned_128loop):
stvx v2,0,r10
stvx v2,r10,r6
stvx v2,r10,r7
stvx v2,r10,r8
addi r10,r10,64
stvx v2,0,r10
stvx v2,r10,r6
stvx v2,r10,r7
stvx v2,r10,r8
addi r10,r10,64
bdnz L(aligned_128loop)
/* Write remaining 1~127 bytes. */
L(aligned_tail):
mtocrf 0x01,r5
bf 25,32f
stvx v2,0,r10
stvx v2,r10,r6
stvx v2,r10,r7
stvx v2,r10,r8
addi r10,r10,64
32: bf 26,16f
stvx v2,0,r10
stvx v2,r10,r6
addi r10,r10,32
16: bf 27,8f
stvx v2,0,r10
addi r10,r10,16
8: bf 28,4f
std r4,0(r10)
addi r10,r10,8
/* Copies 4~7 bytes. */
4: bf 29,L(tail2)
stw r4,0(r10)
bf 30,L(tail5)
sth r4,4(r10)
bflr 31
stb r4,6(r10)
/* Return original DST pointer. */
blr
/* Special case when value is 0 and we have a long length to deal
with. Use dcbz to zero out a full cacheline of 128 bytes at a time.
Before using dcbz though, we need to get the destination 128-byte
aligned. */
.align 4
L(huge_dcbz):
andi. r11,r10,127
neg r0,r10
beq L(huge_dcbz_aligned)
clrldi r0,r0,57
subf r5,r0,r5
srdi r0,r0,3
mtocrf 0x01,r0
/* Write 1~128 bytes until DST is aligned to 128 bytes. */
8: bf 28,4f
std r4,0(r10)
std r4,8(r10)
std r4,16(r10)
std r4,24(r10)
std r4,32(r10)
std r4,40(r10)
std r4,48(r10)
std r4,56(r10)
addi r10,r10,64
.align 4
4: bf 29,2f
std r4,0(r10)
std r4,8(r10)
std r4,16(r10)
std r4,24(r10)
addi r10,r10,32
.align 4
2: bf 30,1f
std r4,0(r10)
std r4,8(r10)
addi r10,r10,16
.align 4
1: bf 31,L(huge_dcbz_aligned)
std r4,0(r10)
addi r10,r10,8
L(huge_dcbz_aligned):
/* Setup dcbz unroll offsets and count numbers. */
srdi r8,r5,9
clrldi r11,r5,55
cmpldi cr6,r11,0
li r9,128
cmpdi r8,0
beq L(huge_tail)
li r7,256
li r6,384
mtctr r8
.align 4
L(huge_loop):
/* Sets 512 bytes to zero in each iteration, the loop unrolling shows
a throughput boost for large sizes (2048 bytes or higher). */
dcbz 0,r10
dcbz r9,r10
dcbz r7,r10
dcbz r6,r10
addi r10,r10,512
bdnz L(huge_loop)
beqlr cr6
L(huge_tail):
srdi r6,r11,8
srdi r7,r11,4
clrldi r8,r11,4
cmpldi cr6,r8,0
mtocrf 0x01,r6
beq cr6,L(tail)
/* We have 1~511 bytes remaining. */
.align 4
32: bf 31,16f
dcbz 0,r10
dcbz r9,r10
addi r10,r10,256
.align 4
16: mtocrf 0x01,r7
bf 28,8f
dcbz 0,r10
addi r10,r10,128
.align 4
8: bf 29,4f
std r4,0(r10)
std r4,8(r10)
std r4,16(r10)
std r4,24(r10)
std r4,32(r10)
std r4,40(r10)
std r4,48(r10)
std r4,56(r10)
addi r10,r10,64
.align 4
4: bf 30,2f
std r4,0(r10)
std r4,8(r10)
std r4,16(r10)
std r4,24(r10)
addi r10,r10,32
.align 4
2: bf 31,L(tail)
std r4,0(r10)
std r4,8(r10)
addi r10,r10,16
.align 4
/* Remaining 1~15 bytes. */
L(tail):
mtocrf 0x01,r8
.align
8: bf 28,4f
std r4,0(r10)
addi r10,r10,8
.align 4
4: bf 29,2f
stw r4,0(r10)
addi r10,r10,4
.align 4
2: bf 30,1f
sth r4,0(r10)
addi r10,r10,2
.align 4
1: bflr 31
stb r4,0(r10)
blr
/* Handle short copies of 0~31 bytes. Best throughput is achieved
by just unrolling all operations. */
.align 4
L(write_LT_32):
cmpldi cr6,5,8
mtocrf 0x01,r5
ble cr6,L(write_LE_8)
/* At least 9 bytes to go. */
neg r8,r4
andi. r0,r8,3
cmpldi cr1,r5,16
beq L(write_LT_32_aligned)
/* Force 4-byte alignment for SRC. */
mtocrf 0x01,r0
subf r5,r0,r5
2: bf 30,1f
/* Use stb instead of sth because it doesn't generate
alignment interrupts on cache-inhibited storage. */
stb r4,0(r10)
stb r4,1(r10)
addi r10,r10,2
1: bf 31,L(end_4bytes_alignment)
stb r4,0(r10)
addi r10,r10,1
.align 4
L(end_4bytes_alignment):
cmpldi cr1,r5,16
mtocrf 0x01,r5
L(write_LT_32_aligned):
blt cr1,8f
stw r4,0(r10)
stw r4,4(r10)
stw r4,8(r10)
stw r4,12(r10)
addi r10,r10,16
8: bf 28,L(tail4)
stw r4,0(r10)
stw r4,4(r10)
addi r10,r10,8
.align 4
/* Copies 4~7 bytes. */
L(tail4):
bf 29,L(tail2)
stw r4,0(r10)
bf 30,L(tail5)
sth r4,4(r10)
bflr 31
stb r4,6(r10)
blr
.align 4
/* Copies 2~3 bytes. */
L(tail2):
bf 30,1f
sth r4,0(r10)
bflr 31
stb r4,2(r10)
blr
.align 4
L(tail5):
bflr 31
stb r4,4(r10)
blr
.align 4
1: bflr 31
stb r4,0(r10)
blr
/* Handles copies of 0~8 bytes. */
.align 4
L(write_LE_8):
bne cr6,L(LE7_tail4)
/* If input is word aligned, use stw, else use stb. */
andi. r0,r10,3
bne L(8_unalign)
stw r4,0(r10)
stw r4,4(r10)
blr
/* Unaligned input and size is 8. */
.align 4
L(8_unalign):
andi. r0,r10,1
beq L(8_hwalign)
stb r4,0(r10)
sth r4,1(r10)
sth r4,3(r10)
sth r4,5(r10)
stb r4,7(r10)
blr
/* Halfword aligned input and size is 8. */
.align 4
L(8_hwalign):
sth r4,0(r10)
sth r4,2(r10)
sth r4,4(r10)
sth r4,6(r10)
blr
.align 4
/* Copies 4~7 bytes. */
L(LE7_tail4):
/* Use stb instead of sth because it doesn't generate
alignment interrupts on cache-inhibited storage. */
bf 29,L(LE7_tail2)
stb r4,0(r10)
stb r4,1(r10)
stb r4,2(r10)
stb r4,3(r10)
bf 30,L(LE7_tail5)
stb r4,4(r10)
stb r4,5(r10)
bflr 31
stb r4,6(r10)
blr
.align 4
/* Copies 2~3 bytes. */
L(LE7_tail2):
bf 30,1f
stb r4,0(r10)
stb r4,1(r10)
bflr 31
stb r4,2(r10)
blr
.align 4
L(LE7_tail5):
bflr 31
stb r4,4(r10)
blr
.align 4
1: bflr 31
stb r4,0(r10)
blr
END_GEN_TB (MEMSET,TB_TOCLESS)
libc_hidden_builtin_def (memset)
/* Copied from bzero.S to prevent the linker from inserting a stub
between bzero and memset. */
ENTRY_TOCLESS (__bzero)
CALL_MCOUNT 3
mr r5,r4
li r4,0
b L(_memset)
END (__bzero)
#ifndef __bzero
weak_alias (__bzero, bzero)
#endif