/* Optimized memset implementation for PowerPC64/POWER7.

   Copyright (C) 2010-2018 Free Software Foundation, Inc.

   Contributed by Luis Machado <luisgpm@br.ibm.com>.

   This file is part of the GNU C Library.

   The GNU C Library is free software; you can redistribute it and/or

   modify it under the terms of the GNU Lesser General Public

   License as published by the Free Software Foundation; either

   version 2.1 of the License, or (at your option) any later version.

   The GNU C Library is distributed in the hope that it will be useful,

   but WITHOUT ANY WARRANTY; without even the implied warranty of

   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU

   Lesser General Public License for more details.

   You should have received a copy of the GNU Lesser General Public

   License along with the GNU C Library; if not, see

   <http://www.gnu.org/licenses/>.  */

#include <sysdep.h>

/* void * [r3] memset (void *s [r3], int c [r4], size_t n [r5]));

   Returns 's'.  */

#ifndef MEMSET

# define MEMSET memset

#endif

	.machine power7

ENTRY_TOCLESS (MEMSET, 5)

	CALL_MCOUNT 3

L(_memset):

	cmpldi	cr7,5,31

	cmpldi	cr6,5,8

	mr	10,3

	/* Replicate byte to word.  */

	insrdi	4,4,8,48

	insrdi	4,4,16,32

	ble	cr6,L(small)	/* If length <= 8, use short copy code.  */

	neg	0,3

	ble	cr7,L(medium)	/* If length < 32, use medium copy code.  */

	andi.	11,10,7		/* Check alignment of SRC.  */

	insrdi	4,4,32,0	/* Replicate word to double word.  */

	mr	12,5

	beq	L(big_aligned)

	clrldi	0,0,61

	mtocrf	0x01,0

	subf	5,0,5

	/* Get DST aligned to 8 bytes.  */

1:	bf	31,2f

	stb	4,0(10)

	addi	10,10,1

2:	bf	30,4f

	sth	4,0(10)

	addi	10,10,2

4:	bf	29,L(big_aligned)

	stw	4,0(10)

	addi	10,10,4

	.align	4

L(big_aligned):

	cmpldi	cr5,5,255

	li	0,32

	dcbtst	0,10

	cmpldi	cr6,4,0

	srdi	9,5,3	/* Number of full doublewords remaining.  */

	crand	27,26,21

	mtocrf	0x01,9

	bt	27,L(huge)

	/* From this point on, we'll copy 32+ bytes and the value

	   isn't 0 (so we can't use dcbz).  */

	srdi	8,5,5

	clrldi	11,5,61

	cmpldi	cr6,11,0

	cmpldi	cr1,9,4

	mtctr	8

	/* Copy 1~3 doublewords so the main loop starts

	at a multiple of 32 bytes.  */

	bf	30,1f

	std	4,0(10)

	std	4,8(10)

	addi	10,10,16

	bf	31,L(big_loop)

	std	4,0(10)

	addi	10,10,8

	mr	12,10

	blt	cr1,L(tail_bytes)

	b	L(big_loop)

	.align	4

1:	/* Copy 1 doubleword.  */

	bf	31,L(big_loop)

	std	4,0(10)

	addi	10,10,8

	/* Main aligned copy loop.  Copies 32-bytes at a time and

	   ping-pong through r10 and r12 to avoid AGEN delays.  */

	.align	4

L(big_loop):

	addi	12,10,32

	std	4,0(10)

	std	4,8(10)

	std	4,16(10)

	std	4,24(10)

	bdz	L(tail_bytes)

	addi	10,10,64

	std	4,0(12)

	std	4,8(12)

	std	4,16(12)

	std	4,24(12)

	bdnz	L(big_loop)

	mr	12,10

	b	L(tail_bytes)

	.align	4

L(tail_bytes):

	/* Check for tail bytes.  */

	beqlr	cr6

	clrldi	0,5,61

	mtocrf	0x01,0

	/*  At this point we have a tail of 0-7 bytes and we know that the

	destination is doubleword-aligned.  */

4:	/* Copy 4 bytes.  */

	bf	29,2f

	stw	4,0(12)

	addi	12,12,4

2:	/* Copy 2 bytes.  */

	bf	30,1f

	sth	4,0(12)

	addi	12,12,2

1:	/* Copy 1 byte.  */

	bflr	31

	stb	4,0(12)

	blr

	/* Special case when value is 0 and we have a long length to deal

	   with.  Use dcbz to zero out 128-bytes at a time.  Before using

	   dcbz though, we need to get the destination 128-bytes aligned.  */

	.align	4

L(huge):

	andi.	11,10,127

	neg	0,10

	beq	L(huge_aligned)

	clrldi	0,0,57

	subf	5,0,5

	srdi	0,0,3

	mtocrf	0x01,0

	/* Get DST aligned to 128 bytes.  */

8:	bf	28,4f

	std	4,0(10)

	std	4,8(10)

	std	4,16(10)

	std	4,24(10)

	std	4,32(10)

	std	4,40(10)

	std	4,48(10)

	std	4,56(10)

	addi	10,10,64

	.align	4

4:	bf	29,2f

	std	4,0(10)

	std	4,8(10)

	std	4,16(10)

	std	4,24(10)

	addi	10,10,32

	.align	4

2:	bf	30,1f

	std	4,0(10)

	std	4,8(10)

	addi	10,10,16

	.align	4

1:	bf	31,L(huge_aligned)

	std	4,0(10)

	addi	10,10,8

L(huge_aligned):

	srdi	8,5,7

	clrldi	11,5,57

	cmpldi	cr6,11,0

	mtctr	8

	.align	4

L(huge_loop):

	dcbz	0,10

	addi	10,10,128

	bdnz	L(huge_loop)

	/* Check how many bytes are still left.  */

	beqlr	cr6

	subf	9,3,10

	subf	5,9,12

	srdi	8,5,3

	cmpldi	cr6,8,0

	mtocrf	0x01,8

	/* We have a tail o 1~127 bytes.  Copy up to 15 doublewords for

	speed.  We'll handle the resulting tail bytes later.  */

	beq	cr6,L(tail)

8:	bf	28,4f

	std	4,0(10)

	std	4,8(10)

	std	4,16(10)

	std	4,24(10)

	std	4,32(10)

	std	4,40(10)

	std	4,48(10)

	std	4,56(10)

	addi	10,10,64

	.align	4

4:	bf	29,2f

	std	4,0(10)

	std	4,8(10)

	std	4,16(10)

	std	4,24(10)

	addi	10,10,32

	.align	4

2:	bf	30,1f

	std	4,0(10)

	std	4,8(10)

	addi	10,10,16

	.align	4

1:	bf	31,L(tail)

	std	4,0(10)

	addi	10,10,8

	/* Handle the rest of the tail bytes here.  */

L(tail):

	mtocrf	0x01,5

	.align	4

4:	bf	29,2f

	stw	4,0(10)

	addi	10,10,4

	.align	4

2:	bf	30,1f

	sth	4,0(10)

	addi	10,10,2

	.align	4

1:	bflr	31

	stb	4,0(10)

	blr

	/* Expanded tree to copy tail bytes without increments.  */

	.align	4

L(copy_tail):

	bf	29,L(FXX)

	stw	4,0(10)

	bf	30,L(TFX)

	sth	4,4(10)

	bflr	31

	stb	4,6(10)

	blr

	.align	4

L(FXX):	bf	30,L(FFX)

	sth	4,0(10)

	bflr	31

	stb	4,2(10)

	blr

	.align	4

L(TFX):	bflr	31

	stb	4,4(10)

	blr

	.align	4

L(FFX):	bflr	31

	stb	4,0(10)

	blr

	/* Handle copies of 9~31 bytes.  */

	.align	4

L(medium):

	/* At least 9 bytes to go.  */

	andi.	11,10,3

	clrldi	0,0,62

	beq	L(medium_aligned)

	/* Force 4-bytes alignment for DST.  */

	mtocrf	0x01,0

	subf	5,0,5

1:	/* Copy 1 byte.  */

	bf	31,2f

	stb	4,0(10)

	addi	10,10,1

2:	/* Copy 2 bytes.  */

	bf	30,L(medium_aligned)

	sth	4,0(10)

	addi	10,10,2

	.align	4

L(medium_aligned):

	/* At least 6 bytes to go, and DST is word-aligned.  */

	cmpldi	cr1,5,16

	mtocrf	0x01,5

	blt	cr1,8f

	/* Copy 16 bytes.  */

	stw	4,0(10)

	stw	4,4(10)

	stw	4,8(10)

	stw	4,12(10)

	addi	10,10,16

8:	/* Copy 8 bytes.  */

	bf	28,4f

	stw	4,0(10)

	stw	4,4(10)

	addi	10,10,8

4:	/* Copy 4 bytes.  */

	bf	29,2f

	stw	4,0(10)

	addi	10,10,4

2:	/* Copy 2-3 bytes.  */

	bf	30,1f

	sth	4,0(10)

	addi	10,10,2

1:	/* Copy 1 byte.  */

	bflr	31

	stb	4,0(10)

	blr

	/* Handles copies of 0~8 bytes.  */

	.align	4

L(small):

	mtocrf	0x01,5

	bne	cr6,L(copy_tail)

	stw	4,0(10)

	stw	4,4(10)

	blr

END_GEN_TB (MEMSET,TB_TOCLESS)

libc_hidden_builtin_def (memset)

/* Copied from bzero.S to prevent the linker from inserting a stub

   between bzero and memset.  */

ENTRY_TOCLESS (__bzero)

	CALL_MCOUNT 3

	mr	r5,r4

	li	r4,0

	b	L(_memset)

END (__bzero)

#ifndef __bzero

weak_alias (__bzero, bzero)

#endif