/* memset optimized with AVX512 for KNL hardware.

   Copyright (C) 2015-2018 Free Software Foundation, Inc.

   This file is part of the GNU C Library.

   The GNU C Library is free software; you can redistribute it and/or

   modify it under the terms of the GNU Lesser General Public

   License as published by the Free Software Foundation; either

   version 2.1 of the License, or (at your option) any later version.

   The GNU C Library is distributed in the hope that it will be useful,

   but WITHOUT ANY WARRANTY; without even the implied warranty of

   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU

   Lesser General Public License for more details.

   You should have received a copy of the GNU Lesser General Public

   License along with the GNU C Library; if not, see

   <http://www.gnu.org/licenses/>.  */

#include <sysdep.h>

#if IS_IN (libc)

#include "asm-syntax.h"

#ifndef MEMSET

# define MEMSET __memset_avx512_no_vzeroupper

# define MEMSET_CHK __memset_chk_avx512_no_vzeroupper

#endif

	.section .text.avx512,"ax",@progbits

#if defined PIC

ENTRY (MEMSET_CHK)

	cmpq	%rdx, %rcx

	jb	HIDDEN_JUMPTARGET (__chk_fail)

END (MEMSET_CHK)

#endif

ENTRY (MEMSET)

	vpxor	%xmm0, %xmm0, %xmm0

	vmovd	%esi, %xmm1

	lea	(%rdi, %rdx), %rsi

	mov	%rdi, %rax

	vpshufb	%xmm0, %xmm1, %xmm0

	cmp	$16, %rdx

	jb	L(less_16bytes)

	cmp	$512, %rdx

	vbroadcastss	%xmm0, %zmm2

	ja	L(512bytesormore)

	cmp	$256, %rdx

	jb	L(less_256bytes)

	vmovups	%zmm2, (%rdi)

	vmovups %zmm2, 0x40(%rdi)

	vmovups %zmm2, 0x80(%rdi)

	vmovups %zmm2, 0xC0(%rdi)

	vmovups %zmm2, -0x100(%rsi)

	vmovups %zmm2, -0xC0(%rsi)

	vmovups %zmm2, -0x80(%rsi)

	vmovups %zmm2, -0x40(%rsi)

	ret

L(less_256bytes):

	cmp	$128, %dl

	jb	L(less_128bytes)

	vmovups	%zmm2, (%rdi)

	vmovups %zmm2, 0x40(%rdi)

	vmovups %zmm2, -0x80(%rsi)

	vmovups %zmm2, -0x40(%rsi)

	ret

L(less_128bytes):

	cmp	$64, %dl

	jb	L(less_64bytes)

	vmovups	%zmm2, (%rdi)

	vmovups	%zmm2, -0x40(%rsi)

	ret

L(less_64bytes):

	cmp	$32, %dl

	jb	L(less_32bytes)

	vmovdqu	%ymm2, (%rdi)

	vmovdqu %ymm2, -0x20(%rsi)

	ret

L(less_32bytes):

	vmovdqu %xmm0, (%rdi)

	vmovdqu %xmm0, -0x10(%rsi)

	ret

L(less_16bytes):

	cmp	$8, %dl

	jb	L(less_8bytes)

	vmovq	%xmm0, (%rdi)

	vmovq	%xmm0, -0x08(%rsi)

	ret

L(less_8bytes):

	vmovd	%xmm0, %ecx

	cmp	$4, %dl

	jb	L(less_4bytes)

	mov	%ecx, (%rdi)

	mov	%ecx, -0x04(%rsi)

	ret

L(less_4bytes):

	cmp	$2, %dl

	jb	L(less_2bytes)

	mov	%cx, (%rdi)

	mov	%cx, -0x02(%rsi)

	ret

L(less_2bytes):

	cmp	$1, %dl

	jb	L(less_1bytes)

	mov	%cl, (%rdi)

L(less_1bytes):

	ret

L(512bytesormore):

	mov	__x86_shared_cache_size_half(%rip), %rcx

	cmp	%rcx, %rdx

	ja	L(preloop_large)

	cmp	$1024, %rdx

	ja	L(1024bytesormore)

	vmovups	%zmm2, (%rdi)

	vmovups	%zmm2, 0x40(%rdi)

	vmovups	%zmm2, 0x80(%rdi)

	vmovups	%zmm2, 0xC0(%rdi)

	vmovups	%zmm2, 0x100(%rdi)

	vmovups	%zmm2, 0x140(%rdi)

	vmovups	%zmm2, 0x180(%rdi)

	vmovups	%zmm2, 0x1C0(%rdi)

	vmovups %zmm2, -0x200(%rsi)

	vmovups %zmm2, -0x1C0(%rsi)

	vmovups %zmm2, -0x180(%rsi)

	vmovups %zmm2, -0x140(%rsi)

	vmovups %zmm2, -0x100(%rsi)

	vmovups %zmm2, -0xC0(%rsi)

	vmovups %zmm2, -0x80(%rsi)

	vmovups %zmm2, -0x40(%rsi)

	ret

/* Align on 64 and loop with aligned stores.  */

L(1024bytesormore):

	sub	$0x100, %rsi

	vmovups	%zmm2, (%rax)

	and	$-0x40, %rdi

	add	$0x40, %rdi

L(gobble_256bytes_loop):

	vmovaps	%zmm2, (%rdi)

	vmovaps	%zmm2, 0x40(%rdi)

	vmovaps	%zmm2, 0x80(%rdi)

	vmovaps	%zmm2, 0xC0(%rdi)

	add	$0x100, %rdi

	cmp	%rsi, %rdi

	jb	L(gobble_256bytes_loop)

	vmovups %zmm2, (%rsi)

	vmovups %zmm2, 0x40(%rsi)

	vmovups %zmm2, 0x80(%rsi)

	vmovups %zmm2, 0xC0(%rsi)

	ret

/* Align on 128 and loop with non-temporal stores.  */

L(preloop_large):

	and	$-0x80, %rdi

	add	$0x80, %rdi

	vmovups	%zmm2, (%rax)

	vmovups	%zmm2, 0x40(%rax)

	sub	$0x200, %rsi

L(gobble_512bytes_nt_loop):

	vmovntdq %zmm2, (%rdi)

	vmovntdq %zmm2, 0x40(%rdi)

	vmovntdq %zmm2, 0x80(%rdi)

	vmovntdq %zmm2, 0xC0(%rdi)

	vmovntdq %zmm2, 0x100(%rdi)

	vmovntdq %zmm2, 0x140(%rdi)

	vmovntdq %zmm2, 0x180(%rdi)

	vmovntdq %zmm2, 0x1C0(%rdi)

	add	$0x200, %rdi

	cmp	%rsi, %rdi

	jb	L(gobble_512bytes_nt_loop)

	sfence

	vmovups %zmm2, (%rsi)

	vmovups %zmm2, 0x40(%rsi)

	vmovups %zmm2, 0x80(%rsi)

	vmovups %zmm2, 0xC0(%rsi)

	vmovups	%zmm2, 0x100(%rsi)

	vmovups	%zmm2, 0x140(%rsi)

	vmovups	%zmm2, 0x180(%rsi)

	vmovups	%zmm2, 0x1C0(%rsi)

	ret

END (MEMSET)

#endif