/* memcpy - copy a block from source to destination. 31/64 bit S/390 version.
Copyright (C) 2012-2018 Free Software Foundation, Inc.
This file is part of the GNU C Library.
The GNU C Library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
The GNU C Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with the GNU C Library; if not, see
<http://www.gnu.org/licenses/>. */
#include <sysdep.h>
#include "asm-syntax.h"
#include <ifunc-memcpy.h>
/* INPUT PARAMETERS
%r2 = address of destination memory area
%r3 = address of source memory area
%r4 = number of bytes to copy. */
.text
#if defined __s390x__
# define LTGR ltgr
# define CGHI cghi
# define LGR lgr
# define AGHI aghi
# define BRCTG brctg
#else
# define LTGR ltr
# define CGHI chi
# define LGR lr
# define AGHI ahi
# define BRCTG brct
#endif /* ! defined __s390x__ */
#if HAVE_MEMCPY_Z900_G5
ENTRY(MEMPCPY_Z900_G5)
# if defined __s390x__
.machine "z900"
# else
.machine "g5"
# endif /* ! defined __s390x__ */
LGR %r1,%r2 # Use as dest
la %r2,0(%r4,%r2) # Return dest + n
j .L_Z900_G5_start
END(MEMPCPY_Z900_G5)
ENTRY(MEMCPY_Z900_G5)
# if defined __s390x__
.machine "z900"
# else
.machine "g5"
# endif /* ! defined __s390x__ */
LGR %r1,%r2 # r1: Use as dest ; r2: Return dest
.L_Z900_G5_start:
LTGR %r4,%r4
je .L_Z900_G5_4
AGHI %r4,-1
# if defined __s390x__
srlg %r5,%r4,8
# else
lr %r5,%r4
srl %r5,8
# endif /* ! defined __s390x__ */
LTGR %r5,%r5
jne .L_Z900_G5_13
.L_Z900_G5_3:
# if defined __s390x__
larl %r5,.L_Z900_G5_15
# define Z900_G5_EX_D 0
# else
basr %r5,0
.L_Z900_G5_14:
# define Z900_G5_EX_D .L_Z900_G5_15-.L_Z900_G5_14
# endif /* ! defined __s390x__ */
ex %r4,Z900_G5_EX_D(%r5)
.L_Z900_G5_4:
br %r14
.L_Z900_G5_13:
CGHI %r5,4096 # Switch to mvcle for copies >1MB
jh __memcpy_mvcle
.L_Z900_G5_12:
mvc 0(256,%r1),0(%r3)
la %r1,256(%r1)
la %r3,256(%r3)
BRCTG %r5,.L_Z900_G5_12
j .L_Z900_G5_3
.L_Z900_G5_15:
mvc 0(1,%r1),0(%r3)
END(MEMCPY_Z900_G5)
#endif /* HAVE_MEMCPY_Z900_G5 */
ENTRY(__memcpy_mvcle)
# Using as standalone function will result in unexpected
# results since the length field is incremented by 1 in order to
# compensate the changes already done in the functions above.
LGR %r0,%r2 # backup return dest [ + n ]
AGHI %r4,1 # length + 1
LGR %r5,%r4 # source length
LGR %r4,%r3 # source address
LGR %r2,%r1 # destination address
LGR %r3,%r5 # destination length = source length
.L_MVCLE_1:
mvcle %r2,%r4,0 # thats it, MVCLE is your friend
jo .L_MVCLE_1
LGR %r2,%r0 # return destination address
br %r14
END(__memcpy_mvcle)
#undef LTGR
#undef CGHI
#undef LGR
#undef AGHI
#undef BRCTG
#if HAVE_MEMCPY_Z10
ENTRY(MEMPCPY_Z10)
.machine "z10"
.machinemode "zarch_nohighgprs"
lgr %r1,%r2 # Use as dest
la %r2,0(%r4,%r2) # Return dest + n
j .L_Z10_start
END(MEMPCPY_Z10)
ENTRY(MEMCPY_Z10)
.machine "z10"
.machinemode "zarch_nohighgprs"
lgr %r1,%r2 # r1: Use as dest ; r2: Return dest
.L_Z10_start:
# if !defined __s390x__
llgfr %r4,%r4
# endif /* !defined __s390x__ */
cgije %r4,0,.L_Z10_4
aghi %r4,-1
srlg %r5,%r4,8
cgijlh %r5,0,.L_Z10_13
.L_Z10_3:
exrl %r4,.L_Z10_15
.L_Z10_4:
br %r14
.L_Z10_13:
cgfi %r5,65535 # Switch to mvcle for copies >16MB
jh __memcpy_mvcle
.L_Z10_12:
pfd 1,768(%r3)
pfd 2,768(%r1)
mvc 0(256,%r1),0(%r3)
la %r1,256(%r1)
la %r3,256(%r3)
brctg %r5,.L_Z10_12
j .L_Z10_3
.L_Z10_15:
mvc 0(1,%r1),0(%r3)
END(MEMCPY_Z10)
#endif /* HAVE_MEMCPY_Z10 */
#if HAVE_MEMCPY_Z196
ENTRY(MEMPCPY_Z196)
.machine "z196"
.machinemode "zarch_nohighgprs"
lgr %r1,%r2 # Use as dest
la %r2,0(%r4,%r2) # Return dest + n
j .L_Z196_start
END(MEMPCPY_Z196)
ENTRY(MEMCPY_Z196)
.machine "z196"
.machinemode "zarch_nohighgprs"
lgr %r1,%r2 # r1: Use as dest ; r2: Return dest
.L_Z196_start:
# if !defined __s390x__
llgfr %r4,%r4
# endif /* !defined __s390x__ */
ltgr %r4,%r4
je .L_Z196_4
.L_Z196_start2:
aghi %r4,-1
risbg %r5,%r4,8,128+63,56 # r0 = r5 / 256
jne .L_Z196_5
.L_Z196_3:
exrl %r4,.L_Z196_14
.L_Z196_4:
br %r14
.L_Z196_5:
cgfi %r5,255 # Switch to loop with pfd for copies >=64kB
jh .L_Z196_6
.L_Z196_2:
mvc 0(256,%r1),0(%r3)
aghi %r5,-1
la %r1,256(%r1)
la %r3,256(%r3)
jne .L_Z196_2
j .L_Z196_3
.L_Z196_6:
cgfi %r5,262144 # Switch to mvcle for copies >64MB
jh __memcpy_mvcle
.L_Z196_7:
pfd 1,1024(%r3)
pfd 2,1024(%r1)
mvc 0(256,%r1),0(%r3)
aghi %r5,-1
la %r1,256(%r1)
la %r3,256(%r3)
jne .L_Z196_7
j .L_Z196_3
.L_Z196_14:
mvc 0(1,%r1),0(%r3)
END(MEMCPY_Z196)
#endif /* HAVE_MEMCPY_Z196 */
#if HAVE_MEMMOVE_Z13
ENTRY(MEMMOVE_Z13)
.machine "z13"
.machinemode "zarch_nohighgprs"
# if !defined __s390x__
/* Note: The 31bit dst and src pointers are prefixed with zeroes. */
llgfr %r4,%r4
llgfr %r3,%r3
llgfr %r2,%r2
# endif /* !defined __s390x__ */
sgrk %r0,%r2,%r3
clgijh %r4,16,.L_MEMMOVE_Z13_LARGE
aghik %r5,%r4,-1
.L_MEMMOVE_Z13_SMALL:
jl .L_MEMMOVE_Z13_END /* Jump away if len was zero. */
/* Store up to 16 bytes with vll/vstl which needs the index
instead of lengths. */
vll %v16,%r5,0(%r3)
vstl %v16,%r5,0(%r2)
.L_MEMMOVE_Z13_END:
br %r14
.L_MEMMOVE_Z13_LARGE:
lgr %r1,%r2 /* For memcpy: r1: Use as dest ;
r2: Return dest */
/* The unsigned comparison (dst - src >= len) determines if we can
execute the forward case with memcpy. */
#if ! HAVE_MEMCPY_Z196
# error The z13 variant of memmove needs the z196 variant of memcpy!
#endif
clgrjhe %r0,%r4,.L_Z196_start2
risbgn %r5,%r4,4,128+63,60 /* r5 = r4 / 16 */
aghi %r4,-16
clgijhe %r5,8,.L_MEMMOVE_Z13_LARGE_64B
.L_MEMMOVE_Z13_LARGE_16B_LOOP:
/* Store at least 16 bytes with vl/vst. The number of 16byte blocks
is stored in r5. */
vl %v16,0(%r4,%r3)
vst %v16,0(%r4,%r2)
aghi %r4,-16
brctg %r5,.L_MEMMOVE_Z13_LARGE_16B_LOOP
aghik %r5,%r4,15
j .L_MEMMOVE_Z13_SMALL
.L_MEMMOVE_Z13_LARGE_64B:
/* Store at least 128 bytes with 4x vl/vst. The number of 64byte blocks
will be stored in r0. */
aghi %r4,-48
srlg %r0,%r5,2 /* r5 = %r0 / 4
=> Number of 64byte blocks. */
.L_MEMMOVE_Z13_LARGE_64B_LOOP:
vl %v20,48(%r4,%r3)
vl %v19,32(%r4,%r3)
vl %v18,16(%r4,%r3)
vl %v17,0(%r4,%r3)
vst %v20,48(%r4,%r2)
vst %v19,32(%r4,%r2)
vst %v18,16(%r4,%r2)
vst %v17,0(%r4,%r2)
aghi %r4,-64
brctg %r0,.L_MEMMOVE_Z13_LARGE_64B_LOOP
aghi %r4,48
/* Recalculate the number of 16byte blocks. */
risbg %r5,%r5,62,128+63,0 /* r5 = r5 & 3
=> Remaining 16byte blocks. */
jne .L_MEMMOVE_Z13_LARGE_16B_LOOP
aghik %r5,%r4,15
j .L_MEMMOVE_Z13_SMALL
END(MEMMOVE_Z13)
#endif /* HAVE_MEMMOVE_Z13 */
#if HAVE_MEMMOVE_ARCH13
ENTRY(MEMMOVE_ARCH13)
.machine "arch13"
.machinemode "zarch_nohighgprs"
# if ! defined __s390x__
/* Note: The 31bit dst and src pointers are prefixed with zeroes. */
llgfr %r4,%r4
llgfr %r3,%r3
llgfr %r2,%r2
# endif /* ! defined __s390x__ */
sgrk %r5,%r2,%r3
aghik %r0,%r4,-1 /* Both vstl and mvcrl needs highest index. */
clgijh %r4,16,.L_MEMMOVE_ARCH13_LARGE
.L_MEMMOVE_ARCH13_SMALL:
jl .L_MEMMOVE_ARCH13_END /* Return if len was zero (cc of aghik). */
/* Store up to 16 bytes with vll/vstl (needs highest index). */
vll %v16,%r0,0(%r3)
vstl %v16,%r0,0(%r2)
.L_MEMMOVE_ARCH13_END:
br %r14
.L_MEMMOVE_ARCH13_LARGE:
lgr %r1,%r2 /* For memcpy: r1: Use as dest ; r2: Return dest */
/* The unsigned comparison (dst - src >= len) determines if we can
execute the forward case with memcpy. */
#if ! HAVE_MEMCPY_Z196
# error The arch13 variant of memmove needs the z196 variant of memcpy!
#endif
/* Backward case. */
clgrjhe %r5,%r4,.L_Z196_start2
clgijh %r0,255,.L_MEMMOVE_ARCH13_LARGER_256B
/* Move up to 256bytes with mvcrl (move right to left). */
mvcrl 0(%r1),0(%r3) /* Move (r0 + 1) bytes from r3 to r1. */
br %r14
.L_MEMMOVE_ARCH13_LARGER_256B:
/* First move the "remaining" block of up to 256 bytes at the end of
src/dst buffers. Then move blocks of 256bytes in a loop starting
with the block at the end.
(If src/dst pointers are aligned e.g. to 256 bytes, then the pointers
passed to mvcrl instructions are aligned, too) */
risbgn %r5,%r0,8,128+63,56 /* r5 = r0 / 256 */
risbgn %r0,%r0,56,128+63,0 /* r0 = r0 & 0xFF */
slgr %r4,%r0
lay %r1,-1(%r4,%r1)
lay %r3,-1(%r4,%r3)
mvcrl 0(%r1),0(%r3) /* Move (r0 + 1) bytes from r3 to r1. */
lghi %r0,255 /* Always copy 256 bytes in the loop below! */
.L_MEMMOVE_ARCH13_LARGE_256B_LOOP:
aghi %r1,-256
aghi %r3,-256
mvcrl 0(%r1),0(%r3) /* Move (r0 + 1) bytes from r3 to r1. */
brctg %r5,.L_MEMMOVE_ARCH13_LARGE_256B_LOOP
br %r14
END(MEMMOVE_ARCH13)
#endif /* HAVE_MEMMOVE_ARCH13 */
#if ! HAVE_MEMCPY_IFUNC
/* If we don't use ifunc, define an alias for mem[p]cpy here.
Otherwise see sysdeps/s390/mem[p]cpy.c. */
strong_alias (MEMCPY_DEFAULT, memcpy)
strong_alias (MEMPCPY_DEFAULT, __mempcpy)
weak_alias (__mempcpy, mempcpy)
#endif
#if ! HAVE_MEMMOVE_IFUNC
/* If we don't use ifunc, define an alias for memmove here.
Otherwise see sysdeps/s390/memmove.c. */
# if ! HAVE_MEMMOVE_C
/* If the c variant is needed, then sysdeps/s390/memmove-c.c
defines memmove.
Otherwise MEMMOVE_DEFAULT is implemented here and we have to define it. */
strong_alias (MEMMOVE_DEFAULT, memmove)
# endif
#endif
#if defined SHARED && IS_IN (libc)
/* Defines the internal symbols.
Compare to libc_hidden_[builtin_]def (mem[p]cpy) in string/mem[p]cpy.c. */
strong_alias (MEMCPY_DEFAULT, __GI_memcpy)
strong_alias (MEMPCPY_DEFAULT, __GI_mempcpy)
strong_alias (MEMPCPY_DEFAULT, __GI___mempcpy)
# if ! HAVE_MEMMOVE_C
/* If the c variant is needed, then sysdeps/s390/memmove-c.c
defines the internal symbol.
Otherwise MEMMOVE_DEFAULT is implemented here and we have to define it. */
strong_alias (MEMMOVE_DEFAULT, __GI_memmove)
# endif
#endif