|
Packit |
1e2b20 |
/* memcpy - copy a block from source to destination. 31/64 bit S/390 version.
|
|
Packit |
6c4009 |
Copyright (C) 2012-2018 Free Software Foundation, Inc.
|
|
Packit |
6c4009 |
This file is part of the GNU C Library.
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
The GNU C Library is free software; you can redistribute it and/or
|
|
Packit |
6c4009 |
modify it under the terms of the GNU Lesser General Public
|
|
Packit |
6c4009 |
License as published by the Free Software Foundation; either
|
|
Packit |
6c4009 |
version 2.1 of the License, or (at your option) any later version.
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
The GNU C Library is distributed in the hope that it will be useful,
|
|
Packit |
6c4009 |
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
Packit |
6c4009 |
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
Packit |
6c4009 |
Lesser General Public License for more details.
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
You should have received a copy of the GNU Lesser General Public
|
|
Packit |
6c4009 |
License along with the GNU C Library; if not, see
|
|
Packit |
6c4009 |
<http://www.gnu.org/licenses/>. */
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
#include <sysdep.h>
|
|
Packit |
6c4009 |
#include "asm-syntax.h"
|
|
Packit |
a65cea |
#include <ifunc-memcpy.h>
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
/* INPUT PARAMETERS
|
|
Packit |
6c4009 |
%r2 = address of destination memory area
|
|
Packit |
6c4009 |
%r3 = address of source memory area
|
|
Packit |
6c4009 |
%r4 = number of bytes to copy. */
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
.text
|
|
Packit |
1e2b20 |
|
|
Packit |
1e2b20 |
#if defined __s390x__
|
|
Packit |
1e2b20 |
# define LTGR ltgr
|
|
Packit |
1e2b20 |
# define CGHI cghi
|
|
Packit |
1e2b20 |
# define LGR lgr
|
|
Packit |
1e2b20 |
# define AGHI aghi
|
|
Packit |
1e2b20 |
# define BRCTG brctg
|
|
Packit |
1e2b20 |
#else
|
|
Packit |
1e2b20 |
# define LTGR ltr
|
|
Packit |
1e2b20 |
# define CGHI chi
|
|
Packit |
1e2b20 |
# define LGR lr
|
|
Packit |
1e2b20 |
# define AGHI ahi
|
|
Packit |
1e2b20 |
# define BRCTG brct
|
|
Packit |
1e2b20 |
#endif /* ! defined __s390x__ */
|
|
Packit |
1e2b20 |
|
|
Packit |
a65cea |
#if HAVE_MEMCPY_Z900_G5
|
|
Packit |
a65cea |
ENTRY(MEMPCPY_Z900_G5)
|
|
Packit |
a65cea |
# if defined __s390x__
|
|
Packit |
6c4009 |
.machine "z900"
|
|
Packit |
a65cea |
# else
|
|
Packit |
1e2b20 |
.machine "g5"
|
|
Packit |
a65cea |
# endif /* ! defined __s390x__ */
|
|
Packit |
1e2b20 |
LGR %r1,%r2 # Use as dest
|
|
Packit |
6c4009 |
la %r2,0(%r4,%r2) # Return dest + n
|
|
Packit |
1e2b20 |
j .L_Z900_G5_start
|
|
Packit |
a65cea |
END(MEMPCPY_Z900_G5)
|
|
Packit |
6c4009 |
|
|
Packit |
a65cea |
ENTRY(MEMCPY_Z900_G5)
|
|
Packit |
a65cea |
# if defined __s390x__
|
|
Packit |
6c4009 |
.machine "z900"
|
|
Packit |
a65cea |
# else
|
|
Packit |
1e2b20 |
.machine "g5"
|
|
Packit |
a65cea |
# endif /* ! defined __s390x__ */
|
|
Packit |
1e2b20 |
LGR %r1,%r2 # r1: Use as dest ; r2: Return dest
|
|
Packit |
1e2b20 |
.L_Z900_G5_start:
|
|
Packit |
1e2b20 |
LTGR %r4,%r4
|
|
Packit |
1e2b20 |
je .L_Z900_G5_4
|
|
Packit |
1e2b20 |
AGHI %r4,-1
|
|
Packit |
a65cea |
# if defined __s390x__
|
|
Packit |
1e2b20 |
srlg %r5,%r4,8
|
|
Packit |
a65cea |
# else
|
|
Packit |
1e2b20 |
lr %r5,%r4
|
|
Packit |
1e2b20 |
srl %r5,8
|
|
Packit |
a65cea |
# endif /* ! defined __s390x__ */
|
|
Packit |
1e2b20 |
LTGR %r5,%r5
|
|
Packit |
1e2b20 |
jne .L_Z900_G5_13
|
|
Packit |
1e2b20 |
.L_Z900_G5_3:
|
|
Packit |
a65cea |
# if defined __s390x__
|
|
Packit |
1e2b20 |
larl %r5,.L_Z900_G5_15
|
|
Packit |
a65cea |
# define Z900_G5_EX_D 0
|
|
Packit |
a65cea |
# else
|
|
Packit |
1e2b20 |
basr %r5,0
|
|
Packit |
1e2b20 |
.L_Z900_G5_14:
|
|
Packit |
a65cea |
# define Z900_G5_EX_D .L_Z900_G5_15-.L_Z900_G5_14
|
|
Packit |
a65cea |
# endif /* ! defined __s390x__ */
|
|
Packit |
1e2b20 |
ex %r4,Z900_G5_EX_D(%r5)
|
|
Packit |
1e2b20 |
.L_Z900_G5_4:
|
|
Packit |
6c4009 |
br %r14
|
|
Packit |
1e2b20 |
.L_Z900_G5_13:
|
|
Packit |
1e2b20 |
CGHI %r5,4096 # Switch to mvcle for copies >1MB
|
|
Packit |
6c4009 |
jh __memcpy_mvcle
|
|
Packit |
1e2b20 |
.L_Z900_G5_12:
|
|
Packit |
6c4009 |
mvc 0(256,%r1),0(%r3)
|
|
Packit |
6c4009 |
la %r1,256(%r1)
|
|
Packit |
6c4009 |
la %r3,256(%r3)
|
|
Packit |
1e2b20 |
BRCTG %r5,.L_Z900_G5_12
|
|
Packit |
1e2b20 |
j .L_Z900_G5_3
|
|
Packit |
1e2b20 |
.L_Z900_G5_15:
|
|
Packit |
6c4009 |
mvc 0(1,%r1),0(%r3)
|
|
Packit |
a65cea |
END(MEMCPY_Z900_G5)
|
|
Packit |
a65cea |
#endif /* HAVE_MEMCPY_Z900_G5 */
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
ENTRY(__memcpy_mvcle)
|
|
Packit |
6c4009 |
# Using as standalone function will result in unexpected
|
|
Packit |
6c4009 |
# results since the length field is incremented by 1 in order to
|
|
Packit |
6c4009 |
# compensate the changes already done in the functions above.
|
|
Packit |
1e2b20 |
LGR %r0,%r2 # backup return dest [ + n ]
|
|
Packit |
1e2b20 |
AGHI %r4,1 # length + 1
|
|
Packit |
1e2b20 |
LGR %r5,%r4 # source length
|
|
Packit |
1e2b20 |
LGR %r4,%r3 # source address
|
|
Packit |
1e2b20 |
LGR %r2,%r1 # destination address
|
|
Packit |
1e2b20 |
LGR %r3,%r5 # destination length = source length
|
|
Packit |
6c4009 |
.L_MVCLE_1:
|
|
Packit |
6c4009 |
mvcle %r2,%r4,0 # thats it, MVCLE is your friend
|
|
Packit |
6c4009 |
jo .L_MVCLE_1
|
|
Packit |
1e2b20 |
LGR %r2,%r0 # return destination address
|
|
Packit |
6c4009 |
br %r14
|
|
Packit |
6c4009 |
END(__memcpy_mvcle)
|
|
Packit |
1e2b20 |
|
|
Packit |
1e2b20 |
#undef LTGR
|
|
Packit |
1e2b20 |
#undef CGHI
|
|
Packit |
1e2b20 |
#undef LGR
|
|
Packit |
1e2b20 |
#undef AGHI
|
|
Packit |
1e2b20 |
#undef BRCTG
|
|
Packit |
a65cea |
|
|
Packit |
a65cea |
#if HAVE_MEMCPY_Z10
|
|
Packit |
a65cea |
ENTRY(MEMPCPY_Z10)
|
|
Packit |
a65cea |
.machine "z10"
|
|
Packit |
a65cea |
.machinemode "zarch_nohighgprs"
|
|
Packit |
a65cea |
lgr %r1,%r2 # Use as dest
|
|
Packit |
a65cea |
la %r2,0(%r4,%r2) # Return dest + n
|
|
Packit |
a65cea |
j .L_Z10_start
|
|
Packit |
a65cea |
END(MEMPCPY_Z10)
|
|
Packit |
a65cea |
|
|
Packit |
a65cea |
ENTRY(MEMCPY_Z10)
|
|
Packit |
a65cea |
.machine "z10"
|
|
Packit |
a65cea |
.machinemode "zarch_nohighgprs"
|
|
Packit |
a65cea |
lgr %r1,%r2 # r1: Use as dest ; r2: Return dest
|
|
Packit |
a65cea |
.L_Z10_start:
|
|
Packit |
a65cea |
# if !defined __s390x__
|
|
Packit |
a65cea |
llgfr %r4,%r4
|
|
Packit |
a65cea |
# endif /* !defined __s390x__ */
|
|
Packit |
a65cea |
cgije %r4,0,.L_Z10_4
|
|
Packit |
a65cea |
aghi %r4,-1
|
|
Packit |
a65cea |
srlg %r5,%r4,8
|
|
Packit |
a65cea |
cgijlh %r5,0,.L_Z10_13
|
|
Packit |
a65cea |
.L_Z10_3:
|
|
Packit |
a65cea |
exrl %r4,.L_Z10_15
|
|
Packit |
a65cea |
.L_Z10_4:
|
|
Packit |
a65cea |
br %r14
|
|
Packit |
a65cea |
.L_Z10_13:
|
|
Packit |
a65cea |
cgfi %r5,65535 # Switch to mvcle for copies >16MB
|
|
Packit |
a65cea |
jh __memcpy_mvcle
|
|
Packit |
a65cea |
.L_Z10_12:
|
|
Packit |
a65cea |
pfd 1,768(%r3)
|
|
Packit |
a65cea |
pfd 2,768(%r1)
|
|
Packit |
a65cea |
mvc 0(256,%r1),0(%r3)
|
|
Packit |
a65cea |
la %r1,256(%r1)
|
|
Packit |
a65cea |
la %r3,256(%r3)
|
|
Packit |
a65cea |
brctg %r5,.L_Z10_12
|
|
Packit |
a65cea |
j .L_Z10_3
|
|
Packit |
a65cea |
.L_Z10_15:
|
|
Packit |
a65cea |
mvc 0(1,%r1),0(%r3)
|
|
Packit |
a65cea |
END(MEMCPY_Z10)
|
|
Packit |
a65cea |
#endif /* HAVE_MEMCPY_Z10 */
|
|
Packit |
a65cea |
|
|
Packit |
a65cea |
#if HAVE_MEMCPY_Z196
|
|
Packit |
a65cea |
ENTRY(MEMPCPY_Z196)
|
|
Packit |
a65cea |
.machine "z196"
|
|
Packit |
a65cea |
.machinemode "zarch_nohighgprs"
|
|
Packit |
a65cea |
lgr %r1,%r2 # Use as dest
|
|
Packit |
a65cea |
la %r2,0(%r4,%r2) # Return dest + n
|
|
Packit |
a65cea |
j .L_Z196_start
|
|
Packit |
a65cea |
END(MEMPCPY_Z196)
|
|
Packit |
a65cea |
|
|
Packit |
a65cea |
ENTRY(MEMCPY_Z196)
|
|
Packit |
a65cea |
.machine "z196"
|
|
Packit |
a65cea |
.machinemode "zarch_nohighgprs"
|
|
Packit |
a65cea |
lgr %r1,%r2 # r1: Use as dest ; r2: Return dest
|
|
Packit |
a65cea |
.L_Z196_start:
|
|
Packit |
a65cea |
# if !defined __s390x__
|
|
Packit |
a65cea |
llgfr %r4,%r4
|
|
Packit |
a65cea |
# endif /* !defined __s390x__ */
|
|
Packit |
a65cea |
ltgr %r4,%r4
|
|
Packit |
a65cea |
je .L_Z196_4
|
|
Packit |
61cd83 |
.L_Z196_start2:
|
|
Packit |
a65cea |
aghi %r4,-1
|
|
Packit |
a65cea |
srlg %r5,%r4,8
|
|
Packit |
a65cea |
ltgr %r5,%r5
|
|
Packit |
a65cea |
jne .L_Z196_5
|
|
Packit |
a65cea |
.L_Z196_3:
|
|
Packit |
a65cea |
exrl %r4,.L_Z196_14
|
|
Packit |
a65cea |
.L_Z196_4:
|
|
Packit |
a65cea |
br %r14
|
|
Packit |
a65cea |
.L_Z196_5:
|
|
Packit |
a65cea |
cgfi %r5,262144 # Switch to mvcle for copies >64MB
|
|
Packit |
a65cea |
jh __memcpy_mvcle
|
|
Packit |
a65cea |
.L_Z196_2:
|
|
Packit |
a65cea |
pfd 1,768(%r3)
|
|
Packit |
a65cea |
pfd 2,768(%r1)
|
|
Packit |
a65cea |
mvc 0(256,%r1),0(%r3)
|
|
Packit |
a65cea |
aghi %r5,-1
|
|
Packit |
a65cea |
la %r1,256(%r1)
|
|
Packit |
a65cea |
la %r3,256(%r3)
|
|
Packit |
a65cea |
jne .L_Z196_2
|
|
Packit |
a65cea |
j .L_Z196_3
|
|
Packit |
a65cea |
.L_Z196_14:
|
|
Packit |
a65cea |
mvc 0(1,%r1),0(%r3)
|
|
Packit |
a65cea |
END(MEMCPY_Z196)
|
|
Packit |
a65cea |
#endif /* HAVE_MEMCPY_Z196 */
|
|
Packit |
a65cea |
|
|
Packit |
61cd83 |
#if HAVE_MEMMOVE_Z13
|
|
Packit |
61cd83 |
ENTRY(MEMMOVE_Z13)
|
|
Packit |
61cd83 |
.machine "z13"
|
|
Packit |
61cd83 |
.machinemode "zarch_nohighgprs"
|
|
Packit |
61cd83 |
# if !defined __s390x__
|
|
Packit |
61cd83 |
/* Note: The 31bit dst and src pointers are prefixed with zeroes. */
|
|
Packit |
61cd83 |
llgfr %r4,%r4
|
|
Packit |
61cd83 |
llgfr %r3,%r3
|
|
Packit |
61cd83 |
llgfr %r2,%r2
|
|
Packit |
61cd83 |
# endif /* !defined __s390x__ */
|
|
Packit |
61cd83 |
sgrk %r0,%r2,%r3
|
|
Packit |
61cd83 |
clgijh %r4,16,.L_MEMMOVE_Z13_LARGE
|
|
Packit |
61cd83 |
aghik %r5,%r4,-1
|
|
Packit |
61cd83 |
.L_MEMMOVE_Z13_SMALL:
|
|
Packit |
61cd83 |
jl .L_MEMMOVE_Z13_END /* Jump away if len was zero. */
|
|
Packit |
61cd83 |
/* Store up to 16 bytes with vll/vstl which needs the index
|
|
Packit |
61cd83 |
instead of lengths. */
|
|
Packit |
61cd83 |
vll %v16,%r5,0(%r3)
|
|
Packit |
61cd83 |
vstl %v16,%r5,0(%r2)
|
|
Packit |
61cd83 |
.L_MEMMOVE_Z13_END:
|
|
Packit |
61cd83 |
br %r14
|
|
Packit |
61cd83 |
.L_MEMMOVE_Z13_LARGE:
|
|
Packit |
61cd83 |
lgr %r1,%r2 /* For memcpy: r1: Use as dest ;
|
|
Packit |
61cd83 |
r2: Return dest */
|
|
Packit |
61cd83 |
/* The unsigned comparison (dst - src >= len) determines if we can
|
|
Packit |
61cd83 |
execute the forward case with memcpy. */
|
|
Packit |
61cd83 |
#if ! HAVE_MEMCPY_Z196
|
|
Packit |
61cd83 |
# error The z13 variant of memmove needs the z196 variant of memcpy!
|
|
Packit |
61cd83 |
#endif
|
|
Packit |
61cd83 |
clgrjhe %r0,%r4,.L_Z196_start2
|
|
Packit |
61cd83 |
risbgn %r5,%r4,4,128+63,60 /* r5 = r4 / 16 */
|
|
Packit |
61cd83 |
aghi %r4,-16
|
|
Packit |
61cd83 |
clgijhe %r5,8,.L_MEMMOVE_Z13_LARGE_64B
|
|
Packit |
61cd83 |
.L_MEMMOVE_Z13_LARGE_16B_LOOP:
|
|
Packit |
61cd83 |
/* Store at least 16 bytes with vl/vst. The number of 16byte blocks
|
|
Packit |
61cd83 |
is stored in r5. */
|
|
Packit |
61cd83 |
vl %v16,0(%r4,%r3)
|
|
Packit |
61cd83 |
vst %v16,0(%r4,%r2)
|
|
Packit |
61cd83 |
aghi %r4,-16
|
|
Packit |
61cd83 |
brctg %r5,.L_MEMMOVE_Z13_LARGE_16B_LOOP
|
|
Packit |
61cd83 |
aghik %r5,%r4,15
|
|
Packit |
61cd83 |
j .L_MEMMOVE_Z13_SMALL
|
|
Packit |
61cd83 |
.L_MEMMOVE_Z13_LARGE_64B:
|
|
Packit |
61cd83 |
/* Store at least 128 bytes with 4x vl/vst. The number of 64byte blocks
|
|
Packit |
61cd83 |
will be stored in r0. */
|
|
Packit |
61cd83 |
aghi %r4,-48
|
|
Packit |
61cd83 |
srlg %r0,%r5,2 /* r5 = %r0 / 4
|
|
Packit |
61cd83 |
=> Number of 64byte blocks. */
|
|
Packit |
61cd83 |
.L_MEMMOVE_Z13_LARGE_64B_LOOP:
|
|
Packit |
61cd83 |
vl %v20,48(%r4,%r3)
|
|
Packit |
61cd83 |
vl %v19,32(%r4,%r3)
|
|
Packit |
61cd83 |
vl %v18,16(%r4,%r3)
|
|
Packit |
61cd83 |
vl %v17,0(%r4,%r3)
|
|
Packit |
61cd83 |
vst %v20,48(%r4,%r2)
|
|
Packit |
61cd83 |
vst %v19,32(%r4,%r2)
|
|
Packit |
61cd83 |
vst %v18,16(%r4,%r2)
|
|
Packit |
61cd83 |
vst %v17,0(%r4,%r2)
|
|
Packit |
61cd83 |
aghi %r4,-64
|
|
Packit |
61cd83 |
brctg %r0,.L_MEMMOVE_Z13_LARGE_64B_LOOP
|
|
Packit |
61cd83 |
aghi %r4,48
|
|
Packit |
61cd83 |
/* Recalculate the number of 16byte blocks. */
|
|
Packit |
61cd83 |
risbg %r5,%r5,62,128+63,0 /* r5 = r5 & 3
|
|
Packit |
61cd83 |
=> Remaining 16byte blocks. */
|
|
Packit |
61cd83 |
jne .L_MEMMOVE_Z13_LARGE_16B_LOOP
|
|
Packit |
61cd83 |
aghik %r5,%r4,15
|
|
Packit |
61cd83 |
j .L_MEMMOVE_Z13_SMALL
|
|
Packit |
61cd83 |
END(MEMMOVE_Z13)
|
|
Packit |
61cd83 |
#endif /* HAVE_MEMMOVE_Z13 */
|
|
Packit |
61cd83 |
|
|
Packit |
a65cea |
#if ! HAVE_MEMCPY_IFUNC
|
|
Packit |
a65cea |
/* If we don't use ifunc, define an alias for mem[p]cpy here.
|
|
Packit |
a65cea |
Otherwise see sysdeps/s390/mem[p]cpy.c. */
|
|
Packit |
a65cea |
strong_alias (MEMCPY_DEFAULT, memcpy)
|
|
Packit |
a65cea |
strong_alias (MEMPCPY_DEFAULT, __mempcpy)
|
|
Packit |
a65cea |
weak_alias (__mempcpy, mempcpy)
|
|
Packit |
a65cea |
#endif
|
|
Packit |
a65cea |
|
|
Packit |
61cd83 |
#if ! HAVE_MEMMOVE_IFUNC
|
|
Packit |
61cd83 |
/* If we don't use ifunc, define an alias for memmove here.
|
|
Packit |
61cd83 |
Otherwise see sysdeps/s390/memmove.c. */
|
|
Packit |
61cd83 |
# if ! HAVE_MEMMOVE_C
|
|
Packit |
61cd83 |
/* If the c variant is needed, then sysdeps/s390/memmove-c.c
|
|
Packit |
61cd83 |
defines memmove.
|
|
Packit |
61cd83 |
Otherwise MEMMOVE_DEFAULT is implemented here and we have to define it. */
|
|
Packit |
61cd83 |
strong_alias (MEMMOVE_DEFAULT, memmove)
|
|
Packit |
61cd83 |
# endif
|
|
Packit |
61cd83 |
#endif
|
|
Packit |
61cd83 |
|
|
Packit |
a65cea |
#if defined SHARED && IS_IN (libc)
|
|
Packit |
a65cea |
/* Defines the internal symbols.
|
|
Packit |
a65cea |
Compare to libc_hidden_[builtin_]def (mem[p]cpy) in string/mem[p]cpy.c. */
|
|
Packit |
a65cea |
strong_alias (MEMCPY_DEFAULT, __GI_memcpy)
|
|
Packit |
a65cea |
strong_alias (MEMPCPY_DEFAULT, __GI_mempcpy)
|
|
Packit |
a65cea |
strong_alias (MEMPCPY_DEFAULT, __GI___mempcpy)
|
|
Packit |
61cd83 |
# if ! HAVE_MEMMOVE_C
|
|
Packit |
61cd83 |
/* If the c variant is needed, then sysdeps/s390/memmove-c.c
|
|
Packit |
61cd83 |
defines the internal symbol.
|
|
Packit |
61cd83 |
Otherwise MEMMOVE_DEFAULT is implemented here and we have to define it. */
|
|
Packit |
61cd83 |
strong_alias (MEMMOVE_DEFAULT, __GI_memmove)
|
|
Packit |
61cd83 |
# endif
|
|
Packit |
a65cea |
#endif
|