|
Packit |
5c3484 |
dnl AMD64 mpn_mul_basecase.
|
|
Packit |
5c3484 |
|
|
Packit |
5c3484 |
dnl Contributed to the GNU project by Torbjorn Granlund and David Harvey.
|
|
Packit |
5c3484 |
|
|
Packit |
5c3484 |
dnl Copyright 2008, 2012 Free Software Foundation, Inc.
|
|
Packit |
5c3484 |
|
|
Packit |
5c3484 |
dnl This file is part of the GNU MP Library.
|
|
Packit |
5c3484 |
dnl
|
|
Packit |
5c3484 |
dnl The GNU MP Library is free software; you can redistribute it and/or modify
|
|
Packit |
5c3484 |
dnl it under the terms of either:
|
|
Packit |
5c3484 |
dnl
|
|
Packit |
5c3484 |
dnl * the GNU Lesser General Public License as published by the Free
|
|
Packit |
5c3484 |
dnl Software Foundation; either version 3 of the License, or (at your
|
|
Packit |
5c3484 |
dnl option) any later version.
|
|
Packit |
5c3484 |
dnl
|
|
Packit |
5c3484 |
dnl or
|
|
Packit |
5c3484 |
dnl
|
|
Packit |
5c3484 |
dnl * the GNU General Public License as published by the Free Software
|
|
Packit |
5c3484 |
dnl Foundation; either version 2 of the License, or (at your option) any
|
|
Packit |
5c3484 |
dnl later version.
|
|
Packit |
5c3484 |
dnl
|
|
Packit |
5c3484 |
dnl or both in parallel, as here.
|
|
Packit |
5c3484 |
dnl
|
|
Packit |
5c3484 |
dnl The GNU MP Library is distributed in the hope that it will be useful, but
|
|
Packit |
5c3484 |
dnl WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
|
|
Packit |
5c3484 |
dnl or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
|
|
Packit |
5c3484 |
dnl for more details.
|
|
Packit |
5c3484 |
dnl
|
|
Packit |
5c3484 |
dnl You should have received copies of the GNU General Public License and the
|
|
Packit |
5c3484 |
dnl GNU Lesser General Public License along with the GNU MP Library. If not,
|
|
Packit |
5c3484 |
dnl see https://www.gnu.org/licenses/.
|
|
Packit |
5c3484 |
|
|
Packit |
5c3484 |
include(`../config.m4')
|
|
Packit |
5c3484 |
|
|
Packit |
5c3484 |
C cycles/limb
|
|
Packit |
5c3484 |
C AMD K8,K9 2.375
|
|
Packit |
5c3484 |
C AMD K10 2.375
|
|
Packit |
5c3484 |
C Intel P4 15-16
|
|
Packit |
5c3484 |
C Intel core2 4.45
|
|
Packit |
5c3484 |
C Intel corei 4.35
|
|
Packit |
5c3484 |
C Intel atom ?
|
|
Packit |
5c3484 |
C VIA nano 4.5
|
|
Packit |
5c3484 |
|
|
Packit |
5c3484 |
C The inner loops of this code are the result of running a code generation and
|
|
Packit |
5c3484 |
C optimization tool suite written by David Harvey and Torbjorn Granlund.
|
|
Packit |
5c3484 |
|
|
Packit |
5c3484 |
C TODO
|
|
Packit |
5c3484 |
C * Use fewer registers. (how??? I can't see it -- david)
|
|
Packit |
5c3484 |
C * Avoid some "mov $0,r" and instead use "xor r,r".
|
|
Packit |
5c3484 |
C * Can the top of each L(addmul_outer_n) prologue be folded into the
|
|
Packit |
5c3484 |
C mul_1/mul_2 prologues, saving a LEA (%rip)? It would slow down the
|
|
Packit |
5c3484 |
C case where vn = 1 or 2; is it worth it?
|
|
Packit |
5c3484 |
|
|
Packit |
5c3484 |
C INPUT PARAMETERS
|
|
Packit |
5c3484 |
define(`rp', `%rdi')
|
|
Packit |
5c3484 |
define(`up', `%rsi')
|
|
Packit |
5c3484 |
define(`un_param',`%rdx')
|
|
Packit |
5c3484 |
define(`vp', `%rcx')
|
|
Packit |
5c3484 |
define(`vn', `%r8')
|
|
Packit |
5c3484 |
|
|
Packit |
5c3484 |
define(`v0', `%r12')
|
|
Packit |
5c3484 |
define(`v1', `%r9')
|
|
Packit |
5c3484 |
|
|
Packit |
5c3484 |
define(`w0', `%rbx')
|
|
Packit |
5c3484 |
define(`w1', `%r15')
|
|
Packit |
5c3484 |
define(`w2', `%rbp')
|
|
Packit |
5c3484 |
define(`w3', `%r10')
|
|
Packit |
5c3484 |
|
|
Packit |
5c3484 |
define(`n', `%r11')
|
|
Packit |
5c3484 |
define(`outer_addr', `%r14')
|
|
Packit |
5c3484 |
define(`un', `%r13')
|
|
Packit |
5c3484 |
|
|
Packit |
5c3484 |
ABI_SUPPORT(DOS64)
|
|
Packit |
5c3484 |
ABI_SUPPORT(STD64)
|
|
Packit |
5c3484 |
|
|
Packit |
5c3484 |
ASM_START()
|
|
Packit |
5c3484 |
TEXT
|
|
Packit |
5c3484 |
ALIGN(16)
|
|
Packit |
5c3484 |
PROLOGUE(mpn_mul_basecase)
|
|
Packit |
5c3484 |
FUNC_ENTRY(4)
|
|
Packit |
5c3484 |
IFDOS(` mov 56(%rsp), %r8d ')
|
|
Packit |
5c3484 |
push %rbx
|
|
Packit |
5c3484 |
push %rbp
|
|
Packit |
5c3484 |
push %r12
|
|
Packit |
5c3484 |
push %r13
|
|
Packit |
5c3484 |
push %r14
|
|
Packit |
5c3484 |
push %r15
|
|
Packit |
5c3484 |
|
|
Packit |
5c3484 |
xor R32(un), R32(un)
|
|
Packit |
5c3484 |
mov (up), %rax
|
|
Packit |
5c3484 |
mov (vp), v0
|
|
Packit |
5c3484 |
|
|
Packit |
5c3484 |
sub un_param, un C rdx used by mul
|
|
Packit |
5c3484 |
mov un, n
|
|
Packit |
5c3484 |
mov R32(un_param), R32(w0)
|
|
Packit |
5c3484 |
|
|
Packit |
5c3484 |
lea (rp,un_param,8), rp
|
|
Packit |
5c3484 |
lea (up,un_param,8), up
|
|
Packit |
5c3484 |
|
|
Packit |
5c3484 |
mul v0
|
|
Packit |
5c3484 |
|
|
Packit |
5c3484 |
test $1, R8(vn)
|
|
Packit |
5c3484 |
jz L(mul_2)
|
|
Packit |
5c3484 |
|
|
Packit |
5c3484 |
C ===========================================================
|
|
Packit |
5c3484 |
C mul_1 for vp[0] if vn is odd
|
|
Packit |
5c3484 |
|
|
Packit |
5c3484 |
L(mul_1):
|
|
Packit |
5c3484 |
and $3, R32(w0)
|
|
Packit |
5c3484 |
jz L(mul_1_prologue_0)
|
|
Packit |
5c3484 |
cmp $2, R32(w0)
|
|
Packit |
5c3484 |
jc L(mul_1_prologue_1)
|
|
Packit |
5c3484 |
jz L(mul_1_prologue_2)
|
|
Packit |
5c3484 |
|
|
Packit |
5c3484 |
L(mul_1_prologue_3):
|
|
Packit |
5c3484 |
add $-1, n
|
|
Packit |
5c3484 |
lea L(addmul_outer_3)(%rip), outer_addr
|
|
Packit |
5c3484 |
mov %rax, w3
|
|
Packit |
5c3484 |
mov %rdx, w0
|
|
Packit |
5c3484 |
jmp L(mul_1_entry_3)
|
|
Packit |
5c3484 |
|
|
Packit |
5c3484 |
L(mul_1_prologue_0):
|
|
Packit |
5c3484 |
mov %rax, w2
|
|
Packit |
5c3484 |
mov %rdx, w3 C note: already w0 == 0
|
|
Packit |
5c3484 |
lea L(addmul_outer_0)(%rip), outer_addr
|
|
Packit |
5c3484 |
jmp L(mul_1_entry_0)
|
|
Packit |
5c3484 |
|
|
Packit |
5c3484 |
L(mul_1_prologue_1):
|
|
Packit |
5c3484 |
cmp $-1, un
|
|
Packit |
5c3484 |
jne 2f
|
|
Packit |
5c3484 |
mov %rax, -8(rp)
|
|
Packit |
5c3484 |
mov %rdx, (rp)
|
|
Packit |
5c3484 |
jmp L(ret)
|
|
Packit |
5c3484 |
2: add $1, n
|
|
Packit |
5c3484 |
lea L(addmul_outer_1)(%rip), outer_addr
|
|
Packit |
5c3484 |
mov %rax, w1
|
|
Packit |
5c3484 |
mov %rdx, w2
|
|
Packit |
5c3484 |
xor R32(w3), R32(w3)
|
|
Packit |
5c3484 |
mov (up,n,8), %rax
|
|
Packit |
5c3484 |
jmp L(mul_1_entry_1)
|
|
Packit |
5c3484 |
|
|
Packit |
5c3484 |
L(mul_1_prologue_2):
|
|
Packit |
5c3484 |
add $-2, n
|
|
Packit |
5c3484 |
lea L(addmul_outer_2)(%rip), outer_addr
|
|
Packit |
5c3484 |
mov %rax, w0
|
|
Packit |
5c3484 |
mov %rdx, w1
|
|
Packit |
5c3484 |
mov 24(up,n,8), %rax
|
|
Packit |
5c3484 |
xor R32(w2), R32(w2)
|
|
Packit |
5c3484 |
xor R32(w3), R32(w3)
|
|
Packit |
5c3484 |
jmp L(mul_1_entry_2)
|
|
Packit |
5c3484 |
|
|
Packit |
5c3484 |
|
|
Packit |
5c3484 |
C this loop is 10 c/loop = 2.5 c/l on K8, for all up/rp alignments
|
|
Packit |
5c3484 |
|
|
Packit |
5c3484 |
ALIGN(16)
|
|
Packit |
5c3484 |
L(mul_1_top):
|
|
Packit |
5c3484 |
mov w0, -16(rp,n,8)
|
|
Packit |
5c3484 |
add %rax, w1
|
|
Packit |
5c3484 |
mov (up,n,8), %rax
|
|
Packit |
5c3484 |
adc %rdx, w2
|
|
Packit |
5c3484 |
L(mul_1_entry_1):
|
|
Packit |
5c3484 |
xor R32(w0), R32(w0)
|
|
Packit |
5c3484 |
mul v0
|
|
Packit |
5c3484 |
mov w1, -8(rp,n,8)
|
|
Packit |
5c3484 |
add %rax, w2
|
|
Packit |
5c3484 |
adc %rdx, w3
|
|
Packit |
5c3484 |
L(mul_1_entry_0):
|
|
Packit |
5c3484 |
mov 8(up,n,8), %rax
|
|
Packit |
5c3484 |
mul v0
|
|
Packit |
5c3484 |
mov w2, (rp,n,8)
|
|
Packit |
5c3484 |
add %rax, w3
|
|
Packit |
5c3484 |
adc %rdx, w0
|
|
Packit |
5c3484 |
L(mul_1_entry_3):
|
|
Packit |
5c3484 |
mov 16(up,n,8), %rax
|
|
Packit |
5c3484 |
mul v0
|
|
Packit |
5c3484 |
mov w3, 8(rp,n,8)
|
|
Packit |
5c3484 |
xor R32(w2), R32(w2) C zero
|
|
Packit |
5c3484 |
mov w2, w3 C zero
|
|
Packit |
5c3484 |
add %rax, w0
|
|
Packit |
5c3484 |
mov 24(up,n,8), %rax
|
|
Packit |
5c3484 |
mov w2, w1 C zero
|
|
Packit |
5c3484 |
adc %rdx, w1
|
|
Packit |
5c3484 |
L(mul_1_entry_2):
|
|
Packit |
5c3484 |
mul v0
|
|
Packit |
5c3484 |
add $4, n
|
|
Packit |
5c3484 |
js L(mul_1_top)
|
|
Packit |
5c3484 |
|
|
Packit |
5c3484 |
mov w0, -16(rp)
|
|
Packit |
5c3484 |
add %rax, w1
|
|
Packit |
5c3484 |
mov w1, -8(rp)
|
|
Packit |
5c3484 |
adc %rdx, w2
|
|
Packit |
5c3484 |
mov w2, (rp)
|
|
Packit |
5c3484 |
|
|
Packit |
5c3484 |
add $-1, vn C vn -= 1
|
|
Packit |
5c3484 |
jz L(ret)
|
|
Packit |
5c3484 |
|
|
Packit |
5c3484 |
mov 8(vp), v0
|
|
Packit |
5c3484 |
mov 16(vp), v1
|
|
Packit |
5c3484 |
|
|
Packit |
5c3484 |
lea 8(vp), vp C vp += 1
|
|
Packit |
5c3484 |
lea 8(rp), rp C rp += 1
|
|
Packit |
5c3484 |
|
|
Packit |
5c3484 |
jmp *outer_addr
|
|
Packit |
5c3484 |
|
|
Packit |
5c3484 |
C ===========================================================
|
|
Packit |
5c3484 |
C mul_2 for vp[0], vp[1] if vn is even
|
|
Packit |
5c3484 |
|
|
Packit |
5c3484 |
ALIGN(16)
|
|
Packit |
5c3484 |
L(mul_2):
|
|
Packit |
5c3484 |
mov 8(vp), v1
|
|
Packit |
5c3484 |
|
|
Packit |
5c3484 |
and $3, R32(w0)
|
|
Packit |
5c3484 |
jz L(mul_2_prologue_0)
|
|
Packit |
5c3484 |
cmp $2, R32(w0)
|
|
Packit |
5c3484 |
jz L(mul_2_prologue_2)
|
|
Packit |
5c3484 |
jc L(mul_2_prologue_1)
|
|
Packit |
5c3484 |
|
|
Packit |
5c3484 |
L(mul_2_prologue_3):
|
|
Packit |
5c3484 |
lea L(addmul_outer_3)(%rip), outer_addr
|
|
Packit |
5c3484 |
add $2, n
|
|
Packit |
5c3484 |
mov %rax, -16(rp,n,8)
|
|
Packit |
5c3484 |
mov %rdx, w2
|
|
Packit |
5c3484 |
xor R32(w3), R32(w3)
|
|
Packit |
5c3484 |
xor R32(w0), R32(w0)
|
|
Packit |
5c3484 |
mov -16(up,n,8), %rax
|
|
Packit |
5c3484 |
jmp L(mul_2_entry_3)
|
|
Packit |
5c3484 |
|
|
Packit |
5c3484 |
ALIGN(16)
|
|
Packit |
5c3484 |
L(mul_2_prologue_0):
|
|
Packit |
5c3484 |
add $3, n
|
|
Packit |
5c3484 |
mov %rax, w0
|
|
Packit |
5c3484 |
mov %rdx, w1
|
|
Packit |
5c3484 |
xor R32(w2), R32(w2)
|
|
Packit |
5c3484 |
mov -24(up,n,8), %rax
|
|
Packit |
5c3484 |
lea L(addmul_outer_0)(%rip), outer_addr
|
|
Packit |
5c3484 |
jmp L(mul_2_entry_0)
|
|
Packit |
5c3484 |
|
|
Packit |
5c3484 |
ALIGN(16)
|
|
Packit |
5c3484 |
L(mul_2_prologue_1):
|
|
Packit |
5c3484 |
mov %rax, w3
|
|
Packit |
5c3484 |
mov %rdx, w0
|
|
Packit |
5c3484 |
xor R32(w1), R32(w1)
|
|
Packit |
5c3484 |
lea L(addmul_outer_1)(%rip), outer_addr
|
|
Packit |
5c3484 |
jmp L(mul_2_entry_1)
|
|
Packit |
5c3484 |
|
|
Packit |
5c3484 |
ALIGN(16)
|
|
Packit |
5c3484 |
L(mul_2_prologue_2):
|
|
Packit |
5c3484 |
add $1, n
|
|
Packit |
5c3484 |
lea L(addmul_outer_2)(%rip), outer_addr
|
|
Packit |
5c3484 |
mov $0, R32(w0)
|
|
Packit |
5c3484 |
mov $0, R32(w1)
|
|
Packit |
5c3484 |
mov %rax, w2
|
|
Packit |
5c3484 |
mov -8(up,n,8), %rax
|
|
Packit |
5c3484 |
mov %rdx, w3
|
|
Packit |
5c3484 |
jmp L(mul_2_entry_2)
|
|
Packit |
5c3484 |
|
|
Packit |
5c3484 |
C this loop is 18 c/loop = 2.25 c/l on K8, for all up/rp alignments
|
|
Packit |
5c3484 |
|
|
Packit |
5c3484 |
ALIGN(16)
|
|
Packit |
5c3484 |
L(mul_2_top):
|
|
Packit |
5c3484 |
mov -32(up,n,8), %rax
|
|
Packit |
5c3484 |
mul v1
|
|
Packit |
5c3484 |
add %rax, w0
|
|
Packit |
5c3484 |
adc %rdx, w1
|
|
Packit |
5c3484 |
mov -24(up,n,8), %rax
|
|
Packit |
5c3484 |
xor R32(w2), R32(w2)
|
|
Packit |
5c3484 |
mul v0
|
|
Packit |
5c3484 |
add %rax, w0
|
|
Packit |
5c3484 |
mov -24(up,n,8), %rax
|
|
Packit |
5c3484 |
adc %rdx, w1
|
|
Packit |
5c3484 |
adc $0, R32(w2)
|
|
Packit |
5c3484 |
L(mul_2_entry_0):
|
|
Packit |
5c3484 |
mul v1
|
|
Packit |
5c3484 |
add %rax, w1
|
|
Packit |
5c3484 |
mov w0, -24(rp,n,8)
|
|
Packit |
5c3484 |
adc %rdx, w2
|
|
Packit |
5c3484 |
mov -16(up,n,8), %rax
|
|
Packit |
5c3484 |
mul v0
|
|
Packit |
5c3484 |
mov $0, R32(w3)
|
|
Packit |
5c3484 |
add %rax, w1
|
|
Packit |
5c3484 |
adc %rdx, w2
|
|
Packit |
5c3484 |
mov -16(up,n,8), %rax
|
|
Packit |
5c3484 |
adc $0, R32(w3)
|
|
Packit |
5c3484 |
mov $0, R32(w0)
|
|
Packit |
5c3484 |
mov w1, -16(rp,n,8)
|
|
Packit |
5c3484 |
L(mul_2_entry_3):
|
|
Packit |
5c3484 |
mul v1
|
|
Packit |
5c3484 |
add %rax, w2
|
|
Packit |
5c3484 |
mov -8(up,n,8), %rax
|
|
Packit |
5c3484 |
adc %rdx, w3
|
|
Packit |
5c3484 |
mov $0, R32(w1)
|
|
Packit |
5c3484 |
mul v0
|
|
Packit |
5c3484 |
add %rax, w2
|
|
Packit |
5c3484 |
mov -8(up,n,8), %rax
|
|
Packit |
5c3484 |
adc %rdx, w3
|
|
Packit |
5c3484 |
adc R32(w1), R32(w0) C adc $0, w0
|
|
Packit |
5c3484 |
L(mul_2_entry_2):
|
|
Packit |
5c3484 |
mul v1
|
|
Packit |
5c3484 |
add %rax, w3
|
|
Packit |
5c3484 |
mov w2, -8(rp,n,8)
|
|
Packit |
5c3484 |
adc %rdx, w0
|
|
Packit |
5c3484 |
mov (up,n,8), %rax
|
|
Packit |
5c3484 |
mul v0
|
|
Packit |
5c3484 |
add %rax, w3
|
|
Packit |
5c3484 |
adc %rdx, w0
|
|
Packit |
5c3484 |
adc $0, R32(w1)
|
|
Packit |
5c3484 |
L(mul_2_entry_1):
|
|
Packit |
5c3484 |
add $4, n
|
|
Packit |
5c3484 |
mov w3, -32(rp,n,8)
|
|
Packit |
5c3484 |
js L(mul_2_top)
|
|
Packit |
5c3484 |
|
|
Packit |
5c3484 |
mov -32(up,n,8), %rax C FIXME: n is constant
|
|
Packit |
5c3484 |
mul v1
|
|
Packit |
5c3484 |
add %rax, w0
|
|
Packit |
5c3484 |
mov w0, (rp)
|
|
Packit |
5c3484 |
adc %rdx, w1
|
|
Packit |
5c3484 |
mov w1, 8(rp)
|
|
Packit |
5c3484 |
|
|
Packit |
5c3484 |
add $-2, vn C vn -= 2
|
|
Packit |
5c3484 |
jz L(ret)
|
|
Packit |
5c3484 |
|
|
Packit |
5c3484 |
mov 16(vp), v0
|
|
Packit |
5c3484 |
mov 24(vp), v1
|
|
Packit |
5c3484 |
|
|
Packit |
5c3484 |
lea 16(vp), vp C vp += 2
|
|
Packit |
5c3484 |
lea 16(rp), rp C rp += 2
|
|
Packit |
5c3484 |
|
|
Packit |
5c3484 |
jmp *outer_addr
|
|
Packit |
5c3484 |
|
|
Packit |
5c3484 |
|
|
Packit |
5c3484 |
C ===========================================================
|
|
Packit |
5c3484 |
C addmul_2 for remaining vp's
|
|
Packit |
5c3484 |
|
|
Packit |
5c3484 |
C in the following prologues, we reuse un to store the
|
|
Packit |
5c3484 |
C adjusted value of n that is reloaded on each iteration
|
|
Packit |
5c3484 |
|
|
Packit |
5c3484 |
L(addmul_outer_0):
|
|
Packit |
5c3484 |
add $3, un
|
|
Packit |
5c3484 |
lea 0(%rip), outer_addr
|
|
Packit |
5c3484 |
|
|
Packit |
5c3484 |
mov un, n
|
|
Packit |
5c3484 |
mov -24(up,un,8), %rax
|
|
Packit |
5c3484 |
mul v0
|
|
Packit |
5c3484 |
mov %rax, w0
|
|
Packit |
5c3484 |
mov -24(up,un,8), %rax
|
|
Packit |
5c3484 |
mov %rdx, w1
|
|
Packit |
5c3484 |
xor R32(w2), R32(w2)
|
|
Packit |
5c3484 |
jmp L(addmul_entry_0)
|
|
Packit |
5c3484 |
|
|
Packit |
5c3484 |
L(addmul_outer_1):
|
|
Packit |
5c3484 |
mov un, n
|
|
Packit |
5c3484 |
mov (up,un,8), %rax
|
|
Packit |
5c3484 |
mul v0
|
|
Packit |
5c3484 |
mov %rax, w3
|
|
Packit |
5c3484 |
mov (up,un,8), %rax
|
|
Packit |
5c3484 |
mov %rdx, w0
|
|
Packit |
5c3484 |
xor R32(w1), R32(w1)
|
|
Packit |
5c3484 |
jmp L(addmul_entry_1)
|
|
Packit |
5c3484 |
|
|
Packit |
5c3484 |
L(addmul_outer_2):
|
|
Packit |
5c3484 |
add $1, un
|
|
Packit |
5c3484 |
lea 0(%rip), outer_addr
|
|
Packit |
5c3484 |
|
|
Packit |
5c3484 |
mov un, n
|
|
Packit |
5c3484 |
mov -8(up,un,8), %rax
|
|
Packit |
5c3484 |
mul v0
|
|
Packit |
5c3484 |
xor R32(w0), R32(w0)
|
|
Packit |
5c3484 |
mov %rax, w2
|
|
Packit |
5c3484 |
xor R32(w1), R32(w1)
|
|
Packit |
5c3484 |
mov %rdx, w3
|
|
Packit |
5c3484 |
mov -8(up,un,8), %rax
|
|
Packit |
5c3484 |
jmp L(addmul_entry_2)
|
|
Packit |
5c3484 |
|
|
Packit |
5c3484 |
L(addmul_outer_3):
|
|
Packit |
5c3484 |
add $2, un
|
|
Packit |
5c3484 |
lea 0(%rip), outer_addr
|
|
Packit |
5c3484 |
|
|
Packit |
5c3484 |
mov un, n
|
|
Packit |
5c3484 |
mov -16(up,un,8), %rax
|
|
Packit |
5c3484 |
xor R32(w3), R32(w3)
|
|
Packit |
5c3484 |
mul v0
|
|
Packit |
5c3484 |
mov %rax, w1
|
|
Packit |
5c3484 |
mov -16(up,un,8), %rax
|
|
Packit |
5c3484 |
mov %rdx, w2
|
|
Packit |
5c3484 |
jmp L(addmul_entry_3)
|
|
Packit |
5c3484 |
|
|
Packit |
5c3484 |
C this loop is 19 c/loop = 2.375 c/l on K8, for all up/rp alignments
|
|
Packit |
5c3484 |
|
|
Packit |
5c3484 |
ALIGN(16)
|
|
Packit |
5c3484 |
L(addmul_top):
|
|
Packit |
5c3484 |
add w3, -32(rp,n,8)
|
|
Packit |
5c3484 |
adc %rax, w0
|
|
Packit |
5c3484 |
mov -24(up,n,8), %rax
|
|
Packit |
5c3484 |
adc %rdx, w1
|
|
Packit |
5c3484 |
xor R32(w2), R32(w2)
|
|
Packit |
5c3484 |
mul v0
|
|
Packit |
5c3484 |
add %rax, w0
|
|
Packit |
5c3484 |
mov -24(up,n,8), %rax
|
|
Packit |
5c3484 |
adc %rdx, w1
|
|
Packit |
5c3484 |
adc R32(w2), R32(w2) C adc $0, w2
|
|
Packit |
5c3484 |
L(addmul_entry_0):
|
|
Packit |
5c3484 |
mul v1
|
|
Packit |
5c3484 |
xor R32(w3), R32(w3)
|
|
Packit |
5c3484 |
add w0, -24(rp,n,8)
|
|
Packit |
5c3484 |
adc %rax, w1
|
|
Packit |
5c3484 |
mov -16(up,n,8), %rax
|
|
Packit |
5c3484 |
adc %rdx, w2
|
|
Packit |
5c3484 |
mul v0
|
|
Packit |
5c3484 |
add %rax, w1
|
|
Packit |
5c3484 |
mov -16(up,n,8), %rax
|
|
Packit |
5c3484 |
adc %rdx, w2
|
|
Packit |
5c3484 |
adc $0, R32(w3)
|
|
Packit |
5c3484 |
L(addmul_entry_3):
|
|
Packit |
5c3484 |
mul v1
|
|
Packit |
5c3484 |
add w1, -16(rp,n,8)
|
|
Packit |
5c3484 |
adc %rax, w2
|
|
Packit |
5c3484 |
mov -8(up,n,8), %rax
|
|
Packit |
5c3484 |
adc %rdx, w3
|
|
Packit |
5c3484 |
mul v0
|
|
Packit |
5c3484 |
xor R32(w0), R32(w0)
|
|
Packit |
5c3484 |
add %rax, w2
|
|
Packit |
5c3484 |
adc %rdx, w3
|
|
Packit |
5c3484 |
mov $0, R32(w1)
|
|
Packit |
5c3484 |
mov -8(up,n,8), %rax
|
|
Packit |
5c3484 |
adc R32(w1), R32(w0) C adc $0, w0
|
|
Packit |
5c3484 |
L(addmul_entry_2):
|
|
Packit |
5c3484 |
mul v1
|
|
Packit |
5c3484 |
add w2, -8(rp,n,8)
|
|
Packit |
5c3484 |
adc %rax, w3
|
|
Packit |
5c3484 |
adc %rdx, w0
|
|
Packit |
5c3484 |
mov (up,n,8), %rax
|
|
Packit |
5c3484 |
mul v0
|
|
Packit |
5c3484 |
add %rax, w3
|
|
Packit |
5c3484 |
mov (up,n,8), %rax
|
|
Packit |
5c3484 |
adc %rdx, w0
|
|
Packit |
5c3484 |
adc $0, R32(w1)
|
|
Packit |
5c3484 |
L(addmul_entry_1):
|
|
Packit |
5c3484 |
mul v1
|
|
Packit |
5c3484 |
add $4, n
|
|
Packit |
5c3484 |
js L(addmul_top)
|
|
Packit |
5c3484 |
|
|
Packit |
5c3484 |
add w3, -8(rp)
|
|
Packit |
5c3484 |
adc %rax, w0
|
|
Packit |
5c3484 |
mov w0, (rp)
|
|
Packit |
5c3484 |
adc %rdx, w1
|
|
Packit |
5c3484 |
mov w1, 8(rp)
|
|
Packit |
5c3484 |
|
|
Packit |
5c3484 |
add $-2, vn C vn -= 2
|
|
Packit |
5c3484 |
jz L(ret)
|
|
Packit |
5c3484 |
|
|
Packit |
5c3484 |
lea 16(rp), rp C rp += 2
|
|
Packit |
5c3484 |
lea 16(vp), vp C vp += 2
|
|
Packit |
5c3484 |
|
|
Packit |
5c3484 |
mov (vp), v0
|
|
Packit |
5c3484 |
mov 8(vp), v1
|
|
Packit |
5c3484 |
|
|
Packit |
5c3484 |
jmp *outer_addr
|
|
Packit |
5c3484 |
|
|
Packit |
5c3484 |
ALIGN(16)
|
|
Packit |
5c3484 |
L(ret): pop %r15
|
|
Packit |
5c3484 |
pop %r14
|
|
Packit |
5c3484 |
pop %r13
|
|
Packit |
5c3484 |
pop %r12
|
|
Packit |
5c3484 |
pop %rbp
|
|
Packit |
5c3484 |
pop %rbx
|
|
Packit |
5c3484 |
FUNC_EXIT()
|
|
Packit |
5c3484 |
ret
|
|
Packit |
5c3484 |
|
|
Packit |
5c3484 |
EPILOGUE()
|