dnl ARM v6 mpn_sqr_basecase.
dnl Contributed to the GNU project by Torbjörn Granlund.
dnl Copyright 2012, 2013, 2015 Free Software Foundation, Inc.
dnl This file is part of the GNU MP Library.
dnl
dnl The GNU MP Library is free software; you can redistribute it and/or modify
dnl it under the terms of either:
dnl
dnl * the GNU Lesser General Public License as published by the Free
dnl Software Foundation; either version 3 of the License, or (at your
dnl option) any later version.
dnl
dnl or
dnl
dnl * the GNU General Public License as published by the Free Software
dnl Foundation; either version 2 of the License, or (at your option) any
dnl later version.
dnl
dnl or both in parallel, as here.
dnl
dnl The GNU MP Library is distributed in the hope that it will be useful, but
dnl WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
dnl or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
dnl for more details.
dnl
dnl You should have received copies of the GNU General Public License and the
dnl GNU Lesser General Public License along with the GNU MP Library. If not,
dnl see https://www.gnu.org/licenses/.
include(`../config.m4')
C Code structure:
C
C
C m_2(0m4) m_2(2m4) m_2(1m4) m_2(3m4)
C | | | |
C | | | |
C | | | |
C \|/ \|/ \|/ \|/
C ____________ ____________
C / \ / \
C \|/ \ \|/ \
C am_2(3m4) am_2(1m4) am_2(0m4) am_2(2m4)
C \ /|\ \ /|\
C \____________/ \____________/
C \ /
C \ /
C \ /
C cor3 cor2
C \ /
C \ /
C sqr_diag_addlsh1
C TODO
C * Align more labels.
C * Further tweak counter and updates in outer loops. (This could save
C perhaps 5n cycles).
C * Avoid sub-with-lsl in outer loops. We could keep n up-shifted, then
C initialise loop counter i with a right shift.
C * Try to use fewer register. Perhaps coalesce r9 branch target and n_saved.
C (This could save 2-3 cycles for n > 4.)
C * Optimise sqr_diag_addlsh1 loop. The current code uses old-style carry
C propagation.
C * Stop loops earlier suppressing writes of upper-most rp[] values.
C * The addmul_2 loops here runs well on all cores, but mul_2 runs poorly
C particularly on Cortex-A8.
define(`rp', r0)
define(`up', r1)
define(`n', r2)
define(`v0', r3)
define(`v1', r6)
define(`i', r8)
define(`n_saved', r14)
define(`cya', r11)
define(`cyb', r12)
define(`u0', r7)
define(`u1', r9)
ASM_START()
PROLOGUE(mpn_sqr_basecase)
and r12, n, #3
cmp n, #4
addgt r12, r12, #4
add pc, pc, r12, lsl #2
nop
b L(4)
b L(1)
b L(2)
b L(3)
b L(0m4)
b L(1m4)
b L(2m4)
b L(3m4)
L(1m4): push {r4-r11, r14}
mov n_saved, n
sub i, n, #4
sub n, n, #2
add r10, pc, #L(am2_2m4)-.-8
ldm up, {v0,v1,u0}
sub up, up, #4
mov cyb, #0
mov r5, #0
umull r4, cya, v1, v0
str r4, [rp], #-12
mov r4, #0
b L(ko0)
L(3m4): push {r4-r11, r14}
mov n_saved, n
sub i, n, #4
sub n, n, #2
add r10, pc, #L(am2_0m4)-.-8
ldm up, {v0,v1,u0}
add up, up, #4
mov cyb, #0
mov r5, #0
umull r4, cya, v1, v0
str r4, [rp], #-4
mov r4, #0
b L(ko2)
L(2m4): push {r4-r11, r14}
mov n_saved, n
sub i, n, #4
sub n, n, #2
add r10, pc, #L(am2_3m4)-.-8
ldm up, {v0,v1,u1}
mov cyb, #0
mov r4, #0
umull r5, cya, v1, v0
str r5, [rp], #-8
mov r5, #0
b L(ko1)
L(0m4): push {r4-r11, r14}
mov n_saved, n
sub i, n, #4
sub n, n, #2
add r10, pc, #L(am2_1m4)-.-8
ldm up, {v0,v1,u1}
mov cyb, #0
mov r4, #0
add up, up, #8
umull r5, cya, v1, v0
str r5, [rp, #0]
mov r5, #0
L(top): ldr u0, [up, #4]
umaal r4, cya, u1, v0
str r4, [rp, #4]
mov r4, #0
umaal r5, cyb, u1, v1
L(ko2): ldr u1, [up, #8]
umaal r5, cya, u0, v0
str r5, [rp, #8]
mov r5, #0
umaal r4, cyb, u0, v1
L(ko1): ldr u0, [up, #12]
umaal r4, cya, u1, v0
str r4, [rp, #12]
mov r4, #0
umaal r5, cyb, u1, v1
L(ko0): ldr u1, [up, #16]!
umaal r5, cya, u0, v0
str r5, [rp, #16]!
mov r5, #0
umaal r4, cyb, u0, v1
subs i, i, #4
bhi L(top)
umaal r4, cya, u1, v0
ldr u0, [up, #4]
umaal r5, cyb, u1, v1
str r4, [rp, #4]
umaal r5, cya, u0, v0
umaal cya, cyb, u0, v1
str r5, [rp, #8]
str cya, [rp, #12]
str cyb, [rp, #16]
add up, up, #4
sub n, n, #1
add rp, rp, #8
bx r10
L(evnloop):
subs i, n, #6
sub n, n, #2
blt L(cor2)
ldm up, {v0,v1,u1}
add up, up, #8
mov cya, #0
mov cyb, #0
ldr r4, [rp, #-4]
umaal r4, cya, v1, v0
str r4, [rp, #-4]
ldr r4, [rp, #0]
ALIGN(16)
L(ua2): ldr r5, [rp, #4]
umaal r4, cya, u1, v0
ldr u0, [up, #4]
umaal r5, cyb, u1, v1
str r4, [rp, #0]
ldr r4, [rp, #8]
umaal r5, cya, u0, v0
ldr u1, [up, #8]
umaal r4, cyb, u0, v1
str r5, [rp, #4]
ldr r5, [rp, #12]
umaal r4, cya, u1, v0
ldr u0, [up, #12]
umaal r5, cyb, u1, v1
str r4, [rp, #8]
ldr r4, [rp, #16]!
umaal r5, cya, u0, v0
ldr u1, [up, #16]!
umaal r4, cyb, u0, v1
str r5, [rp, #-4]
subs i, i, #4
bhs L(ua2)
umaal r4, cya, u1, v0
umaal cya, cyb, u1, v1
str r4, [rp, #0]
str cya, [rp, #4]
str cyb, [rp, #8]
L(am2_0m4):
sub rp, rp, n, lsl #2
sub up, up, n, lsl #2
add rp, rp, #8
sub i, n, #4
sub n, n, #2
ldm up, {v0,v1,u1}
mov cya, #0
mov cyb, #0
ldr r4, [rp, #4]
umaal r4, cya, v1, v0
str r4, [rp, #4]
ldr r4, [rp, #8]
b L(lo0)
ALIGN(16)
L(ua0): ldr r5, [rp, #4]
umaal r4, cya, u1, v0
ldr u0, [up, #4]
umaal r5, cyb, u1, v1
str r4, [rp, #0]
ldr r4, [rp, #8]
umaal r5, cya, u0, v0
ldr u1, [up, #8]
umaal r4, cyb, u0, v1
str r5, [rp, #4]
L(lo0): ldr r5, [rp, #12]
umaal r4, cya, u1, v0
ldr u0, [up, #12]
umaal r5, cyb, u1, v1
str r4, [rp, #8]
ldr r4, [rp, #16]!
umaal r5, cya, u0, v0
ldr u1, [up, #16]!
umaal r4, cyb, u0, v1
str r5, [rp, #-4]
subs i, i, #4
bhs L(ua0)
umaal r4, cya, u1, v0
umaal cya, cyb, u1, v1
str r4, [rp, #0]
str cya, [rp, #4]
str cyb, [rp, #8]
L(am2_2m4):
sub rp, rp, n, lsl #2
sub up, up, n, lsl #2
add rp, rp, #16
b L(evnloop)
L(oddloop):
sub i, n, #5
sub n, n, #2
ldm up, {v0,v1,u0}
mov cya, #0
mov cyb, #0
ldr r5, [rp, #0]
umaal r5, cya, v1, v0
str r5, [rp, #0]
ldr r5, [rp, #4]
add up, up, #4
b L(lo1)
ALIGN(16)
L(ua1): ldr r5, [rp, #4]
umaal r4, cya, u1, v0
ldr u0, [up, #4]
umaal r5, cyb, u1, v1
str r4, [rp, #0]
L(lo1): ldr r4, [rp, #8]
umaal r5, cya, u0, v0
ldr u1, [up, #8]
umaal r4, cyb, u0, v1
str r5, [rp, #4]
ldr r5, [rp, #12]
umaal r4, cya, u1, v0
ldr u0, [up, #12]
umaal r5, cyb, u1, v1
str r4, [rp, #8]
ldr r4, [rp, #16]!
umaal r5, cya, u0, v0
ldr u1, [up, #16]!
umaal r4, cyb, u0, v1
str r5, [rp, #-4]
subs i, i, #4
bhs L(ua1)
umaal r4, cya, u1, v0
umaal cya, cyb, u1, v1
str r4, [rp, #0]
str cya, [rp, #4]
str cyb, [rp, #8]
L(am2_3m4):
sub rp, rp, n, lsl #2
sub up, up, n, lsl #2
add rp, rp, #4
subs i, n, #3
beq L(cor3)
sub n, n, #2
ldm up, {v0,v1,u0}
mov cya, #0
mov cyb, #0
ldr r5, [rp, #8]
sub up, up, #4
umaal r5, cya, v1, v0
str r5, [rp, #8]
ldr r5, [rp, #12]
b L(lo3)
ALIGN(16)
L(ua3): ldr r5, [rp, #4]
umaal r4, cya, u1, v0
ldr u0, [up, #4]
umaal r5, cyb, u1, v1
str r4, [rp, #0]
ldr r4, [rp, #8]
umaal r5, cya, u0, v0
ldr u1, [up, #8]
umaal r4, cyb, u0, v1
str r5, [rp, #4]
ldr r5, [rp, #12]
umaal r4, cya, u1, v0
ldr u0, [up, #12]
umaal r5, cyb, u1, v1
str r4, [rp, #8]
L(lo3): ldr r4, [rp, #16]!
umaal r5, cya, u0, v0
ldr u1, [up, #16]!
umaal r4, cyb, u0, v1
str r5, [rp, #-4]
subs i, i, #4
bhs L(ua3)
umaal r4, cya, u1, v0
umaal cya, cyb, u1, v1
str r4, [rp, #0]
str cya, [rp, #4]
str cyb, [rp, #8]
L(am2_1m4):
sub rp, rp, n, lsl #2
sub up, up, n, lsl #2
add rp, rp, #12
b L(oddloop)
L(cor3):ldm up, {v0,v1,u0}
ldr r5, [rp, #8]
mov cya, #0
mov cyb, #0
umaal r5, cya, v1, v0
str r5, [rp, #8]
ldr r5, [rp, #12]
ldr r4, [rp, #16]
umaal r5, cya, u0, v0
ldr u1, [up, #12]
umaal r4, cyb, u0, v1
str r5, [rp, #12]
umaal r4, cya, u1, v0
umaal cya, cyb, u1, v1
str r4, [rp, #16]
str cya, [rp, #20]
str cyb, [rp, #24]
add up, up, #16
mov cya, cyb
adds rp, rp, #36 C clear cy
mov cyb, #0
umaal cya, cyb, u1, u0
b L(sqr_diag_addlsh1)
L(cor2):
ldm up!, {v0,v1,u0}
mov r4, cya
mov r5, cyb
mov cya, #0
umaal r4, cya, v1, v0
mov cyb, #0
umaal r5, cya, u0, v0
strd r4, r5, [rp, #-4]
umaal cya, cyb, u0, v1
add rp, rp, #16
C b L(sqr_diag_addlsh1)
define(`w0', r6)
define(`w1', r7)
define(`w2', r8)
define(`rbx', r9)
L(sqr_diag_addlsh1):
str cya, [rp, #-12]
str cyb, [rp, #-8]
sub n, n_saved, #1
sub up, up, n_saved, lsl #2
sub rp, rp, n_saved, lsl #3
ldr r3, [up], #4
umull w1, r5, r3, r3
mov w2, #0
mov r10, #0
C cmn r0, #0 C clear cy (already clear)
b L(lm)
L(tsd): adds w0, w0, rbx
adcs w1, w1, r4
str w0, [rp, #0]
L(lm): ldr w0, [rp, #4]
str w1, [rp, #4]
ldr w1, [rp, #8]!
add rbx, r5, w2
adcs w0, w0, w0
ldr r3, [up], #4
adcs w1, w1, w1
adc w2, r10, r10
umull r4, r5, r3, r3
subs n, n, #1
bne L(tsd)
adds w0, w0, rbx
adcs w1, w1, r4
adc w2, r5, w2
stm rp, {w0,w1,w2}
pop {r4-r11, pc}
C Straight line code for n <= 4
L(1): ldr r3, [up, #0]
umull r1, r2, r3, r3
stm rp, {r1,r2}
bx r14
L(2): push {r4-r5}
ldm up, {r5,r12}
umull r1, r2, r5, r5
umull r3, r4, r12, r12
umull r5, r12, r5, r12
adds r5, r5, r5
adcs r12, r12, r12
adc r4, r4, #0
adds r2, r2, r5
adcs r3, r3, r12
adc r4, r4, #0
stm rp, {r1,r2,r3,r4}
pop {r4-r5}
bx r14
L(3): push {r4-r11}
ldm up, {r7,r8,r9}
umull r1, r2, r7, r7
umull r3, r4, r8, r8
umull r5, r6, r9, r9
umull r10, r11, r7, r8
mov r12, #0
umlal r11, r12, r7, r9
mov r7, #0
umlal r12, r7, r8, r9
adds r10, r10, r10
adcs r11, r11, r11
adcs r12, r12, r12
adcs r7, r7, r7
adc r6, r6, #0
adds r2, r2, r10
adcs r3, r3, r11
adcs r4, r4, r12
adcs r5, r5, r7
adc r6, r6, #0
stm rp, {r1,r2,r3,r4,r5,r6}
pop {r4-r11}
bx r14
L(4): push {r4-r11, r14}
ldm up, {r9,r10,r11,r12}
umull r1, r2, r9, r9
umull r3, r4, r10, r10
umull r5, r6, r11, r11
umull r7, r8, r12, r12
stm rp, {r1,r2,r3,r4,r5,r6,r7}
umull r1, r2, r9, r10
mov r3, #0
umlal r2, r3, r9, r11
mov r4, #0
umlal r3, r4, r9, r12
mov r5, #0
umlal r3, r5, r10, r11
umaal r4, r5, r10, r12
mov r6, #0
umlal r5, r6, r11, r12
adds r1, r1, r1
adcs r2, r2, r2
adcs r3, r3, r3
adcs r4, r4, r4
adcs r5, r5, r5
adcs r6, r6, r6
add rp, rp, #4
adc r7, r8, #0
ldm rp, {r8,r9,r10,r11,r12,r14}
adds r1, r1, r8
adcs r2, r2, r9
adcs r3, r3, r10
adcs r4, r4, r11
adcs r5, r5, r12
adcs r6, r6, r14
adc r7, r7, #0
stm rp, {r1,r2,r3,r4,r5,r6,r7}
pop {r4-r11, pc}
EPILOGUE()