dnl Intel Pentium MMX mpn_mul_1 -- mpn by limb multiplication.
dnl Copyright 2000-2002 Free Software Foundation, Inc.
dnl This file is part of the GNU MP Library.
dnl
dnl The GNU MP Library is free software; you can redistribute it and/or modify
dnl it under the terms of either:
dnl
dnl * the GNU Lesser General Public License as published by the Free
dnl Software Foundation; either version 3 of the License, or (at your
dnl option) any later version.
dnl
dnl or
dnl
dnl * the GNU General Public License as published by the Free Software
dnl Foundation; either version 2 of the License, or (at your option) any
dnl later version.
dnl
dnl or both in parallel, as here.
dnl
dnl The GNU MP Library is distributed in the hope that it will be useful, but
dnl WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
dnl or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
dnl for more details.
dnl
dnl You should have received copies of the GNU General Public License and the
dnl GNU Lesser General Public License along with the GNU MP Library. If not,
dnl see https://www.gnu.org/licenses/.
include(`../config.m4')
C cycles/limb
C P5: 12.0 for 32-bit multiplier
C 7.0 for 16-bit multiplier
C mp_limb_t mpn_mul_1 (mp_ptr dst, mp_srcptr src, mp_size_t size,
C mp_limb_t multiplier);
C
C When the multiplier is 16 bits some special case MMX code is used. Small
C multipliers might arise reasonably often from mpz_mul_ui etc. If the size
C is odd there's roughly a 5 cycle penalty, so times for say size==7 and
C size==8 end up being quite close. If src isn't aligned to an 8 byte
C boundary then one limb is processed separately with roughly a 5 cycle
C penalty, so in that case it's say size==8 and size==9 which are close.
C
C Alternatives:
C
C MMX is not believed to be of any use for 32-bit multipliers, since for
C instance the current method would just have to be more or less duplicated
C for the high and low halves of the multiplier, and would probably
C therefore run at about 14 cycles, which is slower than the plain integer
C at 12.
C
C Adding the high and low MMX products using integer code seems best. An
C attempt at using paddd and carry bit propagation with pcmpgtd didn't give
C any joy. Perhaps something could be done keeping the values signed and
C thereby avoiding adjustments to make pcmpgtd into an unsigned compare, or
C perhaps not.
C
C Future:
C
C An mpn_mul_1c entrypoint would need a double carry out of the low result
C limb in the 16-bit code, unless it could be assumed the carry fits in 16
C bits, possibly as carry<multiplier, this being true of a big calculation
C done piece by piece. But let's worry about that if/when mul_1c is
C actually used.
defframe(PARAM_MULTIPLIER,16)
defframe(PARAM_SIZE, 12)
defframe(PARAM_SRC, 8)
defframe(PARAM_DST, 4)
TEXT
ALIGN(8)
PROLOGUE(mpn_mul_1)
deflit(`FRAME',0)
movl PARAM_SIZE, %ecx
movl PARAM_SRC, %edx
cmpl $1, %ecx
jne L(two_or_more)
C one limb only
movl PARAM_MULTIPLIER, %eax
movl PARAM_DST, %ecx
mull (%edx)
movl %eax, (%ecx)
movl %edx, %eax
ret
L(two_or_more):
C eax size
C ebx
C ecx carry
C edx
C esi src
C edi
C ebp
pushl %esi FRAME_pushl()
pushl %edi FRAME_pushl()
movl %edx, %esi C src
movl PARAM_DST, %edi
movl PARAM_MULTIPLIER, %eax
pushl %ebx FRAME_pushl()
leal (%esi,%ecx,4), %esi C src end
leal (%edi,%ecx,4), %edi C dst end
negl %ecx C -size
pushl %ebp FRAME_pushl()
cmpl $65536, %eax
jb L(small)
L(big):
xorl %ebx, %ebx C carry limb
sarl %ecx C -size/2
jnc L(top) C with carry flag clear
C size was odd, process one limb separately
mull 4(%esi,%ecx,8) C m * src[0]
movl %eax, 4(%edi,%ecx,8)
incl %ecx
orl %edx, %ebx C carry limb, and clear carry flag
L(top):
C eax
C ebx carry
C ecx counter, negative
C edx
C esi src end
C edi dst end
C ebp (scratch carry)
adcl $0, %ebx
movl (%esi,%ecx,8), %eax
mull PARAM_MULTIPLIER
movl %edx, %ebp
addl %eax, %ebx
adcl $0, %ebp
movl 4(%esi,%ecx,8), %eax
mull PARAM_MULTIPLIER
movl %ebx, (%edi,%ecx,8)
addl %ebp, %eax
movl %eax, 4(%edi,%ecx,8)
incl %ecx
movl %edx, %ebx
jnz L(top)
adcl $0, %ebx
popl %ebp
movl %ebx, %eax
popl %ebx
popl %edi
popl %esi
ret
L(small):
C Special case for 16-bit multiplier.
C
C eax multiplier
C ebx
C ecx -size
C edx src
C esi src end
C edi dst end
C ebp multiplier
C size<3 not supported here. At size==3 we're already a couple of
C cycles faster, so there's no threshold as such, just use the MMX
C as soon as possible.
cmpl $-3, %ecx
ja L(big)
movd %eax, %mm7 C m
pxor %mm6, %mm6 C initial carry word
punpcklwd %mm7, %mm7 C m replicated 2 times
addl $2, %ecx C -size+2
punpckldq %mm7, %mm7 C m replicated 4 times
andl $4, %edx C test alignment, clear carry flag
movq %mm7, %mm0 C m
jz L(small_entry)
C Source is unaligned, process one limb separately.
C
C Plain integer code is used here, since it's smaller and is about
C the same 13 cycles as an mmx block would be.
C
C An "addl $1,%ecx" doesn't clear the carry flag when size==3, hence
C the use of separate incl and orl.
mull -8(%esi,%ecx,4) C m * src[0]
movl %eax, -8(%edi,%ecx,4) C dst[0]
incl %ecx C one limb processed
movd %edx, %mm6 C initial carry
orl %eax, %eax C clear carry flag
jmp L(small_entry)
C The scheduling here is quite tricky, since so many instructions have
C pairing restrictions. In particular the js won't pair with a movd, and
C can't be paired with an adc since it wants flags from the inc, so
C instructions are rotated to the top of the loop to find somewhere useful
C for it.
C
C Trouble has been taken to avoid overlapping successive loop iterations,
C since that would greatly increase the size of the startup and finishup
C code. Actually there's probably not much advantage to be had from
C overlapping anyway, since the difficulties are mostly with pairing, not
C with latencies as such.
C
C In the comments x represents the src data and m the multiplier (16
C bits, but replicated 4 times).
C
C The m signs calculated in %mm3 are a loop invariant and could be held in
C say %mm5, but that would save only one instruction and hence be no faster.
L(small_top):
C eax l.low, then l.high
C ebx (h.low)
C ecx counter, -size+2 to 0 or 1
C edx (h.high)
C esi &src[size]
C edi &dst[size]
C ebp
C
C %mm0 (high products)
C %mm1 (low products)
C %mm2 (adjust for m using x signs)
C %mm3 (adjust for x using m signs)
C %mm4
C %mm5
C %mm6 h.low, then carry
C %mm7 m replicated 4 times
movd %mm6, %ebx C h.low
psrlq $32, %mm1 C l.high
movd %mm0, %edx C h.high
movq %mm0, %mm6 C new c
adcl %eax, %ebx
incl %ecx
movd %mm1, %eax C l.high
movq %mm7, %mm0
adcl %eax, %edx
movl %ebx, -16(%edi,%ecx,4)
movl %edx, -12(%edi,%ecx,4)
psrlq $32, %mm6 C c
L(small_entry):
pmulhw -8(%esi,%ecx,4), %mm0 C h = (x*m).high
movq %mm7, %mm1
pmullw -8(%esi,%ecx,4), %mm1 C l = (x*m).low
movq %mm7, %mm3
movq -8(%esi,%ecx,4), %mm2 C x
psraw $15, %mm3 C m signs
pand -8(%esi,%ecx,4), %mm3 C x selected by m signs
psraw $15, %mm2 C x signs
paddw %mm3, %mm0 C add x to h if m neg
pand %mm7, %mm2 C m selected by x signs
paddw %mm2, %mm0 C add m to h if x neg
incl %ecx
movd %mm1, %eax C l.low
punpcklwd %mm0, %mm6 C c + h.low << 16
psrlq $16, %mm0 C h.high
js L(small_top)
movd %mm6, %ebx C h.low
psrlq $32, %mm1 C l.high
adcl %eax, %ebx
popl %ebp FRAME_popl()
movd %mm0, %edx C h.high
psrlq $32, %mm0 C l.high
movd %mm1, %eax C l.high
adcl %eax, %edx
movl %ebx, -12(%edi,%ecx,4)
movd %mm0, %eax C c
adcl $0, %eax
movl %edx, -8(%edi,%ecx,4)
orl %ecx, %ecx
jnz L(small_done) C final %ecx==1 means even, ==0 odd
C Size odd, one extra limb to process.
C Plain integer code is used here, since it's smaller and is about
C the same speed as another mmx block would be.
movl %eax, %ecx
movl PARAM_MULTIPLIER, %eax
mull -4(%esi)
addl %ecx, %eax
adcl $0, %edx
movl %eax, -4(%edi)
movl %edx, %eax
L(small_done):
popl %ebx
popl %edi
popl %esi
emms
ret
EPILOGUE()