dnl  IA-64 mpn_mul_1, mpn_mul_1c -- Multiply a limb vector with a limb and
dnl  store the result in a second limb vector.

dnl  Contributed to the GNU project by Torbjorn Granlund.

dnl  Copyright 2000-2004, 2006, 2007 Free Software Foundation, Inc.

dnl  This file is part of the GNU MP Library.
dnl
dnl  The GNU MP Library is free software; you can redistribute it and/or modify
dnl  it under the terms of either:
dnl
dnl    * the GNU Lesser General Public License as published by the Free
dnl      Software Foundation; either version 3 of the License, or (at your
dnl      option) any later version.
dnl
dnl  or
dnl
dnl    * the GNU General Public License as published by the Free Software
dnl      Foundation; either version 2 of the License, or (at your option) any
dnl      later version.
dnl
dnl  or both in parallel, as here.
dnl
dnl  The GNU MP Library is distributed in the hope that it will be useful, but
dnl  WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
dnl  or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
dnl  for more details.
dnl
dnl  You should have received copies of the GNU General Public License and the
dnl  GNU Lesser General Public License along with the GNU MP Library.  If not,
dnl  see https://www.gnu.org/licenses/.

include(`../config.m4')

C         cycles/limb
C Itanium:    4.0
C Itanium 2:  2.0

C TODO
C  * Further optimize feed-in and wind-down code, both for speed and code size.
C  * Handle low limb input and results specially, using a common stf8 in the
C    epilogue.
C  * Use 1 c/l carry propagation scheme in wind-down code.
C  * Use extra pointer register for `up' to speed up feed-in loads.
C  * Work out final differences with addmul_1.asm.

C INPUT PARAMETERS
define(`rp', `r32')
define(`up', `r33')
define(`n', `r34')
define(`vl', `r35')
define(`cy', `r36')	C for mpn_mul_1c

ASM_START()
PROLOGUE(mpn_mul_1)
	.prologue
	.save	ar.lc, r2
	.body

ifdef(`HAVE_ABI_32',
`	addp4		rp = 0, rp		C M I
	addp4		up = 0, up		C M I
	zxt4		n = n			C I
	;;
')
{.mfi
	adds		r15 = -1, n		C M I
	mov		f9 = f0			C F
	mov.i		r2 = ar.lc		C I0
}
{.mmi
	ldf8		f7 = [up], 8		C M
	nop.m		0			C M
	and		r14 = 3, n		C M I
	;;
}
.Lcommon:
{.mii
	setf.sig	f6 = vl			C M2 M3
	shr.u		r31 = r15, 2		C I0
	cmp.eq		p10, p0 = 0, r14	C M I
}
{.mii
	cmp.eq		p11, p0 = 2, r14	C M I
	cmp.eq		p12, p0 = 3, r14	C M I
	nop.i		0			C I
	;;
}
{.mii
	cmp.ne		p6, p7 = r0, r0		C M I
	mov.i		ar.lc = r31		C I0
	cmp.ne		p8, p9 = r0, r0		C M I
}
{.bbb
  (p10)	br.dptk		.Lb00			C B
  (p11)	br.dptk		.Lb10			C B
  (p12)	br.dptk		.Lb11			C B
	;;
}

.Lb01:	mov		r20 = 0
	br.cloop.dptk	.grt1			C B

	xma.l		f39 = f7, f6, f9	C F
	xma.hu		f43 = f7, f6, f9	C F
	;;
	getf.sig	r8 = f43		C M2
	stf8		[rp] = f39		C M2 M3
	mov.i		ar.lc = r2		C I0
	br.ret.sptk.many b0			C B

.grt1:
	ldf8		f32 = [up], 8
	;;
	ldf8		f33 = [up], 8
	;;
	ldf8		f34 = [up], 8
	xma.l		f39 = f7, f6, f9
	xma.hu		f43 = f7, f6, f9
	;;
	ldf8		f35 = [up], 8
	br.cloop.dptk	.grt5

	xma.l		f36 = f32, f6, f0
	xma.hu		f40 = f32, f6, f0
	;;
	stf8		[rp] = f39, 8
	xma.l		f37 = f33, f6, f0
	xma.hu		f41 = f33, f6, f0
	;;
	getf.sig	r21 = f43
	getf.sig	r18 = f36
	xma.l		f38 = f34, f6, f0
	xma.hu		f42 = f34, f6, f0
	;;
	getf.sig	r22 = f40
	getf.sig	r19 = f37
	xma.l		f39 = f35, f6, f0
	xma.hu		f43 = f35, f6, f0
	;;
	getf.sig	r23 = f41
	getf.sig	r16 = f38
	br		.Lcj5

.grt5:
	xma.l		f36 = f32, f6, f0
	xma.hu		f40 = f32, f6, f0
	;;
	getf.sig	r17 = f39
	ldf8		f32 = [up], 8
	xma.l		f37 = f33, f6, f0
	xma.hu		f41 = f33, f6, f0
	;;
	getf.sig	r21 = f43
	ldf8		f33 = [up], 8
	xma.l		f38 = f34, f6, f0
	;;
	getf.sig	r18 = f36
	xma.hu		f42 = f34, f6, f0
	;;
	getf.sig	r22 = f40
	ldf8		f34 = [up], 8
	xma.l		f39 = f35, f6, f0
	;;
	getf.sig	r19 = f37
	xma.hu		f43 = f35, f6, f0
	br		.LL01


.Lb10:	ldf8		f35 = [up], 8
	mov		r23 = 0
	br.cloop.dptk	.grt2

	xma.l		f38 = f7, f6, f9
	xma.hu		f42 = f7, f6, f9
	;;
	stf8		[rp] = f38, 8
	xma.l		f39 = f35, f6, f42
	xma.hu		f43 = f35, f6, f42
	;;
	getf.sig	r8 = f43
	stf8		[rp] = f39
	mov.i		ar.lc = r2
	br.ret.sptk.many b0


.grt2:
	ldf8		f32 = [up], 8
	;;
	ldf8		f33 = [up], 8
	xma.l		f38 = f7, f6, f9
	xma.hu		f42 = f7, f6, f9
	;;
	ldf8		f34 = [up], 8
	xma.l		f39 = f35, f6, f0
	xma.hu		f43 = f35, f6, f0
	;;
	ldf8		f35 = [up], 8
	br.cloop.dptk	.grt6

	stf8		[rp] = f38, 8
	xma.l		f36 = f32, f6, f0
	xma.hu		f40 = f32, f6, f0
	;;
	getf.sig	r20 = f42
	getf.sig	r17 = f39
	xma.l		f37 = f33, f6, f0
	xma.hu		f41 = f33, f6, f0
	;;
	getf.sig	r21 = f43
	getf.sig	r18 = f36
	xma.l		f38 = f34, f6, f0
	xma.hu		f42 = f34, f6, f0
	;;
	getf.sig	r22 = f40
	getf.sig	r19 = f37
	xma.l		f39 = f35, f6, f0
	xma.hu		f43 = f35, f6, f0
	br		.Lcj6

.grt6:
	getf.sig	r16 = f38
	xma.l		f36 = f32, f6, f0
	xma.hu		f40 = f32, f6, f0
	;;
	getf.sig	r20 = f42
	ldf8		f32 = [up], 8
	xma.l		f37 = f33, f6, f0
	;;
	getf.sig	r17 = f39
	xma.hu		f41 = f33, f6, f0
	;;
	getf.sig	r21 = f43
	ldf8		f33 = [up], 8
	xma.l		f38 = f34, f6, f0
	;;
	getf.sig	r18 = f36
	xma.hu		f42 = f34, f6, f0
	br		.LL10


.Lb11:	ldf8		f34 = [up], 8
	mov		r22 = 0
	;;
	ldf8		f35 = [up], 8
	br.cloop.dptk	.grt3
	;;

	xma.l		f37 = f7, f6, f9
	xma.hu		f41 = f7, f6, f9
	xma.l		f38 = f34, f6, f0
	xma.hu		f42 = f34, f6, f0
	xma.l		f39 = f35, f6, f0
	xma.hu		f43 = f35, f6, f0
	;;
	getf.sig	r23 = f41
	stf8		[rp] = f37, 8
	getf.sig	r16 = f38
	getf.sig	r20 = f42
	getf.sig	r17 = f39
	getf.sig	r8 = f43
	br		.Lcj3

.grt3:
	ldf8		f32 = [up], 8
	xma.l		f37 = f7, f6, f9
	xma.hu		f41 = f7, f6, f9
	;;
	ldf8		f33 = [up], 8
	xma.l		f38 = f34, f6, f0
	xma.hu		f42 = f34, f6, f0
	;;
	getf.sig	r19 = f37
	ldf8		f34 = [up], 8
	xma.l		f39 = f35, f6, f0
	xma.hu		f43 = f35, f6, f0
	;;
	getf.sig	r23 = f41
	ldf8		f35 = [up], 8
	br.cloop.dptk	.grt7

	getf.sig	r16 = f38
	xma.l		f36 = f32, f6, f0
	getf.sig	r20 = f42
	xma.hu		f40 = f32, f6, f0
	;;
	getf.sig	r17 = f39
	xma.l		f37 = f33, f6, f0
	getf.sig	r21 = f43
	xma.hu		f41 = f33, f6, f0
	;;
	getf.sig	r18 = f36
	st8		[rp] = r19, 8
	xma.l		f38 = f34, f6, f0
	xma.hu		f42 = f34, f6, f0
	br		.Lcj7

.grt7:
	getf.sig	r16 = f38
	xma.l		f36 = f32, f6, f0
	xma.hu		f40 = f32, f6, f0
	;;
	getf.sig	r20 = f42
	ldf8		f32 = [up], 8
	xma.l		f37 = f33, f6, f0
	;;
	getf.sig	r17 = f39
	xma.hu		f41 = f33, f6, f0
	br		.LL11


.Lb00:	ldf8		f33 = [up], 8
	mov		r21 = 0
	;;
	ldf8		f34 = [up], 8
	;;
	ldf8		f35 = [up], 8
	xma.l		f36 = f7, f6, f9
	xma.hu		f40 = f7, f6, f9
	br.cloop.dptk	.grt4

	xma.l		f37 = f33, f6, f0
	xma.hu		f41 = f33, f6, f0
	xma.l		f38 = f34, f6, f0
	xma.hu		f42 = f34, f6, f0
	;;
	getf.sig	r22 = f40
	stf8		[rp] = f36, 8
	xma.l		f39 = f35, f6, f0
	getf.sig	r19 = f37
	xma.hu		f43 = f35, f6, f0
	;;
	getf.sig	r23 = f41
	getf.sig	r16 = f38
	getf.sig	r20 = f42
	getf.sig	r17 = f39
	br		.Lcj4

.grt4:
	ldf8		f32 = [up], 8
	xma.l		f37 = f33, f6, f0
	xma.hu		f41 = f33, f6, f0
	;;
	getf.sig	r18 = f36
	ldf8		f33 = [up], 8
	xma.l		f38 = f34, f6, f0
	xma.hu		f42 = f34, f6, f0
	;;
	getf.sig	r22 = f40
	ldf8		f34 = [up], 8
	xma.l		f39 = f35, f6, f0
	;;
	getf.sig	r19 = f37
	getf.sig	r23 = f41
	xma.hu		f43 = f35, f6, f0
	ldf8		f35 = [up], 8
	br.cloop.dptk	.grt8

	getf.sig	r16 = f38
	xma.l		f36 = f32, f6, f0
	getf.sig	r20 = f42
	xma.hu		f40 = f32, f6, f0
	;;
	getf.sig	r17 = f39
	st8		[rp] = r18, 8
	xma.l		f37 = f33, f6, f0
	xma.hu		f41 = f33, f6, f0
	br		.Lcj8

.grt8:
	getf.sig	r16 = f38
	xma.l		f36 = f32, f6, f0
	xma.hu		f40 = f32, f6, f0
	br		.LL00


C *** MAIN LOOP START ***
	ALIGN(32)
.Loop:
	.pred.rel "mutex",p6,p7
	getf.sig	r16 = f38
	xma.l		f36 = f32, f6, f0
   (p6)	cmp.leu		p8, p9 = r24, r17
	st8		[rp] = r24, 8
	xma.hu		f40 = f32, f6, f0
   (p7)	cmp.ltu		p8, p9 = r24, r17
	;;
.LL00:
	.pred.rel "mutex",p8,p9
	getf.sig	r20 = f42
   (p8)	add		r24 = r18, r21, 1
	nop.b		0
	ldf8		f32 = [up], 8
   (p9)	add		r24 = r18, r21
	nop.b		0
	;;
	.pred.rel "mutex",p8,p9
	getf.sig	r17 = f39
	xma.l		f37 = f33, f6, f0
   (p8)	cmp.leu		p6, p7 = r24, r18
	st8		[rp] = r24, 8
	xma.hu		f41 = f33, f6, f0
   (p9)	cmp.ltu		p6, p7 = r24, r18
	;;
.LL11:
	.pred.rel "mutex",p6,p7
	getf.sig	r21 = f43
   (p6)	add		r24 = r19, r22, 1
	nop.b		0
	ldf8		f33 = [up], 8
   (p7)	add		r24 = r19, r22
	nop.b		0
	;;
	.pred.rel "mutex",p6,p7
	getf.sig	r18 = f36
	xma.l		f38 = f34, f6, f0
   (p6)	cmp.leu		p8, p9 = r24, r19
	st8		[rp] = r24, 8
	xma.hu		f42 = f34, f6, f0
   (p7)	cmp.ltu		p8, p9 = r24, r19
	;;
.LL10:
	.pred.rel "mutex",p8,p9
	getf.sig	r22 = f40
   (p8)	add		r24 = r16, r23, 1
	nop.b		0
	ldf8		f34 = [up], 8
   (p9)	add		r24 = r16, r23
	nop.b		0
	;;
	.pred.rel "mutex",p8,p9
	getf.sig	r19 = f37
	xma.l		f39 = f35, f6, f0
   (p8)	cmp.leu		p6, p7 = r24, r16
	st8		[rp] = r24, 8
	xma.hu		f43 = f35, f6, f0
   (p9)	cmp.ltu		p6, p7 = r24, r16
	;;
.LL01:
	.pred.rel "mutex",p6,p7
	getf.sig	r23 = f41
   (p6)	add		r24 = r17, r20, 1
	nop.b		0
	ldf8		f35 = [up], 8
   (p7)	add		r24 = r17, r20
	br.cloop.dptk	.Loop
C *** MAIN LOOP END ***
	;;

.Lcj9:
	.pred.rel "mutex",p6,p7
	getf.sig	r16 = f38
	xma.l		f36 = f32, f6, f0
   (p6)	cmp.leu		p8, p9 = r24, r17
	st8		[rp] = r24, 8
	xma.hu		f40 = f32, f6, f0
   (p7)	cmp.ltu		p8, p9 = r24, r17
	;;
	.pred.rel "mutex",p8,p9
	getf.sig	r20 = f42
   (p8)	add		r24 = r18, r21, 1
   (p9)	add		r24 = r18, r21
	;;
	.pred.rel "mutex",p8,p9
	getf.sig	r17 = f39
	xma.l		f37 = f33, f6, f0
   (p8)	cmp.leu		p6, p7 = r24, r18
	st8		[rp] = r24, 8
	xma.hu		f41 = f33, f6, f0
   (p9)	cmp.ltu		p6, p7 = r24, r18
	;;
.Lcj8:
	.pred.rel "mutex",p6,p7
	getf.sig	r21 = f43
   (p6)	add		r24 = r19, r22, 1
   (p7)	add		r24 = r19, r22
	;;
	.pred.rel "mutex",p6,p7
	getf.sig	r18 = f36
	xma.l		f38 = f34, f6, f0
   (p6)	cmp.leu		p8, p9 = r24, r19
	st8		[rp] = r24, 8
	xma.hu		f42 = f34, f6, f0
   (p7)	cmp.ltu		p8, p9 = r24, r19
	;;
.Lcj7:
	.pred.rel "mutex",p8,p9
	getf.sig	r22 = f40
   (p8)	add		r24 = r16, r23, 1
   (p9)	add		r24 = r16, r23
	;;
	.pred.rel "mutex",p8,p9
	getf.sig	r19 = f37
	xma.l		f39 = f35, f6, f0
   (p8)	cmp.leu		p6, p7 = r24, r16
	st8		[rp] = r24, 8
	xma.hu		f43 = f35, f6, f0
   (p9)	cmp.ltu		p6, p7 = r24, r16
	;;
.Lcj6:
	.pred.rel "mutex",p6,p7
	getf.sig	r23 = f41
   (p6)	add		r24 = r17, r20, 1
   (p7)	add		r24 = r17, r20
	;;
	.pred.rel "mutex",p6,p7
   (p6)	cmp.leu		p8, p9 = r24, r17
   (p7)	cmp.ltu		p8, p9 = r24, r17
	getf.sig	r16 = f38
	st8		[rp] = r24, 8
	;;
.Lcj5:
	.pred.rel "mutex",p8,p9
	getf.sig	r20 = f42
   (p8)	add		r24 = r18, r21, 1
   (p9)	add		r24 = r18, r21
	;;
	.pred.rel "mutex",p8,p9
   (p8)	cmp.leu		p6, p7 = r24, r18
   (p9)	cmp.ltu		p6, p7 = r24, r18
	getf.sig	r17 = f39
	st8		[rp] = r24, 8
	;;
.Lcj4:
	.pred.rel "mutex",p6,p7
	getf.sig	r8 = f43
   (p6)	add		r24 = r19, r22, 1
   (p7)	add		r24 = r19, r22
	;;
	.pred.rel "mutex",p6,p7
	st8		[rp] = r24, 8
   (p6)	cmp.leu		p8, p9 = r24, r19
   (p7)	cmp.ltu		p8, p9 = r24, r19
	;;
.Lcj3:
	.pred.rel "mutex",p8,p9
   (p8)	add		r24 = r16, r23, 1
   (p9)	add		r24 = r16, r23
	;;
	.pred.rel "mutex",p8,p9
	st8		[rp] = r24, 8
   (p8)	cmp.leu		p6, p7 = r24, r16
   (p9)	cmp.ltu		p6, p7 = r24, r16
	;;
.Lcj2:
	.pred.rel "mutex",p6,p7
   (p6)	add		r24 = r17, r20, 1
   (p7)	add		r24 = r17, r20
	;;
	.pred.rel "mutex",p6,p7
	st8		[rp] = r24, 8
   (p6)	cmp.leu		p8, p9 = r24, r17
   (p7)	cmp.ltu		p8, p9 = r24, r17
	;;
   (p8)	add		r8 = 1, r8
	mov.i		ar.lc = r2
	br.ret.sptk.many b0
EPILOGUE()

PROLOGUE(mpn_mul_1c)
	.prologue
	.save	ar.lc, r2
	.body

ifdef(`HAVE_ABI_32',
`	addp4		rp = 0, rp		C M I
	addp4		up = 0, up		C M I
	zxt4		n = n			C I
	;;
')
{.mmi
	adds		r15 = -1, n		C M I
	setf.sig	f9 = cy			C M2 M3
	mov.i		r2 = ar.lc		C I0
}
{.mmb
	ldf8		f7 = [up], 8		C M
	and		r14 = 3, n		C M I
	br.sptk		.Lcommon
	;;
}
EPILOGUE()
ASM_END()