/* longlong.h -- definitions for mixed size 32/64 bit arithmetic.

Copyright 1991-1994, 1996, 1997, 1999-2005, 2007-2009, 2011-2016 Free Software

Foundation, Inc.

This file is part of the GNU MP Library.

The GNU MP Library is free software; you can redistribute it and/or modify

it under the terms of either:

  * the GNU Lesser General Public License as published by the Free

    Software Foundation; either version 3 of the License, or (at your

    option) any later version.

or

  * the GNU General Public License as published by the Free Software

    Foundation; either version 2 of the License, or (at your option) any

    later version.

or both in parallel, as here.

The GNU MP Library is distributed in the hope that it will be useful, but

WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY

or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

for more details.

You should have received copies of the GNU General Public License and the

GNU Lesser General Public License along with the GNU MP Library.  If not,

see https://www.gnu.org/licenses/.  */

/* You have to define the following before including this file:

   UWtype -- An unsigned type, default type for operations (typically a "word")

   UHWtype -- An unsigned type, at least half the size of UWtype

   UDWtype -- An unsigned type, at least twice as large a UWtype

   W_TYPE_SIZE -- size in bits of UWtype

   SItype, USItype -- Signed and unsigned 32 bit types

   DItype, UDItype -- Signed and unsigned 64 bit types

   On a 32 bit machine UWtype should typically be USItype;

   on a 64 bit machine, UWtype should typically be UDItype.

   Optionally, define:

   LONGLONG_STANDALONE -- Avoid code that needs machine-dependent support files

   NO_ASM -- Disable inline asm

   CAUTION!  Using this version of longlong.h outside of GMP is not safe.  You

   need to include gmp.h and gmp-impl.h, or certain things might not work as

   expected.

*/

#define __BITS4 (W_TYPE_SIZE / 4)

#define __ll_B ((UWtype) 1 << (W_TYPE_SIZE / 2))

#define __ll_lowpart(t) ((UWtype) (t) & (__ll_B - 1))

#define __ll_highpart(t) ((UWtype) (t) >> (W_TYPE_SIZE / 2))

/* This is used to make sure no undesirable sharing between different libraries

   that use this file takes place.  */

#ifndef __MPN

#define __MPN(x) __##x

#endif

/* Define auxiliary asm macros.

   1) umul_ppmm(high_prod, low_prod, multiplier, multiplicand) multiplies two

   UWtype integers MULTIPLIER and MULTIPLICAND, and generates a two UWtype

   word product in HIGH_PROD and LOW_PROD.

   2) __umulsidi3(a,b) multiplies two UWtype integers A and B, and returns a

   UDWtype product.  This is just a variant of umul_ppmm.

   3) udiv_qrnnd(quotient, remainder, high_numerator, low_numerator,

   denominator) divides a UDWtype, composed by the UWtype integers

   HIGH_NUMERATOR and LOW_NUMERATOR, by DENOMINATOR and places the quotient

   in QUOTIENT and the remainder in REMAINDER.  HIGH_NUMERATOR must be less

   than DENOMINATOR for correct operation.  If, in addition, the most

   significant bit of DENOMINATOR must be 1, then the pre-processor symbol

   UDIV_NEEDS_NORMALIZATION is defined to 1.

   4) sdiv_qrnnd(quotient, remainder, high_numerator, low_numerator,

   denominator).  Like udiv_qrnnd but the numbers are signed.  The quotient

   is rounded towards 0.

   5) count_leading_zeros(count, x) counts the number of zero-bits from the

   msb to the first non-zero bit in the UWtype X.  This is the number of

   steps X needs to be shifted left to set the msb.  Undefined for X == 0,

   unless the symbol COUNT_LEADING_ZEROS_0 is defined to some value.

   6) count_trailing_zeros(count, x) like count_leading_zeros, but counts

   from the least significant end.

   7) add_ssaaaa(high_sum, low_sum, high_addend_1, low_addend_1,

   high_addend_2, low_addend_2) adds two UWtype integers, composed by

   HIGH_ADDEND_1 and LOW_ADDEND_1, and HIGH_ADDEND_2 and LOW_ADDEND_2

   respectively.  The result is placed in HIGH_SUM and LOW_SUM.  Overflow

   (i.e. carry out) is not stored anywhere, and is lost.

   8) sub_ddmmss(high_difference, low_difference, high_minuend, low_minuend,

   high_subtrahend, low_subtrahend) subtracts two two-word UWtype integers,

   composed by HIGH_MINUEND_1 and LOW_MINUEND_1, and HIGH_SUBTRAHEND_2 and

   LOW_SUBTRAHEND_2 respectively.  The result is placed in HIGH_DIFFERENCE

   and LOW_DIFFERENCE.  Overflow (i.e. carry out) is not stored anywhere,

   and is lost.

   If any of these macros are left undefined for a particular CPU,

   C macros are used.

   Notes:

   For add_ssaaaa the two high and two low addends can both commute, but

   unfortunately gcc only supports one "%" commutative in each asm block.

   This has always been so but is only documented in recent versions

   (eg. pre-release 3.3).  Having two or more "%"s can cause an internal

   compiler error in certain rare circumstances.

   Apparently it was only the last "%" that was ever actually respected, so

   the code has been updated to leave just that.  Clearly there's a free

   choice whether high or low should get it, if there's a reason to favour

   one over the other.  Also obviously when the constraints on the two

   operands are identical there's no benefit to the reloader in any "%" at

   all.

   */

/* The CPUs come in alphabetical order below.

   Please add support for more CPUs here, or improve the current support

   for the CPUs below!  */

/* count_leading_zeros_gcc_clz is count_leading_zeros implemented with gcc

   3.4 __builtin_clzl or __builtin_clzll, according to our limb size.

   Similarly count_trailing_zeros_gcc_ctz using __builtin_ctzl or

   __builtin_ctzll.

   These builtins are only used when we check what code comes out, on some

   chips they're merely libgcc calls, where we will instead want an inline

   in that case (either asm or generic C).

   These builtins are better than an asm block of the same insn, since an

   asm block doesn't give gcc any information about scheduling or resource

   usage.  We keep an asm block for use on prior versions of gcc though.

   For reference, __builtin_ffs existed in gcc prior to __builtin_clz, but

   it's not used (for count_leading_zeros) because it generally gives extra

   code to ensure the result is 0 when the input is 0, which we don't need

   or want.  */

#ifdef _LONG_LONG_LIMB

#define count_leading_zeros_gcc_clz(count,x)	\

  do {						\

    ASSERT ((x) != 0);				\

    (count) = __builtin_clzll (x);		\

  } while (0)

#else

#define count_leading_zeros_gcc_clz(count,x)	\

  do {						\

    ASSERT ((x) != 0);				\

    (count) = __builtin_clzl (x);		\

  } while (0)

#endif

#ifdef _LONG_LONG_LIMB

#define count_trailing_zeros_gcc_ctz(count,x)	\

  do {						\

    ASSERT ((x) != 0);				\

    (count) = __builtin_ctzll (x);		\

  } while (0)

#else

#define count_trailing_zeros_gcc_ctz(count,x)	\

  do {						\

    ASSERT ((x) != 0);				\

    (count) = __builtin_ctzl (x);		\

  } while (0)

#endif

/* FIXME: The macros using external routines like __MPN(count_leading_zeros)

   don't need to be under !NO_ASM */

#if ! defined (NO_ASM)

#if defined (__alpha) && W_TYPE_SIZE == 64

/* Most alpha-based machines, except Cray systems. */

#if defined (__GNUC__)

#if __GMP_GNUC_PREREQ (3,3)

#define umul_ppmm(ph, pl, m0, m1) \

  do {									\

    UDItype __m0 = (m0), __m1 = (m1);					\

    (ph) = __builtin_alpha_umulh (__m0, __m1);				\

    (pl) = __m0 * __m1;							\

  } while (0)

#else

#define umul_ppmm(ph, pl, m0, m1) \

  do {									\

    UDItype __m0 = (m0), __m1 = (m1);					\

    __asm__ ("umulh %r1,%2,%0"						\

	     : "=r" (ph)						\

	     : "%rJ" (__m0), "rI" (__m1));				\

    (pl) = __m0 * __m1;							\

  } while (0)

#endif

#define UMUL_TIME 18

#else /* ! __GNUC__ */

#include <machine/builtins.h>

#define umul_ppmm(ph, pl, m0, m1) \

  do {									\

    UDItype __m0 = (m0), __m1 = (m1);					\

    (ph) = __UMULH (__m0, __m1);					\

    (pl) = __m0 * __m1;							\

  } while (0)

#endif

#ifndef LONGLONG_STANDALONE

#define udiv_qrnnd(q, r, n1, n0, d) \

  do { UWtype __di;							\

    __di = __MPN(invert_limb) (d);					\

    udiv_qrnnd_preinv (q, r, n1, n0, d, __di);				\

  } while (0)

#define UDIV_PREINV_ALWAYS  1

#define UDIV_NEEDS_NORMALIZATION 1

#define UDIV_TIME 220

#endif /* LONGLONG_STANDALONE */

/* clz_tab is required in all configurations, since mpn/alpha/cntlz.asm

   always goes into libgmp.so, even when not actually used.  */

#define COUNT_LEADING_ZEROS_NEED_CLZ_TAB

#if defined (__GNUC__) && HAVE_HOST_CPU_alpha_CIX

#define count_leading_zeros(COUNT,X) \

  __asm__("ctlz %1,%0" : "=r"(COUNT) : "r"(X))

#define count_trailing_zeros(COUNT,X) \

  __asm__("cttz %1,%0" : "=r"(COUNT) : "r"(X))

#endif /* clz/ctz using cix */

#if ! defined (count_leading_zeros)				\

  && defined (__GNUC__) && ! defined (LONGLONG_STANDALONE)

/* ALPHA_CMPBGE_0 gives "cmpbge $31,src,dst", ie. test src bytes == 0.

   "$31" is written explicitly in the asm, since an "r" constraint won't

   select reg 31.  There seems no need to worry about "r31" syntax for cray,

   since gcc itself (pre-release 3.4) emits just $31 in various places.	 */

#define ALPHA_CMPBGE_0(dst, src)					\

  do { asm ("cmpbge $31, %1, %0" : "=r" (dst) : "r" (src)); } while (0)

/* Zero bytes are turned into bits with cmpbge, a __clz_tab lookup counts

   them, locating the highest non-zero byte.  A second __clz_tab lookup

   counts the leading zero bits in that byte, giving the result.  */

#define count_leading_zeros(count, x)					\

  do {									\

    UWtype  __clz__b, __clz__c, __clz__x = (x);				\

    ALPHA_CMPBGE_0 (__clz__b,  __clz__x);	    /* zero bytes */	\

    __clz__b = __clz_tab [(__clz__b >> 1) ^ 0x7F];  /* 8 to 1 byte */	\

    __clz__b = __clz__b * 8 - 7;		    /* 57 to 1 shift */ \

    __clz__x >>= __clz__b;						\

    __clz__c = __clz_tab [__clz__x];		    /* 8 to 1 bit */	\

    __clz__b = 65 - __clz__b;						\

    (count) = __clz__b - __clz__c;					\

  } while (0)

#define COUNT_LEADING_ZEROS_NEED_CLZ_TAB

#endif /* clz using cmpbge */

#if ! defined (count_leading_zeros) && ! defined (LONGLONG_STANDALONE)

#if HAVE_ATTRIBUTE_CONST

long __MPN(count_leading_zeros) (UDItype) __attribute__ ((const));

#else

long __MPN(count_leading_zeros) (UDItype);

#endif

#define count_leading_zeros(count, x) \

  ((count) = __MPN(count_leading_zeros) (x))

#endif /* clz using mpn */

#endif /* __alpha */

#if defined (__AVR) && W_TYPE_SIZE == 8

#define umul_ppmm(ph, pl, m0, m1) \

  do {									\

    unsigned short __p = (unsigned short) (m0) * (m1);			\

    (ph) = __p >> 8;							\

    (pl) = __p;								\

  } while (0)

#endif /* AVR */

#if defined (_CRAY) && W_TYPE_SIZE == 64

#include <intrinsics.h>

#define UDIV_PREINV_ALWAYS  1

#define UDIV_NEEDS_NORMALIZATION 1

#define UDIV_TIME 220

long __MPN(count_leading_zeros) (UDItype);

#define count_leading_zeros(count, x) \

  ((count) = _leadz ((UWtype) (x)))

#if defined (_CRAYIEEE)		/* I.e., Cray T90/ieee, T3D, and T3E */

#define umul_ppmm(ph, pl, m0, m1) \

  do {									\

    UDItype __m0 = (m0), __m1 = (m1);					\

    (ph) = _int_mult_upper (__m0, __m1);				\

    (pl) = __m0 * __m1;							\

  } while (0)

#ifndef LONGLONG_STANDALONE

#define udiv_qrnnd(q, r, n1, n0, d) \

  do { UWtype __di;							\

    __di = __MPN(invert_limb) (d);					\

    udiv_qrnnd_preinv (q, r, n1, n0, d, __di);				\

  } while (0)

#endif /* LONGLONG_STANDALONE */

#endif /* _CRAYIEEE */

#endif /* _CRAY */

#if defined (__ia64) && W_TYPE_SIZE == 64

/* This form encourages gcc (pre-release 3.4 at least) to emit predicated

   "sub r=r,r" and "sub r=r,r,1", giving a 2 cycle latency.  The generic

   code using "al

   register, which takes an extra cycle.  */

#define sub_ddmmss(sh, sl, ah, al, bh, bl)      \

  do {						\

    UWtype __x;					\

    __x = (al) - (bl);				\

    if ((al) < (bl))				\

      (sh) = (ah) - (bh) - 1;			\

    else					\

      (sh) = (ah) - (bh);			\

    (sl) = __x;					\

  } while (0)

#if defined (__GNUC__) && ! defined (__INTEL_COMPILER)

/* Do both product parts in assembly, since that gives better code with

   all gcc versions.  Some callers will just use the upper part, and in

   that situation we waste an instruction, but not any cycles.  */

#define umul_ppmm(ph, pl, m0, m1) \

    __asm__ ("xma.hu %0 = %2, %3, f0\n\txma.l %1 = %2, %3, f0"		\

	     : "=&f" (ph), "=f" (pl)					\

	     : "f" (m0), "f" (m1))

#define UMUL_TIME 14

#define count_leading_zeros(count, x) \

  do {									\

    UWtype _x = (x), _y, _a, _c;					\

    __asm__ ("mux1 %0 = %1, @rev" : "=r" (_y) : "r" (_x));		\

    __asm__ ("czx1.l %0 = %1" : "=r" (_a) : "r" (-_y | _y));		\

    _c = (_a - 1) << 3;							\

    _x >>= _c;								\

    if (_x >= 1 << 4)							\

      _x >>= 4, _c += 4;						\

    if (_x >= 1 << 2)							\

      _x >>= 2, _c += 2;						\

    _c += _x >> 1;							\

    (count) =  W_TYPE_SIZE - 1 - _c;					\

  } while (0)

/* similar to what gcc does for __builtin_ffs, but 0 based rather than 1

   based, and we don't need a special case for x==0 here */

#define count_trailing_zeros(count, x)					\

  do {									\

    UWtype __ctz_x = (x);						\

    __asm__ ("popcnt %0 = %1"						\

	     : "=r" (count)						\

	     : "r" ((__ctz_x-1) & ~__ctz_x));				\

  } while (0)

#endif

#if defined (__INTEL_COMPILER)

#include <ia64intrin.h>

#define umul_ppmm(ph, pl, m0, m1)					\

  do {									\

    UWtype __m0 = (m0), __m1 = (m1);					\

    ph = _m64_xmahu (__m0, __m1, 0);					\

    pl = __m0 * __m1;							\

  } while (0)

#endif

#ifndef LONGLONG_STANDALONE

#define udiv_qrnnd(q, r, n1, n0, d) \

  do { UWtype __di;							\

    __di = __MPN(invert_limb) (d);					\

    udiv_qrnnd_preinv (q, r, n1, n0, d, __di);				\

  } while (0)

#define UDIV_PREINV_ALWAYS  1

#define UDIV_NEEDS_NORMALIZATION 1

#endif

#define UDIV_TIME 220

#endif

#if defined (__GNUC__)

/* We sometimes need to clobber "cc" with gcc2, but that would not be

   understood by gcc1.  Use cpp to avoid major code duplication.  */

#if __GNUC__ < 2

#define __CLOBBER_CC

#define __AND_CLOBBER_CC

#else /* __GNUC__ >= 2 */

#define __CLOBBER_CC : "cc"

#define __AND_CLOBBER_CC , "cc"

#endif /* __GNUC__ < 2 */

#if (defined (__a29k__) || defined (_AM29K)) && W_TYPE_SIZE == 32

#define add_ssaaaa(sh, sl, ah, al, bh, bl) \

  __asm__ ("add %1,%4,%5\n\taddc %0,%2,%3"				\

	   : "=r" (sh), "=&r" (sl)					\

	   : "r" (ah), "rI" (bh), "%r" (al), "rI" (bl))

#define sub_ddmmss(sh, sl, ah, al, bh, bl) \

  __asm__ ("sub %1,%4,%5\n\tsubc %0,%2,%3"				\

	   : "=r" (sh), "=&r" (sl)					\

	   : "r" (ah), "rI" (bh), "r" (al), "rI" (bl))

#define umul_ppmm(xh, xl, m0, m1) \

  do {									\

    USItype __m0 = (m0), __m1 = (m1);					\

    __asm__ ("multiplu %0,%1,%2"					\

	     : "=r" (xl)						\

	     : "r" (__m0), "r" (__m1));					\

    __asm__ ("multmu %0,%1,%2"						\

	     : "=r" (xh)						\

	     : "r" (__m0), "r" (__m1));					\

  } while (0)

#define udiv_qrnnd(q, r, n1, n0, d) \

  __asm__ ("dividu %0,%3,%4"						\

	   : "=r" (q), "=q" (r)						\

	   : "1" (n1), "r" (n0), "r" (d))

#define count_leading_zeros(count, x) \

    __asm__ ("clz %0,%1"						\

	     : "=r" (count)						\

	     : "r" (x))

#define COUNT_LEADING_ZEROS_0 32

#endif /* __a29k__ */

#if defined (__arc__)

#define add_ssaaaa(sh, sl, ah, al, bh, bl) \

  __asm__ ("add.f\t%1, %4, %5\n\tadc\t%0, %2, %3"			\

	   : "=r" (sh),							\

	     "=&r" (sl)							\

	   : "r"  ((USItype) (ah)),					\

	     "rICal" ((USItype) (bh)),					\

	     "%r" ((USItype) (al)),					\

	     "rICal" ((USItype) (bl)))

#define sub_ddmmss(sh, sl, ah, al, bh, bl) \

  __asm__ ("sub.f\t%1, %4, %5\n\tsbc\t%0, %2, %3"			\

	   : "=r" (sh),							\

	     "=&r" (sl)							\

	   : "r" ((USItype) (ah)),					\

	     "rICal" ((USItype) (bh)),					\

	     "r" ((USItype) (al)),					\

	     "rICal" ((USItype) (bl)))

#endif

#if defined (__arm__) && (defined (__thumb2__) || !defined (__thumb__)) \

    && W_TYPE_SIZE == 32

#define add_ssaaaa(sh, sl, ah, al, bh, bl) \

  __asm__ ("adds\t%1, %4, %5\n\tadc\t%0, %2, %3"			\

	   : "=r" (sh), "=&r" (sl)					\

	   : "r" (ah), "rI" (bh), "%r" (al), "rI" (bl) __CLOBBER_CC)

#define sub_ddmmss(sh, sl, ah, al, bh, bl) \

  do {									\

    if (__builtin_constant_p (al))					\

      {									\

	if (__builtin_constant_p (ah))					\

	  __asm__ ("rsbs\t%1, %5, %4\n\trsc\t%0, %3, %2"		\

		   : "=r" (sh), "=&r" (sl)				\

		   : "rI" (ah), "r" (bh), "rI" (al), "r" (bl) __CLOBBER_CC); \

	else								\

	  __asm__ ("rsbs\t%1, %5, %4\n\tsbc\t%0, %2, %3"		\

		   : "=r" (sh), "=&r" (sl)				\

		   : "r" (ah), "rI" (bh), "rI" (al), "r" (bl) __CLOBBER_CC); \

      }									\

    else if (__builtin_constant_p (ah))					\

      {									\

	if (__builtin_constant_p (bl))					\

	  __asm__ ("subs\t%1, %4, %5\n\trsc\t%0, %3, %2"		\

		   : "=r" (sh), "=&r" (sl)				\

		   : "rI" (ah), "r" (bh), "r" (al), "rI" (bl) __CLOBBER_CC); \

	else								\

	  __asm__ ("rsbs\t%1, %5, %4\n\trsc\t%0, %3, %2"		\

		   : "=r" (sh), "=&r" (sl)				\

		   : "rI" (ah), "r" (bh), "rI" (al), "r" (bl) __CLOBBER_CC); \

      }									\

    else if (__builtin_constant_p (bl))					\

      {									\

	if (__builtin_constant_p (bh))					\

	  __asm__ ("subs\t%1, %4, %5\n\tsbc\t%0, %2, %3"		\

		   : "=r" (sh), "=&r" (sl)				\

		   : "r" (ah), "rI" (bh), "r" (al), "rI" (bl) __CLOBBER_CC); \

	else								\

	  __asm__ ("subs\t%1, %4, %5\n\trsc\t%0, %3, %2"		\

		   : "=r" (sh), "=&r" (sl)				\

		   : "rI" (ah), "r" (bh), "r" (al), "rI" (bl) __CLOBBER_CC); \

      }									\

    else /* only bh might be a constant */				\

      __asm__ ("subs\t%1, %4, %5\n\tsbc\t%0, %2, %3"			\

	       : "=r" (sh), "=&r" (sl)					\

	       : "r" (ah), "rI" (bh), "r" (al), "rI" (bl) __CLOBBER_CC);\

    } while (0)

#if defined (__ARM_ARCH_2__) || defined (__ARM_ARCH_2A__) \

    || defined (__ARM_ARCH_3__)

#define umul_ppmm(xh, xl, a, b)						\

  do {									\

    register USItype __t0, __t1, __t2;					\

    __asm__ ("%@ Inlined umul_ppmm\n"					\

	   "	mov	%2, %5, lsr #16\n"				\

	   "	mov	%0, %6, lsr #16\n"				\

	   "	bic	%3, %5, %2, lsl #16\n"				\

	   "	bic	%4, %6, %0, lsl #16\n"				\

	   "	mul	%1, %3, %4\n"					\

	   "	mul	%4, %2, %4\n"					\

	   "	mul	%3, %0, %3\n"					\

	   "	mul	%0, %2, %0\n"					\

	   "	adds	%3, %4, %3\n"					\

	   "	addcs	%0, %0, #65536\n"				\

	   "	adds	%1, %1, %3, lsl #16\n"				\

	   "	adc	%0, %0, %3, lsr #16"				\

	   : "=&r" ((USItype) (xh)), "=r" ((USItype) (xl)),		\

	     "=&r" (__t0), "=&r" (__t1), "=r" (__t2)			\

	   : "r" ((USItype) (a)), "r" ((USItype) (b)) __CLOBBER_CC);	\

  } while (0)

#define UMUL_TIME 20

#ifndef LONGLONG_STANDALONE

#define udiv_qrnnd(q, r, n1, n0, d) \

  do { UWtype __r;							\

    (q) = __MPN(udiv_qrnnd) (&__r, (n1), (n0), (d));			\

    (r) = __r;								\

  } while (0)

extern UWtype __MPN(udiv_qrnnd) (UWtype *, UWtype, UWtype, UWtype);

#define UDIV_TIME 200

#endif /* LONGLONG_STANDALONE */

#else /* ARMv4 or newer */

#define umul_ppmm(xh, xl, a, b) \

  __asm__ ("umull %0,%1,%2,%3" : "=&r" (xl), "=&r" (xh) : "r" (a), "r" (b))

#define UMUL_TIME 5

#define smul_ppmm(xh, xl, a, b) \

  __asm__ ("smull %0,%1,%2,%3" : "=&r" (xl), "=&r" (xh) : "r" (a), "r" (b))

#ifndef LONGLONG_STANDALONE

#define udiv_qrnnd(q, r, n1, n0, d) \

  do { UWtype __di;							\

    __di = __MPN(invert_limb) (d);					\

    udiv_qrnnd_preinv (q, r, n1, n0, d, __di);				\

  } while (0)

#define UDIV_PREINV_ALWAYS  1

#define UDIV_NEEDS_NORMALIZATION 1

#define UDIV_TIME 70

#endif /* LONGLONG_STANDALONE */

#endif /* defined(__ARM_ARCH_2__) ... */

#define count_leading_zeros(count, x)  count_leading_zeros_gcc_clz(count, x)

#define count_trailing_zeros(count, x)  count_trailing_zeros_gcc_ctz(count, x)

#define COUNT_LEADING_ZEROS_0 32

#endif /* __arm__ */

#if defined (__aarch64__) && W_TYPE_SIZE == 64

/* FIXME: Extend the immediate range for the low word by using both

   ADDS and SUBS, since they set carry in the same way.  */

#define add_ssaaaa(sh, sl, ah, al, bh, bl) \

  __asm__ ("adds\t%1, %x4, %5\n\tadc\t%0, %x2, %x3"			\

	   : "=r" (sh), "=&r" (sl)					\

	   : "rZ" ((UDItype)(ah)), "rZ" ((UDItype)(bh)),		\

	     "%r" ((UDItype)(al)), "rI" ((UDItype)(bl)) __CLOBBER_CC)

#define sub_ddmmss(sh, sl, ah, al, bh, bl) \

  __asm__ ("subs\t%1, %x4, %5\n\tsbc\t%0, %x2, %x3"			\

	   : "=r,r" (sh), "=&r,&r" (sl)					\

	   : "rZ,rZ" ((UDItype)(ah)), "rZ,rZ" ((UDItype)(bh)),		\

	     "r,Z"   ((UDItype)(al)), "rI,r"  ((UDItype)(bl)) __CLOBBER_CC)

#define umul_ppmm(ph, pl, m0, m1) \

  do {									\

    UDItype __m0 = (m0), __m1 = (m1);					\

    __asm__ ("umulh\t%0, %1, %2" : "=r" (ph) : "r" (__m0), "r" (__m1));	\

    (pl) = __m0 * __m1;							\

  } while (0)

#define count_leading_zeros(count, x)  count_leading_zeros_gcc_clz(count, x)

#define count_trailing_zeros(count, x)  count_trailing_zeros_gcc_ctz(count, x)

#define COUNT_LEADING_ZEROS_0 64

#endif /* __aarch64__ */

#if defined (__clipper__) && W_TYPE_SIZE == 32

#define umul_ppmm(w1, w0, u, v) \

  ({union {UDItype __ll;						\

	   struct {USItype __l, __h;} __i;				\

	  } __x;							\

  __asm__ ("mulwux %2,%0"						\

	   : "=r" (__x.__ll)						\

	   : "%0" ((USItype)(u)), "r" ((USItype)(v)));			\

  (w1) = __x.__i.__h; (w0) = __x.__i.__l;})

#define smul_ppmm(w1, w0, u, v) \

  ({union {DItype __ll;							\

	   struct {SItype __l, __h;} __i;				\

	  } __x;							\

  __asm__ ("mulwx %2,%0"						\

	   : "=r" (__x.__ll)						\

	   : "%0" ((SItype)(u)), "r" ((SItype)(v)));			\

  (w1) = __x.__i.__h; (w0) = __x.__i.__l;})

#define __umulsidi3(u, v) \

  ({UDItype __w;							\

    __asm__ ("mulwux %2,%0"						\

	     : "=r" (__w) : "%0" ((USItype)(u)), "r" ((USItype)(v)));	\

    __w; })

#endif /* __clipper__ */

/* Fujitsu vector computers.  */

#if defined (__uxp__) && W_TYPE_SIZE == 32

#define umul_ppmm(ph, pl, u, v) \

  do {									\

    union {UDItype __ll;						\

	   struct {USItype __h, __l;} __i;				\

	  } __x;							\

    __asm__ ("mult.lu %1,%2,%0"	: "=r" (__x.__ll) : "%r" (u), "rK" (v));\

    (ph) = __x.__i.__h;							\

    (pl) = __x.__i.__l;							\

  } while (0)

#define smul_ppmm(ph, pl, u, v) \

  do {									\

    union {UDItype __ll;						\

	   struct {USItype __h, __l;} __i;				\

	  } __x;							\

    __asm__ ("mult.l %1,%2,%0" : "=r" (__x.__ll) : "%r" (u), "rK" (v));	\

    (ph) = __x.__i.__h;							\

    (pl) = __x.__i.__l;							\

  } while (0)

#endif

#if defined (__gmicro__) && W_TYPE_SIZE == 32

#define add_ssaaaa(sh, sl, ah, al, bh, bl) \

  __asm__ ("add.w %5,%1\n\taddx %3,%0"					\

	   : "=g" (sh), "=&g" (sl)					\

	   : "0"  ((USItype)(ah)), "g" ((USItype)(bh)),			\

	     "%1" ((USItype)(al)), "g" ((USItype)(bl)))

#define sub_ddmmss(sh, sl, ah, al, bh, bl) \

  __asm__ ("sub.w %5,%1\n\tsubx %3,%0"					\

	   : "=g" (sh), "=&g" (sl)					\

	   : "0" ((USItype)(ah)), "g" ((USItype)(bh)),			\

	     "1" ((USItype)(al)), "g" ((USItype)(bl)))

#define umul_ppmm(ph, pl, m0, m1) \

  __asm__ ("mulx %3,%0,%1"						\

	   : "=g" (ph), "=r" (pl)					\

	   : "%0" ((USItype)(m0)), "g" ((USItype)(m1)))

#define udiv_qrnnd(q, r, nh, nl, d) \

  __asm__ ("divx %4,%0,%1"						\

	   : "=g" (q), "=r" (r)						\

	   : "1" ((USItype)(nh)), "0" ((USItype)(nl)), "g" ((USItype)(d)))

#define count_leading_zeros(count, x) \

  __asm__ ("bsch/1 %1,%0"						\

	   : "=g" (count) : "g" ((USItype)(x)), "0" ((USItype)0))

#endif

#if defined (__hppa) && W_TYPE_SIZE == 32

#define add_ssaaaa(sh, sl, ah, al, bh, bl) \

  __asm__ ("add%I5 %5,%r4,%1\n\taddc %r2,%r3,%0"			\

	   : "=r" (sh), "=&r" (sl)					\

	   : "rM" (ah), "rM" (bh), "%rM" (al), "rI" (bl))

#define sub_ddmmss(sh, sl, ah, al, bh, bl) \

  __asm__ ("sub%I4 %4,%r5,%1\n\tsubb %r2,%r3,%0"			\

	   : "=r" (sh), "=&r" (sl)					\

	   : "rM" (ah), "rM" (bh), "rI" (al), "rM" (bl))

#if defined (_PA_RISC1_1)

#define umul_ppmm(wh, wl, u, v) \

  do {									\

    union {UDItype __ll;						\

	   struct {USItype __h, __l;} __i;				\

	  } __x;							\

    __asm__ ("xmpyu %1,%2,%0" : "=*f" (__x.__ll) : "*f" (u), "*f" (v));	\

    (wh) = __x.__i.__h;							\

    (wl) = __x.__i.__l;							\

  } while (0)

#define UMUL_TIME 8

#define UDIV_TIME 60

#else

#define UMUL_TIME 40

#define UDIV_TIME 80

#endif

#define count_leading_zeros(count, x) \

  do {									\

    USItype __tmp;							\

    __asm__ (								\

       "ldi		1,%0\n"						\

"	extru,=		%1,15,16,%%r0	; Bits 31..16 zero?\n"		\

"	extru,tr	%1,15,16,%1	; No.  Shift down, skip add.\n"	\

"	ldo		16(%0),%0	; Yes.  Perform add.\n"		\

"	extru,=		%1,23,8,%%r0	; Bits 15..8 zero?\n"		\

"	extru,tr	%1,23,8,%1	; No.  Shift down, skip add.\n"	\

"	ldo		8(%0),%0	; Yes.  Perform add.\n"		\

"	extru,=		%1,27,4,%%r0	; Bits 7..4 zero?\n"		\

"	extru,tr	%1,27,4,%1	; No.  Shift down, skip add.\n"	\

"	ldo		4(%0),%0	; Yes.  Perform add.\n"		\

"	extru,=		%1,29,2,%%r0	; Bits 3..2 zero?\n"		\

"	extru,tr	%1,29,2,%1	; No.  Shift down, skip add.\n"	\

"	ldo		2(%0),%0	; Yes.  Perform add.\n"		\

"	extru		%1,30,1,%1	; Extract bit 1.\n"		\

"	sub		%0,%1,%0	; Subtract it.\n"		\

	: "=r" (count), "=r" (__tmp) : "1" (x));			\

  } while (0)

#endif /* hppa */

/* These macros are for ABI=2.0w.  In ABI=2.0n they can't be used, since GCC

   (3.2) puts longlong into two adjacent 32-bit registers.  Presumably this

   is just a case of no direct support for 2.0n but treating it like 1.0. */

#if defined (__hppa) && W_TYPE_SIZE == 64 && ! defined (_LONG_LONG_LIMB)

#define add_ssaaaa(sh, sl, ah, al, bh, bl) \

  __asm__ ("add%I5 %5,%r4,%1\n\tadd,dc %r2,%r3,%0"			\

	   : "=r" (sh), "=&r" (sl)					\

	   : "rM" (ah), "rM" (bh), "%rM" (al), "rI" (bl))

#define sub_ddmmss(sh, sl, ah, al, bh, bl) \

  __asm__ ("sub%I4 %4,%r5,%1\n\tsub,db %r2,%r3,%0"			\

	   : "=r" (sh), "=&r" (sl)					\

	   : "rM" (ah), "rM" (bh), "rI" (al), "rM" (bl))

#endif /* hppa */

#if (defined (__i370__) || defined (__s390__) || defined (__mvs__)) && W_TYPE_SIZE == 32

#if defined (__zarch__) || defined (HAVE_HOST_CPU_s390_zarch)

#define add_ssaaaa(sh, sl, ah, al, bh, bl)				\

  do {									\

/*  if (__builtin_constant_p (bl))					\

      __asm__ ("alfi\t%1,%o5\n\talcr\t%0,%3"				\

	       : "=r" (sh), "=&r" (sl)					\

	       : "0"  (ah), "r" (bh), "%1" (al), "n" (bl) __CLOBBER_CC);\

    else								\

*/    __asm__ ("alr\t%1,%5\n\talcr\t%0,%3"				\

	       : "=r" (sh), "=&r" (sl)					\

	       : "0"  (ah), "r" (bh), "%1" (al), "r" (bl)__CLOBBER_CC);	\

  } while (0)

#define sub_ddmmss(sh, sl, ah, al, bh, bl)				\

  do {									\

/*  if (__builtin_constant_p (bl))					\

      __asm__ ("slfi\t%1,%o5\n\tslbr\t%0,%3"				\

	       : "=r" (sh), "=&r" (sl)					\

	       : "0" (ah), "r" (bh), "1" (al), "n" (bl) __CLOBBER_CC);	\

    else								\

*/    __asm__ ("slr\t%1,%5\n\tslbr\t%0,%3"				\

	       : "=r" (sh), "=&r" (sl)					\

	       : "0" (ah), "r" (bh), "1" (al), "r" (bl) __CLOBBER_CC);	\

  } while (0)

#if __GMP_GNUC_PREREQ (4,5)

#define umul_ppmm(xh, xl, m0, m1)					\

  do {									\

    union {UDItype __ll;						\

	   struct {USItype __h, __l;} __i;				\

	  } __x;							\

    __x.__ll = (UDItype) (m0) * (UDItype) (m1);				\

    (xh) = __x.__i.__h; (xl) = __x.__i.__l;				\

  } while (0)

#else

#if 0

/* FIXME: this fails if gcc knows about the 64-bit registers.  Use only

   with a new enough processor pretending we have 32-bit registers.  */

#define umul_ppmm(xh, xl, m0, m1)					\

  do {									\

    union {UDItype __ll;						\

	   struct {USItype __h, __l;} __i;				\

	  } __x;							\

    __asm__ ("mlr\t%0,%2"						\

	     : "=r" (__x.__ll)						\

	     : "%0" (m0), "r" (m1));					\

    (xh) = __x.__i.__h; (xl) = __x.__i.__l;				\

  } while (0)

#else

#define umul_ppmm(xh, xl, m0, m1)					\

  do {									\

  /* When we have 64-bit regs and gcc is aware of that, we cannot simply use

     DImode for the product, since that would be allocated to a single 64-bit

     register, whereas mlr uses the low 32-bits of an even-odd register pair.

  */									\

    register USItype __r0 __asm__ ("0");				\

    register USItype __r1 __asm__ ("1") = (m0);				\

    __asm__ ("mlr\t%0,%3"						\

	     : "=r" (__r0), "=r" (__r1)					\

	     : "r" (__r1), "r" (m1));					\

    (xh) = __r0; (xl) = __r1;						\

  } while (0)

#endif /* if 0 */

#endif

#if 0

/* FIXME: this fails if gcc knows about the 64-bit registers.  Use only

   with a new enough processor pretending we have 32-bit registers.  */

#define udiv_qrnnd(q, r, n1, n0, d)					\

  do {									\

    union {UDItype __ll;						\

	   struct {USItype __h, __l;} __i;				\

	  } __x;							\

    __x.__i.__h = n1; __x.__i.__l = n0;					\

    __asm__ ("dlr\t%0,%2"						\

	     : "=r" (__x.__ll)						\

	     : "0" (__x.__ll), "r" (d));				\

    (q) = __x.__i.__l; (r) = __x.__i.__h;				\

  } while (0)

#else

#define udiv_qrnnd(q, r, n1, n0, d)					\

  do {									\

    register USItype __r0 __asm__ ("0") = (n1);				\

    register USItype __r1 __asm__ ("1") = (n0);				\

    __asm__ ("dlr\t%0,%4"						\

	     : "=r" (__r0), "=r" (__r1)					\

	     : "r" (__r0), "r" (__r1), "r" (d));			\

    (q) = __r1; (r) = __r0;						\

  } while (0)

#endif /* if 0 */

#else /* if __zarch__ */

/* FIXME: this fails if gcc knows about the 64-bit registers.  */

#define smul_ppmm(xh, xl, m0, m1)					\

  do {									\

    union {DItype __ll;							\

	   struct {USItype __h, __l;} __i;				\

	  } __x;							\

    __asm__ ("mr\t%0,%2"						\

	     : "=r" (__x.__ll)						\

	     : "%0" (m0), "r" (m1));					\

    (xh) = __x.__i.__h; (xl) = __x.__i.__l;				\

  } while (0)

/* FIXME: this fails if gcc knows about the 64-bit registers.  */

#define sdiv_qrnnd(q, r, n1, n0, d)					\

  do {									\

    union {DItype __ll;							\

	   struct {USItype __h, __l;} __i;				\

	  } __x;							\

    __x.__i.__h = n1; __x.__i.__l = n0;					\

    __asm__ ("dr\t%0,%2"						\

	     : "=r" (__x.__ll)						\

	     : "0" (__x.__ll), "r" (d));				\

    (q) = __x.__i.__l; (r) = __x.__i.__h;				\

  } while (0)

#endif /* if __zarch__ */

#endif

#if defined (__s390x__) && W_TYPE_SIZE == 64

/* We need to cast operands with register constraints, otherwise their types

   will be assumed to be SImode by gcc.  For these machines, such operations

   will insert a value into the low 32 bits, and leave the high 32 bits with

   garbage.  */

#define add_ssaaaa(sh, sl, ah, al, bh, bl)				\

  do {									\

    __asm__ ("algr\t%1,%5\n\talcgr\t%0,%3"				\

	       : "=r" (sh), "=&r" (sl)					\

	       : "0"  ((UDItype)(ah)), "r" ((UDItype)(bh)),		\

		 "%1" ((UDItype)(al)), "r" ((UDItype)(bl)) __CLOBBER_CC); \

  } while (0)

#define sub_ddmmss(sh, sl, ah, al, bh, bl)				\

  do {									\

    __asm__ ("slgr\t%1,%5\n\tslbgr\t%0,%3"				\

	     : "=r" (sh), "=&r" (sl)					\

	     : "0" ((UDItype)(ah)), "r" ((UDItype)(bh)),		\

	       "1" ((UDItype)(al)), "r" ((UDItype)(bl)) __CLOBBER_CC);	\

  } while (0)

#define umul_ppmm(xh, xl, m0, m1)					\

  do {									\

    union {unsigned int __attribute__ ((mode(TI))) __ll;		\

	   struct {UDItype __h, __l;} __i;				\

	  } __x;							\

    __asm__ ("mlgr\t%0,%2"						\

	     : "=r" (__x.__ll)						\

	     : "%0" ((UDItype)(m0)), "r" ((UDItype)(m1)));		\

    (xh) = __x.__i.__h; (xl) = __x.__i.__l;				\

  } while (0)

#define udiv_qrnnd(q, r, n1, n0, d)					\

  do {									\

    union {unsigned int __attribute__ ((mode(TI))) __ll;		\

	   struct {UDItype __h, __l;} __i;				\

	  } __x;							\

    __x.__i.__h = n1; __x.__i.__l = n0;					\

    __asm__ ("dlgr\t%0,%2"						\

	     : "=r" (__x.__ll)						\

	     : "0" (__x.__ll), "r" ((UDItype)(d)));			\

    (q) = __x.__i.__l; (r) = __x.__i.__h;				\

  } while (0)

#if 0 /* FIXME: Enable for z10 (?) */

#define count_leading_zeros(cnt, x)					\

  do {									\

    union {unsigned int __attribute__ ((mode(TI))) __ll;		\

	   struct {UDItype __h, __l;} __i;				\

	  } __clr_cnt;							\

    __asm__ ("flogr\t%0,%1"						\

	     : "=r" (__clr_cnt.__ll)					\

	     : "r" (x) __CLOBBER_CC);					\

    (cnt) = __clr_cnt.__i.__h;						\

  } while (0)

#endif

#endif

/* On x86 and x86_64, every asm implicitly clobbers "flags" and "fpsr",

   so we don't need __CLOBBER_CC.  */

#if (defined (__i386__) || defined (__i486__)) && W_TYPE_SIZE == 32

#define add_ssaaaa(sh, sl, ah, al, bh, bl) \

  __asm__ ("addl %5,%k1\n\tadcl %3,%k0"					\

	   : "=r" (sh), "=&r" (sl)					\

	   : "0"  ((USItype)(ah)), "g" ((USItype)(bh)),			\

	     "%1" ((USItype)(al)), "g" ((USItype)(bl)))

#define sub_ddmmss(sh, sl, ah, al, bh, bl) \

  __asm__ ("subl %5,%k1\n\tsbbl %3,%k0"					\

	   : "=r" (sh), "=&r" (sl)					\

	   : "0" ((USItype)(ah)), "g" ((USItype)(bh)),			\

	     "1" ((USItype)(al)), "g" ((USItype)(bl)))

#define umul_ppmm(w1, w0, u, v) \

  __asm__ ("mull %3"							\

	   : "=a" (w0), "=d" (w1)					\