#! /usr/bin/env perl

# Copyright 2013-2020 The OpenSSL Project Authors. All Rights Reserved.

# Copyright (c) 2012, Intel Corporation. All Rights Reserved.

#

# Licensed under the OpenSSL license (the "License").  You may not use

# this file except in compliance with the License.  You can obtain a copy

# in the file LICENSE in the source distribution or at

# https://www.openssl.org/source/license.html

#

# Originally written by Shay Gueron (1, 2), and Vlad Krasnov (1)

# (1) Intel Corporation, Israel Development Center, Haifa, Israel

# (2) University of Haifa, Israel

#

# References:

# [1] S. Gueron, "Efficient Software Implementations of Modular

#     Exponentiation", http://eprint.iacr.org/2011/239

# [2] S. Gueron, V. Krasnov. "Speeding up Big-Numbers Squaring".

#     IEEE Proceedings of 9th International Conference on Information

#     Technology: New Generations (ITNG 2012), 821-823 (2012).

# [3] S. Gueron, Efficient Software Implementations of Modular Exponentiation

#     Journal of Cryptographic Engineering 2:31-43 (2012).

# [4] S. Gueron, V. Krasnov: "[PATCH] Efficient and side channel analysis

#     resistant 512-bit and 1024-bit modular exponentiation for optimizing

#     RSA1024 and RSA2048 on x86_64 platforms",

#     http://rt.openssl.org/Ticket/Display.html?id=2582&user=guest&pass=guest

#

# While original submission covers 512- and 1024-bit exponentiation,

# this module is limited to 512-bit version only (and as such

# accelerates RSA1024 sign). This is because improvement for longer

# keys is not high enough to justify the effort, highest measured

# was ~5% on Westmere. [This is relative to OpenSSL 1.0.2, upcoming

# for the moment of this writing!] Nor does this module implement

# "monolithic" complete exponentiation jumbo-subroutine, but adheres

# to more modular mixture of C and assembly. And it's optimized even

# for processors other than Intel Core family (see table below for

# improvement coefficients).

# 						<appro@openssl.org>

#

# RSA1024 sign/sec	this/original	|this/rsax(*)	this/fips(*)

#			----------------+---------------------------

# Opteron		+13%		|+5%		+20%

# Bulldozer		-0%		|-1%		+10%

# P4			+11%		|+7%		+8%

# Westmere		+5%		|+14%		+17%

# Sandy Bridge		+2%		|+12%		+29%

# Ivy Bridge		+1%		|+11%		+35%

# Haswell(**)		-0%		|+12%		+39%

# Atom			+13%		|+11%		+4%

# VIA Nano		+70%		|+9%		+25%

#

# (*)	rsax engine and fips numbers are presented for reference

#	purposes;

# (**)	MULX was attempted, but found to give only marginal improvement;

$flavour = shift;

$output  = shift;

if ($flavour =~ /\./) { $output = $flavour; undef $flavour; }

$win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/);

$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;

( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or

( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or

die "can't locate x86_64-xlate.pl";

open OUT,"| \"$^X\" \"$xlate\" $flavour \"$output\"";

*STDOUT=*OUT;

if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1`

		=~ /GNU assembler version ([2-9]\.[0-9]+)/) {

	$addx = ($1>=2.23);

}

if (!$addx && $win64 && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/) &&

	    `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)/) {

	$addx = ($1>=2.10);

}

if (!$addx && $win64 && ($flavour =~ /masm/ || $ENV{ASM} =~ /ml64/) &&

	    `ml64 2>&1` =~ /Version ([0-9]+)\./) {

	$addx = ($1>=12);

}

if (!$addx && `$ENV{CC} -v 2>&1` =~ /((?:^clang|LLVM) version|.*based on LLVM) ([0-9]+)\.([0-9]+)/) {

	my $ver = $2 + $3/100.0;	# 3.1->3.01, 3.10->3.10

	$addx = ($ver>=3.03);

}

($out, $inp, $mod) = ("%rdi", "%rsi", "%rbp");	# common internal API

{

my ($out,$inp,$mod,$n0,$times) = ("%rdi","%rsi","%rdx","%rcx","%r8d");

$code.=<<___;

.text

.extern	OPENSSL_ia32cap_P

.globl	rsaz_512_sqr

.type	rsaz_512_sqr,\@function,5

.align	32

rsaz_512_sqr:				# 25-29% faster than rsaz_512_mul

.cfi_startproc

	push	%rbx

.cfi_push	%rbx

	push	%rbp

.cfi_push	%rbp

	push	%r12

.cfi_push	%r12

	push	%r13

.cfi_push	%r13

	push	%r14

.cfi_push	%r14

	push	%r15

.cfi_push	%r15

	subq	\$128+24, %rsp

.cfi_adjust_cfa_offset	128+24

.Lsqr_body:

	movq	$mod, %xmm1		# common off-load

	movq	($inp), %rdx

	movq	8($inp), %rax

	movq	$n0, 128(%rsp)

___

$code.=<<___ if ($addx);

	movl	\$0x80100,%r11d

	andl	OPENSSL_ia32cap_P+8(%rip),%r11d

	cmpl	\$0x80100,%r11d		# check for MULX and ADO/CX

	je	.Loop_sqrx

___

$code.=<<___;

	jmp	.Loop_sqr

.align	32

.Loop_sqr:

	movl	$times,128+8(%rsp)

#first iteration

	movq	%rdx, %rbx		# 0($inp)

	mov	%rax, %rbp		# 8($inp)

	mulq	%rdx

	movq	%rax, %r8

	movq	16($inp), %rax

	movq	%rdx, %r9

	mulq	%rbx

	addq	%rax, %r9

	movq	24($inp), %rax

	movq	%rdx, %r10

	adcq	\$0, %r10

	mulq	%rbx

	addq	%rax, %r10

	movq	32($inp), %rax

	movq	%rdx, %r11

	adcq	\$0, %r11

	mulq	%rbx

	addq	%rax, %r11

	movq	40($inp), %rax

	movq	%rdx, %r12

	adcq	\$0, %r12

	mulq	%rbx

	addq	%rax, %r12

	movq	48($inp), %rax

	movq	%rdx, %r13

	adcq	\$0, %r13

	mulq	%rbx

	addq	%rax, %r13

	movq	56($inp), %rax

	movq	%rdx, %r14

	adcq	\$0, %r14

	mulq	%rbx

	addq	%rax, %r14

	movq	%rbx, %rax

	adcq	\$0, %rdx

	xorq	%rcx,%rcx		# rcx:r8 = r8 << 1

	addq	%r8, %r8

	 movq	%rdx, %r15

	adcq	\$0, %rcx

	mulq	%rax

	addq	%r8, %rdx

	adcq	\$0, %rcx

	movq	%rax, (%rsp)

	movq	%rdx, 8(%rsp)

#second iteration

	movq	16($inp), %rax

	mulq	%rbp

	addq	%rax, %r10

	movq	24($inp), %rax

	movq	%rdx, %rbx

	adcq	\$0, %rbx

	mulq	%rbp

	addq	%rax, %r11

	movq	32($inp), %rax

	adcq	\$0, %rdx

	addq	%rbx, %r11

	movq	%rdx, %rbx

	adcq	\$0, %rbx

	mulq	%rbp

	addq	%rax, %r12

	movq	40($inp), %rax

	adcq	\$0, %rdx

	addq	%rbx, %r12

	movq	%rdx, %rbx

	adcq	\$0, %rbx

	mulq	%rbp

	addq	%rax, %r13

	movq	48($inp), %rax

	adcq	\$0, %rdx

	addq	%rbx, %r13

	movq	%rdx, %rbx

	adcq	\$0, %rbx

	mulq	%rbp

	addq	%rax, %r14

	movq	56($inp), %rax

	adcq	\$0, %rdx

	addq	%rbx, %r14

	movq	%rdx, %rbx

	adcq	\$0, %rbx

	mulq	%rbp

	addq	%rax, %r15

	movq	%rbp, %rax

	adcq	\$0, %rdx

	addq	%rbx, %r15

	adcq	\$0, %rdx

	xorq	%rbx, %rbx		# rbx:r10:r9 = r10:r9 << 1

	addq	%r9, %r9

	 movq	%rdx, %r8

	adcq	%r10, %r10

	adcq	\$0, %rbx

	mulq	%rax

	# rcx <= 2 and rax <= 0xFFFF..F9, so carry must be zero here

	addq	%rcx, %rax

	 movq	16($inp), %rbp

	addq	%rax, %r9

	 movq	24($inp), %rax

	adcq	%rdx, %r10

	adcq	\$0, %rbx

	movq	%r9, 16(%rsp)

	movq	%r10, 24(%rsp)

#third iteration

	mulq	%rbp

	addq	%rax, %r12

	movq	32($inp), %rax

	movq	%rdx, %rcx

	adcq	\$0, %rcx

	mulq	%rbp

	addq	%rax, %r13

	movq	40($inp), %rax

	adcq	\$0, %rdx

	addq	%rcx, %r13

	movq	%rdx, %rcx

	adcq	\$0, %rcx

	mulq	%rbp

	addq	%rax, %r14

	movq	48($inp), %rax

	adcq	\$0, %rdx

	addq	%rcx, %r14

	movq	%rdx, %rcx

	adcq	\$0, %rcx

	mulq	%rbp

	addq	%rax, %r15

	movq	56($inp), %rax

	adcq	\$0, %rdx

	addq	%rcx, %r15

	movq	%rdx, %rcx

	adcq	\$0, %rcx

	mulq	%rbp

	addq	%rax, %r8

	movq	%rbp, %rax

	adcq	\$0, %rdx

	addq	%rcx, %r8

	adcq	\$0, %rdx

	xorq	%rcx, %rcx		# rcx:r12:r11 = r12:r11 << 1

	addq	%r11, %r11

	 movq	%rdx, %r9

	adcq	%r12, %r12

	adcq	\$0, %rcx

	mulq	%rax

	# rbx <= 2 and rax <= 0xFFFF..F9, so carry must be zero here

	addq	%rbx, %rax

	 movq	24($inp), %r10

	addq	%rax, %r11

	 movq	32($inp), %rax

	adcq	%rdx, %r12

	adcq	\$0, %rcx

	movq	%r11, 32(%rsp)

	movq	%r12, 40(%rsp)

#fourth iteration

	mov	%rax, %r11		# 32($inp)

	mulq	%r10

	addq	%rax, %r14

	movq	40($inp), %rax

	movq	%rdx, %rbx

	adcq	\$0, %rbx

	mov	%rax, %r12		# 40($inp)

	mulq	%r10

	addq	%rax, %r15

	movq	48($inp), %rax

	adcq	\$0, %rdx

	addq	%rbx, %r15

	movq	%rdx, %rbx

	adcq	\$0, %rbx

	mov	%rax, %rbp		# 48($inp)

	mulq	%r10

	addq	%rax, %r8

	movq	56($inp), %rax

	adcq	\$0, %rdx

	addq	%rbx, %r8

	movq	%rdx, %rbx

	adcq	\$0, %rbx

	mulq	%r10

	addq	%rax, %r9

	movq	%r10, %rax

	adcq	\$0, %rdx

	addq	%rbx, %r9

	adcq	\$0, %rdx

	xorq	%rbx, %rbx		# rbx:r13:r14 = r13:r14 << 1

	addq	%r13, %r13

	 movq	%rdx, %r10

	adcq	%r14, %r14

	adcq	\$0, %rbx

	mulq	%rax

	# rcx <= 2 and rax <= 0xFFFF..F9, so carry must be zero here

	addq	%rcx, %rax

	addq	%rax, %r13

	 movq	%r12, %rax		# 40($inp)

	adcq	%rdx, %r14

	adcq	\$0, %rbx

	movq	%r13, 48(%rsp)

	movq	%r14, 56(%rsp)

#fifth iteration

	mulq	%r11

	addq	%rax, %r8

	movq	%rbp, %rax		# 48($inp)

	movq	%rdx, %rcx

	adcq	\$0, %rcx

	mulq	%r11

	addq	%rax, %r9

	movq	56($inp), %rax

	adcq	\$0, %rdx

	addq	%rcx, %r9

	movq	%rdx, %rcx

	adcq	\$0, %rcx

	mov	%rax, %r14		# 56($inp)

	mulq	%r11

	addq	%rax, %r10

	movq	%r11, %rax

	adcq	\$0, %rdx

	addq	%rcx, %r10

	adcq	\$0, %rdx

	xorq	%rcx, %rcx		# rcx:r8:r15 = r8:r15 << 1

	addq	%r15, %r15

	 movq	%rdx, %r11

	adcq	%r8, %r8

	adcq	\$0, %rcx

	mulq	%rax

	# rbx <= 2 and rax <= 0xFFFF..F9, so carry must be zero here

	addq	%rbx, %rax

	addq	%rax, %r15

	 movq	%rbp, %rax		# 48($inp)

	adcq	%rdx, %r8

	adcq	\$0, %rcx

	movq	%r15, 64(%rsp)

	movq	%r8, 72(%rsp)

#sixth iteration

	mulq	%r12

	addq	%rax, %r10

	movq	%r14, %rax		# 56($inp)

	movq	%rdx, %rbx

	adcq	\$0, %rbx

	mulq	%r12

	addq	%rax, %r11

	movq	%r12, %rax

	adcq	\$0, %rdx

	addq	%rbx, %r11

	adcq	\$0, %rdx

	xorq	%rbx, %rbx		# rbx:r10:r9 = r10:r9 << 1

	addq	%r9, %r9

	 movq	%rdx, %r12

	adcq	%r10, %r10

	adcq	\$0, %rbx

	mulq	%rax

	# rcx <= 2 and rax <= 0xFFFF..F9, so carry must be zero here

	addq	%rcx, %rax

	addq	%rax, %r9

	 movq	%r14, %rax		# 56($inp)

	adcq	%rdx, %r10

	adcq	\$0, %rbx

	movq	%r9, 80(%rsp)

	movq	%r10, 88(%rsp)

#seventh iteration

	mulq	%rbp

	addq	%rax, %r12

	movq	%rbp, %rax

	adcq	\$0, %rdx

	xorq	%rcx, %rcx		# rcx:r12:r11 = r12:r11 << 1

	addq	%r11, %r11

	 movq	%rdx, %r13

	adcq	%r12, %r12

	adcq	\$0, %rcx

	mulq	%rax

	# rbx <= 2 and rax <= 0xFFFF..F9, so carry must be zero here

	addq	%rbx, %rax

	addq	%rax, %r11

	 movq	%r14, %rax		# 56($inp)

	adcq	%rdx, %r12

	adcq	\$0, %rcx

	movq	%r11, 96(%rsp)

	movq	%r12, 104(%rsp)

#eighth iteration

	xorq	%rbx, %rbx		# rbx:r13 = r13 << 1

	addq	%r13, %r13

	adcq	\$0, %rbx

	mulq	%rax

	# rcx <= 2 and rax <= 0xFFFF..F9, so carry must be zero here

	addq	%rcx, %rax

	addq	%r13, %rax

	adcq	%rbx, %rdx

	movq	(%rsp), %r8

	movq	8(%rsp), %r9

	movq	16(%rsp), %r10

	movq	24(%rsp), %r11

	movq	32(%rsp), %r12

	movq	40(%rsp), %r13

	movq	48(%rsp), %r14

	movq	56(%rsp), %r15

	movq	%xmm1, %rbp

	movq	%rax, 112(%rsp)

	movq	%rdx, 120(%rsp)

	call	__rsaz_512_reduce

	addq	64(%rsp), %r8

	adcq	72(%rsp), %r9

	adcq	80(%rsp), %r10

	adcq	88(%rsp), %r11

	adcq	96(%rsp), %r12

	adcq	104(%rsp), %r13

	adcq	112(%rsp), %r14

	adcq	120(%rsp), %r15

	sbbq	%rcx, %rcx

	call	__rsaz_512_subtract

	movq	%r8, %rdx

	movq	%r9, %rax

	movl	128+8(%rsp), $times

	movq	$out, $inp

	decl	$times

	jnz	.Loop_sqr

___

if ($addx) {

$code.=<<___;

	jmp	.Lsqr_tail

.align	32

.Loop_sqrx:

	movl	$times,128+8(%rsp)

	movq	$out, %xmm0		# off-load

#first iteration

	mulx	%rax, %r8, %r9

	mov	%rax, %rbx

	mulx	16($inp), %rcx, %r10

	xor	%rbp, %rbp		# cf=0, of=0

	mulx	24($inp), %rax, %r11

	adcx	%rcx, %r9

	.byte	0xc4,0x62,0xf3,0xf6,0xa6,0x20,0x00,0x00,0x00	# mulx	32($inp), %rcx, %r12

	adcx	%rax, %r10

	.byte	0xc4,0x62,0xfb,0xf6,0xae,0x28,0x00,0x00,0x00	# mulx	40($inp), %rax, %r13

	adcx	%rcx, %r11

	mulx	48($inp), %rcx, %r14

	adcx	%rax, %r12

	adcx	%rcx, %r13

	mulx	56($inp), %rax, %r15

	adcx	%rax, %r14

	adcx	%rbp, %r15		# %rbp is 0

	mulx	%rdx, %rax, $out

	 mov	%rbx, %rdx		# 8($inp)

	xor	%rcx, %rcx

	adox	%r8, %r8

	adcx	$out, %r8

	adox	%rbp, %rcx

	adcx	%rbp, %rcx

	mov	%rax, (%rsp)

	mov	%r8, 8(%rsp)

#second iteration

	.byte	0xc4,0xe2,0xfb,0xf6,0x9e,0x10,0x00,0x00,0x00	# mulx	16($inp), %rax, %rbx

	adox	%rax, %r10

	adcx	%rbx, %r11

	mulx	24($inp), $out, %r8

	adox	$out, %r11

	.byte	0x66

	adcx	%r8, %r12

	mulx	32($inp), %rax, %rbx

	adox	%rax, %r12

	adcx	%rbx, %r13

	mulx	40($inp), $out, %r8

	adox	$out, %r13

	adcx	%r8, %r14

	.byte	0xc4,0xe2,0xfb,0xf6,0x9e,0x30,0x00,0x00,0x00	# mulx	48($inp), %rax, %rbx

	adox	%rax, %r14

	adcx	%rbx, %r15

	.byte	0xc4,0x62,0xc3,0xf6,0x86,0x38,0x00,0x00,0x00	# mulx	56($inp), $out, %r8

	adox	$out, %r15

	adcx	%rbp, %r8

	 mulx	%rdx, %rax, $out

	adox	%rbp, %r8

	 .byte	0x48,0x8b,0x96,0x10,0x00,0x00,0x00		# mov	16($inp), %rdx

	xor	%rbx, %rbx

	 adox	%r9, %r9

	# rcx <= 2 and rax <= 0xFFFF..F9, so carry must be zero here

	adcx	%rcx, %rax

	adox	%r10, %r10

	adcx	%rax, %r9

	adox	%rbp, %rbx

	adcx	$out, %r10

	adcx	%rbp, %rbx

	mov	%r9, 16(%rsp)

	.byte	0x4c,0x89,0x94,0x24,0x18,0x00,0x00,0x00		# mov	%r10, 24(%rsp)

#third iteration

	mulx	24($inp), $out, %r9

	adox	$out, %r12

	adcx	%r9, %r13

	mulx	32($inp), %rax, %rcx

	adox	%rax, %r13

	adcx	%rcx, %r14

	.byte	0xc4,0x62,0xc3,0xf6,0x8e,0x28,0x00,0x00,0x00	# mulx	40($inp), $out, %r9

	adox	$out, %r14

	adcx	%r9, %r15

	.byte	0xc4,0xe2,0xfb,0xf6,0x8e,0x30,0x00,0x00,0x00	# mulx	48($inp), %rax, %rcx

	adox	%rax, %r15

	adcx	%rcx, %r8

	mulx	56($inp), $out, %r9

	adox	$out, %r8

	adcx	%rbp, %r9

	 mulx	%rdx, %rax, $out

	adox	%rbp, %r9

	 mov	24($inp), %rdx

	xor	%rcx, %rcx

	 adox	%r11, %r11

	# rbx <= 2 and rax <= 0xFFFF..F9, so carry must be zero here

	adcx	%rbx, %rax

	adox	%r12, %r12

	adcx	%rax, %r11

	adox	%rbp, %rcx

	adcx	$out, %r12

	adcx	%rbp, %rcx

	mov	%r11, 32(%rsp)

	mov	%r12, 40(%rsp)

#fourth iteration

	mulx	32($inp), %rax, %rbx

	adox	%rax, %r14

	adcx	%rbx, %r15

	mulx	40($inp), $out, %r10

	adox	$out, %r15

	adcx	%r10, %r8

	mulx	48($inp), %rax, %rbx

	adox	%rax, %r8

	adcx	%rbx, %r9

	mulx	56($inp), $out, %r10

	adox	$out, %r9

	adcx	%rbp, %r10

	 mulx	%rdx, %rax, $out

	adox	%rbp, %r10

	 mov	32($inp), %rdx

	xor	%rbx, %rbx

	 adox	%r13, %r13

	# rcx <= 2 and rax <= 0xFFFF..F9, so carry must be zero here

	adcx	%rcx, %rax

	adox	%r14, %r14

	adcx	%rax, %r13

	adox	%rbp, %rbx

	adcx	$out, %r14

	adcx	%rbp, %rbx

	mov	%r13, 48(%rsp)

	mov	%r14, 56(%rsp)

#fifth iteration

	mulx	40($inp), $out, %r11

	adox	$out, %r8

	adcx	%r11, %r9

	mulx	48($inp), %rax, %rcx

	adox	%rax, %r9

	adcx	%rcx, %r10

	mulx	56($inp), $out, %r11

	adox	$out, %r10

	adcx	%rbp, %r11

	 mulx	%rdx, %rax, $out

	 mov	40($inp), %rdx

	adox	%rbp, %r11

	xor	%rcx, %rcx

	 adox	%r15, %r15

	# rbx <= 2 and rax <= 0xFFFF..F9, so carry must be zero here

	adcx	%rbx, %rax

	adox	%r8, %r8

	adcx	%rax, %r15

	adox	%rbp, %rcx

	adcx	$out, %r8

	adcx	%rbp, %rcx

	mov	%r15, 64(%rsp)

	mov	%r8, 72(%rsp)

#sixth iteration

	.byte	0xc4,0xe2,0xfb,0xf6,0x9e,0x30,0x00,0x00,0x00	# mulx	48($inp), %rax, %rbx

	adox	%rax, %r10

	adcx	%rbx, %r11

	.byte	0xc4,0x62,0xc3,0xf6,0xa6,0x38,0x00,0x00,0x00	# mulx	56($inp), $out, %r12

	adox	$out, %r11

	adcx	%rbp, %r12

	 mulx	%rdx, %rax, $out

	adox	%rbp, %r12

	 mov	48($inp), %rdx

	xor	%rbx, %rbx

	 adox	%r9, %r9

	# rcx <= 2 and rax <= 0xFFFF..F9, so carry must be zero here

	adcx	%rcx, %rax

	adox	%r10, %r10

	adcx	%rax, %r9

	adcx	$out, %r10

	adox	%rbp, %rbx

	adcx	%rbp, %rbx

	mov	%r9, 80(%rsp)

	mov	%r10, 88(%rsp)

#seventh iteration

	.byte	0xc4,0x62,0xfb,0xf6,0xae,0x38,0x00,0x00,0x00	# mulx	56($inp), %rax, %r13

	adox	%rax, %r12

	adox	%rbp, %r13

	mulx	%rdx, %rax, $out

	xor	%rcx, %rcx

	 mov	56($inp), %rdx

	 adox	%r11, %r11

	# rbx <= 2 and rax <= 0xFFFF..F9, so carry must be zero here

	adcx	%rbx, %rax

	adox	%r12, %r12

	adcx	%rax, %r11

	adox	%rbp, %rcx

	adcx	$out, %r12

	adcx	%rbp, %rcx

	.byte	0x4c,0x89,0x9c,0x24,0x60,0x00,0x00,0x00		# mov	%r11, 96(%rsp)

	.byte	0x4c,0x89,0xa4,0x24,0x68,0x00,0x00,0x00		# mov	%r12, 104(%rsp)

#eighth iteration

	mulx	%rdx, %rax, %rdx

	xor	%rbx, %rbx

	 adox	%r13, %r13

	# rcx <= 2 and rax <= 0xFFFF..F9, so carry must be zero here

	adcx	%rcx, %rax

	adox	%rbp, %rbx

	adcx	%r13, %rax

	adcx	%rdx, %rbx

	movq	%xmm0, $out

	movq	%xmm1, %rbp

	movq	128(%rsp), %rdx		# pull $n0

	movq	(%rsp), %r8

	movq	8(%rsp), %r9

	movq	16(%rsp), %r10

	movq	24(%rsp), %r11

	movq	32(%rsp), %r12

	movq	40(%rsp), %r13

	movq	48(%rsp), %r14

	movq	56(%rsp), %r15

	movq	%rax, 112(%rsp)

	movq	%rbx, 120(%rsp)

	call	__rsaz_512_reducex

	addq	64(%rsp), %r8

	adcq	72(%rsp), %r9

	adcq	80(%rsp), %r10

	adcq	88(%rsp), %r11

	adcq	96(%rsp), %r12

	adcq	104(%rsp), %r13

	adcq	112(%rsp), %r14

	adcq	120(%rsp), %r15

	sbbq	%rcx, %rcx

	call	__rsaz_512_subtract

	movq	%r8, %rdx

	movq	%r9, %rax

	movl	128+8(%rsp), $times

	movq	$out, $inp

	decl	$times

	jnz	.Loop_sqrx

.Lsqr_tail:

___

}

$code.=<<___;

	leaq	128+24+48(%rsp), %rax

.cfi_def_cfa	%rax,8

	movq	-48(%rax), %r15

.cfi_restore	%r15

	movq	-40(%rax), %r14

.cfi_restore	%r14

	movq	-32(%rax), %r13

.cfi_restore	%r13

	movq	-24(%rax), %r12

.cfi_restore	%r12

	movq	-16(%rax), %rbp

.cfi_restore	%rbp

	movq	-8(%rax), %rbx

.cfi_restore	%rbx

	leaq	(%rax), %rsp

.cfi_def_cfa_register	%rsp

.Lsqr_epilogue:

	ret

.cfi_endproc

.size	rsaz_512_sqr,.-rsaz_512_sqr

___

}

{

my ($out,$ap,$bp,$mod,$n0) = ("%rdi","%rsi","%rdx","%rcx","%r8");

$code.=<<___;

.globl	rsaz_512_mul

.type	rsaz_512_mul,\@function,5

.align	32

rsaz_512_mul:

.cfi_startproc

	push	%rbx

.cfi_push	%rbx

	push	%rbp

.cfi_push	%rbp

	push	%r12

.cfi_push	%r12

	push	%r13

.cfi_push	%r13

	push	%r14

.cfi_push	%r14

	push	%r15

.cfi_push	%r15

	subq	\$128+24, %rsp

.cfi_adjust_cfa_offset	128+24

.Lmul_body:

	movq	$out, %xmm0		# off-load arguments

	movq	$mod, %xmm1

	movq	$n0, 128(%rsp)

___

$code.=<<___ if ($addx);

	movl	\$0x80100,%r11d

	andl	OPENSSL_ia32cap_P+8(%rip),%r11d

	cmpl	\$0x80100,%r11d		# check for MULX and ADO/CX

	je	.Lmulx

___

$code.=<<___;

	movq	($bp), %rbx		# pass b[0]

	movq	$bp, %rbp		# pass argument

	call	__rsaz_512_mul

	movq	%xmm0, $out

	movq	%xmm1, %rbp

	movq	(%rsp), %r8

	movq	8(%rsp), %r9

	movq	16(%rsp), %r10

	movq	24(%rsp), %r11

	movq	32(%rsp), %r12

	movq	40(%rsp), %r13

	movq	48(%rsp), %r14

	movq	56(%rsp), %r15

	call	__rsaz_512_reduce

___

$code.=<<___ if ($addx);

	jmp	.Lmul_tail

.align	32

.Lmulx:

	movq	$bp, %rbp		# pass argument

	movq	($bp), %rdx		# pass b[0]

	call	__rsaz_512_mulx

	movq	%xmm0, $out

	movq	%xmm1, %rbp

	movq	128(%rsp), %rdx		# pull $n0

	movq	(%rsp), %r8

	movq	8(%rsp), %r9

	movq	16(%rsp), %r10

	movq	24(%rsp), %r11

	movq	32(%rsp), %r12

	movq	40(%rsp), %r13

	movq	48(%rsp), %r14

	movq	56(%rsp), %r15