#! /usr/bin/env perl

# Copyright 2011-2018 The OpenSSL Project Authors. All Rights Reserved.

#

# Licensed under the OpenSSL license (the "License").  You may not use

# this file except in compliance with the License.  You can obtain a copy

# in the file LICENSE in the source distribution or at

# https://www.openssl.org/source/license.html

# ====================================================================

# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL

# project. The module is, however, dual licensed under OpenSSL and

# CRYPTOGAMS licenses depending on where you obtain it. For further

# details see http://www.openssl.org/~appro/cryptogams/.

# ====================================================================

# September 2011

#

# Assembler helpers for Padlock engine. See even e_padlock-x86.pl for

# details.

$flavour = shift;

$output  = shift;

if ($flavour =~ /\./) { $output = $flavour; undef $flavour; }

$win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/);

$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;

( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or

( $xlate="${dir}../../crypto/perlasm/x86_64-xlate.pl" and -f $xlate) or

die "can't locate x86_64-xlate.pl";

open OUT,"| \"$^X\" \"$xlate\" $flavour \"$output\"";

*STDOUT=*OUT;

$code=".text\n";

%PADLOCK_PREFETCH=(ecb=>128, cbc=>64, ctr32=>32);	# prefetch errata

$PADLOCK_CHUNK=512;	# Must be a power of 2 between 32 and 2^20

$ctx="%rdx";

$out="%rdi";

$inp="%rsi";

$len="%rcx";

$chunk="%rbx";

($arg1,$arg2,$arg3,$arg4)=$win64?("%rcx","%rdx","%r8", "%r9") : # Win64 order

                                 ("%rdi","%rsi","%rdx","%rcx"); # Unix order

$code.=<<___;

.globl	padlock_capability

.type	padlock_capability,\@abi-omnipotent

.align	16

padlock_capability:

	mov	%rbx,%r8

	xor	%eax,%eax

	cpuid

	xor	%eax,%eax

	cmp	\$`"0x".unpack("H*",'tneC')`,%ebx

	jne	.Lzhaoxin

	cmp	\$`"0x".unpack("H*",'Hrua')`,%edx

	jne	.Lnoluck

	cmp	\$`"0x".unpack("H*",'slua')`,%ecx

	jne	.Lnoluck

	jmp	.LzhaoxinEnd

.Lzhaoxin:

	cmp	\$`"0x".unpack("H*",'hS  ')`,%ebx

	jne	.Lnoluck

	cmp	\$`"0x".unpack("H*",'hgna')`,%edx

	jne	.Lnoluck

	cmp	\$`"0x".unpack("H*",'  ia')`,%ecx

	jne	.Lnoluck

.LzhaoxinEnd:

	mov	\$0xC0000000,%eax

	cpuid

	mov	%eax,%edx

	xor	%eax,%eax

	cmp	\$0xC0000001,%edx

	jb	.Lnoluck

	mov	\$0xC0000001,%eax

	cpuid

	mov	%edx,%eax

	and	\$0xffffffef,%eax

	or	\$0x10,%eax		# set Nano bit#4

.Lnoluck:

	mov	%r8,%rbx

	ret

.size	padlock_capability,.-padlock_capability

.globl	padlock_key_bswap

.type	padlock_key_bswap,\@abi-omnipotent,0

.align	16

padlock_key_bswap:

	mov	240($arg1),%edx

.Lbswap_loop:

	mov	($arg1),%eax

	bswap	%eax

	mov	%eax,($arg1)

	lea	4($arg1),$arg1

	sub	\$1,%edx

	jnz	.Lbswap_loop

	ret

.size	padlock_key_bswap,.-padlock_key_bswap

.globl	padlock_verify_context

.type	padlock_verify_context,\@abi-omnipotent

.align	16

padlock_verify_context:

	mov	$arg1,$ctx

	pushf

	lea	.Lpadlock_saved_context(%rip),%rax

	call	_padlock_verify_ctx

	lea	8(%rsp),%rsp

	ret

.size	padlock_verify_context,.-padlock_verify_context

.type	_padlock_verify_ctx,\@abi-omnipotent

.align	16

_padlock_verify_ctx:

	mov	8(%rsp),%r8

	bt	\$30,%r8

	jnc	.Lverified

	cmp	(%rax),$ctx

	je	.Lverified

	pushf

	popf

.Lverified:

	mov	$ctx,(%rax)

	ret

.size	_padlock_verify_ctx,.-_padlock_verify_ctx

.globl	padlock_reload_key

.type	padlock_reload_key,\@abi-omnipotent

.align	16

padlock_reload_key:

	pushf

	popf

	ret

.size	padlock_reload_key,.-padlock_reload_key

.globl	padlock_aes_block

.type	padlock_aes_block,\@function,3

.align	16

padlock_aes_block:

	mov	%rbx,%r8

	mov	\$1,$len

	lea	32($ctx),%rbx		# key

	lea	16($ctx),$ctx		# control word

	.byte	0xf3,0x0f,0xa7,0xc8	# rep xcryptecb

	mov	%r8,%rbx

	ret

.size	padlock_aes_block,.-padlock_aes_block

.globl	padlock_xstore

.type	padlock_xstore,\@function,2

.align	16

padlock_xstore:

	mov	%esi,%edx

	.byte	0x0f,0xa7,0xc0		# xstore

	ret

.size	padlock_xstore,.-padlock_xstore

.globl	padlock_sha1_oneshot

.type	padlock_sha1_oneshot,\@function,3

.align	16

padlock_sha1_oneshot:

	mov	%rdx,%rcx

	mov	%rdi,%rdx		# put aside %rdi

	movups	(%rdi),%xmm0		# copy-in context

	sub	\$128+8,%rsp

	mov	16(%rdi),%eax

	movaps	%xmm0,(%rsp)

	mov	%rsp,%rdi

	mov	%eax,16(%rsp)

	xor	%rax,%rax

	.byte	0xf3,0x0f,0xa6,0xc8	# rep xsha1

	movaps	(%rsp),%xmm0

	mov	16(%rsp),%eax

	add	\$128+8,%rsp

	movups	%xmm0,(%rdx)		# copy-out context

	mov	%eax,16(%rdx)

	ret

.size	padlock_sha1_oneshot,.-padlock_sha1_oneshot

.globl	padlock_sha1_blocks

.type	padlock_sha1_blocks,\@function,3

.align	16

padlock_sha1_blocks:

	mov	%rdx,%rcx

	mov	%rdi,%rdx		# put aside %rdi

	movups	(%rdi),%xmm0		# copy-in context

	sub	\$128+8,%rsp

	mov	16(%rdi),%eax

	movaps	%xmm0,(%rsp)

	mov	%rsp,%rdi

	mov	%eax,16(%rsp)

	mov	\$-1,%rax

	.byte	0xf3,0x0f,0xa6,0xc8	# rep xsha1

	movaps	(%rsp),%xmm0

	mov	16(%rsp),%eax

	add	\$128+8,%rsp

	movups	%xmm0,(%rdx)		# copy-out context

	mov	%eax,16(%rdx)

	ret

.size	padlock_sha1_blocks,.-padlock_sha1_blocks

.globl	padlock_sha256_oneshot

.type	padlock_sha256_oneshot,\@function,3

.align	16

padlock_sha256_oneshot:

	mov	%rdx,%rcx

	mov	%rdi,%rdx		# put aside %rdi

	movups	(%rdi),%xmm0		# copy-in context

	sub	\$128+8,%rsp

	movups	16(%rdi),%xmm1

	movaps	%xmm0,(%rsp)

	mov	%rsp,%rdi

	movaps	%xmm1,16(%rsp)

	xor	%rax,%rax

	.byte	0xf3,0x0f,0xa6,0xd0	# rep xsha256

	movaps	(%rsp),%xmm0

	movaps	16(%rsp),%xmm1

	add	\$128+8,%rsp

	movups	%xmm0,(%rdx)		# copy-out context

	movups	%xmm1,16(%rdx)

	ret

.size	padlock_sha256_oneshot,.-padlock_sha256_oneshot

.globl	padlock_sha256_blocks

.type	padlock_sha256_blocks,\@function,3

.align	16

padlock_sha256_blocks:

	mov	%rdx,%rcx

	mov	%rdi,%rdx		# put aside %rdi

	movups	(%rdi),%xmm0		# copy-in context

	sub	\$128+8,%rsp

	movups	16(%rdi),%xmm1

	movaps	%xmm0,(%rsp)

	mov	%rsp,%rdi

	movaps	%xmm1,16(%rsp)

	mov	\$-1,%rax

	.byte	0xf3,0x0f,0xa6,0xd0	# rep xsha256

	movaps	(%rsp),%xmm0

	movaps	16(%rsp),%xmm1

	add	\$128+8,%rsp

	movups	%xmm0,(%rdx)		# copy-out context

	movups	%xmm1,16(%rdx)

	ret

.size	padlock_sha256_blocks,.-padlock_sha256_blocks

.globl	padlock_sha512_blocks

.type	padlock_sha512_blocks,\@function,3

.align	16

padlock_sha512_blocks:

	mov	%rdx,%rcx

	mov	%rdi,%rdx		# put aside %rdi

	movups	(%rdi),%xmm0		# copy-in context

	sub	\$128+8,%rsp

	movups	16(%rdi),%xmm1

	movups	32(%rdi),%xmm2

	movups	48(%rdi),%xmm3

	movaps	%xmm0,(%rsp)

	mov	%rsp,%rdi

	movaps	%xmm1,16(%rsp)

	movaps	%xmm2,32(%rsp)

	movaps	%xmm3,48(%rsp)

	.byte	0xf3,0x0f,0xa6,0xe0	# rep xha512

	movaps	(%rsp),%xmm0

	movaps	16(%rsp),%xmm1

	movaps	32(%rsp),%xmm2

	movaps	48(%rsp),%xmm3

	add	\$128+8,%rsp

	movups	%xmm0,(%rdx)		# copy-out context

	movups	%xmm1,16(%rdx)

	movups	%xmm2,32(%rdx)

	movups	%xmm3,48(%rdx)

	ret

.size	padlock_sha512_blocks,.-padlock_sha512_blocks

___

sub generate_mode {

my ($mode,$opcode) = @_;

# int padlock_$mode_encrypt(void *out, const void *inp,

#		struct padlock_cipher_data *ctx, size_t len);

$code.=<<___;

.globl	padlock_${mode}_encrypt

.type	padlock_${mode}_encrypt,\@function,4

.align	16

padlock_${mode}_encrypt:

	push	%rbp

	push	%rbx

	xor	%eax,%eax

	test	\$15,$ctx

	jnz	.L${mode}_abort

	test	\$15,$len

	jnz	.L${mode}_abort

	lea	.Lpadlock_saved_context(%rip),%rax

	pushf

	cld

	call	_padlock_verify_ctx

	lea	16($ctx),$ctx		# control word

	xor	%eax,%eax

	xor	%ebx,%ebx

	testl	\$`1<<5`,($ctx)		# align bit in control word

	jnz	.L${mode}_aligned

	test	\$0x0f,$out

	setz	%al			# !out_misaligned

	test	\$0x0f,$inp

	setz	%bl			# !inp_misaligned

	test	%ebx,%eax

	jnz	.L${mode}_aligned

	neg	%rax

	mov	\$$PADLOCK_CHUNK,$chunk

	not	%rax			# out_misaligned?-1:0

	lea	(%rsp),%rbp

	cmp	$chunk,$len

	cmovc	$len,$chunk		# chunk=len>PADLOCK_CHUNK?PADLOCK_CHUNK:len

	and	$chunk,%rax		# out_misaligned?chunk:0

	mov	$len,$chunk

	neg	%rax

	and	\$$PADLOCK_CHUNK-1,$chunk	# chunk%=PADLOCK_CHUNK

	lea	(%rax,%rbp),%rsp

	mov	\$$PADLOCK_CHUNK,%rax

	cmovz	%rax,$chunk			# chunk=chunk?:PADLOCK_CHUNK

___

$code.=<<___				if ($mode eq "ctr32");

.L${mode}_reenter:

	mov	-4($ctx),%eax		# pull 32-bit counter

	bswap	%eax

	neg	%eax

	and	\$`$PADLOCK_CHUNK/16-1`,%eax

	mov	\$$PADLOCK_CHUNK,$chunk

	shl	\$4,%eax

	cmovz	$chunk,%rax

	cmp	%rax,$len

	cmova	%rax,$chunk		# don't let counter cross PADLOCK_CHUNK

	cmovbe	$len,$chunk

___

$code.=<<___				if ($PADLOCK_PREFETCH{$mode});

	cmp	$chunk,$len

	ja	.L${mode}_loop

	mov	$inp,%rax		# check if prefetch crosses page

	cmp	%rsp,%rbp

	cmove	$out,%rax

	add	$len,%rax

	neg	%rax

	and	\$0xfff,%rax		# distance to page boundary

	cmp	\$$PADLOCK_PREFETCH{$mode},%rax

	mov	\$-$PADLOCK_PREFETCH{$mode},%rax

	cmovae	$chunk,%rax		# mask=distance

	and	%rax,$chunk

	jz	.L${mode}_unaligned_tail

___

$code.=<<___;

	jmp	.L${mode}_loop

.align	16

.L${mode}_loop:

	cmp	$len,$chunk		# ctr32 artefact

	cmova	$len,$chunk		# ctr32 artefact

	mov	$out,%r8		# save parameters

	mov	$inp,%r9

	mov	$len,%r10

	mov	$chunk,$len

	mov	$chunk,%r11

	test	\$0x0f,$out		# out_misaligned

	cmovnz	%rsp,$out

	test	\$0x0f,$inp		# inp_misaligned

	jz	.L${mode}_inp_aligned

	shr	\$3,$len

	.byte	0xf3,0x48,0xa5		# rep movsq

	sub	$chunk,$out

	mov	$chunk,$len

	mov	$out,$inp

.L${mode}_inp_aligned:

	lea	-16($ctx),%rax		# ivp

	lea	16($ctx),%rbx		# key

	shr	\$4,$len

	.byte	0xf3,0x0f,0xa7,$opcode	# rep xcrypt*

___

$code.=<<___				if ($mode !~ /ecb|ctr/);

	movdqa	(%rax),%xmm0

	movdqa	%xmm0,-16($ctx)		# copy [or refresh] iv

___

$code.=<<___				if ($mode eq "ctr32");

	mov	-4($ctx),%eax		# pull 32-bit counter

	test	\$0xffff0000,%eax

	jnz	.L${mode}_no_carry

	bswap	%eax

	add	\$0x10000,%eax

	bswap	%eax

	mov	%eax,-4($ctx)

.L${mode}_no_carry:

___

$code.=<<___;

	mov	%r8,$out		# restore parameters

	mov	%r11,$chunk

	test	\$0x0f,$out

	jz	.L${mode}_out_aligned

	mov	$chunk,$len

	lea	(%rsp),$inp

	shr	\$3,$len

	.byte	0xf3,0x48,0xa5		# rep movsq

	sub	$chunk,$out

.L${mode}_out_aligned:

	mov	%r9,$inp

	mov	%r10,$len

	add	$chunk,$out

	add	$chunk,$inp

	sub	$chunk,$len

	mov	\$$PADLOCK_CHUNK,$chunk

___

					if (!$PADLOCK_PREFETCH{$mode}) {

$code.=<<___;

	jnz	.L${mode}_loop

___

					} else {

$code.=<<___;

	jz	.L${mode}_break

	cmp	$chunk,$len

	jae	.L${mode}_loop

___

$code.=<<___				if ($mode eq "ctr32");

	mov	$len,$chunk

	mov	$inp,%rax		# check if prefetch crosses page

	cmp	%rsp,%rbp

	cmove	$out,%rax

	add	$len,%rax

	neg	%rax

	and	\$0xfff,%rax		# distance to page boundary

	cmp	\$$PADLOCK_PREFETCH{$mode},%rax

	mov	\$-$PADLOCK_PREFETCH{$mode},%rax

	cmovae	$chunk,%rax

	and	%rax,$chunk

	jnz	.L${mode}_loop

___

$code.=<<___;

.L${mode}_unaligned_tail:

	xor	%eax,%eax

	cmp	%rsp,%rbp

	cmove	$len,%rax

	mov	$out,%r8		# save parameters

	mov	$len,$chunk

	sub	%rax,%rsp		# alloca

	shr	\$3,$len

	lea	(%rsp),$out

	.byte	0xf3,0x48,0xa5		# rep movsq

	mov	%rsp,$inp

	mov	%r8, $out		# restore parameters

	mov	$chunk,$len

	jmp	.L${mode}_loop

.align	16

.L${mode}_break:

___

					}

$code.=<<___;

	cmp	%rbp,%rsp

	je	.L${mode}_done

	pxor	%xmm0,%xmm0

	lea	(%rsp),%rax

.L${mode}_bzero:

	movaps	%xmm0,(%rax)

	lea	16(%rax),%rax

	cmp	%rax,%rbp

	ja	.L${mode}_bzero

.L${mode}_done:

	lea	(%rbp),%rsp

	jmp	.L${mode}_exit

.align	16

.L${mode}_aligned:

___

$code.=<<___				if ($mode eq "ctr32");

	mov	-4($ctx),%eax		# pull 32-bit counter

	bswap	%eax

	neg	%eax

	and	\$0xffff,%eax

	mov	\$`16*0x10000`,$chunk

	shl	\$4,%eax

	cmovz	$chunk,%rax

	cmp	%rax,$len

	cmova	%rax,$chunk		# don't let counter cross 2^16

	cmovbe	$len,$chunk

	jbe	.L${mode}_aligned_skip

.L${mode}_aligned_loop:

	mov	$len,%r10		# save parameters

	mov	$chunk,$len

	mov	$chunk,%r11

	lea	-16($ctx),%rax		# ivp

	lea	16($ctx),%rbx		# key

	shr	\$4,$len		# len/=AES_BLOCK_SIZE

	.byte	0xf3,0x0f,0xa7,$opcode	# rep xcrypt*

	mov	-4($ctx),%eax		# pull 32-bit counter

	bswap	%eax

	add	\$0x10000,%eax

	bswap	%eax

	mov	%eax,-4($ctx)

	mov	%r10,$len		# restore parameters

	sub	%r11,$len

	mov	\$`16*0x10000`,$chunk

	jz	.L${mode}_exit

	cmp	$chunk,$len

	jae	.L${mode}_aligned_loop

.L${mode}_aligned_skip:

___

$code.=<<___				if ($PADLOCK_PREFETCH{$mode});

	lea	($inp,$len),%rbp

	neg	%rbp

	and	\$0xfff,%rbp		# distance to page boundary

	xor	%eax,%eax

	cmp	\$$PADLOCK_PREFETCH{$mode},%rbp

	mov	\$$PADLOCK_PREFETCH{$mode}-1,%rbp

	cmovae	%rax,%rbp

	and	$len,%rbp		# remainder

	sub	%rbp,$len

	jz	.L${mode}_aligned_tail

___

$code.=<<___;

	lea	-16($ctx),%rax		# ivp

	lea	16($ctx),%rbx		# key

	shr	\$4,$len		# len/=AES_BLOCK_SIZE

	.byte	0xf3,0x0f,0xa7,$opcode	# rep xcrypt*

___

$code.=<<___				if ($mode !~ /ecb|ctr/);

	movdqa	(%rax),%xmm0

	movdqa	%xmm0,-16($ctx)		# copy [or refresh] iv

___

$code.=<<___				if ($PADLOCK_PREFETCH{$mode});

	test	%rbp,%rbp		# check remainder

	jz	.L${mode}_exit

.L${mode}_aligned_tail:

	mov	$out,%r8

	mov	%rbp,$chunk

	mov	%rbp,$len

	lea	(%rsp),%rbp

	sub	$len,%rsp

	shr	\$3,$len

	lea	(%rsp),$out

	.byte	0xf3,0x48,0xa5		# rep movsq

	lea	(%r8),$out

	lea	(%rsp),$inp

	mov	$chunk,$len

	jmp	.L${mode}_loop

___

$code.=<<___;

.L${mode}_exit:

	mov	\$1,%eax

	lea	8(%rsp),%rsp

.L${mode}_abort:

	pop	%rbx

	pop	%rbp

	ret

.size	padlock_${mode}_encrypt,.-padlock_${mode}_encrypt

___

}

&generate_mode("ecb",0xc8);

&generate_mode("cbc",0xd0);

&generate_mode("cfb",0xe0);

&generate_mode("ofb",0xe8);

&generate_mode("ctr32",0xd8);	# all 64-bit CPUs have working CTR...

$code.=<<___;

.asciz	"VIA Padlock x86_64 module, CRYPTOGAMS by <appro\@openssl.org>"

.align	16

.data

.align	8

.Lpadlock_saved_context:

	.quad	0

___

$code =~ s/\`([^\`]*)\`/eval($1)/gem;

print $code;

close STDOUT;