morus1280-avx2-asm.S

/*
 * AVX2 implementation of MORUS-1280
 *
 * Copyright (c) 2017-2018 Ondrej Mosnacek <omosnacek@gmail.com>
 * Copyright (C) 2017-2018 Red Hat, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License version 2 as published
 * by the Free Software Foundation.
 */

#include <linux/linkage.h>
#include <asm/frame.h>

#define SHUFFLE_MASK(i0, i1, i2, i3) \
	(i0 | (i1 << 2) | (i2 << 4) | (i3 << 6))

#define MASK1 SHUFFLE_MASK(3, 0, 1, 2)
#define MASK2 SHUFFLE_MASK(2, 3, 0, 1)
#define MASK3 SHUFFLE_MASK(1, 2, 3, 0)

#define STATE0		%ymm0
#define STATE0_LOW	%xmm0
#define STATE1		%ymm1
#define STATE2		%ymm2
#define STATE3		%ymm3
#define STATE4		%ymm4
#define KEY		%ymm5
#define MSG		%ymm5
#define MSG_LOW		%xmm5
#define T0		%ymm6
#define T0_LOW		%xmm6
#define T1		%ymm7

.section .rodata.cst32.morus1280_const, "aM", @progbits, 32
.align 32
.Lmorus1280_const:
	.byte 0x00, 0x01, 0x01, 0x02, 0x03, 0x05, 0x08, 0x0d
	.byte 0x15, 0x22, 0x37, 0x59, 0x90, 0xe9, 0x79, 0x62
	.byte 0xdb, 0x3d, 0x18, 0x55, 0x6d, 0xc2, 0x2f, 0xf1
	.byte 0x20, 0x11, 0x31, 0x42, 0x73, 0xb5, 0x28, 0xdd

.section .rodata.cst32.morus1280_counter, "aM", @progbits, 32
.align 32
.Lmorus1280_counter:
	.byte 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07
	.byte 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f
	.byte 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17
	.byte 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f

.text

.macro morus1280_round s0, s1, s2, s3, s4, b, w
	vpand \s1, \s2, T0
	vpxor T0, \s0, \s0
	vpxor \s3, \s0, \s0
	vpsllq $\b, \s0, T0
	vpsrlq $(64 - \b), \s0, \s0
	vpxor T0, \s0, \s0
	vpermq $\w, \s3, \s3
.endm

/*
 * __morus1280_update: internal ABI
 * input:
 *   STATE[0-4] - input state
 *   MSG        - message block
 * output:
 *   STATE[0-4] - output state
 * changed:
 *   T0
 */
__morus1280_update:
	morus1280_round STATE0, STATE1, STATE2, STATE3, STATE4, 13, MASK1
	vpxor MSG, STATE1, STATE1
	morus1280_round STATE1, STATE2, STATE3, STATE4, STATE0, 46, MASK2
	vpxor MSG, STATE2, STATE2
	morus1280_round STATE2, STATE3, STATE4, STATE0, STATE1, 38, MASK3
	vpxor MSG, STATE3, STATE3
	morus1280_round STATE3, STATE4, STATE0, STATE1, STATE2,  7, MASK2
	vpxor MSG, STATE4, STATE4
	morus1280_round STATE4, STATE0, STATE1, STATE2, STATE3,  4, MASK1
	ret
ENDPROC(__morus1280_update)

/*
 * __morus1280_update_zero: internal ABI
 * input:
 *   STATE[0-4] - input state
 * output:
 *   STATE[0-4] - output state
 * changed:
 *   T0
 */
__morus1280_update_zero:
	morus1280_round STATE0, STATE1, STATE2, STATE3, STATE4, 13, MASK1
	morus1280_round STATE1, STATE2, STATE3, STATE4, STATE0, 46, MASK2
	morus1280_round STATE2, STATE3, STATE4, STATE0, STATE1, 38, MASK3
	morus1280_round STATE3, STATE4, STATE0, STATE1, STATE2,  7, MASK2
	morus1280_round STATE4, STATE0, STATE1, STATE2, STATE3,  4, MASK1
	ret
ENDPROC(__morus1280_update_zero)

/*
 * __load_partial: internal ABI
 * input:
 *   %rsi - src
 *   %rcx - bytes
 * output:
 *   MSG  - message block
 * changed:
 *   %r8
 *   %r9
 */
__load_partial:
	xor %r9d, %r9d
	vpxor MSG, MSG, MSG

	mov %rcx, %r8
	and $0x1, %r8
	jz .Lld_partial_1

	mov %rcx, %r8
	and $0x1E, %r8
	add %rsi, %r8
	mov (%r8), %r9b

.Lld_partial_1:
	mov %rcx, %r8
	and $0x2, %r8
	jz .Lld_partial_2

	mov %rcx, %r8
	and $0x1C, %r8
	add %rsi, %r8
	shl $16, %r9
	mov (%r8), %r9w

.Lld_partial_2:
	mov %rcx, %r8
	and $0x4, %r8
	jz .Lld_partial_4

	mov %rcx, %r8
	and $0x18, %r8
	add %rsi, %r8
	shl $32, %r9
	mov (%r8), %r8d
	xor %r8, %r9

.Lld_partial_4:
	movq %r9, MSG_LOW

	mov %rcx, %r8
	and $0x8, %r8
	jz .Lld_partial_8

	mov %rcx, %r8
	and $0x10, %r8
	add %rsi, %r8
	pshufd $MASK2, MSG_LOW, MSG_LOW
	pinsrq $0, (%r8), MSG_LOW

.Lld_partial_8:
	mov %rcx, %r8
	and $0x10, %r8
	jz .Lld_partial_16

	vpermq $MASK2, MSG, MSG
	movdqu (%rsi), MSG_LOW

.Lld_partial_16:
	ret
ENDPROC(__load_partial)

/*
 * __store_partial: internal ABI
 * input:
 *   %rdx - dst
 *   %rcx - bytes
 * output:
 *   T0   - message block
 * changed:
 *   %r8
 *   %r9
 *   %r10
 */
__store_partial:
	mov %rcx, %r8
	mov %rdx, %r9

	cmp $16, %r8
	jl .Lst_partial_16

	movdqu T0_LOW, (%r9)
	vpermq $MASK2, T0, T0

	sub $16, %r8
	add $16, %r9

.Lst_partial_16:
	movq T0_LOW, %r10

	cmp $8, %r8
	jl .Lst_partial_8

	mov %r10, (%r9)
	pextrq $1, T0_LOW, %r10

	sub $8, %r8
	add $8, %r9

.Lst_partial_8:
	cmp $4, %r8
	jl .Lst_partial_4

	mov %r10d, (%r9)
	shr $32, %r10

	sub $4, %r8
	add $4, %r9

.Lst_partial_4:
	cmp $2, %r8
	jl .Lst_partial_2

	mov %r10w, (%r9)
	shr $16, %r10

	sub $2, %r8
	add $2, %r9

.Lst_partial_2:
	cmp $1, %r8
	jl .Lst_partial_1

	mov %r10b, (%r9)

.Lst_partial_1:
	ret
ENDPROC(__store_partial)

/*
 * void crypto_morus1280_avx2_init(void *state, const void *key,
 *                                 const void *iv);
 */
ENTRY(crypto_morus1280_avx2_init)
	FRAME_BEGIN

	/* load IV: */
	vpxor STATE0, STATE0, STATE0
	movdqu (%rdx), STATE0_LOW
	/* load key: */
	vmovdqu (%rsi), KEY
	vmovdqa KEY, STATE1
	/* load all ones: */
	vpcmpeqd STATE2, STATE2, STATE2
	/* load all zeros: */
	vpxor STATE3, STATE3, STATE3
	/* load the constant: */
	vmovdqa .Lmorus1280_const, STATE4

	/* update 16 times with zero: */
	call __morus1280_update_zero
	call __morus1280_update_zero
	call __morus1280_update_zero
	call __morus1280_update_zero
	call __morus1280_update_zero
	call __morus1280_update_zero
	call __morus1280_update_zero
	call __morus1280_update_zero
	call __morus1280_update_zero
	call __morus1280_update_zero
	call __morus1280_update_zero
	call __morus1280_update_zero
	call __morus1280_update_zero
	call __morus1280_update_zero
	call __morus1280_update_zero
	call __morus1280_update_zero

	/* xor-in the key again after updates: */
	vpxor KEY, STATE1, STATE1

	/* store the state: */
	vmovdqu STATE0, (0 * 32)(%rdi)
	vmovdqu STATE1, (1 * 32)(%rdi)
	vmovdqu STATE2, (2 * 32)(%rdi)
	vmovdqu STATE3, (3 * 32)(%rdi)
	vmovdqu STATE4, (4 * 32)(%rdi)

	FRAME_END
	ret
ENDPROC(crypto_morus1280_avx2_init)

/*
 * void crypto_morus1280_avx2_ad(void *state, const void *data,
 *                               unsigned int length);
 */
ENTRY(crypto_morus1280_avx2_ad)
	FRAME_BEGIN

	cmp $32, %rdx
	jb .Lad_out

	/* load the state: */
	vmovdqu (0 * 32)(%rdi), STATE0
	vmovdqu (1 * 32)(%rdi), STATE1
	vmovdqu (2 * 32)(%rdi), STATE2
	vmovdqu (3 * 32)(%rdi), STATE3
	vmovdqu (4 * 32)(%rdi), STATE4

	mov %rsi,  %r8
	and $0x1F, %r8
	jnz .Lad_u_loop

.align 4
.Lad_a_loop:
	vmovdqa (%rsi), MSG
	call __morus1280_update
	sub $32, %rdx
	add $32, %rsi
	cmp $32, %rdx
	jge .Lad_a_loop

	jmp .Lad_cont
.align 4
.Lad_u_loop:
	vmovdqu (%rsi), MSG
	call __morus1280_update
	sub $32, %rdx
	add $32, %rsi
	cmp $32, %rdx
	jge .Lad_u_loop

.Lad_cont:
	/* store the state: */
	vmovdqu STATE0, (0 * 32)(%rdi)
	vmovdqu STATE1, (1 * 32)(%rdi)
	vmovdqu STATE2, (2 * 32)(%rdi)
	vmovdqu STATE3, (3 * 32)(%rdi)
	vmovdqu STATE4, (4 * 32)(%rdi)

.Lad_out:
	FRAME_END
	ret
ENDPROC(crypto_morus1280_avx2_ad)

/*
 * void crypto_morus1280_avx2_enc(void *state, const void *src, void *dst,
 *                                unsigned int length);
 */
ENTRY(crypto_morus1280_avx2_enc)
	FRAME_BEGIN

	cmp $32, %rcx
	jb .Lenc_out

	/* load the state: */
	vmovdqu (0 * 32)(%rdi), STATE0
	vmovdqu (1 * 32)(%rdi), STATE1
	vmovdqu (2 * 32)(%rdi), STATE2
	vmovdqu (3 * 32)(%rdi), STATE3
	vmovdqu (4 * 32)(%rdi), STATE4

	mov %rsi,  %r8
	or  %rdx,  %r8
	and $0x1F, %r8
	jnz .Lenc_u_loop

.align 4
.Lenc_a_loop:
	vmovdqa (%rsi), MSG
	vmovdqa MSG, T0
	vpxor STATE0, T0, T0
	vpermq $MASK3, STATE1, T1
	vpxor T1, T0, T0
	vpand STATE2, STATE3, T1
	vpxor T1, T0, T0
	vmovdqa T0, (%rdx)

	call __morus1280_update
	sub $32, %rcx
	add $32, %rsi
	add $32, %rdx
	cmp $32, %rcx
	jge .Lenc_a_loop

	jmp .Lenc_cont
.align 4
.Lenc_u_loop:
	vmovdqu (%rsi), MSG
	vmovdqa MSG, T0
	vpxor STATE0, T0, T0
	vpermq $MASK3, STATE1, T1
	vpxor T1, T0, T0
	vpand STATE2, STATE3, T1
	vpxor T1, T0, T0
	vmovdqu T0, (%rdx)

	call __morus1280_update
	sub $32, %rcx
	add $32, %rsi
	add $32, %rdx
	cmp $32, %rcx
	jge .Lenc_u_loop

.Lenc_cont:
	/* store the state: */
	vmovdqu STATE0, (0 * 32)(%rdi)
	vmovdqu STATE1, (1 * 32)(%rdi)
	vmovdqu STATE2, (2 * 32)(%rdi)
	vmovdqu STATE3, (3 * 32)(%rdi)
	vmovdqu STATE4, (4 * 32)(%rdi)

.Lenc_out:
	FRAME_END
	ret
ENDPROC(crypto_morus1280_avx2_enc)

/*
 * void crypto_morus1280_avx2_enc_tail(void *state, const void *src, void *dst,
 *                                     unsigned int length);
 */
ENTRY(crypto_morus1280_avx2_enc_tail)
	FRAME_BEGIN

	/* load the state: */
	vmovdqu (0 * 32)(%rdi), STATE0
	vmovdqu (1 * 32)(%rdi), STATE1
	vmovdqu (2 * 32)(%rdi), STATE2
	vmovdqu (3 * 32)(%rdi), STATE3
	vmovdqu (4 * 32)(%rdi), STATE4

	/* encrypt message: */
	call __load_partial

	vmovdqa MSG, T0
	vpxor STATE0, T0, T0
	vpermq $MASK3, STATE1, T1
	vpxor T1, T0, T0
	vpand STATE2, STATE3, T1
	vpxor T1, T0, T0

	call __store_partial

	call __morus1280_update

	/* store the state: */
	vmovdqu STATE0, (0 * 32)(%rdi)
	vmovdqu STATE1, (1 * 32)(%rdi)
	vmovdqu STATE2, (2 * 32)(%rdi)
	vmovdqu STATE3, (3 * 32)(%rdi)
	vmovdqu STATE4, (4 * 32)(%rdi)

	FRAME_END
	ret
ENDPROC(crypto_morus1280_avx2_enc_tail)

/*
 * void crypto_morus1280_avx2_dec(void *state, const void *src, void *dst,
 *                                unsigned int length);
 */
ENTRY(crypto_morus1280_avx2_dec)
	FRAME_BEGIN

	cmp $32, %rcx
	jb .Ldec_out

	/* load the state: */
	vmovdqu (0 * 32)(%rdi), STATE0
	vmovdqu (1 * 32)(%rdi), STATE1
	vmovdqu (2 * 32)(%rdi), STATE2
	vmovdqu (3 * 32)(%rdi), STATE3
	vmovdqu (4 * 32)(%rdi), STATE4

	mov %rsi,  %r8
	or  %rdx,  %r8
	and $0x1F, %r8
	jnz .Ldec_u_loop

.align 4
.Ldec_a_loop:
	vmovdqa (%rsi), MSG
	vpxor STATE0, MSG, MSG
	vpermq $MASK3, STATE1, T0
	vpxor T0, MSG, MSG
	vpand STATE2, STATE3, T0
	vpxor T0, MSG, MSG
	vmovdqa MSG, (%rdx)

	call __morus1280_update
	sub $32, %rcx
	add $32, %rsi
	add $32, %rdx
	cmp $32, %rcx
	jge .Ldec_a_loop

	jmp .Ldec_cont
.align 4
.Ldec_u_loop:
	vmovdqu (%rsi), MSG
	vpxor STATE0, MSG, MSG
	vpermq $MASK3, STATE1, T0
	vpxor T0, MSG, MSG
	vpand STATE2, STATE3, T0
	vpxor T0, MSG, MSG
	vmovdqu MSG, (%rdx)

	call __morus1280_update
	sub $32, %rcx
	add $32, %rsi
	add $32, %rdx
	cmp $32, %rcx
	jge .Ldec_u_loop

.Ldec_cont:
	/* store the state: */
	vmovdqu STATE0, (0 * 32)(%rdi)
	vmovdqu STATE1, (1 * 32)(%rdi)
	vmovdqu STATE2, (2 * 32)(%rdi)
	vmovdqu STATE3, (3 * 32)(%rdi)
	vmovdqu STATE4, (4 * 32)(%rdi)

.Ldec_out:
	FRAME_END
	ret
ENDPROC(crypto_morus1280_avx2_dec)

/*
 * void crypto_morus1280_avx2_dec_tail(void *state, const void *src, void *dst,
 *                                     unsigned int length);
 */
ENTRY(crypto_morus1280_avx2_dec_tail)
	FRAME_BEGIN

	/* load the state: */
	vmovdqu (0 * 32)(%rdi), STATE0
	vmovdqu (1 * 32)(%rdi), STATE1
	vmovdqu (2 * 32)(%rdi), STATE2
	vmovdqu (3 * 32)(%rdi), STATE3
	vmovdqu (4 * 32)(%rdi), STATE4

	/* decrypt message: */
	call __load_partial

	vpxor STATE0, MSG, MSG
	vpermq $MASK3, STATE1, T0
	vpxor T0, MSG, MSG
	vpand STATE2, STATE3, T0
	vpxor T0, MSG, MSG
	vmovdqa MSG, T0

	call __store_partial

	/* mask with byte count: */
	movq %rcx, T0_LOW
	vpbroadcastb T0_LOW, T0
	vmovdqa .Lmorus1280_counter, T1
	vpcmpgtb T1, T0, T0
	vpand T0, MSG, MSG

	call __morus1280_update

	/* store the state: */
	vmovdqu STATE0, (0 * 32)(%rdi)
	vmovdqu STATE1, (1 * 32)(%rdi)
	vmovdqu STATE2, (2 * 32)(%rdi)
	vmovdqu STATE3, (3 * 32)(%rdi)
	vmovdqu STATE4, (4 * 32)(%rdi)

	FRAME_END
	ret
ENDPROC(crypto_morus1280_avx2_dec_tail)

/*
 * void crypto_morus1280_avx2_final(void *state, void *tag_xor,
 *                                  u64 assoclen, u64 cryptlen);
 */
ENTRY(crypto_morus1280_avx2_final)
	FRAME_BEGIN

	/* load the state: */
	vmovdqu (0 * 32)(%rdi), STATE0
	vmovdqu (1 * 32)(%rdi), STATE1
	vmovdqu (2 * 32)(%rdi), STATE2
	vmovdqu (3 * 32)(%rdi), STATE3
	vmovdqu (4 * 32)(%rdi), STATE4

	/* xor state[0] into state[4]: */
	vpxor STATE0, STATE4, STATE4

	/* prepare length block: */
	vpxor MSG, MSG, MSG
	vpinsrq $0, %rdx, MSG_LOW, MSG_LOW
	vpinsrq $1, %rcx, MSG_LOW, MSG_LOW
	vpsllq $3, MSG, MSG /* multiply by 8 (to get bit count) */

	/* update state: */
	call __morus1280_update
	call __morus1280_update
	call __morus1280_update
	call __morus1280_update
	call __morus1280_update
	call __morus1280_update
	call __morus1280_update
	call __morus1280_update
	call __morus1280_update
	call __morus1280_update

	/* xor tag: */
	vmovdqu (%rsi), MSG

	vpxor STATE0, MSG, MSG
	vpermq $MASK3, STATE1, T0
	vpxor T0, MSG, MSG
	vpand STATE2, STATE3, T0
	vpxor T0, MSG, MSG
	vmovdqu MSG, (%rsi)

	FRAME_END
	ret
ENDPROC(crypto_morus1280_avx2_final)