crypto/aes: use s390x KMA instruction for AES-GCM if available

Adds support for the cipher message with authentication (KMA)
instruction added in message-security-assist extension 8. This
instruction encapsulates most of the operations required for
AES-GCM and is faster than executing the operations independently.

name          old speed      new speed       delta
AESGCMSeal1K  1.96GB/s ± 0%   6.79GB/s ± 0%  +246.47%  (p=0.000 n=8+10)
AESGCMOpen1K  1.85GB/s ± 0%   5.76GB/s ± 0%  +211.18%  (p=0.000 n=10+10)
AESGCMSign8K  12.0GB/s ± 0%   14.5GB/s ± 0%   +20.43%  (p=0.000 n=10+8)
AESGCMSeal8K  3.75GB/s ± 0%  14.16GB/s ± 0%  +277.57%  (p=0.000 n=9+10)
AESGCMOpen8K  3.70GB/s ± 0%  13.57GB/s ± 0%  +266.50%  (p=0.000 n=10+9)

Change-Id: I57c46573fc5a0bd63c32ce5cba6e37cab85e3de6
Reviewed-on: https://go-review.googlesource.com/73550
Run-TryBot: Michael Munday <[email protected]>
Reviewed-by: Bill O'Farrell <[email protected]>
Reviewed-by: Volodymyr Paprotski <[email protected]>
Reviewed-by: Adam Langley <[email protected]>
TryBot-Result: Gobot Gobot <[email protected]>

src/crypto/aes/asm_s390x.s

@@ -150,3 +150,65 @@ loop:
 	MVC     $16, (R1), (R8)
 	MOVD	$0, R0
 	RET
+
+// func supportsKMA() bool
+TEXT ·supportsKMA(SB),NOSPLIT,$24-1
+	MOVD	$tmp-24(SP), R1
+	MOVD	$2, R0       // store 24-bytes
+	XC	$24, (R1), (R1)
+	WORD	$0xb2b01000  // STFLE (R1)
+	MOVWZ	16(R1), R2
+	ANDW	$(1<<13), R2 // test bit 146 (message-security-assist 8)
+	BEQ	no
+
+	MOVD	$0, R0       // KMA-Query
+	XC	$16, (R1), (R1)
+	WORD	$0xb9296024  // kma %r6,%r2,%r4
+	MOVWZ	(R1), R2
+	WORD	$0xa7213800  // TMLL R2, $0x3800
+	BVS	yes
+no:
+	MOVB	$0, ret+0(FP)
+	RET
+yes:
+	MOVB	$1, ret+0(FP)
+	RET
+
+// func kmaGCM(fn code, key, dst, src, aad []byte, tag *[16]byte, cnt *gcmCount)
+TEXT ·kmaGCM(SB),NOSPLIT,$112-120
+	MOVD	fn+0(FP), R0
+	MOVD	$params-112(SP), R1
+
+	// load ptr/len pairs
+	LMG	dst+32(FP), R2, R3 // R2=base R3=len
+	LMG	src+56(FP), R4, R5 // R4=base R5=len
+	LMG	aad+80(FP), R6, R7 // R6=base R7=len
+
+	// setup parameters
+	MOVD	cnt+112(FP), R8
+	XC	$12, (R1), (R1)     // reserved
+	MVC	$4, 12(R8), 12(R1)  // set chain value
+	MVC	$16, (R8), 64(R1)   // set initial counter value
+	XC	$32, 16(R1), 16(R1) // set hash subkey and tag
+	SLD	$3, R7, R12
+	MOVD	R12, 48(R1)         // set total AAD length
+	SLD	$3, R5, R12
+	MOVD	R12, 56(R1)         // set total plaintext/ciphertext length
+
+	LMG	key+8(FP), R8, R9   // R8=base R9=len
+	MVC	$16, (R8), 80(R1)   // set key
+	CMPBEQ	R9, $16, kma
+	MVC	$8, 16(R8), 96(R1)
+	CMPBEQ	R9, $24, kma
+	MVC	$8, 24(R8), 104(R1)
+
+kma:
+	WORD	$0xb9296024 // kma %r6,%r2,%r4
+	BVS	kma
+
+	MOVD	tag+104(FP), R2
+	MVC	$16, 16(R1), 0(R2) // copy tag to output
+	MOVD	cnt+112(FP), R8
+	MVC	$4, 12(R1), 12(R8) // update counter value
+
+	RET

src/crypto/aes/gcm_s390x.go

@@ -10,6 +10,11 @@ import (
 	"errors"
 )
 
+// This file contains two implementations of AES-GCM. The first implementation
+// (gcmAsm) uses the KMCTR instruction to encrypt using AES in counter mode and
+// the KIMD instruction for GHASH. The second implementation (gcmKMA) uses the
+// newer KMA instruction which performs both operations.
+
 // gcmCount represents a 16-byte big-endian count value.
 type gcmCount [16]byte
 
@@ -71,12 +76,16 @@ var _ gcmAble = (*aesCipherAsm)(nil)
 func (c *aesCipherAsm) NewGCM(nonceSize int) (cipher.AEAD, error) {
 	var hk gcmHashKey
 	c.Encrypt(hk[:], hk[:])
-	g := &gcmAsm{
+	g := gcmAsm{
 		block:     c,
 		hashKey:   hk,
 		nonceSize: nonceSize,
 	}
-	return g, nil
+	if hasKMA {
+		g := gcmKMA{g}
+		return &g, nil
+	}
+	return &g, nil
 }
 
 func (g *gcmAsm) NonceSize() int {
@@ -268,3 +277,90 @@ func (g *gcmAsm) Open(dst, nonce, ciphertext, data []byte) ([]byte, error) {
 	g.counterCrypt(out, ciphertext, &counter)
 	return ret, nil
 }
+
+// supportsKMA reports whether the message-security-assist 8 facility is available.
+// This function call may be expensive so hasKMA should be queried instead.
+func supportsKMA() bool
+
+// hasKMA contains the result of supportsKMA.
+var hasKMA = supportsKMA()
+
+// gcmKMA implements the cipher.AEAD interface using the KMA instruction. It should
+// only be used if hasKMA is true.
+type gcmKMA struct {
+	gcmAsm
+}
+
+// flags for the KMA instruction
+const (
+	kmaHS      = 1 << 10 // hash subkey supplied
+	kmaLAAD    = 1 << 9  // last series of additional authenticated data
+	kmaLPC     = 1 << 8  // last series of plaintext or ciphertext blocks
+	kmaDecrypt = 1 << 7  // decrypt
+)
+
+// kmaGCM executes the encryption or decryption operation given by fn. The tag
+// will be calculated and written to tag. cnt should contain the current
+// counter state and will be overwritten with the updated counter state.
+// TODO(mundaym): could pass in hash subkey
+//go:noescape
+func kmaGCM(fn code, key, dst, src, aad []byte, tag *[16]byte, cnt *gcmCount)
+
+// Seal encrypts and authenticates plaintext. See the cipher.AEAD interface for
+// details.
+func (g *gcmKMA) Seal(dst, nonce, plaintext, data []byte) []byte {
+	if len(nonce) != g.nonceSize {
+		panic("cipher: incorrect nonce length given to GCM")
+	}
+	if uint64(len(plaintext)) > ((1<<32)-2)*BlockSize {
+		panic("cipher: message too large for GCM")
+	}
+
+	ret, out := sliceForAppend(dst, len(plaintext)+gcmTagSize)
+
+	counter := g.deriveCounter(nonce)
+	fc := g.block.function | kmaLAAD | kmaLPC
+
+	var tag [gcmTagSize]byte
+	kmaGCM(fc, g.block.key, out[:len(plaintext)], plaintext, data, &tag, &counter)
+	copy(out[len(plaintext):], tag[:])
+
+	return ret
+}
+
+// Open authenticates and decrypts ciphertext. See the cipher.AEAD interface
+// for details.
+func (g *gcmKMA) Open(dst, nonce, ciphertext, data []byte) ([]byte, error) {
+	if len(nonce) != g.nonceSize {
+		panic("cipher: incorrect nonce length given to GCM")
+	}
+	if len(ciphertext) < gcmTagSize {
+		return nil, errOpen
+	}
+	if uint64(len(ciphertext)) > ((1<<32)-2)*BlockSize+gcmTagSize {
+		return nil, errOpen
+	}
+
+	tag := ciphertext[len(ciphertext)-gcmTagSize:]
+	ciphertext = ciphertext[:len(ciphertext)-gcmTagSize]
+	ret, out := sliceForAppend(dst, len(ciphertext))
+
+	counter := g.deriveCounter(nonce)
+	fc := g.block.function | kmaLAAD | kmaLPC | kmaDecrypt
+
+	var expectedTag [gcmTagSize]byte
+	kmaGCM(fc, g.block.key, out[:len(ciphertext)], ciphertext, data, &expectedTag, &counter)
+
+	if subtle.ConstantTimeCompare(expectedTag[:], tag) != 1 {
+		// The AESNI code decrypts and authenticates concurrently, and
+		// so overwrites dst in the event of a tag mismatch. That
+		// behavior is mimicked here in order to be consistent across
+		// platforms.
+		for i := range out {
+			out[i] = 0
+		}
+		return nil, errOpen
+	}
+
+	return ret, nil
+}