internal/chacha20: improve performance for ppc64le

Add asm implementation for chacha20 using vector instructions on ppc64le.
Below, the difference using the new code:

name               old speed     new speed      delta
ChaCha20/32-16     167MB/s ± 0%   129MB/s ± 0%   -22.60%  (p=0.008 n=5+5)
ChaCha20/63-16     308MB/s ± 0%   249MB/s ± 0%   -19.00%  (p=0.008 n=5+5)
ChaCha20/64-16     357MB/s ± 0%   251MB/s ± 0%   -29.57%  (p=0.008 n=5+5)
ChaCha20/256-16    398MB/s ± 0%  1199MB/s ± 0%  +201.20%  (p=0.008 n=5+5)
ChaCha20/1024-16   413MB/s ± 0%  1398MB/s ± 0%  +238.67%  (p=0.008 n=5+5)
ChaCha20/1350-16   395MB/s ± 0%  1189MB/s ± 0%  +200.71%  (p=0.008 n=5+5)
ChaCha20/65536-16  420MB/s ± 0%  1489MB/s ± 0%  +254.10%  (p=0.008 n=5+5)

Small sizes are slower due the fact that it always calculates using
256 bytes of key stream.

This change was originally from Paulo Flabiano Smorigo <pfsmorigo@linux.vnet.ibm.com>
and started as CL 108999 (https://go-review.googlesource.com/c/crypto/+/108999).

Fixes golang/go#25051

Change-Id: Ie510494249b227379e23d993467256b3d4088035
Reviewed-on: https://go-review.googlesource.com/c/crypto/+/172177
Run-TryBot: Carlos Eduardo Seo <cseo@linux.vnet.ibm.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Ian Lance Taylor <iant@golang.org>
Reviewed-by: Lynn Boger <laboger@linux.vnet.ibm.com>

internal/chacha20/asm_ppc64le.s

@@ -0,0 +1,668 @@
+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Based on CRYPTOGAMS code with the following comment:
+// # ====================================================================
+// # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
+// # project. The module is, however, dual licensed under OpenSSL and
+// # CRYPTOGAMS licenses depending on where you obtain it. For further
+// # details see http://www.openssl.org/~appro/cryptogams/.
+// # ====================================================================
+
+// Original code can be found at the link below:
+// https://github.com/dot-asm/cryptogams/commit/a60f5b50ed908e91e5c39ca79126a4a876d5d8ff
+
+// There are some differences between CRYPTOGAMS code and this one. The round
+// loop for "_int" isn't the same as the original. Some adjustments were
+// necessary because there are less vector registers available.  For example, some
+// X variables (r12, r13, r14, and r15) share the same register used by the
+// counter. The original code uses ctr to name the counter. Here we use CNT
+// because golang uses CTR as the counter register name.
+
+// +build ppc64le,!gccgo,!appengine
+
+#include "textflag.h"
+
+#define OUT  R3
+#define INP  R4
+#define LEN  R5
+#define KEY  R6
+#define CNT  R7
+
+#define TEMP R8
+
+#define X0   R11
+#define X1   R12
+#define X2   R14
+#define X3   R15
+#define X4   R16
+#define X5   R17
+#define X6   R18
+#define X7   R19
+#define X8   R20
+#define X9   R21
+#define X10  R22
+#define X11  R23
+#define X12  R24
+#define X13  R25
+#define X14  R26
+#define X15  R27
+
+#define CON0 X0
+#define CON1 X1
+#define CON2 X2
+#define CON3 X3
+
+#define KEY0 X4
+#define KEY1 X5
+#define KEY2 X6
+#define KEY3 X7
+#define KEY4 X8
+#define KEY5 X9
+#define KEY6 X10
+#define KEY7 X11
+
+#define CNT0 X12
+#define CNT1 X13
+#define CNT2 X14
+#define CNT3 X15
+
+#define TMP0 R9
+#define TMP1 R10
+#define TMP2 R28
+#define TMP3 R29
+
+#define CONSTS  R8
+
+#define A0      V0
+#define B0      V1
+#define C0      V2
+#define D0      V3
+#define A1      V4
+#define B1      V5
+#define C1      V6
+#define D1      V7
+#define A2      V8
+#define B2      V9
+#define C2      V10
+#define D2      V11
+#define T0      V12
+#define T1      V13
+#define T2      V14
+
+#define K0      V15
+#define K1      V16
+#define K2      V17
+#define K3      V18
+#define K4      V19
+#define K5      V20
+
+#define FOUR    V21
+#define SIXTEEN V22
+#define TWENTY4 V23
+#define TWENTY  V24
+#define TWELVE  V25
+#define TWENTY5 V26
+#define SEVEN   V27
+
+#define INPPERM V28
+#define OUTPERM V29
+#define OUTMASK V30
+
+#define DD0     V31
+#define DD1     SEVEN
+#define DD2     T0
+#define DD3     T1
+#define DD4     T2
+
+DATA  ·consts+0x00(SB)/8, $0x3320646e61707865
+DATA  ·consts+0x08(SB)/8, $0x6b20657479622d32
+DATA  ·consts+0x10(SB)/8, $0x0000000000000001
+DATA  ·consts+0x18(SB)/8, $0x0000000000000000
+DATA  ·consts+0x20(SB)/8, $0x0000000000000004
+DATA  ·consts+0x28(SB)/8, $0x0000000000000000
+DATA  ·consts+0x30(SB)/8, $0x0a0b08090e0f0c0d
+DATA  ·consts+0x38(SB)/8, $0x0203000106070405
+DATA  ·consts+0x40(SB)/8, $0x090a0b080d0e0f0c
+DATA  ·consts+0x48(SB)/8, $0x0102030005060704
+GLOBL ·consts(SB), RODATA, $80
+
+//func chaCha20_ctr32_vmx(out, inp *byte, len int, key *[32]byte, counter *[16]byte)
+TEXT ·chaCha20_ctr32_vmx(SB),NOSPLIT|NOFRAME,$0
+	// Load the arguments inside the registers
+	MOVD out+0(FP), OUT
+	MOVD inp+8(FP), INP
+	MOVD len+16(FP), LEN
+	MOVD key+24(FP), KEY
+	MOVD cnt+32(FP), CNT
+
+	MOVD $·consts(SB), CONSTS // point to consts addr
+
+	MOVD $16, X0
+	MOVD $32, X1
+	MOVD $48, X2
+	MOVD $64, X3
+	MOVD $31, X4
+	MOVD $15, X5
+
+	// Load key
+	LVX  (KEY)(R0), K1
+	LVSR (KEY)(R0), T0
+	LVX  (KEY)(X0), K2
+	LVX  (KEY)(X4), DD0
+
+	// Load counter
+	LVX  (CNT)(R0), K3
+	LVSR (CNT)(R0), T1
+	LVX  (CNT)(X5), DD1
+
+	// Load constants
+	LVX (CONSTS)(R0), K0
+	LVX (CONSTS)(X0), K5
+	LVX (CONSTS)(X1), FOUR
+	LVX (CONSTS)(X2), SIXTEEN
+	LVX (CONSTS)(X3), TWENTY4
+
+	// Align key and counter
+	VPERM K2,  K1, T0, K1
+	VPERM DD0, K2, T0, K2
+	VPERM DD1, K3, T1, K3
+
+	// Load counter to GPR
+	MOVWZ 0(CNT), CNT0
+	MOVWZ 4(CNT), CNT1
+	MOVWZ 8(CNT), CNT2
+	MOVWZ 12(CNT), CNT3
+
+	// Adjust vectors for the initial state
+	VADDUWM K3, K5, K3
+	VADDUWM K3, K5, K4
+	VADDUWM K4, K5, K5
+
+	// Synthesized constants
+	VSPLTISW $-12, TWENTY
+	VSPLTISW $12, TWELVE
+	VSPLTISW $-7, TWENTY5
+
+	VXOR T0, T0, T0
+	VSPLTISW $-1, OUTMASK
+	LVSR (INP)(R0), INPPERM
+	LVSL (OUT)(R0), OUTPERM
+	VPERM OUTMASK, T0, OUTPERM, OUTMASK
+
+loop_outer_vmx:
+	// Load constant
+	MOVD $0x61707865, CON0
+	MOVD $0x3320646e, CON1
+	MOVD $0x79622d32, CON2
+	MOVD $0x6b206574, CON3
+
+	VOR K0, K0, A0
+	VOR K0, K0, A1
+	VOR K0, K0, A2
+	VOR K1, K1, B0
+
+	MOVD $10, TEMP
+
+	// Load key to GPR
+	MOVWZ 0(KEY), X4
+	MOVWZ 4(KEY), X5
+	MOVWZ 8(KEY), X6
+	MOVWZ 12(KEY), X7
+	VOR K1, K1, B1
+	VOR K1, K1, B2
+	MOVWZ 16(KEY), X8
+	MOVWZ  0(CNT), X12
+	MOVWZ 20(KEY), X9
+	MOVWZ 4(CNT), X13
+	VOR K2, K2, C0
+	VOR K2, K2, C1
+	MOVWZ 24(KEY), X10
+	MOVWZ 8(CNT), X14
+	VOR K2, K2, C2
+	VOR K3, K3, D0
+	MOVWZ 28(KEY), X11
+	MOVWZ 12(CNT), X15
+	VOR K4, K4, D1
+	VOR K5, K5, D2
+
+	MOVD X4, TMP0
+	MOVD X5, TMP1
+	MOVD X6, TMP2
+	MOVD X7, TMP3
+	VSPLTISW $7, SEVEN
+
+	MOVD TEMP, CTR
+
+loop_vmx:
+	// CRYPTOGAMS uses a macro to create a loop using perl. This isn't possible
+	// using assembly macros.  Therefore, the macro expansion result was used
+	// in order to maintain the algorithm efficiency.
+	// This loop generates three keystream blocks using VMX instructions and,
+	// in parallel, one keystream block using scalar instructions.
+	ADD X4, X0, X0
+	ADD X5, X1, X1
+	VADDUWM A0, B0, A0
+	VADDUWM A1, B1, A1
+	ADD X6, X2, X2
+	ADD X7, X3, X3
+	VADDUWM A2, B2, A2
+	VXOR D0, A0, D0
+	XOR X0, X12, X12
+	XOR X1, X13, X13
+	VXOR D1, A1, D1
+	VXOR D2, A2, D2
+	XOR X2, X14, X14
+	XOR X3, X15, X15
+	VPERM D0, D0, SIXTEEN, D0
+	VPERM D1, D1, SIXTEEN, D1
+	ROTLW $16, X12, X12
+	ROTLW $16, X13, X13
+	VPERM D2, D2, SIXTEEN, D2
+	VADDUWM C0, D0, C0
+	ROTLW $16, X14, X14
+	ROTLW $16, X15, X15
+	VADDUWM C1, D1, C1
+	VADDUWM C2, D2, C2
+	ADD X12, X8, X8
+	ADD X13, X9, X9
+	VXOR B0, C0, T0
+	VXOR B1, C1, T1
+	ADD X14, X10, X10
+	ADD X15, X11, X11
+	VXOR B2, C2, T2
+	VRLW T0, TWELVE, B0
+	XOR X8, X4, X4
+	XOR X9, X5, X5
+	VRLW T1, TWELVE, B1
+	VRLW T2, TWELVE, B2
+	XOR X10, X6, X6
+	XOR X11, X7, X7
+	VADDUWM A0, B0, A0
+	VADDUWM A1, B1, A1
+	ROTLW $12, X4, X4
+	ROTLW $12, X5, X5
+	VADDUWM A2, B2, A2
+	VXOR D0, A0, D0
+	ROTLW $12, X6, X6
+	ROTLW $12, X7, X7
+	VXOR D1, A1, D1
+	VXOR D2, A2, D2
+	ADD X4, X0, X0
+	ADD X5, X1, X1
+	VPERM D0, D0, TWENTY4, D0
+	VPERM D1, D1, TWENTY4, D1
+	ADD X6, X2, X2
+	ADD X7, X3, X3
+	VPERM D2, D2, TWENTY4, D2
+	VADDUWM C0, D0, C0
+	XOR X0, X12, X12
+	XOR X1, X13, X13
+	VADDUWM C1, D1, C1
+	VADDUWM C2, D2, C2
+	XOR X2, X14, X14
+	XOR X3, X15, X15
+	VXOR B0, C0, T0
+	VXOR B1, C1, T1
+	ROTLW $8, X12, X12
+	ROTLW $8, X13, X13
+	VXOR B2, C2, T2
+	VRLW T0, SEVEN, B0
+	ROTLW $8, X14, X14
+	ROTLW $8, X15, X15
+	VRLW T1, SEVEN, B1
+	VRLW T2, SEVEN, B2
+	ADD X12, X8, X8
+	ADD X13, X9, X9
+	VSLDOI $8, C0, C0, C0
+	VSLDOI $8, C1, C1, C1
+	ADD X14, X10, X10
+	ADD X15, X11, X11
+	VSLDOI $8, C2, C2, C2
+	VSLDOI $12, B0, B0, B0
+	XOR X8, X4, X4
+	XOR X9, X5, X5
+	VSLDOI $12, B1, B1, B1
+	VSLDOI $12, B2, B2, B2
+	XOR X10, X6, X6
+	XOR X11, X7, X7
+	VSLDOI $4, D0, D0, D0
+	VSLDOI $4, D1, D1, D1
+	ROTLW $7, X4, X4
+	ROTLW $7, X5, X5
+	VSLDOI $4, D2, D2, D2
+	VADDUWM A0, B0, A0
+	ROTLW $7, X6, X6
+	ROTLW $7, X7, X7
+	VADDUWM A1, B1, A1
+	VADDUWM A2, B2, A2
+	ADD X5, X0, X0
+	ADD X6, X1, X1
+	VXOR D0, A0, D0
+	VXOR D1, A1, D1
+	ADD X7, X2, X2
+	ADD X4, X3, X3
+	VXOR D2, A2, D2
+	VPERM D0, D0, SIXTEEN, D0
+	XOR X0, X15, X15
+	XOR X1, X12, X12
+	VPERM D1, D1, SIXTEEN, D1
+	VPERM D2, D2, SIXTEEN, D2
+	XOR X2, X13, X13
+	XOR X3, X14, X14
+	VADDUWM C0, D0, C0
+	VADDUWM C1, D1, C1
+	ROTLW $16, X15, X15
+	ROTLW $16, X12, X12
+	VADDUWM C2, D2, C2
+	VXOR B0, C0, T0
+	ROTLW $16, X13, X13
+	ROTLW $16, X14, X14
+	VXOR B1, C1, T1
+	VXOR B2, C2, T2
+	ADD X15, X10, X10
+	ADD X12, X11, X11
+	VRLW T0, TWELVE, B0
+	VRLW T1, TWELVE, B1
+	ADD X13, X8, X8
+	ADD X14, X9, X9
+	VRLW T2, TWELVE, B2
+	VADDUWM A0, B0, A0
+	XOR X10, X5, X5
+	XOR X11, X6, X6
+	VADDUWM A1, B1, A1
+	VADDUWM A2, B2, A2
+	XOR X8, X7, X7
+	XOR X9, X4, X4
+	VXOR D0, A0, D0
+	VXOR D1, A1, D1
+	ROTLW $12, X5, X5
+	ROTLW $12, X6, X6
+	VXOR D2, A2, D2
+	VPERM D0, D0, TWENTY4, D0
+	ROTLW $12, X7, X7
+	ROTLW $12, X4, X4
+	VPERM D1, D1, TWENTY4, D1
+	VPERM D2, D2, TWENTY4, D2
+	ADD X5, X0, X0
+	ADD X6, X1, X1
+	VADDUWM C0, D0, C0
+	VADDUWM C1, D1, C1
+	ADD X7, X2, X2
+	ADD X4, X3, X3
+	VADDUWM C2, D2, C2
+	VXOR B0, C0, T0
+	XOR X0, X15, X15
+	XOR X1, X12, X12
+	VXOR B1, C1, T1
+	VXOR B2, C2, T2
+	XOR X2, X13, X13
+	XOR X3, X14, X14
+	VRLW T0, SEVEN, B0
+	VRLW T1, SEVEN, B1
+	ROTLW $8, X15, X15
+	ROTLW $8, X12, X12
+	VRLW T2, SEVEN, B2
+	VSLDOI $8, C0, C0, C0
+	ROTLW $8, X13, X13
+	ROTLW $8, X14, X14
+	VSLDOI $8, C1, C1, C1
+	VSLDOI $8, C2, C2, C2
+	ADD X15, X10, X10
+	ADD X12, X11, X11
+	VSLDOI $4, B0, B0, B0
+	VSLDOI $4, B1, B1, B1
+	ADD X13, X8, X8
+	ADD X14, X9, X9
+	VSLDOI $4, B2, B2, B2
+	VSLDOI $12, D0, D0, D0
+	XOR X10, X5, X5
+	XOR X11, X6, X6
+	VSLDOI $12, D1, D1, D1
+	VSLDOI $12, D2, D2, D2
+	XOR X8, X7, X7
+	XOR X9, X4, X4
+	ROTLW $7, X5, X5
+	ROTLW $7, X6, X6
+	ROTLW $7, X7, X7
+	ROTLW $7, X4, X4
+	BC 0x10, 0, loop_vmx
+
+	SUB $256, LEN, LEN
+
+	// Accumulate key block
+	ADD $0x61707865, X0, X0
+	ADD $0x3320646e, X1, X1
+	ADD $0x79622d32, X2, X2
+	ADD $0x6b206574, X3, X3
+	ADD TMP0, X4, X4
+	ADD TMP1, X5, X5
+	ADD TMP2, X6, X6
+	ADD TMP3, X7, X7
+	MOVWZ 16(KEY), TMP0
+	MOVWZ 20(KEY), TMP1
+	MOVWZ 24(KEY), TMP2
+	MOVWZ 28(KEY), TMP3
+	ADD TMP0, X8, X8
+	ADD TMP1, X9, X9
+	ADD TMP2, X10, X10
+	ADD TMP3, X11, X11
+
+	MOVWZ 12(CNT), TMP0
+	MOVWZ 8(CNT), TMP1
+	MOVWZ 4(CNT), TMP2
+	MOVWZ 0(CNT), TEMP
+	ADD TMP0, X15, X15
+	ADD TMP1, X14, X14
+	ADD TMP2, X13, X13
+	ADD TEMP, X12, X12
+
+	// Accumulate key block
+	VADDUWM A0, K0, A0
+	VADDUWM A1, K0, A1
+	VADDUWM A2, K0, A2
+	VADDUWM B0, K1, B0
+	VADDUWM B1, K1, B1
+	VADDUWM B2, K1, B2
+	VADDUWM C0, K2, C0
+	VADDUWM C1, K2, C1
+	VADDUWM C2, K2, C2
+	VADDUWM D0, K3, D0
+	VADDUWM D1, K4, D1
+	VADDUWM D2, K5, D2
+
+	// Increment counter
+	ADD $4, TEMP, TEMP
+	MOVW TEMP, 0(CNT)
+
+	VADDUWM K3, FOUR, K3
+	VADDUWM K4, FOUR, K4
+	VADDUWM K5, FOUR, K5
+
+	// XOR the input slice (INP) with the keystream, which is stored in GPRs (X0-X3).
+
+	// Load input (aligned or not)
+	MOVWZ 0(INP), TMP0
+	MOVWZ 4(INP), TMP1
+	MOVWZ 8(INP), TMP2
+	MOVWZ 12(INP), TMP3
+
+	// XOR with input
+	XOR TMP0, X0, X0
+	XOR TMP1, X1, X1
+	XOR TMP2, X2, X2
+	XOR TMP3, X3, X3
+	MOVWZ 16(INP), TMP0
+	MOVWZ 20(INP), TMP1
+	MOVWZ 24(INP), TMP2
+	MOVWZ 28(INP), TMP3
+	XOR TMP0, X4, X4
+	XOR TMP1, X5, X5
+	XOR TMP2, X6, X6
+	XOR TMP3, X7, X7
+	MOVWZ 32(INP), TMP0
+	MOVWZ 36(INP), TMP1
+	MOVWZ 40(INP), TMP2
+	MOVWZ 44(INP), TMP3
+	XOR TMP0, X8, X8
+	XOR TMP1, X9, X9
+	XOR TMP2, X10, X10
+	XOR TMP3, X11, X11
+	MOVWZ 48(INP), TMP0
+	MOVWZ 52(INP), TMP1
+	MOVWZ 56(INP), TMP2
+	MOVWZ 60(INP), TMP3
+	XOR TMP0, X12, X12
+	XOR TMP1, X13, X13
+	XOR TMP2, X14, X14
+	XOR TMP3, X15, X15
+
+	// Store output (aligned or not)
+	MOVW X0, 0(OUT)
+	MOVW X1, 4(OUT)
+	MOVW X2, 8(OUT)
+	MOVW X3, 12(OUT)
+
+	ADD $64, INP, INP // INP points to the end of the slice for the alignment code below
+
+	MOVW X4, 16(OUT)
+	MOVD $16, TMP0
+	MOVW X5, 20(OUT)
+	MOVD $32, TMP1
+	MOVW X6, 24(OUT)
+	MOVD $48, TMP2
+	MOVW X7, 28(OUT)
+	MOVD $64, TMP3
+	MOVW X8, 32(OUT)
+	MOVW X9, 36(OUT)
+	MOVW X10, 40(OUT)
+	MOVW X11, 44(OUT)
+	MOVW X12, 48(OUT)
+	MOVW X13, 52(OUT)
+	MOVW X14, 56(OUT)
+	MOVW X15, 60(OUT)
+	ADD $64, OUT, OUT
+
+	// Load input
+	LVX (INP)(R0), DD0
+	LVX (INP)(TMP0), DD1
+	LVX (INP)(TMP1), DD2
+	LVX (INP)(TMP2), DD3
+	LVX (INP)(TMP3), DD4
+	ADD $64, INP, INP
+
+	VPERM DD1, DD0, INPPERM, DD0 // Align input
+	VPERM DD2, DD1, INPPERM, DD1
+	VPERM DD3, DD2, INPPERM, DD2
+	VPERM DD4, DD3, INPPERM, DD3
+	VXOR A0, DD0, A0 // XOR with input
+	VXOR B0, DD1, B0
+	LVX (INP)(TMP0), DD1 // Keep loading input
+	VXOR C0, DD2, C0
+	LVX (INP)(TMP1), DD2
+	VXOR D0, DD3, D0
+	LVX (INP)(TMP2), DD3
+	LVX (INP)(TMP3), DD0
+	ADD $64, INP, INP
+	MOVD $63, TMP3 // 63 is not a typo
+	VPERM A0, A0, OUTPERM, A0
+	VPERM B0, B0, OUTPERM, B0
+	VPERM C0, C0, OUTPERM, C0
+	VPERM D0, D0, OUTPERM, D0
+
+	VPERM DD1, DD4, INPPERM, DD4 // Align input
+	VPERM DD2, DD1, INPPERM, DD1
+	VPERM DD3, DD2, INPPERM, DD2
+	VPERM DD0, DD3, INPPERM, DD3
+	VXOR A1, DD4, A1
+	VXOR B1, DD1, B1
+	LVX (INP)(TMP0), DD1 // Keep loading
+	VXOR C1, DD2, C1
+	LVX (INP)(TMP1), DD2
+	VXOR D1, DD3, D1
+	LVX (INP)(TMP2), DD3
+
+	// Note that the LVX address is always rounded down to the nearest 16-byte
+	// boundary, and that it always points to at most 15 bytes beyond the end of
+	// the slice, so we cannot cross a page boundary.
+	LVX (INP)(TMP3), DD4 // Redundant in aligned case.
+	ADD $64, INP, INP
+	VPERM A1, A1, OUTPERM, A1 // Pre-misalign output
+	VPERM B1, B1, OUTPERM, B1
+	VPERM C1, C1, OUTPERM, C1
+	VPERM D1, D1, OUTPERM, D1
+
+	VPERM DD1, DD0, INPPERM, DD0 // Align Input
+	VPERM DD2, DD1, INPPERM, DD1
+	VPERM DD3, DD2, INPPERM, DD2
+	VPERM DD4, DD3, INPPERM, DD3
+	VXOR A2, DD0, A2
+	VXOR B2, DD1, B2
+	VXOR C2, DD2, C2
+	VXOR D2, DD3, D2
+	VPERM A2, A2, OUTPERM, A2
+	VPERM B2, B2, OUTPERM, B2
+	VPERM C2, C2, OUTPERM, C2
+	VPERM D2, D2, OUTPERM, D2
+
+	ANDCC $15, OUT, X1 // Is out aligned?
+	MOVD OUT, X0
+
+	VSEL A0, B0, OUTMASK, DD0 // Collect pre-misaligned output
+	VSEL B0, C0, OUTMASK, DD1
+	VSEL C0, D0, OUTMASK, DD2
+	VSEL D0, A1, OUTMASK, DD3
+	VSEL A1, B1, OUTMASK, B0
+	VSEL B1, C1, OUTMASK, C0
+	VSEL C1, D1, OUTMASK, D0
+	VSEL D1, A2, OUTMASK, A1
+	VSEL A2, B2, OUTMASK, B1
+	VSEL B2, C2, OUTMASK, C1
+	VSEL C2, D2, OUTMASK, D1
+
+	STVX DD0, (OUT+TMP0)
+	STVX DD1, (OUT+TMP1)
+	STVX DD2, (OUT+TMP2)
+	ADD $64, OUT, OUT
+	STVX DD3, (OUT+R0)
+	STVX B0, (OUT+TMP0)
+	STVX C0, (OUT+TMP1)
+	STVX D0, (OUT+TMP2)
+	ADD $64, OUT, OUT
+	STVX A1, (OUT+R0)
+	STVX B1, (OUT+TMP0)
+	STVX C1, (OUT+TMP1)
+	STVX D1, (OUT+TMP2)
+	ADD $64, OUT, OUT
+
+	BEQ aligned_vmx
+
+	SUB X1, OUT, X2 // in misaligned case edges
+	MOVD $0, X3 // are written byte-by-byte
+
+unaligned_tail_vmx:
+	STVEBX D2, (X2+X3)
+	ADD $1, X3, X3
+	CMPW X3, X1
+	BNE unaligned_tail_vmx
+	SUB X1, X0, X2
+
+unaligned_head_vmx:
+	STVEBX A0, (X2+X1)
+	CMPW X1, $15
+	ADD $1, X1, X1
+	BNE unaligned_head_vmx
+
+	CMPU LEN, $255 // done with 256-byte block yet?
+	BGT loop_outer_vmx
+
+	JMP done_vmx
+
+aligned_vmx:
+	STVX A0, (X0+R0)
+	CMPU LEN, $255 // done with 256-byte block yet?
+	BGT loop_outer_vmx
+
+done_vmx:
+	RET

internal/chacha20/chacha_noasm.go

@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build !arm64,!s390x arm64,!go1.11 gccgo appengine
+// +build !ppc64le,!arm64,!s390x arm64,!go1.11 gccgo appengine
 
 package chacha20
 

internal/chacha20/chacha_ppc64le.go

@@ -0,0 +1,51 @@
+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build ppc64le,!gccgo,!appengine
+
+package chacha20
+
+import "encoding/binary"
+
+var haveAsm = true
+
+const bufSize = 256
+
+//go:noescape
+func chaCha20_ctr32_vmx(out, inp *byte, len int, key *[8]uint32, counter *uint32)
+
+func (c *Cipher) xorKeyStreamAsm(dst, src []byte) {
+	if len(src) >= bufSize {
+		chaCha20_ctr32_vmx(&dst[0], &src[0], len(src)-len(src)%bufSize, &c.key, &c.counter)
+	}
+	if len(src)%bufSize != 0 {
+		chaCha20_ctr32_vmx(&c.buf[0], &c.buf[0], bufSize, &c.key, &c.counter)
+		start := len(src) - len(src)%bufSize
+		ts, td, tb := src[start:], dst[start:], c.buf[:]
+		// Unroll loop to XOR 32 bytes per iteration.
+		for i := 0; i < len(ts)-32; i += 32 {
+			td, tb = td[:len(ts)], tb[:len(ts)] // bounds check elimination
+			s0 := binary.LittleEndian.Uint64(ts[0:8])
+			s1 := binary.LittleEndian.Uint64(ts[8:16])
+			s2 := binary.LittleEndian.Uint64(ts[16:24])
+			s3 := binary.LittleEndian.Uint64(ts[24:32])
+			b0 := binary.LittleEndian.Uint64(tb[0:8])
+			b1 := binary.LittleEndian.Uint64(tb[8:16])
+			b2 := binary.LittleEndian.Uint64(tb[16:24])
+			b3 := binary.LittleEndian.Uint64(tb[24:32])
+			binary.LittleEndian.PutUint64(td[0:8], s0^b0)
+			binary.LittleEndian.PutUint64(td[8:16], s1^b1)
+			binary.LittleEndian.PutUint64(td[16:24], s2^b2)
+			binary.LittleEndian.PutUint64(td[24:32], s3^b3)
+			ts, td, tb = ts[32:], td[32:], tb[32:]
+		}
+		td, tb = td[:len(ts)], tb[:len(ts)] // bounds check elimination
+		for i, v := range ts {
+			td[i] = tb[i] ^ v
+		}
+		c.len = bufSize - (len(src) % bufSize)
+
+	}
+
+}