go.crypto/sha3: change keccakF to stateless function

Taken from my implementation: https://bitbucket.org/ede/sha3
Performance gain from using less memory and more registers.

benchmark                       old ns/op    new ns/op    delta
BenchmarkPermutationFunction         1484         1118  -24.66%
BenchmarkBulkKeccak512             374993       295178  -21.28%
BenchmarkBulkKeccak256             215496       172335  -20.03%

benchmark                        old MB/s     new MB/s  speedup
BenchmarkPermutationFunction       134.76       178.80    1.33x
BenchmarkBulkKeccak512              43.69        55.51    1.27x
BenchmarkBulkKeccak256              76.03        95.07    1.25x

R=jcb, agl
CC=golang-dev, nigeltao
https://golang.org/cl/8088044

sha3/keccakf.go

@@ -37,135 +37,129 @@ var rc = [...]uint64{
 	0x8000000080008008,
 }
 
-// ro_xx represent the rotation offsets for use in the χ step.
-// Defining them as const instead of in an array allows the compiler to insert constant shifts.
-const (
-	ro_00 = 0
-	ro_01 = 36
-	ro_02 = 3
-	ro_03 = 41
-	ro_04 = 18
-	ro_05 = 1
-	ro_06 = 44
-	ro_07 = 10
-	ro_08 = 45
-	ro_09 = 2
-	ro_10 = 62
-	ro_11 = 6
-	ro_12 = 43
-	ro_13 = 15
-	ro_14 = 61
-	ro_15 = 28
-	ro_16 = 55
-	ro_17 = 25
-	ro_18 = 21
-	ro_19 = 56
-	ro_20 = 27
-	ro_21 = 20
-	ro_22 = 39
-	ro_23 = 8
-	ro_24 = 14
-)
-
 // keccakF computes the complete Keccak-f function consisting of 24 rounds with a different
 // constant (rc) in each round. This implementation fully unrolls the round function to avoid
 // inner loops, as well as pre-calculating shift offsets.
-func (d *digest) keccakF() {
+func keccakF(a *[numLanes]uint64) {
+	var t, bc0, bc1, bc2, bc3, bc4 uint64
 	for _, roundConstant := range rc {
 		// θ step
-		d.c[0] = d.a[0] ^ d.a[5] ^ d.a[10] ^ d.a[15] ^ d.a[20]
-		d.c[1] = d.a[1] ^ d.a[6] ^ d.a[11] ^ d.a[16] ^ d.a[21]
-		d.c[2] = d.a[2] ^ d.a[7] ^ d.a[12] ^ d.a[17] ^ d.a[22]
-		d.c[3] = d.a[3] ^ d.a[8] ^ d.a[13] ^ d.a[18] ^ d.a[23]
-		d.c[4] = d.a[4] ^ d.a[9] ^ d.a[14] ^ d.a[19] ^ d.a[24]
-
-		d.d[0] = d.c[4] ^ (d.c[1]<<1 ^ d.c[1]>>63)
-		d.d[1] = d.c[0] ^ (d.c[2]<<1 ^ d.c[2]>>63)
-		d.d[2] = d.c[1] ^ (d.c[3]<<1 ^ d.c[3]>>63)
-		d.d[3] = d.c[2] ^ (d.c[4]<<1 ^ d.c[4]>>63)
-		d.d[4] = d.c[3] ^ (d.c[0]<<1 ^ d.c[0]>>63)
-
-		d.a[0] ^= d.d[0]
-		d.a[1] ^= d.d[1]
-		d.a[2] ^= d.d[2]
-		d.a[3] ^= d.d[3]
-		d.a[4] ^= d.d[4]
-		d.a[5] ^= d.d[0]
-		d.a[6] ^= d.d[1]
-		d.a[7] ^= d.d[2]
-		d.a[8] ^= d.d[3]
-		d.a[9] ^= d.d[4]
-		d.a[10] ^= d.d[0]
-		d.a[11] ^= d.d[1]
-		d.a[12] ^= d.d[2]
-		d.a[13] ^= d.d[3]
-		d.a[14] ^= d.d[4]
-		d.a[15] ^= d.d[0]
-		d.a[16] ^= d.d[1]
-		d.a[17] ^= d.d[2]
-		d.a[18] ^= d.d[3]
-		d.a[19] ^= d.d[4]
-		d.a[20] ^= d.d[0]
-		d.a[21] ^= d.d[1]
-		d.a[22] ^= d.d[2]
-		d.a[23] ^= d.d[3]
-		d.a[24] ^= d.d[4]
+		bc0 = a[0] ^ a[5] ^ a[10] ^ a[15] ^ a[20]
+		bc1 = a[1] ^ a[6] ^ a[11] ^ a[16] ^ a[21]
+		bc2 = a[2] ^ a[7] ^ a[12] ^ a[17] ^ a[22]
+		bc3 = a[3] ^ a[8] ^ a[13] ^ a[18] ^ a[23]
+		bc4 = a[4] ^ a[9] ^ a[14] ^ a[19] ^ a[24]
+		t = bc4 ^ (bc1<<1 ^ bc1>>63)
+		a[0] ^= t
+		a[5] ^= t
+		a[10] ^= t
+		a[15] ^= t
+		a[20] ^= t
+		t = bc0 ^ (bc2<<1 ^ bc2>>63)
+		a[1] ^= t
+		a[6] ^= t
+		a[11] ^= t
+		a[16] ^= t
+		a[21] ^= t
+		t = bc1 ^ (bc3<<1 ^ bc3>>63)
+		a[2] ^= t
+		a[7] ^= t
+		a[12] ^= t
+		a[17] ^= t
+		a[22] ^= t
+		t = bc2 ^ (bc4<<1 ^ bc4>>63)
+		a[3] ^= t
+		a[8] ^= t
+		a[13] ^= t
+		a[18] ^= t
+		a[23] ^= t
+		t = bc3 ^ (bc0<<1 ^ bc0>>63)
+		a[4] ^= t
+		a[9] ^= t
+		a[14] ^= t
+		a[19] ^= t
+		a[24] ^= t
 
 		// ρ and π steps
-		d.b[0] = d.a[0]
-		d.b[1] = d.a[6]<<ro_06 ^ d.a[6]>>(64-ro_06)
-		d.b[2] = d.a[12]<<ro_12 ^ d.a[12]>>(64-ro_12)
-		d.b[3] = d.a[18]<<ro_18 ^ d.a[18]>>(64-ro_18)
-		d.b[4] = d.a[24]<<ro_24 ^ d.a[24]>>(64-ro_24)
-		d.b[5] = d.a[3]<<ro_15 ^ d.a[3]>>(64-ro_15)
-		d.b[6] = d.a[9]<<ro_21 ^ d.a[9]>>(64-ro_21)
-		d.b[7] = d.a[10]<<ro_02 ^ d.a[10]>>(64-ro_02)
-		d.b[8] = d.a[16]<<ro_08 ^ d.a[16]>>(64-ro_08)
-		d.b[9] = d.a[22]<<ro_14 ^ d.a[22]>>(64-ro_14)
-		d.b[10] = d.a[1]<<ro_05 ^ d.a[1]>>(64-ro_05)
-		d.b[11] = d.a[7]<<ro_11 ^ d.a[7]>>(64-ro_11)
-		d.b[12] = d.a[13]<<ro_17 ^ d.a[13]>>(64-ro_17)
-		d.b[13] = d.a[19]<<ro_23 ^ d.a[19]>>(64-ro_23)
-		d.b[14] = d.a[20]<<ro_04 ^ d.a[20]>>(64-ro_04)
-		d.b[15] = d.a[4]<<ro_20 ^ d.a[4]>>(64-ro_20)
-		d.b[16] = d.a[5]<<ro_01 ^ d.a[5]>>(64-ro_01)
-		d.b[17] = d.a[11]<<ro_07 ^ d.a[11]>>(64-ro_07)
-		d.b[18] = d.a[17]<<ro_13 ^ d.a[17]>>(64-ro_13)
-		d.b[19] = d.a[23]<<ro_19 ^ d.a[23]>>(64-ro_19)
-		d.b[20] = d.a[2]<<ro_10 ^ d.a[2]>>(64-ro_10)
-		d.b[21] = d.a[8]<<ro_16 ^ d.a[8]>>(64-ro_16)
-		d.b[22] = d.a[14]<<ro_22 ^ d.a[14]>>(64-ro_22)
-		d.b[23] = d.a[15]<<ro_03 ^ d.a[15]>>(64-ro_03)
-		d.b[24] = d.a[21]<<ro_09 ^ d.a[21]>>(64-ro_09)
+		t = a[1]
+		t, a[10] = a[10], t<<1^t>>(64-1)
+		t, a[7] = a[7], t<<3^t>>(64-3)
+		t, a[11] = a[11], t<<6^t>>(64-6)
+		t, a[17] = a[17], t<<10^t>>(64-10)
+		t, a[18] = a[18], t<<15^t>>(64-15)
+		t, a[3] = a[3], t<<21^t>>(64-21)
+		t, a[5] = a[5], t<<28^t>>(64-28)
+		t, a[16] = a[16], t<<36^t>>(64-36)
+		t, a[8] = a[8], t<<45^t>>(64-45)
+		t, a[21] = a[21], t<<55^t>>(64-55)
+		t, a[24] = a[24], t<<2^t>>(64-2)
+		t, a[4] = a[4], t<<14^t>>(64-14)
+		t, a[15] = a[15], t<<27^t>>(64-27)
+		t, a[23] = a[23], t<<41^t>>(64-41)
+		t, a[19] = a[19], t<<56^t>>(64-56)
+		t, a[13] = a[13], t<<8^t>>(64-8)
+		t, a[12] = a[12], t<<25^t>>(64-25)
+		t, a[2] = a[2], t<<43^t>>(64-43)
+		t, a[20] = a[20], t<<62^t>>(64-62)
+		t, a[14] = a[14], t<<18^t>>(64-18)
+		t, a[22] = a[22], t<<39^t>>(64-39)
+		t, a[9] = a[9], t<<61^t>>(64-61)
+		t, a[6] = a[6], t<<20^t>>(64-20)
+		a[1] = t<<44 ^ t>>(64-44)
 
 		// χ step
-		d.a[0] = d.b[0] ^ (^d.b[1] & d.b[2])
-		d.a[1] = d.b[1] ^ (^d.b[2] & d.b[3])
-		d.a[2] = d.b[2] ^ (^d.b[3] & d.b[4])
-		d.a[3] = d.b[3] ^ (^d.b[4] & d.b[0])
-		d.a[4] = d.b[4] ^ (^d.b[0] & d.b[1])
-		d.a[5] = d.b[5] ^ (^d.b[6] & d.b[7])
-		d.a[6] = d.b[6] ^ (^d.b[7] & d.b[8])
-		d.a[7] = d.b[7] ^ (^d.b[8] & d.b[9])
-		d.a[8] = d.b[8] ^ (^d.b[9] & d.b[5])
-		d.a[9] = d.b[9] ^ (^d.b[5] & d.b[6])
-		d.a[10] = d.b[10] ^ (^d.b[11] & d.b[12])
-		d.a[11] = d.b[11] ^ (^d.b[12] & d.b[13])
-		d.a[12] = d.b[12] ^ (^d.b[13] & d.b[14])
-		d.a[13] = d.b[13] ^ (^d.b[14] & d.b[10])
-		d.a[14] = d.b[14] ^ (^d.b[10] & d.b[11])
-		d.a[15] = d.b[15] ^ (^d.b[16] & d.b[17])
-		d.a[16] = d.b[16] ^ (^d.b[17] & d.b[18])
-		d.a[17] = d.b[17] ^ (^d.b[18] & d.b[19])
-		d.a[18] = d.b[18] ^ (^d.b[19] & d.b[15])
-		d.a[19] = d.b[19] ^ (^d.b[15] & d.b[16])
-		d.a[20] = d.b[20] ^ (^d.b[21] & d.b[22])
-		d.a[21] = d.b[21] ^ (^d.b[22] & d.b[23])
-		d.a[22] = d.b[22] ^ (^d.b[23] & d.b[24])
-		d.a[23] = d.b[23] ^ (^d.b[24] & d.b[20])
-		d.a[24] = d.b[24] ^ (^d.b[20] & d.b[21])
+		bc0 = a[0]
+		bc1 = a[1]
+		bc2 = a[2]
+		bc3 = a[3]
+		bc4 = a[4]
+		a[0] ^= bc2 &^ bc1
+		a[1] ^= bc3 &^ bc2
+		a[2] ^= bc4 &^ bc3
+		a[3] ^= bc0 &^ bc4
+		a[4] ^= bc1 &^ bc0
+		bc0 = a[5]
+		bc1 = a[6]
+		bc2 = a[7]
+		bc3 = a[8]
+		bc4 = a[9]
+		a[5] ^= bc2 &^ bc1
+		a[6] ^= bc3 &^ bc2
+		a[7] ^= bc4 &^ bc3
+		a[8] ^= bc0 &^ bc4
+		a[9] ^= bc1 &^ bc0
+		bc0 = a[10]
+		bc1 = a[11]
+		bc2 = a[12]
+		bc3 = a[13]
+		bc4 = a[14]
+		a[10] ^= bc2 &^ bc1
+		a[11] ^= bc3 &^ bc2
+		a[12] ^= bc4 &^ bc3
+		a[13] ^= bc0 &^ bc4
+		a[14] ^= bc1 &^ bc0
+		bc0 = a[15]
+		bc1 = a[16]
+		bc2 = a[17]
+		bc3 = a[18]
+		bc4 = a[19]
+		a[15] ^= bc2 &^ bc1
+		a[16] ^= bc3 &^ bc2
+		a[17] ^= bc4 &^ bc3
+		a[18] ^= bc0 &^ bc4
+		a[19] ^= bc1 &^ bc0
+		bc0 = a[20]
+		bc1 = a[21]
+		bc2 = a[22]
+		bc3 = a[23]
+		bc4 = a[24]
+		a[20] ^= bc2 &^ bc1
+		a[21] ^= bc3 &^ bc2
+		a[22] ^= bc4 &^ bc3
+		a[23] ^= bc0 &^ bc4
+		a[24] ^= bc1 &^ bc0
 
 		// ι step
-		d.a[0] ^= roundConstant
+		a[0] ^= roundConstant
 	}
 }

sha3/sha3.go

@@ -38,13 +38,10 @@ const stateSize = laneSize * numLanes
 // O(2^{outputSize/2}) computations (the birthday lower bound). Future standards may modify the
 // capacity/outputSize ratio to allow for more output with lower cryptographic security.
 type digest struct {
-	a          [numLanes]uint64  // main state of the hash
-	b          [numLanes]uint64  // intermediate states
-	c          [sliceSize]uint64 // intermediate states
-	d          [sliceSize]uint64 // intermediate states
-	outputSize int               // desired output size in bytes
-	capacity   int               // number of bytes to leave untouched during squeeze/absorb
-	absorbed   int               // number of bytes absorbed thus far
+	a          [numLanes]uint64 // main state of the hash
+	outputSize int              // desired output size in bytes
+	capacity   int              // number of bytes to leave untouched during squeeze/absorb
+	absorbed   int              // number of bytes absorbed thus far
 }
 
 // minInt returns the lesser of two integer arguments, to simplify the absorption routine.
@@ -116,7 +113,7 @@ func (d *digest) Write(p []byte) (int, error) {
 
 		// For every rate() bytes absorbed, the state must be permuted via the F Function.
 		if (d.absorbed)%d.rate() == 0 {
-			d.keccakF()
+			keccakF(&d.a)
 		}
 	}
 
@@ -134,7 +131,7 @@ func (d *digest) Write(p []byte) (int, error) {
 		d.absorbed += (lastLane - firstLane) * laneSize
 		// For every rate() bytes absorbed, the state must be permuted via the F Function.
 		if (d.absorbed)%d.rate() == 0 {
-			d.keccakF()
+			keccakF(&d.a)
 		}
 
 		offset = 0
@@ -167,7 +164,7 @@ func (d *digest) pad() {
 // finalize prepares the hash to output data by padding and one final permutation of the state.
 func (d *digest) finalize() {
 	d.pad()
-	d.keccakF()
+	keccakF(&d.a)
 }
 
 // squeeze outputs an arbitrary number of bytes from the hash state.
@@ -192,7 +189,7 @@ func (d *digest) squeeze(in []byte, toSqueeze int) []byte {
 			out = out[laneSize:]
 		}
 		if len(out) > 0 {
-			d.keccakF()
+			keccakF(&d.a)
 		}
 	}
 	return in[:len(in)+toSqueeze] // Re-slice in case we wrote extra data.

sha3/sha3_test.go

@@ -212,16 +212,11 @@ func benchmarkBlockWrite(b *testing.B, d *digest) {
 
 // BenchmarkPermutationFunction measures the speed of the permutation function with no input data.
 func BenchmarkPermutationFunction(b *testing.B) {
-	b.StopTimer()
-	d := testDigests["Keccak512"]
-	d.Reset()
 	b.SetBytes(int64(stateSize))
-	b.StartTimer()
+	var lanes [numLanes]uint64
 	for i := 0; i < b.N; i++ {
-		d.keccakF()
+		keccakF(&lanes)
 	}
-	b.StopTimer()
-	d.Reset()
 }
 
 // BenchmarkSingleByteWrite tests the latency from writing a single byte