|
@@ -209,7 +209,7 @@ GLOBL ·andMask<>(SB), (NOPTR+RODATA), $240
|
|
|
#define polyMul polyMulStage1; polyMulStage2; polyMulStage3; polyMulReduceStage
|
|
#define polyMul polyMulStage1; polyMulStage2; polyMulStage3; polyMulReduceStage
|
|
|
#define polyMulAVX2 polyMulStage1_AVX2; polyMulStage2_AVX2; polyMulStage3_AVX2; polyMulReduceStage
|
|
#define polyMulAVX2 polyMulStage1_AVX2; polyMulStage2_AVX2; polyMulStage3_AVX2; polyMulReduceStage
|
|
|
// ----------------------------------------------------------------------------
|
|
// ----------------------------------------------------------------------------
|
|
|
-TEXT polyHashADInternal(SB), NOSPLIT, $0
|
|
|
|
|
|
|
+TEXT polyHashADInternal<>(SB), NOSPLIT, $0
|
|
|
// adp points to beginning of additional data
|
|
// adp points to beginning of additional data
|
|
|
// itr2 holds ad length
|
|
// itr2 holds ad length
|
|
|
XORQ acc0, acc0
|
|
XORQ acc0, acc0
|
|
@@ -315,7 +315,7 @@ openSSEPreparePolyKey:
|
|
|
|
|
|
|
|
// Hash AAD
|
|
// Hash AAD
|
|
|
MOVQ ad_len+80(FP), itr2
|
|
MOVQ ad_len+80(FP), itr2
|
|
|
- CALL polyHashADInternal(SB)
|
|
|
|
|
|
|
+ CALL polyHashADInternal<>(SB)
|
|
|
|
|
|
|
|
openSSEMainLoop:
|
|
openSSEMainLoop:
|
|
|
CMPQ inl, $256
|
|
CMPQ inl, $256
|
|
@@ -476,7 +476,7 @@ openSSE128InnerCipherLoop:
|
|
|
|
|
|
|
|
// Hash
|
|
// Hash
|
|
|
MOVQ ad_len+80(FP), itr2
|
|
MOVQ ad_len+80(FP), itr2
|
|
|
- CALL polyHashADInternal(SB)
|
|
|
|
|
|
|
+ CALL polyHashADInternal<>(SB)
|
|
|
|
|
|
|
|
openSSE128Open:
|
|
openSSE128Open:
|
|
|
CMPQ inl, $16
|
|
CMPQ inl, $16
|
|
@@ -822,7 +822,7 @@ openAVX2PreparePolyKey:
|
|
|
|
|
|
|
|
// Hash AD + first 64 bytes
|
|
// Hash AD + first 64 bytes
|
|
|
MOVQ ad_len+80(FP), itr2
|
|
MOVQ ad_len+80(FP), itr2
|
|
|
- CALL polyHashADInternal(SB)
|
|
|
|
|
|
|
+ CALL polyHashADInternal<>(SB)
|
|
|
XORQ itr1, itr1
|
|
XORQ itr1, itr1
|
|
|
|
|
|
|
|
openAVX2InitialHash64:
|
|
openAVX2InitialHash64:
|
|
@@ -1014,7 +1014,7 @@ openAVX2192InnerCipherLoop:
|
|
|
openAVX2ShortOpen:
|
|
openAVX2ShortOpen:
|
|
|
// Hash
|
|
// Hash
|
|
|
MOVQ ad_len+80(FP), itr2
|
|
MOVQ ad_len+80(FP), itr2
|
|
|
- CALL polyHashADInternal(SB)
|
|
|
|
|
|
|
+ CALL polyHashADInternal<>(SB)
|
|
|
|
|
|
|
|
openAVX2ShortOpenLoop:
|
|
openAVX2ShortOpenLoop:
|
|
|
CMPQ inl, $32
|
|
CMPQ inl, $32
|
|
@@ -1547,7 +1547,7 @@ sealSSEIntroLoop:
|
|
|
|
|
|
|
|
// Hash AAD
|
|
// Hash AAD
|
|
|
MOVQ ad_len+80(FP), itr2
|
|
MOVQ ad_len+80(FP), itr2
|
|
|
- CALL polyHashADInternal(SB)
|
|
|
|
|
|
|
+ CALL polyHashADInternal<>(SB)
|
|
|
|
|
|
|
|
MOVOU (0*16)(inp), A0; MOVOU (1*16)(inp), B0; MOVOU (2*16)(inp), C0; MOVOU (3*16)(inp), D0
|
|
MOVOU (0*16)(inp), A0; MOVOU (1*16)(inp), B0; MOVOU (2*16)(inp), C0; MOVOU (3*16)(inp), D0
|
|
|
PXOR A0, A1; PXOR B0, B1; PXOR C0, C1; PXOR D0, D1
|
|
PXOR A0, A1; PXOR B0, B1; PXOR C0, C1; PXOR D0, D1
|
|
@@ -1852,7 +1852,7 @@ sealSSE128InnerCipherLoop:
|
|
|
|
|
|
|
|
// Hash
|
|
// Hash
|
|
|
MOVQ ad_len+80(FP), itr2
|
|
MOVQ ad_len+80(FP), itr2
|
|
|
- CALL polyHashADInternal(SB)
|
|
|
|
|
|
|
+ CALL polyHashADInternal<>(SB)
|
|
|
XORQ itr1, itr1
|
|
XORQ itr1, itr1
|
|
|
|
|
|
|
|
sealSSE128SealHash:
|
|
sealSSE128SealHash:
|
|
@@ -2027,7 +2027,7 @@ sealAVX2IntroLoop:
|
|
|
|
|
|
|
|
// Hash AD
|
|
// Hash AD
|
|
|
MOVQ ad_len+80(FP), itr2
|
|
MOVQ ad_len+80(FP), itr2
|
|
|
- CALL polyHashADInternal(SB)
|
|
|
|
|
|
|
+ CALL polyHashADInternal<>(SB)
|
|
|
|
|
|
|
|
// Can store at least 320 bytes
|
|
// Can store at least 320 bytes
|
|
|
VPXOR (0*32)(inp), AA0, AA0
|
|
VPXOR (0*32)(inp), AA0, AA0
|
|
@@ -2290,7 +2290,7 @@ sealAVX2192InnerCipherLoop:
|
|
|
sealAVX2ShortSeal:
|
|
sealAVX2ShortSeal:
|
|
|
// Hash aad
|
|
// Hash aad
|
|
|
MOVQ ad_len+80(FP), itr2
|
|
MOVQ ad_len+80(FP), itr2
|
|
|
- CALL polyHashADInternal(SB)
|
|
|
|
|
|
|
+ CALL polyHashADInternal<>(SB)
|
|
|
XORQ itr1, itr1
|
|
XORQ itr1, itr1
|
|
|
|
|
|
|
|
sealAVX2SealHash:
|
|
sealAVX2SealHash:
|