24 #ifndef CRYPTOPP_IMPORTS 28 #if (CRYPTOPP_CLMUL_AVAILABLE) 29 # include <emmintrin.h> 30 # include <wmmintrin.h> 33 #if (CRYPTOPP_ARM_PMULL_AVAILABLE) 37 #if defined(CRYPTOPP_ALTIVEC_AVAILABLE) 41 ANONYMOUS_NAMESPACE_BEGIN
47 #if (CRYPTOPP_ARM_PMULL_AVAILABLE) 51 F2N_Multiply_128x128_ARMv8(uint64x2_t& c1, uint64x2_t& c0,
const uint64x2_t& a,
const uint64x2_t& b)
53 uint64x2_t t1, t2, z0={0};
57 t1 = vmovq_n_u64(vgetq_lane_u64(a, 1));
58 t1 = veorq_u64(a, t1);
59 t2 = vmovq_n_u64(vgetq_lane_u64(b, 1));
60 t2 = veorq_u64(b, t2);
62 t1 = veorq_u64(c0, t1);
63 t1 = veorq_u64(c1, t1);
65 t1 = vextq_u64(z0, t1, 1);
66 t2 = vextq_u64(t2, z0, 1);
67 c0 = veorq_u64(c0, t1);
68 c1 = veorq_u64(c1, t2);
73 F2N_Multiply_256x256_ARMv8(uint64x2_t& c3, uint64x2_t& c2, uint64x2_t& c1, uint64x2_t& c0,
74 const uint64x2_t& b1,
const uint64x2_t& b0,
const uint64x2_t& a1,
const uint64x2_t& a0)
77 uint64x2_t x0=a0, x1=a1, y0=b0, y1=b1;
79 F2N_Multiply_128x128_ARMv8(c1, c0, x0, y0);
80 F2N_Multiply_128x128_ARMv8(c3, c2, x1, y1);
82 x0 = veorq_u64(x0, x1);
83 y0 = veorq_u64(y0, y1);
85 F2N_Multiply_128x128_ARMv8(c5, c4, x0, y0);
87 c4 = veorq_u64(c4, c0);
88 c4 = veorq_u64(c4, c2);
89 c5 = veorq_u64(c5, c1);
90 c5 = veorq_u64(c5, c3);
91 c1 = veorq_u64(c1, c4);
92 c2 = veorq_u64(c2, c5);
97 F2N_Square_256_ARMv8(uint64x2_t& c3, uint64x2_t& c2, uint64x2_t& c1,
98 uint64x2_t& c0,
const uint64x2_t& a1,
const uint64x2_t& a0)
107 template <
unsigned int N>
108 inline uint64x2_t ShiftLeft128_ARMv8(uint64x2_t x)
110 uint64x2_t u=x, v, z={0};
111 x = vshlq_n_u64(x, N);
112 u = vshrq_n_u64(u, (64-N));
113 v = vcombine_u64(vget_low_u64(z), vget_low_u64(u));
121 GF2NT_233_Reduce_ARMv8(uint64x2_t& c3, uint64x2_t& c2, uint64x2_t& c1, uint64x2_t& c0)
123 const unsigned int mask[4] = {
124 0xffffffff, 0xffffffff, 0xffffffff, 0x000001ff,
127 uint64x2_t b3, b2, b1, a1, a0, m0, z0={0};
128 m0 = vreinterpretq_u64_u32(vld1q_u32(mask));
130 a0 = vcombine_u64(vget_low_u64(c1), vget_low_u64(z0));
131 a1 = vshlq_n_u64(a1, 23);
132 a1 = vshrq_n_u64(a1, 23);
133 c1 = vorrq_u64(a1, a0);
134 b2 = vshrq_n_u64(c2, (64-23));
135 c3 = ShiftLeft128_ARMv8<23>(c3);
136 a0 = vcombine_u64(vget_high_u64(b2), vget_high_u64(z0));
137 c3 = vorrq_u64(c3, a0);
138 b1 = vshrq_n_u64(b1, (64-23));
139 c2 = ShiftLeft128_ARMv8<23>(c2);
140 a0 = vcombine_u64(vget_high_u64(b1), vget_high_u64(z0));
141 c2 = vorrq_u64(c2, a0);
143 b2 = vshrq_n_u64(c2, (64-10));
144 b3 = ShiftLeft128_ARMv8<10>(b3);
145 a0 = vcombine_u64(vget_high_u64(b2), vget_high_u64(z0));
146 b3 = vorrq_u64(b3, a0);
147 a0 = vcombine_u64(vget_high_u64(c3), vget_high_u64(z0));
148 b3 = veorq_u64(b3, a0);
149 b1 = vshrq_n_u64(b3, (64-23));
150 b3 = ShiftLeft128_ARMv8<23>(b3);
151 b3 = vcombine_u64(vget_high_u64(b3), vget_high_u64(z0));
152 b3 = vorrq_u64(b3, b1);
153 c2 = veorq_u64(c2, b3);
155 b2 = vshrq_n_u64(c2, (64-10));
156 b3 = ShiftLeft128_ARMv8<10>(b3);
157 b2 = vcombine_u64(vget_high_u64(b2), vget_high_u64(z0));
158 b3 = vorrq_u64(b3, b2);
160 b2 = ShiftLeft128_ARMv8<10>(b2);
161 a0 = vcombine_u64(vget_low_u64(z0), vget_low_u64(b2));
162 c2 = veorq_u64(c2, a0);
163 a0 = vcombine_u64(vget_low_u64(z0), vget_low_u64(b3));
164 a1 = vcombine_u64(vget_high_u64(b2), vget_high_u64(z0));
165 a0 = vorrq_u64(a0, a1);
166 c3 = veorq_u64(c3, a0);
167 c0 = veorq_u64(c0, c2);
168 c1 = veorq_u64(c1, c3);
169 c1 = vandq_u64(c1, m0);
176 #if (CRYPTOPP_CLMUL_AVAILABLE) 178 using CryptoPP::word;
182 F2N_Multiply_128x128_CLMUL(__m128i& c1, __m128i& c0,
const __m128i& a,
const __m128i& b)
186 c0 = _mm_clmulepi64_si128(a, b, 0x00);
187 c1 = _mm_clmulepi64_si128(a, b, 0x11);
188 t1 = _mm_shuffle_epi32(a, 0xEE);
189 t1 = _mm_xor_si128(a, t1);
190 t2 = _mm_shuffle_epi32(b, 0xEE);
191 t2 = _mm_xor_si128(b, t2);
192 t1 = _mm_clmulepi64_si128(t1, t2, 0x00);
193 t1 = _mm_xor_si128(c0, t1);
194 t1 = _mm_xor_si128(c1, t1);
196 t1 = _mm_slli_si128(t1, 8);
197 t2 = _mm_srli_si128(t2, 8);
198 c0 = _mm_xor_si128(c0, t1);
199 c1 = _mm_xor_si128(c1, t2);
204 F2N_Multiply_256x256_CLMUL(__m128i& c3, __m128i& c2, __m128i& c1, __m128i& c0,
205 const __m128i& b1,
const __m128i& b0,
const __m128i& a1,
const __m128i& a0)
208 __m128i x0=a0, x1=a1, y0=b0, y1=b1;
210 F2N_Multiply_128x128_CLMUL(c1, c0, x0, y0);
211 F2N_Multiply_128x128_CLMUL(c3, c2, x1, y1);
213 x0 = _mm_xor_si128(x0, x1);
214 y0 = _mm_xor_si128(y0, y1);
216 F2N_Multiply_128x128_CLMUL(c5, c4, x0, y0);
218 c4 = _mm_xor_si128(c4, c0);
219 c4 = _mm_xor_si128(c4, c2);
220 c5 = _mm_xor_si128(c5, c1);
221 c5 = _mm_xor_si128(c5, c3);
222 c1 = _mm_xor_si128(c1, c4);
223 c2 = _mm_xor_si128(c2, c5);
228 F2N_Square_256_CLMUL(__m128i& c3, __m128i& c2, __m128i& c1,
229 __m128i& c0,
const __m128i& a1,
const __m128i& a0)
231 c0 = _mm_clmulepi64_si128(a0, a0, 0x00);
232 c1 = _mm_clmulepi64_si128(a0, a0, 0x11);
233 c2 = _mm_clmulepi64_si128(a1, a1, 0x00);
234 c3 = _mm_clmulepi64_si128(a1, a1, 0x11);
238 template <
unsigned int N>
239 inline __m128i ShiftLeft128_SSE(__m128i x,
const __m128i& z)
242 x = _mm_slli_epi64(x, N);
243 u = _mm_srli_epi64(u, (64-N));
244 v = _mm_unpacklo_epi64(z, u);
245 x = _mm_or_si128(x, v);
252 GF2NT_233_Reduce_CLMUL(__m128i& c3, __m128i& c2, __m128i& c1, __m128i& c0)
254 const unsigned int m[4] = {
255 0xffffffff, 0xffffffff, 0xffffffff, 0x000001ff
258 __m128i b3, b2, b1, a1, a0, m0, z0;
259 m0 = _mm_set_epi32(m[3], m[2], m[1], m[0]);
260 z0 = _mm_setzero_si128();
262 a0 = _mm_move_epi64(c1);
263 a1 = _mm_slli_epi64(a1, 23);
264 a1 = _mm_srli_epi64(a1, 23);
265 c1 = _mm_or_si128(a1, a0);
266 b2 = _mm_srli_epi64(c2, (64-23));
267 c3 = ShiftLeft128_SSE<23>(c3, z0);
268 a0 = _mm_unpackhi_epi64(b2, z0);
269 c3 = _mm_or_si128(c3, a0);
270 b1 = _mm_srli_epi64(b1, (64-23));
271 c2 = ShiftLeft128_SSE<23>(c2, z0);
272 a0 = _mm_unpackhi_epi64(b1, z0);
273 c2 = _mm_or_si128(c2, a0);
275 b2 = _mm_srli_epi64(c2, (64-10));
276 b3 = ShiftLeft128_SSE<10>(b3, z0);
277 a0 = _mm_unpackhi_epi64(b2, z0);
278 b3 = _mm_or_si128(b3, a0);
279 a0 = _mm_unpackhi_epi64(c3, z0);
280 b3 = _mm_xor_si128(b3, a0);
281 b1 = _mm_srli_epi64(b3, (64-23));
282 b3 = ShiftLeft128_SSE<23>(b3, z0);
283 b3 = _mm_unpackhi_epi64(b3, z0);
284 b3 = _mm_or_si128(b3, b1);
285 c2 = _mm_xor_si128(c2, b3);
287 b2 = _mm_srli_epi64(c2, (64-10));
288 b3 = ShiftLeft128_SSE<10>(b3, z0);
289 b2 = _mm_unpackhi_epi64(b2, z0);
290 b3 = _mm_or_si128(b3, b2);
292 b2 = ShiftLeft128_SSE<10>(b2, z0);
293 a0 = _mm_unpacklo_epi64(z0, b2);
294 c2 = _mm_xor_si128(c2, a0);
295 a0 = _mm_unpacklo_epi64(z0, b3);
296 a1 = _mm_unpackhi_epi64(b2, z0);
297 a0 = _mm_or_si128(a0, a1);
298 c3 = _mm_xor_si128(c3, a0);
299 c0 = _mm_xor_si128(c0, c2);
300 c1 = _mm_xor_si128(c1, c3);
301 c1 = _mm_and_si128(c1, m0);
308 #if (CRYPTOPP_POWER8_VMULL_AVAILABLE) 310 using CryptoPP::byte;
311 using CryptoPP::word;
362 F2N_Multiply_128x128_POWER8(c1, c0, x0, y0);
363 F2N_Multiply_128x128_POWER8(c3, c2, x1, y1);
368 F2N_Multiply_128x128_POWER8(c5, c4, x0, y0);
390 template <
unsigned int N>
396 x = VecShiftLeft<N>(x);
408 const uint64_t mod[] = {W64LIT(0xffffffffffffffff), W64LIT(0x01ffffffffff)};
416 a1 = VecShiftLeft<23>(a1);
417 a1 = VecShiftRight<23>(a1);
420 c3 = ShiftLeft128_POWER8<23>(c3);
424 c2 = ShiftLeft128_POWER8<23>(c2);
429 b3 = ShiftLeft128_POWER8<10>(b3);
435 b3 = ShiftLeft128_POWER8<23>(b3);
441 b3 = ShiftLeft128_POWER8<10>(b3);
445 b2 = ShiftLeft128_POWER8<10>(b2);
459 ANONYMOUS_NAMESPACE_END
463 #if (CRYPTOPP_CLMUL_AVAILABLE) 466 GF2NT_233_Multiply_Reduce_CLMUL(
const word* pA,
const word* pB, word* pC)
468 const __m128i* pAA =
reinterpret_cast<const __m128i*
>(pA);
469 const __m128i* pBB =
reinterpret_cast<const __m128i*
>(pB);
470 __m128i a0 = _mm_loadu_si128(pAA+0);
471 __m128i a1 = _mm_loadu_si128(pAA+1);
472 __m128i b0 = _mm_loadu_si128(pBB+0);
473 __m128i b1 = _mm_loadu_si128(pBB+1);
475 __m128i c0, c1, c2, c3;
476 F2N_Multiply_256x256_CLMUL(c3, c2, c1, c0, a1, a0, b1, b0);
477 GF2NT_233_Reduce_CLMUL(c3, c2, c1, c0);
479 __m128i* pCC =
reinterpret_cast<__m128i*
>(pC);
480 _mm_storeu_si128(pCC+0, c0);
481 _mm_storeu_si128(pCC+1, c1);
485 GF2NT_233_Square_Reduce_CLMUL(
const word* pA, word* pC)
487 const __m128i* pAA =
reinterpret_cast<const __m128i*
>(pA);
488 __m128i a0 = _mm_loadu_si128(pAA+0);
489 __m128i a1 = _mm_loadu_si128(pAA+1);
491 __m128i c0, c1, c2, c3;
492 F2N_Square_256_CLMUL(c3, c2, c1, c0, a1, a0);
493 GF2NT_233_Reduce_CLMUL(c3, c2, c1, c0);
495 __m128i* pCC =
reinterpret_cast<__m128i*
>(pC);
496 _mm_storeu_si128(pCC+0, c0);
497 _mm_storeu_si128(pCC+1, c1);
500 #elif (CRYPTOPP_ARM_PMULL_AVAILABLE) 503 GF2NT_233_Multiply_Reduce_ARMv8(
const word* pA,
const word* pB, word* pC)
507 const uint32_t* pAA =
reinterpret_cast<const uint32_t*
>(pA);
508 const uint32_t* pBB =
reinterpret_cast<const uint32_t*
>(pB);
510 uint64x2_t a0 = vreinterpretq_u64_u32(vld1q_u32(pAA+0));
511 uint64x2_t a1 = vreinterpretq_u64_u32(vld1q_u32(pAA+4));
512 uint64x2_t b0 = vreinterpretq_u64_u32(vld1q_u32(pBB+0));
513 uint64x2_t b1 = vreinterpretq_u64_u32(vld1q_u32(pBB+4));
515 uint64x2_t c0, c1, c2, c3;
516 F2N_Multiply_256x256_ARMv8(c3, c2, c1, c0, a1, a0, b1, b0);
517 GF2NT_233_Reduce_ARMv8(c3, c2, c1, c0);
519 uint32_t* pCC =
reinterpret_cast<uint32_t*
>(pC);
520 vst1q_u32(pCC+0, vreinterpretq_u32_u64(c0));
521 vst1q_u32(pCC+4, vreinterpretq_u32_u64(c1));
525 GF2NT_233_Square_Reduce_ARMv8(
const word* pA, word* pC)
529 const uint32_t* pAA =
reinterpret_cast<const uint32_t*
>(pA);
530 uint64x2_t a0 = vreinterpretq_u64_u32(vld1q_u32(pAA+0));
531 uint64x2_t a1 = vreinterpretq_u64_u32(vld1q_u32(pAA+4));
533 uint64x2_t c0, c1, c2, c3;
534 F2N_Square_256_ARMv8(c3, c2, c1, c0, a1, a0);
535 GF2NT_233_Reduce_ARMv8(c3, c2, c1, c0);
537 uint32_t* pCC =
reinterpret_cast<uint32_t*
>(pC);
538 vst1q_u32(pCC+0, vreinterpretq_u32_u64(c0));
539 vst1q_u32(pCC+4, vreinterpretq_u32_u64(c1));
542 #elif (CRYPTOPP_POWER8_VMULL_AVAILABLE) 545 GF2NT_233_Multiply_Reduce_POWER8(
const word* pA,
const word* pB, word* pC)
549 const byte* pAA =
reinterpret_cast<const byte*
>(pA);
550 const byte* pBB =
reinterpret_cast<const byte*
>(pB);
557 #if (CRYPTOPP_BIG_ENDIAN) 558 const uint8_t mb[] = {4,5,6,7, 0,1,2,3, 12,13,14,15, 8,9,10,11};
567 F2N_Multiply_256x256_POWER8(c3, c2, c1, c0, a1, a0, b1, b0);
568 GF2NT_233_Reduce_POWER8(c3, c2, c1, c0);
570 #if (CRYPTOPP_BIG_ENDIAN) 575 byte* pCC =
reinterpret_cast<byte*
>(pC);
581 GF2NT_233_Square_Reduce_POWER8(
const word* pA, word* pC)
585 const byte* pAA =
reinterpret_cast<const byte*
>(pA);
589 #if (CRYPTOPP_BIG_ENDIAN) 590 const uint8_t mb[] = {4,5,6,7, 0,1,2,3, 12,13,14,15, 8,9,10,11};
597 F2N_Square_256_POWER8(c3, c2, c1, c0, a1, a0);
598 GF2NT_233_Reduce_POWER8(c3, c2, c1, c0);
600 #if (CRYPTOPP_BIG_ENDIAN) 605 byte* pCC =
reinterpret_cast<byte*
>(pC);
614 #endif // CRYPTOPP_IMPORTS Library configuration file.
uint32x4_p VecShiftLeft(const uint32x4_p vec)
Shift a packed vector left.
T1 VecPermute(const T1 vec, const T2 mask)
Permutes a vector.
uint64x2_p VecPolyMultiply00LE(const uint64x2_p &a, const uint64x2_p &b)
Polynomial multiplication.
uint64x2_t PMULL_00(const uint64x2_t a, const uint64x2_t b)
Polynomial multiplication.
Support functions for PowerPC and vector operations.
T VecMergeHigh(const T vec1, const T vec2)
Merge two vectors.
void VecStore(const T data, byte dest[16])
Stores a vector to a byte array.
Classes and functions for schemes over GF(2^n)
T VecMergeLow(const T vec1, const T vec2)
Merge two vectors.
T1 VecXor(const T1 vec1, const T2 vec2)
XOR two vectors.
uint32x4_p VecShiftRight(const uint32x4_p vec)
Shift a packed vector right.
__vector unsigned long long uint64x2_p
Vector of 64-bit elements.
T1 VecOr(const T1 vec1, const T2 vec2)
OR two vectors.
uint64x2_t PMULL_11(const uint64x2_t a, const uint64x2_t b)
Polynomial multiplication.
Crypto++ library namespace.
uint32x4_p VecLoad(const byte src[16])
Loads a vector from a byte array.
__vector unsigned char uint8x16_p
Vector of 8-bit elements.
T1 VecAnd(const T1 vec1, const T2 vec2)
AND two vectors.
uint64x2_p VecPolyMultiply11LE(const uint64x2_p &a, const uint64x2_p &b)
Polynomial multiplication.
Support functions for ARM and vector operations.