Crypto++  8.2
Free C++ class library of cryptographic schemes
rijndael_simd.cpp
1 // rijndael_simd.cpp - written and placed in the public domain by
2 // Jeffrey Walton, Uri Blumenthal and Marcel Raad.
3 // AES-NI code originally written by Wei Dai.
4 //
5 // This source file uses intrinsics and built-ins to gain access to
6 // AES-NI, ARMv8a AES and Power8 AES instructions. A separate source
7 // file is needed because additional CXXFLAGS are required to enable
8 // the appropriate instructions sets in some build configurations.
9 //
10 // ARMv8a AES code based on CriticalBlue code from Johannes Schneiders,
11 // Skip Hovsmith and Barry O'Rourke for the mbedTLS project. Stepping
12 // mbedTLS under a debugger was helped for us to determine problems
13 // with our subkey generation and scheduling.
14 //
15 // AltiVec and Power8 code based on http://github.com/noloader/AES-Intrinsics and
16 // http://www.ibm.com/developerworks/library/se-power8-in-core-cryptography/
17 // For Power8 do not remove the casts, even when const-ness is cast away. It causes
18 // failed compiles and a 0.3 to 0.6 cpb drop in performance. The IBM documentation
19 // absolutely sucks. Thanks to Andy Polyakov, Paul R and Trudeaun for answering
20 // questions and filling the gaps in the IBM documentation.
21 //
22 
23 #include "pch.h"
24 #include "config.h"
25 #include "misc.h"
26 
27 #if (CRYPTOPP_AESNI_AVAILABLE)
28 # include "adv_simd.h"
29 # include <emmintrin.h>
30 # include <smmintrin.h>
31 # include <wmmintrin.h>
32 #endif
33 
34 // C1189: error: This header is specific to ARM targets
35 #if (CRYPTOPP_ARM_NEON_AVAILABLE)
36 # include "adv_simd.h"
37 # ifndef _M_ARM64
38 # include <arm_neon.h>
39 # endif
40 #endif
41 
42 #if (CRYPTOPP_ARM_ACLE_AVAILABLE)
43 # include <stdint.h>
44 # include <arm_acle.h>
45 #endif
46 
47 #if defined(CRYPTOPP_POWER8_AES_AVAILABLE)
48 # include "adv_simd.h"
49 # include "ppc_simd.h"
50 #endif
51 
52 #ifdef CRYPTOPP_GNU_STYLE_INLINE_ASSEMBLY
53 # include <signal.h>
54 # include <setjmp.h>
55 #endif
56 
57 #ifndef EXCEPTION_EXECUTE_HANDLER
58 # define EXCEPTION_EXECUTE_HANDLER 1
59 #endif
60 
61 // Clang __m128i casts, http://bugs.llvm.org/show_bug.cgi?id=20670
62 #define M128_CAST(x) ((__m128i *)(void *)(x))
63 #define CONST_M128_CAST(x) ((const __m128i *)(const void *)(x))
64 
65 // Squash MS LNK4221 and libtool warnings
66 extern const char RIJNDAEL_SIMD_FNAME[] = __FILE__;
67 
68 NAMESPACE_BEGIN(CryptoPP)
69 
70 // ************************* Feature Probes ************************* //
71 
72 #ifdef CRYPTOPP_GNU_STYLE_INLINE_ASSEMBLY
73 extern "C" {
74  typedef void (*SigHandler)(int);
75 
76  static jmp_buf s_jmpSIGILL;
77  static void SigIllHandler(int)
78  {
79  longjmp(s_jmpSIGILL, 1);
80  }
81 }
82 #endif // Not CRYPTOPP_MS_STYLE_INLINE_ASSEMBLY
83 
84 #if (CRYPTOPP_BOOL_ARM32 || CRYPTOPP_BOOL_ARMV8)
85 bool CPU_ProbeAES()
86 {
87 #if defined(CRYPTOPP_NO_CPU_FEATURE_PROBES)
88  return false;
89 #elif (CRYPTOPP_ARM_AES_AVAILABLE)
90 # if defined(CRYPTOPP_MS_STYLE_INLINE_ASSEMBLY)
91  volatile bool result = true;
92  __try
93  {
94  // AES encrypt and decrypt
95  uint8x16_t data = vdupq_n_u8(0), key = vdupq_n_u8(0);
96  uint8x16_t r1 = vaeseq_u8(data, key);
97  uint8x16_t r2 = vaesdq_u8(data, key);
98  r1 = vaesmcq_u8(r1);
99  r2 = vaesimcq_u8(r2);
100 
101  result = !!(vgetq_lane_u8(r1,0) | vgetq_lane_u8(r2,7));
102  }
103  __except (EXCEPTION_EXECUTE_HANDLER)
104  {
105  return false;
106  }
107  return result;
108 # else
109 
110  // longjmp and clobber warnings. Volatile is required.
111  // http://github.com/weidai11/cryptopp/issues/24 and http://stackoverflow.com/q/7721854
112  volatile bool result = true;
113 
114  volatile SigHandler oldHandler = signal(SIGILL, SigIllHandler);
115  if (oldHandler == SIG_ERR)
116  return false;
117 
118  volatile sigset_t oldMask;
119  if (sigprocmask(0, NULLPTR, (sigset_t*)&oldMask))
120  return false;
121 
122  if (setjmp(s_jmpSIGILL))
123  result = false;
124  else
125  {
126  uint8x16_t data = vdupq_n_u8(0), key = vdupq_n_u8(0);
127  uint8x16_t r1 = vaeseq_u8(data, key);
128  uint8x16_t r2 = vaesdq_u8(data, key);
129  r1 = vaesmcq_u8(r1);
130  r2 = vaesimcq_u8(r2);
131 
132  // Hack... GCC optimizes away the code and returns true
133  result = !!(vgetq_lane_u8(r1,0) | vgetq_lane_u8(r2,7));
134  }
135 
136  sigprocmask(SIG_SETMASK, (sigset_t*)&oldMask, NULLPTR);
137  signal(SIGILL, oldHandler);
138  return result;
139 # endif
140 #else
141  return false;
142 #endif // CRYPTOPP_ARM_AES_AVAILABLE
143 }
144 #endif // ARM32 or ARM64
145 
146 // ***************************** ARMv8 ***************************** //
147 
148 #if (CRYPTOPP_ARM_AES_AVAILABLE)
149 
150 ANONYMOUS_NAMESPACE_BEGIN
151 
152 static inline void ARMV8_Enc_Block(uint64x2_t &data, const word32 *subkeys, unsigned int rounds)
153 {
154  CRYPTOPP_ASSERT(subkeys);
155  const byte *keys = reinterpret_cast<const byte*>(subkeys);
156  uint8x16_t block = vreinterpretq_u8_u64(data);
157 
158  // AES single round encryption
159  block = vaeseq_u8(block, vld1q_u8(keys+0*16));
160  // AES mix columns
161  block = vaesmcq_u8(block);
162 
163  for (unsigned int i=1; i<rounds-1; i+=2)
164  {
165  // AES single round encryption
166  block = vaeseq_u8(block, vld1q_u8(keys+i*16));
167  // AES mix columns
168  block = vaesmcq_u8(block);
169  // AES single round encryption
170  block = vaeseq_u8(block, vld1q_u8(keys+(i+1)*16));
171  // AES mix columns
172  block = vaesmcq_u8(block);
173  }
174 
175  // AES single round encryption
176  block = vaeseq_u8(block, vld1q_u8(keys+(rounds-1)*16));
177  // Final Add (bitwise Xor)
178  block = veorq_u8(block, vld1q_u8(keys+rounds*16));
179 
180  data = vreinterpretq_u64_u8(block);
181 }
182 
183 static inline void ARMV8_Enc_6_Blocks(uint64x2_t &data0, uint64x2_t &data1,
184  uint64x2_t &data2, uint64x2_t &data3, uint64x2_t &data4, uint64x2_t &data5,
185  const word32 *subkeys, unsigned int rounds)
186 {
187  CRYPTOPP_ASSERT(subkeys);
188  const byte *keys = reinterpret_cast<const byte*>(subkeys);
189 
190  uint8x16_t block0 = vreinterpretq_u8_u64(data0);
191  uint8x16_t block1 = vreinterpretq_u8_u64(data1);
192  uint8x16_t block2 = vreinterpretq_u8_u64(data2);
193  uint8x16_t block3 = vreinterpretq_u8_u64(data3);
194  uint8x16_t block4 = vreinterpretq_u8_u64(data4);
195  uint8x16_t block5 = vreinterpretq_u8_u64(data5);
196 
197  uint8x16_t key;
198  for (unsigned int i=0; i<rounds-1; ++i)
199  {
200  key = vld1q_u8(keys+i*16);
201  // AES single round encryption
202  block0 = vaeseq_u8(block0, key);
203  // AES mix columns
204  block0 = vaesmcq_u8(block0);
205  // AES single round encryption
206  block1 = vaeseq_u8(block1, key);
207  // AES mix columns
208  block1 = vaesmcq_u8(block1);
209  // AES single round encryption
210  block2 = vaeseq_u8(block2, key);
211  // AES mix columns
212  block2 = vaesmcq_u8(block2);
213  // AES single round encryption
214  block3 = vaeseq_u8(block3, key);
215  // AES mix columns
216  block3 = vaesmcq_u8(block3);
217  // AES single round encryption
218  block4 = vaeseq_u8(block4, key);
219  // AES mix columns
220  block4 = vaesmcq_u8(block4);
221  // AES single round encryption
222  block5 = vaeseq_u8(block5, key);
223  // AES mix columns
224  block5 = vaesmcq_u8(block5);
225  }
226 
227  // AES single round encryption
228  key = vld1q_u8(keys+(rounds-1)*16);
229  block0 = vaeseq_u8(block0, key);
230  block1 = vaeseq_u8(block1, key);
231  block2 = vaeseq_u8(block2, key);
232  block3 = vaeseq_u8(block3, key);
233  block4 = vaeseq_u8(block4, key);
234  block5 = vaeseq_u8(block5, key);
235 
236  // Final Add (bitwise Xor)
237  key = vld1q_u8(keys+rounds*16);
238  data0 = vreinterpretq_u64_u8(veorq_u8(block0, key));
239  data1 = vreinterpretq_u64_u8(veorq_u8(block1, key));
240  data2 = vreinterpretq_u64_u8(veorq_u8(block2, key));
241  data3 = vreinterpretq_u64_u8(veorq_u8(block3, key));
242  data4 = vreinterpretq_u64_u8(veorq_u8(block4, key));
243  data5 = vreinterpretq_u64_u8(veorq_u8(block5, key));
244 }
245 
246 static inline void ARMV8_Dec_Block(uint64x2_t &data, const word32 *subkeys, unsigned int rounds)
247 {
248  CRYPTOPP_ASSERT(subkeys);
249  const byte *keys = reinterpret_cast<const byte*>(subkeys);
250  uint8x16_t block = vreinterpretq_u8_u64(data);
251 
252  // AES single round decryption
253  block = vaesdq_u8(block, vld1q_u8(keys+0*16));
254  // AES inverse mix columns
255  block = vaesimcq_u8(block);
256 
257  for (unsigned int i=1; i<rounds-1; i+=2)
258  {
259  // AES single round decryption
260  block = vaesdq_u8(block, vld1q_u8(keys+i*16));
261  // AES inverse mix columns
262  block = vaesimcq_u8(block);
263  // AES single round decryption
264  block = vaesdq_u8(block, vld1q_u8(keys+(i+1)*16));
265  // AES inverse mix columns
266  block = vaesimcq_u8(block);
267  }
268 
269  // AES single round decryption
270  block = vaesdq_u8(block, vld1q_u8(keys+(rounds-1)*16));
271  // Final Add (bitwise Xor)
272  block = veorq_u8(block, vld1q_u8(keys+rounds*16));
273 
274  data = vreinterpretq_u64_u8(block);
275 }
276 
277 static inline void ARMV8_Dec_6_Blocks(uint64x2_t &data0, uint64x2_t &data1,
278  uint64x2_t &data2, uint64x2_t &data3, uint64x2_t &data4, uint64x2_t &data5,
279  const word32 *subkeys, unsigned int rounds)
280 {
281  CRYPTOPP_ASSERT(subkeys);
282  const byte *keys = reinterpret_cast<const byte*>(subkeys);
283 
284  uint8x16_t block0 = vreinterpretq_u8_u64(data0);
285  uint8x16_t block1 = vreinterpretq_u8_u64(data1);
286  uint8x16_t block2 = vreinterpretq_u8_u64(data2);
287  uint8x16_t block3 = vreinterpretq_u8_u64(data3);
288  uint8x16_t block4 = vreinterpretq_u8_u64(data4);
289  uint8x16_t block5 = vreinterpretq_u8_u64(data5);
290 
291  uint8x16_t key;
292  for (unsigned int i=0; i<rounds-1; ++i)
293  {
294  key = vld1q_u8(keys+i*16);
295  // AES single round decryption
296  block0 = vaesdq_u8(block0, key);
297  // AES inverse mix columns
298  block0 = vaesimcq_u8(block0);
299  // AES single round decryption
300  block1 = vaesdq_u8(block1, key);
301  // AES inverse mix columns
302  block1 = vaesimcq_u8(block1);
303  // AES single round decryption
304  block2 = vaesdq_u8(block2, key);
305  // AES inverse mix columns
306  block2 = vaesimcq_u8(block2);
307  // AES single round decryption
308  block3 = vaesdq_u8(block3, key);
309  // AES inverse mix columns
310  block3 = vaesimcq_u8(block3);
311  // AES single round decryption
312  block4 = vaesdq_u8(block4, key);
313  // AES inverse mix columns
314  block4 = vaesimcq_u8(block4);
315  // AES single round decryption
316  block5 = vaesdq_u8(block5, key);
317  // AES inverse mix columns
318  block5 = vaesimcq_u8(block5);
319  }
320 
321  // AES single round decryption
322  key = vld1q_u8(keys+(rounds-1)*16);
323  block0 = vaesdq_u8(block0, key);
324  block1 = vaesdq_u8(block1, key);
325  block2 = vaesdq_u8(block2, key);
326  block3 = vaesdq_u8(block3, key);
327  block4 = vaesdq_u8(block4, key);
328  block5 = vaesdq_u8(block5, key);
329 
330  // Final Add (bitwise Xor)
331  key = vld1q_u8(keys+rounds*16);
332  data0 = vreinterpretq_u64_u8(veorq_u8(block0, key));
333  data1 = vreinterpretq_u64_u8(veorq_u8(block1, key));
334  data2 = vreinterpretq_u64_u8(veorq_u8(block2, key));
335  data3 = vreinterpretq_u64_u8(veorq_u8(block3, key));
336  data4 = vreinterpretq_u64_u8(veorq_u8(block4, key));
337  data5 = vreinterpretq_u64_u8(veorq_u8(block5, key));
338 }
339 
340 ANONYMOUS_NAMESPACE_END
341 
342 size_t Rijndael_Enc_AdvancedProcessBlocks_ARMV8(const word32 *subKeys, size_t rounds,
343  const byte *inBlocks, const byte *xorBlocks, byte *outBlocks, size_t length, word32 flags)
344 {
345  return AdvancedProcessBlocks128_6x1_NEON(ARMV8_Enc_Block, ARMV8_Enc_6_Blocks,
346  subKeys, rounds, inBlocks, xorBlocks, outBlocks, length, flags);
347 }
348 
349 size_t Rijndael_Dec_AdvancedProcessBlocks_ARMV8(const word32 *subKeys, size_t rounds,
350  const byte *inBlocks, const byte *xorBlocks, byte *outBlocks, size_t length, word32 flags)
351 {
352  return AdvancedProcessBlocks128_6x1_NEON(ARMV8_Dec_Block, ARMV8_Dec_6_Blocks,
353  subKeys, rounds, inBlocks, xorBlocks, outBlocks, length, flags);
354 }
355 
356 #endif // CRYPTOPP_ARM_AES_AVAILABLE
357 
358 // ***************************** AES-NI ***************************** //
359 
360 #if (CRYPTOPP_AESNI_AVAILABLE)
361 
362 ANONYMOUS_NAMESPACE_BEGIN
363 
364 /* for 128-bit blocks, Rijndael never uses more than 10 rcon values */
365 CRYPTOPP_ALIGN_DATA(16)
366 const word32 s_rconLE[] = {
367  0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x1B, 0x36
368 };
369 
370 static inline void AESNI_Enc_Block(__m128i &block, MAYBE_CONST word32 *subkeys, unsigned int rounds)
371 {
372  const __m128i* skeys = reinterpret_cast<const __m128i*>(subkeys);
373 
374  block = _mm_xor_si128(block, skeys[0]);
375  for (unsigned int i=1; i<rounds-1; i+=2)
376  {
377  block = _mm_aesenc_si128(block, skeys[i]);
378  block = _mm_aesenc_si128(block, skeys[i+1]);
379  }
380  block = _mm_aesenc_si128(block, skeys[rounds-1]);
381  block = _mm_aesenclast_si128(block, skeys[rounds]);
382 }
383 
384 static inline void AESNI_Enc_4_Blocks(__m128i &block0, __m128i &block1, __m128i &block2, __m128i &block3,
385  MAYBE_CONST word32 *subkeys, unsigned int rounds)
386 {
387  const __m128i* skeys = reinterpret_cast<const __m128i*>(subkeys);
388 
389  __m128i rk = skeys[0];
390  block0 = _mm_xor_si128(block0, rk);
391  block1 = _mm_xor_si128(block1, rk);
392  block2 = _mm_xor_si128(block2, rk);
393  block3 = _mm_xor_si128(block3, rk);
394  for (unsigned int i=1; i<rounds; i++)
395  {
396  rk = skeys[i];
397  block0 = _mm_aesenc_si128(block0, rk);
398  block1 = _mm_aesenc_si128(block1, rk);
399  block2 = _mm_aesenc_si128(block2, rk);
400  block3 = _mm_aesenc_si128(block3, rk);
401  }
402  rk = skeys[rounds];
403  block0 = _mm_aesenclast_si128(block0, rk);
404  block1 = _mm_aesenclast_si128(block1, rk);
405  block2 = _mm_aesenclast_si128(block2, rk);
406  block3 = _mm_aesenclast_si128(block3, rk);
407 }
408 
409 static inline void AESNI_Dec_Block(__m128i &block, MAYBE_CONST word32 *subkeys, unsigned int rounds)
410 {
411  const __m128i* skeys = reinterpret_cast<const __m128i*>(subkeys);
412 
413  block = _mm_xor_si128(block, skeys[0]);
414  for (unsigned int i=1; i<rounds-1; i+=2)
415  {
416  block = _mm_aesdec_si128(block, skeys[i]);
417  block = _mm_aesdec_si128(block, skeys[i+1]);
418  }
419  block = _mm_aesdec_si128(block, skeys[rounds-1]);
420  block = _mm_aesdeclast_si128(block, skeys[rounds]);
421 }
422 
423 static inline void AESNI_Dec_4_Blocks(__m128i &block0, __m128i &block1, __m128i &block2, __m128i &block3,
424  MAYBE_CONST word32 *subkeys, unsigned int rounds)
425 {
426  const __m128i* skeys = reinterpret_cast<const __m128i*>(subkeys);
427 
428  __m128i rk = skeys[0];
429  block0 = _mm_xor_si128(block0, rk);
430  block1 = _mm_xor_si128(block1, rk);
431  block2 = _mm_xor_si128(block2, rk);
432  block3 = _mm_xor_si128(block3, rk);
433  for (unsigned int i=1; i<rounds; i++)
434  {
435  rk = skeys[i];
436  block0 = _mm_aesdec_si128(block0, rk);
437  block1 = _mm_aesdec_si128(block1, rk);
438  block2 = _mm_aesdec_si128(block2, rk);
439  block3 = _mm_aesdec_si128(block3, rk);
440  }
441  rk = skeys[rounds];
442  block0 = _mm_aesdeclast_si128(block0, rk);
443  block1 = _mm_aesdeclast_si128(block1, rk);
444  block2 = _mm_aesdeclast_si128(block2, rk);
445  block3 = _mm_aesdeclast_si128(block3, rk);
446 }
447 
448 ANONYMOUS_NAMESPACE_END
449 
450 void Rijndael_UncheckedSetKey_SSE4_AESNI(const byte *userKey, size_t keyLen, word32 *rk)
451 {
452  const size_t rounds = keyLen / 4 + 6;
453  const word32 *rc = s_rconLE;
454 
455  __m128i temp = _mm_loadu_si128(M128_CAST(userKey+keyLen-16));
456  std::memcpy(rk, userKey, keyLen);
457 
458  // keySize: m_key allocates 4*(rounds+1) word32's.
459  const size_t keySize = 4*(rounds+1);
460  const word32* end = rk + keySize;
461 
462  while (true)
463  {
464  rk[keyLen/4] = rk[0] ^ _mm_extract_epi32(_mm_aeskeygenassist_si128(temp, 0), 3) ^ *(rc++);
465  rk[keyLen/4+1] = rk[1] ^ rk[keyLen/4];
466  rk[keyLen/4+2] = rk[2] ^ rk[keyLen/4+1];
467  rk[keyLen/4+3] = rk[3] ^ rk[keyLen/4+2];
468 
469  if (rk + keyLen/4 + 4 == end)
470  break;
471 
472  if (keyLen == 24)
473  {
474  rk[10] = rk[ 4] ^ rk[ 9];
475  rk[11] = rk[ 5] ^ rk[10];
476  temp = _mm_insert_epi32(temp, rk[11], 3);
477  }
478  else if (keyLen == 32)
479  {
480  temp = _mm_insert_epi32(temp, rk[11], 3);
481  rk[12] = rk[ 4] ^ _mm_extract_epi32(_mm_aeskeygenassist_si128(temp, 0), 2);
482  rk[13] = rk[ 5] ^ rk[12];
483  rk[14] = rk[ 6] ^ rk[13];
484  rk[15] = rk[ 7] ^ rk[14];
485  temp = _mm_insert_epi32(temp, rk[15], 3);
486  }
487  else
488  {
489  temp = _mm_insert_epi32(temp, rk[7], 3);
490  }
491 
492  rk += keyLen/4;
493  }
494 }
495 
496 void Rijndael_UncheckedSetKeyRev_AESNI(word32 *key, unsigned int rounds)
497 {
498  unsigned int i, j;
499  __m128i temp;
500 
501  vec_swap(*M128_CAST(key), *M128_CAST(key+4*rounds));
502 
503  for (i = 4, j = 4*rounds-4; i < j; i += 4, j -= 4)
504  {
505  temp = _mm_aesimc_si128(*M128_CAST(key+i));
506  *M128_CAST(key+i) = _mm_aesimc_si128(*M128_CAST(key+j));
507  *M128_CAST(key+j) = temp;
508  }
509 
510  *M128_CAST(key+i) = _mm_aesimc_si128(*M128_CAST(key+i));
511 }
512 
513 size_t Rijndael_Enc_AdvancedProcessBlocks_AESNI(const word32 *subKeys, size_t rounds,
514  const byte *inBlocks, const byte *xorBlocks, byte *outBlocks, size_t length, word32 flags)
515 {
516  // SunCC workaround
517  MAYBE_CONST word32* sk = MAYBE_UNCONST_CAST(word32*, subKeys);
518  MAYBE_CONST byte* ib = MAYBE_UNCONST_CAST(byte*, inBlocks);
519  MAYBE_CONST byte* xb = MAYBE_UNCONST_CAST(byte*, xorBlocks);
520 
521  return AdvancedProcessBlocks128_4x1_SSE(AESNI_Enc_Block, AESNI_Enc_4_Blocks,
522  sk, rounds, ib, xb, outBlocks, length, flags);
523 }
524 
525 size_t Rijndael_Dec_AdvancedProcessBlocks_AESNI(const word32 *subKeys, size_t rounds,
526  const byte *inBlocks, const byte *xorBlocks, byte *outBlocks, size_t length, word32 flags)
527 {
528  MAYBE_CONST word32* sk = MAYBE_UNCONST_CAST(word32*, subKeys);
529  MAYBE_CONST byte* ib = MAYBE_UNCONST_CAST(byte*, inBlocks);
530  MAYBE_CONST byte* xb = MAYBE_UNCONST_CAST(byte*, xorBlocks);
531 
532  return AdvancedProcessBlocks128_4x1_SSE(AESNI_Dec_Block, AESNI_Dec_4_Blocks,
533  sk, rounds, ib, xb, outBlocks, length, flags);
534 }
535 
536 #endif // CRYPTOPP_AESNI_AVAILABLE
537 
538 // ************************** Power 8 Crypto ************************** //
539 
540 #if (CRYPTOPP_POWER8_AES_AVAILABLE)
541 
542 ANONYMOUS_NAMESPACE_BEGIN
543 
544 /* for 128-bit blocks, Rijndael never uses more than 10 rcon values */
545 CRYPTOPP_ALIGN_DATA(16)
546 static const uint32_t s_rconBE[] = {
547  0x01000000, 0x02000000, 0x04000000, 0x08000000,
548  0x10000000, 0x20000000, 0x40000000, 0x80000000,
549  0x1B000000, 0x36000000
550 };
551 
552 static inline void POWER8_Enc_Block(uint32x4_p &block, const word32 *subkeys, unsigned int rounds)
553 {
554  CRYPTOPP_ASSERT(IsAlignedOn(subkeys, 16));
555  const byte *keys = reinterpret_cast<const byte*>(subkeys);
556 
557  uint32x4_p k = VecLoad(keys);
558  block = VecXor(block, k);
559 
560  for (size_t i=1; i<rounds-1; i+=2)
561  {
562  block = VecEncrypt(block, VecLoad( i*16, keys));
563  block = VecEncrypt(block, VecLoad((i+1)*16, keys));
564  }
565 
566  block = VecEncrypt(block, VecLoad((rounds-1)*16, keys));
567  block = VecEncryptLast(block, VecLoad(rounds*16, keys));
568 }
569 
570 static inline void POWER8_Enc_6_Blocks(uint32x4_p &block0, uint32x4_p &block1,
571  uint32x4_p &block2, uint32x4_p &block3, uint32x4_p &block4,
572  uint32x4_p &block5, const word32 *subkeys, unsigned int rounds)
573 {
574  CRYPTOPP_ASSERT(IsAlignedOn(subkeys, 16));
575  const byte *keys = reinterpret_cast<const byte*>(subkeys);
576 
577  uint32x4_p k = VecLoad(keys);
578  block0 = VecXor(block0, k);
579  block1 = VecXor(block1, k);
580  block2 = VecXor(block2, k);
581  block3 = VecXor(block3, k);
582  block4 = VecXor(block4, k);
583  block5 = VecXor(block5, k);
584 
585  for (size_t i=1; i<rounds; ++i)
586  {
587  k = VecLoad(i*16, keys);
588  block0 = VecEncrypt(block0, k);
589  block1 = VecEncrypt(block1, k);
590  block2 = VecEncrypt(block2, k);
591  block3 = VecEncrypt(block3, k);
592  block4 = VecEncrypt(block4, k);
593  block5 = VecEncrypt(block5, k);
594  }
595 
596  k = VecLoad(rounds*16, keys);
597  block0 = VecEncryptLast(block0, k);
598  block1 = VecEncryptLast(block1, k);
599  block2 = VecEncryptLast(block2, k);
600  block3 = VecEncryptLast(block3, k);
601  block4 = VecEncryptLast(block4, k);
602  block5 = VecEncryptLast(block5, k);
603 }
604 
605 static inline void POWER8_Dec_Block(uint32x4_p &block, const word32 *subkeys, unsigned int rounds)
606 {
607  CRYPTOPP_ASSERT(IsAlignedOn(subkeys, 16));
608  const byte *keys = reinterpret_cast<const byte*>(subkeys);
609 
610  uint32x4_p k = VecLoad(rounds*16, keys);
611  block = VecXor(block, k);
612 
613  for (size_t i=rounds-1; i>1; i-=2)
614  {
615  block = VecDecrypt(block, VecLoad( i*16, keys));
616  block = VecDecrypt(block, VecLoad((i-1)*16, keys));
617  }
618 
619  block = VecDecrypt(block, VecLoad(16, keys));
620  block = VecDecryptLast(block, VecLoad(0, keys));
621 }
622 
623 static inline void POWER8_Dec_6_Blocks(uint32x4_p &block0, uint32x4_p &block1,
624  uint32x4_p &block2, uint32x4_p &block3, uint32x4_p &block4,
625  uint32x4_p &block5, const word32 *subkeys, unsigned int rounds)
626 {
627  CRYPTOPP_ASSERT(IsAlignedOn(subkeys, 16));
628  const byte *keys = reinterpret_cast<const byte*>(subkeys);
629 
630  uint32x4_p k = VecLoad(rounds*16, keys);
631  block0 = VecXor(block0, k);
632  block1 = VecXor(block1, k);
633  block2 = VecXor(block2, k);
634  block3 = VecXor(block3, k);
635  block4 = VecXor(block4, k);
636  block5 = VecXor(block5, k);
637 
638  for (size_t i=rounds-1; i>0; --i)
639  {
640  k = VecLoad(i*16, keys);
641  block0 = VecDecrypt(block0, k);
642  block1 = VecDecrypt(block1, k);
643  block2 = VecDecrypt(block2, k);
644  block3 = VecDecrypt(block3, k);
645  block4 = VecDecrypt(block4, k);
646  block5 = VecDecrypt(block5, k);
647  }
648 
649  k = VecLoad(0, keys);
650  block0 = VecDecryptLast(block0, k);
651  block1 = VecDecryptLast(block1, k);
652  block2 = VecDecryptLast(block2, k);
653  block3 = VecDecryptLast(block3, k);
654  block4 = VecDecryptLast(block4, k);
655  block5 = VecDecryptLast(block5, k);
656 }
657 
658 ANONYMOUS_NAMESPACE_END
659 
660 void Rijndael_UncheckedSetKey_POWER8(const byte* userKey, size_t keyLen, word32* rk, const byte* Se)
661 {
662  const size_t rounds = keyLen / 4 + 6;
663  const word32 *rc = s_rconBE;
664  word32 *rkey = rk, temp;
665 
666  GetUserKey(BIG_ENDIAN_ORDER, rkey, keyLen/4, userKey, keyLen);
667 
668  // keySize: m_key allocates 4*(rounds+1) word32's.
669  const size_t keySize = 4*(rounds+1);
670  const word32* end = rkey + keySize;
671 
672  while (true)
673  {
674  temp = rkey[keyLen/4-1];
675  word32 x = (word32(Se[GETBYTE(temp, 2)]) << 24) ^ (word32(Se[GETBYTE(temp, 1)]) << 16) ^
676  (word32(Se[GETBYTE(temp, 0)]) << 8) ^ Se[GETBYTE(temp, 3)];
677  rkey[keyLen/4] = rkey[0] ^ x ^ *(rc++);
678  rkey[keyLen/4+1] = rkey[1] ^ rkey[keyLen/4];
679  rkey[keyLen/4+2] = rkey[2] ^ rkey[keyLen/4+1];
680  rkey[keyLen/4+3] = rkey[3] ^ rkey[keyLen/4+2];
681 
682  if (rkey + keyLen/4 + 4 == end)
683  break;
684 
685  if (keyLen == 24)
686  {
687  rkey[10] = rkey[ 4] ^ rkey[ 9];
688  rkey[11] = rkey[ 5] ^ rkey[10];
689  }
690  else if (keyLen == 32)
691  {
692  temp = rkey[11];
693  rkey[12] = rkey[ 4] ^ (word32(Se[GETBYTE(temp, 3)]) << 24) ^ (word32(Se[GETBYTE(temp, 2)]) << 16) ^ (word32(Se[GETBYTE(temp, 1)]) << 8) ^ Se[GETBYTE(temp, 0)];
694  rkey[13] = rkey[ 5] ^ rkey[12];
695  rkey[14] = rkey[ 6] ^ rkey[13];
696  rkey[15] = rkey[ 7] ^ rkey[14];
697  }
698  rkey += keyLen/4;
699  }
700 
701 #if (CRYPTOPP_LITTLE_ENDIAN)
702  rkey = rk;
703  const uint8x16_p mask = ((uint8x16_p){12,13,14,15, 8,9,10,11, 4,5,6,7, 0,1,2,3});
704  const uint8x16_p zero = {0};
705 
706  unsigned int i=0;
707  for (i=0; i<rounds; i+=2, rkey+=8)
708  {
709  const uint8x16_p d1 = vec_vsx_ld( 0, (uint8_t*)rkey);
710  const uint8x16_p d2 = vec_vsx_ld(16, (uint8_t*)rkey);
711  vec_vsx_st(VecPermute(d1, zero, mask), 0, (uint8_t*)rkey);
712  vec_vsx_st(VecPermute(d2, zero, mask), 16, (uint8_t*)rkey);
713  }
714 
715  for ( ; i<rounds+1; i++, rkey+=4)
716  {
717  const uint8x16_p d = vec_vsx_ld( 0, (uint8_t*)rkey);
718  vec_vsx_st(VecPermute(d, zero, mask), 0, (uint8_t*)rkey);
719  }
720 #endif
721 }
722 
723 size_t Rijndael_Enc_AdvancedProcessBlocks128_6x1_ALTIVEC(const word32 *subKeys, size_t rounds,
724  const byte *inBlocks, const byte *xorBlocks, byte *outBlocks, size_t length, word32 flags)
725 {
726  return AdvancedProcessBlocks128_6x1_ALTIVEC(POWER8_Enc_Block, POWER8_Enc_6_Blocks,
727  subKeys, rounds, inBlocks, xorBlocks, outBlocks, length, flags);
728 }
729 
730 size_t Rijndael_Dec_AdvancedProcessBlocks128_6x1_ALTIVEC(const word32 *subKeys, size_t rounds,
731  const byte *inBlocks, const byte *xorBlocks, byte *outBlocks, size_t length, word32 flags)
732 {
733  return AdvancedProcessBlocks128_6x1_ALTIVEC(POWER8_Dec_Block, POWER8_Dec_6_Blocks,
734  subKeys, rounds, inBlocks, xorBlocks, outBlocks, length, flags);
735 }
736 
737 #endif // CRYPTOPP_POWER8_AES_AVAILABLE
738 NAMESPACE_END
T1 VecDecryptLast(const T1 state, const T2 key)
Final round of AES decryption.
Definition: ppc_simd.h:1668
Utility functions for the Crypto++ library.
Library configuration file.
bool IsAlignedOn(const void *ptr, unsigned int alignment)
Determines whether ptr is aligned to a minimum value.
Definition: misc.h:1143
T1 VecPermute(const T1 vec, const T2 mask)
Permutes a vector.
Definition: ppc_simd.h:1010
__vector unsigned int uint32x4_p
Vector of 32-bit elements.
Definition: ppc_simd.h:129
Support functions for PowerPC and vector operations.
Template for AdvancedProcessBlocks and SIMD processing.
Precompiled header file.
byte order is big-endian
Definition: cryptlib.h:147
#define CRYPTOPP_ASSERT(exp)
Debugging and diagnostic assertion.
Definition: trap.h:69
T1 VecXor(const T1 vec1, const T2 vec2)
XOR two vectors.
Definition: ppc_simd.h:916
T1 VecEncryptLast(const T1 state, const T2 key)
Final round of AES encryption.
Definition: ppc_simd.h:1618
Crypto++ library namespace.
T1 VecDecrypt(const T1 state, const T2 key)
One round of AES decryption.
Definition: ppc_simd.h:1643
uint32x4_p VecLoad(const byte src[16])
Loads a vector from a byte array.
Definition: ppc_simd.h:253
__vector unsigned char uint8x16_p
Vector of 8-bit elements.
Definition: ppc_simd.h:119
T1 VecEncrypt(const T1 state, const T2 key)
One round of AES encryption.
Definition: ppc_simd.h:1593
void vec_swap(T &a, T &b)
Swaps two variables which are arrays.
Definition: misc.h:531