sshsh256.c 18 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513
  1. /*
  2. * SHA-256 algorithm as described at
  3. *
  4. * http://csrc.nist.gov/cryptval/shs.html
  5. */
  6. #include "ssh.h"
  7. #include <assert.h>
  8. /* ----------------------------------------------------------------------
  9. * Core SHA256 algorithm: processes 16-word blocks into a message digest.
  10. */
  11. #define ror(x,y) ( ((x) << (32-y)) | (((uint32_t)(x)) >> (y)) )
  12. #define shr(x,y) ( (((uint32_t)(x)) >> (y)) )
  13. #define Ch(x,y,z) ( ((x) & (y)) ^ (~(x) & (z)) )
  14. #define Maj(x,y,z) ( ((x) & (y)) ^ ((x) & (z)) ^ ((y) & (z)) )
  15. #define bigsigma0(x) ( ror((x),2) ^ ror((x),13) ^ ror((x),22) )
  16. #define bigsigma1(x) ( ror((x),6) ^ ror((x),11) ^ ror((x),25) )
  17. #define smallsigma0(x) ( ror((x),7) ^ ror((x),18) ^ shr((x),3) )
  18. #define smallsigma1(x) ( ror((x),17) ^ ror((x),19) ^ shr((x),10) )
  19. typedef struct SHA256_State {
  20. uint32_t h[8];
  21. unsigned char block[64];
  22. int blkused;
  23. uint64_t len;
  24. void (*sha256)(struct SHA256_State * s, const unsigned char *p, int len);
  25. BinarySink_IMPLEMENTATION;
  26. } SHA256_State;
  27. static void SHA256_sw(SHA256_State *s, const unsigned char *q, int len);
  28. static void SHA256_ni(SHA256_State *s, const unsigned char *q, int len);
  29. void SHA256_Core_Init(SHA256_State *s) {
  30. s->h[0] = 0x6a09e667;
  31. s->h[1] = 0xbb67ae85;
  32. s->h[2] = 0x3c6ef372;
  33. s->h[3] = 0xa54ff53a;
  34. s->h[4] = 0x510e527f;
  35. s->h[5] = 0x9b05688c;
  36. s->h[6] = 0x1f83d9ab;
  37. s->h[7] = 0x5be0cd19;
  38. }
  39. void SHA256_Block(SHA256_State *s, uint32_t *block) {
  40. uint32_t w[80];
  41. uint32_t a,b,c,d,e,f,g,h;
  42. static const int k[] = {
  43. 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5,
  44. 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5,
  45. 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3,
  46. 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174,
  47. 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc,
  48. 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da,
  49. 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7,
  50. 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967,
  51. 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13,
  52. 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85,
  53. 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3,
  54. 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070,
  55. 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5,
  56. 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3,
  57. 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208,
  58. 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2,
  59. };
  60. int t;
  61. for (t = 0; t < 16; t++)
  62. w[t] = block[t];
  63. for (t = 16; t < 64; t++)
  64. w[t] = smallsigma1(w[t-2]) + w[t-7] + smallsigma0(w[t-15]) + w[t-16];
  65. a = s->h[0]; b = s->h[1]; c = s->h[2]; d = s->h[3];
  66. e = s->h[4]; f = s->h[5]; g = s->h[6]; h = s->h[7];
  67. for (t = 0; t < 64; t+=8) {
  68. uint32_t t1, t2;
  69. #define ROUND(j,a,b,c,d,e,f,g,h) \
  70. t1 = h + bigsigma1(e) + Ch(e,f,g) + k[j] + w[j]; \
  71. t2 = bigsigma0(a) + Maj(a,b,c); \
  72. d = d + t1; h = t1 + t2;
  73. ROUND(t+0, a,b,c,d,e,f,g,h);
  74. ROUND(t+1, h,a,b,c,d,e,f,g);
  75. ROUND(t+2, g,h,a,b,c,d,e,f);
  76. ROUND(t+3, f,g,h,a,b,c,d,e);
  77. ROUND(t+4, e,f,g,h,a,b,c,d);
  78. ROUND(t+5, d,e,f,g,h,a,b,c);
  79. ROUND(t+6, c,d,e,f,g,h,a,b);
  80. ROUND(t+7, b,c,d,e,f,g,h,a);
  81. }
  82. s->h[0] += a; s->h[1] += b; s->h[2] += c; s->h[3] += d;
  83. s->h[4] += e; s->h[5] += f; s->h[6] += g; s->h[7] += h;
  84. }
  85. /* ----------------------------------------------------------------------
  86. * Outer SHA256 algorithm: take an arbitrary length byte string,
  87. * convert it into 16-word blocks with the prescribed padding at
  88. * the end, and pass those blocks to the core SHA256 algorithm.
  89. */
  90. #define BLKSIZE 64
  91. static void SHA256_BinarySink_write(BinarySink *bs,
  92. const void *p, size_t len);
  93. void SHA256_Init(SHA256_State *s) {
  94. SHA256_Core_Init(s);
  95. s->blkused = 0;
  96. s->len = 0;
  97. if (supports_sha_ni())
  98. s->sha256 = &SHA256_ni;
  99. else
  100. s->sha256 = &SHA256_sw;
  101. BinarySink_INIT(s, SHA256_BinarySink_write);
  102. }
  103. static void SHA256_BinarySink_write(BinarySink *bs,
  104. const void *p, size_t len)
  105. {
  106. struct SHA256_State *s = BinarySink_DOWNCAST(bs, struct SHA256_State);
  107. unsigned char *q = (unsigned char *)p;
  108. /*
  109. * Update the length field.
  110. */
  111. s->len += len;
  112. (*(s->sha256))(s, q, len);
  113. }
  114. static void SHA256_sw(SHA256_State *s, const unsigned char *q, int len) {
  115. uint32_t wordblock[16];
  116. int i;
  117. if (s->blkused && s->blkused+len < BLKSIZE) {
  118. /*
  119. * Trivial case: just add to the block.
  120. */
  121. memcpy(s->block + s->blkused, q, len);
  122. s->blkused += len;
  123. } else {
  124. /*
  125. * We must complete and process at least one block.
  126. */
  127. while (s->blkused + len >= BLKSIZE) {
  128. memcpy(s->block + s->blkused, q, BLKSIZE - s->blkused);
  129. q += BLKSIZE - s->blkused;
  130. len -= BLKSIZE - s->blkused;
  131. /* Now process the block. Gather bytes big-endian into words */
  132. for (i = 0; i < 16; i++) {
  133. wordblock[i] =
  134. ( ((uint32_t)s->block[i*4+0]) << 24 ) |
  135. ( ((uint32_t)s->block[i*4+1]) << 16 ) |
  136. ( ((uint32_t)s->block[i*4+2]) << 8 ) |
  137. ( ((uint32_t)s->block[i*4+3]) << 0 );
  138. }
  139. SHA256_Block(s, wordblock);
  140. s->blkused = 0;
  141. }
  142. memcpy(s->block, q, len);
  143. s->blkused = len;
  144. }
  145. }
  146. void SHA256_Final(SHA256_State *s, unsigned char *digest) {
  147. int i;
  148. int pad;
  149. unsigned char c[64];
  150. uint64_t len;
  151. if (s->blkused >= 56)
  152. pad = 56 + 64 - s->blkused;
  153. else
  154. pad = 56 - s->blkused;
  155. len = (s->len << 3);
  156. memset(c, 0, pad);
  157. c[0] = 0x80;
  158. put_data(s, &c, pad);
  159. put_uint64(s, len);
  160. for (i = 0; i < 8; i++) {
  161. digest[i*4+0] = (s->h[i] >> 24) & 0xFF;
  162. digest[i*4+1] = (s->h[i] >> 16) & 0xFF;
  163. digest[i*4+2] = (s->h[i] >> 8) & 0xFF;
  164. digest[i*4+3] = (s->h[i] >> 0) & 0xFF;
  165. }
  166. }
  167. void SHA256_Simple(const void *p, int len, unsigned char *output) {
  168. SHA256_State s;
  169. SHA256_Init(&s);
  170. put_data(&s, p, len);
  171. SHA256_Final(&s, output);
  172. smemclr(&s, sizeof(s));
  173. }
  174. /*
  175. * Thin abstraction for things where hashes are pluggable.
  176. */
  177. struct sha256_hash {
  178. SHA256_State state;
  179. ssh_hash hash;
  180. };
  181. static ssh_hash *sha256_new(const ssh_hashalg *alg)
  182. {
  183. struct sha256_hash *h = snew(struct sha256_hash);
  184. SHA256_Init(&h->state);
  185. h->hash.vt = alg;
  186. BinarySink_DELEGATE_INIT(&h->hash, &h->state);
  187. return &h->hash;
  188. }
  189. static ssh_hash *sha256_copy(ssh_hash *hashold)
  190. {
  191. struct sha256_hash *hold, *hnew;
  192. ssh_hash *hashnew = sha256_new(hashold->vt);
  193. hold = container_of(hashold, struct sha256_hash, hash);
  194. hnew = container_of(hashnew, struct sha256_hash, hash);
  195. hnew->state = hold->state;
  196. BinarySink_COPIED(&hnew->state);
  197. return hashnew;
  198. }
  199. static void sha256_free(ssh_hash *hash)
  200. {
  201. struct sha256_hash *h = container_of(hash, struct sha256_hash, hash);
  202. smemclr(h, sizeof(*h));
  203. sfree(h);
  204. }
  205. static void sha256_final(ssh_hash *hash, unsigned char *output)
  206. {
  207. struct sha256_hash *h = container_of(hash, struct sha256_hash, hash);
  208. SHA256_Final(&h->state, output);
  209. sha256_free(hash);
  210. }
  211. const ssh_hashalg ssh_sha256 = {
  212. sha256_new, sha256_copy, sha256_final, sha256_free, 32, 64, "SHA-256"
  213. };
  214. #ifdef COMPILER_SUPPORTS_SHA_NI
  215. #if defined _MSC_VER && defined _M_AMD64
  216. # include <intrin.h>
  217. #endif
  218. /*
  219. * Set target architecture for Clang and GCC
  220. */
  221. #if !defined(__clang__) && defined(__GNUC__)
  222. # pragma GCC target("sha")
  223. # pragma GCC target("sse4.1")
  224. #endif
  225. #if defined(__clang__) || (defined(__GNUC__) && (__GNUC__ >= 5))
  226. # define FUNC_ISA __attribute__ ((target("sse4.1,sha")))
  227. #else
  228. # define FUNC_ISA
  229. #endif
  230. #include <wmmintrin.h>
  231. #include <smmintrin.h>
  232. #include <immintrin.h>
  233. #if defined(__clang__) || defined(__GNUC__)
  234. #include <shaintrin.h>
  235. #endif
  236. /* SHA256 implementation using new instructions
  237. The code is based on Jeffrey Walton's SHA256 implementation:
  238. https://github.com/noloader/SHA-Intrinsics
  239. */
  240. FUNC_ISA
  241. static void SHA256_ni_(SHA256_State * s, const unsigned char *q, int len) {
  242. if (s->blkused && s->blkused+len < BLKSIZE) {
  243. /*
  244. * Trivial case: just add to the block.
  245. */
  246. memcpy(s->block + s->blkused, q, len);
  247. s->blkused += len;
  248. } else {
  249. __m128i STATE0, STATE1;
  250. __m128i MSG, TMP;
  251. __m128i MSG0, MSG1, MSG2, MSG3;
  252. __m128i ABEF_SAVE, CDGH_SAVE;
  253. const __m128i MASK = _mm_set_epi64x(0x0c0d0e0f08090a0bULL, 0x0405060700010203ULL);
  254. /* Load initial values */
  255. TMP = _mm_loadu_si128((const __m128i*) &s->h[0]);
  256. STATE1 = _mm_loadu_si128((const __m128i*) &s->h[4]);
  257. TMP = _mm_shuffle_epi32(TMP, 0xB1); /* CDAB */
  258. STATE1 = _mm_shuffle_epi32(STATE1, 0x1B); /* EFGH */
  259. STATE0 = _mm_alignr_epi8(TMP, STATE1, 8); /* ABEF */
  260. STATE1 = _mm_blend_epi16(STATE1, TMP, 0xF0); /* CDGH */
  261. /*
  262. * We must complete and process at least one block.
  263. */
  264. while (s->blkused + len >= BLKSIZE) {
  265. memcpy(s->block + s->blkused, q, BLKSIZE - s->blkused);
  266. q += BLKSIZE - s->blkused;
  267. len -= BLKSIZE - s->blkused;
  268. /* Save current state */
  269. ABEF_SAVE = STATE0;
  270. CDGH_SAVE = STATE1;
  271. /* Rounds 0-3 */
  272. MSG = _mm_loadu_si128((const __m128i*) (s->block + 0));
  273. MSG0 = _mm_shuffle_epi8(MSG, MASK);
  274. MSG = _mm_add_epi32(MSG0, _mm_set_epi64x(0xE9B5DBA5B5C0FBCFULL, 0x71374491428A2F98ULL));
  275. STATE1 = _mm_sha256rnds2_epu32(STATE1, STATE0, MSG);
  276. MSG = _mm_shuffle_epi32(MSG, 0x0E);
  277. STATE0 = _mm_sha256rnds2_epu32(STATE0, STATE1, MSG);
  278. /* Rounds 4-7 */
  279. MSG1 = _mm_loadu_si128((const __m128i*) (s->block + 16));
  280. MSG1 = _mm_shuffle_epi8(MSG1, MASK);
  281. MSG = _mm_add_epi32(MSG1, _mm_set_epi64x(0xAB1C5ED5923F82A4ULL, 0x59F111F13956C25BULL));
  282. STATE1 = _mm_sha256rnds2_epu32(STATE1, STATE0, MSG);
  283. MSG = _mm_shuffle_epi32(MSG, 0x0E);
  284. STATE0 = _mm_sha256rnds2_epu32(STATE0, STATE1, MSG);
  285. MSG0 = _mm_sha256msg1_epu32(MSG0, MSG1);
  286. /* Rounds 8-11 */
  287. MSG2 = _mm_loadu_si128((const __m128i*) (s->block + 32));
  288. MSG2 = _mm_shuffle_epi8(MSG2, MASK);
  289. MSG = _mm_add_epi32(MSG2, _mm_set_epi64x(0x550C7DC3243185BEULL, 0x12835B01D807AA98ULL));
  290. STATE1 = _mm_sha256rnds2_epu32(STATE1, STATE0, MSG);
  291. MSG = _mm_shuffle_epi32(MSG, 0x0E);
  292. STATE0 = _mm_sha256rnds2_epu32(STATE0, STATE1, MSG);
  293. MSG1 = _mm_sha256msg1_epu32(MSG1, MSG2);
  294. /* Rounds 12-15 */
  295. MSG3 = _mm_loadu_si128((const __m128i*) (s->block + 48));
  296. MSG3 = _mm_shuffle_epi8(MSG3, MASK);
  297. MSG = _mm_add_epi32(MSG3, _mm_set_epi64x(0xC19BF1749BDC06A7ULL, 0x80DEB1FE72BE5D74ULL));
  298. STATE1 = _mm_sha256rnds2_epu32(STATE1, STATE0, MSG);
  299. TMP = _mm_alignr_epi8(MSG3, MSG2, 4);
  300. MSG0 = _mm_add_epi32(MSG0, TMP);
  301. MSG0 = _mm_sha256msg2_epu32(MSG0, MSG3);
  302. MSG = _mm_shuffle_epi32(MSG, 0x0E);
  303. STATE0 = _mm_sha256rnds2_epu32(STATE0, STATE1, MSG);
  304. MSG2 = _mm_sha256msg1_epu32(MSG2, MSG3);
  305. /* Rounds 16-19 */
  306. MSG = _mm_add_epi32(MSG0, _mm_set_epi64x(0x240CA1CC0FC19DC6ULL, 0xEFBE4786E49B69C1ULL));
  307. STATE1 = _mm_sha256rnds2_epu32(STATE1, STATE0, MSG);
  308. TMP = _mm_alignr_epi8(MSG0, MSG3, 4);
  309. MSG1 = _mm_add_epi32(MSG1, TMP);
  310. MSG1 = _mm_sha256msg2_epu32(MSG1, MSG0);
  311. MSG = _mm_shuffle_epi32(MSG, 0x0E);
  312. STATE0 = _mm_sha256rnds2_epu32(STATE0, STATE1, MSG);
  313. MSG3 = _mm_sha256msg1_epu32(MSG3, MSG0);
  314. /* Rounds 20-23 */
  315. MSG = _mm_add_epi32(MSG1, _mm_set_epi64x(0x76F988DA5CB0A9DCULL, 0x4A7484AA2DE92C6FULL));
  316. STATE1 = _mm_sha256rnds2_epu32(STATE1, STATE0, MSG);
  317. TMP = _mm_alignr_epi8(MSG1, MSG0, 4);
  318. MSG2 = _mm_add_epi32(MSG2, TMP);
  319. MSG2 = _mm_sha256msg2_epu32(MSG2, MSG1);
  320. MSG = _mm_shuffle_epi32(MSG, 0x0E);
  321. STATE0 = _mm_sha256rnds2_epu32(STATE0, STATE1, MSG);
  322. MSG0 = _mm_sha256msg1_epu32(MSG0, MSG1);
  323. /* Rounds 24-27 */
  324. MSG = _mm_add_epi32(MSG2, _mm_set_epi64x(0xBF597FC7B00327C8ULL, 0xA831C66D983E5152ULL));
  325. STATE1 = _mm_sha256rnds2_epu32(STATE1, STATE0, MSG);
  326. TMP = _mm_alignr_epi8(MSG2, MSG1, 4);
  327. MSG3 = _mm_add_epi32(MSG3, TMP);
  328. MSG3 = _mm_sha256msg2_epu32(MSG3, MSG2);
  329. MSG = _mm_shuffle_epi32(MSG, 0x0E);
  330. STATE0 = _mm_sha256rnds2_epu32(STATE0, STATE1, MSG);
  331. MSG1 = _mm_sha256msg1_epu32(MSG1, MSG2);
  332. /* Rounds 28-31 */
  333. MSG = _mm_add_epi32(MSG3, _mm_set_epi64x(0x1429296706CA6351ULL, 0xD5A79147C6E00BF3ULL));
  334. STATE1 = _mm_sha256rnds2_epu32(STATE1, STATE0, MSG);
  335. TMP = _mm_alignr_epi8(MSG3, MSG2, 4);
  336. MSG0 = _mm_add_epi32(MSG0, TMP);
  337. MSG0 = _mm_sha256msg2_epu32(MSG0, MSG3);
  338. MSG = _mm_shuffle_epi32(MSG, 0x0E);
  339. STATE0 = _mm_sha256rnds2_epu32(STATE0, STATE1, MSG);
  340. MSG2 = _mm_sha256msg1_epu32(MSG2, MSG3);
  341. /* Rounds 32-35 */
  342. MSG = _mm_add_epi32(MSG0, _mm_set_epi64x(0x53380D134D2C6DFCULL, 0x2E1B213827B70A85ULL));
  343. STATE1 = _mm_sha256rnds2_epu32(STATE1, STATE0, MSG);
  344. TMP = _mm_alignr_epi8(MSG0, MSG3, 4);
  345. MSG1 = _mm_add_epi32(MSG1, TMP);
  346. MSG1 = _mm_sha256msg2_epu32(MSG1, MSG0);
  347. MSG = _mm_shuffle_epi32(MSG, 0x0E);
  348. STATE0 = _mm_sha256rnds2_epu32(STATE0, STATE1, MSG);
  349. MSG3 = _mm_sha256msg1_epu32(MSG3, MSG0);
  350. /* Rounds 36-39 */
  351. MSG = _mm_add_epi32(MSG1, _mm_set_epi64x(0x92722C8581C2C92EULL, 0x766A0ABB650A7354ULL));
  352. STATE1 = _mm_sha256rnds2_epu32(STATE1, STATE0, MSG);
  353. TMP = _mm_alignr_epi8(MSG1, MSG0, 4);
  354. MSG2 = _mm_add_epi32(MSG2, TMP);
  355. MSG2 = _mm_sha256msg2_epu32(MSG2, MSG1);
  356. MSG = _mm_shuffle_epi32(MSG, 0x0E);
  357. STATE0 = _mm_sha256rnds2_epu32(STATE0, STATE1, MSG);
  358. MSG0 = _mm_sha256msg1_epu32(MSG0, MSG1);
  359. /* Rounds 40-43 */
  360. MSG = _mm_add_epi32(MSG2, _mm_set_epi64x(0xC76C51A3C24B8B70ULL, 0xA81A664BA2BFE8A1ULL));
  361. STATE1 = _mm_sha256rnds2_epu32(STATE1, STATE0, MSG);
  362. TMP = _mm_alignr_epi8(MSG2, MSG1, 4);
  363. MSG3 = _mm_add_epi32(MSG3, TMP);
  364. MSG3 = _mm_sha256msg2_epu32(MSG3, MSG2);
  365. MSG = _mm_shuffle_epi32(MSG, 0x0E);
  366. STATE0 = _mm_sha256rnds2_epu32(STATE0, STATE1, MSG);
  367. MSG1 = _mm_sha256msg1_epu32(MSG1, MSG2);
  368. /* Rounds 44-47 */
  369. MSG = _mm_add_epi32(MSG3, _mm_set_epi64x(0x106AA070F40E3585ULL, 0xD6990624D192E819ULL));
  370. STATE1 = _mm_sha256rnds2_epu32(STATE1, STATE0, MSG);
  371. TMP = _mm_alignr_epi8(MSG3, MSG2, 4);
  372. MSG0 = _mm_add_epi32(MSG0, TMP);
  373. MSG0 = _mm_sha256msg2_epu32(MSG0, MSG3);
  374. MSG = _mm_shuffle_epi32(MSG, 0x0E);
  375. STATE0 = _mm_sha256rnds2_epu32(STATE0, STATE1, MSG);
  376. MSG2 = _mm_sha256msg1_epu32(MSG2, MSG3);
  377. /* Rounds 48-51 */
  378. MSG = _mm_add_epi32(MSG0, _mm_set_epi64x(0x34B0BCB52748774CULL, 0x1E376C0819A4C116ULL));
  379. STATE1 = _mm_sha256rnds2_epu32(STATE1, STATE0, MSG);
  380. TMP = _mm_alignr_epi8(MSG0, MSG3, 4);
  381. MSG1 = _mm_add_epi32(MSG1, TMP);
  382. MSG1 = _mm_sha256msg2_epu32(MSG1, MSG0);
  383. MSG = _mm_shuffle_epi32(MSG, 0x0E);
  384. STATE0 = _mm_sha256rnds2_epu32(STATE0, STATE1, MSG);
  385. MSG3 = _mm_sha256msg1_epu32(MSG3, MSG0);
  386. /* Rounds 52-55 */
  387. MSG = _mm_add_epi32(MSG1, _mm_set_epi64x(0x682E6FF35B9CCA4FULL, 0x4ED8AA4A391C0CB3ULL));
  388. STATE1 = _mm_sha256rnds2_epu32(STATE1, STATE0, MSG);
  389. TMP = _mm_alignr_epi8(MSG1, MSG0, 4);
  390. MSG2 = _mm_add_epi32(MSG2, TMP);
  391. MSG2 = _mm_sha256msg2_epu32(MSG2, MSG1);
  392. MSG = _mm_shuffle_epi32(MSG, 0x0E);
  393. STATE0 = _mm_sha256rnds2_epu32(STATE0, STATE1, MSG);
  394. /* Rounds 56-59 */
  395. MSG = _mm_add_epi32(MSG2, _mm_set_epi64x(0x8CC7020884C87814ULL, 0x78A5636F748F82EEULL));
  396. STATE1 = _mm_sha256rnds2_epu32(STATE1, STATE0, MSG);
  397. TMP = _mm_alignr_epi8(MSG2, MSG1, 4);
  398. MSG3 = _mm_add_epi32(MSG3, TMP);
  399. MSG3 = _mm_sha256msg2_epu32(MSG3, MSG2);
  400. MSG = _mm_shuffle_epi32(MSG, 0x0E);
  401. STATE0 = _mm_sha256rnds2_epu32(STATE0, STATE1, MSG);
  402. /* Rounds 60-63 */
  403. MSG = _mm_add_epi32(MSG3, _mm_set_epi64x(0xC67178F2BEF9A3F7ULL, 0xA4506CEB90BEFFFAULL));
  404. STATE1 = _mm_sha256rnds2_epu32(STATE1, STATE0, MSG);
  405. MSG = _mm_shuffle_epi32(MSG, 0x0E);
  406. STATE0 = _mm_sha256rnds2_epu32(STATE0, STATE1, MSG);
  407. /* Combine state */
  408. STATE0 = _mm_add_epi32(STATE0, ABEF_SAVE);
  409. STATE1 = _mm_add_epi32(STATE1, CDGH_SAVE);
  410. s->blkused = 0;
  411. }
  412. TMP = _mm_shuffle_epi32(STATE0, 0x1B); /* FEBA */
  413. STATE1 = _mm_shuffle_epi32(STATE1, 0xB1); /* DCHG */
  414. STATE0 = _mm_blend_epi16(TMP, STATE1, 0xF0); /* DCBA */
  415. STATE1 = _mm_alignr_epi8(STATE1, TMP, 8); /* ABEF */
  416. /* Save state */
  417. _mm_storeu_si128((__m128i*) &s->h[0], STATE0);
  418. _mm_storeu_si128((__m128i*) &s->h[4], STATE1);
  419. memcpy(s->block, q, len);
  420. s->blkused = len;
  421. }
  422. }
  423. /*
  424. * Workaround LLVM bug https://bugs.llvm.org/show_bug.cgi?id=34980
  425. */
  426. static void SHA256_ni(SHA256_State * s, const unsigned char *q, int len)
  427. {
  428. SHA256_ni_(s, q, len);
  429. }
  430. #else /* COMPILER_SUPPORTS_AES_NI */
  431. static void SHA256_ni(SHA256_State * s, const unsigned char *q, int len)
  432. {
  433. unreachable("SHA256_ni not compiled in");
  434. }
  435. #endif /* COMPILER_SUPPORTS_AES_NI */