sshsha.c 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701
  1. /*
  2. * SHA1 hash algorithm. Used in SSH-2 as a MAC, and the transform is
  3. * also used as a `stirring' function for the PuTTY random number
  4. * pool. Implemented directly from the specification by Simon
  5. * Tatham.
  6. */
  7. #include "ssh.h"
  8. #include <assert.h>
  9. /* ----------------------------------------------------------------------
  10. * Core SHA algorithm: processes 16-word blocks into a message digest.
  11. */
  12. #define rol(x,y) ( ((x) << (y)) | (((uint32)x) >> (32-y)) )
  13. static void sha1_sw(SHA_State * s, const unsigned char *q, int len);
  14. static void sha1_ni(SHA_State * s, const unsigned char *q, int len);
  15. static void SHA_Core_Init(uint32 h[5])
  16. {
  17. h[0] = 0x67452301;
  18. h[1] = 0xefcdab89;
  19. h[2] = 0x98badcfe;
  20. h[3] = 0x10325476;
  21. h[4] = 0xc3d2e1f0;
  22. }
  23. void SHATransform(word32 * digest, word32 * block)
  24. {
  25. word32 w[80];
  26. word32 a, b, c, d, e;
  27. int t;
  28. #ifdef RANDOM_DIAGNOSTICS
  29. {
  30. extern int random_diagnostics;
  31. if (random_diagnostics) {
  32. int i;
  33. printf("SHATransform:");
  34. for (i = 0; i < 5; i++)
  35. printf(" %08x", digest[i]);
  36. printf(" +");
  37. for (i = 0; i < 16; i++)
  38. printf(" %08x", block[i]);
  39. }
  40. }
  41. #endif
  42. for (t = 0; t < 16; t++)
  43. w[t] = block[t];
  44. for (t = 16; t < 80; t++) {
  45. word32 tmp = w[t - 3] ^ w[t - 8] ^ w[t - 14] ^ w[t - 16];
  46. w[t] = rol(tmp, 1);
  47. }
  48. a = digest[0];
  49. b = digest[1];
  50. c = digest[2];
  51. d = digest[3];
  52. e = digest[4];
  53. for (t = 0; t < 20; t++) {
  54. word32 tmp =
  55. rol(a, 5) + ((b & c) | (d & ~b)) + e + w[t] + 0x5a827999;
  56. e = d;
  57. d = c;
  58. c = rol(b, 30);
  59. b = a;
  60. a = tmp;
  61. }
  62. for (t = 20; t < 40; t++) {
  63. word32 tmp = rol(a, 5) + (b ^ c ^ d) + e + w[t] + 0x6ed9eba1;
  64. e = d;
  65. d = c;
  66. c = rol(b, 30);
  67. b = a;
  68. a = tmp;
  69. }
  70. for (t = 40; t < 60; t++) {
  71. word32 tmp = rol(a,
  72. 5) + ((b & c) | (b & d) | (c & d)) + e + w[t] +
  73. 0x8f1bbcdc;
  74. e = d;
  75. d = c;
  76. c = rol(b, 30);
  77. b = a;
  78. a = tmp;
  79. }
  80. for (t = 60; t < 80; t++) {
  81. word32 tmp = rol(a, 5) + (b ^ c ^ d) + e + w[t] + 0xca62c1d6;
  82. e = d;
  83. d = c;
  84. c = rol(b, 30);
  85. b = a;
  86. a = tmp;
  87. }
  88. digest[0] += a;
  89. digest[1] += b;
  90. digest[2] += c;
  91. digest[3] += d;
  92. digest[4] += e;
  93. #ifdef RANDOM_DIAGNOSTICS
  94. {
  95. extern int random_diagnostics;
  96. if (random_diagnostics) {
  97. int i;
  98. printf(" =");
  99. for (i = 0; i < 5; i++)
  100. printf(" %08x", digest[i]);
  101. printf("\n");
  102. }
  103. }
  104. #endif
  105. }
  106. /* ----------------------------------------------------------------------
  107. * Outer SHA algorithm: take an arbitrary length byte string,
  108. * convert it into 16-word blocks with the prescribed padding at
  109. * the end, and pass those blocks to the core SHA algorithm.
  110. */
  111. static void SHA_BinarySink_write(BinarySink *bs, const void *p, size_t len);
  112. void SHA_Init(SHA_State * s)
  113. {
  114. SHA_Core_Init(s->h);
  115. s->blkused = 0;
  116. s->lenhi = s->lenlo = 0;
  117. if (supports_sha_ni())
  118. s->sha1 = &sha1_ni;
  119. else
  120. s->sha1 = &sha1_sw;
  121. BinarySink_INIT(s, SHA_BinarySink_write);
  122. }
  123. static void SHA_BinarySink_write(BinarySink *bs, const void *p, size_t len)
  124. {
  125. struct SHA_State *s = BinarySink_DOWNCAST(bs, struct SHA_State);
  126. const unsigned char *q = (const unsigned char *) p;
  127. uint32 lenw = len;
  128. assert(lenw == len);
  129. /*
  130. * Update the length field.
  131. */
  132. s->lenlo += lenw;
  133. s->lenhi += (s->lenlo < lenw);
  134. (*(s->sha1))(s, q, len);
  135. }
  136. static void sha1_sw(SHA_State * s, const unsigned char *q, int len)
  137. {
  138. uint32 wordblock[16];
  139. int i;
  140. if (s->blkused && s->blkused + len < 64) {
  141. /*
  142. * Trivial case: just add to the block.
  143. */
  144. memcpy(s->block + s->blkused, q, len);
  145. s->blkused += len;
  146. } else {
  147. /*
  148. * We must complete and process at least one block.
  149. */
  150. while (s->blkused + len >= 64) {
  151. memcpy(s->block + s->blkused, q, 64 - s->blkused);
  152. q += 64 - s->blkused;
  153. len -= 64 - s->blkused;
  154. /* Now process the block. Gather bytes big-endian into words */
  155. for (i = 0; i < 16; i++) {
  156. wordblock[i] =
  157. (((uint32) s->block[i * 4 + 0]) << 24) |
  158. (((uint32) s->block[i * 4 + 1]) << 16) |
  159. (((uint32) s->block[i * 4 + 2]) << 8) |
  160. (((uint32) s->block[i * 4 + 3]) << 0);
  161. }
  162. SHATransform(s->h, wordblock);
  163. s->blkused = 0;
  164. }
  165. #ifdef MPEXT
  166. if (len > 0)
  167. #endif
  168. memcpy(s->block, q, len);
  169. s->blkused = len;
  170. }
  171. }
  172. void SHA_Final(SHA_State * s, unsigned char *output)
  173. {
  174. int i;
  175. int pad;
  176. unsigned char c[64];
  177. uint32 lenhi, lenlo;
  178. if (s->blkused >= 56)
  179. pad = 56 + 64 - s->blkused;
  180. else
  181. pad = 56 - s->blkused;
  182. lenhi = (s->lenhi << 3) | (s->lenlo >> (32 - 3));
  183. lenlo = (s->lenlo << 3);
  184. memset(c, 0, pad);
  185. c[0] = 0x80;
  186. put_data(s, &c, pad);
  187. put_uint32(s, lenhi);
  188. put_uint32(s, lenlo);
  189. for (i = 0; i < 5; i++) {
  190. output[i * 4] = (s->h[i] >> 24) & 0xFF;
  191. output[i * 4 + 1] = (s->h[i] >> 16) & 0xFF;
  192. output[i * 4 + 2] = (s->h[i] >> 8) & 0xFF;
  193. output[i * 4 + 3] = (s->h[i]) & 0xFF;
  194. }
  195. }
  196. void SHA_Simple(const void *p, int len, unsigned char *output)
  197. {
  198. SHA_State s;
  199. SHA_Init(&s);
  200. put_data(&s, p, len);
  201. SHA_Final(&s, output);
  202. smemclr(&s, sizeof(s));
  203. }
  204. /*
  205. * Thin abstraction for things where hashes are pluggable.
  206. */
  207. struct sha1_hash {
  208. SHA_State state;
  209. ssh_hash hash;
  210. };
  211. static ssh_hash *sha1_new(const struct ssh_hashalg *alg)
  212. {
  213. struct sha1_hash *h = snew(struct sha1_hash);
  214. SHA_Init(&h->state);
  215. h->hash.vt = alg;
  216. BinarySink_DELEGATE_INIT(&h->hash, &h->state);
  217. return &h->hash;
  218. }
  219. static ssh_hash *sha1_copy(ssh_hash *hashold)
  220. {
  221. struct sha1_hash *hold, *hnew;
  222. ssh_hash *hashnew = sha1_new(hashold->vt);
  223. hold = container_of(hashold, struct sha1_hash, hash);
  224. hnew = container_of(hashnew, struct sha1_hash, hash);
  225. hnew->state = hold->state;
  226. BinarySink_COPIED(&hnew->state);
  227. return hashnew;
  228. }
  229. static void sha1_free(ssh_hash *hash)
  230. {
  231. struct sha1_hash *h = container_of(hash, struct sha1_hash, hash);
  232. smemclr(h, sizeof(*h));
  233. sfree(h);
  234. }
  235. static void sha1_final(ssh_hash *hash, unsigned char *output)
  236. {
  237. struct sha1_hash *h = container_of(hash, struct sha1_hash, hash);
  238. SHA_Final(&h->state, output);
  239. sha1_free(hash);
  240. }
  241. const struct ssh_hashalg ssh_sha1 = {
  242. sha1_new, sha1_copy, sha1_final, sha1_free, 20, "SHA-1"
  243. };
  244. /* ----------------------------------------------------------------------
  245. * The above is the SHA-1 algorithm itself. Now we implement the
  246. * HMAC wrapper on it.
  247. */
  248. struct hmacsha1 {
  249. SHA_State sha[3];
  250. ssh2_mac mac;
  251. };
  252. static ssh2_mac *hmacsha1_new(
  253. const struct ssh2_macalg *alg, ssh2_cipher *cipher)
  254. {
  255. struct hmacsha1 *ctx = snew(struct hmacsha1);
  256. ctx->mac.vt = alg;
  257. BinarySink_DELEGATE_INIT(&ctx->mac, &ctx->sha[2]);
  258. return &ctx->mac;
  259. }
  260. static void hmacsha1_free(ssh2_mac *mac)
  261. {
  262. struct hmacsha1 *ctx = container_of(mac, struct hmacsha1, mac);
  263. smemclr(ctx, sizeof(*ctx));
  264. sfree(ctx);
  265. }
  266. static void sha1_key_internal(SHA_State *keys,
  267. const unsigned char *key, int len)
  268. {
  269. unsigned char foo[64];
  270. int i;
  271. memset(foo, 0x36, 64);
  272. for (i = 0; i < len && i < 64; i++)
  273. foo[i] ^= key[i];
  274. SHA_Init(&keys[0]);
  275. put_data(&keys[0], foo, 64);
  276. memset(foo, 0x5C, 64);
  277. for (i = 0; i < len && i < 64; i++)
  278. foo[i] ^= key[i];
  279. SHA_Init(&keys[1]);
  280. put_data(&keys[1], foo, 64);
  281. smemclr(foo, 64); /* burn the evidence */
  282. }
  283. static void hmacsha1_key(ssh2_mac *mac, const void *key)
  284. {
  285. struct hmacsha1 *ctx = container_of(mac, struct hmacsha1, mac);
  286. /* Reading the key length out of the ssh2_macalg structure means
  287. * this same method can be used for the _buggy variants which use
  288. * a shorter key */
  289. sha1_key_internal(ctx->sha, key, ctx->mac.vt->keylen);
  290. }
  291. static void hmacsha1_start(ssh2_mac *mac)
  292. {
  293. struct hmacsha1 *ctx = container_of(mac, struct hmacsha1, mac);
  294. ctx->sha[2] = ctx->sha[0]; /* structure copy */
  295. BinarySink_COPIED(&ctx->sha[2]);
  296. }
  297. static void hmacsha1_genresult(ssh2_mac *mac, unsigned char *hmac)
  298. {
  299. struct hmacsha1 *ctx = container_of(mac, struct hmacsha1, mac);
  300. SHA_State s;
  301. unsigned char intermediate[20];
  302. s = ctx->sha[2]; /* structure copy */
  303. BinarySink_COPIED(&s);
  304. SHA_Final(&s, intermediate);
  305. s = ctx->sha[1]; /* structure copy */
  306. BinarySink_COPIED(&s);
  307. put_data(&s, intermediate, 20);
  308. SHA_Final(&s, intermediate);
  309. memcpy(hmac, intermediate, ctx->mac.vt->len);
  310. smemclr(intermediate, sizeof(intermediate));
  311. }
  312. void hmac_sha1_simple(const void *key, int keylen,
  313. const void *data, int datalen,
  314. unsigned char *output) {
  315. SHA_State states[2];
  316. unsigned char intermediate[20];
  317. sha1_key_internal(states, key, keylen);
  318. put_data(&states[0], data, datalen);
  319. SHA_Final(&states[0], intermediate);
  320. put_data(&states[1], intermediate, 20);
  321. SHA_Final(&states[1], output);
  322. }
  323. const struct ssh2_macalg ssh_hmac_sha1 = {
  324. hmacsha1_new, hmacsha1_free, hmacsha1_key,
  325. hmacsha1_start, hmacsha1_genresult,
  326. "hmac-sha1", "[email protected]",
  327. 20, 20,
  328. "HMAC-SHA1"
  329. };
  330. const struct ssh2_macalg ssh_hmac_sha1_96 = {
  331. hmacsha1_new, hmacsha1_free, hmacsha1_key,
  332. hmacsha1_start, hmacsha1_genresult,
  333. "hmac-sha1-96", "[email protected]",
  334. 12, 20,
  335. "HMAC-SHA1-96"
  336. };
  337. const struct ssh2_macalg ssh_hmac_sha1_buggy = {
  338. hmacsha1_new, hmacsha1_free, hmacsha1_key,
  339. hmacsha1_start, hmacsha1_genresult,
  340. "hmac-sha1", NULL,
  341. 20, 16,
  342. "bug-compatible HMAC-SHA1"
  343. };
  344. const struct ssh2_macalg ssh_hmac_sha1_96_buggy = {
  345. hmacsha1_new, hmacsha1_free, hmacsha1_key,
  346. hmacsha1_start, hmacsha1_genresult,
  347. "hmac-sha1-96", NULL,
  348. 12, 16,
  349. "bug-compatible HMAC-SHA1-96"
  350. };
  351. #ifdef COMPILER_SUPPORTS_SHA_NI
  352. #if defined _MSC_VER && defined _M_AMD64
  353. # include <intrin.h>
  354. #endif
  355. /*
  356. * Set target architecture for Clang and GCC
  357. */
  358. #if !defined(__clang__) && defined(__GNUC__)
  359. # pragma GCC target("sha")
  360. # pragma GCC target("sse4.1")
  361. #endif
  362. #if defined(__clang__) || (defined(__GNUC__) && (__GNUC__ >= 5))
  363. # define FUNC_ISA __attribute__ ((target("sse4.1,sha")))
  364. #else
  365. # define FUNC_ISA
  366. #endif
  367. #include <wmmintrin.h>
  368. #include <smmintrin.h>
  369. #include <immintrin.h>
  370. #if defined(__clang__) || defined(__GNUC__)
  371. #include <shaintrin.h>
  372. #endif
  373. /*
  374. * Determinators of CPU type
  375. */
  376. #if defined(__clang__) || defined(__GNUC__)
  377. #include <cpuid.h>
  378. int supports_sha_ni(void)
  379. {
  380. unsigned int CPUInfo[4];
  381. __cpuid(0, CPUInfo[0], CPUInfo[1], CPUInfo[2], CPUInfo[3]);
  382. if (CPUInfo[0] < 7)
  383. return 0;
  384. __cpuid_count(7, 0, CPUInfo[0], CPUInfo[1], CPUInfo[2], CPUInfo[3]);
  385. return CPUInfo[1] & (1 << 29); /* SHA */
  386. }
  387. #else /* defined(__clang__) || defined(__GNUC__) */
  388. int supports_sha_ni(void)
  389. {
  390. unsigned int CPUInfo[4];
  391. __cpuid(CPUInfo, 0);
  392. if (CPUInfo[0] < 7)
  393. return 0;
  394. __cpuidex(CPUInfo, 7, 0);
  395. return CPUInfo[1] & (1 << 29); /* Check SHA */
  396. }
  397. #endif /* defined(__clang__) || defined(__GNUC__) */
  398. /* SHA1 implementation using new instructions
  399. The code is based on Jeffrey Walton's SHA1 implementation:
  400. https://github.com/noloader/SHA-Intrinsics
  401. */
  402. FUNC_ISA
  403. static void sha1_ni_(SHA_State * s, const unsigned char *q, int len)
  404. {
  405. if (s->blkused && s->blkused + len < 64) {
  406. /*
  407. * Trivial case: just add to the block.
  408. */
  409. memcpy(s->block + s->blkused, q, len);
  410. s->blkused += len;
  411. } else {
  412. __m128i ABCD, ABCD_SAVE, E0, E0_SAVE, E1;
  413. const __m128i MASK = _mm_set_epi64x(0x0001020304050607ULL, 0x08090a0b0c0d0e0fULL);
  414. ABCD = _mm_loadu_si128((const __m128i*) s->h);
  415. E0 = _mm_set_epi32(s->h[4], 0, 0, 0);
  416. ABCD = _mm_shuffle_epi32(ABCD, 0x1B);
  417. /*
  418. * We must complete and process at least one block.
  419. */
  420. while (s->blkused + len >= 64)
  421. {
  422. __m128i MSG0, MSG1, MSG2, MSG3;
  423. memcpy(s->block + s->blkused, q, 64 - s->blkused);
  424. q += 64 - s->blkused;
  425. len -= 64 - s->blkused;
  426. /* Save current state */
  427. ABCD_SAVE = ABCD;
  428. E0_SAVE = E0;
  429. /* Rounds 0-3 */
  430. MSG0 = _mm_loadu_si128((const __m128i*)(s->block + 0));
  431. MSG0 = _mm_shuffle_epi8(MSG0, MASK);
  432. E0 = _mm_add_epi32(E0, MSG0);
  433. E1 = ABCD;
  434. ABCD = _mm_sha1rnds4_epu32(ABCD, E0, 0);
  435. /* Rounds 4-7 */
  436. MSG1 = _mm_loadu_si128((const __m128i*)(s->block + 16));
  437. MSG1 = _mm_shuffle_epi8(MSG1, MASK);
  438. E1 = _mm_sha1nexte_epu32(E1, MSG1);
  439. E0 = ABCD;
  440. ABCD = _mm_sha1rnds4_epu32(ABCD, E1, 0);
  441. MSG0 = _mm_sha1msg1_epu32(MSG0, MSG1);
  442. /* Rounds 8-11 */
  443. MSG2 = _mm_loadu_si128((const __m128i*)(s->block + 32));
  444. MSG2 = _mm_shuffle_epi8(MSG2, MASK);
  445. E0 = _mm_sha1nexte_epu32(E0, MSG2);
  446. E1 = ABCD;
  447. ABCD = _mm_sha1rnds4_epu32(ABCD, E0, 0);
  448. MSG1 = _mm_sha1msg1_epu32(MSG1, MSG2);
  449. MSG0 = _mm_xor_si128(MSG0, MSG2);
  450. /* Rounds 12-15 */
  451. MSG3 = _mm_loadu_si128((const __m128i*)(s->block + 48));
  452. MSG3 = _mm_shuffle_epi8(MSG3, MASK);
  453. E1 = _mm_sha1nexte_epu32(E1, MSG3);
  454. E0 = ABCD;
  455. MSG0 = _mm_sha1msg2_epu32(MSG0, MSG3);
  456. ABCD = _mm_sha1rnds4_epu32(ABCD, E1, 0);
  457. MSG2 = _mm_sha1msg1_epu32(MSG2, MSG3);
  458. MSG1 = _mm_xor_si128(MSG1, MSG3);
  459. /* Rounds 16-19 */
  460. E0 = _mm_sha1nexte_epu32(E0, MSG0);
  461. E1 = ABCD;
  462. MSG1 = _mm_sha1msg2_epu32(MSG1, MSG0);
  463. ABCD = _mm_sha1rnds4_epu32(ABCD, E0, 0);
  464. MSG3 = _mm_sha1msg1_epu32(MSG3, MSG0);
  465. MSG2 = _mm_xor_si128(MSG2, MSG0);
  466. /* Rounds 20-23 */
  467. E1 = _mm_sha1nexte_epu32(E1, MSG1);
  468. E0 = ABCD;
  469. MSG2 = _mm_sha1msg2_epu32(MSG2, MSG1);
  470. ABCD = _mm_sha1rnds4_epu32(ABCD, E1, 1);
  471. MSG0 = _mm_sha1msg1_epu32(MSG0, MSG1);
  472. MSG3 = _mm_xor_si128(MSG3, MSG1);
  473. /* Rounds 24-27 */
  474. E0 = _mm_sha1nexte_epu32(E0, MSG2);
  475. E1 = ABCD;
  476. MSG3 = _mm_sha1msg2_epu32(MSG3, MSG2);
  477. ABCD = _mm_sha1rnds4_epu32(ABCD, E0, 1);
  478. MSG1 = _mm_sha1msg1_epu32(MSG1, MSG2);
  479. MSG0 = _mm_xor_si128(MSG0, MSG2);
  480. /* Rounds 28-31 */
  481. E1 = _mm_sha1nexte_epu32(E1, MSG3);
  482. E0 = ABCD;
  483. MSG0 = _mm_sha1msg2_epu32(MSG0, MSG3);
  484. ABCD = _mm_sha1rnds4_epu32(ABCD, E1, 1);
  485. MSG2 = _mm_sha1msg1_epu32(MSG2, MSG3);
  486. MSG1 = _mm_xor_si128(MSG1, MSG3);
  487. /* Rounds 32-35 */
  488. E0 = _mm_sha1nexte_epu32(E0, MSG0);
  489. E1 = ABCD;
  490. MSG1 = _mm_sha1msg2_epu32(MSG1, MSG0);
  491. ABCD = _mm_sha1rnds4_epu32(ABCD, E0, 1);
  492. MSG3 = _mm_sha1msg1_epu32(MSG3, MSG0);
  493. MSG2 = _mm_xor_si128(MSG2, MSG0);
  494. /* Rounds 36-39 */
  495. E1 = _mm_sha1nexte_epu32(E1, MSG1);
  496. E0 = ABCD;
  497. MSG2 = _mm_sha1msg2_epu32(MSG2, MSG1);
  498. ABCD = _mm_sha1rnds4_epu32(ABCD, E1, 1);
  499. MSG0 = _mm_sha1msg1_epu32(MSG0, MSG1);
  500. MSG3 = _mm_xor_si128(MSG3, MSG1);
  501. /* Rounds 40-43 */
  502. E0 = _mm_sha1nexte_epu32(E0, MSG2);
  503. E1 = ABCD;
  504. MSG3 = _mm_sha1msg2_epu32(MSG3, MSG2);
  505. ABCD = _mm_sha1rnds4_epu32(ABCD, E0, 2);
  506. MSG1 = _mm_sha1msg1_epu32(MSG1, MSG2);
  507. MSG0 = _mm_xor_si128(MSG0, MSG2);
  508. /* Rounds 44-47 */
  509. E1 = _mm_sha1nexte_epu32(E1, MSG3);
  510. E0 = ABCD;
  511. MSG0 = _mm_sha1msg2_epu32(MSG0, MSG3);
  512. ABCD = _mm_sha1rnds4_epu32(ABCD, E1, 2);
  513. MSG2 = _mm_sha1msg1_epu32(MSG2, MSG3);
  514. MSG1 = _mm_xor_si128(MSG1, MSG3);
  515. /* Rounds 48-51 */
  516. E0 = _mm_sha1nexte_epu32(E0, MSG0);
  517. E1 = ABCD;
  518. MSG1 = _mm_sha1msg2_epu32(MSG1, MSG0);
  519. ABCD = _mm_sha1rnds4_epu32(ABCD, E0, 2);
  520. MSG3 = _mm_sha1msg1_epu32(MSG3, MSG0);
  521. MSG2 = _mm_xor_si128(MSG2, MSG0);
  522. /* Rounds 52-55 */
  523. E1 = _mm_sha1nexte_epu32(E1, MSG1);
  524. E0 = ABCD;
  525. MSG2 = _mm_sha1msg2_epu32(MSG2, MSG1);
  526. ABCD = _mm_sha1rnds4_epu32(ABCD, E1, 2);
  527. MSG0 = _mm_sha1msg1_epu32(MSG0, MSG1);
  528. MSG3 = _mm_xor_si128(MSG3, MSG1);
  529. /* Rounds 56-59 */
  530. E0 = _mm_sha1nexte_epu32(E0, MSG2);
  531. E1 = ABCD;
  532. MSG3 = _mm_sha1msg2_epu32(MSG3, MSG2);
  533. ABCD = _mm_sha1rnds4_epu32(ABCD, E0, 2);
  534. MSG1 = _mm_sha1msg1_epu32(MSG1, MSG2);
  535. MSG0 = _mm_xor_si128(MSG0, MSG2);
  536. /* Rounds 60-63 */
  537. E1 = _mm_sha1nexte_epu32(E1, MSG3);
  538. E0 = ABCD;
  539. MSG0 = _mm_sha1msg2_epu32(MSG0, MSG3);
  540. ABCD = _mm_sha1rnds4_epu32(ABCD, E1, 3);
  541. MSG2 = _mm_sha1msg1_epu32(MSG2, MSG3);
  542. MSG1 = _mm_xor_si128(MSG1, MSG3);
  543. /* Rounds 64-67 */
  544. E0 = _mm_sha1nexte_epu32(E0, MSG0);
  545. E1 = ABCD;
  546. MSG1 = _mm_sha1msg2_epu32(MSG1, MSG0);
  547. ABCD = _mm_sha1rnds4_epu32(ABCD, E0, 3);
  548. MSG3 = _mm_sha1msg1_epu32(MSG3, MSG0);
  549. MSG2 = _mm_xor_si128(MSG2, MSG0);
  550. /* Rounds 68-71 */
  551. E1 = _mm_sha1nexte_epu32(E1, MSG1);
  552. E0 = ABCD;
  553. MSG2 = _mm_sha1msg2_epu32(MSG2, MSG1);
  554. ABCD = _mm_sha1rnds4_epu32(ABCD, E1, 3);
  555. MSG3 = _mm_xor_si128(MSG3, MSG1);
  556. /* Rounds 72-75 */
  557. E0 = _mm_sha1nexte_epu32(E0, MSG2);
  558. E1 = ABCD;
  559. MSG3 = _mm_sha1msg2_epu32(MSG3, MSG2);
  560. ABCD = _mm_sha1rnds4_epu32(ABCD, E0, 3);
  561. /* Rounds 76-79 */
  562. E1 = _mm_sha1nexte_epu32(E1, MSG3);
  563. E0 = ABCD;
  564. ABCD = _mm_sha1rnds4_epu32(ABCD, E1, 3);
  565. /* Combine state */
  566. E0 = _mm_sha1nexte_epu32(E0, E0_SAVE);
  567. ABCD = _mm_add_epi32(ABCD, ABCD_SAVE);
  568. s->blkused = 0;
  569. }
  570. ABCD = _mm_shuffle_epi32(ABCD, 0x1B);
  571. /* Save state */
  572. _mm_storeu_si128((__m128i*) s->h, ABCD);
  573. s->h[4] = _mm_extract_epi32(E0, 3);
  574. memcpy(s->block, q, len);
  575. s->blkused = len;
  576. }
  577. }
  578. /*
  579. * Workaround LLVM bug https://bugs.llvm.org/show_bug.cgi?id=34980
  580. */
  581. static void sha1_ni(SHA_State * s, const unsigned char *q, int len)
  582. {
  583. sha1_ni_(s, q, len);
  584. }
  585. #else /* COMPILER_SUPPORTS_AES_NI */
  586. static void sha1_ni(SHA_State * s, const unsigned char *q, int len)
  587. {
  588. assert(0);
  589. }
  590. int supports_sha_ni(void)
  591. {
  592. return 0;
  593. }
  594. #endif /* COMPILER_SUPPORTS_AES_NI */