sshsha.c 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699
  1. /*
  2. * SHA1 hash algorithm. Used in SSH-2 as a MAC, and the transform is
  3. * also used as a `stirring' function for the PuTTY random number
  4. * pool. Implemented directly from the specification by Simon
  5. * Tatham.
  6. */
  7. #include "ssh.h"
  8. #include <assert.h>
  9. typedef struct SHA_State {
  10. uint32_t h[5];
  11. unsigned char block[64];
  12. int blkused;
  13. uint64_t len;
  14. void (*sha1)(struct SHA_State * s, const unsigned char *p, int len);
  15. BinarySink_IMPLEMENTATION;
  16. } SHA_State;
  17. /* ----------------------------------------------------------------------
  18. * Core SHA algorithm: processes 16-word blocks into a message digest.
  19. */
  20. #define rol(x,y) ( ((x) << (y)) | (((uint32_t)x) >> (32-y)) )
  21. static void sha1_sw(SHA_State * s, const unsigned char *q, int len);
  22. static void sha1_ni(SHA_State * s, const unsigned char *q, int len);
  23. static void SHA_Core_Init(uint32_t h[5])
  24. {
  25. h[0] = 0x67452301;
  26. h[1] = 0xefcdab89;
  27. h[2] = 0x98badcfe;
  28. h[3] = 0x10325476;
  29. h[4] = 0xc3d2e1f0;
  30. }
  31. void SHATransform(uint32_t * digest, uint32_t * block)
  32. {
  33. uint32_t w[80];
  34. uint32_t a, b, c, d, e;
  35. int t;
  36. #ifdef RANDOM_DIAGNOSTICS
  37. {
  38. extern int random_diagnostics;
  39. if (random_diagnostics) {
  40. int i;
  41. printf("SHATransform:");
  42. for (i = 0; i < 5; i++)
  43. printf(" %08x", digest[i]);
  44. printf(" +");
  45. for (i = 0; i < 16; i++)
  46. printf(" %08x", block[i]);
  47. }
  48. }
  49. #endif
  50. for (t = 0; t < 16; t++)
  51. w[t] = block[t];
  52. for (t = 16; t < 80; t++) {
  53. uint32_t tmp = w[t - 3] ^ w[t - 8] ^ w[t - 14] ^ w[t - 16];
  54. w[t] = rol(tmp, 1);
  55. }
  56. a = digest[0];
  57. b = digest[1];
  58. c = digest[2];
  59. d = digest[3];
  60. e = digest[4];
  61. for (t = 0; t < 20; t++) {
  62. uint32_t tmp =
  63. rol(a, 5) + ((b & c) | (d & ~b)) + e + w[t] + 0x5a827999;
  64. e = d;
  65. d = c;
  66. c = rol(b, 30);
  67. b = a;
  68. a = tmp;
  69. }
  70. for (t = 20; t < 40; t++) {
  71. uint32_t tmp = rol(a, 5) + (b ^ c ^ d) + e + w[t] + 0x6ed9eba1;
  72. e = d;
  73. d = c;
  74. c = rol(b, 30);
  75. b = a;
  76. a = tmp;
  77. }
  78. for (t = 40; t < 60; t++) {
  79. uint32_t tmp = rol(a,
  80. 5) + ((b & c) | (b & d) | (c & d)) + e + w[t] +
  81. 0x8f1bbcdc;
  82. e = d;
  83. d = c;
  84. c = rol(b, 30);
  85. b = a;
  86. a = tmp;
  87. }
  88. for (t = 60; t < 80; t++) {
  89. uint32_t tmp = rol(a, 5) + (b ^ c ^ d) + e + w[t] + 0xca62c1d6;
  90. e = d;
  91. d = c;
  92. c = rol(b, 30);
  93. b = a;
  94. a = tmp;
  95. }
  96. digest[0] += a;
  97. digest[1] += b;
  98. digest[2] += c;
  99. digest[3] += d;
  100. digest[4] += e;
  101. #ifdef RANDOM_DIAGNOSTICS
  102. {
  103. extern int random_diagnostics;
  104. if (random_diagnostics) {
  105. int i;
  106. printf(" =");
  107. for (i = 0; i < 5; i++)
  108. printf(" %08x", digest[i]);
  109. printf("\n");
  110. }
  111. }
  112. #endif
  113. }
  114. /* ----------------------------------------------------------------------
  115. * Outer SHA algorithm: take an arbitrary length byte string,
  116. * convert it into 16-word blocks with the prescribed padding at
  117. * the end, and pass those blocks to the core SHA algorithm.
  118. */
  119. static void SHA_BinarySink_write(BinarySink *bs, const void *p, size_t len);
  120. void SHA_Init(SHA_State * s)
  121. {
  122. SHA_Core_Init(s->h);
  123. s->blkused = 0;
  124. s->len = 0;
  125. if (supports_sha_ni())
  126. s->sha1 = &sha1_ni;
  127. else
  128. s->sha1 = &sha1_sw;
  129. BinarySink_INIT(s, SHA_BinarySink_write);
  130. }
  131. static void SHA_BinarySink_write(BinarySink *bs, const void *p, size_t len)
  132. {
  133. struct SHA_State *s = BinarySink_DOWNCAST(bs, struct SHA_State);
  134. const unsigned char *q = (const unsigned char *) p;
  135. /*
  136. * Update the length field.
  137. */
  138. s->len += len;
  139. (*(s->sha1))(s, q, len);
  140. }
  141. static void sha1_sw(SHA_State * s, const unsigned char *q, int len)
  142. {
  143. uint32_t wordblock[16];
  144. int i;
  145. if (s->blkused && s->blkused + len < 64) {
  146. /*
  147. * Trivial case: just add to the block.
  148. */
  149. memcpy(s->block + s->blkused, q, len);
  150. s->blkused += len;
  151. } else {
  152. /*
  153. * We must complete and process at least one block.
  154. */
  155. while (s->blkused + len >= 64) {
  156. memcpy(s->block + s->blkused, q, 64 - s->blkused);
  157. q += 64 - s->blkused;
  158. len -= 64 - s->blkused;
  159. /* Now process the block. Gather bytes big-endian into words */
  160. for (i = 0; i < 16; i++) {
  161. wordblock[i] =
  162. (((uint32_t) s->block[i * 4 + 0]) << 24) |
  163. (((uint32_t) s->block[i * 4 + 1]) << 16) |
  164. (((uint32_t) s->block[i * 4 + 2]) << 8) |
  165. (((uint32_t) s->block[i * 4 + 3]) << 0);
  166. }
  167. SHATransform(s->h, wordblock);
  168. s->blkused = 0;
  169. }
  170. memcpy(s->block, q, len);
  171. s->blkused = len;
  172. }
  173. }
  174. void SHA_Final(SHA_State * s, unsigned char *output)
  175. {
  176. int i;
  177. int pad;
  178. unsigned char c[64];
  179. uint64_t len;
  180. if (s->blkused >= 56)
  181. pad = 56 + 64 - s->blkused;
  182. else
  183. pad = 56 - s->blkused;
  184. len = (s->len << 3);
  185. memset(c, 0, pad);
  186. c[0] = 0x80;
  187. put_data(s, &c, pad);
  188. put_uint64(s, len);
  189. for (i = 0; i < 5; i++) {
  190. output[i * 4] = (s->h[i] >> 24) & 0xFF;
  191. output[i * 4 + 1] = (s->h[i] >> 16) & 0xFF;
  192. output[i * 4 + 2] = (s->h[i] >> 8) & 0xFF;
  193. output[i * 4 + 3] = (s->h[i]) & 0xFF;
  194. }
  195. }
  196. void SHA_Simple(const void *p, int len, unsigned char *output)
  197. {
  198. SHA_State s;
  199. SHA_Init(&s);
  200. put_data(&s, p, len);
  201. SHA_Final(&s, output);
  202. smemclr(&s, sizeof(s));
  203. }
  204. /*
  205. * Thin abstraction for things where hashes are pluggable.
  206. */
  207. struct sha1_hash {
  208. SHA_State state;
  209. ssh_hash hash;
  210. };
  211. static ssh_hash *sha1_new(const ssh_hashalg *alg)
  212. {
  213. struct sha1_hash *h = snew(struct sha1_hash);
  214. SHA_Init(&h->state);
  215. h->hash.vt = alg;
  216. BinarySink_DELEGATE_INIT(&h->hash, &h->state);
  217. return &h->hash;
  218. }
  219. static ssh_hash *sha1_copy(ssh_hash *hashold)
  220. {
  221. struct sha1_hash *hold, *hnew;
  222. ssh_hash *hashnew = sha1_new(hashold->vt);
  223. hold = container_of(hashold, struct sha1_hash, hash);
  224. hnew = container_of(hashnew, struct sha1_hash, hash);
  225. hnew->state = hold->state;
  226. BinarySink_COPIED(&hnew->state);
  227. return hashnew;
  228. }
  229. static void sha1_free(ssh_hash *hash)
  230. {
  231. struct sha1_hash *h = container_of(hash, struct sha1_hash, hash);
  232. smemclr(h, sizeof(*h));
  233. sfree(h);
  234. }
  235. static void sha1_final(ssh_hash *hash, unsigned char *output)
  236. {
  237. struct sha1_hash *h = container_of(hash, struct sha1_hash, hash);
  238. SHA_Final(&h->state, output);
  239. sha1_free(hash);
  240. }
  241. const ssh_hashalg ssh_sha1 = {
  242. sha1_new, sha1_copy, sha1_final, sha1_free, 20, "SHA-1"
  243. };
  244. /* ----------------------------------------------------------------------
  245. * The above is the SHA-1 algorithm itself. Now we implement the
  246. * HMAC wrapper on it.
  247. */
  248. struct hmacsha1 {
  249. SHA_State sha[3];
  250. ssh2_mac mac;
  251. };
  252. static ssh2_mac *hmacsha1_new(const ssh2_macalg *alg, ssh_cipher *cipher)
  253. {
  254. struct hmacsha1 *ctx = snew(struct hmacsha1);
  255. ctx->mac.vt = alg;
  256. BinarySink_DELEGATE_INIT(&ctx->mac, &ctx->sha[2]);
  257. return &ctx->mac;
  258. }
  259. static void hmacsha1_free(ssh2_mac *mac)
  260. {
  261. struct hmacsha1 *ctx = container_of(mac, struct hmacsha1, mac);
  262. smemclr(ctx, sizeof(*ctx));
  263. sfree(ctx);
  264. }
  265. static void sha1_key_internal(SHA_State *keys,
  266. const unsigned char *key, int len)
  267. {
  268. unsigned char foo[64];
  269. int i;
  270. memset(foo, 0x36, 64);
  271. for (i = 0; i < len && i < 64; i++)
  272. foo[i] ^= key[i];
  273. SHA_Init(&keys[0]);
  274. put_data(&keys[0], foo, 64);
  275. memset(foo, 0x5C, 64);
  276. for (i = 0; i < len && i < 64; i++)
  277. foo[i] ^= key[i];
  278. SHA_Init(&keys[1]);
  279. put_data(&keys[1], foo, 64);
  280. smemclr(foo, 64); /* burn the evidence */
  281. }
  282. static void hmacsha1_key(ssh2_mac *mac, ptrlen key)
  283. {
  284. struct hmacsha1 *ctx = container_of(mac, struct hmacsha1, mac);
  285. sha1_key_internal(ctx->sha, key.ptr, key.len);
  286. }
  287. static void hmacsha1_start(ssh2_mac *mac)
  288. {
  289. struct hmacsha1 *ctx = container_of(mac, struct hmacsha1, mac);
  290. ctx->sha[2] = ctx->sha[0]; /* structure copy */
  291. BinarySink_COPIED(&ctx->sha[2]);
  292. }
  293. static void hmacsha1_genresult(ssh2_mac *mac, unsigned char *hmac)
  294. {
  295. struct hmacsha1 *ctx = container_of(mac, struct hmacsha1, mac);
  296. SHA_State s;
  297. unsigned char intermediate[20];
  298. s = ctx->sha[2]; /* structure copy */
  299. BinarySink_COPIED(&s);
  300. SHA_Final(&s, intermediate);
  301. s = ctx->sha[1]; /* structure copy */
  302. BinarySink_COPIED(&s);
  303. put_data(&s, intermediate, 20);
  304. SHA_Final(&s, intermediate);
  305. memcpy(hmac, intermediate, ctx->mac.vt->len);
  306. smemclr(intermediate, sizeof(intermediate));
  307. }
  308. void hmac_sha1_simple(const void *key, int keylen,
  309. const void *data, int datalen,
  310. unsigned char *output) {
  311. SHA_State states[2];
  312. unsigned char intermediate[20];
  313. sha1_key_internal(states, key, keylen);
  314. put_data(&states[0], data, datalen);
  315. SHA_Final(&states[0], intermediate);
  316. put_data(&states[1], intermediate, 20);
  317. SHA_Final(&states[1], output);
  318. }
  319. const ssh2_macalg ssh_hmac_sha1 = {
  320. hmacsha1_new, hmacsha1_free, hmacsha1_key,
  321. hmacsha1_start, hmacsha1_genresult,
  322. "hmac-sha1", "[email protected]",
  323. 20, 20,
  324. "HMAC-SHA1"
  325. };
  326. const ssh2_macalg ssh_hmac_sha1_96 = {
  327. hmacsha1_new, hmacsha1_free, hmacsha1_key,
  328. hmacsha1_start, hmacsha1_genresult,
  329. "hmac-sha1-96", "[email protected]",
  330. 12, 20,
  331. "HMAC-SHA1-96"
  332. };
  333. const ssh2_macalg ssh_hmac_sha1_buggy = {
  334. hmacsha1_new, hmacsha1_free, hmacsha1_key,
  335. hmacsha1_start, hmacsha1_genresult,
  336. "hmac-sha1", NULL,
  337. 20, 16,
  338. "bug-compatible HMAC-SHA1"
  339. };
  340. const ssh2_macalg ssh_hmac_sha1_96_buggy = {
  341. hmacsha1_new, hmacsha1_free, hmacsha1_key,
  342. hmacsha1_start, hmacsha1_genresult,
  343. "hmac-sha1-96", NULL,
  344. 12, 16,
  345. "bug-compatible HMAC-SHA1-96"
  346. };
  347. #ifdef COMPILER_SUPPORTS_SHA_NI
  348. #if defined _MSC_VER && defined _M_AMD64
  349. # include <intrin.h>
  350. #endif
  351. /*
  352. * Set target architecture for Clang and GCC
  353. */
  354. #if !defined(__clang__) && defined(__GNUC__)
  355. # pragma GCC target("sha")
  356. # pragma GCC target("sse4.1")
  357. #endif
  358. #if defined(__clang__) || (defined(__GNUC__) && (__GNUC__ >= 5))
  359. # define FUNC_ISA __attribute__ ((target("sse4.1,sha")))
  360. #else
  361. # define FUNC_ISA
  362. #endif
  363. #include <wmmintrin.h>
  364. #include <smmintrin.h>
  365. #include <immintrin.h>
  366. #if defined(__clang__) || defined(__GNUC__)
  367. #include <shaintrin.h>
  368. #endif
  369. /*
  370. * Determinators of CPU type
  371. */
  372. #if defined(__clang__) || defined(__GNUC__)
  373. #include <cpuid.h>
  374. bool supports_sha_ni(void)
  375. {
  376. unsigned int CPUInfo[4];
  377. __cpuid(0, CPUInfo[0], CPUInfo[1], CPUInfo[2], CPUInfo[3]);
  378. if (CPUInfo[0] < 7)
  379. return false;
  380. __cpuid_count(7, 0, CPUInfo[0], CPUInfo[1], CPUInfo[2], CPUInfo[3]);
  381. return CPUInfo[1] & (1 << 29); /* SHA */
  382. }
  383. #else /* defined(__clang__) || defined(__GNUC__) */
  384. bool supports_sha_ni(void)
  385. {
  386. unsigned int CPUInfo[4];
  387. __cpuid(CPUInfo, 0);
  388. if (CPUInfo[0] < 7)
  389. return false;
  390. __cpuidex(CPUInfo, 7, 0);
  391. return CPUInfo[1] & (1 << 29); /* Check SHA */
  392. }
  393. #endif /* defined(__clang__) || defined(__GNUC__) */
  394. /* SHA1 implementation using new instructions
  395. The code is based on Jeffrey Walton's SHA1 implementation:
  396. https://github.com/noloader/SHA-Intrinsics
  397. */
  398. FUNC_ISA
  399. static void sha1_ni_(SHA_State * s, const unsigned char *q, int len)
  400. {
  401. if (s->blkused && s->blkused + len < 64) {
  402. /*
  403. * Trivial case: just add to the block.
  404. */
  405. memcpy(s->block + s->blkused, q, len);
  406. s->blkused += len;
  407. } else {
  408. __m128i ABCD, ABCD_SAVE, E0, E0_SAVE, E1;
  409. const __m128i MASK = _mm_set_epi64x(0x0001020304050607ULL, 0x08090a0b0c0d0e0fULL);
  410. ABCD = _mm_loadu_si128((const __m128i*) s->h);
  411. E0 = _mm_set_epi32(s->h[4], 0, 0, 0);
  412. ABCD = _mm_shuffle_epi32(ABCD, 0x1B);
  413. /*
  414. * We must complete and process at least one block.
  415. */
  416. while (s->blkused + len >= 64)
  417. {
  418. __m128i MSG0, MSG1, MSG2, MSG3;
  419. memcpy(s->block + s->blkused, q, 64 - s->blkused);
  420. q += 64 - s->blkused;
  421. len -= 64 - s->blkused;
  422. /* Save current state */
  423. ABCD_SAVE = ABCD;
  424. E0_SAVE = E0;
  425. /* Rounds 0-3 */
  426. MSG0 = _mm_loadu_si128((const __m128i*)(s->block + 0));
  427. MSG0 = _mm_shuffle_epi8(MSG0, MASK);
  428. E0 = _mm_add_epi32(E0, MSG0);
  429. E1 = ABCD;
  430. ABCD = _mm_sha1rnds4_epu32(ABCD, E0, 0);
  431. /* Rounds 4-7 */
  432. MSG1 = _mm_loadu_si128((const __m128i*)(s->block + 16));
  433. MSG1 = _mm_shuffle_epi8(MSG1, MASK);
  434. E1 = _mm_sha1nexte_epu32(E1, MSG1);
  435. E0 = ABCD;
  436. ABCD = _mm_sha1rnds4_epu32(ABCD, E1, 0);
  437. MSG0 = _mm_sha1msg1_epu32(MSG0, MSG1);
  438. /* Rounds 8-11 */
  439. MSG2 = _mm_loadu_si128((const __m128i*)(s->block + 32));
  440. MSG2 = _mm_shuffle_epi8(MSG2, MASK);
  441. E0 = _mm_sha1nexte_epu32(E0, MSG2);
  442. E1 = ABCD;
  443. ABCD = _mm_sha1rnds4_epu32(ABCD, E0, 0);
  444. MSG1 = _mm_sha1msg1_epu32(MSG1, MSG2);
  445. MSG0 = _mm_xor_si128(MSG0, MSG2);
  446. /* Rounds 12-15 */
  447. MSG3 = _mm_loadu_si128((const __m128i*)(s->block + 48));
  448. MSG3 = _mm_shuffle_epi8(MSG3, MASK);
  449. E1 = _mm_sha1nexte_epu32(E1, MSG3);
  450. E0 = ABCD;
  451. MSG0 = _mm_sha1msg2_epu32(MSG0, MSG3);
  452. ABCD = _mm_sha1rnds4_epu32(ABCD, E1, 0);
  453. MSG2 = _mm_sha1msg1_epu32(MSG2, MSG3);
  454. MSG1 = _mm_xor_si128(MSG1, MSG3);
  455. /* Rounds 16-19 */
  456. E0 = _mm_sha1nexte_epu32(E0, MSG0);
  457. E1 = ABCD;
  458. MSG1 = _mm_sha1msg2_epu32(MSG1, MSG0);
  459. ABCD = _mm_sha1rnds4_epu32(ABCD, E0, 0);
  460. MSG3 = _mm_sha1msg1_epu32(MSG3, MSG0);
  461. MSG2 = _mm_xor_si128(MSG2, MSG0);
  462. /* Rounds 20-23 */
  463. E1 = _mm_sha1nexte_epu32(E1, MSG1);
  464. E0 = ABCD;
  465. MSG2 = _mm_sha1msg2_epu32(MSG2, MSG1);
  466. ABCD = _mm_sha1rnds4_epu32(ABCD, E1, 1);
  467. MSG0 = _mm_sha1msg1_epu32(MSG0, MSG1);
  468. MSG3 = _mm_xor_si128(MSG3, MSG1);
  469. /* Rounds 24-27 */
  470. E0 = _mm_sha1nexte_epu32(E0, MSG2);
  471. E1 = ABCD;
  472. MSG3 = _mm_sha1msg2_epu32(MSG3, MSG2);
  473. ABCD = _mm_sha1rnds4_epu32(ABCD, E0, 1);
  474. MSG1 = _mm_sha1msg1_epu32(MSG1, MSG2);
  475. MSG0 = _mm_xor_si128(MSG0, MSG2);
  476. /* Rounds 28-31 */
  477. E1 = _mm_sha1nexte_epu32(E1, MSG3);
  478. E0 = ABCD;
  479. MSG0 = _mm_sha1msg2_epu32(MSG0, MSG3);
  480. ABCD = _mm_sha1rnds4_epu32(ABCD, E1, 1);
  481. MSG2 = _mm_sha1msg1_epu32(MSG2, MSG3);
  482. MSG1 = _mm_xor_si128(MSG1, MSG3);
  483. /* Rounds 32-35 */
  484. E0 = _mm_sha1nexte_epu32(E0, MSG0);
  485. E1 = ABCD;
  486. MSG1 = _mm_sha1msg2_epu32(MSG1, MSG0);
  487. ABCD = _mm_sha1rnds4_epu32(ABCD, E0, 1);
  488. MSG3 = _mm_sha1msg1_epu32(MSG3, MSG0);
  489. MSG2 = _mm_xor_si128(MSG2, MSG0);
  490. /* Rounds 36-39 */
  491. E1 = _mm_sha1nexte_epu32(E1, MSG1);
  492. E0 = ABCD;
  493. MSG2 = _mm_sha1msg2_epu32(MSG2, MSG1);
  494. ABCD = _mm_sha1rnds4_epu32(ABCD, E1, 1);
  495. MSG0 = _mm_sha1msg1_epu32(MSG0, MSG1);
  496. MSG3 = _mm_xor_si128(MSG3, MSG1);
  497. /* Rounds 40-43 */
  498. E0 = _mm_sha1nexte_epu32(E0, MSG2);
  499. E1 = ABCD;
  500. MSG3 = _mm_sha1msg2_epu32(MSG3, MSG2);
  501. ABCD = _mm_sha1rnds4_epu32(ABCD, E0, 2);
  502. MSG1 = _mm_sha1msg1_epu32(MSG1, MSG2);
  503. MSG0 = _mm_xor_si128(MSG0, MSG2);
  504. /* Rounds 44-47 */
  505. E1 = _mm_sha1nexte_epu32(E1, MSG3);
  506. E0 = ABCD;
  507. MSG0 = _mm_sha1msg2_epu32(MSG0, MSG3);
  508. ABCD = _mm_sha1rnds4_epu32(ABCD, E1, 2);
  509. MSG2 = _mm_sha1msg1_epu32(MSG2, MSG3);
  510. MSG1 = _mm_xor_si128(MSG1, MSG3);
  511. /* Rounds 48-51 */
  512. E0 = _mm_sha1nexte_epu32(E0, MSG0);
  513. E1 = ABCD;
  514. MSG1 = _mm_sha1msg2_epu32(MSG1, MSG0);
  515. ABCD = _mm_sha1rnds4_epu32(ABCD, E0, 2);
  516. MSG3 = _mm_sha1msg1_epu32(MSG3, MSG0);
  517. MSG2 = _mm_xor_si128(MSG2, MSG0);
  518. /* Rounds 52-55 */
  519. E1 = _mm_sha1nexte_epu32(E1, MSG1);
  520. E0 = ABCD;
  521. MSG2 = _mm_sha1msg2_epu32(MSG2, MSG1);
  522. ABCD = _mm_sha1rnds4_epu32(ABCD, E1, 2);
  523. MSG0 = _mm_sha1msg1_epu32(MSG0, MSG1);
  524. MSG3 = _mm_xor_si128(MSG3, MSG1);
  525. /* Rounds 56-59 */
  526. E0 = _mm_sha1nexte_epu32(E0, MSG2);
  527. E1 = ABCD;
  528. MSG3 = _mm_sha1msg2_epu32(MSG3, MSG2);
  529. ABCD = _mm_sha1rnds4_epu32(ABCD, E0, 2);
  530. MSG1 = _mm_sha1msg1_epu32(MSG1, MSG2);
  531. MSG0 = _mm_xor_si128(MSG0, MSG2);
  532. /* Rounds 60-63 */
  533. E1 = _mm_sha1nexte_epu32(E1, MSG3);
  534. E0 = ABCD;
  535. MSG0 = _mm_sha1msg2_epu32(MSG0, MSG3);
  536. ABCD = _mm_sha1rnds4_epu32(ABCD, E1, 3);
  537. MSG2 = _mm_sha1msg1_epu32(MSG2, MSG3);
  538. MSG1 = _mm_xor_si128(MSG1, MSG3);
  539. /* Rounds 64-67 */
  540. E0 = _mm_sha1nexte_epu32(E0, MSG0);
  541. E1 = ABCD;
  542. MSG1 = _mm_sha1msg2_epu32(MSG1, MSG0);
  543. ABCD = _mm_sha1rnds4_epu32(ABCD, E0, 3);
  544. MSG3 = _mm_sha1msg1_epu32(MSG3, MSG0);
  545. MSG2 = _mm_xor_si128(MSG2, MSG0);
  546. /* Rounds 68-71 */
  547. E1 = _mm_sha1nexte_epu32(E1, MSG1);
  548. E0 = ABCD;
  549. MSG2 = _mm_sha1msg2_epu32(MSG2, MSG1);
  550. ABCD = _mm_sha1rnds4_epu32(ABCD, E1, 3);
  551. MSG3 = _mm_xor_si128(MSG3, MSG1);
  552. /* Rounds 72-75 */
  553. E0 = _mm_sha1nexte_epu32(E0, MSG2);
  554. E1 = ABCD;
  555. MSG3 = _mm_sha1msg2_epu32(MSG3, MSG2);
  556. ABCD = _mm_sha1rnds4_epu32(ABCD, E0, 3);
  557. /* Rounds 76-79 */
  558. E1 = _mm_sha1nexte_epu32(E1, MSG3);
  559. E0 = ABCD;
  560. ABCD = _mm_sha1rnds4_epu32(ABCD, E1, 3);
  561. /* Combine state */
  562. E0 = _mm_sha1nexte_epu32(E0, E0_SAVE);
  563. ABCD = _mm_add_epi32(ABCD, ABCD_SAVE);
  564. s->blkused = 0;
  565. }
  566. ABCD = _mm_shuffle_epi32(ABCD, 0x1B);
  567. /* Save state */
  568. _mm_storeu_si128((__m128i*) s->h, ABCD);
  569. s->h[4] = _mm_extract_epi32(E0, 3);
  570. memcpy(s->block, q, len);
  571. s->blkused = len;
  572. }
  573. }
  574. /*
  575. * Workaround LLVM bug https://bugs.llvm.org/show_bug.cgi?id=34980
  576. */
  577. static void sha1_ni(SHA_State * s, const unsigned char *q, int len)
  578. {
  579. sha1_ni_(s, q, len);
  580. }
  581. #else /* COMPILER_SUPPORTS_AES_NI */
  582. static void sha1_ni(SHA_State * s, const unsigned char *q, int len)
  583. {
  584. unreachable("sha1_ni not compiled in");
  585. }
  586. bool supports_sha_ni(void)
  587. {
  588. return false;
  589. }
  590. #endif /* COMPILER_SUPPORTS_AES_NI */