content_encoding.c 24 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823
  1. /***************************************************************************
  2. * _ _ ____ _
  3. * Project ___| | | | _ \| |
  4. * / __| | | | |_) | |
  5. * | (__| |_| | _ <| |___
  6. * \___|\___/|_| \_\_____|
  7. *
  8. * Copyright (C) Daniel Stenberg, <[email protected]>, et al.
  9. *
  10. * This software is licensed as described in the file COPYING, which
  11. * you should have received as part of this distribution. The terms
  12. * are also available at https://curl.se/docs/copyright.html.
  13. *
  14. * You may opt to use, copy, modify, merge, publish, distribute and/or sell
  15. * copies of the Software, and permit persons to whom the Software is
  16. * furnished to do so, under the terms of the COPYING file.
  17. *
  18. * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
  19. * KIND, either express or implied.
  20. *
  21. * SPDX-License-Identifier: curl
  22. *
  23. ***************************************************************************/
  24. #include "curl_setup.h"
  25. #include "urldata.h"
  26. #include "curlx/dynbuf.h"
  27. #ifdef HAVE_LIBZ
  28. #include <cm3p/zlib.h>
  29. #endif
  30. #ifdef HAVE_BROTLI
  31. #if defined(__GNUC__) || defined(__clang__)
  32. /* Ignore -Wvla warnings in brotli headers */
  33. #pragma GCC diagnostic push
  34. #pragma GCC diagnostic ignored "-Wvla"
  35. #endif
  36. #include <brotli/decode.h>
  37. #if defined(__GNUC__) || defined(__clang__)
  38. #pragma GCC diagnostic pop
  39. #endif
  40. #endif
  41. #ifdef HAVE_ZSTD
  42. #include <zstd.h>
  43. #endif
  44. #include "sendf.h"
  45. #include "curl_trc.h"
  46. #include "content_encoding.h"
  47. #define CONTENT_ENCODING_DEFAULT "identity"
  48. #ifndef CURL_DISABLE_HTTP
  49. /* allow no more than 5 "chained" compression steps */
  50. #define MAX_ENCODE_STACK 5
  51. #if defined(HAVE_LIBZ) || defined(HAVE_BROTLI) || defined(HAVE_ZSTD)
  52. #define DECOMPRESS_BUFFER_SIZE 16384 /* buffer size for decompressed data */
  53. #endif
  54. #ifdef HAVE_LIBZ
  55. #if !defined(ZLIB_VERNUM) || (ZLIB_VERNUM < 0x1252)
  56. #error "requires zlib 1.2.5.2 or newer"
  57. #endif
  58. typedef enum {
  59. ZLIB_UNINIT, /* uninitialized */
  60. ZLIB_INIT, /* initialized */
  61. ZLIB_INFLATING, /* inflating started. */
  62. ZLIB_EXTERNAL_TRAILER, /* reading external trailer */
  63. ZLIB_INIT_GZIP /* initialized in transparent gzip mode */
  64. } zlibInitState;
  65. /* Deflate and gzip writer. */
  66. struct zlib_writer {
  67. struct Curl_cwriter super;
  68. zlibInitState zlib_init; /* zlib init state */
  69. char buffer[DECOMPRESS_BUFFER_SIZE]; /* Put the decompressed data here. */
  70. uInt trailerlen; /* Remaining trailer byte count. */
  71. z_stream z; /* State structure for zlib. */
  72. };
  73. static voidpf zalloc_cb(voidpf opaque, unsigned int items, unsigned int size)
  74. {
  75. (void)opaque;
  76. /* not a typo, keep it curlx_calloc() */
  77. return (voidpf)curlx_calloc(items, size);
  78. }
  79. static void zfree_cb(voidpf opaque, voidpf ptr)
  80. {
  81. (void)opaque;
  82. curlx_free(ptr);
  83. }
  84. static CURLcode process_zlib_error(struct Curl_easy *data, z_stream *z)
  85. {
  86. if(z->msg)
  87. failf(data, "Error while processing content unencoding: %s",
  88. z->msg);
  89. else
  90. failf(data, "Error while processing content unencoding: "
  91. "Unknown failure within decompression software.");
  92. return CURLE_BAD_CONTENT_ENCODING;
  93. }
  94. static CURLcode exit_zlib(struct Curl_easy *data, z_stream *z,
  95. zlibInitState *zlib_init, CURLcode result)
  96. {
  97. if(*zlib_init != ZLIB_UNINIT) {
  98. if(inflateEnd(z) != Z_OK && result == CURLE_OK)
  99. result = process_zlib_error(data, z);
  100. *zlib_init = ZLIB_UNINIT;
  101. }
  102. return result;
  103. }
  104. static CURLcode process_trailer(struct Curl_easy *data, struct zlib_writer *zp)
  105. {
  106. z_stream *z = &zp->z;
  107. CURLcode result = CURLE_OK;
  108. uInt len = z->avail_in < zp->trailerlen ? z->avail_in : zp->trailerlen;
  109. /* Consume expected trailer bytes. Terminate stream if exhausted.
  110. Issue an error if unexpected bytes follow. */
  111. zp->trailerlen -= len;
  112. z->avail_in -= len;
  113. z->next_in += len;
  114. if(z->avail_in)
  115. result = CURLE_WRITE_ERROR;
  116. if(result || !zp->trailerlen)
  117. result = exit_zlib(data, z, &zp->zlib_init, result);
  118. else {
  119. /* Only occurs for gzip with zlib < 1.2.0.4 or raw deflate. */
  120. zp->zlib_init = ZLIB_EXTERNAL_TRAILER;
  121. }
  122. return result;
  123. }
  124. static CURLcode inflate_stream(struct Curl_easy *data,
  125. struct Curl_cwriter *writer, int type,
  126. zlibInitState started)
  127. {
  128. struct zlib_writer *zp = (struct zlib_writer *)writer;
  129. z_stream *z = &zp->z; /* zlib state structure */
  130. uInt nread = z->avail_in;
  131. z_const Bytef *orig_in = z->next_in;
  132. bool done = FALSE;
  133. CURLcode result = CURLE_OK; /* Curl_client_write status */
  134. /* Check state. */
  135. if(zp->zlib_init != ZLIB_INIT &&
  136. zp->zlib_init != ZLIB_INFLATING &&
  137. zp->zlib_init != ZLIB_INIT_GZIP)
  138. return exit_zlib(data, z, &zp->zlib_init, CURLE_WRITE_ERROR);
  139. /* because the buffer size is fixed, iteratively decompress and transfer to
  140. the client via next_write function. */
  141. while(!done) {
  142. int status; /* zlib status */
  143. done = TRUE;
  144. /* (re)set buffer for decompressed output for every iteration */
  145. z->next_out = (Bytef *)zp->buffer;
  146. z->avail_out = DECOMPRESS_BUFFER_SIZE;
  147. status = inflate(z, Z_BLOCK);
  148. /* Flush output data if some. */
  149. if(z->avail_out != DECOMPRESS_BUFFER_SIZE) {
  150. if(status == Z_OK || status == Z_STREAM_END) {
  151. zp->zlib_init = started; /* Data started. */
  152. result = Curl_cwriter_write(data, writer->next, type, zp->buffer,
  153. DECOMPRESS_BUFFER_SIZE - z->avail_out);
  154. if(result) {
  155. exit_zlib(data, z, &zp->zlib_init, result);
  156. break;
  157. }
  158. }
  159. }
  160. /* Dispatch by inflate() status. */
  161. switch(status) {
  162. case Z_OK:
  163. /* Always loop: there may be unflushed latched data in zlib state. */
  164. done = FALSE;
  165. break;
  166. case Z_BUF_ERROR:
  167. /* No more data to flush: just exit loop. */
  168. break;
  169. case Z_STREAM_END:
  170. result = process_trailer(data, zp);
  171. break;
  172. case Z_DATA_ERROR:
  173. /* some servers seem to not generate zlib headers, so this is an attempt
  174. to fix and continue anyway */
  175. if(zp->zlib_init == ZLIB_INIT) {
  176. if(inflateReset2(z, -MAX_WBITS) == Z_OK) {
  177. z->next_in = orig_in;
  178. z->avail_in = nread;
  179. zp->zlib_init = ZLIB_INFLATING;
  180. zp->trailerlen = 4; /* Tolerate up to 4 unknown trailer bytes. */
  181. done = FALSE;
  182. break;
  183. }
  184. zp->zlib_init = ZLIB_UNINIT; /* inflateEnd() already called. */
  185. }
  186. result = exit_zlib(data, z, &zp->zlib_init, process_zlib_error(data, z));
  187. break;
  188. default:
  189. result = exit_zlib(data, z, &zp->zlib_init, process_zlib_error(data, z));
  190. break;
  191. }
  192. }
  193. /* We are about to leave this call so the `nread' data bytes will not be seen
  194. again. If we are in a state that would wrongly allow restart in raw mode
  195. at the next call, assume output has already started. */
  196. if(nread && zp->zlib_init == ZLIB_INIT)
  197. zp->zlib_init = started; /* Cannot restart anymore. */
  198. return result;
  199. }
  200. /* Deflate handler. */
  201. static CURLcode deflate_do_init(struct Curl_easy *data,
  202. struct Curl_cwriter *writer)
  203. {
  204. struct zlib_writer *zp = (struct zlib_writer *)writer;
  205. z_stream *z = &zp->z; /* zlib state structure */
  206. /* Initialize zlib */
  207. z->zalloc = (alloc_func)zalloc_cb;
  208. z->zfree = (free_func)zfree_cb;
  209. if(inflateInit(z) != Z_OK)
  210. return process_zlib_error(data, z);
  211. zp->zlib_init = ZLIB_INIT;
  212. return CURLE_OK;
  213. }
  214. static CURLcode deflate_do_write(struct Curl_easy *data,
  215. struct Curl_cwriter *writer, int type,
  216. const char *buf, size_t nbytes)
  217. {
  218. struct zlib_writer *zp = (struct zlib_writer *)writer;
  219. z_stream *z = &zp->z; /* zlib state structure */
  220. if(!(type & CLIENTWRITE_BODY) || !nbytes)
  221. return Curl_cwriter_write(data, writer->next, type, buf, nbytes);
  222. /* Set the compressed input when this function is called */
  223. z->next_in = (z_const Bytef *)buf;
  224. z->avail_in = (uInt)nbytes;
  225. if(zp->zlib_init == ZLIB_EXTERNAL_TRAILER)
  226. return process_trailer(data, zp);
  227. /* Now uncompress the data */
  228. return inflate_stream(data, writer, type, ZLIB_INFLATING);
  229. }
  230. static void deflate_do_close(struct Curl_easy *data,
  231. struct Curl_cwriter *writer)
  232. {
  233. struct zlib_writer *zp = (struct zlib_writer *)writer;
  234. z_stream *z = &zp->z; /* zlib state structure */
  235. exit_zlib(data, z, &zp->zlib_init, CURLE_OK);
  236. }
  237. static const struct Curl_cwtype deflate_encoding = {
  238. "deflate",
  239. NULL,
  240. deflate_do_init,
  241. deflate_do_write,
  242. deflate_do_close,
  243. sizeof(struct zlib_writer)
  244. };
  245. /*
  246. * Gzip handler.
  247. */
  248. static CURLcode gzip_do_init(struct Curl_easy *data,
  249. struct Curl_cwriter *writer)
  250. {
  251. struct zlib_writer *zp = (struct zlib_writer *)writer;
  252. z_stream *z = &zp->z; /* zlib state structure */
  253. /* Initialize zlib */
  254. z->zalloc = (alloc_func)zalloc_cb;
  255. z->zfree = (free_func)zfree_cb;
  256. if(inflateInit2(z, MAX_WBITS + 32) != Z_OK)
  257. return process_zlib_error(data, z);
  258. zp->zlib_init = ZLIB_INIT_GZIP; /* Transparent gzip decompress state */
  259. return CURLE_OK;
  260. }
  261. static CURLcode gzip_do_write(struct Curl_easy *data,
  262. struct Curl_cwriter *writer, int type,
  263. const char *buf, size_t nbytes)
  264. {
  265. struct zlib_writer *zp = (struct zlib_writer *)writer;
  266. z_stream *z = &zp->z; /* zlib state structure */
  267. if(!(type & CLIENTWRITE_BODY) || !nbytes)
  268. return Curl_cwriter_write(data, writer->next, type, buf, nbytes);
  269. if(zp->zlib_init == ZLIB_INIT_GZIP) {
  270. /* Let zlib handle the gzip decompression entirely */
  271. z->next_in = (z_const Bytef *)buf;
  272. z->avail_in = (uInt)nbytes;
  273. /* Now uncompress the data */
  274. return inflate_stream(data, writer, type, ZLIB_INIT_GZIP);
  275. }
  276. /* We are running with an old version: return error. */
  277. return exit_zlib(data, z, &zp->zlib_init, CURLE_WRITE_ERROR);
  278. }
  279. static void gzip_do_close(struct Curl_easy *data,
  280. struct Curl_cwriter *writer)
  281. {
  282. struct zlib_writer *zp = (struct zlib_writer *)writer;
  283. z_stream *z = &zp->z; /* zlib state structure */
  284. exit_zlib(data, z, &zp->zlib_init, CURLE_OK);
  285. }
  286. static const struct Curl_cwtype gzip_encoding = {
  287. "gzip",
  288. "x-gzip",
  289. gzip_do_init,
  290. gzip_do_write,
  291. gzip_do_close,
  292. sizeof(struct zlib_writer)
  293. };
  294. #endif /* HAVE_LIBZ */
  295. #ifdef HAVE_BROTLI
  296. /* Brotli writer. */
  297. struct brotli_writer {
  298. struct Curl_cwriter super;
  299. char buffer[DECOMPRESS_BUFFER_SIZE];
  300. BrotliDecoderState *br; /* State structure for brotli. */
  301. };
  302. static CURLcode brotli_map_error(BrotliDecoderErrorCode be)
  303. {
  304. switch(be) {
  305. case BROTLI_DECODER_ERROR_FORMAT_EXUBERANT_NIBBLE:
  306. case BROTLI_DECODER_ERROR_FORMAT_EXUBERANT_META_NIBBLE:
  307. case BROTLI_DECODER_ERROR_FORMAT_SIMPLE_HUFFMAN_ALPHABET:
  308. case BROTLI_DECODER_ERROR_FORMAT_SIMPLE_HUFFMAN_SAME:
  309. case BROTLI_DECODER_ERROR_FORMAT_CL_SPACE:
  310. case BROTLI_DECODER_ERROR_FORMAT_HUFFMAN_SPACE:
  311. case BROTLI_DECODER_ERROR_FORMAT_CONTEXT_MAP_REPEAT:
  312. case BROTLI_DECODER_ERROR_FORMAT_BLOCK_LENGTH_1:
  313. case BROTLI_DECODER_ERROR_FORMAT_BLOCK_LENGTH_2:
  314. case BROTLI_DECODER_ERROR_FORMAT_TRANSFORM:
  315. case BROTLI_DECODER_ERROR_FORMAT_DICTIONARY:
  316. case BROTLI_DECODER_ERROR_FORMAT_WINDOW_BITS:
  317. case BROTLI_DECODER_ERROR_FORMAT_PADDING_1:
  318. case BROTLI_DECODER_ERROR_FORMAT_PADDING_2:
  319. #ifdef BROTLI_DECODER_ERROR_COMPOUND_DICTIONARY /* brotli v1.1.0+ */
  320. case BROTLI_DECODER_ERROR_COMPOUND_DICTIONARY:
  321. #endif
  322. case BROTLI_DECODER_ERROR_DICTIONARY_NOT_SET:
  323. case BROTLI_DECODER_ERROR_INVALID_ARGUMENTS:
  324. return CURLE_BAD_CONTENT_ENCODING;
  325. case BROTLI_DECODER_ERROR_ALLOC_CONTEXT_MODES:
  326. case BROTLI_DECODER_ERROR_ALLOC_TREE_GROUPS:
  327. case BROTLI_DECODER_ERROR_ALLOC_CONTEXT_MAP:
  328. case BROTLI_DECODER_ERROR_ALLOC_RING_BUFFER_1:
  329. case BROTLI_DECODER_ERROR_ALLOC_RING_BUFFER_2:
  330. case BROTLI_DECODER_ERROR_ALLOC_BLOCK_TYPE_TREES:
  331. return CURLE_OUT_OF_MEMORY;
  332. default:
  333. break;
  334. }
  335. return CURLE_WRITE_ERROR;
  336. }
  337. static CURLcode brotli_do_init(struct Curl_easy *data,
  338. struct Curl_cwriter *writer)
  339. {
  340. struct brotli_writer *bp = (struct brotli_writer *)writer;
  341. (void)data;
  342. bp->br = BrotliDecoderCreateInstance(NULL, NULL, NULL);
  343. return bp->br ? CURLE_OK : CURLE_OUT_OF_MEMORY;
  344. }
  345. static CURLcode brotli_do_write(struct Curl_easy *data,
  346. struct Curl_cwriter *writer, int type,
  347. const char *buf, size_t nbytes)
  348. {
  349. struct brotli_writer *bp = (struct brotli_writer *)writer;
  350. const uint8_t *src = (const uint8_t *)buf;
  351. uint8_t *dst;
  352. size_t dstleft;
  353. CURLcode result = CURLE_OK;
  354. BrotliDecoderResult r = BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT;
  355. if(!(type & CLIENTWRITE_BODY) || !nbytes)
  356. return Curl_cwriter_write(data, writer->next, type, buf, nbytes);
  357. if(!bp->br)
  358. return CURLE_WRITE_ERROR; /* Stream already ended. */
  359. while((nbytes || r == BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT) &&
  360. result == CURLE_OK) {
  361. dst = (uint8_t *)bp->buffer;
  362. dstleft = DECOMPRESS_BUFFER_SIZE;
  363. r = BrotliDecoderDecompressStream(bp->br,
  364. &nbytes, &src, &dstleft, &dst, NULL);
  365. result = Curl_cwriter_write(data, writer->next, type,
  366. bp->buffer, DECOMPRESS_BUFFER_SIZE - dstleft);
  367. if(result)
  368. break;
  369. switch(r) {
  370. case BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT:
  371. case BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT:
  372. break;
  373. case BROTLI_DECODER_RESULT_SUCCESS:
  374. BrotliDecoderDestroyInstance(bp->br);
  375. bp->br = NULL;
  376. if(nbytes)
  377. result = CURLE_WRITE_ERROR;
  378. break;
  379. default:
  380. result = brotli_map_error(BrotliDecoderGetErrorCode(bp->br));
  381. break;
  382. }
  383. }
  384. return result;
  385. }
  386. static void brotli_do_close(struct Curl_easy *data,
  387. struct Curl_cwriter *writer)
  388. {
  389. struct brotli_writer *bp = (struct brotli_writer *)writer;
  390. (void)data;
  391. if(bp->br) {
  392. BrotliDecoderDestroyInstance(bp->br);
  393. bp->br = NULL;
  394. }
  395. }
  396. static const struct Curl_cwtype brotli_encoding = {
  397. "br",
  398. NULL,
  399. brotli_do_init,
  400. brotli_do_write,
  401. brotli_do_close,
  402. sizeof(struct brotli_writer)
  403. };
  404. #endif
  405. #ifdef HAVE_ZSTD
  406. /* Zstd writer. */
  407. struct zstd_writer {
  408. struct Curl_cwriter super;
  409. ZSTD_DStream *zds; /* State structure for zstd. */
  410. char buffer[DECOMPRESS_BUFFER_SIZE];
  411. };
  412. #ifdef ZSTD_STATIC_LINKING_ONLY
  413. static void *Curl_zstd_alloc(void *opaque, size_t size)
  414. {
  415. (void)opaque;
  416. return Curl_cmalloc(size);
  417. }
  418. static void Curl_zstd_free(void *opaque, void *address)
  419. {
  420. (void)opaque;
  421. Curl_cfree(address);
  422. }
  423. #endif
  424. static CURLcode zstd_do_init(struct Curl_easy *data,
  425. struct Curl_cwriter *writer)
  426. {
  427. struct zstd_writer *zp = (struct zstd_writer *)writer;
  428. (void)data;
  429. #ifdef ZSTD_STATIC_LINKING_ONLY
  430. zp->zds = ZSTD_createDStream_advanced((ZSTD_customMem) {
  431. .customAlloc = Curl_zstd_alloc,
  432. .customFree = Curl_zstd_free,
  433. .opaque = NULL
  434. });
  435. #else
  436. zp->zds = ZSTD_createDStream();
  437. #endif
  438. return zp->zds ? CURLE_OK : CURLE_OUT_OF_MEMORY;
  439. }
  440. static CURLcode zstd_do_write(struct Curl_easy *data,
  441. struct Curl_cwriter *writer, int type,
  442. const char *buf, size_t nbytes)
  443. {
  444. CURLcode result = CURLE_OK;
  445. struct zstd_writer *zp = (struct zstd_writer *)writer;
  446. ZSTD_inBuffer in;
  447. ZSTD_outBuffer out;
  448. size_t errorCode;
  449. if(!(type & CLIENTWRITE_BODY) || !nbytes)
  450. return Curl_cwriter_write(data, writer->next, type, buf, nbytes);
  451. in.pos = 0;
  452. in.src = buf;
  453. in.size = nbytes;
  454. for(;;) {
  455. out.pos = 0;
  456. out.dst = zp->buffer;
  457. out.size = DECOMPRESS_BUFFER_SIZE;
  458. errorCode = ZSTD_decompressStream(zp->zds, &out, &in);
  459. if(ZSTD_isError(errorCode)) {
  460. return CURLE_BAD_CONTENT_ENCODING;
  461. }
  462. if(out.pos > 0) {
  463. result = Curl_cwriter_write(data, writer->next, type,
  464. zp->buffer, out.pos);
  465. if(result)
  466. break;
  467. }
  468. if((in.pos == nbytes) && (out.pos < out.size))
  469. break;
  470. }
  471. return result;
  472. }
  473. static void zstd_do_close(struct Curl_easy *data,
  474. struct Curl_cwriter *writer)
  475. {
  476. struct zstd_writer *zp = (struct zstd_writer *)writer;
  477. (void)data;
  478. if(zp->zds) {
  479. ZSTD_freeDStream(zp->zds);
  480. zp->zds = NULL;
  481. }
  482. }
  483. static const struct Curl_cwtype zstd_encoding = {
  484. "zstd",
  485. NULL,
  486. zstd_do_init,
  487. zstd_do_write,
  488. zstd_do_close,
  489. sizeof(struct zstd_writer)
  490. };
  491. #endif
  492. /* Identity handler. */
  493. static const struct Curl_cwtype identity_encoding = {
  494. "identity",
  495. "none",
  496. Curl_cwriter_def_init,
  497. Curl_cwriter_def_write,
  498. Curl_cwriter_def_close,
  499. sizeof(struct Curl_cwriter)
  500. };
  501. /* supported general content decoders. */
  502. static const struct Curl_cwtype * const general_unencoders[] = {
  503. &identity_encoding,
  504. #ifdef HAVE_LIBZ
  505. &deflate_encoding,
  506. &gzip_encoding,
  507. #endif
  508. #ifdef HAVE_BROTLI
  509. &brotli_encoding,
  510. #endif
  511. #ifdef HAVE_ZSTD
  512. &zstd_encoding,
  513. #endif
  514. NULL
  515. };
  516. /* supported content decoders only for transfer encodings */
  517. static const struct Curl_cwtype * const transfer_unencoders[] = {
  518. #ifndef CURL_DISABLE_HTTP
  519. &Curl_httpchunk_unencoder,
  520. #endif
  521. NULL
  522. };
  523. /* Return the list of comma-separated names of supported encodings.
  524. */
  525. char *Curl_get_content_encodings(void)
  526. {
  527. struct dynbuf enc;
  528. const struct Curl_cwtype * const *cep;
  529. CURLcode result = CURLE_OK;
  530. curlx_dyn_init(&enc, 255);
  531. for(cep = general_unencoders; *cep && !result; cep++) {
  532. const struct Curl_cwtype *ce = *cep;
  533. if(!curl_strequal(ce->name, CONTENT_ENCODING_DEFAULT)) {
  534. if(curlx_dyn_len(&enc))
  535. result = curlx_dyn_addn(&enc, ", ", 2);
  536. if(!result)
  537. result = curlx_dyn_add(&enc, ce->name);
  538. }
  539. }
  540. if(!result)
  541. return curlx_dyn_ptr(&enc);
  542. return NULL;
  543. }
  544. /* Deferred error dummy writer. */
  545. static CURLcode error_do_init(struct Curl_easy *data,
  546. struct Curl_cwriter *writer)
  547. {
  548. (void)data;
  549. (void)writer;
  550. return CURLE_OK;
  551. }
  552. static CURLcode error_do_write(struct Curl_easy *data,
  553. struct Curl_cwriter *writer, int type,
  554. const char *buf, size_t nbytes)
  555. {
  556. (void)writer;
  557. (void)buf;
  558. (void)nbytes;
  559. if(!(type & CLIENTWRITE_BODY) || !nbytes)
  560. return Curl_cwriter_write(data, writer->next, type, buf, nbytes);
  561. failf(data, "Unrecognized content encoding type");
  562. return CURLE_BAD_CONTENT_ENCODING;
  563. }
  564. static void error_do_close(struct Curl_easy *data,
  565. struct Curl_cwriter *writer)
  566. {
  567. (void)data;
  568. (void)writer;
  569. }
  570. static const struct Curl_cwtype error_writer = {
  571. "ce-error",
  572. NULL,
  573. error_do_init,
  574. error_do_write,
  575. error_do_close,
  576. sizeof(struct Curl_cwriter)
  577. };
  578. /* Find the content encoding by name. */
  579. static const struct Curl_cwtype *find_unencode_writer(const char *name,
  580. size_t len,
  581. Curl_cwriter_phase phase)
  582. {
  583. const struct Curl_cwtype * const *cep;
  584. if(phase == CURL_CW_TRANSFER_DECODE) {
  585. for(cep = transfer_unencoders; *cep; cep++) {
  586. const struct Curl_cwtype *ce = *cep;
  587. if((curl_strnequal(name, ce->name, len) && !ce->name[len]) ||
  588. (ce->alias && curl_strnequal(name, ce->alias, len)
  589. && !ce->alias[len]))
  590. return ce;
  591. }
  592. }
  593. /* look among the general decoders */
  594. for(cep = general_unencoders; *cep; cep++) {
  595. const struct Curl_cwtype *ce = *cep;
  596. if((curl_strnequal(name, ce->name, len) && !ce->name[len]) ||
  597. (ce->alias && curl_strnequal(name, ce->alias, len) && !ce->alias[len]))
  598. return ce;
  599. }
  600. return NULL;
  601. }
  602. /* Setup the unencoding stack from the Content-Encoding header value.
  603. * See RFC 7231 section 3.1.2.2. */
  604. CURLcode Curl_build_unencoding_stack(struct Curl_easy *data,
  605. const char *enclist, int is_transfer)
  606. {
  607. Curl_cwriter_phase phase = is_transfer ?
  608. CURL_CW_TRANSFER_DECODE : CURL_CW_CONTENT_DECODE;
  609. CURLcode result;
  610. bool has_chunked = FALSE;
  611. do {
  612. const char *name;
  613. size_t namelen;
  614. bool is_chunked = FALSE;
  615. /* Parse a single encoding name. */
  616. while(ISBLANK(*enclist) || *enclist == ',')
  617. enclist++;
  618. name = enclist;
  619. for(namelen = 0; *enclist && *enclist != ','; enclist++)
  620. if(*enclist > ' ')
  621. namelen = enclist - name + 1;
  622. if(namelen) {
  623. const struct Curl_cwtype *cwt;
  624. struct Curl_cwriter *writer;
  625. CURL_TRC_WRITE(data, "looking for %s decoder: %.*s",
  626. is_transfer ? "transfer" : "content", (int)namelen, name);
  627. is_chunked = (is_transfer && (namelen == 7) &&
  628. curl_strnequal(name, "chunked", 7));
  629. /* if we skip the decoding in this phase, do not look further.
  630. * Exception is "chunked" transfer-encoding which always must happen */
  631. if((is_transfer && !data->set.http_transfer_encoding && !is_chunked) ||
  632. (!is_transfer && data->set.http_ce_skip)) {
  633. bool is_identity = curl_strnequal(name, "identity", 8);
  634. /* not requested, ignore */
  635. CURL_TRC_WRITE(data, "decoder not requested, ignored: %.*s",
  636. (int)namelen, name);
  637. if(is_transfer && !data->set.http_te_skip) {
  638. if(has_chunked)
  639. failf(data, "A Transfer-Encoding (%.*s) was listed after chunked",
  640. (int)namelen, name);
  641. else if(is_identity)
  642. continue;
  643. else
  644. failf(data, "Unsolicited Transfer-Encoding (%.*s) found",
  645. (int)namelen, name);
  646. return CURLE_BAD_CONTENT_ENCODING;
  647. }
  648. return CURLE_OK;
  649. }
  650. if(Curl_cwriter_count(data, phase) + 1 >= MAX_ENCODE_STACK) {
  651. failf(data, "Reject response due to more than %u content encodings",
  652. MAX_ENCODE_STACK);
  653. return CURLE_BAD_CONTENT_ENCODING;
  654. }
  655. cwt = find_unencode_writer(name, namelen, phase);
  656. if(cwt && is_chunked && Curl_cwriter_get_by_type(data, cwt)) {
  657. /* A 'chunked' transfer encoding has already been added.
  658. * Ignore duplicates. See #13451.
  659. * Also RFC 9112, ch. 6.1:
  660. * "A sender MUST NOT apply the chunked transfer coding more than
  661. * once to a message body."
  662. */
  663. CURL_TRC_WRITE(data, "ignoring duplicate 'chunked' decoder");
  664. return CURLE_OK;
  665. }
  666. if(is_transfer && !is_chunked &&
  667. Curl_cwriter_get_by_name(data, "chunked")) {
  668. /* RFC 9112, ch. 6.1:
  669. * "If any transfer coding other than chunked is applied to a
  670. * response's content, the sender MUST either apply chunked as the
  671. * final transfer coding or terminate the message by closing the
  672. * connection."
  673. * "chunked" must be the last added to be the first in its phase,
  674. * reject this.
  675. */
  676. failf(data, "Reject response due to 'chunked' not being the last "
  677. "Transfer-Encoding");
  678. return CURLE_BAD_CONTENT_ENCODING;
  679. }
  680. if(!cwt)
  681. cwt = &error_writer; /* Defer error at use. */
  682. result = Curl_cwriter_create(&writer, data, cwt, phase);
  683. CURL_TRC_WRITE(data, "added %s decoder %s -> %d",
  684. is_transfer ? "transfer" : "content", cwt->name, result);
  685. if(result)
  686. return result;
  687. result = Curl_cwriter_add(data, writer);
  688. if(result) {
  689. Curl_cwriter_free(data, writer);
  690. return result;
  691. }
  692. if(is_chunked)
  693. has_chunked = TRUE;
  694. }
  695. } while(*enclist);
  696. return CURLE_OK;
  697. }
  698. #else
  699. /* Stubs for builds without HTTP. */
  700. CURLcode Curl_build_unencoding_stack(struct Curl_easy *data,
  701. const char *enclist, int is_transfer)
  702. {
  703. (void)data;
  704. (void)enclist;
  705. (void)is_transfer;
  706. return CURLE_NOT_BUILT_IN;
  707. }
  708. char *Curl_get_content_encodings(void)
  709. {
  710. return curlx_strdup(CONTENT_ENCODING_DEFAULT);
  711. }
  712. #endif /* CURL_DISABLE_HTTP */