content_encoding.c 24 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858
  1. /***************************************************************************
  2. * _ _ ____ _
  3. * Project ___| | | | _ \| |
  4. * / __| | | | |_) | |
  5. * | (__| |_| | _ <| |___
  6. * \___|\___/|_| \_\_____|
  7. *
  8. * Copyright (C) Daniel Stenberg, <[email protected]>, et al.
  9. *
  10. * This software is licensed as described in the file COPYING, which
  11. * you should have received as part of this distribution. The terms
  12. * are also available at https://curl.se/docs/copyright.html.
  13. *
  14. * You may opt to use, copy, modify, merge, publish, distribute and/or sell
  15. * copies of the Software, and permit persons to whom the Software is
  16. * furnished to do so, under the terms of the COPYING file.
  17. *
  18. * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
  19. * KIND, either express or implied.
  20. *
  21. * SPDX-License-Identifier: curl
  22. *
  23. ***************************************************************************/
  24. #include "curl_setup.h"
  25. #include "urldata.h"
  26. #include <curl/curl.h>
  27. #include <stddef.h>
  28. #ifdef HAVE_LIBZ
  29. #include <zlib.h>
  30. #endif
  31. #ifdef HAVE_BROTLI
  32. #if defined(__GNUC__) || defined(__clang__)
  33. /* Ignore -Wvla warnings in brotli headers */
  34. #pragma GCC diagnostic push
  35. #pragma GCC diagnostic ignored "-Wvla"
  36. #endif
  37. #include <brotli/decode.h>
  38. #if defined(__GNUC__) || defined(__clang__)
  39. #pragma GCC diagnostic pop
  40. #endif
  41. #endif
  42. #ifdef HAVE_ZSTD
  43. #include <zstd.h>
  44. #endif
  45. #include "sendf.h"
  46. #include "http.h"
  47. #include "content_encoding.h"
  48. #include "strdup.h"
  49. #include "strcase.h"
  50. /* The last 3 #include files should be in this order */
  51. #include "curl_printf.h"
  52. #include "curl_memory.h"
  53. #include "memdebug.h"
  54. #define CONTENT_ENCODING_DEFAULT "identity"
  55. #ifndef CURL_DISABLE_HTTP
  56. /* allow no more than 5 "chained" compression steps */
  57. #define MAX_ENCODE_STACK 5
  58. #define DECOMPRESS_BUFFER_SIZE 16384 /* buffer size for decompressed data */
  59. #ifdef HAVE_LIBZ
  60. typedef enum {
  61. ZLIB_UNINIT, /* uninitialized */
  62. ZLIB_INIT, /* initialized */
  63. ZLIB_INFLATING, /* inflating started. */
  64. ZLIB_EXTERNAL_TRAILER, /* reading external trailer */
  65. ZLIB_INIT_GZIP /* initialized in transparent gzip mode */
  66. } zlibInitState;
  67. /* Deflate and gzip writer. */
  68. struct zlib_writer {
  69. struct Curl_cwriter super;
  70. zlibInitState zlib_init; /* zlib init state */
  71. char buffer[DECOMPRESS_BUFFER_SIZE]; /* Put the decompressed data here. */
  72. uInt trailerlen; /* Remaining trailer byte count. */
  73. z_stream z; /* State structure for zlib. */
  74. };
  75. static voidpf
  76. zalloc_cb(voidpf opaque, unsigned int items, unsigned int size)
  77. {
  78. (void) opaque;
  79. /* not a typo, keep it calloc() */
  80. return (voidpf) calloc(items, size);
  81. }
  82. static void
  83. zfree_cb(voidpf opaque, voidpf ptr)
  84. {
  85. (void) opaque;
  86. free(ptr);
  87. }
  88. static CURLcode
  89. process_zlib_error(struct Curl_easy *data, z_stream *z)
  90. {
  91. if(z->msg)
  92. failf(data, "Error while processing content unencoding: %s",
  93. z->msg);
  94. else
  95. failf(data, "Error while processing content unencoding: "
  96. "Unknown failure within decompression software.");
  97. return CURLE_BAD_CONTENT_ENCODING;
  98. }
  99. static CURLcode
  100. exit_zlib(struct Curl_easy *data,
  101. z_stream *z, zlibInitState *zlib_init, CURLcode result)
  102. {
  103. if(*zlib_init != ZLIB_UNINIT) {
  104. if(inflateEnd(z) != Z_OK && result == CURLE_OK)
  105. result = process_zlib_error(data, z);
  106. *zlib_init = ZLIB_UNINIT;
  107. }
  108. return result;
  109. }
  110. static CURLcode process_trailer(struct Curl_easy *data,
  111. struct zlib_writer *zp)
  112. {
  113. z_stream *z = &zp->z;
  114. CURLcode result = CURLE_OK;
  115. uInt len = z->avail_in < zp->trailerlen ? z->avail_in : zp->trailerlen;
  116. /* Consume expected trailer bytes. Terminate stream if exhausted.
  117. Issue an error if unexpected bytes follow. */
  118. zp->trailerlen -= len;
  119. z->avail_in -= len;
  120. z->next_in += len;
  121. if(z->avail_in)
  122. result = CURLE_WRITE_ERROR;
  123. if(result || !zp->trailerlen)
  124. result = exit_zlib(data, z, &zp->zlib_init, result);
  125. else {
  126. /* Only occurs for gzip with zlib < 1.2.0.4 or raw deflate. */
  127. zp->zlib_init = ZLIB_EXTERNAL_TRAILER;
  128. }
  129. return result;
  130. }
  131. static CURLcode inflate_stream(struct Curl_easy *data,
  132. struct Curl_cwriter *writer, int type,
  133. zlibInitState started)
  134. {
  135. struct zlib_writer *zp = (struct zlib_writer *) writer;
  136. z_stream *z = &zp->z; /* zlib state structure */
  137. uInt nread = z->avail_in;
  138. Bytef *orig_in = z->next_in;
  139. bool done = FALSE;
  140. CURLcode result = CURLE_OK; /* Curl_client_write status */
  141. /* Check state. */
  142. if(zp->zlib_init != ZLIB_INIT &&
  143. zp->zlib_init != ZLIB_INFLATING &&
  144. zp->zlib_init != ZLIB_INIT_GZIP)
  145. return exit_zlib(data, z, &zp->zlib_init, CURLE_WRITE_ERROR);
  146. /* because the buffer size is fixed, iteratively decompress and transfer to
  147. the client via next_write function. */
  148. while(!done) {
  149. int status; /* zlib status */
  150. done = TRUE;
  151. /* (re)set buffer for decompressed output for every iteration */
  152. z->next_out = (Bytef *) zp->buffer;
  153. z->avail_out = DECOMPRESS_BUFFER_SIZE;
  154. #ifdef Z_BLOCK
  155. /* Z_BLOCK is only available in zlib ver. >= 1.2.0.5 */
  156. status = inflate(z, Z_BLOCK);
  157. #else
  158. /* fallback for zlib ver. < 1.2.0.5 */
  159. status = inflate(z, Z_SYNC_FLUSH);
  160. #endif
  161. /* Flush output data if some. */
  162. if(z->avail_out != DECOMPRESS_BUFFER_SIZE) {
  163. if(status == Z_OK || status == Z_STREAM_END) {
  164. zp->zlib_init = started; /* Data started. */
  165. result = Curl_cwriter_write(data, writer->next, type, zp->buffer,
  166. DECOMPRESS_BUFFER_SIZE - z->avail_out);
  167. if(result) {
  168. exit_zlib(data, z, &zp->zlib_init, result);
  169. break;
  170. }
  171. }
  172. }
  173. /* Dispatch by inflate() status. */
  174. switch(status) {
  175. case Z_OK:
  176. /* Always loop: there may be unflushed latched data in zlib state. */
  177. done = FALSE;
  178. break;
  179. case Z_BUF_ERROR:
  180. /* No more data to flush: just exit loop. */
  181. break;
  182. case Z_STREAM_END:
  183. result = process_trailer(data, zp);
  184. break;
  185. case Z_DATA_ERROR:
  186. /* some servers seem to not generate zlib headers, so this is an attempt
  187. to fix and continue anyway */
  188. if(zp->zlib_init == ZLIB_INIT) {
  189. /* Do not use inflateReset2(): only available since zlib 1.2.3.4. */
  190. (void) inflateEnd(z); /* do not care about the return code */
  191. if(inflateInit2(z, -MAX_WBITS) == Z_OK) {
  192. z->next_in = orig_in;
  193. z->avail_in = nread;
  194. zp->zlib_init = ZLIB_INFLATING;
  195. zp->trailerlen = 4; /* Tolerate up to 4 unknown trailer bytes. */
  196. done = FALSE;
  197. break;
  198. }
  199. zp->zlib_init = ZLIB_UNINIT; /* inflateEnd() already called. */
  200. }
  201. result = exit_zlib(data, z, &zp->zlib_init, process_zlib_error(data, z));
  202. break;
  203. default:
  204. result = exit_zlib(data, z, &zp->zlib_init, process_zlib_error(data, z));
  205. break;
  206. }
  207. }
  208. /* We are about to leave this call so the `nread' data bytes will not be seen
  209. again. If we are in a state that would wrongly allow restart in raw mode
  210. at the next call, assume output has already started. */
  211. if(nread && zp->zlib_init == ZLIB_INIT)
  212. zp->zlib_init = started; /* Cannot restart anymore. */
  213. return result;
  214. }
  215. /* Deflate handler. */
  216. static CURLcode deflate_do_init(struct Curl_easy *data,
  217. struct Curl_cwriter *writer)
  218. {
  219. struct zlib_writer *zp = (struct zlib_writer *) writer;
  220. z_stream *z = &zp->z; /* zlib state structure */
  221. /* Initialize zlib */
  222. z->zalloc = (alloc_func) zalloc_cb;
  223. z->zfree = (free_func) zfree_cb;
  224. if(inflateInit(z) != Z_OK)
  225. return process_zlib_error(data, z);
  226. zp->zlib_init = ZLIB_INIT;
  227. return CURLE_OK;
  228. }
  229. static CURLcode deflate_do_write(struct Curl_easy *data,
  230. struct Curl_cwriter *writer, int type,
  231. const char *buf, size_t nbytes)
  232. {
  233. struct zlib_writer *zp = (struct zlib_writer *) writer;
  234. z_stream *z = &zp->z; /* zlib state structure */
  235. if(!(type & CLIENTWRITE_BODY) || !nbytes)
  236. return Curl_cwriter_write(data, writer->next, type, buf, nbytes);
  237. /* Set the compressed input when this function is called */
  238. z->next_in = (Bytef *) buf;
  239. z->avail_in = (uInt) nbytes;
  240. if(zp->zlib_init == ZLIB_EXTERNAL_TRAILER)
  241. return process_trailer(data, zp);
  242. /* Now uncompress the data */
  243. return inflate_stream(data, writer, type, ZLIB_INFLATING);
  244. }
  245. static void deflate_do_close(struct Curl_easy *data,
  246. struct Curl_cwriter *writer)
  247. {
  248. struct zlib_writer *zp = (struct zlib_writer *) writer;
  249. z_stream *z = &zp->z; /* zlib state structure */
  250. exit_zlib(data, z, &zp->zlib_init, CURLE_OK);
  251. }
  252. static const struct Curl_cwtype deflate_encoding = {
  253. "deflate",
  254. NULL,
  255. deflate_do_init,
  256. deflate_do_write,
  257. deflate_do_close,
  258. sizeof(struct zlib_writer)
  259. };
  260. /* Gzip handler. */
  261. static CURLcode gzip_do_init(struct Curl_easy *data,
  262. struct Curl_cwriter *writer)
  263. {
  264. struct zlib_writer *zp = (struct zlib_writer *) writer;
  265. z_stream *z = &zp->z; /* zlib state structure */
  266. const char *v = zlibVersion();
  267. /* Initialize zlib */
  268. z->zalloc = (alloc_func) zalloc_cb;
  269. z->zfree = (free_func) zfree_cb;
  270. if(strcmp(v, "1.2.0.4") >= 0) {
  271. /* zlib version >= 1.2.0.4 supports transparent gzip decompressing */
  272. if(inflateInit2(z, MAX_WBITS + 32) != Z_OK) {
  273. return process_zlib_error(data, z);
  274. }
  275. zp->zlib_init = ZLIB_INIT_GZIP; /* Transparent gzip decompress state */
  276. }
  277. else {
  278. failf(data, "too old zlib version: %s", v);
  279. return CURLE_FAILED_INIT;
  280. }
  281. return CURLE_OK;
  282. }
  283. static CURLcode gzip_do_write(struct Curl_easy *data,
  284. struct Curl_cwriter *writer, int type,
  285. const char *buf, size_t nbytes)
  286. {
  287. struct zlib_writer *zp = (struct zlib_writer *) writer;
  288. z_stream *z = &zp->z; /* zlib state structure */
  289. if(!(type & CLIENTWRITE_BODY) || !nbytes)
  290. return Curl_cwriter_write(data, writer->next, type, buf, nbytes);
  291. if(zp->zlib_init == ZLIB_INIT_GZIP) {
  292. /* Let zlib handle the gzip decompression entirely */
  293. z->next_in = (Bytef *) buf;
  294. z->avail_in = (uInt) nbytes;
  295. /* Now uncompress the data */
  296. return inflate_stream(data, writer, type, ZLIB_INIT_GZIP);
  297. }
  298. /* We are running with an old version: return error. */
  299. return exit_zlib(data, z, &zp->zlib_init, CURLE_WRITE_ERROR);
  300. }
  301. static void gzip_do_close(struct Curl_easy *data,
  302. struct Curl_cwriter *writer)
  303. {
  304. struct zlib_writer *zp = (struct zlib_writer *) writer;
  305. z_stream *z = &zp->z; /* zlib state structure */
  306. exit_zlib(data, z, &zp->zlib_init, CURLE_OK);
  307. }
  308. static const struct Curl_cwtype gzip_encoding = {
  309. "gzip",
  310. "x-gzip",
  311. gzip_do_init,
  312. gzip_do_write,
  313. gzip_do_close,
  314. sizeof(struct zlib_writer)
  315. };
  316. #endif /* HAVE_LIBZ */
  317. #ifdef HAVE_BROTLI
  318. /* Brotli writer. */
  319. struct brotli_writer {
  320. struct Curl_cwriter super;
  321. char buffer[DECOMPRESS_BUFFER_SIZE];
  322. BrotliDecoderState *br; /* State structure for brotli. */
  323. };
  324. static CURLcode brotli_map_error(BrotliDecoderErrorCode be)
  325. {
  326. switch(be) {
  327. case BROTLI_DECODER_ERROR_FORMAT_EXUBERANT_NIBBLE:
  328. case BROTLI_DECODER_ERROR_FORMAT_EXUBERANT_META_NIBBLE:
  329. case BROTLI_DECODER_ERROR_FORMAT_SIMPLE_HUFFMAN_ALPHABET:
  330. case BROTLI_DECODER_ERROR_FORMAT_SIMPLE_HUFFMAN_SAME:
  331. case BROTLI_DECODER_ERROR_FORMAT_CL_SPACE:
  332. case BROTLI_DECODER_ERROR_FORMAT_HUFFMAN_SPACE:
  333. case BROTLI_DECODER_ERROR_FORMAT_CONTEXT_MAP_REPEAT:
  334. case BROTLI_DECODER_ERROR_FORMAT_BLOCK_LENGTH_1:
  335. case BROTLI_DECODER_ERROR_FORMAT_BLOCK_LENGTH_2:
  336. case BROTLI_DECODER_ERROR_FORMAT_TRANSFORM:
  337. case BROTLI_DECODER_ERROR_FORMAT_DICTIONARY:
  338. case BROTLI_DECODER_ERROR_FORMAT_WINDOW_BITS:
  339. case BROTLI_DECODER_ERROR_FORMAT_PADDING_1:
  340. case BROTLI_DECODER_ERROR_FORMAT_PADDING_2:
  341. #ifdef BROTLI_DECODER_ERROR_COMPOUND_DICTIONARY
  342. case BROTLI_DECODER_ERROR_COMPOUND_DICTIONARY:
  343. #endif
  344. #ifdef BROTLI_DECODER_ERROR_DICTIONARY_NOT_SET
  345. case BROTLI_DECODER_ERROR_DICTIONARY_NOT_SET:
  346. #endif
  347. case BROTLI_DECODER_ERROR_INVALID_ARGUMENTS:
  348. return CURLE_BAD_CONTENT_ENCODING;
  349. case BROTLI_DECODER_ERROR_ALLOC_CONTEXT_MODES:
  350. case BROTLI_DECODER_ERROR_ALLOC_TREE_GROUPS:
  351. case BROTLI_DECODER_ERROR_ALLOC_CONTEXT_MAP:
  352. case BROTLI_DECODER_ERROR_ALLOC_RING_BUFFER_1:
  353. case BROTLI_DECODER_ERROR_ALLOC_RING_BUFFER_2:
  354. case BROTLI_DECODER_ERROR_ALLOC_BLOCK_TYPE_TREES:
  355. return CURLE_OUT_OF_MEMORY;
  356. default:
  357. break;
  358. }
  359. return CURLE_WRITE_ERROR;
  360. }
  361. static CURLcode brotli_do_init(struct Curl_easy *data,
  362. struct Curl_cwriter *writer)
  363. {
  364. struct brotli_writer *bp = (struct brotli_writer *) writer;
  365. (void) data;
  366. bp->br = BrotliDecoderCreateInstance(NULL, NULL, NULL);
  367. return bp->br ? CURLE_OK : CURLE_OUT_OF_MEMORY;
  368. }
  369. static CURLcode brotli_do_write(struct Curl_easy *data,
  370. struct Curl_cwriter *writer, int type,
  371. const char *buf, size_t nbytes)
  372. {
  373. struct brotli_writer *bp = (struct brotli_writer *) writer;
  374. const uint8_t *src = (const uint8_t *) buf;
  375. uint8_t *dst;
  376. size_t dstleft;
  377. CURLcode result = CURLE_OK;
  378. BrotliDecoderResult r = BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT;
  379. if(!(type & CLIENTWRITE_BODY) || !nbytes)
  380. return Curl_cwriter_write(data, writer->next, type, buf, nbytes);
  381. if(!bp->br)
  382. return CURLE_WRITE_ERROR; /* Stream already ended. */
  383. while((nbytes || r == BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT) &&
  384. result == CURLE_OK) {
  385. dst = (uint8_t *) bp->buffer;
  386. dstleft = DECOMPRESS_BUFFER_SIZE;
  387. r = BrotliDecoderDecompressStream(bp->br,
  388. &nbytes, &src, &dstleft, &dst, NULL);
  389. result = Curl_cwriter_write(data, writer->next, type,
  390. bp->buffer, DECOMPRESS_BUFFER_SIZE - dstleft);
  391. if(result)
  392. break;
  393. switch(r) {
  394. case BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT:
  395. case BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT:
  396. break;
  397. case BROTLI_DECODER_RESULT_SUCCESS:
  398. BrotliDecoderDestroyInstance(bp->br);
  399. bp->br = NULL;
  400. if(nbytes)
  401. result = CURLE_WRITE_ERROR;
  402. break;
  403. default:
  404. result = brotli_map_error(BrotliDecoderGetErrorCode(bp->br));
  405. break;
  406. }
  407. }
  408. return result;
  409. }
  410. static void brotli_do_close(struct Curl_easy *data,
  411. struct Curl_cwriter *writer)
  412. {
  413. struct brotli_writer *bp = (struct brotli_writer *) writer;
  414. (void) data;
  415. if(bp->br) {
  416. BrotliDecoderDestroyInstance(bp->br);
  417. bp->br = NULL;
  418. }
  419. }
  420. static const struct Curl_cwtype brotli_encoding = {
  421. "br",
  422. NULL,
  423. brotli_do_init,
  424. brotli_do_write,
  425. brotli_do_close,
  426. sizeof(struct brotli_writer)
  427. };
  428. #endif
  429. #ifdef HAVE_ZSTD
  430. /* Zstd writer. */
  431. struct zstd_writer {
  432. struct Curl_cwriter super;
  433. ZSTD_DStream *zds; /* State structure for zstd. */
  434. char buffer[DECOMPRESS_BUFFER_SIZE];
  435. };
  436. #ifdef ZSTD_STATIC_LINKING_ONLY
  437. static void *Curl_zstd_alloc(void *opaque, size_t size)
  438. {
  439. (void)opaque;
  440. return Curl_cmalloc(size);
  441. }
  442. static void Curl_zstd_free(void *opaque, void *address)
  443. {
  444. (void)opaque;
  445. Curl_cfree(address);
  446. }
  447. #endif
  448. static CURLcode zstd_do_init(struct Curl_easy *data,
  449. struct Curl_cwriter *writer)
  450. {
  451. struct zstd_writer *zp = (struct zstd_writer *) writer;
  452. (void)data;
  453. #ifdef ZSTD_STATIC_LINKING_ONLY
  454. zp->zds = ZSTD_createDStream_advanced((ZSTD_customMem) {
  455. .customAlloc = Curl_zstd_alloc,
  456. .customFree = Curl_zstd_free,
  457. .opaque = NULL
  458. });
  459. #else
  460. zp->zds = ZSTD_createDStream();
  461. #endif
  462. return zp->zds ? CURLE_OK : CURLE_OUT_OF_MEMORY;
  463. }
  464. static CURLcode zstd_do_write(struct Curl_easy *data,
  465. struct Curl_cwriter *writer, int type,
  466. const char *buf, size_t nbytes)
  467. {
  468. CURLcode result = CURLE_OK;
  469. struct zstd_writer *zp = (struct zstd_writer *) writer;
  470. ZSTD_inBuffer in;
  471. ZSTD_outBuffer out;
  472. size_t errorCode;
  473. if(!(type & CLIENTWRITE_BODY) || !nbytes)
  474. return Curl_cwriter_write(data, writer->next, type, buf, nbytes);
  475. in.pos = 0;
  476. in.src = buf;
  477. in.size = nbytes;
  478. for(;;) {
  479. out.pos = 0;
  480. out.dst = zp->buffer;
  481. out.size = DECOMPRESS_BUFFER_SIZE;
  482. errorCode = ZSTD_decompressStream(zp->zds, &out, &in);
  483. if(ZSTD_isError(errorCode)) {
  484. return CURLE_BAD_CONTENT_ENCODING;
  485. }
  486. if(out.pos > 0) {
  487. result = Curl_cwriter_write(data, writer->next, type,
  488. zp->buffer, out.pos);
  489. if(result)
  490. break;
  491. }
  492. if((in.pos == nbytes) && (out.pos < out.size))
  493. break;
  494. }
  495. return result;
  496. }
  497. static void zstd_do_close(struct Curl_easy *data,
  498. struct Curl_cwriter *writer)
  499. {
  500. struct zstd_writer *zp = (struct zstd_writer *) writer;
  501. (void)data;
  502. if(zp->zds) {
  503. ZSTD_freeDStream(zp->zds);
  504. zp->zds = NULL;
  505. }
  506. }
  507. static const struct Curl_cwtype zstd_encoding = {
  508. "zstd",
  509. NULL,
  510. zstd_do_init,
  511. zstd_do_write,
  512. zstd_do_close,
  513. sizeof(struct zstd_writer)
  514. };
  515. #endif
  516. /* Identity handler. */
  517. static const struct Curl_cwtype identity_encoding = {
  518. "identity",
  519. "none",
  520. Curl_cwriter_def_init,
  521. Curl_cwriter_def_write,
  522. Curl_cwriter_def_close,
  523. sizeof(struct Curl_cwriter)
  524. };
  525. /* supported general content decoders. */
  526. static const struct Curl_cwtype * const general_unencoders[] = {
  527. &identity_encoding,
  528. #ifdef HAVE_LIBZ
  529. &deflate_encoding,
  530. &gzip_encoding,
  531. #endif
  532. #ifdef HAVE_BROTLI
  533. &brotli_encoding,
  534. #endif
  535. #ifdef HAVE_ZSTD
  536. &zstd_encoding,
  537. #endif
  538. NULL
  539. };
  540. /* supported content decoders only for transfer encodings */
  541. static const struct Curl_cwtype * const transfer_unencoders[] = {
  542. #ifndef CURL_DISABLE_HTTP
  543. &Curl_httpchunk_unencoder,
  544. #endif
  545. NULL
  546. };
  547. /* Provide a list of comma-separated names of supported encodings.
  548. */
  549. void Curl_all_content_encodings(char *buf, size_t blen)
  550. {
  551. size_t len = 0;
  552. const struct Curl_cwtype * const *cep;
  553. const struct Curl_cwtype *ce;
  554. DEBUGASSERT(buf);
  555. DEBUGASSERT(blen);
  556. buf[0] = 0;
  557. for(cep = general_unencoders; *cep; cep++) {
  558. ce = *cep;
  559. if(!strcasecompare(ce->name, CONTENT_ENCODING_DEFAULT))
  560. len += strlen(ce->name) + 2;
  561. }
  562. if(!len) {
  563. if(blen >= sizeof(CONTENT_ENCODING_DEFAULT))
  564. strcpy(buf, CONTENT_ENCODING_DEFAULT);
  565. }
  566. else if(blen > len) {
  567. char *p = buf;
  568. for(cep = general_unencoders; *cep; cep++) {
  569. ce = *cep;
  570. if(!strcasecompare(ce->name, CONTENT_ENCODING_DEFAULT)) {
  571. strcpy(p, ce->name);
  572. p += strlen(p);
  573. *p++ = ',';
  574. *p++ = ' ';
  575. }
  576. }
  577. p[-2] = '\0';
  578. }
  579. }
  580. /* Deferred error dummy writer. */
  581. static CURLcode error_do_init(struct Curl_easy *data,
  582. struct Curl_cwriter *writer)
  583. {
  584. (void)data;
  585. (void)writer;
  586. return CURLE_OK;
  587. }
  588. static CURLcode error_do_write(struct Curl_easy *data,
  589. struct Curl_cwriter *writer, int type,
  590. const char *buf, size_t nbytes)
  591. {
  592. (void) writer;
  593. (void) buf;
  594. (void) nbytes;
  595. if(!(type & CLIENTWRITE_BODY) || !nbytes)
  596. return Curl_cwriter_write(data, writer->next, type, buf, nbytes);
  597. else {
  598. char all[256];
  599. (void)Curl_all_content_encodings(all, sizeof(all));
  600. failf(data, "Unrecognized content encoding type. "
  601. "libcurl understands %s content encodings.", all);
  602. }
  603. return CURLE_BAD_CONTENT_ENCODING;
  604. }
  605. static void error_do_close(struct Curl_easy *data,
  606. struct Curl_cwriter *writer)
  607. {
  608. (void) data;
  609. (void) writer;
  610. }
  611. static const struct Curl_cwtype error_writer = {
  612. "ce-error",
  613. NULL,
  614. error_do_init,
  615. error_do_write,
  616. error_do_close,
  617. sizeof(struct Curl_cwriter)
  618. };
  619. /* Find the content encoding by name. */
  620. static const struct Curl_cwtype *find_unencode_writer(const char *name,
  621. size_t len,
  622. Curl_cwriter_phase phase)
  623. {
  624. const struct Curl_cwtype * const *cep;
  625. if(phase == CURL_CW_TRANSFER_DECODE) {
  626. for(cep = transfer_unencoders; *cep; cep++) {
  627. const struct Curl_cwtype *ce = *cep;
  628. if((strncasecompare(name, ce->name, len) && !ce->name[len]) ||
  629. (ce->alias && strncasecompare(name, ce->alias, len)
  630. && !ce->alias[len]))
  631. return ce;
  632. }
  633. }
  634. /* look among the general decoders */
  635. for(cep = general_unencoders; *cep; cep++) {
  636. const struct Curl_cwtype *ce = *cep;
  637. if((strncasecompare(name, ce->name, len) && !ce->name[len]) ||
  638. (ce->alias && strncasecompare(name, ce->alias, len) && !ce->alias[len]))
  639. return ce;
  640. }
  641. return NULL;
  642. }
  643. /* Setup the unencoding stack from the Content-Encoding header value.
  644. * See RFC 7231 section 3.1.2.2. */
  645. CURLcode Curl_build_unencoding_stack(struct Curl_easy *data,
  646. const char *enclist, int is_transfer)
  647. {
  648. Curl_cwriter_phase phase = is_transfer ?
  649. CURL_CW_TRANSFER_DECODE : CURL_CW_CONTENT_DECODE;
  650. CURLcode result;
  651. do {
  652. const char *name;
  653. size_t namelen;
  654. bool is_chunked = FALSE;
  655. /* Parse a single encoding name. */
  656. while(ISBLANK(*enclist) || *enclist == ',')
  657. enclist++;
  658. name = enclist;
  659. for(namelen = 0; *enclist && *enclist != ','; enclist++)
  660. if(!ISSPACE(*enclist))
  661. namelen = enclist - name + 1;
  662. if(namelen) {
  663. const struct Curl_cwtype *cwt;
  664. struct Curl_cwriter *writer;
  665. CURL_TRC_WRITE(data, "looking for %s decoder: %.*s",
  666. is_transfer ? "transfer" : "content", (int)namelen, name);
  667. is_chunked = (is_transfer && (namelen == 7) &&
  668. strncasecompare(name, "chunked", 7));
  669. /* if we skip the decoding in this phase, do not look further.
  670. * Exception is "chunked" transfer-encoding which always must happen */
  671. if((is_transfer && !data->set.http_transfer_encoding && !is_chunked) ||
  672. (!is_transfer && data->set.http_ce_skip)) {
  673. /* not requested, ignore */
  674. CURL_TRC_WRITE(data, "decoder not requested, ignored: %.*s",
  675. (int)namelen, name);
  676. return CURLE_OK;
  677. }
  678. if(Curl_cwriter_count(data, phase) + 1 >= MAX_ENCODE_STACK) {
  679. failf(data, "Reject response due to more than %u content encodings",
  680. MAX_ENCODE_STACK);
  681. return CURLE_BAD_CONTENT_ENCODING;
  682. }
  683. cwt = find_unencode_writer(name, namelen, phase);
  684. if(cwt && is_chunked && Curl_cwriter_get_by_type(data, cwt)) {
  685. /* A 'chunked' transfer encoding has already been added.
  686. * Ignore duplicates. See #13451.
  687. * Also RFC 9112, ch. 6.1:
  688. * "A sender MUST NOT apply the chunked transfer coding more than
  689. * once to a message body."
  690. */
  691. CURL_TRC_WRITE(data, "ignoring duplicate 'chunked' decoder");
  692. return CURLE_OK;
  693. }
  694. if(is_transfer && !is_chunked &&
  695. Curl_cwriter_get_by_name(data, "chunked")) {
  696. /* RFC 9112, ch. 6.1:
  697. * "If any transfer coding other than chunked is applied to a
  698. * response's content, the sender MUST either apply chunked as the
  699. * final transfer coding or terminate the message by closing the
  700. * connection."
  701. * "chunked" must be the last added to be the first in its phase,
  702. * reject this.
  703. */
  704. failf(data, "Reject response due to 'chunked' not being the last "
  705. "Transfer-Encoding");
  706. return CURLE_BAD_CONTENT_ENCODING;
  707. }
  708. if(!cwt)
  709. cwt = &error_writer; /* Defer error at use. */
  710. result = Curl_cwriter_create(&writer, data, cwt, phase);
  711. CURL_TRC_WRITE(data, "added %s decoder %s -> %d",
  712. is_transfer ? "transfer" : "content", cwt->name, result);
  713. if(result)
  714. return result;
  715. result = Curl_cwriter_add(data, writer);
  716. if(result) {
  717. Curl_cwriter_free(data, writer);
  718. return result;
  719. }
  720. }
  721. } while(*enclist);
  722. return CURLE_OK;
  723. }
  724. #else
  725. /* Stubs for builds without HTTP. */
  726. CURLcode Curl_build_unencoding_stack(struct Curl_easy *data,
  727. const char *enclist, int is_transfer)
  728. {
  729. (void) data;
  730. (void) enclist;
  731. (void) is_transfer;
  732. return CURLE_NOT_BUILT_IN;
  733. }
  734. void Curl_all_content_encodings(char *buf, size_t blen)
  735. {
  736. DEBUGASSERT(buf);
  737. DEBUGASSERT(blen);
  738. if(blen < sizeof(CONTENT_ENCODING_DEFAULT))
  739. buf[0] = 0;
  740. else
  741. strcpy(buf, CONTENT_ENCODING_DEFAULT);
  742. }
  743. #endif /* CURL_DISABLE_HTTP */