Merge branch 'thirdparty'

Source commit: 340ee263503e937c1454d0421b246ff2696c41d7

libs/expat/CMake.README

@@ -3,25 +3,25 @@
 The cmake based buildsystem for expat works on Windows (cygwin, mingw, Visual
 Studio) and should work on all other platform cmake supports.
 
-Assuming ~/expat-2.6.1 is the source directory of expat, add a subdirectory
+Assuming ~/expat-2.6.2 is the source directory of expat, add a subdirectory
 build and change into that directory:
-~/expat-2.6.1$ mkdir build && cd build
-~/expat-2.6.1/build$
+~/expat-2.6.2$ mkdir build && cd build
+~/expat-2.6.2/build$
 
 From that directory, call cmake first, then call make, make test and
 make install in the usual way:
-~/expat-2.6.1/build$ cmake ..
+~/expat-2.6.2/build$ cmake ..
 -- The C compiler identification is GNU
 -- The CXX compiler identification is GNU
 ....
 -- Configuring done
 -- Generating done
--- Build files have been written to: /home/patrick/expat-2.6.1/build
+-- Build files have been written to: /home/patrick/expat-2.6.2/build
 
 If you want to specify the install location for your files, append
 -DCMAKE_INSTALL_PREFIX=/your/install/path to the cmake call.
 
-~/expat-2.6.1/build$ make && make test && make install
+~/expat-2.6.2/build$ make && make test && make install
 Scanning dependencies of target expat
 [  5%] Building C object CMakeFiles/expat.dir/lib/xmlparse.c.o
 [ 11%] Building C object CMakeFiles/expat.dir/lib/xmlrole.c.o

libs/expat/CMakeLists.txt

@@ -38,7 +38,7 @@ cmake_minimum_required(VERSION 3.5.0)
 
 project(expat
     VERSION
-        2.6.1
+        2.6.2
     LANGUAGES
         C
 )
@@ -466,7 +466,7 @@ foreach(build_type_upper
 endforeach()
 
 set(LIBCURRENT 10)  # sync
-set(LIBREVISION 1)  # with
+set(LIBREVISION 2)  # with
 set(LIBAGE 9)       # configure.ac!
 math(EXPR LIBCURRENT_MINUS_AGE "${LIBCURRENT} - ${LIBAGE}")
 

libs/expat/Changes

@@ -1,6 +1,59 @@
-NOTE: We are looking for help with a few things:
-      https://github.com/libexpat/libexpat/labels/help%20wanted
-      If you can help, please get in touch.  Thanks!
+                           __  __            _
+                        ___\ \/ /_ __   __ _| |_
+                       / _ \\  /| '_ \ / _` | __|
+                      |  __//  \| |_) | (_| | |_
+                       \___/_/\_\ .__/ \__,_|\__|
+                                |_| XML parser
+
+!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+!! <blink>Expat is UNDERSTAFFED and WITHOUT FUNDING.</blink>                 !!
+!!                 ~~~~~~~~~~~~                                              !!
+!! The following topics need *additional skilled C developers* to progress   !!
+!! in a timely manner or at all (loosely ordered by descending priority):    !!
+!!                                                                           !!
+!! - <blink>fixing a complex non-public security issue</blink>,              !!
+!! - teaming up on researching and fixing future security reports and        !!
+!!   ClusterFuzz findings with few-days-max response times in communication  !!
+!!   in order to (1) have a sound fix ready before the end of a 90 days      !!
+!!   grace period and (2) in a sustainable manner,                           !!
+!! - implementing and auto-testing XML 1.0r5 support                         !!
+!!   (needs discussion before pull requests),                                !!
+!! - smart ideas on fixing the Autotools CMake files generation issue        !!
+!!   without breaking CI (needs discussion before pull requests),            !!
+!! - the Windows binaries topic (needs requirements engineering first),      !!
+!! - pushing migration from `int` to `size_t` further                        !!
+!!   including edge-cases test coverage (needs discussion before anything).  !!
+!!                                                                           !!
+!! For details, please reach out via e-mail to [email protected] so we   !!
+!! can schedule a voice call on the topic, in English or German.             !!
+!!                                                                           !!
+!! THANK YOU!                        Sebastian Pipping -- Berlin, 2024-03-09 !!
+!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+
+Release 2.6.2 Wed March 13 2024
+        Security fixes:
+       #839 #842  CVE-2024-28757 -- Prevent billion laughs attacks with
+                    isolated use of external parsers.  Please see the commit
+                    message of commit 1d50b80cf31de87750103656f6eb693746854aa8
+                    for details.
+
+        Bug fixes:
+       #839 #841  Reject direct parameter entity recursion
+                    and avoid the related undefined behavior
+
+        Other changes:
+            #847  Autotools: Fix build for DOCBOOK_TO_MAN containing spaces
+            #837  Add missing #821 and #824 to 2.6.1 change log
+       #838 #843  Version info bumped from 10:1:9 (libexpat*.so.1.9.1)
+                    to 10:2:9 (libexpat*.so.1.9.2); see https://verbump.de/
+                    for what these numbers do
+
+        Special thanks to:
+            Philippe Antoine
+            Tomas Korbar
+                 and
+            Clang UndefinedBehaviorSanitizer
+            OSS-Fuzz / ClusterFuzz
 
 Release 2.6.1 Thu February 29 2024
         Bug fixes:
@@ -11,6 +64,8 @@ Release 2.6.1 Thu February 29 2024
         Other changes:
             #829  Hide test-only code behind new internal macro
             #833  Autotools: Reject expat_config.h.in defining SIZEOF_VOID_P
+       #821 #824  Autotools: Fix "make clean" for case:
+                    ./configure --without-docbook && make clean all
             #819  Address compiler warnings
        #832 #834  Version info bumped from 10:0:9 (libexpat*.so.1.9.0)
                     to 10:1:9 (libexpat*.so.1.9.1); see https://verbump.de/

libs/expat/README.md

@@ -5,7 +5,7 @@
 [![Downloads GitHub](https://img.shields.io/github/downloads/libexpat/libexpat/total?label=Downloads%20GitHub)](https://github.com/libexpat/libexpat/releases)
 
 
-# Expat, Release 2.6.1
+# Expat, Release 2.6.2
 
 This is Expat, a C99 library for parsing
 [XML 1.0 Fourth Edition](https://www.w3.org/TR/2006/REC-xml-20060816/), started by

libs/expat/configure

@@ -1,6 +1,6 @@
 #! /bin/sh
 # Guess values for system-dependent variables and create Makefiles.
-# Generated by GNU Autoconf 2.71 for expat 2.6.1.
+# Generated by GNU Autoconf 2.71 for expat 2.6.2.
 #
 # Report bugs to <https://github.com/libexpat/libexpat/issues>.
 #
@@ -621,8 +621,8 @@ MAKEFLAGS=
 # Identity of this package.
 PACKAGE_NAME='expat'
 PACKAGE_TARNAME='expat'
-PACKAGE_VERSION='2.6.1'
-PACKAGE_STRING='expat 2.6.1'
+PACKAGE_VERSION='2.6.2'
+PACKAGE_STRING='expat 2.6.2'
 PACKAGE_BUGREPORT='https://github.com/libexpat/libexpat/issues'
 PACKAGE_URL=''
 
@@ -1424,7 +1424,7 @@ if test "$ac_init_help" = "long"; then
   # Omit some internal or obsolete options to make the list less imposing.
   # This message is too long to be a string in the A/UX 3.1 sh.
   cat <<_ACEOF
-\`configure' configures expat 2.6.1 to adapt to many kinds of systems.
+\`configure' configures expat 2.6.2 to adapt to many kinds of systems.
 
 Usage: $0 [OPTION]... [VAR=VALUE]...
 
@@ -1495,7 +1495,7 @@ fi
 
 if test -n "$ac_init_help"; then
   case $ac_init_help in
-     short | recursive ) echo "Configuration of expat 2.6.1:";;
+     short | recursive ) echo "Configuration of expat 2.6.2:";;
    esac
   cat <<\_ACEOF
 
@@ -1632,7 +1632,7 @@ fi
 test -n "$ac_init_help" && exit $ac_status
 if $ac_init_version; then
   cat <<\_ACEOF
-expat configure 2.6.1
+expat configure 2.6.2
 generated by GNU Autoconf 2.71
 
 Copyright (C) 2021 Free Software Foundation, Inc.
@@ -2263,7 +2263,7 @@ cat >config.log <<_ACEOF
 This file contains any messages produced by compilers while
 running configure, to aid debugging if configure makes a mistake.
 
-It was created by expat $as_me 2.6.1, which was
+It was created by expat $as_me 2.6.2, which was
 generated by GNU Autoconf 2.71.  Invocation command line was
 
   $ $0$ac_configure_args_raw
@@ -3829,7 +3829,7 @@ fi
 
 # Define the identity of the package.
  PACKAGE='expat'
- VERSION='2.6.1'
+ VERSION='2.6.2'
 
 
 printf "%s\n" "#define PACKAGE \"$PACKAGE\"" >>confdefs.h
@@ -3960,7 +3960,7 @@ fi
 
 
 LIBCURRENT=10  # sync
-LIBREVISION=1  # with
+LIBREVISION=2  # with
 LIBAGE=9       # CMakeLists.txt!
 
 ac_config_headers="$ac_config_headers expat_config.h"
@@ -20977,7 +20977,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
 # report actual input values of CONFIG_FILES etc. instead of their
 # values after options handling.
 ac_log="
-This file was extended by expat $as_me 2.6.1, which was
+This file was extended by expat $as_me 2.6.2, which was
 generated by GNU Autoconf 2.71.  Invocation command line was
 
   CONFIG_FILES    = $CONFIG_FILES
@@ -21045,7 +21045,7 @@ ac_cs_config_escaped=`printf "%s\n" "$ac_cs_config" | sed "s/^ //; s/'/'\\\\\\\\
 cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
 ac_cs_config='$ac_cs_config_escaped'
 ac_cs_version="\\
-expat config.status 2.6.1
+expat config.status 2.6.2
 configured by $0, generated by GNU Autoconf 2.71,
   with options \\"\$ac_cs_config\\"
 

libs/expat/configure.ac

@@ -83,7 +83,7 @@ dnl If the API changes incompatibly set LIBAGE back to 0
 dnl
 
 LIBCURRENT=10  # sync
-LIBREVISION=1  # with
+LIBREVISION=2  # with
 LIBAGE=9       # CMakeLists.txt!
 
 AC_CONFIG_HEADERS([expat_config.h])

libs/expat/doc/Makefile.am

@@ -37,7 +37,7 @@ dist_man_MANS = xmlwf.1
 
 xmlwf.1: xmlwf.xml
 	-rm -f $@
-	test x$(DOCBOOK_TO_MAN) != x && $(DOCBOOK_TO_MAN) $<
+	test "x$(DOCBOOK_TO_MAN)" != x && $(DOCBOOK_TO_MAN) $<
 	test -f $@ || mv XMLWF.1 $@
 endif
 

libs/expat/doc/Makefile.in

@@ -605,7 +605,7 @@ uninstall-man: uninstall-man1
 
 @[email protected]: xmlwf.xml
 @WITH_MANPAGE_TRUE@	-rm -f $@
-@WITH_MANPAGE_TRUE@	test x$(DOCBOOK_TO_MAN) != x && $(DOCBOOK_TO_MAN) $<
+@WITH_MANPAGE_TRUE@	test "x$(DOCBOOK_TO_MAN)" != x && $(DOCBOOK_TO_MAN) $<
 @WITH_MANPAGE_TRUE@	test -f $@ || mv XMLWF.1 $@
 
 @WITH_DISTRIBUTABLE_MANPAGE_FALSE@dist-hook:

libs/expat/doc/reference.html

@@ -52,7 +52,7 @@
   <div>
     <h1>
       The Expat XML Parser
-      <small>Release 2.6.1</small>
+      <small>Release 2.6.2</small>
     </h1>
   </div>
 <div class="content">

libs/expat/doc/xmlwf.1

@@ -5,7 +5,7 @@
 \\$2 \(la\\$1\(ra\\$3
 ..
 .if \n(.g .mso www.tmac
-.TH XMLWF 1 "February 29, 2024" "" ""
+.TH XMLWF 1 "March 13, 2024" "" ""
 .SH NAME
 xmlwf \- Determines if an XML document is well-formed
 .SH SYNOPSIS

libs/expat/doc/xmlwf.xml

@@ -21,7 +21,7 @@
           "http://www.oasis-open.org/docbook/xml/4.2/docbookx.dtd" [
   <!ENTITY dhfirstname "<firstname>Scott</firstname>">
   <!ENTITY dhsurname   "<surname>Bronson</surname>">
-  <!ENTITY dhdate      "<date>February 29, 2024</date>">
+  <!ENTITY dhdate      "<date>March 13, 2024</date>">
   <!-- Please adjust this^^ date whenever cutting a new release. -->
   <!ENTITY dhsection   "<manvolnum>1</manvolnum>">
   <!ENTITY dhemail     "<email>[email protected]</email>">

libs/expat/expat_config.h

@@ -84,7 +84,7 @@
 #define PACKAGE_NAME "expat"
 
 /* Define to the full name and version of this package. */
-#define PACKAGE_STRING "expat 2.6.1"
+#define PACKAGE_STRING "expat 2.6.2"
 
 /* Define to the one symbol short name of this package. */
 #define PACKAGE_TARNAME "expat"
@@ -93,7 +93,7 @@
 #define PACKAGE_URL ""
 
 /* Define to the version of this package. */
-#define PACKAGE_VERSION "2.6.1"
+#define PACKAGE_VERSION "2.6.2"
 
 /* Define to 1 if all of the C90 standard headers exist (not just the ones
    required in a freestanding environment). This macro is provided for
@@ -101,7 +101,7 @@
 #define STDC_HEADERS 1
 
 /* Version number of package */
-#define VERSION "2.6.1"
+#define VERSION "2.6.2"
 
 /* Define WORDS_BIGENDIAN to 1 if your processor stores words with the most
    significant byte first (like Motorola and SPARC, unlike Intel). */

libs/expat/lib/expat.h

@@ -1066,7 +1066,7 @@ XML_SetReparseDeferralEnabled(XML_Parser parser, XML_Bool enabled);
 */
 #define XML_MAJOR_VERSION 2
 #define XML_MINOR_VERSION 6
-#define XML_MICRO_VERSION 1
+#define XML_MICRO_VERSION 2
 
 #ifdef __cplusplus
 }

libs/expat/lib/xmlparse.c

@@ -1,4 +1,4 @@
-/* dd2a9703e301882afe16d198a82689ab225277057f5eab9d079d8606eab736b4 (2.6.1+)
+/* 2a14271ad4d35e82bde8ba210b4edb7998794bcbae54deab114046a300f9639a (2.6.2+)
                             __  __            _
                          ___\ \/ /_ __   __ _| |_
                         / _ \\  /| '_ \ / _` | __|
@@ -6266,7 +6266,7 @@ storeEntityValue(XML_Parser parser, const ENCODING *enc,
           dtd->keepProcessing = dtd->standalone;
           goto endEntityValue;
         }
-        if (entity->open) {
+        if (entity->open || (entity == parser->m_declEntity)) {
           if (enc == parser->m_encoding)
             parser->m_eventPtr = entityTextPtr;
           result = XML_ERROR_RECURSIVE_ENTITY_REF;
@@ -7825,6 +7825,8 @@ copyString(const XML_Char *s, const XML_Memory_Handling_Suite *memsuite) {
 
 static float
 accountingGetCurrentAmplification(XML_Parser rootParser) {
+  //                                          1.........1.........12 => 22
+  const size_t lenOfShortestInclude = sizeof("<!ENTITY a SYSTEM 'b'>") - 1;
   const XmlBigCount countBytesOutput
       = rootParser->m_accounting.countBytesDirect
         + rootParser->m_accounting.countBytesIndirect;
@@ -7832,7 +7834,9 @@ accountingGetCurrentAmplification(XML_Parser rootParser) {
       = rootParser->m_accounting.countBytesDirect
             ? (countBytesOutput
                / (float)(rootParser->m_accounting.countBytesDirect))
-            : 1.0f;
+            : ((lenOfShortestInclude
+                + rootParser->m_accounting.countBytesIndirect)
+               / (float)lenOfShortestInclude);
   assert(! rootParser->m_parentParser);
   return amplificationFactor;
 }

libs/expat/tests/acc_tests.c

@@ -378,6 +378,63 @@ START_TEST(test_helper_unsigned_char_to_printable) {
     fail("unsignedCharToPrintable result mistaken");
 }
 END_TEST
+
+START_TEST(test_amplification_isolated_external_parser) {
+  // NOTE: Length 44 is precisely twice the length of "<!ENTITY a SYSTEM 'b'>"
+  // (22) that is used in function accountingGetCurrentAmplification in
+  // xmlparse.c.
+  //                  1.........1.........1.........1.........1..4 => 44
+  const char doc[] = "<!ENTITY % p1 '123456789_123456789_1234567'>";
+  const int docLen = (int)sizeof(doc) - 1;
+  const float maximumToleratedAmplification = 2.0f;
+
+  struct TestCase {
+    int offsetOfThreshold;
+    enum XML_Status expectedStatus;
+  };
+
+  struct TestCase cases[] = {
+      {-2, XML_STATUS_ERROR}, {-1, XML_STATUS_ERROR}, {0, XML_STATUS_ERROR},
+      {+1, XML_STATUS_OK},    {+2, XML_STATUS_OK},
+  };
+
+  for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) {
+    const int offsetOfThreshold = cases[i].offsetOfThreshold;
+    const enum XML_Status expectedStatus = cases[i].expectedStatus;
+    const unsigned long long activationThresholdBytes
+        = docLen + offsetOfThreshold;
+
+    set_subtest("offsetOfThreshold=%d, expectedStatus=%d", offsetOfThreshold,
+                expectedStatus);
+
+    XML_Parser parser = XML_ParserCreate(NULL);
+    assert_true(parser != NULL);
+
+    assert_true(XML_SetBillionLaughsAttackProtectionMaximumAmplification(
+                    parser, maximumToleratedAmplification)
+                == XML_TRUE);
+    assert_true(XML_SetBillionLaughsAttackProtectionActivationThreshold(
+                    parser, activationThresholdBytes)
+                == XML_TRUE);
+
+    XML_Parser ext_parser = XML_ExternalEntityParserCreate(parser, NULL, NULL);
+    assert_true(ext_parser != NULL);
+
+    const enum XML_Status actualStatus
+        = _XML_Parse_SINGLE_BYTES(ext_parser, doc, docLen, XML_TRUE);
+
+    assert_true(actualStatus == expectedStatus);
+    if (actualStatus != XML_STATUS_OK) {
+      assert_true(XML_GetErrorCode(ext_parser)
+                  == XML_ERROR_AMPLIFICATION_LIMIT_BREACH);
+    }
+
+    XML_ParserFree(ext_parser);
+    XML_ParserFree(parser);
+  }
+}
+END_TEST
+
 #endif // XML_GE == 1
 
 void
@@ -390,6 +447,8 @@ make_accounting_test_case(Suite *s) {
   tcase_add_test(tc_accounting, test_accounting_precision);
   tcase_add_test(tc_accounting, test_billion_laughs_attack_protection_api);
   tcase_add_test(tc_accounting, test_helper_unsigned_char_to_printable);
+  tcase_add_test__ifdef_xml_dtd(tc_accounting,
+                                test_amplification_isolated_external_parser);
 #else
   UNUSED_P(s);
 #endif /* XML_GE == 1 */

libs/expat/tests/basic_tests.c

@@ -1202,6 +1202,49 @@ START_TEST(test_wfc_no_recursive_entity_refs) {
 }
 END_TEST
 
+START_TEST(test_recursive_external_parameter_entity_2) {
+  struct TestCase {
+    const char *doc;
+    enum XML_Status expectedStatus;
+  };
+
+  struct TestCase cases[] = {
+      {"<!ENTITY % p1 '%p1;'>", XML_STATUS_ERROR},
+      {"<!ENTITY % p1 '%p1;'>"
+       "<!ENTITY % p1 'first declaration wins'>",
+       XML_STATUS_ERROR},
+      {"<!ENTITY % p1 'first declaration wins'>"
+       "<!ENTITY % p1 '%p1;'>",
+       XML_STATUS_OK},
+      {"<!ENTITY % p1 '&#37;p1;'>", XML_STATUS_OK},
+  };
+
+  for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) {
+    const char *const doc = cases[i].doc;
+    const enum XML_Status expectedStatus = cases[i].expectedStatus;
+    set_subtest("%s", doc);
+
+    XML_Parser parser = XML_ParserCreate(NULL);
+    assert_true(parser != NULL);
+
+    XML_Parser ext_parser = XML_ExternalEntityParserCreate(parser, NULL, NULL);
+    assert_true(ext_parser != NULL);
+
+    const enum XML_Status actualStatus
+        = _XML_Parse_SINGLE_BYTES(ext_parser, doc, (int)strlen(doc), XML_TRUE);
+
+    assert_true(actualStatus == expectedStatus);
+    if (actualStatus != XML_STATUS_OK) {
+      assert_true(XML_GetErrorCode(ext_parser)
+                  == XML_ERROR_RECURSIVE_ENTITY_REF);
+    }
+
+    XML_ParserFree(ext_parser);
+    XML_ParserFree(parser);
+  }
+}
+END_TEST
+
 /* Test incomplete external entities are faulted */
 START_TEST(test_ext_entity_invalid_parse) {
   const char *text = "<!DOCTYPE doc [\n"
@@ -5944,6 +5987,8 @@ make_basic_test_case(Suite *s) {
   tcase_add_test__ifdef_xml_dtd(tc_basic, test_skipped_parameter_entity);
   tcase_add_test__ifdef_xml_dtd(tc_basic,
                                 test_recursive_external_parameter_entity);
+  tcase_add_test__ifdef_xml_dtd(tc_basic,
+                                test_recursive_external_parameter_entity_2);
   tcase_add_test(tc_basic, test_undefined_ext_entity_in_external_dtd);
   tcase_add_test(tc_basic, test_suspend_xdecl);
   tcase_add_test(tc_basic, test_abort_epilog);

libs/expat/tests/misc_tests.c

@@ -208,7 +208,7 @@ START_TEST(test_misc_version) {
   if (! versions_equal(&read_version, &parsed_version))
     fail("Version mismatch");
 
-  if (xcstrcmp(version_text, XCS("expat_2.6.1"))) /* needs bump on releases */
+  if (xcstrcmp(version_text, XCS("expat_2.6.2"))) /* needs bump on releases */
     fail("XML_*_VERSION in expat.h out of sync?\n");
 }
 END_TEST

libs/expat/win32/expat.iss

@@ -37,7 +37,7 @@
 ; OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
 ; USE OR OTHER DEALINGS IN THE SOFTWARE.
 
-#define expatVer "2.6.1"
+#define expatVer "2.6.2"
 
 [Setup]
 AppName=Expat