Merge topic 'execute_process-encoding'

e782811cfe execute_process: Change default ENCODING to UTF-8
9a4533405b cmExecuteProcessCommand: Explicitly ignore unknown ENCODING values

Acked-by: Kitware Robot <[email protected]>
Acked-by: buildbot <[email protected]>
Merge-request: !9822

Help/command/execute_process.rst

@@ -147,7 +147,7 @@ Options:
    Use the current active console's codepage or if that isn't
    available then use ANSI.
 
-   This is the default since CMake 3.15.
+   This was the default in CMake 3.15 through 3.30.
 
  ``ANSI``
    Use the ANSI codepage.
@@ -160,6 +160,8 @@ Options:
 
    Use the UTF-8 codepage.
 
+   This is the default.  See policy :policy:`CMP0176`.
+
  ``UTF8``
    Use the UTF-8 codepage.  Use of this name is discouraged in favor
    of ``UTF-8`` to match the `UTF-8 RFC <https://www.ietf.org/rfc/rfc3629>`_

Help/manual/cmake-policies.7.rst

@@ -57,6 +57,7 @@ Policies Introduced by CMake 3.31
 .. toctree::
    :maxdepth: 1
 
+   CMP0176: execute_process() ENCODING is UTF-8 by default. </policy/CMP0176>
    CMP0175: add_custom_command() rejects invalid arguments. </policy/CMP0175>
    CMP0174: cmake_parse_arguments(PARSE_ARGV) defines a variable for an empty string after a single-value keyword. </policy/CMP0174>
    CMP0173: The CMakeFindFrameworks module is removed. </policy/CMP0173>

Help/policy/CMP0176.rst

@@ -0,0 +1,27 @@
+CMP0176
+-------
+
+.. versionadded:: 3.31
+
+:command:`execute_process` ``ENCODING`` is ``UTF-8`` by default.
+
+The ``ENCODING`` option is meaningful only on Windows.  It specifies the
+character encoding expected in the process's output on stdout and stderr.
+In CMake 3.14 and below the default encoding was ``NONE``, which corresponds
+to CMake's internal UTF-8 encoding.  In CMake 3.15 through CMake 3.30 the
+default encoding was accidentally changed to ``AUTO``, but the change went
+unnoticed and was not documented.
+
+CMake 3.31 and above prefer the ``ENCODING`` default to be ``UTF-8``.
+This policy provides compatibility with projects that may have been
+relying on the default being ``AUTO``.
+
+The ``OLD`` behavior of this policy is for :command:`execute_process`
+to use ``AUTO`` by default if no ``ENCODING`` is specified.  The ``NEW``
+behavior for this policy is to use ``UTF-8`` as the default ``ENCODING``.
+
+.. |INTRODUCED_IN_CMAKE_VERSION| replace:: 3.31
+.. |WARNS_OR_DOES_NOT_WARN| replace:: does *not* warn
+.. include:: STANDARD_ADVICE.txt
+
+.. include:: DEPRECATED.txt

Help/release/dev/execute_process-encoding.rst

@@ -0,0 +1,6 @@
+execute_process-encoding
+------------------------
+
+* The :command:`execute_process` command's ``ENCODING`` option,
+  meaningful on Windows, now defaults to ``UTF-8``.
+  See policy :policy:`CMP0176`.

Source/cmExecuteProcessCommand.cxx

@@ -11,6 +11,7 @@
 #include <utility>
 #include <vector>
 
+#include <cm/optional>
 #include <cm/string_view>
 #include <cmext/algorithm>
 #include <cmext/string_view>
@@ -22,6 +23,7 @@
 #include "cmList.h"
 #include "cmMakefile.h"
 #include "cmMessageType.h"
+#include "cmPolicies.h"
 #include "cmProcessOutput.h"
 #include "cmStringAlgorithms.h"
 #include "cmSystemTools.h"
@@ -69,7 +71,7 @@ bool cmExecuteProcessCommand(std::vector<std::string> const& args,
     bool ErrorStripTrailingWhitespace = false;
     bool EchoOutputVariable = false;
     bool EchoErrorVariable = false;
-    std::string Encoding;
+    cm::optional<std::string> Encoding;
     std::string CommandErrorIsFatal;
   };
 
@@ -296,8 +298,19 @@ bool cmExecuteProcessCommand(std::vector<std::string> const& args,
   };
   ReadData outputData;
   ReadData errorData;
-  cmProcessOutput processOutput(
-    cmProcessOutput::FindEncoding(arguments.Encoding));
+  cmPolicies::PolicyStatus const cmp0176 =
+    status.GetMakefile().GetPolicyStatus(cmPolicies::CMP0176);
+  cmProcessOutput::Encoding encoding =
+    cmp0176 == cmPolicies::OLD || cmp0176 == cmPolicies::WARN
+    ? cmProcessOutput::Auto
+    : cmProcessOutput::UTF8;
+  if (arguments.Encoding) {
+    if (cm::optional<cmProcessOutput::Encoding> maybeEncoding =
+          cmProcessOutput::FindEncoding(*arguments.Encoding)) {
+      encoding = *maybeEncoding;
+    }
+  }
+  cmProcessOutput processOutput(encoding);
   std::string strdata;
 
   std::unique_ptr<cmUVStreamReadHandle> outputHandle;

Source/cmPolicies.h

@@ -539,6 +539,8 @@ class cmMakefile;
          "string after a single-value keyword.",                              \
          3, 31, 0, cmPolicies::WARN)                                          \
   SELECT(POLICY, CMP0175, "add_custom_command() rejects invalid arguments.",  \
+         3, 31, 0, cmPolicies::WARN)                                          \
+  SELECT(POLICY, CMP0176, "execute_process() ENCODING is UTF-8 by default.",  \
          3, 31, 0, cmPolicies::WARN)
 
 #define CM_SELECT_ID(F, A1, A2, A3, A4, A5, A6) F(A1)

Source/cmProcessOutput.cxx

@@ -12,16 +12,18 @@ unsigned int cmProcessOutput::defaultCodepage =
   KWSYS_ENCODING_DEFAULT_CODEPAGE;
 #endif
 
-cmProcessOutput::Encoding cmProcessOutput::FindEncoding(
+cm::optional<cmProcessOutput::Encoding> cmProcessOutput::FindEncoding(
   std::string const& name)
 {
-  Encoding encoding = Auto;
+  cm::optional<Encoding> encoding;
   if ((name == "UTF8") || (name == "UTF-8")) {
     encoding = UTF8;
   } else if (name == "NONE") {
     encoding = None;
   } else if (name == "ANSI") {
     encoding = ANSI;
+  } else if (name == "AUTO") {
+    encoding = Auto;
   } else if (name == "OEM") {
     encoding = OEM;
   }

Source/cmProcessOutput.h

@@ -8,6 +8,8 @@
 #include <string>
 #include <vector>
 
+#include <cm/optional>
+
 /** \class cmProcessOutput
  * \brief Decode text data to internal encoding.
  *
@@ -31,7 +33,7 @@ public:
    * \param name a encoding name.
    * \return encoding enum value or Auto if \a name was not found.
    */
-  static Encoding FindEncoding(std::string const& name);
+  static cm::optional<Encoding> FindEncoding(std::string const& name);
 
   /// The code page that is used as internal encoding to which we will encode.
   static unsigned int defaultCodepage;

Tests/RunCMake/execute_process/Encoding-common.cmake

@@ -2,7 +2,12 @@ if(ENCODING)
   set(maybe_ENCODING ENCODING ${ENCODING})
 else()
   set(maybe_ENCODING "")
-  set(ENCODING AUTO) # execute_process's default ENCODING
+  cmake_policy(GET CMP0176 cmp0176)
+  if(cmp0176 STREQUAL "NEW")
+    set(ENCODING UTF-8) # execute_process's default ENCODING
+  else()
+    set(ENCODING AUTO) # execute_process's default ENCODING
+  endif()
 endif()
 execute_process(
   COMMAND ${TEST_ENCODING_EXE} ${ENCODING} ${CMAKE_CURRENT_LIST_DIR}/Encoding${ENCODING}-stderr.txt

Tests/RunCMake/execute_process/Encoding-windows.cmake

@@ -1,3 +1,7 @@
+if(CMP0176 STREQUAL "NEW")
+  cmake_policy(SET CMP0176 NEW)
+endif()
+
 # Set the console code page.
 execute_process(COMMAND cmd /c chcp ${CODEPAGE})
 

Tests/RunCMake/execute_process/Encoding.cmake

@@ -1,9 +1,12 @@
 if(CMAKE_HOST_WIN32 AND CODEPAGE)
+  cmake_policy(GET CMP0176 CMP0176)
+
   # Run cmake in a new Window to isolate its console code page.
   execute_process(COMMAND cmd /c start /min /wait ""
     ${CMAKE_COMMAND} -DTEST_ENCODING_EXE=${TEST_ENCODING_EXE}
                      -DENCODING=${ENCODING}
                      -DCODEPAGE=${CODEPAGE}
+                     -DCMP0176=${CMP0176}
                      -P ${CMAKE_CURRENT_LIST_DIR}/Encoding-windows.cmake)
 
   # Load our internal UTF-8 representation of the output.

Tests/RunCMake/execute_process/EncodingCMP0176-NEW-stderr.txt

@@ -0,0 +1,2 @@
+Chinese Hindi  Greek English Russian
+注意    यूनिकोड είναι very    здорово!

Tests/RunCMake/execute_process/EncodingCMP0176-NEW.cmake

@@ -0,0 +1,3 @@
+cmake_policy(SET CMP0176 NEW)
+# No explicit ENCODING option; fall back to default.
+include(${CMAKE_CURRENT_LIST_DIR}/Encoding.cmake)

Tests/RunCMake/execute_process/EncodingDefault.cmake → Tests/RunCMake/execute_process/EncodingCMP0176-OLD.cmake

@@ -1,3 +1,4 @@
+cmake_policy(SET CMP0176 OLD)
 # No explicit ENCODING option; fall back to default.
 set(CODEPAGE 54936)
 include(${CMAKE_CURRENT_LIST_DIR}/Encoding.cmake)

Tests/RunCMake/execute_process/RunCMakeTest.cmake

@@ -9,7 +9,8 @@ run_cmake_command(MergeOutputVars ${CMAKE_COMMAND} -P ${RunCMake_SOURCE_DIR}/Mer
 
 run_cmake(EncodingMissing)
 if(TEST_ENCODING_EXE)
-  run_cmake_script(EncodingDefault -DTEST_ENCODING_EXE=${TEST_ENCODING_EXE})
+  run_cmake_script(EncodingCMP0176-NEW -DTEST_ENCODING_EXE=${TEST_ENCODING_EXE})
+  run_cmake_script(EncodingCMP0176-OLD -DTEST_ENCODING_EXE=${TEST_ENCODING_EXE})
   run_cmake_script(EncodingAUTO -DTEST_ENCODING_EXE=${TEST_ENCODING_EXE})
   run_cmake_script(EncodingUTF-8 -DTEST_ENCODING_EXE=${TEST_ENCODING_EXE})
   run_cmake_script(EncodingUTF8 -DTEST_ENCODING_EXE=${TEST_ENCODING_EXE})