Merge topic 'ninja-showIncludes-encoding' into release-3.25

a0d4e3bf34 cmGeneratedFileStream: Drop unused WriteRaw method
2e5af30ce0 Ninja: Match showIncludes dependencies using console output code page
e1c1679148 cm_codecvt: Add support for the Windows console output code page
328c15189d cmGeneratedFileStream: Add support for a temporary alternate encoding

Acked-by: Kitware Robot <[email protected]>
Merge-request: !7845

Modules/CMakeDetermineCompilerId.cmake

@@ -1127,7 +1127,7 @@ function(CMAKE_DETERMINE_MSVC_SHOWINCLUDES_PREFIX lang userflags)
     OUTPUT_VARIABLE out
     ERROR_VARIABLE err
     RESULT_VARIABLE res
-    ENCODING AUTO # cl prints in current code page
+    ENCODING AUTO # cl prints in console output code page
     )
   if(res EQUAL 0 AND "${out}" MATCHES "(^|\n)([^:\n]*:[^:\n]*:[ \t]*)")
     set(CMAKE_${lang}_CL_SHOWINCLUDES_PREFIX "${CMAKE_MATCH_2}" PARENT_SCOPE)

Source/cmGeneratedFileStream.cxx

@@ -14,11 +14,10 @@
 #endif
 
 cmGeneratedFileStream::cmGeneratedFileStream(Encoding encoding)
-  : OriginalLocale(this->getloc())
 {
 #ifndef CMAKE_BOOTSTRAP
   if (encoding != codecvt::None) {
-    this->imbue(std::locale(this->OriginalLocale, new codecvt(encoding)));
+    this->imbue(std::locale(this->getloc(), new codecvt(encoding)));
   }
 #else
   static_cast<void>(encoding);
@@ -239,13 +238,16 @@ void cmGeneratedFileStream::SetTempExt(std::string const& ext)
   this->TempExt = ext;
 }
 
-void cmGeneratedFileStream::WriteRaw(std::string const& data)
+void cmGeneratedFileStream::WriteAltEncoding(std::string const& data,
+                                             Encoding encoding)
 {
 #ifndef CMAKE_BOOTSTRAP
-  std::locale activeLocale = this->imbue(this->OriginalLocale);
+  std::locale prevLocale =
+    this->imbue(std::locale(this->getloc(), new codecvt(encoding)));
   this->write(data.data(), data.size());
-  this->imbue(activeLocale);
+  this->imbue(prevLocale);
 #else
+  static_cast<void>(encoding);
   this->write(data.data(), data.size());
 #endif
 }

Source/cmGeneratedFileStream.h

@@ -148,12 +148,8 @@ public:
   void SetTempExt(std::string const& ext);
 
   /**
-   * Writes the given string directly to the file without changing the
-   * encoding.
+   * Write a specific string using an alternate encoding.
+   * Afterward, the original encoding is restored.
    */
-  void WriteRaw(std::string const& data);
-
-private:
-  // The original locale of the stream (performs no encoding conversion).
-  std::locale OriginalLocale;
+  void WriteAltEncoding(std::string const& data, Encoding encoding);
 };

Source/cmLocalNinjaGenerator.cxx

@@ -88,27 +88,11 @@ void cmLocalNinjaGenerator::Generate()
       cmGlobalNinjaGenerator::WriteComment(this->GetRulesFileStream(),
                                            "localized /showIncludes string");
       this->GetRulesFileStream() << "msvc_deps_prefix = ";
-#ifdef _WIN32
-      // Ninja uses the ANSI Windows APIs, so strings in the rules file
-      // typically need to be ANSI encoded. However, in this case the compiler
-      // is being invoked using the UTF-8 codepage so the /showIncludes prefix
-      // will be UTF-8 encoded on stdout. Ninja can't successfully compare this
-      // UTF-8 encoded prefix to the ANSI encoded msvc_deps_prefix if it
-      // contains any non-ASCII characters and dependency checking will fail.
-      // As a workaround, leave the msvc_deps_prefix UTF-8 encoded even though
-      // the rest of the file is ANSI encoded.
-      if (GetConsoleOutputCP() == CP_UTF8 && GetACP() != CP_UTF8 &&
-          this->GetGlobalGenerator()->GetMakefileEncoding() != codecvt::None) {
-        this->GetRulesFileStream().WriteRaw(showIncludesPrefix);
-      } else {
-        // Ninja 1.11 and above uses the UTF-8 code page if it's supported, so
-        // in that case we can write it normally without using raw bytes.
-        this->GetRulesFileStream() << showIncludesPrefix;
-      }
-#else
-      // It's safe to use the standard encoding on other platforms.
-      this->GetRulesFileStream() << showIncludesPrefix;
-#endif
+      // 'cl /showIncludes' encodes output in the console output code page.
+      // It may differ from the encoding used for file paths in 'build.ninja'.
+      // Ninja matches the showIncludes prefix using its raw byte sequence.
+      this->GetRulesFileStream().WriteAltEncoding(
+        showIncludesPrefix, cmGeneratedFileStream::Encoding::ConsoleOutput);
       this->GetRulesFileStream() << "\n\n";
     }
   }

Source/cm_codecvt.cxx

@@ -19,6 +19,12 @@ codecvt::codecvt(Encoding e)
 #endif
 {
   switch (e) {
+    case codecvt::ConsoleOutput:
+#if defined(_WIN32)
+      m_noconv = false;
+      m_codepage = GetConsoleOutputCP();
+      break;
+#endif
     case codecvt::ANSI:
 #if defined(_WIN32)
       m_noconv = false;

Source/cm_codecvt.hxx

@@ -15,7 +15,8 @@ public:
     None,
     UTF8,
     UTF8_WITH_BOM,
-    ANSI
+    ANSI,
+    ConsoleOutput,
   };
 
 #ifndef CMAKE_BOOTSTRAP

Tests/RunCMake/CMakeLists.txt

@@ -197,6 +197,10 @@ if(CMAKE_GENERATOR MATCHES "Ninja")
       ${ninja_qt_args}
     )
   endif()
+  if(WIN32)
+    add_executable(showIncludes showIncludes.c)
+    list(APPEND Ninja_ARGS -DshowIncludes=$<TARGET_FILE:showIncludes>)
+  endif()
   add_RunCMake_test(Ninja)
   set(NinjaMultiConfig_ARGS
     -DCYGWIN=${CYGWIN} -DMSYS=${MSYS}

Tests/RunCMake/Ninja/RunCMakeTest.cmake

@@ -42,6 +42,15 @@ function(run_Intl)
 endfunction()
 run_Intl()
 
+if(WIN32)
+  if(RunCMake_MAKE_PROGRAM)
+    set(maybe_MAKE_PROGRAM "-DRunCMake_MAKE_PROGRAM=${RunCMake_MAKE_PROGRAM}")
+  endif()
+  run_cmake_script(ShowIncludes-54936 -DshowIncludes=${showIncludes} ${maybe_MAKE_PROGRAM})
+  run_cmake_script(ShowIncludes-65001 -DshowIncludes=${showIncludes} ${maybe_MAKE_PROGRAM})
+  unset(maybe_MAKE_PROGRAM)
+endif()
+
 function(run_NoWorkToDo)
   run_cmake(NoWorkToDo)
   set(RunCMake_TEST_NO_CLEAN 1)

Tests/RunCMake/Ninja/ShowIncludes-54936-check.cmake

@@ -0,0 +1,3 @@
+# 'cl /showIncludes' prefix with 'VSLANG=2052' and 'chcp 54936'.
+string(ASCII 215 162 210 226 58 32 176 252 186 172 206 196 188 254 58 expect)
+include(${CMAKE_CURRENT_LIST_DIR}/ShowIncludes-check.cmake)

Tests/RunCMake/Ninja/ShowIncludes-54936-stdout.txt

@@ -0,0 +1 @@
+-- showIncludes='注意: 包含文件:'

Tests/RunCMake/Ninja/ShowIncludes-54936.cmake

@@ -0,0 +1,2 @@
+set(CODEPAGE 54936)
+include(${CMAKE_CURRENT_LIST_DIR}/ShowIncludes.cmake)

Tests/RunCMake/Ninja/ShowIncludes-65001-check.cmake

@@ -0,0 +1,3 @@
+# 'cl /showIncludes' prefix with 'VSLANG=2052' and 'chcp 65001'.
+string(ASCII 230 179 168 230 132 143 58 32 229 140 133 229 144 171 230 150 135 228 187 182 58 expect)
+include(${CMAKE_CURRENT_LIST_DIR}/ShowIncludes-check.cmake)

Tests/RunCMake/Ninja/ShowIncludes-65001-stdout.txt

@@ -0,0 +1 @@
+-- showIncludes='注意: 包含文件:'

Tests/RunCMake/Ninja/ShowIncludes-65001.cmake

@@ -0,0 +1,2 @@
+set(CODEPAGE 65001)
+include(${CMAKE_CURRENT_LIST_DIR}/ShowIncludes.cmake)

Tests/RunCMake/Ninja/ShowIncludes-check.cmake

@@ -0,0 +1,17 @@
+set(rules_ninja "${RunCMake_TEST_BINARY_DIR}/CMakeFiles/rules.ninja")
+if(NOT EXISTS "${rules_ninja}")
+  set(RunCMake_TEST_FAILED "Generator output file is missing:\n ${rules_ninja}")
+  return()
+endif()
+
+file(READ "${rules_ninja}" rules_ninja)
+if(rules_ninja MATCHES "msvc_deps_prefix = ([^\r\n]*)\n")
+  set(actual "${CMAKE_MATCH_1}")
+endif()
+
+if(NOT actual STREQUAL expect)
+  string(HEX "${actual}" actual_hex)
+  string(HEX "${expect}" expect_hex)
+  set(RunCMake_TEST_FAILED "Expected byte sequence\n '${expect}' (${expect_hex})\nbut got\n '${actual}' (${actual_hex})")
+  return()
+endif()

Tests/RunCMake/Ninja/ShowIncludes-cmake.cmake

@@ -0,0 +1,7 @@
+# Set the console code page.
+execute_process(COMMAND cmd /c chcp "${CODEPAGE}")
+
+if(RunCMake_MAKE_PROGRAM)
+  set(maybe_MAKE_PROGRAM "-DCMAKE_MAKE_PROGRAM=${RunCMake_MAKE_PROGRAM}")
+endif()
+execute_process(COMMAND "${CMAKE_COMMAND}" . -G Ninja ${maybe_MAKE_PROGRAM})

Tests/RunCMake/Ninja/ShowIncludes.cmake

@@ -0,0 +1,22 @@
+# Create a project to do showIncludes prefix detection.
+file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/CMakeLists.txt" "
+cmake_minimum_required(VERSION 3.25)
+project(ShowIncludes NONE)
+include(CMakeDetermineCompilerId)
+set(CMAKE_dummy_COMPILER \"${showIncludes}\")
+CMAKE_DETERMINE_MSVC_SHOWINCLUDES_PREFIX(dummy \"\")
+set(CMAKE_CL_SHOWINCLUDES_PREFIX \"\${CMAKE_dummy_CL_SHOWINCLUDES_PREFIX}\")
+file(WRITE \"\${CMAKE_CURRENT_BINARY_DIR}/showIncludes.txt\" \"\${CMAKE_CL_SHOWINCLUDES_PREFIX}\")
+")
+
+if(RunCMake_MAKE_PROGRAM)
+  set(maybe_MAKE_PROGRAM "-DRunCMake_MAKE_PROGRAM=${RunCMake_MAKE_PROGRAM}")
+endif()
+
+# Run cmake in a new Window to isolate its console code page.
+execute_process(COMMAND cmd /c start /min /wait ""
+  ${CMAKE_COMMAND} -DCODEPAGE=${CODEPAGE} ${maybe_MAKE_PROGRAM} -P ${CMAKE_CURRENT_LIST_DIR}/ShowIncludes-cmake.cmake)
+
+# Print our internal UTF-8 representation of the showIncludes prefix.
+file(READ "${CMAKE_CURRENT_BINARY_DIR}/showIncludes.txt" showIncludes_txt)
+message(STATUS "showIncludes='${showIncludes_txt}'")

Tests/RunCMake/showIncludes.c

@@ -0,0 +1,33 @@
+#if defined(_MSC_VER) && _MSC_VER >= 1928
+#  pragma warning(disable : 5105) /* macro expansion warning in windows.h */
+#endif
+#include <windows.h>
+
+#include <stdio.h>
+
+int main()
+{
+  /* 'cl /showIncludes' encodes output in the console output code page.  */
+  unsigned int cp = GetConsoleOutputCP();
+  printf("Console output code page: %u\n", cp);
+  printf("Console input code page: %u\n", GetConsoleCP());
+  printf("ANSI code page: %u\n", GetACP());
+  printf("OEM code page: %u\n", GetOEMCP());
+
+  if (cp == 54936 || cp == 936) {
+    /* VSLANG=2052 */
+    printf("\xd7\xa2\xd2\xe2: "
+           "\xb0\xfc\xba\xac\xce\xc4\xbc\xfe:\n");
+    return 0;
+  }
+
+  if (cp == 65001) {
+    /* VSLANG=2052  */
+    printf("\xe6\xb3\xa8\xe6\x84\x8f: "
+           "\xe5\x8c\x85\xe5\x90\xab\xe6\x96\x87\xe4\xbb\xb6:\n");
+    return 0;
+  }
+
+  fprintf(stderr, "No example showIncludes for console's output code page.\n");
+  return 1;
+}