Merge topic 'autogen-rsp'

232610e60e Autogen: Use new API for limiting autogen command line lengths
7a07887055 Autogen: Add support for response files for moc predef targets
7eb5ab2c63 Autogen: Generalize MaybeWriteMocResponseFile to MaybeWriteResponseFile

Acked-by: Kitware Robot <[email protected]>
Tested-by: buildbot <[email protected]>
Merge-request: !8944

Auxiliary/vim/syntax/cmake.vim

@@ -70,6 +70,7 @@ syn keyword cmakeProperty contained
             \ ATTACHED_FILES
             \ ATTACHED_FILES_ON_FAIL
             \ AUTOGEN_BUILD_DIR
+            \ AUTOGEN_COMMAND_LINE_LENGTH_MAX
             \ AUTOGEN_ORIGIN_DEPENDS
             \ AUTOGEN_PARALLEL
             \ AUTOGEN_SOURCE_GROUP
@@ -764,6 +765,7 @@ syn keyword cmakeVariable contained
             \ CMAKE_ASM_STANDARD_REQUIRED
             \ CMAKE_ASM_SUPPORTED
             \ CMAKE_ASM_VISIBILITY_PRESET
+            \ CMAKE_AUTOGEN_COMMAND_LINE_LENGTH_MAX
             \ CMAKE_AUTOGEN_ORIGIN_DEPENDS
             \ CMAKE_AUTOGEN_PARALLEL
             \ CMAKE_AUTOGEN_USE_SYSTEM_INCLUDE

Help/manual/cmake-properties.7.rst

@@ -130,6 +130,7 @@ Properties on Targets
    /prop_tgt/ARCHIVE_OUTPUT_NAME
    /prop_tgt/ARCHIVE_OUTPUT_NAME_CONFIG
    /prop_tgt/AUTOGEN_BUILD_DIR
+   /prop_tgt/AUTOGEN_COMMAND_LINE_LENGTH_MAX
    /prop_tgt/AUTOGEN_ORIGIN_DEPENDS
    /prop_tgt/AUTOGEN_PARALLEL
    /prop_tgt/AUTOGEN_TARGET_DEPENDS

Help/manual/cmake-variables.7.rst

@@ -400,6 +400,7 @@ Variables that Control the Build
    /variable/CMAKE_APPLE_SILICON_PROCESSOR
    /variable/CMAKE_ARCHIVE_OUTPUT_DIRECTORY
    /variable/CMAKE_ARCHIVE_OUTPUT_DIRECTORY_CONFIG
+   /variable/CMAKE_AUTOGEN_COMMAND_LINE_LENGTH_MAX
    /variable/CMAKE_AUTOGEN_ORIGIN_DEPENDS
    /variable/CMAKE_AUTOGEN_PARALLEL
    /variable/CMAKE_AUTOGEN_USE_SYSTEM_INCLUDE

Help/prop_tgt/AUTOGEN_COMMAND_LINE_LENGTH_MAX.rst

@@ -0,0 +1,18 @@
+AUTOGEN_COMMAND_LINE_LENGTH_MAX
+-------------------------------
+
+.. versionadded:: 3.29
+
+Command line length limit for autogen targets, i.e. ``moc`` or ``uic``,
+that triggers the use of response files on Windows instead of passing all
+arguments to the command line.
+
+- An empty (or unset) value sets the limit to 32000
+- A positive non zero integer value sets the exact command line length
+  limit.
+
+By default ``AUTOGEN_COMMAND_LINE_LENGTH_MAX`` is initialized from
+:variable:`CMAKE_AUTOGEN_COMMAND_LINE_LENGTH_MAX`.
+
+See the :manual:`cmake-qt(7)` manual for more information on using CMake
+with Qt.

Help/prop_tgt/AUTOMOC.rst

@@ -247,5 +247,9 @@ will be generated when this variable is ``ON``.
 This target property controls the number of ``moc`` or ``uic`` processes to
 start in parallel during builds.
 
+:prop_tgt:`AUTOGEN_COMMAND_LINE_LENGTH_MAX`:
+This target property controls the limit when to use response files for
+``moc`` or ``uic`` processes on Windows.
+
 See the :manual:`cmake-qt(7)` manual for more information on using CMake
 with Qt.

Help/prop_tgt/AUTOUIC.rst

@@ -81,5 +81,9 @@ will be generated when this variable is ``ON``.
 This target property controls the number of ``moc`` or ``uic`` processes to
 start in parallel during builds.
 
+:prop_tgt:`AUTOGEN_COMMAND_LINE_LENGTH_MAX`:
+This target property controls the limit when to use response files for
+``moc`` or ``uic`` processes on Windows.
+
 See the :manual:`cmake-qt(7)` manual for more information on using CMake
 with Qt.

Help/variable/CMAKE_AUTOGEN_COMMAND_LINE_LENGTH_MAX.rst

@@ -0,0 +1,10 @@
+CMAKE_AUTOGEN_COMMAND_LINE_LENGTH_MAX
+-------------------------------------
+
+.. versionadded:: 3.29
+
+Command line length limit for autogen targets, i.e. ``moc`` or ``uic``,
+that triggers the use of response files on Windows instead of passing all
+arguments to the command line.
+
+By default ``CMAKE_AUTOGEN_COMMAND_LINE_LENGTH_MAX`` is unset.

Source/cmQtAutoGen.cxx

@@ -76,13 +76,6 @@ static void MergeOptions(std::vector<std::string>& baseOpts,
 
 unsigned int const cmQtAutoGen::ParallelMax = 64;
 
-#ifdef _WIN32
-// Actually 32767 (see
-// https://devblogs.microsoft.com/oldnewthing/20031210-00/?p=41553) but we
-// allow for a small margin
-size_t const cmQtAutoGen::CommandLineLengthMax = 32000;
-#endif
-
 cm::string_view cmQtAutoGen::GeneratorName(GenT genType)
 {
   switch (genType) {

Source/cmQtAutoGen.h

@@ -64,11 +64,6 @@ public:
   /// @brief Maximum number of parallel threads/processes in a generator
   static unsigned int const ParallelMax;
 
-#ifdef _WIN32
-  /// @brief Maximum number of characters on command line
-  static size_t const CommandLineLengthMax;
-#endif
-
   /// @brief Returns the generator name
   static cm::string_view GeneratorName(GenT genType);
   /// @brief Returns the generator name in upper case

Source/cmQtAutoGenInitializer.cxx

@@ -485,6 +485,38 @@ bool cmQtAutoGenInitializer::InitCustomTargets()
       }
     }
 
+#ifdef _WIN32
+    {
+      const auto& value =
+        this->GenTarget->GetProperty("AUTOGEN_COMMAND_LINE_LENGTH_MAX");
+      if (value.IsSet()) {
+        using maxCommandLineLengthType =
+          decltype(this->AutogenTarget.MaxCommandLineLength);
+        unsigned long propInt = 0;
+        if (cmStrToULong(value, &propInt) && propInt > 0 &&
+            propInt <= std::numeric_limits<maxCommandLineLengthType>::max()) {
+          this->AutogenTarget.MaxCommandLineLength =
+            static_cast<maxCommandLineLengthType>(propInt);
+        } else {
+          // Warn the project author that AUTOGEN_PARALLEL is not valid.
+          this->Makefile->IssueMessage(
+            MessageType::AUTHOR_WARNING,
+            cmStrCat("AUTOGEN_COMMAND_LINE_LENGTH_MAX=\"", *value,
+                     "\" for target \"", this->GenTarget->GetName(),
+                     "\" is not valid. Using no limit for "
+                     "AUTOGEN_COMMAND_LINE_LENGTH_MAX"));
+          this->AutogenTarget.MaxCommandLineLength =
+            std::numeric_limits<maxCommandLineLengthType>::max();
+        }
+      } else {
+        // Actually 32767 (see
+        // https://devblogs.microsoft.com/oldnewthing/20031210-00/?p=41553) but
+        // we allow for a small margin
+        this->AutogenTarget.MaxCommandLineLength = 32000;
+      }
+    }
+#endif
+
     // Autogen target info and settings files
     {
       // Info file
@@ -1692,6 +1724,10 @@ bool cmQtAutoGenInitializer::SetupWriteAutogenInfo()
   // General
   info.SetBool("MULTI_CONFIG", this->MultiConfig);
   info.SetUInt("PARALLEL", this->AutogenTarget.Parallel);
+#ifdef _WIN32
+  info.SetUInt("AUTOGEN_COMMAND_LINE_LENGTH_MAX",
+               this->AutogenTarget.MaxCommandLineLength);
+#endif
   info.SetUInt("VERBOSITY", this->Verbosity);
 
   // Directories

Source/cmQtAutoGenInitializer.h

@@ -5,6 +5,7 @@
 #include "cmConfigure.h" // IWYU pragma: keep
 
 #include <cstddef>
+#include <limits>
 #include <memory>
 #include <set>
 #include <string>
@@ -194,6 +195,8 @@ private:
     bool GlobalTarget = false;
     // Settings
     unsigned int Parallel = 1;
+    unsigned int MaxCommandLineLength =
+      std::numeric_limits<unsigned int>::max();
     // Configuration files
     std::string InfoFile;
     ConfigString SettingsFile;

Source/cmQtAutoMocUic.cxx

@@ -5,6 +5,7 @@
 #include <algorithm>
 #include <atomic>
 #include <cstddef>
+#include <limits>
 #include <map>
 #include <mutex>
 #include <set>
@@ -172,6 +173,8 @@ public:
     bool MultiConfig = false;
     IntegerVersion QtVersion = { 4, 0 };
     unsigned int ThreadCount = 0;
+    unsigned int MaxCommandLineLength =
+      std::numeric_limits<unsigned int>::max();
     // - Directories
     std::string AutogenBuildDir;
     std::string AutogenIncludeDir;
@@ -333,6 +336,13 @@ public:
                          std::vector<std::string> const& command,
                          std::string const& output) const;
 
+    /*
+     * Check if command line exceeds maximum length supported by OS
+     * (if on Windows) and switch to using a response file instead.
+     */
+    void MaybeWriteResponseFile(std::string const& outputFile,
+                                std::vector<std::string>& cmd) const;
+
     /** @brief Run an external process. Use only during Process() call!  */
     bool RunProcess(GenT genType, cmWorkerPool::ProcessResultT& result,
                     std::vector<std::string> const& command,
@@ -498,10 +508,6 @@ public:
 
   protected:
     ParseCacheT::FileHandleT CacheEntry;
-
-  private:
-    void MaybeWriteMocResponseFile(std::string const& outputFile,
-                                   std::vector<std::string>& cmd) const;
   };
 
   /** uic compiles a file.  */
@@ -795,6 +801,51 @@ void cmQtAutoMocUicT::JobT::LogCommandError(
   this->Gen()->Log().ErrorCommand(genType, message, command, output);
 }
 
+/*
+ * Check if command line exceeds maximum length supported by OS
+ * (if on Windows) and switch to using a response file instead.
+ */
+void cmQtAutoMocUicT::JobT::MaybeWriteResponseFile(
+  std::string const& outputFile, std::vector<std::string>& cmd) const
+{
+#ifdef _WIN32
+  // Ensure cmd is less than CommandLineLengthMax characters
+  size_t commandLineLength = cmd.size(); // account for separating spaces
+  for (std::string const& str : cmd) {
+    commandLineLength += str.length();
+  }
+  if (commandLineLength >= this->BaseConst().MaxCommandLineLength) {
+    // Command line exceeds maximum size allowed by OS
+    // => create response file
+    std::string const responseFile = cmStrCat(outputFile, ".rsp");
+
+    cmsys::ofstream fout(responseFile.c_str());
+    if (!fout) {
+      this->LogError(
+        GenT::MOC,
+        cmStrCat("AUTOMOC was unable to create a response file at\n  ",
+                 this->MessagePath(responseFile)));
+      return;
+    }
+
+    auto it = cmd.begin();
+    while (++it != cmd.end()) {
+      fout << *it << "\n";
+    }
+    fout.close();
+
+    // Keep all but executable
+    cmd.resize(1);
+
+    // Specify response file
+    cmd.emplace_back(cmStrCat('@', responseFile));
+  }
+#else
+  static_cast<void>(outputFile);
+  static_cast<void>(cmd);
+#endif
+}
+
 bool cmQtAutoMocUicT::JobT::RunProcess(GenT genType,
                                        cmWorkerPool::ProcessResultT& result,
                                        std::vector<std::string> const& command,
@@ -836,6 +887,8 @@ void cmQtAutoMocUicT::JobMocPredefsT::Process()
       cm::append(cmd, this->MocConst().OptionsDefinitions);
       // Add includes
       cm::append(cmd, this->MocConst().OptionsIncludes);
+      // Check if response file is necessary
+      MaybeWriteResponseFile(this->MocConst().PredefsFileAbs, cmd);
       // Execute command
       if (!this->RunProcess(GenT::MOC, result, cmd, reason.get())) {
         this->LogCommandError(GenT::MOC,
@@ -2034,7 +2087,7 @@ void cmQtAutoMocUicT::JobCompileMocT::Process()
     // Add source file
     cmd.push_back(sourceFile);
 
-    MaybeWriteMocResponseFile(outputFile, cmd);
+    MaybeWriteResponseFile(outputFile, cmd);
   }
 
   // Execute moc command
@@ -2080,51 +2133,6 @@ void cmQtAutoMocUicT::JobCompileMocT::Process()
   }
 }
 
-/*
- * Check if command line exceeds maximum length supported by OS
- * (if on Windows) and switch to using a response file instead.
- */
-void cmQtAutoMocUicT::JobCompileMocT::MaybeWriteMocResponseFile(
-  std::string const& outputFile, std::vector<std::string>& cmd) const
-{
-#ifdef _WIN32
-  // Ensure cmd is less than CommandLineLengthMax characters
-  size_t commandLineLength = cmd.size(); // account for separating spaces
-  for (std::string const& str : cmd) {
-    commandLineLength += str.length();
-  }
-  if (commandLineLength >= CommandLineLengthMax) {
-    // Command line exceeds maximum size allowed by OS
-    // => create response file
-    std::string const responseFile = cmStrCat(outputFile, ".rsp");
-
-    cmsys::ofstream fout(responseFile.c_str());
-    if (!fout) {
-      this->LogError(
-        GenT::MOC,
-        cmStrCat("AUTOMOC was unable to create a response file at\n  ",
-                 this->MessagePath(responseFile)));
-      return;
-    }
-
-    auto it = cmd.begin();
-    while (++it != cmd.end()) {
-      fout << *it << "\n";
-    }
-    fout.close();
-
-    // Keep all but executable
-    cmd.resize(1);
-
-    // Specify response file
-    cmd.emplace_back(cmStrCat('@', responseFile));
-  }
-#else
-  static_cast<void>(outputFile);
-  static_cast<void>(cmd);
-#endif
-}
-
 void cmQtAutoMocUicT::JobCompileUicT::Process()
 {
   std::string const& sourceFile = this->Mapping->SourceFile->FileName;
@@ -2377,6 +2385,10 @@ bool cmQtAutoMocUicT::InitFromInfo(InfoT const& info)
       !info.GetUInt("QT_VERSION_MINOR", this->BaseConst_.QtVersion.Minor,
                     true) ||
       !info.GetUInt("PARALLEL", this->BaseConst_.ThreadCount, false) ||
+#ifdef _WIN32
+      !info.GetUInt("AUTOGEN_COMMAND_LINE_LENGTH_MAX",
+                    this->BaseConst_.MaxCommandLineLength, false) ||
+#endif
       !info.GetString("BUILD_DIR", this->BaseConst_.AutogenBuildDir, true) ||
       !info.GetStringConfig("INCLUDE_DIR", this->BaseConst_.AutogenIncludeDir,
                             true) ||

Source/cmTarget.cxx

@@ -551,6 +551,7 @@ TargetProperty const StaticTargetProperties[] = {
   { "ANDROID_PROCESS_MAX"_s, IC::CanCompileSources },
   { "ANDROID_SKIP_ANT_STEP"_s, IC::CanCompileSources },
   // -- Autogen
+  { "AUTOGEN_COMMAND_LINE_LENGTH_MAX"_s, IC::CanCompileSources },
   { "AUTOGEN_ORIGIN_DEPENDS"_s, IC::CanCompileSources },
   { "AUTOGEN_PARALLEL"_s, IC::CanCompileSources },
   { "AUTOGEN_USE_SYSTEM_INCLUDE"_s, IC::CanCompileSources },