Add an option for explicit BYPRODUCTS of custom commands (#14963)

A common idiom in CMake-based build systems is to have custom commands
that generate files not listed explicitly as outputs so that these
files do not have to be newer than the inputs.  The file modification
times of such "byproducts" are updated only when their content changes.
Then other build rules can depend on the byproducts explicitly so that
their dependents rebuild when the content of the original byproducts
really does change.

This "undeclared byproduct" approach is necessary for Makefile, VS, and
Xcode build tools because if a byproduct were listed as an output of a
rule then the rule would always rerun when the input is newer than the
byproduct but the byproduct may never be updated.

Ninja solves this problem by offering a 'restat' feature to check
whether an output was really modified after running a rule and tracking
the fact that it is up to date separately from its timestamp.  However,
Ninja also stats all dependencies up front and will only restat files
that are listed as outputs of rules with the 'restat' option enabled.
Therefore an undeclared byproduct that does not exist at the start of
the build will be considered missing and the build will fail even if
other dependencies would cause the byproduct to be available before its
dependents build.

CMake works around this limitation by adding 'phony' build rules for
custom command dependencies in the build tree that do not have any
explicit specification of what produces them.  This is not optimal
because it prevents Ninja from reporting an error when an input to a
rule really is missing.  A better approach is to allow projects to
explicitly specify the byproducts of their custom commands so that no
phony rules are needed for them.  In order to work with the non-Ninja
generators, the byproducts must be known separately from the outputs.

Add a new "BYPRODUCTS" option to the add_custom_command and
add_custom_target commands to specify byproducts explicitly.  Teach the
Ninja generator to specify byproducts as outputs of the custom commands.
In the case of POST_BUILD, PRE_LINK, and PRE_BUILD events on targets
that link, the byproducts must be specified as outputs of the link rule
that runs the commands.  Activate 'restat' for such rules so that Ninja
knows it needs to check the byproducts, but not for link rules that have
no byproducts.

Help/command/add_custom_command.rst

@@ -15,6 +15,7 @@ The first signature is for adding a custom command to produce an output::
                      [COMMAND command2 [ARGS] [args2...] ...]
                      [MAIN_DEPENDENCY depend]
                      [DEPENDS [depends...]]
+                     [BYPRODUCTS [files...]]
                      [IMPLICIT_DEPENDS <lang1> depend1
                                       [<lang2> depend2] ...]
                      [WORKING_DIRECTORY dir]
@@ -44,6 +45,27 @@ The options are:
   options are currently ignored when APPEND is given, but may be
   used in the future.
 
+``BYPRODUCTS``
+  Specify the files the command is expected to produce but whose
+  modification time may or may not be newer than the dependencies.
+  If a byproduct name is a relative path it will be interpreted
+  relative to the build tree directory corresponding to the
+  current source directory.
+  Each byproduct file will be marked with the :prop_sf:`GENERATED`
+  source file property automatically.
+
+  Explicit specification of byproducts is supported by the
+  :generator:`Ninja` generator to tell the ``ninja`` build tool
+  how to regenerate byproducts when they are missing.  It is
+  also useful when other build rules (e.g. custom commands)
+  depend on the byproducts.  Ninja requires a build rule for any
+  generated file on which another rule depends even if there are
+  order-only dependencies to ensure the byproducts will be
+  available before their dependents build.
+
+  The ``BYPRODUCTS`` option is ignored on non-Ninja generators
+  except to mark byproducts ``GENERATED``.
+
 ``COMMAND``
   Specify the command-line(s) to execute at build time.
   If more than one ``COMMAND`` is specified they will be executed in order,
@@ -156,6 +178,7 @@ target is already built, the command will not execute.
                      PRE_BUILD | PRE_LINK | POST_BUILD
                      COMMAND command1 [ARGS] [args1...]
                      [COMMAND command2 [ARGS] [args2...] ...]
+                     [BYPRODUCTS [files...]]
                      [WORKING_DIRECTORY dir]
                      [COMMENT comment]
                      [VERBATIM] [USES_TERMINAL])

Help/command/add_custom_target.rst

@@ -8,6 +8,7 @@ Add a target with no output so it will always be built.
   add_custom_target(Name [ALL] [command1 [args1...]]
                     [COMMAND command2 [args2...] ...]
                     [DEPENDS depend depend depend ... ]
+                    [BYPRODUCTS [files...]]
                     [WORKING_DIRECTORY dir]
                     [COMMENT comment]
                     [VERBATIM] [USES_TERMINAL]
@@ -28,6 +29,27 @@ The options are:
   target so that it will be run every time (the command cannot be
   called ``ALL``).
 
+``BYPRODUCTS``
+  Specify the files the command is expected to produce but whose
+  modification time may or may not be updated on subsequent builds.
+  If a byproduct name is a relative path it will be interpreted
+  relative to the build tree directory corresponding to the
+  current source directory.
+  Each byproduct file will be marked with the :prop_sf:`GENERATED`
+  source file property automatically.
+
+  Explicit specification of byproducts is supported by the
+  :generator:`Ninja` generator to tell the ``ninja`` build tool
+  how to regenerate byproducts when they are missing.  It is
+  also useful when other build rules (e.g. custom commands)
+  depend on the byproducts.  Ninja requires a build rule for any
+  generated file on which another rule depends even if there are
+  order-only dependencies to ensure the byproducts will be
+  available before their dependents build.
+
+  The ``BYPRODUCTS`` option is ignored on non-Ninja generators
+  except to mark byproducts ``GENERATED``.
+
 ``COMMAND``
   Specify the command-line(s) to execute at build time.
   If more than one ``COMMAND`` is specified they will be executed in order,

Source/cmAddCustomCommandCommand.cxx

@@ -32,7 +32,7 @@ bool cmAddCustomCommandCommand
   std::string source, target, main_dependency, working;
   std::string comment_buffer;
   const char* comment = 0;
-  std::vector<std::string> depends, outputs, output;
+  std::vector<std::string> depends, outputs, output, byproducts;
   bool verbatim = false;
   bool append = false;
   bool uses_terminal = false;
@@ -57,6 +57,7 @@ bool cmAddCustomCommandCommand
     doing_main_dependency,
     doing_output,
     doing_outputs,
+    doing_byproducts,
     doing_comment,
     doing_working_directory,
     doing_nothing
@@ -127,6 +128,10 @@ bool cmAddCustomCommandCommand
       {
       doing = doing_output;
       }
+    else if (copy == "BYPRODUCTS")
+      {
+      doing = doing_byproducts;
+      }
     else if (copy == "WORKING_DIRECTORY")
       {
       doing = doing_working_directory;
@@ -150,6 +155,7 @@ bool cmAddCustomCommandCommand
         {
         case doing_output:
         case doing_outputs:
+        case doing_byproducts:
           if (!cmSystemTools::FileIsFullPath(copy.c_str()))
             {
             // This is an output to be generated, so it should be
@@ -233,6 +239,9 @@ bool cmAddCustomCommandCommand
          case doing_outputs:
            outputs.push_back(filename);
            break;
+         case doing_byproducts:
+           byproducts.push_back(filename);
+           break;
          case doing_comment:
            comment_buffer = copy;
            comment = comment_buffer.c_str();
@@ -272,7 +281,9 @@ bool cmAddCustomCommandCommand
     }
 
   // Make sure the output names and locations are safe.
-  if(!this->CheckOutputs(output) || !this->CheckOutputs(outputs))
+  if(!this->CheckOutputs(output) ||
+     !this->CheckOutputs(outputs) ||
+     !this->CheckOutputs(byproducts))
     {
     return false;
     }
@@ -314,7 +325,7 @@ bool cmAddCustomCommandCommand
     {
     // Source is empty, use the target.
     std::vector<std::string> no_depends;
-    this->Makefile->AddCustomCommandToTarget(target, no_depends,
+    this->Makefile->AddCustomCommandToTarget(target, byproducts, no_depends,
                                              commandLines, cctype,
                                              comment, working.c_str(),
                                              escapeOldStyle, uses_terminal);
@@ -322,8 +333,8 @@ bool cmAddCustomCommandCommand
   else if(target.empty())
     {
     // Target is empty, use the output.
-    this->Makefile->AddCustomCommandToOutput(output, depends,
-                                             main_dependency,
+    this->Makefile->AddCustomCommandToOutput(output, byproducts,
+                                             depends, main_dependency,
                                              commandLines, comment,
                                              working.c_str(), false,
                                              escapeOldStyle, uses_terminal);
@@ -351,6 +362,11 @@ bool cmAddCustomCommandCommand
         }
       }
     }
+  else if (!byproducts.empty())
+    {
+    this->SetError("BYPRODUCTS may not be specified with SOURCE signatures");
+    return false;
+    }
   else if (uses_terminal)
     {
     this->SetError("USES_TERMINAL may not be used with SOURCE signatures");

Source/cmAddCustomTargetCommand.cxx

@@ -45,7 +45,7 @@ bool cmAddCustomTargetCommand
   cmCustomCommandLines commandLines;
 
   // Accumulate dependencies.
-  std::vector<std::string> depends;
+  std::vector<std::string> depends, byproducts;
   std::string working_directory;
   bool verbatim = false;
   bool uses_terminal = false;
@@ -57,6 +57,7 @@ bool cmAddCustomTargetCommand
   enum tdoing {
     doing_command,
     doing_depends,
+    doing_byproducts,
     doing_working_directory,
     doing_comment,
     doing_source,
@@ -85,6 +86,10 @@ bool cmAddCustomTargetCommand
       {
       doing = doing_depends;
       }
+    else if(copy == "BYPRODUCTS")
+      {
+      doing = doing_byproducts;
+      }
     else if(copy == "WORKING_DIRECTORY")
       {
       doing = doing_working_directory;
@@ -128,6 +133,19 @@ bool cmAddCustomTargetCommand
         case doing_command:
           currentLine.push_back(copy);
           break;
+        case doing_byproducts:
+          {
+          std::string filename;
+          if (!cmSystemTools::FileIsFullPath(copy.c_str()))
+            {
+            filename = this->Makefile->GetCurrentOutputDirectory();
+            filename += "/";
+            }
+          filename += copy;
+          cmSystemTools::ConvertToUnixSlashes(filename);
+          byproducts.push_back(filename);
+          }
+          break;
         case doing_depends:
           {
           std::string dep = copy;
@@ -227,6 +245,12 @@ bool cmAddCustomTargetCommand
       cmSystemTools::CollapseFullPath(working_directory, build_dir);
     }
 
+  if (commandLines.empty() && !byproducts.empty())
+    {
+    this->Makefile->IssueMessage(cmake::FATAL_ERROR,
+      "BYPRODUCTS may not be specified without any COMMAND");
+    return true;
+    }
   if (commandLines.empty() && uses_terminal)
     {
     this->Makefile->IssueMessage(cmake::FATAL_ERROR,
@@ -238,7 +262,8 @@ bool cmAddCustomTargetCommand
   bool escapeOldStyle = !verbatim;
   cmTarget* target =
     this->Makefile->AddUtilityCommand(targetName, excludeFromAll,
-                                      working_directory.c_str(), depends,
+                                      working_directory.c_str(),
+                                      byproducts, depends,
                                       commandLines, escapeOldStyle, comment,
                                       uses_terminal);
 

Source/cmCPluginAPI.cxx

@@ -353,10 +353,11 @@ void CCONV cmAddCustomCommandToTarget(void *arg, const char* target,
     }
 
   // Pass the call to the makefile instance.
+  std::vector<std::string> no_byproducts;
   std::vector<std::string> no_depends;
   const char* no_comment = 0;
   const char* no_working_dir = 0;
-  mf->AddCustomCommandToTarget(target, no_depends, commandLines,
+  mf->AddCustomCommandToTarget(target, no_byproducts, no_depends, commandLines,
                                cctype, no_comment, no_working_dir);
 }
 

Source/cmCustomCommand.cxx

@@ -28,6 +28,7 @@ cmCustomCommand::cmCustomCommand()
 //----------------------------------------------------------------------------
 cmCustomCommand::cmCustomCommand(const cmCustomCommand& r):
   Outputs(r.Outputs),
+  Byproducts(r.Byproducts),
   Depends(r.Depends),
   CommandLines(r.CommandLines),
   HaveComment(r.HaveComment),
@@ -49,6 +50,7 @@ cmCustomCommand& cmCustomCommand::operator=(cmCustomCommand const& r)
     }
 
   this->Outputs = r.Outputs;
+  this->Byproducts= r.Byproducts;
   this->Depends = r.Depends;
   this->CommandLines = r.CommandLines;
   this->HaveComment = r.HaveComment;
@@ -66,11 +68,13 @@ cmCustomCommand& cmCustomCommand::operator=(cmCustomCommand const& r)
 //----------------------------------------------------------------------------
 cmCustomCommand::cmCustomCommand(cmMakefile const* mf,
                                  const std::vector<std::string>& outputs,
+                                 const std::vector<std::string>& byproducts,
                                  const std::vector<std::string>& depends,
                                  const cmCustomCommandLines& commandLines,
                                  const char* comment,
                                  const char* workingDirectory):
   Outputs(outputs),
+  Byproducts(byproducts),
   Depends(depends),
   CommandLines(commandLines),
   HaveComment(comment?true:false),
@@ -99,6 +103,12 @@ const std::vector<std::string>& cmCustomCommand::GetOutputs() const
   return this->Outputs;
 }
 
+//----------------------------------------------------------------------------
+const std::vector<std::string>& cmCustomCommand::GetByproducts() const
+{
+  return this->Byproducts;
+}
+
 //----------------------------------------------------------------------------
 const std::vector<std::string>& cmCustomCommand::GetDepends() const
 {

Source/cmCustomCommand.h

@@ -32,6 +32,7 @@ public:
   /** Main constructor specifies all information for the command.  */
   cmCustomCommand(cmMakefile const* mf,
                   const std::vector<std::string>& outputs,
+                  const std::vector<std::string>& byproducts,
                   const std::vector<std::string>& depends,
                   const cmCustomCommandLines& commandLines,
                   const char* comment,
@@ -42,6 +43,9 @@ public:
   /** Get the output file produced by the command.  */
   const std::vector<std::string>& GetOutputs() const;
 
+  /** Get the extra files produced by the command.  */
+  const std::vector<std::string>& GetByproducts() const;
+
   /** Get the vector that holds the list of dependencies.  */
   const std::vector<std::string>& GetDepends() const;
 
@@ -86,6 +90,7 @@ public:
 
 private:
   std::vector<std::string> Outputs;
+  std::vector<std::string> Byproducts;
   std::vector<std::string> Depends;
   cmCustomCommandLines CommandLines;
   bool HaveComment;

Source/cmCustomCommandGenerator.cxx

@@ -90,6 +90,12 @@ std::vector<std::string> const& cmCustomCommandGenerator::GetOutputs() const
   return this->CC.GetOutputs();
 }
 
+//----------------------------------------------------------------------------
+std::vector<std::string> const& cmCustomCommandGenerator::GetByproducts() const
+{
+  return this->CC.GetByproducts();
+}
+
 //----------------------------------------------------------------------------
 std::vector<std::string> const& cmCustomCommandGenerator::GetDepends() const
 {

Source/cmCustomCommandGenerator.h

@@ -42,6 +42,7 @@ public:
   const char* GetComment() const;
   std::string GetWorkingDirectory() const;
   std::vector<std::string> const& GetOutputs() const;
+  std::vector<std::string> const& GetByproducts() const;
   std::vector<std::string> const& GetDepends() const;
 };
 

Source/cmGlobalGenerator.cxx

@@ -2510,10 +2510,11 @@ cmTarget cmGlobalGenerator::CreateGlobalTarget(
   target.SetProperty("EXCLUDE_FROM_ALL","TRUE");
 
   std::vector<std::string> no_outputs;
+  std::vector<std::string> no_byproducts;
   std::vector<std::string> no_depends;
   // Store the custom command in the target.
-  cmCustomCommand cc(0, no_outputs, no_depends, *commandLines, 0,
-                     workingDirectory);
+  cmCustomCommand cc(0, no_outputs, no_byproducts, no_depends,
+                     *commandLines, 0, workingDirectory);
   cc.SetUsesTerminal(uses_terminal);
   target.AddPostBuildCommand(cc);
   target.SetProperty("EchoString", message);

Source/cmGlobalVisualStudio8Generator.cxx

@@ -341,9 +341,10 @@ bool cmGlobalVisualStudio8Generator::AddCheckTarget()
   // overwritten by the CreateVCProjBuildRule.
   // (this could be avoided with per-target source files)
   std::string no_main_dependency = "";
+  std::vector<std::string> no_byproducts;
   if(cmSourceFile* file =
      mf->AddCustomCommandToOutput(
-       stamps, listFiles,
+       stamps, no_byproducts, listFiles,
        no_main_dependency, commandLines, "Checking Build System",
        no_working_directory, true))
     {

Source/cmGlobalXCodeGenerator.cxx

@@ -477,7 +477,9 @@ cmGlobalXCodeGenerator::AddExtraTargets(cmLocalGenerator* root,
           this->PostBuildMakeTarget(target.GetName(), "$(CONFIGURATION)");
         cmCustomCommandLines commandLines;
         commandLines.push_back(makeHelper);
+        std::vector<std::string> no_byproducts;
         lg->GetMakefile()->AddCustomCommandToTarget(target.GetName(),
+                                                    no_byproducts,
                                                     no_depends,
                                                     commandLines,
                                                     cmTarget::POST_BUILD,
@@ -1366,6 +1368,7 @@ void cmGlobalXCodeGenerator::CreateCustomCommands(cmXCodeObject* buildPhases,
     cmd[0].push_back(str_link_file);
 
     cmCustomCommand command(this->CurrentMakefile,
+      std::vector<std::string>(),
       std::vector<std::string>(),
       std::vector<std::string>(),
       cmd,

Source/cmLocalNinjaGenerator.cxx

@@ -440,10 +440,18 @@ cmLocalNinjaGenerator::WriteCustomCommandBuildStatement(
   cmCustomCommandGenerator ccg(*cc, this->GetConfigName(), this->Makefile);
 
   const std::vector<std::string> &outputs = ccg.GetOutputs();
-  cmNinjaDeps ninjaOutputs(outputs.size()), ninjaDeps;
+  const std::vector<std::string> &byproducts = ccg.GetByproducts();
+  cmNinjaDeps ninjaOutputs(outputs.size()+byproducts.size()), ninjaDeps;
 
+#if 0
+#error TODO: Once CC in an ExternalProject target must provide the \
+    file of each imported target that has an add_dependencies pointing \
+    at us.  How to know which ExternalProject step actually provides it?
+#endif
   std::transform(outputs.begin(), outputs.end(),
                  ninjaOutputs.begin(), MapToNinjaPath());
+  std::transform(byproducts.begin(), byproducts.end(),
+                 ninjaOutputs.begin() + outputs.size(), MapToNinjaPath());
   this->AppendCustomCommandDeps(ccg, ninjaDeps);
 
   for (cmNinjaDeps::iterator i = ninjaOutputs.begin(); i != ninjaOutputs.end();

Source/cmLocalVisualStudio6Generator.cxx

@@ -821,10 +821,12 @@ cmLocalVisualStudio6Generator::MaybeCreateOutputDir(cmTarget& target,
   command.push_back("make_directory");
   command.push_back(outDir);
   std::vector<std::string> no_output;
+  std::vector<std::string> no_byproducts;
   std::vector<std::string> no_depends;
   cmCustomCommandLines commands;
   commands.push_back(command);
-  pcc.reset(new cmCustomCommand(0, no_output, no_depends, commands, 0, 0));
+  pcc.reset(new cmCustomCommand(0, no_output, no_byproducts,
+                                no_depends, commands, 0, 0));
   pcc->SetEscapeOldStyle(false);
   pcc->SetEscapeAllowMakeVars(true);
   return pcc;

Source/cmLocalVisualStudioGenerator.cxx

@@ -95,10 +95,12 @@ cmLocalVisualStudioGenerator::MaybeCreateImplibDir(cmTarget& target,
   command.push_back("make_directory");
   command.push_back(impDir);
   std::vector<std::string> no_output;
+  std::vector<std::string> no_byproducts;
   std::vector<std::string> no_depends;
   cmCustomCommandLines commands;
   commands.push_back(command);
-  pcc.reset(new cmCustomCommand(0, no_output, no_depends, commands, 0, 0));
+  pcc.reset(new cmCustomCommand(0, no_output, no_byproducts,
+                                no_depends, commands, 0, 0));
   pcc->SetEscapeOldStyle(false);
   pcc->SetEscapeAllowMakeVars(true);
   return pcc;

Source/cmMakefile.cxx

@@ -880,13 +880,14 @@ void cmMakefile::ConfigureFinalPass()
 //----------------------------------------------------------------------------
 void
 cmMakefile::AddCustomCommandToTarget(const std::string& target,
+                                   const std::vector<std::string>& byproducts,
                                      const std::vector<std::string>& depends,
                                      const cmCustomCommandLines& commandLines,
                                      cmTarget::CustomCommandType type,
                                      const char* comment,
                                      const char* workingDir,
                                      bool escapeOldStyle,
-                                     bool uses_terminal) const
+                                     bool uses_terminal)
 {
   // Find the target to which to add the custom command.
   cmTargets::iterator ti = this->Targets.find(target);
@@ -936,9 +937,20 @@ cmMakefile::AddCustomCommandToTarget(const std::string& target,
     this->IssueMessage(cmake::FATAL_ERROR, e.str());
     return;
     }
+
+  // Always create the byproduct sources and mark them generated.
+  for(std::vector<std::string>::const_iterator o = byproducts.begin();
+      o != byproducts.end(); ++o)
+    {
+    if(cmSourceFile* out = this->GetOrCreateSource(*o, true))
+      {
+      out->SetProperty("GENERATED", "1");
+      }
+    }
+
   // Add the command to the appropriate build step for the target.
   std::vector<std::string> no_output;
-  cmCustomCommand cc(this, no_output, depends,
+  cmCustomCommand cc(this, no_output, byproducts, depends,
                      commandLines, comment, workingDir);
   cc.SetEscapeOldStyle(escapeOldStyle);
   cc.SetEscapeAllowMakeVars(true);
@@ -960,6 +972,7 @@ cmMakefile::AddCustomCommandToTarget(const std::string& target,
 //----------------------------------------------------------------------------
 cmSourceFile*
 cmMakefile::AddCustomCommandToOutput(const std::vector<std::string>& outputs,
+                                  const std::vector<std::string>& byproducts,
                                      const std::vector<std::string>& depends,
                                      const std::string& main_dependency,
                                      const cmCustomCommandLines& commandLines,
@@ -1058,6 +1071,14 @@ cmMakefile::AddCustomCommandToOutput(const std::vector<std::string>& outputs,
       out->SetProperty("GENERATED", "1");
       }
     }
+  for(std::vector<std::string>::const_iterator o = byproducts.begin();
+      o != byproducts.end(); ++o)
+    {
+    if(cmSourceFile* out = this->GetOrCreateSource(*o, true))
+      {
+      out->SetProperty("GENERATED", "1");
+      }
+    }
 
   // Attach the custom command to the file.
   if(file)
@@ -1070,8 +1091,8 @@ cmMakefile::AddCustomCommandToOutput(const std::vector<std::string>& outputs,
       }
 
     cmCustomCommand* cc =
-      new cmCustomCommand(this, outputs, depends2, commandLines,
-                          comment, workingDir);
+      new cmCustomCommand(this, outputs, byproducts, depends2,
+                          commandLines, comment, workingDir);
     cc->SetEscapeOldStyle(escapeOldStyle);
     cc->SetEscapeAllowMakeVars(true);
     cc->SetUsesTerminal(uses_terminal);
@@ -1128,7 +1149,9 @@ cmMakefile::AddCustomCommandToOutput(const std::string& output,
 {
   std::vector<std::string> outputs;
   outputs.push_back(output);
-  return this->AddCustomCommandToOutput(outputs, depends, main_dependency,
+  std::vector<std::string> no_byproducts;
+  return this->AddCustomCommandToOutput(outputs, no_byproducts,
+                                        depends, main_dependency,
                                         commandLines, comment, workingDir,
                                         replace, escapeOldStyle,
                                         uses_terminal);
@@ -1150,7 +1173,9 @@ cmMakefile::AddCustomCommandOldStyle(const std::string& target,
     // In the old-style signature if the source and target were the
     // same then it added a post-build rule to the target.  Preserve
     // this behavior.
-    this->AddCustomCommandToTarget(target, depends, commandLines,
+    std::vector<std::string> no_byproducts;
+    this->AddCustomCommandToTarget(target, no_byproducts,
+                                   depends, commandLines,
                                    cmTarget::POST_BUILD, comment, 0);
     return;
     }
@@ -1250,6 +1275,23 @@ cmMakefile::AddUtilityCommand(const std::string& utilityName,
                               const cmCustomCommandLines& commandLines,
                               bool escapeOldStyle, const char* comment,
                               bool uses_terminal)
+{
+  std::vector<std::string> no_byproducts;
+  return this->AddUtilityCommand(utilityName, excludeFromAll, workingDirectory,
+                                 no_byproducts, depends, commandLines,
+                                 escapeOldStyle, comment, uses_terminal);
+}
+
+//----------------------------------------------------------------------------
+cmTarget*
+cmMakefile::AddUtilityCommand(const std::string& utilityName,
+                              bool excludeFromAll,
+                              const char* workingDirectory,
+                              const std::vector<std::string>& byproducts,
+                              const std::vector<std::string>& depends,
+                              const cmCustomCommandLines& commandLines,
+                              bool escapeOldStyle, const char* comment,
+                              bool uses_terminal)
 {
   // Create a target instance for this utility.
   cmTarget* target = this->AddNewTarget(cmTarget::UTILITY, utilityName);
@@ -1270,10 +1312,12 @@ cmMakefile::AddUtilityCommand(const std::string& utilityName,
     force += cmake::GetCMakeFilesDirectory();
     force += "/";
     force += utilityName;
+    std::vector<std::string> forced;
+    forced.push_back(force);
     std::string no_main_dependency = "";
     bool no_replace = false;
-    this->AddCustomCommandToOutput(force, depends,
-                                   no_main_dependency,
+    this->AddCustomCommandToOutput(forced, byproducts,
+                                   depends, no_main_dependency,
                                    commandLines, comment,
                                    workingDirectory, no_replace,
                                    escapeOldStyle, uses_terminal);
@@ -1289,6 +1333,16 @@ cmMakefile::AddUtilityCommand(const std::string& utilityName,
       cmSystemTools::Error("Could not get source file entry for ",
                           force.c_str());
       }
+
+    // Always create the byproduct sources and mark them generated.
+    for(std::vector<std::string>::const_iterator o = byproducts.begin();
+        o != byproducts.end(); ++o)
+      {
+      if(cmSourceFile* out = this->GetOrCreateSource(*o, true))
+        {
+        out->SetProperty("GENERATED", "1");
+        }
+      }
     }
   return target;
 }

Source/cmMakefile.h

@@ -170,14 +170,16 @@ public:
 
   /** Add a custom command to the build.  */
   void AddCustomCommandToTarget(const std::string& target,
+                                const std::vector<std::string>& byproducts,
                                 const std::vector<std::string>& depends,
                                 const cmCustomCommandLines& commandLines,
                                 cmTarget::CustomCommandType type,
                                 const char* comment, const char* workingDir,
                                 bool escapeOldStyle = true,
-                                bool uses_terminal = false) const;
+                                bool uses_terminal = false);
   cmSourceFile* AddCustomCommandToOutput(
     const std::vector<std::string>& outputs,
+    const std::vector<std::string>& byproducts,
     const std::vector<std::string>& depends,
     const std::string& main_dependency,
     const cmCustomCommandLines& commandLines,
@@ -242,6 +244,15 @@ public:
                               bool escapeOldStyle = true,
                               const char* comment = 0,
                               bool uses_terminal = false);
+  cmTarget* AddUtilityCommand(const std::string& utilityName,
+                              bool excludeFromAll,
+                              const char* workingDirectory,
+                              const std::vector<std::string>& byproducts,
+                              const std::vector<std::string>& depends,
+                              const cmCustomCommandLines& commandLines,
+                              bool escapeOldStyle = true,
+                              const char* comment = 0,
+                              bool uses_terminal = false);
 
   /**
    * Add a link library to the build.

Source/cmNinjaNormalTargetGenerator.cxx

@@ -256,7 +256,7 @@ cmNinjaNormalTargetGenerator
                                         /*deptype*/ "",
                                         rspfile,
                                         rspcontent,
-                                        /*restat*/ "",
+                                        /*restat*/ "$RESTAT",
                                         /*generator*/ false);
   }
 
@@ -556,6 +556,7 @@ void cmNinjaNormalTargetGenerator::WriteLinkStatement()
     &postBuildCmdLines
   };
 
+  cmNinjaDeps byproducts;
   for (unsigned i = 0; i != 3; ++i)
     {
     for (std::vector<cmCustomCommand>::const_iterator
@@ -564,6 +565,9 @@ void cmNinjaNormalTargetGenerator::WriteLinkStatement()
       {
       cmCustomCommandGenerator ccg(*ci, cfgName, mf);
       localGen.AppendCustomCommandLines(ccg, *cmdLineLists[i]);
+      std::vector<std::string> const& ccByproducts = ccg.GetByproducts();
+      std::transform(ccByproducts.begin(), ccByproducts.end(),
+                     std::back_inserter(byproducts), MapToNinjaPath());
       }
     }
 
@@ -611,6 +615,17 @@ void cmNinjaNormalTargetGenerator::WriteLinkStatement()
   this->GetLocalGenerator()->AppendTargetDepends(this->GetTarget(),
     orderOnlyDeps);
 
+  // Ninja should restat after linking if and only if there are byproducts.
+  vars["RESTAT"] = byproducts.empty()? "" : "1";
+
+  for (cmNinjaDeps::const_iterator oi = byproducts.begin(),
+         oe = byproducts.end();
+       oi != oe; ++oi)
+    {
+    this->GetGlobalGenerator()->SeenCustomCommandOutput(*oi);
+    outputs.push_back(*oi);
+    }
+
   // Write the build statement for this target.
   globalGen.WriteBuild(this->GetBuildFileStream(),
                         comment.str(),

Source/cmNinjaTargetGenerator.cxx

@@ -538,8 +538,11 @@ cmNinjaTargetGenerator
     cmCustomCommandGenerator ccg(*cc, this->GetConfigName(),
                                  this->GetMakefile());
     const std::vector<std::string>& ccoutputs = ccg.GetOutputs();
+    const std::vector<std::string>& ccbyproducts= ccg.GetByproducts();
     std::transform(ccoutputs.begin(), ccoutputs.end(),
                    std::back_inserter(orderOnlyDeps), MapToNinjaPath());
+    std::transform(ccbyproducts.begin(), ccbyproducts.end(),
+                   std::back_inserter(orderOnlyDeps), MapToNinjaPath());
     }
 
   if (!orderOnlyDeps.empty())

Source/cmNinjaUtilityTargetGenerator.cxx

@@ -27,8 +27,11 @@ cmNinjaUtilityTargetGenerator::~cmNinjaUtilityTargetGenerator() {}
 
 void cmNinjaUtilityTargetGenerator::Generate()
 {
+  std::string utilCommandName = cmake::GetCMakeFilesDirectoryPostSlash();
+  utilCommandName += this->GetTargetName() + ".util";
+
   std::vector<std::string> commands;
-  cmNinjaDeps deps, outputs;
+  cmNinjaDeps deps, outputs, util_outputs(1, utilCommandName);
 
   const std::vector<cmCustomCommand> *cmdLists[2] = {
     &this->GetTarget()->GetPreBuildCommands(),
@@ -44,6 +47,9 @@ void cmNinjaUtilityTargetGenerator::Generate()
                                    this->GetMakefile());
       this->GetLocalGenerator()->AppendCustomCommandDeps(ccg, deps);
       this->GetLocalGenerator()->AppendCustomCommandLines(ccg, commands);
+      std::vector<std::string> const& ccByproducts = ccg.GetByproducts();
+      std::transform(ccByproducts.begin(), ccByproducts.end(),
+                     std::back_inserter(util_outputs), MapToNinjaPath());
       if (ci->GetUsesTerminal())
         uses_terminal = true;
     }
@@ -64,8 +70,11 @@ void cmNinjaUtilityTargetGenerator::Generate()
 
       // Depend on all custom command outputs.
       const std::vector<std::string>& ccOutputs = ccg.GetOutputs();
+      const std::vector<std::string>& ccByproducts = ccg.GetByproducts();
       std::transform(ccOutputs.begin(), ccOutputs.end(),
                      std::back_inserter(deps), MapToNinjaPath());
+      std::transform(ccByproducts.begin(), ccByproducts.end(),
+                     std::back_inserter(deps), MapToNinjaPath());
       }
     }
 
@@ -107,15 +116,19 @@ void cmNinjaUtilityTargetGenerator::Generate()
     if (command.find('$') != std::string::npos)
       return;
 
-    std::string utilCommandName = cmake::GetCMakeFilesDirectoryPostSlash();
-    utilCommandName += this->GetTargetName() + ".util";
+    for (cmNinjaDeps::const_iterator
+           oi = util_outputs.begin(), oe = util_outputs.end();
+         oi != oe; ++oi)
+      {
+      this->GetGlobalGenerator()->SeenCustomCommandOutput(*oi);
+      }
 
     this->GetGlobalGenerator()->WriteCustomCommandBuild(
       command,
       desc,
       "Utility command for " + this->GetTargetName(),
       uses_terminal,
-      cmNinjaDeps(1, utilCommandName),
+      util_outputs,
       deps);
 
     this->GetGlobalGenerator()->WritePhonyBuild(this->GetBuildFileStream(),

Source/cmQtAutoGenerators.cxx

@@ -438,7 +438,8 @@ bool cmQtAutoGenerators::InitializeAutogenTarget(cmTarget* target)
     // rejection in cmMakefile::AddCustomCommandToTarget because we know
     // PRE_BUILD will work for an OBJECT_LIBRARY in this specific case.
     std::vector<std::string> no_output;
-    cmCustomCommand cc(makefile, no_output, depends,
+    std::vector<std::string> no_byproducts;
+    cmCustomCommand cc(makefile, no_output, no_byproducts, depends,
                        commandLines, autogenComment.c_str(),
                        workingDirectory.c_str());
     cc.SetEscapeOldStyle(false);
@@ -451,7 +452,9 @@ bool cmQtAutoGenerators::InitializeAutogenTarget(cmTarget* target)
     cmTarget* autogenTarget = 0;
     if (!rcc_output.empty())
       {
-      makefile->AddCustomCommandToOutput(rcc_output, depends, "",
+      std::vector<std::string> no_byproducts;
+      makefile->AddCustomCommandToOutput(rcc_output, no_byproducts,
+                                         depends, "",
                                          commandLines, 0,
                                          workingDirectory.c_str(),
                                          false, false);

Tests/CMakeLists.txt

@@ -716,6 +716,8 @@ ${CMake_BINARY_DIR}/bin/cmake -DDIR=dev -P ${CMake_SOURCE_DIR}/Utilities/Release
     )
   list(APPEND TEST_BUILD_DIRS "${CMake_BINARY_DIR}/Tests/CustomCommand")
 
+  ADD_TEST_MACRO(CustomCommandByproducts CustomCommandByproducts)
+
   ADD_TEST_MACRO(EmptyDepends ${CMAKE_CTEST_COMMAND})
 
   add_test(CustomCommandWorkingDirectory  ${CMAKE_CTEST_COMMAND}

Tests/CustomCommand/CMakeLists.txt

@@ -117,6 +117,7 @@ add_custom_command(
   COMMAND ${CMAKE_COMMAND} -E echo " Copying doc1pre.txt to doc2post.txt."
   COMMAND ${CMAKE_COMMAND} -E copy ${PROJECT_BINARY_DIR}/doc1pre.txt
                                    ${PROJECT_BINARY_DIR}/doc2post.txt
+  BYPRODUCTS ${PROJECT_BINARY_DIR}/doc2post.txt
   COMMENT "Running TDocument post-build commands"
   )
 

Tests/CustomCommandByproducts/CMakeLists.txt

@@ -0,0 +1,103 @@
+cmake_minimum_required(VERSION 3.1)
+project(CustomCommandByproducts C)
+
+# Generate a byproduct in a rule that runs in the target consuming it.
+add_custom_command(
+  OUTPUT timestamp1.txt
+  COMMAND ${CMAKE_COMMAND} -E copy_if_different
+    ${CMAKE_CURRENT_SOURCE_DIR}/byproduct1.c.in byproduct1.c
+  BYPRODUCTS byproduct1.c
+  COMMAND ${CMAKE_COMMAND} -E touch timestamp1.txt
+  DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/byproduct1.c.in
+  )
+
+# Generate a byproduct in a rule that runs in a dependency of the consumer.
+add_custom_command(
+  OUTPUT timestamp2.txt
+  COMMAND ${CMAKE_COMMAND} -E copy_if_different
+    ${CMAKE_CURRENT_SOURCE_DIR}/byproduct2.c.in byproduct2.c
+  BYPRODUCTS byproduct2.c
+  COMMAND ${CMAKE_COMMAND} -E touch timestamp2.txt
+  DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/byproduct2.c.in
+  )
+add_custom_target(Producer2 DEPENDS timestamp2.txt)
+
+# Generate a byproduct in a custom target.
+add_custom_target(Producer3_4
+  COMMAND ${CMAKE_COMMAND} -E copy_if_different
+    ${CMAKE_CURRENT_SOURCE_DIR}/byproduct3.c.in byproduct3.c
+  BYPRODUCTS byproduct3.c
+  )
+
+# Generate a byproduct in a custom target POST_BUILD command.
+add_custom_command(
+  TARGET Producer3_4 POST_BUILD
+  COMMAND ${CMAKE_COMMAND} -E copy_if_different
+    ${CMAKE_CURRENT_SOURCE_DIR}/byproduct4.c.in byproduct4.c
+  BYPRODUCTS byproduct4.c
+  )
+
+add_executable(ProducerExe ProducerExe.c)
+
+# Generate a byproduct in an executable POST_BUILD command.
+add_custom_command(
+  TARGET ProducerExe POST_BUILD
+  COMMAND ${CMAKE_COMMAND} -E copy_if_different
+    ${CMAKE_CURRENT_SOURCE_DIR}/byproduct5.c.in byproduct5.c
+  BYPRODUCTS byproduct5.c
+  )
+
+# Generate a byproduct in an executable PRE_LINK command.
+add_custom_command(
+  TARGET ProducerExe PRE_LINK
+  COMMAND ${CMAKE_COMMAND} -E copy_if_different
+    ${CMAKE_CURRENT_SOURCE_DIR}/byproduct6.c.in byproduct6.c
+  BYPRODUCTS byproduct6.c
+  )
+
+# Generate a byproduct in an executable PRE_BUILD command.
+add_custom_command(
+  TARGET ProducerExe PRE_BUILD
+  COMMAND ${CMAKE_COMMAND} -E copy_if_different
+    ${CMAKE_CURRENT_SOURCE_DIR}/byproduct7.c.in byproduct7.c
+  BYPRODUCTS byproduct7.c
+  )
+
+# Generate a byproduct in a custom command that consumes other byproducts.
+add_custom_command(OUTPUT timestamp8.txt
+  COMMAND ${CMAKE_COMMAND} -E copy_if_different
+    ${CMAKE_CURRENT_SOURCE_DIR}/byproduct8.c.in byproduct8.c
+  COMMAND ${CMAKE_COMMAND} -E touch timestamp8.txt
+  BYPRODUCTS byproduct8.c
+  DEPENDS
+    ${CMAKE_CURRENT_BINARY_DIR}/byproduct2.c
+    ${CMAKE_CURRENT_BINARY_DIR}/byproduct3.c
+    ${CMAKE_CURRENT_BINARY_DIR}/byproduct4.c
+    ${CMAKE_CURRENT_BINARY_DIR}/byproduct5.c
+    ${CMAKE_CURRENT_BINARY_DIR}/byproduct6.c
+    ${CMAKE_CURRENT_BINARY_DIR}/byproduct7.c
+    ${CMAKE_CURRENT_SOURCE_DIR}/byproduct8.c.in
+  )
+
+# Add an executable consuming all the byproducts.
+add_executable(CustomCommandByproducts
+  CustomCommandByproducts.c
+  byproduct1.c timestamp1.txt
+  byproduct2.c
+  byproduct3.c
+  byproduct4.c
+  byproduct5.c
+  byproduct6.c
+  byproduct7.c
+  byproduct8.c timestamp8.txt
+  )
+add_dependencies(CustomCommandByproducts Producer2)
+add_dependencies(CustomCommandByproducts Producer3_4)
+add_dependencies(CustomCommandByproducts ProducerExe)
+
+if(CMAKE_GENERATOR STREQUAL "Ninja")
+  add_custom_target(CheckNinja ALL
+    COMMENT "Checking build.ninja"
+    COMMAND ${CMAKE_COMMAND} -P ${CMAKE_CURRENT_SOURCE_DIR}/ninja-check.cmake
+    )
+endif()

Tests/CustomCommandByproducts/CustomCommandByproducts.c

@@ -0,0 +1,21 @@
+extern int byproduct1(void);
+extern int byproduct2(void);
+extern int byproduct3(void);
+extern int byproduct4(void);
+extern int byproduct5(void);
+extern int byproduct6(void);
+extern int byproduct7(void);
+extern int byproduct8(void);
+int main(void)
+{
+  return (
+    byproduct1() +
+    byproduct2() +
+    byproduct3() +
+    byproduct4() +
+    byproduct5() +
+    byproduct6() +
+    byproduct7() +
+    byproduct8() +
+    0);
+}

Tests/CustomCommandByproducts/ProducerExe.c

@@ -0,0 +1 @@
+int main(void) { return 0; }

Tests/CustomCommandByproducts/byproduct1.c.in

@@ -0,0 +1 @@
+int byproduct1(void) { return 0; }

Tests/CustomCommandByproducts/byproduct2.c.in

@@ -0,0 +1 @@
+int byproduct2(void) { return 0; }

Tests/CustomCommandByproducts/byproduct3.c.in

@@ -0,0 +1 @@
+int byproduct3(void) { return 0; }

Tests/CustomCommandByproducts/byproduct4.c.in

@@ -0,0 +1 @@
+int byproduct4(void) { return 0; }

Tests/CustomCommandByproducts/byproduct5.c.in

@@ -0,0 +1 @@
+int byproduct5(void) { return 0; }

Tests/CustomCommandByproducts/byproduct6.c.in

@@ -0,0 +1 @@
+int byproduct6(void) { return 0; }

Tests/CustomCommandByproducts/byproduct7.c.in

@@ -0,0 +1 @@
+int byproduct7(void) { return 0; }

Tests/CustomCommandByproducts/byproduct8.c.in

@@ -0,0 +1 @@
+int byproduct8(void) { return 0; }

Tests/CustomCommandByproducts/ninja-check.cmake

@@ -0,0 +1,20 @@
+file(READ build.ninja build_ninja)
+if("${build_ninja}" MATCHES [====[
+# Unknown Build Time Dependencies.
+# Tell Ninja that they may appear as side effects of build rules
+# otherwise ordered by order-only dependencies.
+
+((build [^:]*: phony [^\n]*
+)*)# ========]====])
+  set(phony "${CMAKE_MATCH_1}")
+  if(NOT phony)
+    message(STATUS "build.ninja correctly does not have extra phony rules")
+  else()
+    string(REGEX REPLACE "\n+$" "" phony "${phony}")
+    string(REGEX REPLACE "\n" "\n  " phony "  ${phony}")
+    message(FATAL_ERROR "build.ninja incorrectly has extra phony rules:\n"
+      "${phony}")
+  endif()
+else()
+  message(FATAL_ERROR "build.ninja is incorrectly missing expected block")
+endif()

Tests/RunCMake/add_custom_command/RunCMakeTest.cmake

@@ -6,4 +6,5 @@ run_cmake(BadArgument)
 run_cmake(NoArguments)
 run_cmake(NoOutputOrTarget)
 run_cmake(OutputAndTarget)
+run_cmake(SourceByproducts)
 run_cmake(SourceUsesTerminal)

Tests/RunCMake/add_custom_command/SourceByproducts-result.txt

@@ -0,0 +1 @@
+1

Tests/RunCMake/add_custom_command/SourceByproducts-stderr.txt

@@ -0,0 +1,4 @@
+CMake Error at SourceByproducts.cmake:1 \(add_custom_command\):
+  add_custom_command BYPRODUCTS may not be specified with SOURCE signatures
+Call Stack \(most recent call first\):
+  CMakeLists.txt:3 \(include\)

Tests/RunCMake/add_custom_command/SourceByproducts.cmake

@@ -0,0 +1 @@
+add_custom_command(SOURCE t TARGET t BYPRODUCTS b)

Tests/RunCMake/add_custom_target/ByproductsNoCommand-result.txt

@@ -0,0 +1 @@
+1

Tests/RunCMake/add_custom_target/ByproductsNoCommand-stderr.txt

@@ -0,0 +1,4 @@
+CMake Error at ByproductsNoCommand.cmake:1 \(add_custom_target\):
+  BYPRODUCTS may not be specified without any COMMAND
+Call Stack \(most recent call first\):
+  CMakeLists.txt:3 \(include\)

Tests/RunCMake/add_custom_target/ByproductsNoCommand.cmake

@@ -0,0 +1 @@
+add_custom_target(MyTarget BYPRODUCTS a b c d)

Tests/RunCMake/add_custom_target/RunCMakeTest.cmake

@@ -2,4 +2,5 @@ include(RunCMake)
 
 run_cmake(NoArguments)
 run_cmake(BadTargetName)
+run_cmake(ByproductsNoCommand)
 run_cmake(UsesTerminalNoCommand)