ninja: Add experimental infrastructure for C++20 module dependency scanning

Optionally enable this infrastructure through an undocumented
`CMAKE_EXPERIMENTAL_CXX_MODULE_DYNDEP` variable.  Currently this is
experimental and intended for use by compiler writers to implement their
scanning tools.  Warn as such when the feature is activated.  Later when
compilers provide the needed scanning tools we can enable this variable
from our corresponding compiler information modules.  It is never meant
to be set by project code.

When enabled, generate a build graph similar to what we use for Fortran
module dependencies.  There are some differences needed because we can
scan dependencies without explicit preprocessing, and can directly
compile the original source afterward.

Co-Author: Ben Boeckel <[email protected]>

Help/dev/experimental.rst

@@ -7,4 +7,44 @@ See documentation on `CMake Development`_ for more information.
 
 .. _`CMake Development`: README.rst
 
-No experimental features are under development in this version of CMake.
+C++20 Module Dependencies
+=========================
+
+The Ninja generator has experimental infrastructure supporting C++20 module
+dependency scanning.  This is similar to the Fortran modules support, but
+relies on external tools to scan C++20 translation units for module
+dependencies.  The approach is described by Kitware's `D1483r1`_ paper.
+
+The ``CMAKE_EXPERIMENTAL_CXX_MODULE_DYNDEP`` variable can be set to ``1``
+in order to activate this undocumented experimental infrastructure.  This
+is **intended to make the functionality available to compiler writers** so
+they can use it to develop and test their dependency scanning tool.
+The ``CMAKE_EXPERIMENTAL_CXX_SCANDEP_SOURCE`` variable must also be set
+to tell CMake how to invoke the C++20 module dependency scanning tool.
+
+For example, add code like the following to a test project:
+
+.. code-block:: cmake
+
+  set(CMAKE_EXPERIMENTAL_CXX_MODULE_DYNDEP 1)
+  string(CONCAT CMAKE_EXPERIMENTAL_CXX_SCANDEP_SOURCE
+    "<CMAKE_CXX_COMPILER> <DEFINES> <INCLUDES> <FLAGS> <SOURCE>"
+    " -MT <DYNDEP_FILE> -MD -MF <DEP_FILE>"
+    " ${flags_to_scan_deps} -fdep-file=<DYNDEP_FILE> -fdep-output=<OBJECT>"
+    )
+
+The tool specified by ``CMAKE_EXPERIMENTAL_CXX_SCANDEP_SOURCE`` is
+expected to process the translation unit, write preprocessor dependencies
+to the file specified by the ``<DEP_FILE>`` placeholder, and write module
+dependencies to the file specified by the ``<DYNDEP_FILE>`` placeholder.
+
+The module dependencies should be written in the format described
+by the `P1689r3`_ paper.
+
+Compiler writers may try out their scanning functionality using
+the `cxx-modules-sandbox`_ test project, modified to set variables
+as above for their compiler.
+
+.. _`D1483r1`: https://mathstuf.fedorapeople.org/fortran-modules/fortran-modules.html
+.. _`P1689r3`: http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2020/p1689r3.html
+.. _`cxx-modules-sandbox`: https://github.com/mathstuf/cxx-modules-sandbox

Source/cmGlobalNinjaGenerator.cxx

@@ -555,6 +555,7 @@ void cmGlobalNinjaGenerator::Generate()
   this->TargetAll = this->NinjaOutputPath("all");
   this->CMakeCacheFile = this->NinjaOutputPath("CMakeCache.txt");
   this->DisableCleandead = false;
+  this->DiagnosedCxxModuleSupport = false;
 
   this->PolicyCMP0058 =
     this->LocalGenerators[0]->GetMakefile()->GetPolicyStatus(
@@ -755,6 +756,37 @@ bool cmGlobalNinjaGenerator::CheckLanguages(
   return true;
 }
 
+bool cmGlobalNinjaGenerator::CheckCxxModuleSupport()
+{
+  bool const diagnose = !this->DiagnosedCxxModuleSupport &&
+    !this->CMakeInstance->GetIsInTryCompile();
+  if (diagnose) {
+    this->DiagnosedCxxModuleSupport = true;
+    this->GetCMakeInstance()->IssueMessage(
+      MessageType::AUTHOR_WARNING,
+      "C++20 modules support via CMAKE_EXPERIMENTAL_CXX_MODULE_DYNDEP "
+      "is experimental.  It is meant only for compiler developers to try.");
+  }
+  if (this->NinjaSupportsDyndeps) {
+    return true;
+  }
+  if (diagnose) {
+    std::ostringstream e;
+    /* clang-format off */
+    e <<
+      "The Ninja generator does not support C++20 modules "
+      "using Ninja version \n"
+      "  " << this->NinjaVersion << "\n"
+      "due to lack of required features.  "
+      "Ninja " << RequiredNinjaVersionForDyndeps() << " or higher is required."
+      ;
+    /* clang-format on */
+    this->GetCMakeInstance()->IssueMessage(MessageType::FATAL_ERROR, e.str());
+    cmSystemTools::SetFatalErrorOccured();
+  }
+  return false;
+}
+
 bool cmGlobalNinjaGenerator::CheckFortran(cmMakefile* mf) const
 {
   if (this->NinjaSupportsDyndeps) {

Source/cmGlobalNinjaGenerator.h

@@ -445,6 +445,8 @@ public:
 
   bool IsSingleConfigUtility(cmGeneratorTarget const* target) const;
 
+  bool CheckCxxModuleSupport();
+
 protected:
   void Generate() override;
 
@@ -565,6 +567,8 @@ private:
   bool NinjaSupportsMultipleOutputs = false;
   bool NinjaSupportsMetadataOnRegeneration = false;
 
+  bool DiagnosedCxxModuleSupport = false;
+
 private:
   void InitOutputPathPrefix();
 

Source/cmNinjaTargetGenerator.cxx

@@ -35,6 +35,7 @@
 #include "cmRange.h"
 #include "cmRulePlaceholderExpander.h"
 #include "cmSourceFile.h"
+#include "cmStandardLevelResolver.h"
 #include "cmState.h"
 #include "cmStateTypes.h"
 #include "cmStringAlgorithms.h"
@@ -146,9 +147,26 @@ std::string cmNinjaTargetGenerator::LanguageDyndepRule(
     '_', config);
 }
 
-bool cmNinjaTargetGenerator::NeedDyndep(std::string const& lang) const
+bool cmNinjaTargetGenerator::NeedCxxModuleSupport(
+  std::string const& lang, std::string const& config) const
 {
-  return lang == "Fortran";
+  if (lang != "CXX") {
+    return false;
+  }
+  if (!this->Makefile->IsOn("CMAKE_EXPERIMENTAL_CXX_MODULE_DYNDEP")) {
+    return false;
+  }
+  cmGeneratorTarget const* tgt = this->GetGeneratorTarget();
+  cmStandardLevelResolver standardResolver(this->Makefile);
+  bool const uses_cxx20 =
+    standardResolver.HaveStandardAvailable(tgt, "CXX", config, "cxx_std_20");
+  return uses_cxx20 && this->GetGlobalGenerator()->CheckCxxModuleSupport();
+}
+
+bool cmNinjaTargetGenerator::NeedDyndep(std::string const& lang,
+                                        std::string const& config) const
+{
+  return lang == "Fortran" || this->NeedCxxModuleSupport(lang, config);
 }
 
 std::string cmNinjaTargetGenerator::OrderDependsTargetForTarget(
@@ -531,6 +549,7 @@ cmNinjaRule GetScanRule(
   scanVars.Language = vars.Language;
   scanVars.Object = "$OBJ_FILE";
   scanVars.PreprocessedSource = "$out";
+  scanVars.DynDepFile = "$DYNDEP_INTERMEDIATE_FILE";
   scanVars.DependencyFile = rule.DepFile.c_str();
   scanVars.DependencyTarget = "$out";
 
@@ -585,7 +604,7 @@ void cmNinjaTargetGenerator::WriteCompileRule(const std::string& lang,
   cmMakefile* mf = this->GetMakefile();
 
   // For some cases we scan to dynamically discover dependencies.
-  bool const needDyndep = this->NeedDyndep(lang);
+  bool const needDyndep = this->NeedDyndep(lang, config);
   bool const compilationPreprocesses = !this->NeedExplicitPreprocessing(lang);
 
   std::string flags = "$FLAGS";
@@ -623,16 +642,26 @@ void cmNinjaTargetGenerator::WriteCompileRule(const std::string& lang,
     // Rule to scan dependencies of sources that need preprocessing.
     {
       std::vector<std::string> scanCommands;
-      std::string const& scanRuleName =
-        this->LanguagePreprocessAndScanRule(lang, config);
-      std::string const& ppCommmand = mf->GetRequiredDefinition(
-        cmStrCat("CMAKE_", lang, "_PREPROCESS_SOURCE"));
-      cmExpandList(ppCommmand, scanCommands);
-      for (std::string& i : scanCommands) {
-        i = cmStrCat(launcher, i);
+      std::string scanRuleName;
+      if (compilationPreprocesses) {
+        scanRuleName = this->LanguageScanRule(lang, config);
+        std::string const& scanCommand = mf->GetRequiredDefinition(
+          cmStrCat("CMAKE_EXPERIMENTAL_", lang, "_SCANDEP_SOURCE"));
+        cmExpandList(scanCommand, scanCommands);
+        for (std::string& i : scanCommands) {
+          i = cmStrCat(launcher, i);
+        }
+      } else {
+        scanRuleName = this->LanguagePreprocessAndScanRule(lang, config);
+        std::string const& ppCommmand = mf->GetRequiredDefinition(
+          cmStrCat("CMAKE_", lang, "_PREPROCESS_SOURCE"));
+        cmExpandList(ppCommmand, scanCommands);
+        for (std::string& i : scanCommands) {
+          i = cmStrCat(launcher, i);
+        }
+        scanCommands.emplace_back(GetScanCommand(cmakeCmd, tdi, lang, "$out",
+                                                 "$DYNDEP_INTERMEDIATE_FILE"));
       }
-      scanCommands.emplace_back(GetScanCommand(cmakeCmd, tdi, lang, "$out",
-                                               "$DYNDEP_INTERMEDIATE_FILE"));
 
       auto scanRule = GetScanRule(
         scanRuleName, vars, responseFlag, flags, rulePlaceholderExpander.get(),
@@ -640,12 +669,18 @@ void cmNinjaTargetGenerator::WriteCompileRule(const std::string& lang,
 
       scanRule.Comment =
         cmStrCat("Rule for generating ", lang, " dependencies.");
-      scanRule.Description = cmStrCat("Building ", lang, " preprocessed $out");
+      if (compilationPreprocesses) {
+        scanRule.Description =
+          cmStrCat("Scanning $in for ", lang, " dependencies");
+      } else {
+        scanRule.Description =
+          cmStrCat("Building ", lang, " preprocessed $out");
+      }
 
       this->GetGlobalGenerator()->AddRule(scanRule);
     }
 
-    {
+    if (!compilationPreprocesses) {
       // Compilation will not preprocess, so it does not need the defines
       // unless the compiler wants them for some other purpose.
       if (!this->CompileWithDefines(lang)) {
@@ -1258,7 +1293,7 @@ void cmNinjaTargetGenerator::WriteObjectBuildStatement(
   }
 
   // For some cases we scan to dynamically discover dependencies.
-  bool const needDyndep = this->NeedDyndep(language);
+  bool const needDyndep = this->NeedDyndep(language, config);
   bool const compilationPreprocesses =
     !this->NeedExplicitPreprocessing(language);
 
@@ -1440,17 +1475,26 @@ void cmNinjaTargetGenerator::WriteTargetDependInfo(std::string const& lang,
   tdi["compiler-id"] = this->Makefile->GetSafeDefinition(
     cmStrCat("CMAKE_", lang, "_COMPILER_ID"));
 
+  std::string mod_dir;
   if (lang == "Fortran") {
-    std::string mod_dir = this->GeneratorTarget->GetFortranModuleDirectory(
+    mod_dir = this->GeneratorTarget->GetFortranModuleDirectory(
       this->Makefile->GetHomeOutputDirectory());
-    if (mod_dir.empty()) {
-      mod_dir = this->Makefile->GetCurrentBinaryDirectory();
-    }
-    tdi["module-dir"] = mod_dir;
+  } else if (lang == "CXX") {
+    mod_dir =
+      cmSystemTools::CollapseFullPath(this->GeneratorTarget->ObjectDirectory);
+  }
+  if (mod_dir.empty()) {
+    mod_dir = this->Makefile->GetCurrentBinaryDirectory();
+  }
+  tdi["module-dir"] = mod_dir;
+
+  if (lang == "Fortran") {
     tdi["submodule-sep"] =
       this->Makefile->GetSafeDefinition("CMAKE_Fortran_SUBMODULE_SEP");
     tdi["submodule-ext"] =
       this->Makefile->GetSafeDefinition("CMAKE_Fortran_SUBMODULE_EXT");
+  } else if (lang == "CXX") {
+    // No extra information necessary.
   }
 
   tdi["dir-cur-bld"] = this->Makefile->GetCurrentBinaryDirectory();

Source/cmNinjaTargetGenerator.h

@@ -71,9 +71,11 @@ protected:
                                const std::string& config) const;
   std::string LanguageDyndepRule(std::string const& lang,
                                  const std::string& config) const;
-  bool NeedDyndep(std::string const& lang) const;
+  bool NeedDyndep(std::string const& lang, std::string const& config) const;
   bool NeedExplicitPreprocessing(std::string const& lang) const;
   bool CompileWithDefines(std::string const& lang) const;
+  bool NeedCxxModuleSupport(std::string const& lang,
+                            std::string const& config) const;
 
   std::string OrderDependsTargetForTarget(const std::string& config);
 

Source/cmRulePlaceholderExpander.cxx

@@ -44,6 +44,11 @@ std::string cmRulePlaceholderExpander::ExpandRuleVariable(
       return replaceValues.Source;
     }
   }
+  if (replaceValues.DynDepFile) {
+    if (variable == "DYNDEP_FILE") {
+      return replaceValues.DynDepFile;
+    }
+  }
   if (replaceValues.PreprocessedSource) {
     if (variable == "PREPROCESSED_SOURCE") {
       return replaceValues.PreprocessedSource;

Source/cmRulePlaceholderExpander.h

@@ -41,6 +41,7 @@ public:
     const char* Source = nullptr;
     const char* AssemblySource = nullptr;
     const char* PreprocessedSource = nullptr;
+    const char* DynDepFile = nullptr;
     const char* Output = nullptr;
     const char* Object = nullptr;
     const char* ObjectDir = nullptr;