Merge topic 'ninja-fortran-ddi-format'

f3eed2c49d cmGlobalNinjaGenerator: use P1689 dependency file format for Fortran
a02c4ccabc cmScanDepFormat: add reader and writer for the format from P1689R2
fa18069ebd Ninja: Exclude unused dyndep features during CMake bootstrap

Acked-by: Kitware Robot <[email protected]>
Acked-by: Michael Hirsch, Ph.D. <[email protected]>
Merge-request: !5485

Source/CMakeLists.txt

@@ -834,6 +834,7 @@ endif()
 
 # Ninja support
 set(SRCS ${SRCS}
+  cmScanDepFormat.cxx
   cmGlobalNinjaGenerator.cxx
   cmGlobalNinjaGenerator.h
   cmNinjaTypes.h

Source/cmGlobalNinjaGenerator.cxx

@@ -34,6 +34,7 @@
 #include "cmOutputConverter.h"
 #include "cmProperty.h"
 #include "cmRange.h"
+#include "cmScanDepFormat.h"
 #include "cmState.h"
 #include "cmStateDirectory.h"
 #include "cmStateSnapshot.h"
@@ -2024,6 +2025,8 @@ void cmGlobalNinjaGenerator::StripNinjaOutputPathPrefixAsSuffix(
   cmStripSuffixIfExists(path, this->OutputPathPrefix);
 }
 
+#if !defined(CMAKE_BOOTSTRAP)
+
 /*
 
 We use the following approach to support Fortran.  Each target already
@@ -2103,16 +2106,6 @@ Compilation of source files within a target is split into the following steps:
    (because the latter consumes the module).
 */
 
-struct cmSourceInfo
-{
-  // Set of provided and required modules.
-  std::set<std::string> Provides;
-  std::set<std::string> Requires;
-
-  // Set of files included in the translation unit.
-  std::set<std::string> Includes;
-};
-
 static std::unique_ptr<cmSourceInfo> cmcmd_cmake_ninja_depends_fortran(
   std::string const& arg_tdi, std::string const& arg_pp);
 
@@ -2120,6 +2113,7 @@ int cmcmd_cmake_ninja_depends(std::vector<std::string>::const_iterator argBeg,
                               std::vector<std::string>::const_iterator argEnd)
 {
   std::string arg_tdi;
+  std::string arg_src;
   std::string arg_pp;
   std::string arg_dep;
   std::string arg_obj;
@@ -2128,6 +2122,8 @@ int cmcmd_cmake_ninja_depends(std::vector<std::string>::const_iterator argBeg,
   for (std::string const& arg : cmMakeRange(argBeg, argEnd)) {
     if (cmHasLiteralPrefix(arg, "--tdi=")) {
       arg_tdi = arg.substr(6);
+    } else if (cmHasLiteralPrefix(arg, "--src=")) {
+      arg_src = arg.substr(6);
     } else if (cmHasLiteralPrefix(arg, "--pp=")) {
       arg_pp = arg.substr(5);
     } else if (cmHasLiteralPrefix(arg, "--dep=")) {
@@ -2168,6 +2164,9 @@ int cmcmd_cmake_ninja_depends(std::vector<std::string>::const_iterator argBeg,
     cmSystemTools::Error("-E cmake_ninja_depends requires value for --lang=");
     return 1;
   }
+  if (arg_src.empty()) {
+    arg_src = cmStrCat("<", arg_obj, " input file>");
+  }
 
   std::unique_ptr<cmSourceInfo> info;
   if (arg_lang == "Fortran") {
@@ -2184,6 +2183,8 @@ int cmcmd_cmake_ninja_depends(std::vector<std::string>::const_iterator argBeg,
     return 1;
   }
 
+  info->PrimaryOutput = arg_obj;
+
   {
     cmGeneratedFileStream depfile(arg_dep);
     depfile << cmSystemTools::ConvertToUnixOutputPath(arg_pp) << ":";
@@ -2193,24 +2194,7 @@ int cmcmd_cmake_ninja_depends(std::vector<std::string>::const_iterator argBeg,
     depfile << "\n";
   }
 
-  Json::Value ddi(Json::objectValue);
-  ddi["object"] = arg_obj;
-
-  Json::Value& ddi_provides = ddi["provides"] = Json::arrayValue;
-  for (std::string const& provide : info->Provides) {
-    ddi_provides.append(provide);
-  }
-  Json::Value& ddi_requires = ddi["requires"] = Json::arrayValue;
-  for (std::string const& r : info->Requires) {
-    // Require modules not provided in the same source.
-    if (!info->Provides.count(r)) {
-      ddi_requires.append(r);
-    }
-  }
-
-  cmGeneratedFileStream ddif(arg_ddi);
-  ddif << ddi;
-  if (!ddif) {
+  if (!cmScanDepFormat_P1689_Write(arg_ddi, arg_src, *info)) {
     cmSystemTools::Error(
       cmStrCat("-E cmake_ninja_depends failed to write ", arg_ddi));
     return 1;
@@ -2268,19 +2252,28 @@ std::unique_ptr<cmSourceInfo> cmcmd_cmake_ninja_depends_fortran(
   }
 
   auto info = cm::make_unique<cmSourceInfo>();
-  info->Provides = finfo.Provides;
-  info->Requires = finfo.Requires;
-  info->Includes = finfo.Includes;
+  for (std::string const& provide : finfo.Provides) {
+    cmSourceReqInfo src_info;
+    src_info.LogicalName = provide;
+    src_info.CompiledModulePath = provide;
+    info->Provides.emplace_back(src_info);
+  }
+  for (std::string const& require : finfo.Requires) {
+    // Require modules not provided in the same source.
+    if (finfo.Provides.count(require)) {
+      continue;
+    }
+    cmSourceReqInfo src_info;
+    src_info.LogicalName = require;
+    src_info.CompiledModulePath = require;
+    info->Requires.emplace_back(src_info);
+  }
+  for (std::string const& include : finfo.Includes) {
+    info->Includes.push_back(include);
+  }
   return info;
 }
 
-struct cmDyndepObjectInfo
-{
-  std::string Object;
-  std::vector<std::string> Provides;
-  std::vector<std::string> Requires;
-};
-
 bool cmGlobalNinjaGenerator::WriteDyndepFile(
   std::string const& dir_top_src, std::string const& dir_top_bld,
   std::string const& dir_cur_src, std::string const& dir_cur_bld,
@@ -2302,34 +2295,14 @@ bool cmGlobalNinjaGenerator::WriteDyndepFile(
     this->LocalGenerators.push_back(std::move(lgd));
   }
 
-  std::vector<cmDyndepObjectInfo> objects;
+  std::vector<cmSourceInfo> objects;
   for (std::string const& arg_ddi : arg_ddis) {
-    // Load the ddi file and compute the module file paths it provides.
-    Json::Value ddio;
-    Json::Value const& ddi = ddio;
-    cmsys::ifstream ddif(arg_ddi.c_str(), std::ios::in | std::ios::binary);
-    Json::Reader reader;
-    if (!reader.parse(ddif, ddio, false)) {
-      cmSystemTools::Error(cmStrCat("-E cmake_ninja_dyndep failed to parse ",
-                                    arg_ddi,
-                                    reader.getFormattedErrorMessages()));
+    cmSourceInfo info;
+    if (!cmScanDepFormat_P1689_Parse(arg_ddi, &info)) {
+      cmSystemTools::Error(
+        cmStrCat("-E cmake_ninja_dyndep failed to parse ddi file ", arg_ddi));
       return false;
     }
-
-    cmDyndepObjectInfo info;
-    info.Object = ddi["object"].asString();
-    Json::Value const& ddi_provides = ddi["provides"];
-    if (ddi_provides.isArray()) {
-      for (auto const& ddi_provide : ddi_provides) {
-        info.Provides.push_back(ddi_provide.asString());
-      }
-    }
-    Json::Value const& ddi_requires = ddi["requires"];
-    if (ddi_requires.isArray()) {
-      for (auto const& ddi_require : ddi_requires) {
-        info.Requires.push_back(ddi_require.asString());
-      }
-    }
     objects.push_back(std::move(info));
   }
 
@@ -2360,11 +2333,12 @@ bool cmGlobalNinjaGenerator::WriteDyndepFile(
   // We do this after loading the modules provided by linked targets
   // in case we have one of the same name that must be preferred.
   Json::Value tm = Json::objectValue;
-  for (cmDyndepObjectInfo const& object : objects) {
-    for (std::string const& p : object.Provides) {
-      std::string const mod = cmStrCat(module_dir, p);
-      mod_files[p] = mod;
-      tm[p] = mod;
+  for (cmSourceInfo const& object : objects) {
+    for (auto const& p : object.Provides) {
+      std::string const mod = cmStrCat(
+        module_dir, cmSystemTools::GetFilenameName(p.CompiledModulePath));
+      mod_files[p.LogicalName] = mod;
+      tm[p.LogicalName] = mod;
     }
   }
 
@@ -2374,15 +2348,16 @@ bool cmGlobalNinjaGenerator::WriteDyndepFile(
   {
     cmNinjaBuild build("dyndep");
     build.Outputs.emplace_back("");
-    for (cmDyndepObjectInfo const& object : objects) {
-      build.Outputs[0] = object.Object;
+    for (cmSourceInfo const& object : objects) {
+      build.Outputs[0] = this->ConvertToNinjaPath(object.PrimaryOutput);
       build.ImplicitOuts.clear();
-      for (std::string const& p : object.Provides) {
-        build.ImplicitOuts.push_back(this->ConvertToNinjaPath(mod_files[p]));
+      for (auto const& p : object.Provides) {
+        build.ImplicitOuts.push_back(
+          this->ConvertToNinjaPath(mod_files[p.LogicalName]));
       }
       build.ImplicitDeps.clear();
-      for (std::string const& r : object.Requires) {
-        auto mit = mod_files.find(r);
+      for (auto const& r : object.Requires) {
+        auto mit = mod_files.find(r.LogicalName);
         if (mit != mod_files.end()) {
           build.ImplicitDeps.push_back(this->ConvertToNinjaPath(mit->second));
         }
@@ -2406,11 +2381,6 @@ bool cmGlobalNinjaGenerator::WriteDyndepFile(
   return true;
 }
 
-bool cmGlobalNinjaGenerator::EnableCrossConfigBuild() const
-{
-  return !this->CrossConfigs.empty();
-}
-
 int cmcmd_cmake_ninja_dyndep(std::vector<std::string>::const_iterator argBeg,
                              std::vector<std::string>::const_iterator argEnd)
 {
@@ -2492,6 +2462,13 @@ int cmcmd_cmake_ninja_dyndep(std::vector<std::string>::const_iterator argBeg,
   return 0;
 }
 
+#endif
+
+bool cmGlobalNinjaGenerator::EnableCrossConfigBuild() const
+{
+  return !this->CrossConfigs.empty();
+}
+
 void cmGlobalNinjaGenerator::AppendDirectoryForConfig(
   const std::string& prefix, const std::string& config,
   const std::string& suffix, std::string& dir)

Source/cmNinjaTargetGenerator.cxx

@@ -532,7 +532,7 @@ std::string GetScanCommand(const std::string& cmakeCmd, const std::string& tdi,
                            const std::string& ddiFile)
 {
   return cmStrCat(cmakeCmd, " -E cmake_ninja_depends --tdi=", tdi,
-                  " --lang=", lang, " --pp=", ppFile,
+                  " --lang=", lang, " --src=$in", " --pp=", ppFile,
                   " --dep=$DEP_FILE --obj=$OBJ_FILE --ddi=", ddiFile);
 }
 

Source/cmScanDepFormat.cxx

@@ -0,0 +1,289 @@
+/* Distributed under the OSI-approved BSD 3-Clause License.  See accompanying
+   file Copyright.txt or https://cmake.org/licensing for details.  */
+
+#include "cmScanDepFormat.h"
+
+#include <cctype>
+#include <cstdio>
+
+#include <cm3p/json/reader.h>
+#include <cm3p/json/value.h>
+#include <cm3p/json/writer.h>
+
+#include "cmsys/FStream.hxx"
+
+#include "cm_utf8.h"
+
+#include "cmGeneratedFileStream.h"
+#include "cmStringAlgorithms.h"
+#include "cmSystemTools.h"
+
+static bool ParseFilename(Json::Value const& val, std::string& result)
+{
+  if (val.isObject()) {
+    Json::Value const& format = val["format"];
+    if (format == "raw8") {
+      Json::Value const& data = val["data"];
+      for (auto const& byte : data) {
+        result.push_back(static_cast<char>(byte.asUInt()));
+      }
+    } else /* TODO: if (format == "raw16") */ {
+      return false;
+    }
+  } else if (val.isString()) {
+    result = val.asString();
+  } else {
+    return false;
+  }
+
+  return true;
+}
+
+static Json::Value EncodeFilename(std::string const& path)
+{
+  if (cm_utf8_is_valid(path.c_str())) {
+    std::string valid_data;
+    valid_data.reserve(path.size());
+
+    for (auto const& byte : path) {
+      if (std::iscntrl(byte)) {
+        // Control characters.
+        valid_data.append("\\u");
+        char buf[5];
+        std::snprintf(buf, sizeof(buf), "%04x", byte);
+        valid_data.append(buf);
+      } else if (byte == '"' || byte == '\\') {
+        // Special JSON characters.
+        valid_data.push_back('\\');
+        valid_data.push_back(byte);
+      } else {
+        // Other data.
+        valid_data.push_back(byte);
+      }
+    }
+
+    return valid_data;
+  }
+
+  Json::Value data;
+  data["format"] = "raw8";
+  Json::Value& code_units = data["code-units"];
+  for (auto const& code_unit : path) {
+    code_units.append(static_cast<int>(code_unit));
+  }
+
+  return data;
+}
+
+#define PARSE_BLOB(val, res)                                                  \
+  do {                                                                        \
+    if (!ParseFilename(val, res)) {                                           \
+      cmSystemTools::Error(                                                   \
+        cmStrCat("-E cmake_ninja_depends failed to parse ", arg_pp,           \
+                 ": invalid blob"));                                          \
+      return false;                                                           \
+    }                                                                         \
+  } while (0)
+
+#define PARSE_FILENAME(val, res)                                              \
+  do {                                                                        \
+    if (!ParseFilename(val, res)) {                                           \
+      cmSystemTools::Error(                                                   \
+        cmStrCat("-E cmake_ninja_depends failed to parse ", arg_pp,           \
+                 ": invalid filename"));                                      \
+      return false;                                                           \
+    }                                                                         \
+                                                                              \
+    if (!cmSystemTools::FileIsFullPath(res)) {                                \
+      res = cmStrCat(work_directory, '/', res);                               \
+    }                                                                         \
+  } while (0)
+
+bool cmScanDepFormat_P1689_Parse(std::string const& arg_pp, cmSourceInfo* info)
+{
+  Json::Value ppio;
+  Json::Value const& ppi = ppio;
+  cmsys::ifstream ppf(arg_pp.c_str(), std::ios::in | std::ios::binary);
+  {
+    Json::Reader reader;
+    if (!reader.parse(ppf, ppio, false)) {
+      cmSystemTools::Error(cmStrCat("-E cmake_ninja_depends failed to parse ",
+                                    arg_pp,
+                                    reader.getFormattedErrorMessages()));
+      return false;
+    }
+  }
+
+  Json::Value const& version = ppi["version"];
+  if (version.asUInt() != 0) {
+    cmSystemTools::Error(cmStrCat("-E cmake_ninja_depends failed to parse ",
+                                  arg_pp, ": version ", version.asString()));
+    return false;
+  }
+
+  Json::Value const& workdir = ppi["work-directory"];
+  if (!workdir.isString()) {
+    cmSystemTools::Error(cmStrCat("-E cmake_ninja_depends failed to parse ",
+                                  arg_pp, ": work-directory is not a string"));
+    return false;
+  }
+  std::string work_directory;
+  PARSE_BLOB(workdir, work_directory);
+
+  Json::Value const& rules = ppi["rules"];
+  if (rules.isArray()) {
+    if (rules.size() != 1) {
+      cmSystemTools::Error(cmStrCat("-E cmake_ninja_depends failed to parse ",
+                                    arg_pp, ": expected 1 source entry"));
+      return false;
+    }
+
+    for (auto const& rule : rules) {
+      Json::Value const& depends = rule["depends"];
+      if (depends.isArray()) {
+        std::string depend_filename;
+        for (auto const& depend : depends) {
+          PARSE_FILENAME(depend, depend_filename);
+          info->Includes.push_back(depend_filename);
+        }
+      }
+
+      if (rule.isMember("future-compile")) {
+        Json::Value const& future_compile = rule["future-compile"];
+
+        if (future_compile.isMember("outputs")) {
+          Json::Value const& outputs = future_compile["outputs"];
+          if (outputs.isArray()) {
+            if (outputs.empty()) {
+              cmSystemTools::Error(
+                cmStrCat("-E cmake_ninja_depends failed to parse ", arg_pp,
+                         ": expected at least one 1 output"));
+              return false;
+            }
+
+            PARSE_FILENAME(outputs[0], info->PrimaryOutput);
+          }
+        }
+
+        if (future_compile.isMember("provides")) {
+          Json::Value const& provides = future_compile["provides"];
+          if (provides.isArray()) {
+            for (auto const& provide : provides) {
+              cmSourceReqInfo provide_info;
+
+              Json::Value const& logical_name = provide["logical-name"];
+              PARSE_BLOB(logical_name, provide_info.LogicalName);
+
+              if (provide.isMember("compiled-module-path")) {
+                Json::Value const& compiled_module_path =
+                  provide["compiled-module-path"];
+                PARSE_FILENAME(compiled_module_path,
+                               provide_info.CompiledModulePath);
+              } else {
+                provide_info.CompiledModulePath =
+                  cmStrCat(provide_info.LogicalName, ".mod");
+              }
+
+              info->Provides.push_back(provide_info);
+            }
+          }
+        }
+
+        if (future_compile.isMember("requires")) {
+          Json::Value const& reqs = future_compile["requires"];
+          if (reqs.isArray()) {
+            for (auto const& require : reqs) {
+              cmSourceReqInfo require_info;
+
+              Json::Value const& logical_name = require["logical-name"];
+              PARSE_BLOB(logical_name, require_info.LogicalName);
+
+              if (require.isMember("compiled-module-path")) {
+                Json::Value const& compiled_module_path =
+                  require["compiled-module-path"];
+                PARSE_FILENAME(compiled_module_path,
+                               require_info.CompiledModulePath);
+              }
+
+              info->Requires.push_back(require_info);
+            }
+          }
+        }
+      }
+    }
+  }
+
+  return true;
+}
+
+bool cmScanDepFormat_P1689_Write(std::string const& path,
+                                 std::string const& input,
+                                 cmSourceInfo const& info)
+{
+  Json::Value ddi(Json::objectValue);
+  ddi["version"] = 0;
+  ddi["revision"] = 0;
+  ddi["work-directory"] =
+    EncodeFilename(cmSystemTools::GetCurrentWorkingDirectory());
+
+  Json::Value& rules = ddi["rules"] = Json::arrayValue;
+
+  Json::Value rule(Json::objectValue);
+  Json::Value& inputs = rule["inputs"] = Json::arrayValue;
+  inputs.append(EncodeFilename(input));
+
+  Json::Value& rule_outputs = rule["outputs"] = Json::arrayValue;
+  rule_outputs.append(EncodeFilename(path));
+
+  Json::Value& depends = rule["depends"] = Json::arrayValue;
+  for (auto const& include : info.Includes) {
+    depends.append(EncodeFilename(include));
+  }
+
+  Json::Value& future_compile = rule["future-compile"] = Json::objectValue;
+
+  Json::Value& outputs = future_compile["outputs"] = Json::arrayValue;
+  outputs.append(info.PrimaryOutput);
+
+  Json::Value& provides = future_compile["provides"] = Json::arrayValue;
+  for (auto const& provide : info.Provides) {
+    Json::Value provide_obj(Json::objectValue);
+    auto const encoded = EncodeFilename(provide.LogicalName);
+    provide_obj["logical-name"] = encoded;
+    if (provide.CompiledModulePath.empty()) {
+      provide_obj["compiled-module-path"] = encoded;
+    } else {
+      provide_obj["compiled-module-path"] =
+        EncodeFilename(provide.CompiledModulePath);
+    }
+
+    // TODO: Source file tracking. See below.
+
+    provides.append(provide_obj);
+  }
+
+  Json::Value& reqs = future_compile["requires"] = Json::arrayValue;
+  for (auto const& require : info.Requires) {
+    Json::Value require_obj(Json::objectValue);
+    auto const encoded = EncodeFilename(require.LogicalName);
+    require_obj["logical-name"] = encoded;
+    if (require.CompiledModulePath.empty()) {
+      require_obj["compiled-module-path"] = encoded;
+    } else {
+      require_obj["compiled-module-path"] =
+        EncodeFilename(require.CompiledModulePath);
+    }
+
+    // TODO: Source filename inclusion. Requires collating with the provides
+    // filenames (as a sanity check if available on both sides).
+
+    reqs.append(require_obj);
+  }
+
+  rules.append(rule);
+
+  cmGeneratedFileStream ddif(path);
+  ddif << ddi;
+
+  return !!ddif;
+}

Source/cmScanDepFormat.h

@@ -0,0 +1,30 @@
+/* Distributed under the OSI-approved BSD 3-Clause License.  See accompanying
+   file Copyright.txt or https://cmake.org/licensing for details.  */
+#pragma once
+
+#include <string>
+#include <vector>
+
+struct cmSourceReqInfo
+{
+  std::string LogicalName;
+  std::string CompiledModulePath;
+};
+
+struct cmSourceInfo
+{
+  std::string PrimaryOutput;
+
+  // Set of provided and required modules.
+  std::vector<cmSourceReqInfo> Provides;
+  std::vector<cmSourceReqInfo> Requires;
+
+  // Set of files included in the translation unit.
+  std::vector<std::string> Includes;
+};
+
+bool cmScanDepFormat_P1689_Parse(std::string const& arg_pp,
+                                 cmSourceInfo* info);
+bool cmScanDepFormat_P1689_Write(std::string const& path,
+                                 std::string const& input,
+                                 cmSourceInfo const& info);

Source/cmcmd.cxx

@@ -1167,7 +1167,7 @@ int cmcmd::ExecuteCMakeCommand(std::vector<std::string> const& args,
       return cmcmd::ExecuteLinkScript(args);
     }
 
-#if !defined(CMAKE_BOOTSTRAP) || defined(CMAKE_BOOTSTRAP_NINJA)
+#if !defined(CMAKE_BOOTSTRAP)
     // Internal CMake ninja dependency scanning support.
     if (args[1] == "cmake_ninja_depends") {
       return cmcmd_cmake_ninja_depends(args.begin() + 2, args.end());