Merge topic 'ninja-parallel'

0e5250e63c Ninja: Add option for parallel install
daeb8fffa2 Help: Add Builtin Targets section to Ninja help

Acked-by: Kitware Robot <[email protected]>
Merge-request: !9505

Help/generator/Ninja.rst

@@ -1,11 +1,35 @@
 Ninja
 -----
 
-Generates ``build.ninja`` files.
+Generates a ``build.ninja`` file into the build tree.
 
-A ``build.ninja`` file is generated into the build tree.  Use the ninja
-program to build the project through the ``all`` target and install the
-project through the ``install`` (or ``install/strip``) target.
+Builtin Targets
+^^^^^^^^^^^^^^^
+
+``all``
+
+  Depends on all targets required by the project, except those with
+  :prop_tgt:`EXCLUDE_FROM_ALL` set to true.
+
+``install``
+
+  Runs the install step.
+
+``install/strip``
+
+  .. versionadded:: 3.7
+
+    Runs the install followed by a ``CMAKE_STRIP`` command, if any.
+
+    The ``CMAKE_STRIP`` variable will contain the platform's ``strip`` utility, which
+    removes symbols information from generated binaries.
+
+``install/parallel``
+
+  .. versionadded:: 3.30
+
+    Created only if the :prop_gbl:`INSTALL_PARALLEL` global property is ``ON``.
+    Runs the install step for each subdirectory independently and in parallel.
 
 For each subdirectory ``sub/dir`` of the project, additional targets
 are generated:
@@ -25,12 +49,10 @@ are generated:
 ``sub/dir/install/strip``
 
   .. versionadded:: 3.7
+
     Runs the install step in the subdirectory followed by a ``CMAKE_STRIP`` command,
     if any.
 
-    The ``CMAKE_STRIP`` variable will contain the platform's ``strip`` utility, which
-    removes symbols information from generated binaries.
-
 ``sub/dir/test``
 
   .. versionadded:: 3.7

Help/manual/cmake-properties.7.rst

@@ -39,6 +39,7 @@ Properties of Global Scope
    /prop_gbl/GENERATOR_IS_MULTI_CONFIG
    /prop_gbl/GLOBAL_DEPENDS_DEBUG_MODE
    /prop_gbl/GLOBAL_DEPENDS_NO_CYCLES
+   /prop_gbl/INSTALL_PARALLEL
    /prop_gbl/IN_TRY_COMPILE
    /prop_gbl/JOB_POOLS
    /prop_gbl/PACKAGES_FOUND

Help/prop_gbl/INSTALL_PARALLEL.rst

@@ -0,0 +1,23 @@
+INSTALL_PARALLEL
+----------------
+
+.. versionadded:: 3.30
+
+Enables parallel installation option for the Ninja generator.
+
+When this property is ``ON``, ``install/local`` targets have the
+console pool disabled, allowing them to run concurrently.
+
+This property also provides the target ``install/parallel``, which has an
+explicit dependency on the ``install/local`` target for each subdirectory,
+recursing down the project.
+
+Setting this property has no affect on the behavior of ``cmake --install``.
+The install must be invoked by building the ``install/parallel`` target
+directly.
+
+Calls to :command:`install(CODE)` or :command:`install(SCRIPT)` might depend
+on actions performed by an earlier :command:`install` command in a different
+directory such as files installed or variable settings. If the project has
+such order-dependent installation logic, parallel installation should be
+not be enabled, in order to prevent possible race conditions.

Source/cmGlobalGenerator.cxx

@@ -3048,7 +3048,9 @@ void cmGlobalGenerator::AddGlobalTarget_Install(
     if (const char* install_local = this->GetInstallLocalTargetName()) {
       gti.Name = install_local;
       gti.Message = "Installing only the local directory...";
-      gti.UsesTerminal = true;
+      gti.UsesTerminal =
+        !this->GetCMakeInstance()->GetState()->GetGlobalPropertyAsBool(
+          "INSTALL_PARALLEL");
       gti.CommandLines.clear();
 
       cmCustomCommandLine localCmdLine = singleLine;

Source/cmGlobalNinjaGenerator.cxx

@@ -1835,6 +1835,21 @@ void cmGlobalNinjaGenerator::WriteBuiltinTargets(std::ostream& os)
   if (!this->DefaultFileConfig.empty()) {
     this->WriteTargetDefault(*this->GetDefaultFileStream());
   }
+
+  if (this->InstallTargetEnabled &&
+      this->GetCMakeInstance()->GetState()->GetGlobalPropertyAsBool(
+        "INSTALL_PARALLEL") &&
+      !this->Makefiles[0]->IsOn("CMAKE_SKIP_INSTALL_RULES")) {
+    cmNinjaBuild build("phony");
+    build.Comment = "Install every subdirectory in parallel";
+    build.Outputs.emplace_back(this->GetInstallParallelTargetName());
+    for (auto const& mf : this->Makefiles) {
+      build.ExplicitDeps.emplace_back(
+        this->ConvertToNinjaPath(cmStrCat(mf->GetCurrentBinaryDirectory(), "/",
+                                          this->GetInstallLocalTargetName())));
+    }
+    WriteBuild(os, build);
+  }
 }
 
 void cmGlobalNinjaGenerator::WriteTargetDefault(std::ostream& os)

Source/cmGlobalNinjaGenerator.h

@@ -218,6 +218,10 @@ public:
   {
     return "install/strip";
   }
+  const char* GetInstallParallelTargetName() const
+  {
+    return "install/parallel";
+  }
   const char* GetTestTargetName() const override { return "test"; }
   const char* GetPackageTargetName() const override { return "package"; }
   const char* GetPackageSourceTargetName() const override

Source/cmLocalGenerator.cxx

@@ -730,10 +730,12 @@ void cmLocalGenerator::GenerateInstallRules()
       "  set(CMAKE_INSTALL_MANIFEST \"install_manifest.txt\")\n"
       "endif()\n"
       "\n"
-      "string(REPLACE \";\" \"\\n\" CMAKE_INSTALL_MANIFEST_CONTENT\n"
+      "if(NOT CMAKE_INSTALL_LOCAL_ONLY)\n"
+      "  string(REPLACE \";\" \"\\n\" CMAKE_INSTALL_MANIFEST_CONTENT\n"
       "       \"${CMAKE_INSTALL_MANIFEST_FILES}\")\n"
-      "file(WRITE \"" << homedir << "/${CMAKE_INSTALL_MANIFEST}\"\n"
-      "     \"${CMAKE_INSTALL_MANIFEST_CONTENT}\")\n";
+      "  file(WRITE \"" << homedir << "/${CMAKE_INSTALL_MANIFEST}\"\n"
+      "     \"${CMAKE_INSTALL_MANIFEST_CONTENT}\")\n"
+      "endif()\n";
     /* clang-format on */
   }
 }

Tests/RunCMake/CMakeLists.txt

@@ -252,6 +252,7 @@ if(CMAKE_GENERATOR MATCHES "Ninja")
   add_RunCMake_test(NinjaPrivateDeps
     -DCMAKE_C_OUTPUT_EXTENSION=${CMAKE_C_OUTPUT_EXTENSION}
     -DRunCMake_GENERATOR_IS_MULTI_CONFIG=${_isMultiConfig})
+  add_RunCMake_test(InstallParallel)
 endif()
 add_RunCMake_test(CTest)
 

Tests/RunCMake/InstallParallel/CMakeLists.txt

@@ -0,0 +1,3 @@
+cmake_minimum_required(VERSION 3.29)
+project(${RunCMake_TEST} NONE)
+include(${RunCMake_TEST}.cmake)

Tests/RunCMake/InstallParallel/RunCMakeTest.cmake

@@ -0,0 +1,17 @@
+include(RunCMake)
+
+function(install_test test parallel install_target check_script)
+  set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/${test}-install)
+  set(RunCMake_TEST_OPTIONS -DINSTALL_PARALLEL=${parallel})
+  if (NOT RunCMake_GENERATOR_IS_MULTI_CONFIG)
+    list(APPEND RunCMake_TEST_OPTIONS -DCMAKE_BUILD_TYPE=Debug)
+  endif()
+  run_cmake(install)
+  set(RunCMake_TEST_NO_CLEAN 1)
+  run_cmake_command(${test}-install ${CMAKE_COMMAND} --build . --config Debug -t ${install_target})
+  set(RunCMake_TEST_COMMAND_WORKING_DIRECTORY ${RunCMake_SOURCE_DIR})
+  run_cmake_command(verify-parallel ${CMAKE_COMMAND} -P ${check_script} ${RunCMake_TEST_BINARY_DIR}/.ninja_log)
+endfunction()
+
+install_test(parallel 1 install/parallel check-parallel.cmake)
+install_test(no-parallel 0 install check-single.cmake)

Tests/RunCMake/InstallParallel/check-parallel.cmake

@@ -0,0 +1,15 @@
+include(read-ninja-install.cmake)
+
+foreach(line ${lines})
+  string(REPLACE "\t" ";" line ${line})
+  list(GET line 0 start)
+  list(GET line 1 end)
+  list(APPEND start_times ${start})
+  list(APPEND end_times ${end})
+endforeach()
+list(GET start_times 1 start_2)
+list(GET end_times 0 end_1)
+
+if (NOT start_2 LESS end_1)
+  message(FATAL_ERROR "Install is not parallel")
+endif()

Tests/RunCMake/InstallParallel/check-single.cmake

@@ -0,0 +1,5 @@
+include(read-ninja-install.cmake)
+list(LENGTH lines len)
+if (NOT ${len} STREQUAL "1")
+  message(FATAL_ERROR "Expected single installation call")
+endif()

Tests/RunCMake/InstallParallel/install.cmake

@@ -0,0 +1,6 @@
+install(CODE "message(installing:${CMAKE_CURRENT_SOURCE_DIR})")
+if (INSTALL_PARALLEL)
+  set_property(GLOBAL PROPERTY INSTALL_PARALLEL ON)
+endif()
+add_subdirectory(subdir-1)
+add_subdirectory(subdir-2)

Tests/RunCMake/InstallParallel/no-parallel-install-stderr.txt

@@ -0,0 +1,5 @@
+installing:.*
+installing:.*
+installing:.*
+installing:.*
+installing:.*

Tests/RunCMake/InstallParallel/parallel-install-stdout.txt

@@ -0,0 +1,15 @@
+\[1\/5\] Installing only the local directory...
+\-\- Install configuration: \"Debug\"
+installing:.*
+\[2\/5\] Installing only the local directory...
+\-\- Install configuration: \"Debug\"
+installing:.*
+\[3\/5\] Installing only the local directory...
+\-\- Install configuration: \"Debug\"
+installing:.*
+\[4\/5\] Installing only the local directory...
+\-\- Install configuration: \"Debug\"
+installing:.*
+\[5\/5\] Installing only the local directory...
+\-\- Install configuration: \"Debug\"
+installing:.*

Tests/RunCMake/InstallParallel/read-ninja-install.cmake

@@ -0,0 +1,4 @@
+set(ninja_log ${CMAKE_ARGV3})
+file(STRINGS ${ninja_log} lines)
+list(POP_FRONT lines)
+list(FILTER lines INCLUDE REGEX ".*install.*util")

Tests/RunCMake/InstallParallel/subdir-1/CMakeLists.txt

@@ -0,0 +1,3 @@
+install(CODE "message(installing:${CMAKE_CURRENT_SOURCE_DIR})")
+add_subdirectory(subdir-3)
+add_subdirectory(subdir-4)

Tests/RunCMake/InstallParallel/subdir-1/subdir-3/CMakeLists.txt

@@ -0,0 +1 @@
+install(CODE "message(installing:${CMAKE_CURRENT_SOURCE_DIR})")

Tests/RunCMake/InstallParallel/subdir-1/subdir-4/CMakeLists.txt

@@ -0,0 +1 @@
+install(CODE "message(installing:${CMAKE_CURRENT_SOURCE_DIR})")

Tests/RunCMake/InstallParallel/subdir-2/CMakeLists.txt

@@ -0,0 +1 @@
+install(CODE "message(installing:${CMAKE_CURRENT_SOURCE_DIR})")