ExternalProject: Refactor pre-configure steps to support no-target uses

The mkdir, download, update and patch steps are used by
FetchContent during the configure phase of the main build. Because
these steps need a target, this has so far required a sub-build to be
set up. The changes here factor out the preparation of the scripts
from the creation of the targets, allowing future work to leverage these
steps without a sub-build (see #21703).

As part of the refactoring, some rationalisation of the stamp files,
repository info files and script names was done to make things more
consistent between download methods and step implementations.
Every download method now records its own specific repository info
in a file and that file is a dependency of the download step. The source
directory is also written to that file, so if the SOURCE_DIR changes, the
download will be retriggered (the existing implementation fails in this
scenario). Each download method now also has just one driver script
that implements the whole step (it may pull in other scripts to do its
task though). The patch step gained support for USES_TERMINAL as
a result of generalising the implementation for custom commands.

Fixes: #21748

Help/release/dev/fetchcontent-performance.rst

@@ -0,0 +1,7 @@
+fetchcontent-performance
+------------------------
+
+* The implementation of the :module:`ExternalProject` module was
+  significantly refactored.  The patch step gained support for
+  using the terminal with a new ``USES_TERMINAL_PATCH`` keyword
+  as a by-product of that work.

Modules/ExternalProject-verify.cmake.in

@@ -1,37 +0,0 @@
-# Distributed under the OSI-approved BSD 3-Clause License.  See accompanying
-# file Copyright.txt or https://cmake.org/licensing for details.
-
-cmake_minimum_required(VERSION 3.5)
-
-if("@LOCAL@" STREQUAL "")
-  message(FATAL_ERROR "LOCAL can't be empty")
-endif()
-
-if(NOT EXISTS "@LOCAL@")
-  message(FATAL_ERROR "File not found: @LOCAL@")
-endif()
-
-if("@ALGO@" STREQUAL "")
-  message(WARNING "File will not be verified since no URL_HASH specified")
-  return()
-endif()
-
-if("@EXPECT_VALUE@" STREQUAL "")
-  message(FATAL_ERROR "EXPECT_VALUE can't be empty")
-endif()
-
-message(STATUS "verifying file...
-     file='@LOCAL@'")
-
-file("@ALGO@" "@LOCAL@" actual_value)
-
-if(NOT "${actual_value}" STREQUAL "@EXPECT_VALUE@")
-  message(FATAL_ERROR "error: @ALGO@ hash of
-  @LOCAL@
-does not match expected value
-  expected: '@EXPECT_VALUE@'
-    actual: '${actual_value}'
-")
-endif()
-
-message(STATUS "verifying file... done")

Modules/ExternalProject/RepositoryInfo.txt.in

@@ -0,0 +1 @@
+@repo_info_content@

Modules/ExternalProject/cfgcmd.txt.in

@@ -0,0 +1 @@
+cmd='@cmd@'

Modules/ExternalProject/copydir.cmake.in

@@ -0,0 +1,10 @@
+# Distributed under the OSI-approved BSD 3-Clause License.  See accompanying
+# file Copyright.txt or https://cmake.org/licensing for details.
+
+cmake_minimum_required(VERSION 3.5)
+
+file(REMOVE_RECURSE "@to_dir@")
+
+# Copy the _contents_ of the source dir into the destination dir, hence the
+# trailing slash on the from_dir
+file(COPY "@from_dir@/" DESTINATION "@to_dir@")

Modules/ExternalProject-download.cmake.in → Modules/ExternalProject/download.cmake.in

@@ -79,77 +79,81 @@ if("@REMOTE@" STREQUAL "")
   message(FATAL_ERROR "REMOTE can't be empty")
 endif()
 
-if(EXISTS "@LOCAL@")
-  check_file_hash(has_hash hash_is_good)
-  if(has_hash)
-    if(hash_is_good)
-      message(STATUS "File already exists and hash match (skip download):
+function(download_and_verify)
+  if(EXISTS "@LOCAL@")
+    check_file_hash(has_hash hash_is_good)
+    if(has_hash)
+      if(hash_is_good)
+        message(STATUS
+"File already exists and hash match (skip download):
   file='@LOCAL@'
   @ALGO@='@EXPECT_VALUE@'"
-      )
-      return()
+        )
+        return()
+      else()
+        message(STATUS "File already exists but hash mismatch. Removing...")
+        file(REMOVE "@LOCAL@")
+      endif()
     else()
-      message(STATUS "File already exists but hash mismatch. Removing...")
-      file(REMOVE "@LOCAL@")
-    endif()
-  else()
-    message(STATUS "File already exists but no hash specified (use URL_HASH):
+      message(STATUS
+"File already exists but no hash specified (use URL_HASH):
   file='@LOCAL@'
 Old file will be removed and new file downloaded from URL."
-    )
-    file(REMOVE "@LOCAL@")
+      )
+      file(REMOVE "@LOCAL@")
+    endif()
   endif()
-endif()
 
-set(retry_number 5)
+  set(retry_number 5)
 
-message(STATUS "Downloading...
+  message(STATUS "Downloading...
    dst='@LOCAL@'
    timeout='@TIMEOUT_MSG@'
    inactivity timeout='@INACTIVITY_TIMEOUT_MSG@'"
-)
-set(download_retry_codes 7 6 8 15)
-set(skip_url_list)
-set(status_code)
-foreach(i RANGE ${retry_number})
-  if(status_code IN_LIST download_retry_codes)
-    sleep_before_download(${i})
-  endif()
-  foreach(url @REMOTE@)
-    if(NOT url IN_LIST skip_url_list)
-      message(STATUS "Using src='${url}'")
-
-      @TLS_VERIFY_CODE@
-      @TLS_CAINFO_CODE@
-      @NETRC_CODE@
-      @NETRC_FILE_CODE@
-
-      file(
-        DOWNLOAD
-        "${url}" "@LOCAL@"
-        @SHOW_PROGRESS@
-        @TIMEOUT_ARGS@
-        @INACTIVITY_TIMEOUT_ARGS@
-        STATUS status
-        LOG log
-        @USERPWD_ARGS@
-        @HTTP_HEADERS_ARGS@
-        )
-
-      list(GET status 0 status_code)
-      list(GET status 1 status_string)
-
-      if(status_code EQUAL 0)
-        check_file_hash(has_hash hash_is_good)
-        if(has_hash AND NOT hash_is_good)
-          message(STATUS "Hash mismatch, removing...")
-          file(REMOVE "@LOCAL@")
+  )
+  set(download_retry_codes 7 6 8 15)
+  set(skip_url_list)
+  set(status_code)
+  foreach(i RANGE ${retry_number})
+    if(status_code IN_LIST download_retry_codes)
+      sleep_before_download(${i})
+    endif()
+    foreach(url @REMOTE@)
+      if(NOT url IN_LIST skip_url_list)
+        message(STATUS "Using src='${url}'")
+
+        @TLS_VERIFY_CODE@
+        @TLS_CAINFO_CODE@
+        @NETRC_CODE@
+        @NETRC_FILE_CODE@
+
+        file(
+          DOWNLOAD
+          "${url}" "@LOCAL@"
+          @SHOW_PROGRESS@
+          @TIMEOUT_ARGS@
+          @INACTIVITY_TIMEOUT_ARGS@
+          STATUS status
+          LOG log
+          @USERPWD_ARGS@
+          @HTTP_HEADERS_ARGS@
+          )
+
+        list(GET status 0 status_code)
+        list(GET status 1 status_string)
+
+        if(status_code EQUAL 0)
+          check_file_hash(has_hash hash_is_good)
+          if(has_hash AND NOT hash_is_good)
+            message(STATUS "Hash mismatch, removing...")
+            file(REMOVE "@LOCAL@")
+          else()
+            message(STATUS "Downloading... done")
+            return()
+          endif()
         else()
-          message(STATUS "Downloading... done")
-          return()
-        endif()
-      else()
-        string(APPEND logFailedURLs "error: downloading '${url}' failed
+          string(APPEND logFailedURLs
+"error: downloading '${url}' failed
         status_code: ${status_code}
         status_string: ${status_string}
         log:
@@ -157,17 +161,27 @@ foreach(i RANGE ${retry_number})
         ${log}
         --- LOG END ---
         "
-        )
-      if(NOT status_code IN_LIST download_retry_codes)
-        list(APPEND skip_url_list "${url}")
-        break()
+          )
+        if(NOT status_code IN_LIST download_retry_codes)
+          list(APPEND skip_url_list "${url}")
+          break()
+        endif()
       endif()
     endif()
-  endif()
+    endforeach()
   endforeach()
-endforeach()
 
-message(FATAL_ERROR "Each download failed!
+  message(FATAL_ERROR
+"Each download failed!
   ${logFailedURLs}
   "
-)
+  )
+
+endfunction()
+
+download_and_verify()
+
+set(extract_script @extract_script_filename@)
+if(NOT "${extract_script}" STREQUAL "")
+  include(${extract_script})
+endif()

Modules/ExternalProject/extractfile.cmake.in

@@ -0,0 +1,63 @@
+# Distributed under the OSI-approved BSD 3-Clause License.  See accompanying
+# file Copyright.txt or https://cmake.org/licensing for details.
+
+cmake_minimum_required(VERSION 3.5)
+
+# Make file names absolute:
+#
+get_filename_component(filename "@filename@" ABSOLUTE)
+get_filename_component(directory "@directory@" ABSOLUTE)
+
+message(STATUS "extracting...
+     src='${filename}'
+     dst='${directory}'")
+
+if(NOT EXISTS "${filename}")
+  message(FATAL_ERROR "File to extract does not exist: '${filename}'")
+endif()
+
+# Prepare a space for extracting:
+#
+set(i 1234)
+while(EXISTS "${directory}/../ex-@name@${i}")
+  math(EXPR i "${i} + 1")
+endwhile()
+set(ut_dir "${directory}/../ex-@name@${i}")
+file(MAKE_DIRECTORY "${ut_dir}")
+
+# Extract it:
+#
+message(STATUS "extracting... [tar @args@]")
+execute_process(COMMAND ${CMAKE_COMMAND} -E tar @args@ ${filename}
+  WORKING_DIRECTORY ${ut_dir}
+  RESULT_VARIABLE rv)
+
+if(NOT rv EQUAL 0)
+  message(STATUS "extracting... [error clean up]")
+  file(REMOVE_RECURSE "${ut_dir}")
+  message(FATAL_ERROR "Extract of '${filename}' failed")
+endif()
+
+# Analyze what came out of the tar file:
+#
+message(STATUS "extracting... [analysis]")
+file(GLOB contents "${ut_dir}/*")
+list(REMOVE_ITEM contents "${ut_dir}/.DS_Store")
+list(LENGTH contents n)
+if(NOT n EQUAL 1 OR NOT IS_DIRECTORY "${contents}")
+  set(contents "${ut_dir}")
+endif()
+
+# Move "the one" directory to the final directory:
+#
+message(STATUS "extracting... [rename]")
+file(REMOVE_RECURSE ${directory})
+get_filename_component(contents ${contents} ABSOLUTE)
+file(RENAME ${contents} ${directory})
+
+# Clean up:
+#
+message(STATUS "extracting... [clean up]")
+file(REMOVE_RECURSE "${ut_dir}")
+
+message(STATUS "extracting... done")

Modules/ExternalProject/gitclone.cmake.in

@@ -0,0 +1,67 @@
+# Distributed under the OSI-approved BSD 3-Clause License.  See accompanying
+# file Copyright.txt or https://cmake.org/licensing for details.
+
+cmake_minimum_required(VERSION 3.5)
+
+if(NOT "@gitclone_infofile@" IS_NEWER_THAN "@gitclone_stampfile@")
+  message(STATUS "Avoiding repeated git clone, stamp file is up to date: '@gitclone_stampfile@'")
+  return()
+endif()
+
+execute_process(
+  COMMAND ${CMAKE_COMMAND} -E rm -rf "@source_dir@"
+  RESULT_VARIABLE error_code
+  )
+if(error_code)
+  message(FATAL_ERROR "Failed to remove directory: '@source_dir@'")
+endif()
+
+# try the clone 3 times in case there is an odd git clone issue
+set(error_code 1)
+set(number_of_tries 0)
+while(error_code AND number_of_tries LESS 3)
+  execute_process(
+    COMMAND "@git_EXECUTABLE@" @git_options@ clone @git_clone_options@ "@git_repository@" "@src_name@"
+    WORKING_DIRECTORY "@work_dir@"
+    RESULT_VARIABLE error_code
+    )
+  math(EXPR number_of_tries "${number_of_tries} + 1")
+endwhile()
+if(number_of_tries GREATER 1)
+  message(STATUS "Had to git clone more than once:
+          ${number_of_tries} times.")
+endif()
+if(error_code)
+  message(FATAL_ERROR "Failed to clone repository: '@git_repository@'")
+endif()
+
+execute_process(
+  COMMAND "@git_EXECUTABLE@" @git_options@ checkout "@git_tag@" @git_checkout_explicit--@
+  WORKING_DIRECTORY "@work_dir@/@src_name@"
+  RESULT_VARIABLE error_code
+  )
+if(error_code)
+  message(FATAL_ERROR "Failed to checkout tag: '@git_tag@'")
+endif()
+
+set(init_submodules @init_submodules@)
+if(init_submodules)
+  execute_process(
+    COMMAND "@git_EXECUTABLE@" @git_options@ submodule update @git_submodules_recurse@ --init @git_submodules@
+    WORKING_DIRECTORY "@work_dir@/@src_name@"
+    RESULT_VARIABLE error_code
+    )
+endif()
+if(error_code)
+  message(FATAL_ERROR "Failed to update submodules in: '@work_dir@/@src_name@'")
+endif()
+
+# Complete success, update the script-last-run stamp file:
+#
+execute_process(
+  COMMAND ${CMAKE_COMMAND} -E copy "@gitclone_infofile@" "@gitclone_stampfile@"
+  RESULT_VARIABLE error_code
+  )
+if(error_code)
+  message(FATAL_ERROR "Failed to copy script-last-run stamp file: '@gitclone_stampfile@'")
+endif()

Modules/ExternalProject/hgclone.cmake.in

@@ -0,0 +1,45 @@
+# Distributed under the OSI-approved BSD 3-Clause License.  See accompanying
+# file Copyright.txt or https://cmake.org/licensing for details.
+
+cmake_minimum_required(VERSION 3.5)
+
+if(NOT "@hgclone_infofile@" IS_NEWER_THAN "@hgclone_stampfile@")
+  message(STATUS "Avoiding repeated hg clone, stamp file is up to date: '@hgclone_stampfile@'")
+  return()
+endif()
+
+execute_process(
+  COMMAND ${CMAKE_COMMAND} -E rm -rf "@source_dir@"
+  RESULT_VARIABLE error_code
+  )
+if(error_code)
+  message(FATAL_ERROR "Failed to remove directory: '@source_dir@'")
+endif()
+
+execute_process(
+  COMMAND "@hg_EXECUTABLE@" clone -U "@hg_repository@" "@src_name@"
+  WORKING_DIRECTORY "@work_dir@"
+  RESULT_VARIABLE error_code
+  )
+if(error_code)
+  message(FATAL_ERROR "Failed to clone repository: '@hg_repository@'")
+endif()
+
+execute_process(
+  COMMAND "@hg_EXECUTABLE@" update @hg_tag@
+  WORKING_DIRECTORY "@work_dir@/@src_name@"
+  RESULT_VARIABLE error_code
+  )
+if(error_code)
+  message(FATAL_ERROR "Failed to checkout tag: '@hg_tag@'")
+endif()
+
+# Complete success, update the script-last-run stamp file:
+#
+execute_process(
+  COMMAND ${CMAKE_COMMAND} -E copy "@hgclone_infofile@" "@hgclone_stampfile@"
+  RESULT_VARIABLE error_code
+  )
+if(error_code)
+  message(FATAL_ERROR "Failed to copy script-last-run stamp file: '@hgclone_stampfile@'")
+endif()

Modules/ExternalProject/hgupdate.cmake.in

@@ -0,0 +1,16 @@
+# Distributed under the OSI-approved BSD 3-Clause License.  See accompanying
+# file Copyright.txt or https://cmake.org/licensing for details.
+
+cmake_minimum_required(VERSION 3.19)
+
+execute_process(
+  COMMAND "@hg_EXECUTABLE@" pull
+  COMMAND_ERROR_IS_FATAL ANY
+  WORKING_DIRECTORY "@work_dir@"
+)
+
+execute_process(
+  COMMAND "@hg_EXECUTABLE@" update @hg_tag@
+  COMMAND_ERROR_IS_FATAL ANY
+  WORKING_DIRECTORY "@work_dir@"
+)

Modules/ExternalProject/mkdirs.cmake.in

@@ -0,0 +1,19 @@
+# Distributed under the OSI-approved BSD 3-Clause License.  See accompanying
+# file Copyright.txt or https://cmake.org/licensing for details.
+
+cmake_minimum_required(VERSION 3.5)
+
+file(MAKE_DIRECTORY
+  "@source_dir@"
+  "@binary_dir@"
+  "@install_dir@"
+  "@tmp_dir@"
+  "@stamp_dir@"
+  "@download_dir@"
+  "@log_dir@"
+)
+
+set(configSubDirs @CMAKE_CONFIGURATION_TYPES@)
+foreach(subDir IN LISTS configSubDirs)
+  file(MAKE_DIRECTORY "@stamp_dir@/${subDir}")
+endforeach()

Modules/ExternalProject/verify.cmake.in

@@ -0,0 +1,48 @@
+# Distributed under the OSI-approved BSD 3-Clause License.  See accompanying
+# file Copyright.txt or https://cmake.org/licensing for details.
+
+cmake_minimum_required(VERSION 3.5)
+
+if("@LOCAL@" STREQUAL "")
+  message(FATAL_ERROR "LOCAL can't be empty")
+endif()
+
+if(NOT EXISTS "@LOCAL@")
+  message(FATAL_ERROR "File not found: @LOCAL@")
+endif()
+
+function(do_verify)
+  if("@ALGO@" STREQUAL "")
+    message(WARNING "File will not be verified since no URL_HASH specified")
+    return()
+  endif()
+
+  if("@EXPECT_VALUE@" STREQUAL "")
+    message(FATAL_ERROR "EXPECT_VALUE can't be empty")
+  endif()
+
+  message(STATUS
+"verifying file...
+     file='@LOCAL@'")
+
+  file("@ALGO@" "@LOCAL@" actual_value)
+
+  if(NOT "${actual_value}" STREQUAL "@EXPECT_VALUE@")
+    message(FATAL_ERROR
+"error: @ALGO@ hash of
+  @LOCAL@
+does not match expected value
+  expected: '@EXPECT_VALUE@'
+    actual: '${actual_value}'
+")
+  endif()
+
+  message(STATUS "verifying file... done")
+endfunction()
+
+do_verify()
+
+set(extract_script "@extract_script_filename@")
+if(NOT "${extract_script}" STREQUAL "")
+  include("${extract_script}")
+endif()

Modules/RepositoryInfo.txt.in

@@ -1,3 +0,0 @@
-repository='@repository@'
-module='@module@'
-tag='@tag@'

Tests/RunCMake/ExternalProject/NO_DEPENDS-CMP0114-NEW-stderr.txt

@@ -10,7 +10,7 @@ Call Stack \(most recent call first\):
   [^
 ]*/Modules/ExternalProject.cmake:[0-9]+ \(ExternalProject_Add_Step\)
   [^
-]*/Modules/ExternalProject.cmake:[0-9]+ \(_ep_add_mkdir_command\)
+]*/Modules/ExternalProject.cmake:[0-9]+ \(_ep_add_preconfigure_command\)
   NO_DEPENDS-CMP0114-Common.cmake:[0-9]+ \(ExternalProject_Add\)
   NO_DEPENDS-CMP0114-NEW.cmake:[0-9]+ \(include\)
   CMakeLists.txt:[0-9]+ \(include\)$

Tests/RunCMake/ExternalProject/NO_DEPENDS-CMP0114-WARN-stderr.txt

@@ -13,7 +13,7 @@ Call Stack \(most recent call first\):
   [^
 ]*/Modules/ExternalProject.cmake:[0-9]+ \(ExternalProject_Add_Step\)
   [^
-]*/Modules/ExternalProject.cmake:[0-9]+ \(_ep_add_mkdir_command\)
+]*/Modules/ExternalProject.cmake:[0-9]+ \(_ep_add_preconfigure_command\)
   NO_DEPENDS-CMP0114-Common.cmake:[0-9]+ \(ExternalProject_Add\)
   NO_DEPENDS-CMP0114-WARN.cmake:[0-9]+ \(include\)
   CMakeLists.txt:[0-9]+ \(include\)
@@ -68,7 +68,7 @@ Call Stack \(most recent call first\):
   [^
 ]*/Modules/ExternalProject.cmake:[0-9]+ \(ExternalProject_Add_Step\)
   [^
-]*/Modules/ExternalProject.cmake:[0-9]+ \(_ep_add_mkdir_command\)
+]*/Modules/ExternalProject.cmake:[0-9]+ \(_ep_add_preconfigure_command\)
   NO_DEPENDS-CMP0114-Common.cmake:[0-9]+ \(ExternalProject_Add\)
   NO_DEPENDS-CMP0114-WARN.cmake:[0-9]+ \(include\)
   CMakeLists.txt:[0-9]+ \(include\)

Tests/RunCMake/ExternalProject/NoOptions-stderr.txt

@@ -13,6 +13,6 @@
    \* HG_REPOSITORY
    \* CVS_REPOSITORY and CVS_MODULE
 Call Stack \(most recent call first\):
-  .*/Modules/ExternalProject.cmake:[0-9]+ \(_ep_add_download_command\)
+  .*/Modules/ExternalProject.cmake:[0-9]+ \(_ep_prepare_download\)
   NoOptions.cmake:[0-9]+ \(ExternalProject_Add\)
   CMakeLists.txt:[0-9]+ \(include\)$

Tests/RunCMake/ExternalProject/SourceEmpty-stderr.txt

@@ -13,6 +13,6 @@
    \* HG_REPOSITORY
    \* CVS_REPOSITORY and CVS_MODULE
 Call Stack \(most recent call first\):
-  .*/Modules/ExternalProject.cmake:[0-9]+ \(_ep_add_download_command\)
+  .*/Modules/ExternalProject.cmake:[0-9]+ \(_ep_prepare_download\)
   SourceEmpty.cmake:[0-9]+ \(ExternalProject_Add\)
   CMakeLists.txt:[0-9]+ \(include\)$

Tests/RunCMake/ExternalProject/SourceMissing-stderr.txt

@@ -13,6 +13,6 @@
    \* HG_REPOSITORY
    \* CVS_REPOSITORY and CVS_MODULE
 Call Stack \(most recent call first\):
-  .*/Modules/ExternalProject.cmake:[0-9]+ \(_ep_add_download_command\)
+  .*/Modules/ExternalProject.cmake:[0-9]+ \(_ep_prepare_download\)
   SourceMissing.cmake:[0-9]+ \(ExternalProject_Add\)
   CMakeLists.txt:[0-9]+ \(include\)$

Tests/RunCMake/ExternalProject/UsesTerminal-check.cmake

@@ -19,7 +19,7 @@ cmake_minimum_required(VERSION 3.3)
 # console pool.
 macro(CheckNinjaStep _target _step _require)
   if("${_build}" MATCHES
-"  DESC = Performing ${_step} step for '${_target}'
+"  DESC = Performing ${_step} step (\\([a-zA-Z0-9 ]*\\) )?for '${_target}'
   pool = console"
   )
     if(NOT ${_require})