ExternalProject: Provide choice of git update strategies

Fixes: #16528

Co-Authored-By: Michael Wake <[email protected]>

Modules/ExternalProject-gitupdate.cmake.in

@@ -59,7 +59,7 @@ if(error_code OR is_remote_ref OR NOT ("${tag_sha}" STREQUAL "${head_sha}"))
     message(FATAL_ERROR "Failed to fetch repository '@git_repository@'")
   endif()
 
-  if(is_remote_ref)
+  if(is_remote_ref AND NOT "@git_update_strategy@" STREQUAL "CHECKOUT")
     # Check if stash is needed
     execute_process(
       COMMAND "@git_EXECUTABLE@" status --porcelain
@@ -90,21 +90,61 @@ if(error_code OR is_remote_ref OR NOT ("${tag_sha}" STREQUAL "${head_sha}"))
       COMMAND "@git_EXECUTABLE@" rebase "${git_remote}/${git_tag}"
       WORKING_DIRECTORY "@work_dir@"
       RESULT_VARIABLE error_code
+      OUTPUT_VARIABLE rebase_output
+      ERROR_VARIABLE  rebase_output
       )
     if(error_code)
-      # Rebase failed: Restore previous state.
+      # Rebase failed, undo the rebase attempt before continuing
       execute_process(
         COMMAND "@git_EXECUTABLE@" rebase --abort
         WORKING_DIRECTORY "@work_dir@"
       )
-      if(need_stash)
-        execute_process(
-          COMMAND "@git_EXECUTABLE@" stash pop --index --quiet
-          WORKING_DIRECTORY "@work_dir@"
-          )
+
+      if(NOT "@git_update_strategy@" STREQUAL "REBASE_CHECKOUT")
+        # Not allowed to do a checkout as a fallback, so cannot proceed
+        if(need_stash)
+          execute_process(
+            COMMAND "@git_EXECUTABLE@" stash pop --index --quiet
+            WORKING_DIRECTORY "@work_dir@"
+            )
+        endif()
+        message(FATAL_ERROR "\nFailed to rebase in: '@work_dir@'."
+                            "\nOutput from the attempted rebase follows:"
+                            "\n${rebase_output}"
+                            "\n\nYou will have to resolve the conflicts manually")
       endif()
-      message(FATAL_ERROR "\nFailed to rebase in: '@work_dir@'."
-                          "\nYou will have to resolve the conflicts manually")
+
+      # Fall back to checkout. We create an annotated tag so that the user
+      # can manually inspect the situation and revert if required.
+      # We can't log the failed rebase output because MSVC sees it and
+      # intervenes, causing the build to fail even though it completes.
+      # Write it to a file instead.
+      string(TIMESTAMP tag_timestamp "%Y%m%dT%H%M%S" UTC)
+      set(tag_name _cmake_ExternalProject_moved_from_here_${tag_timestamp}Z)
+      set(error_log_file ${CMAKE_CURRENT_LIST_DIR}/rebase_error_${tag_timestamp}Z.log)
+      file(WRITE ${error_log_file} "${rebase_output}")
+      message(WARNING "Rebase failed, output has been saved to ${error_log_file}"
+                      "\nFalling back to checkout, previous commit tagged as ${tag_name}")
+      execute_process(
+        COMMAND "@git_EXECUTABLE@" tag -a
+                -m "ExternalProject attempting to move from here to ${git_remote}/${git_tag}"
+                ${tag_name}
+        WORKING_DIRECTORY "@work_dir@"
+        RESULT_VARIABLE error_code
+      )
+      if(error_code)
+        message(FATAL_ERROR "Failed to add marker tag")
+      endif()
+
+      execute_process(
+        COMMAND "@git_EXECUTABLE@" checkout ${git_remote}/${git_tag}
+        WORKING_DIRECTORY "@work_dir@"
+        RESULT_VARIABLE error_code
+      )
+      if(error_code)
+        message(FATAL_ERROR "Failed to checkout : '${git_remote}/${git_tag}'")
+      endif()
+
     endif()
 
     if(need_stash)

Modules/ExternalProject.cmake

@@ -294,6 +294,42 @@ External Project Definition
         ``git clone`` command line, with each option required to be in the
         form ``key=value``.
 
+      ``GIT_REMOTE_UPDATE_STRATEGY <strategy>``
+        When ``GIT_TAG`` refers to a remote branch, this option can be used to
+        specify how the update step behaves.  The ``<strategy>`` must be one of
+        the following:
+
+        ``CHECKOUT``
+          Ignore the local branch and always checkout the branch specified by
+          ``GIT_TAG``.
+
+        ``REBASE``
+          Try to rebase the current branch to the one specified by ``GIT_TAG``.
+          If there are local uncommitted changes, they will be stashed first
+          and popped again after rebasing.  If rebasing or popping stashed
+          changes fail, abort the rebase and halt with an error.
+          When ``GIT_REMOTE_UPDATE_STRATEGY`` is not present, this is the
+          default strategy unless the default has been overridden with
+          ``CMAKE_EP_GIT_REMOTE_UPDATE_STRATEGY`` (see below).
+
+        ``REBASE_CHECKOUT``
+          Same as ``REBASE`` except if the rebase fails, an annotated tag will
+          be created at the original ``HEAD`` position from before the rebase
+          and then checkout ``GIT_TAG`` just like the ``CHECKOUT`` strategy.
+          The message stored on the annotated tag will give information about
+          what was attempted and the tag name will include a timestamp so that
+          each failed run will add a new tag.  This strategy ensures no changes
+          will be lost, but updates should always succeed if ``GIT_TAG`` refers
+          to a valid ref unless there are uncommitted changes that cannot be
+          popped successfully.
+
+        The variable ``CMAKE_EP_GIT_REMOTE_UPDATE_STRATEGY`` can be set to
+        override the default strategy.  This variable should not be set by a
+        project, it is intended for the user to set.  It is primarily intended
+        for use in continuous integration scripts to ensure that when history
+        is rewritten on a remote branch, the build doesn't end up with unintended
+        changes or failed builds resulting from conflicts during rebase operations.
+
     *Subversion*
       ``SVN_REPOSITORY <url>``
         URL of the Subversion repository.
@@ -938,6 +974,7 @@ The custom step could then be triggered from the main build like so::
 
 cmake_policy(PUSH)
 cmake_policy(SET CMP0054 NEW) # if() quoted variables not dereferenced
+cmake_policy(SET CMP0057 NEW) # if() supports IN_LIST
 
 # Pre-compute a regex to match documented keywords for each command.
 math(EXPR _ep_documentation_line_count "${CMAKE_CURRENT_LIST_LINE} - 4")
@@ -1242,7 +1279,7 @@ endif()
 endfunction()
 
 
-function(_ep_write_gitupdate_script script_filename git_EXECUTABLE git_tag git_remote_name init_submodules git_submodules_recurse git_submodules git_repository work_dir)
+function(_ep_write_gitupdate_script script_filename git_EXECUTABLE git_tag git_remote_name init_submodules git_submodules_recurse git_submodules git_repository work_dir git_update_strategy)
   if("${git_tag}" STREQUAL "")
     message(FATAL_ERROR "Tag for git checkout should not be empty.")
   endif()
@@ -2631,10 +2668,22 @@ function(_ep_add_update_command name)
       endif()
     endif()
 
+    get_property(git_update_strategy TARGET ${name} PROPERTY _EP_GIT_REMOTE_UPDATE_STRATEGY)
+    if(NOT git_update_strategy)
+      set(git_update_strategy "${CMAKE_EP_GIT_REMOTE_UPDATE_STRATEGY}")
+    endif()
+    if(NOT git_update_strategy)
+      set(git_update_strategy REBASE)
+    endif()
+    set(strategies CHECKOUT REBASE REBASE_CHECKOUT)
+    if(NOT git_update_strategy IN_LIST strategies)
+      message(FATAL_ERROR "'${git_update_strategy}' is not one of the supported strategies: ${strategies}")
+    endif()
+
     _ep_get_git_submodules_recurse(git_submodules_recurse)
 
     _ep_write_gitupdate_script(${tmp_dir}/${name}-gitupdate.cmake
-      ${GIT_EXECUTABLE} ${git_tag} ${git_remote_name} ${git_init_submodules} "${git_submodules_recurse}" "${git_submodules}" ${git_repository} ${work_dir}
+      ${GIT_EXECUTABLE} ${git_tag} ${git_remote_name} ${git_init_submodules} "${git_submodules_recurse}" "${git_submodules}" ${git_repository} ${work_dir} ${git_update_strategy}
       )
     set(cmd ${CMAKE_COMMAND} -P ${tmp_dir}/${name}-gitupdate.cmake)
     set(always 1)

Tests/ExternalProjectUpdate/CMakeLists.txt

@@ -78,6 +78,8 @@ if(do_git_tests)
   ExternalProject_Add(${proj}
     GIT_REPOSITORY "${local_git_repo}"
     GIT_TAG ${TEST_GIT_TAG}
+    GIT_CONFIG "[email protected]"
+               "user.name=testauthor"
     CMAKE_GENERATOR "${CMAKE_GENERATOR}"
     CMAKE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=<INSTALL_DIR>
     INSTALL_COMMAND ""

Tests/ExternalProjectUpdate/ExternalProjectUpdateTest.cmake

@@ -2,7 +2,7 @@
 # resulting checked out version is resulting_sha and rebuild.
 # This check's the correct behavior of the ExternalProject UPDATE_COMMAND.
 # Also verify that a fetch only occurs when fetch_expected is 1.
-macro(check_a_tag desired_tag resulting_sha fetch_expected)
+macro(check_a_tag desired_tag resulting_sha fetch_expected update_strategy)
   message( STATUS "Checking ExternalProjectUpdate to tag: ${desired_tag}" )
 
   # Remove the FETCH_HEAD file, so we can check if it gets replaced with a 'git
@@ -10,11 +10,16 @@ macro(check_a_tag desired_tag resulting_sha fetch_expected)
   set( FETCH_HEAD_file ${ExternalProjectUpdate_BINARY_DIR}/CMakeExternals/Source/TutorialStep1-GIT/.git/FETCH_HEAD )
   file( REMOVE ${FETCH_HEAD_file} )
 
+  # Give ourselves a marker in the output. It is difficult to tell where we
+  # are up to without this
+  message(STATUS "===> check_a_tag ${desired_tag} ${resulting_sha} ${fetch_expected} ${update_strategy}")
+
   # Configure
   execute_process(COMMAND ${CMAKE_COMMAND}
     -G ${CMAKE_GENERATOR} -T "${CMAKE_GENERATOR_TOOLSET}"
     -A "${CMAKE_GENERATOR_PLATFORM}"
     -DTEST_GIT_TAG:STRING=${desired_tag}
+    -DCMAKE_EP_GIT_REMOTE_UPDATE_STRATEGY:STRING=${update_strategy}
     ${ExternalProjectUpdate_SOURCE_DIR}
     WORKING_DIRECTORY ${ExternalProjectUpdate_BINARY_DIR}
     RESULT_VARIABLE error_code
@@ -176,16 +181,48 @@ if(GIT_EXECUTABLE)
   endif()
 endif()
 
+# When re-running tests locally, this ensures we always start afresh
+file(REMOVE_RECURSE ${ExternalProjectUpdate_BINARY_DIR}/CMakeExternals)
+
 if(do_git_tests)
-  check_a_tag(origin/master 5842b503ba4113976d9bb28d57b5aee1ad2736b7 1)
-  check_a_tag(tag1          d1970730310fe8bc07e73f15dc570071f9f9654a 1)
+  check_a_tag(origin/master 5842b503ba4113976d9bb28d57b5aee1ad2736b7 1 REBASE)
+  check_a_tag(tag1          d1970730310fe8bc07e73f15dc570071f9f9654a 1 REBASE)
   # With the Git UPDATE_COMMAND performance patch, this will not required a
   # 'git fetch'
-  check_a_tag(tag1          d1970730310fe8bc07e73f15dc570071f9f9654a 0)
-  check_a_tag(tag2          5842b503ba4113976d9bb28d57b5aee1ad2736b7 1)
-  check_a_tag(d19707303     d1970730310fe8bc07e73f15dc570071f9f9654a 1)
-  check_a_tag(d19707303     d1970730310fe8bc07e73f15dc570071f9f9654a 0)
-  check_a_tag(origin/master 5842b503ba4113976d9bb28d57b5aee1ad2736b7 1)
+  check_a_tag(tag1          d1970730310fe8bc07e73f15dc570071f9f9654a 0 REBASE)
+  check_a_tag(tag2          5842b503ba4113976d9bb28d57b5aee1ad2736b7 1 REBASE)
+  check_a_tag(d19707303     d1970730310fe8bc07e73f15dc570071f9f9654a 1 REBASE)
+  check_a_tag(d19707303     d1970730310fe8bc07e73f15dc570071f9f9654a 0 REBASE)
+  check_a_tag(origin/master 5842b503ba4113976d9bb28d57b5aee1ad2736b7 1 REBASE)
   # This is a remote symbolic ref, so it will always trigger a 'git fetch'
-  check_a_tag(origin/master 5842b503ba4113976d9bb28d57b5aee1ad2736b7 1)
+  check_a_tag(origin/master 5842b503ba4113976d9bb28d57b5aee1ad2736b7 1 REBASE)
+
+  foreach(strategy IN ITEMS CHECKOUT REBASE_CHECKOUT)
+    # Move local master back, then apply a change that will cause a conflict
+    # during rebase. We want to test the fallback to checkout.
+    check_a_tag(master 5842b503ba4113976d9bb28d57b5aee1ad2736b7 1 REBASE)
+    execute_process(COMMAND ${GIT_EXECUTABLE} reset --hard tag1
+      WORKING_DIRECTORY ${ExternalProjectUpdate_BINARY_DIR}/CMakeExternals/Source/TutorialStep1-GIT
+      RESULT_VARIABLE error_code
+      )
+    if(error_code)
+      message(FATAL_ERROR "Could not reset local master back to tag1.")
+    endif()
+    set(cmlFile ${ExternalProjectUpdate_BINARY_DIR}/CMakeExternals/Source/TutorialStep1-GIT/CMakeLists.txt)
+    file(READ ${cmlFile} contents)
+    string(REPLACE "find TutorialConfig.h" "find TutorialConfig.h (conflict here)"
+      conflictingContent "${contents}"
+      )
+    file(WRITE ${cmlFile} "${conflictingContent}")
+    execute_process(COMMAND ${GIT_EXECUTABLE} commit -a -m "This should cause a conflict"
+      WORKING_DIRECTORY ${ExternalProjectUpdate_BINARY_DIR}/CMakeExternals/Source/TutorialStep1-GIT
+      RESULT_VARIABLE error_code
+      )
+    if(error_code)
+      message(FATAL_ERROR "Could not commit conflicting change.")
+    endif()
+    # This should discard our commit but leave behind an annotated tag
+    check_a_tag(master 5842b503ba4113976d9bb28d57b5aee1ad2736b7 1 ${strategy})
+  endforeach()
+
 endif()