Merge topic 'ExternalProject-improve-retry'

89113e12 ExternalProject: Re-implement download logic as a dedicated script
e7d5e4b4 ExternalProject: Re-implement download verification as a dedicated script
ebcc7027 ExternalProject: Avoid repeating download verification
33218f6a ExternalProject: Remove unused verify script logic
e5409d1e ExternalProject: Remove unused 'retries' argument from verify script
d610407c ExternalProject: Use uppercase placeholders for script generation

Modules/ExternalProject-download.cmake.in

@@ -0,0 +1,161 @@
+#=============================================================================
+# Copyright 2008-2013 Kitware, Inc.
+# Copyright 2016 Ruslan Baratov
+#
+# Distributed under the OSI-approved BSD License (the "License");
+# see accompanying file Copyright.txt for details.
+#
+# This software is distributed WITHOUT ANY WARRANTY; without even the
+# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+# See the License for more information.
+#=============================================================================
+# (To distribute this file outside of CMake, substitute the full
+#  License text for the above reference.)
+
+cmake_minimum_required(VERSION 3.5)
+
+function(check_file_hash has_hash hash_is_good)
+  if("${has_hash}" STREQUAL "")
+    message(FATAL_ERROR "has_hash Can't be empty")
+  endif()
+
+  if("${hash_is_good}" STREQUAL "")
+    message(FATAL_ERROR "hash_is_good Can't be empty")
+  endif()
+
+  if("@ALGO@" STREQUAL "")
+    # No check
+    set("${has_hash}" FALSE PARENT_SCOPE)
+    set("${hash_is_good}" FALSE PARENT_SCOPE)
+    return()
+  endif()
+
+  set("${has_hash}" TRUE PARENT_SCOPE)
+
+  message(STATUS "verifying file...
+       file='@LOCAL@'")
+
+  file("@ALGO@" "@LOCAL@" actual_value)
+
+  if(NOT "${actual_value}" STREQUAL "@EXPECT_VALUE@")
+    set("${hash_is_good}" FALSE PARENT_SCOPE)
+    message(STATUS "@ALGO@ hash of
+    @LOCAL@
+  does not match expected value
+    expected: '@EXPECT_VALUE@'
+      actual: '${actual_value}'")
+  else()
+    set("${hash_is_good}" TRUE PARENT_SCOPE)
+  endif()
+endfunction()
+
+function(sleep_before_download attempt)
+  if(attempt EQUAL 0)
+    return()
+  endif()
+
+  if(attempt EQUAL 1)
+    message(STATUS "Retrying...")
+    return()
+  endif()
+
+  set(sleep_seconds 0)
+
+  if(attempt EQUAL 2)
+    set(sleep_seconds 5)
+  elseif(attempt EQUAL 3)
+    set(sleep_seconds 5)
+  elseif(attempt EQUAL 4)
+    set(sleep_seconds 15)
+  elseif(attempt EQUAL 5)
+    set(sleep_seconds 60)
+  elseif(attempt EQUAL 6)
+    set(sleep_seconds 90)
+  elseif(attempt EQUAL 7)
+    set(sleep_seconds 300)
+  else()
+    set(sleep_seconds 1200)
+  endif()
+
+  message(STATUS "Retry after ${sleep_seconds} seconds (attempt #${attempt}) ...")
+
+  execute_process(COMMAND "${CMAKE_COMMAND}" -E sleep "${sleep_seconds}")
+endfunction()
+
+if("@LOCAL@" STREQUAL "")
+  message(FATAL_ERROR "LOCAL can't be empty")
+endif()
+
+if("@REMOTE@" STREQUAL "")
+  message(FATAL_ERROR "REMOTE can't be empty")
+endif()
+
+if(EXISTS "@LOCAL@")
+  check_file_hash(has_hash hash_is_good)
+  if(has_hash)
+    if(hash_is_good)
+      message(STATUS "File already exists and hash match (skip download):
+  file='@LOCAL@'
+  @ALGO@='@EXPECT_VALUE@'"
+      )
+      return()
+    else()
+      message(STATUS "File already exists but hash mismatch. Removing...")
+      file(REMOVE "@LOCAL@")
+    endif()
+  else()
+    message(STATUS "File already exists but no hash specified (use URL_HASH):
+  file='@LOCAL@'
+Old file will be removed and new file downloaded from URL."
+    )
+    file(REMOVE "@LOCAL@")
+  endif()
+endif()
+
+set(retry_number 5)
+
+foreach(i RANGE ${retry_number})
+  sleep_before_download(${i})
+
+  message(STATUS "downloading...
+       src='@REMOTE@'
+       dst='@LOCAL@'
+       timeout='@TIMEOUT_MSG@'")
+
+  @TLS_VERIFY_CODE@
+  @TLS_CAINFO_CODE@
+
+  file(
+      DOWNLOAD
+      "@REMOTE@" "@LOCAL@"
+      @SHOW_PROGRESS@
+      @TIMEOUT_ARGS@
+      STATUS status
+      LOG log
+  )
+
+  list(GET status 0 status_code)
+  list(GET status 1 status_string)
+
+  if(status_code EQUAL 0)
+    check_file_hash(has_hash hash_is_good)
+    if(has_hash AND NOT hash_is_good)
+      message(STATUS "Hash mismatch, removing...")
+      file(REMOVE "@LOCAL@")
+    else()
+      message(STATUS "Downloading... done")
+      return()
+    endif()
+  else()
+    message("error: downloading '@REMOTE@' failed
+  status_code: ${status_code}
+  status_string: ${status_string}
+  log:
+  --- LOG BEGIN ---
+  ${log}
+  --- LOG END ---"
+    )
+  endif()
+endforeach()
+
+message(FATAL_ERROR "Downloading failed")

Modules/ExternalProject-verify.cmake.in

@@ -0,0 +1,48 @@
+#=============================================================================
+# Copyright 2008-2013 Kitware, Inc.
+# Copyright 2016 Ruslan Baratov
+#
+# Distributed under the OSI-approved BSD License (the "License");
+# see accompanying file Copyright.txt for details.
+#
+# This software is distributed WITHOUT ANY WARRANTY; without even the
+# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+# See the License for more information.
+#=============================================================================
+# (To distribute this file outside of CMake, substitute the full
+#  License text for the above reference.)
+
+cmake_minimum_required(VERSION 3.5)
+
+if("@LOCAL@" STREQUAL "")
+  message(FATAL_ERROR "LOCAL can't be empty")
+endif()
+
+if(NOT EXISTS "@LOCAL@")
+  message(FATAL_ERROR "File not found: @LOCAL@")
+endif()
+
+if("@ALGO@" STREQUAL "")
+  message(WARNING "File will not be verified since no URL_HASH specified")
+  return()
+endif()
+
+if("@EXPECT_VALUE@" STREQUAL "")
+  message(FATAL_ERROR "EXPECT_VALUE can't be empty")
+endif()
+
+message(STATUS "verifying file...
+     file='@LOCAL@'")
+
+file("@ALGO@" "@LOCAL@" actual_value)
+
+if(NOT "${actual_value}" STREQUAL "@EXPECT_VALUE@")
+  message(FATAL_ERROR "error: @ALGO@ hash of
+  @LOCAL@
+does not match expected value
+  expected: '@EXPECT_VALUE@'
+    actual: '${actual_value}'
+")
+endif()
+
+message(STATUS "verifying file... done")

Modules/ExternalProject.cmake

@@ -377,6 +377,7 @@ file::
 
 #=============================================================================
 # Copyright 2008-2013 Kitware, Inc.
+# Copyright 2016 Ruslan Baratov
 #
 # Distributed under the OSI-approved BSD License (the "License");
 # see accompanying file Copyright.txt for details.
@@ -418,6 +419,9 @@ endif()
 set(_ep_hash_algos "MD5|SHA1|SHA224|SHA256|SHA384|SHA512")
 set(_ep_hash_regex "^(${_ep_hash_algos})=([0-9A-Fa-f]+)$")
 
+set(_ExternalProject_SELF "${CMAKE_CURRENT_LIST_FILE}")
+get_filename_component(_ExternalProject_SELF_DIR "${_ExternalProject_SELF}" PATH)
+
 function(_ep_parse_arguments f name ns args)
   # Transfer the arguments to this function into target properties for the
   # new custom target we just added so that we can set up all the build steps
@@ -847,43 +851,38 @@ endif()
 
 endfunction(_ep_write_gitupdate_script)
 
-function(_ep_write_downloadfile_script script_filename remote local timeout no_progress hash tls_verify tls_cainfo)
+function(_ep_write_downloadfile_script script_filename REMOTE LOCAL timeout no_progress hash tls_verify tls_cainfo)
   if(timeout)
-    set(timeout_args TIMEOUT ${timeout})
-    set(timeout_msg "${timeout} seconds")
+    set(TIMEOUT_ARGS TIMEOUT ${timeout})
+    set(TIMEOUT_MSG "${timeout} seconds")
   else()
-    set(timeout_args "# no TIMEOUT")
-    set(timeout_msg "none")
+    set(TIMEOUT_ARGS "# no TIMEOUT")
+    set(TIMEOUT_MSG "none")
   endif()
 
   if(no_progress)
-    set(show_progress "")
+    set(SHOW_PROGRESS "")
   else()
-    set(show_progress "SHOW_PROGRESS")
+    set(SHOW_PROGRESS "SHOW_PROGRESS")
   endif()
 
   if("${hash}" MATCHES "${_ep_hash_regex}")
-    string(CONCAT hash_check
-      "if(EXISTS \"${local}\")\n"
-      "  file(\"${CMAKE_MATCH_1}\" \"${local}\" hash_value)\n"
-      "  if(\"x\${hash_value}\" STREQUAL \"x${CMAKE_MATCH_2}\")\n"
-      "    return()\n"
-      "  endif()\n"
-      "endif()\n"
-      )
+    set(ALGO "${CMAKE_MATCH_1}")
+    set(EXPECT_VALUE "${CMAKE_MATCH_2}")
   else()
-    set(hash_check "")
+    set(ALGO "")
+    set(EXPECT_VALUE "")
   endif()
 
-  set(tls_verify_code "")
-  set(tls_cainfo_code "")
+  set(TLS_VERIFY_CODE "")
+  set(TLS_CAINFO_CODE "")
 
   # check for curl globals in the project
   if(DEFINED CMAKE_TLS_VERIFY)
-    set(tls_verify_code "set(CMAKE_TLS_VERIFY ${CMAKE_TLS_VERIFY})")
+    set(TLS_VERIFY_CODE "set(CMAKE_TLS_VERIFY ${CMAKE_TLS_VERIFY})")
   endif()
   if(DEFINED CMAKE_TLS_CAINFO)
-    set(tls_cainfo_code "set(CMAKE_TLS_CAINFO \"${CMAKE_TLS_CAINFO}\")")
+    set(TLS_CAINFO_CODE "set(CMAKE_TLS_CAINFO \"${CMAKE_TLS_CAINFO}\")")
   endif()
 
   # now check for curl locals so that the local values
@@ -892,91 +891,49 @@ function(_ep_write_downloadfile_script script_filename remote local timeout no_p
   # check for tls_verify argument
   string(LENGTH "${tls_verify}" tls_verify_len)
   if(tls_verify_len GREATER 0)
-    set(tls_verify_code "set(CMAKE_TLS_VERIFY ${tls_verify})")
+    set(TLS_VERIFY_CODE "set(CMAKE_TLS_VERIFY ${tls_verify})")
   endif()
   # check for tls_cainfo argument
   string(LENGTH "${tls_cainfo}" tls_cainfo_len)
   if(tls_cainfo_len GREATER 0)
-    set(tls_cainfo_code "set(CMAKE_TLS_CAINFO \"${tls_cainfo}\")")
-  endif()
-
-  file(WRITE ${script_filename}
-"${hash_check}message(STATUS \"downloading...
-     src='${remote}'
-     dst='${local}'
-     timeout='${timeout_msg}'\")
-
-${tls_verify_code}
-${tls_cainfo_code}
-
-file(DOWNLOAD
-  \"${remote}\"
-  \"${local}\"
-  ${show_progress}
-  ${timeout_args}
-  STATUS status
-  LOG log)
-
-list(GET status 0 status_code)
-list(GET status 1 status_string)
-
-if(NOT status_code EQUAL 0)
-  message(FATAL_ERROR \"error: downloading '${remote}' failed
-  status_code: \${status_code}
-  status_string: \${status_string}
-  log: \${log}
-\")
-endif()
-
-message(STATUS \"downloading... done\")
-"
-)
-
+    set(TLS_CAINFO_CODE "set(CMAKE_TLS_CAINFO \"${tls_cainfo}\")")
+  endif()
+
+  # Used variables:
+  # * TLS_VERIFY_CODE
+  # * TLS_CAINFO_CODE
+  # * ALGO
+  # * EXPECT_VALUE
+  # * REMOTE
+  # * LOCAL
+  # * SHOW_PROGRESS
+  # * TIMEOUT_ARGS
+  # * TIMEOUT_MSG
+  configure_file(
+      "${_ExternalProject_SELF_DIR}/ExternalProject-download.cmake.in"
+      "${script_filename}"
+      @ONLY
+  )
 endfunction()
 
-
-function(_ep_write_verifyfile_script script_filename local hash retries download_script)
+function(_ep_write_verifyfile_script script_filename LOCAL hash)
   if("${hash}" MATCHES "${_ep_hash_regex}")
-    set(algo "${CMAKE_MATCH_1}")
-    string(TOLOWER "${CMAKE_MATCH_2}" expect_value)
-    set(script_content "set(expect_value \"${expect_value}\")
-set(attempt 0)
-set(succeeded 0)
-while(\${attempt} LESS ${retries} OR \${attempt} EQUAL ${retries} AND NOT \${succeeded})
-  file(${algo} \"\${file}\" actual_value)
-  if(\"\${actual_value}\" STREQUAL \"\${expect_value}\")
-    set(succeeded 1)
-  elseif(\${attempt} LESS ${retries})
-    message(STATUS \"${algo} hash of \${file}
-does not match expected value
-  expected: \${expect_value}
-    actual: \${actual_value}
-Retrying download.
-\")
-    file(REMOVE \"\${file}\")
-    execute_process(COMMAND \${CMAKE_COMMAND} -P \"${download_script}\")
-  endif()
-  math(EXPR attempt \"\${attempt} + 1\")
-endwhile()
-
-if(\${succeeded})
-  message(STATUS \"verifying file... done\")
-else()
-  message(FATAL_ERROR \"error: ${algo} hash of
-  \${file}
-does not match expected value
-  expected: \${expect_value}
-    actual: \${actual_value}
-\")
-endif()")
+    set(ALGO "${CMAKE_MATCH_1}")
+    string(TOLOWER "${CMAKE_MATCH_2}" EXPECT_VALUE)
   else()
-    set(script_content "message(STATUS \"verifying file... warning: did not verify file - no URL_HASH specified?\")")
+    set(ALGO "")
+    set(EXPECT_VALUE "")
   endif()
-  file(WRITE ${script_filename} "set(file \"${local}\")
-message(STATUS \"verifying file...
-     file='\${file}'\")
-${script_content}
-")
+
+  # Used variables:
+  # * ALGO
+  # * EXPECT_VALUE
+  # * LOCAL
+  configure_file(
+      "${_ExternalProject_SELF_DIR}/ExternalProject-verify.cmake.in"
+      "${script_filename}"
+      @ONLY
+  )
 endfunction()
 
 
@@ -1898,8 +1855,6 @@ function(_ep_add_download_command name)
     set(repository "external project URL")
     set(module "${url}")
     set(tag "${hash}")
-    set(retries 0)
-    set(download_script "")
     configure_file(
       "${CMAKE_ROOT}/Modules/RepositoryInfo.txt.in"
       "${stamp_dir}/${name}-urlinfo.txt"
@@ -1937,13 +1892,13 @@ function(_ep_add_download_command name)
         _ep_write_downloadfile_script("${download_script}" "${url}" "${file}" "${timeout}" "${no_progress}" "${hash}" "${tls_verify}" "${tls_cainfo}")
         set(cmd ${CMAKE_COMMAND} -P "${download_script}"
           COMMAND)
-        set(retries 3)
         if (no_extract)
           set(steps "download and verify")
         else ()
           set(steps "download, verify and extract")
         endif ()
         set(comment "Performing download step (${steps}) for '${name}'")
+        file(WRITE "${stamp_dir}/verify-${name}.cmake" "") # already verified by 'download_script'
       else()
         set(file "${url}")
         if (no_extract)
@@ -1952,8 +1907,8 @@ function(_ep_add_download_command name)
           set(steps "verify and extract")
         endif ()
         set(comment "Performing download step (${steps}) for '${name}'")
+        _ep_write_verifyfile_script("${stamp_dir}/verify-${name}.cmake" "${file}" "${hash}")
       endif()
-      _ep_write_verifyfile_script("${stamp_dir}/verify-${name}.cmake" "${file}" "${hash}" "${retries}" "${download_script}")
       list(APPEND cmd ${CMAKE_COMMAND} -P ${stamp_dir}/verify-${name}.cmake
         COMMAND)
       if (NOT no_extract)