ExternalData: Add support for custom download scripts

Add support for a special URL template to map the fetch operation
to a project-specified .cmake script insead of using file(DOWNLOAD).

Extend the Module.ExternalData test to cover the behavior.
Extend the RunCMake.ExternalData test to cover error cases.

Modules/ExternalData.cmake

@@ -99,6 +99,12 @@ calling any of the functions provided by this module.
   default is ``CMAKE_BINARY_DIR``.  The directory layout will mirror that of
   content links under ``ExternalData_SOURCE_ROOT``.
 
+.. variable:: ExternalData_CUSTOM_SCRIPT_<key>
+
+  Specify a full path to a ``.cmake`` custom fetch script identified by
+  ``<key>`` in entries of the ``ExternalData_URL_TEMPLATES`` list.
+  See `Custom Fetch Scripts`_.
+
 .. variable:: ExternalData_LINK_CONTENT
 
   The ``ExternalData_LINK_CONTENT`` variable may be set to the name of a
@@ -246,6 +252,44 @@ The following hash algorithms are supported::
 
 Note that the hashes are used only for unique data identification and
 download verification.
+
+Custom Fetch Scripts
+^^^^^^^^^^^^^^^^^^^^
+
+When a data file must be fetched from one of the URL templates
+specified in the ``ExternalData_URL_TEMPLATES`` variable, it is
+normally downloaded using the :command:`file(DOWNLOAD)` command.
+One may specify usage of a custom fetch script by using a URL
+template of the form ``ExternalDataCustomScript://<key>/<loc>``.
+The ``<key>`` must be a C identifier, and the ``<loc>`` must
+contain the ``%(algo)`` and ``%(hash)`` placeholders.
+A variable corresponding to the key, ``ExternalData_CUSTOM_SCRIPT_<key>``,
+must be set to the full path to a ``.cmake`` script file.  The script
+will be included to perform the actual fetch, and provided with
+the following variables:
+
+.. variable:: ExternalData_CUSTOM_LOCATION
+
+  When a custom fetch script is loaded, this variable is set to the
+  location part of the URL, which will contain the substituted hash
+  algorithm name and content hash value.
+
+.. variable:: ExternalData_CUSTOM_FILE
+
+  When a custom fetch script is loaded, this variable is set to the
+  full path to a file in which the script must store the fetched
+  content.  The name of the file is unspecified and should not be
+  interpreted in any way.
+
+The custom fetch script is expected to store fetched content in the
+file or set a variable:
+
+.. variable:: ExternalData_CUSTOM_ERROR
+
+  When a custom fetch script fails to fetch the requested content,
+  it must set this variable to a short one-line message describing
+  the reason for failure.
+
 #]=======================================================================]
 
 #=============================================================================
@@ -275,6 +319,37 @@ function(ExternalData_add_target target)
   if(NOT ExternalData_OBJECT_STORES)
     set(ExternalData_OBJECT_STORES ${CMAKE_BINARY_DIR}/ExternalData/Objects)
   endif()
+  set(_ExternalData_CONFIG_CODE "")
+
+  # Store custom script configuration.
+  foreach(url_template IN LISTS ExternalData_URL_TEMPLATES)
+    if("${url_template}" MATCHES "^ExternalDataCustomScript://([^/]*)/(.*)$")
+      set(key "${CMAKE_MATCH_1}")
+      if(key MATCHES "^[A-Za-z_][A-Za-z0-9_]*$")
+        if(ExternalData_CUSTOM_SCRIPT_${key})
+          if(IS_ABSOLUTE "${ExternalData_CUSTOM_SCRIPT_${key}}")
+            string(CONCAT _ExternalData_CONFIG_CODE "${_ExternalData_CONFIG_CODE}\n"
+              "set(ExternalData_CUSTOM_SCRIPT_${key} \"${ExternalData_CUSTOM_SCRIPT_${key}}\")")
+          else()
+            message(FATAL_ERROR
+              "No ExternalData_CUSTOM_SCRIPT_${key} is not set to a full path:\n"
+              " ${ExternalData_CUSTOM_SCRIPT_${key}}")
+          endif()
+        else()
+          message(FATAL_ERROR
+            "No ExternalData_CUSTOM_SCRIPT_${key} is set for URL template:\n"
+            " ${url_template}")
+        endif()
+      else()
+        message(FATAL_ERROR
+          "Bad ExternalDataCustomScript key '${key}' in URL template:\n"
+          " ${url_template}\n"
+          "The key must be a valid C identifier.")
+      endif()
+    endif()
+  endforeach()
+
+  # Store configuration for use by build-time script.
   set(config ${CMAKE_CURRENT_BINARY_DIR}/${target}_config.cmake)
   configure_file(${_ExternalData_SELF_DIR}/ExternalData_config.cmake.in ${config} @ONLY)
 
@@ -781,6 +856,30 @@ function(_ExternalData_download_file url file err_var msg_var)
   set("${msg_var}" "${msg}" PARENT_SCOPE)
 endfunction()
 
+function(_ExternalData_custom_fetch key loc file err_var msg_var)
+  if(NOT ExternalData_CUSTOM_SCRIPT_${key})
+    set(err 1)
+    set(msg "No ExternalData_CUSTOM_SCRIPT_${key} set!")
+  elseif(NOT EXISTS "${ExternalData_CUSTOM_SCRIPT_${key}}")
+    set(err 1)
+    set(msg "No '${ExternalData_CUSTOM_SCRIPT_${key}}' exists!")
+  else()
+    set(ExternalData_CUSTOM_LOCATION "${loc}")
+    set(ExternalData_CUSTOM_FILE "${file}")
+    unset(ExternalData_CUSTOM_ERROR)
+    include("${ExternalData_CUSTOM_SCRIPT_${key}}")
+    if(DEFINED ExternalData_CUSTOM_ERROR)
+      set(err 1)
+      set(msg "${ExternalData_CUSTOM_ERROR}")
+    else()
+      set(err 0)
+      set(msg "no error")
+    endif()
+  endif()
+  set("${err_var}" "${err}" PARENT_SCOPE)
+  set("${msg_var}" "${msg}" PARENT_SCOPE)
+endfunction()
+
 function(_ExternalData_download_object name hash algo var_obj)
   # Search all object stores for an existing object.
   foreach(dir ${ExternalData_OBJECT_STORES})
@@ -804,7 +903,11 @@ function(_ExternalData_download_object name hash algo var_obj)
     string(REPLACE "%(hash)" "${hash}" url_tmp "${url_template}")
     string(REPLACE "%(algo)" "${algo}" url "${url_tmp}")
     message(STATUS "Fetching \"${url}\"")
-    _ExternalData_download_file("${url}" "${tmp}" err errMsg)
+    if(url MATCHES "^ExternalDataCustomScript://([A-Za-z_][A-Za-z0-9_]*)/(.*)$")
+      _ExternalData_custom_fetch("${CMAKE_MATCH_1}" "${CMAKE_MATCH_2}" "${tmp}" err errMsg)
+    else()
+      _ExternalData_download_file("${url}" "${tmp}" err errMsg)
+    endif()
     set(tried "${tried}\n  ${url}")
     if(err)
       set(tried "${tried} (${errMsg})")

Modules/ExternalData_config.cmake.in

@@ -2,3 +2,4 @@ set(ExternalData_OBJECT_STORES "@ExternalData_OBJECT_STORES@")
 set(ExternalData_URL_TEMPLATES "@ExternalData_URL_TEMPLATES@")
 set(ExternalData_TIMEOUT_INACTIVITY "@ExternalData_TIMEOUT_INACTIVITY@")
 set(ExternalData_TIMEOUT_ABSOLUTE "@ExternalData_TIMEOUT_ABSOLUTE@")
+@_ExternalData_CONFIG_CODE@

Tests/Module/ExternalData/CMakeLists.txt

@@ -10,7 +10,9 @@ if(NOT "${CMAKE_CURRENT_SOURCE_DIR}" MATCHES "^/")
 endif()
 set(ExternalData_URL_TEMPLATES
   "file://${slash}${CMAKE_CURRENT_SOURCE_DIR}/%(algo)/%(hash)"
+  "ExternalDataCustomScript://MyScript1/%(algo)/%(hash)"
   )
+set(ExternalData_CUSTOM_SCRIPT_MyScript1 "${CMAKE_CURRENT_SOURCE_DIR}/MyScript1.cmake")
 set(ExternalData_BINARY_ROOT "${CMAKE_CURRENT_BINARY_DIR}/ExternalData")
 file(REMOVE_RECURSE ${ExternalData_BINARY_ROOT}) # clean test
 
@@ -23,6 +25,7 @@ ExternalData_Add_Test(Data1
   COMMAND ${CMAKE_COMMAND}
     -D Data=DATA{Data.dat}
     ${Data1CheckSpaces}
+    -D DataScript=DATA{DataScript.dat}
     -D DataMissing=DATA{DataMissing.dat}
     -D DataMissingWithAssociated=DATA{DataMissing.dat,Data.dat}
     -D SeriesA=DATA{SeriesA.dat,:}

Tests/Module/ExternalData/Data1Check.cmake

@@ -8,6 +8,10 @@ if(DEFINED DataSpace)
     message(SEND_ERROR "Input file:\n  ${DataSpace}\ndoes not have expected content, but [[${lines}]]")
   endif()
 endif()
+file(STRINGS "${DataScript}" lines LIMIT_INPUT 1024)
+if(NOT "x${lines}" STREQUAL "xDataScript")
+  message(SEND_ERROR "Input file:\n  ${DataScript}\ndoes not have expected content, but [[${lines}]]")
+endif()
 if(DataMissing)
   if(EXISTS "${DataMissing}")
     message(SEND_ERROR

Tests/Module/ExternalData/DataScript.dat.md5

@@ -0,0 +1 @@
+fd95c03719e8626c0d10a818f9996dc5

Tests/Module/ExternalData/MyScript1.cmake

@@ -0,0 +1,5 @@
+if(ExternalData_CUSTOM_LOCATION STREQUAL "MD5/fd95c03719e8626c0d10a818f9996dc5")
+  file(WRITE "${ExternalData_CUSTOM_FILE}" "DataScript")
+else()
+  set(ExternalData_CUSTOM_ERROR "no ${ExternalData_CUSTOM_LOCATION} known")
+endif()

Tests/RunCMake/ExternalData/BadCustom1-result.txt

@@ -0,0 +1 @@
+1

Tests/RunCMake/ExternalData/BadCustom1-stderr.txt

@@ -0,0 +1,9 @@
+^CMake Error at .*/Modules/ExternalData.cmake:[0-9]+ \(message\):
+  Bad ExternalDataCustomScript key '0BadKey' in URL template:
+
+   ExternalDataCustomScript://0BadKey/%\(algo\)/%\(hash\)
+
+  The key must be a valid C identifier.
+Call Stack \(most recent call first\):
+  BadCustom1.cmake:[0-9]+ \(ExternalData_Add_Target\)
+  CMakeLists.txt:[0-9]+ \(include\)$

Tests/RunCMake/ExternalData/BadCustom1.cmake

@@ -0,0 +1,5 @@
+include(ExternalData)
+set(ExternalData_URL_TEMPLATES
+  "ExternalDataCustomScript://0BadKey/%(algo)/%(hash)"
+  )
+ExternalData_Add_Target(Data)

Tests/RunCMake/ExternalData/BadCustom2-result.txt

@@ -0,0 +1 @@
+1

Tests/RunCMake/ExternalData/BadCustom2-stderr.txt

@@ -0,0 +1,9 @@
+^CMake Error at .*/Modules/ExternalData.cmake:[0-9]+ \(message\):
+  Bad ExternalDataCustomScript key '' in URL template:
+
+   ExternalDataCustomScript:///%\(algo\)/%\(hash\)
+
+  The key must be a valid C identifier.
+Call Stack \(most recent call first\):
+  BadCustom2.cmake:[0-9]+ \(ExternalData_Add_Target\)
+  CMakeLists.txt:[0-9]+ \(include\)$

Tests/RunCMake/ExternalData/BadCustom2.cmake

@@ -0,0 +1,5 @@
+include(ExternalData)
+set(ExternalData_URL_TEMPLATES
+  "ExternalDataCustomScript:///%(algo)/%(hash)"
+  )
+ExternalData_Add_Target(Data)

Tests/RunCMake/ExternalData/BadCustom3-result.txt

@@ -0,0 +1 @@
+1

Tests/RunCMake/ExternalData/BadCustom3-stderr.txt

@@ -0,0 +1,7 @@
+^CMake Error at .*/Modules/ExternalData.cmake:[0-9]+ \(message\):
+  No ExternalData_CUSTOM_SCRIPT_MissingKey is set for URL template:
+
+   ExternalDataCustomScript://MissingKey/%\(algo\)/%\(hash\)
+Call Stack \(most recent call first\):
+  BadCustom3.cmake:[0-9]+ \(ExternalData_Add_Target\)
+  CMakeLists.txt:[0-9]+ \(include\)$

Tests/RunCMake/ExternalData/BadCustom3.cmake

@@ -0,0 +1,5 @@
+include(ExternalData)
+set(ExternalData_URL_TEMPLATES
+  "ExternalDataCustomScript://MissingKey/%(algo)/%(hash)"
+  )
+ExternalData_Add_Target(Data)

Tests/RunCMake/ExternalData/BadCustom4-result.txt

@@ -0,0 +1 @@
+1

Tests/RunCMake/ExternalData/BadCustom4-stderr.txt

@@ -0,0 +1,7 @@
+^CMake Error at .*/Modules/ExternalData.cmake:[0-9]+ \(message\):
+  No ExternalData_CUSTOM_SCRIPT_RelPathKey is not set to a full path:
+
+   RelPathScript.cmake
+Call Stack \(most recent call first\):
+  BadCustom4.cmake:[0-9]+ \(ExternalData_Add_Target\)
+  CMakeLists.txt:[0-9]+ \(include\)$

Tests/RunCMake/ExternalData/BadCustom4.cmake

@@ -0,0 +1,6 @@
+include(ExternalData)
+set(ExternalData_URL_TEMPLATES
+  "ExternalDataCustomScript://RelPathKey/%(algo)/%(hash)"
+  )
+set(ExternalData_CUSTOM_SCRIPT_RelPathKey "RelPathScript.cmake")
+ExternalData_Add_Target(Data)

Tests/RunCMake/ExternalData/RunCMakeTest.cmake

@@ -1,5 +1,9 @@
 include(RunCMake)
 
+run_cmake(BadCustom1)
+run_cmake(BadCustom2)
+run_cmake(BadCustom3)
+run_cmake(BadCustom4)
 run_cmake(BadHashAlgo1)
 run_cmake(BadOption1)
 run_cmake(BadOption2)