ExternalData: Add option to recursively match under directories

Extend the ``DATA{Dir/,...}`` syntax with a new ``RECURSE:`` option
to enable recursive matching of associated files.  This will allow
an entire directory tree of data to be referenced at once.

Help/release/dev/ExternalData-recursive-match.rst

@@ -0,0 +1,7 @@
+ExternalData-recursive-match
+----------------------------
+
+* The :module:`ExternalData` module learned a new ``RECURSE:``
+  option in ``DATA{}`` references specifying directories.
+  This allows an entire directory tree of associated files
+  to be matched.

Modules/ExternalData.cmake

@@ -244,7 +244,8 @@ associated file options.  For example, the argument
 ``DATA{MyDataDir/,REGEX:.*}`` will pass the full path to a ``MyDataDir``
 directory on the command line and ensure that the directory contains
 files corresponding to every file or content link in the ``MyDataDir``
-source directory.
+source directory.  In order to match associated files in subdirectories,
+specify a ``RECURSE:`` option, e.g. ``DATA{MyDataDir/,RECURSE:,REGEX:.*}``.
 
 Hash Algorithms
 ^^^^^^^^^^^^^^^
@@ -597,6 +598,7 @@ function(_ExternalData_arg target arg options var_file)
 
   # Process options.
   set(series_option "")
+  set(recurse_option "")
   set(associated_files "")
   set(associated_regex "")
   foreach(opt ${options})
@@ -606,6 +608,9 @@ function(_ExternalData_arg target arg options var_file)
     elseif(opt STREQUAL ":")
       # Activate series matching.
       set(series_option "${opt}")
+    elseif(opt STREQUAL "RECURSE:")
+      # Activate recursive matching in directories.
+      set(recurse_option "${opt}")
     elseif("x${opt}" MATCHES "^[^][:/*?]+$")
       # Specific associated file.
       list(APPEND associated_files "${opt}")
@@ -622,6 +627,9 @@ function(_ExternalData_arg target arg options var_file)
     if(associated_files OR associated_regex)
       message(FATAL_ERROR "Series option \"${series_option}\" not allowed with associated files.")
     endif()
+    if(recurse_option)
+      message(FATAL_ERROR "Recurse option \"${recurse_option}\" allowed only with directories.")
+    endif()
     # Load a whole file series.
     _ExternalData_arg_series()
   elseif(data_is_directory)
@@ -634,6 +642,9 @@ function(_ExternalData_arg target arg options var_file)
         "must list associated files.")
     endif()
   else()
+    if(recurse_option)
+      message(FATAL_ERROR "Recurse option \"${recurse_option}\" allowed only with directories.")
+    endif()
     # Load the named data file.
     _ExternalData_arg_single()
     if(associated_files OR associated_regex)
@@ -681,11 +692,17 @@ macro(_ExternalData_arg_associated)
     set(reldir "${reldir}/")
   endif()
   _ExternalData_exact_regex(reldir_regex "${reldir}")
+  if(recurse_option)
+    set(glob GLOB_RECURSE)
+    set(reldir_regex "${reldir_regex}(.+/)?")
+  else()
+    set(glob GLOB)
+  endif()
 
   # Find files named explicitly.
   foreach(file ${associated_files})
     _ExternalData_exact_regex(file_regex "${file}")
-    _ExternalData_arg_find_files(GLOB "${reldir}${file}"
+    _ExternalData_arg_find_files(${glob} "${reldir}${file}"
       "${reldir_regex}${file_regex}")
   endforeach()
 
@@ -696,7 +713,7 @@ macro(_ExternalData_arg_associated)
     set(all "${all}${sep}${reldir_regex}${regex}")
     set(sep "|")
   endforeach()
-  _ExternalData_arg_find_files(GLOB "${reldir}" "${all}")
+  _ExternalData_arg_find_files(${glob} "${reldir}" "${all}")
 endmacro()
 
 macro(_ExternalData_arg_single)

Tests/Module/ExternalData/CMakeLists.txt

@@ -44,6 +44,7 @@ ExternalData_Add_Test(Data1
     -D Paired=DATA{PairedA.dat,PairedB.dat}
     -D Meta=DATA{MetaTop.dat,REGEX:Meta[ABC].dat}
     -D Directory=DATA{Directory/,A.dat,REGEX:[BC].dat}
+    -D DirRecurse=DATA{DirRecurse/,RECURSE:,A.dat,REGEX:[BC].dat}
     -D "Semicolons=DATA{Data.dat}\\;DATA{Data.dat}"
     -P ${CMAKE_CURRENT_SOURCE_DIR}/Data1Check.cmake
   )

Tests/Module/ExternalData/Data1Check.cmake

@@ -90,6 +90,12 @@ foreach(n A B C)
     message(SEND_ERROR "Input file:\n  ${file}\ndoes not exist!")
   endif()
 endforeach()
+foreach(n A Sub1/A Sub2/Dir/A B Sub1/B Sub2/Dir/B C Sub1/C Sub2/Dir/C)
+  set(file "${DirRecurse}/${n}.dat")
+  if(NOT EXISTS "${file}")
+    message(SEND_ERROR "Input file:\n  ${file}\ndoes not exist!")
+  endif()
+endforeach()
 list(LENGTH Semicolons len)
 if("${len}" EQUAL 2)
   foreach(file ${Semicolons})

Tests/Module/ExternalData/DirRecurse/A.dat.md5

@@ -0,0 +1 @@
+9d980b06c2f0fec3d4872d68175b9822

Tests/Module/ExternalData/DirRecurse/B.dat.md5

@@ -0,0 +1 @@
+8f4add4581551facf27237e6577fd662

Tests/Module/ExternalData/DirRecurse/C.dat.md5

@@ -0,0 +1 @@
+c1030719c95f3435d8abc39c0d442946

Tests/Module/ExternalData/DirRecurse/Sub1/A.dat.md5

@@ -0,0 +1 @@
+9d980b06c2f0fec3d4872d68175b9822

Tests/Module/ExternalData/DirRecurse/Sub1/B.dat.md5

@@ -0,0 +1 @@
+8f4add4581551facf27237e6577fd662

Tests/Module/ExternalData/DirRecurse/Sub1/C.dat.md5

@@ -0,0 +1 @@
+c1030719c95f3435d8abc39c0d442946

Tests/Module/ExternalData/DirRecurse/Sub2/Dir/A.dat.md5

@@ -0,0 +1 @@
+9d980b06c2f0fec3d4872d68175b9822

Tests/Module/ExternalData/DirRecurse/Sub2/Dir/B.dat.md5

@@ -0,0 +1 @@
+8f4add4581551facf27237e6577fd662

Tests/Module/ExternalData/DirRecurse/Sub2/Dir/C.dat.md5

@@ -0,0 +1 @@
+c1030719c95f3435d8abc39c0d442946

Tests/RunCMake/ExternalData/BadRecurse1-result.txt

@@ -0,0 +1 @@
+1

Tests/RunCMake/ExternalData/BadRecurse1-stderr.txt

@@ -0,0 +1,6 @@
+CMake Error at .*/Modules/ExternalData.cmake:[0-9]+ \(message\):
+  Recurse option "RECURSE:" allowed only with directories.
+Call Stack \(most recent call first\):
+  .*
+  BadRecurse1.cmake:2 \(ExternalData_Expand_Arguments\)
+  CMakeLists.txt:3 \(include\)

Tests/RunCMake/ExternalData/BadRecurse1.cmake

@@ -0,0 +1,2 @@
+include(ExternalData)
+ExternalData_Expand_Arguments(Data args DATA{Series.txt,:,RECURSE:})

Tests/RunCMake/ExternalData/BadRecurse2-result.txt

@@ -0,0 +1 @@
+1

Tests/RunCMake/ExternalData/BadRecurse2-stderr.txt

@@ -0,0 +1,6 @@
+CMake Error at .*/Modules/ExternalData.cmake:[0-9]+ \(message\):
+  Recurse option "RECURSE:" allowed only with directories.
+Call Stack \(most recent call first\):
+  .*
+  BadRecurse2.cmake:2 \(ExternalData_Expand_Arguments\)
+  CMakeLists.txt:3 \(include\)

Tests/RunCMake/ExternalData/BadRecurse2.cmake

@@ -0,0 +1,2 @@
+include(ExternalData)
+ExternalData_Expand_Arguments(Data args DATA{Data.txt,RECURSE:})

Tests/RunCMake/ExternalData/BadRecurse3-result.txt

@@ -0,0 +1 @@
+1

Tests/RunCMake/ExternalData/BadRecurse3-stderr.txt

@@ -0,0 +1,9 @@
+CMake Error at .*/Modules/ExternalData.cmake:[0-9]+ \(message\):
+  Unknown option "RECURSE:x" in argument
+
+    DATA{Directory1/,RECURSE:x,Data.dat}
+
+Call Stack \(most recent call first\):
+  .*
+  BadRecurse3.cmake:2 \(ExternalData_Expand_Arguments\)
+  CMakeLists.txt:3 \(include\)

Tests/RunCMake/ExternalData/BadRecurse3.cmake

@@ -0,0 +1,2 @@
+include(ExternalData)
+ExternalData_Expand_Arguments(Data args DATA{Directory1/,RECURSE:x,Data.dat})

Tests/RunCMake/ExternalData/RunCMakeTest.cmake

@@ -9,6 +9,9 @@ run_cmake(BadCustom4)
 run_cmake(BadHashAlgo1)
 run_cmake(BadOption1)
 run_cmake(BadOption2)
+run_cmake(BadRecurse1)
+run_cmake(BadRecurse2)
+run_cmake(BadRecurse3)
 run_cmake(BadSeries1)
 run_cmake(BadSeries2)
 run_cmake(BadSeries3)