add_test: Add COMMAND_EXPAND_LISTS option

Add a `COMMAND_EXPAND_LISTS` option to the `add_test` command to cause
`;`-separated lists produced by generator expressions to be expanded
into multiple arguments.  The `add_custom_command` command already
has such an option.

Fixes: #17284

Help/command/add_test.rst

@@ -7,7 +7,8 @@ Add a test to the project to be run by :manual:`ctest(1)`.
 
   add_test(NAME <name> COMMAND <command> [<arg>...]
            [CONFIGURATIONS <config>...]
-           [WORKING_DIRECTORY <dir>])
+           [WORKING_DIRECTORY <dir>]
+           [COMMAND_EXPAND_LISTS])
 
 Adds a test called ``<name>``.  The test name may not contain spaces,
 quotes, or other characters special in CMake syntax.  The options are:
@@ -28,6 +29,11 @@ quotes, or other characters special in CMake syntax.  The options are:
   directory set to the build directory corresponding to the
   current source directory.
 
+``COMMAND_EXPAND_LISTS``
+  Lists in ``COMMAND`` arguments will be expanded, including those
+  created with
+  :manual:`generator expressions <cmake-generator-expressions(7)>`.
+
 The given test command is expected to exit with code ``0`` to pass and
 non-zero to fail, or vice-versa if the :prop_test:`WILL_FAIL` test
 property is set.  Any output written to stdout or stderr will be

Help/release/dev/add_test-expand_lists.rst

@@ -0,0 +1,6 @@
+add_test-expand_lists
+---------------------
+
+* The command :command:`add_test` learned the option ``COMMAND_EXPAND_LISTS``
+  which causes lists in the ``COMMAND`` argument to be expanded, including
+  lists created by generator expressions.

Source/cmAddTestCommand.cxx

@@ -58,6 +58,7 @@ bool cmAddTestCommand::HandleNameMode(std::vector<std::string> const& args)
   std::vector<std::string> configurations;
   std::string working_directory;
   std::vector<std::string> command;
+  bool command_expand_lists = false;
 
   // Read the arguments.
   enum Doing
@@ -88,6 +89,13 @@ bool cmAddTestCommand::HandleNameMode(std::vector<std::string> const& args)
         return false;
       }
       doing = DoingWorkingDirectory;
+    } else if (args[i] == "COMMAND_EXPAND_LISTS") {
+      if (command_expand_lists) {
+        this->SetError(" may be given at most one COMMAND_EXPAND_LISTS.");
+        return false;
+      }
+      command_expand_lists = true;
+      doing = DoingNone;
     } else if (doing == DoingName) {
       name = args[i];
       doing = DoingNone;
@@ -134,6 +142,7 @@ bool cmAddTestCommand::HandleNameMode(std::vector<std::string> const& args)
   if (!working_directory.empty()) {
     test->SetProperty("WORKING_DIRECTORY", working_directory.c_str());
   }
+  test->SetCommandExpandLists(command_expand_lists);
   this->Makefile->AddTestGenerator(new cmTestGenerator(test, configurations));
 
   return true;

Source/cmTest.cxx

@@ -8,7 +8,8 @@
 #include "cmSystemTools.h"
 
 cmTest::cmTest(cmMakefile* mf)
-  : Backtrace(mf->GetBacktrace())
+  : CommandExpandLists(false)
+  , Backtrace(mf->GetBacktrace())
 {
   this->Makefile = mf;
   this->OldStyle = true;
@@ -59,3 +60,13 @@ void cmTest::AppendProperty(const std::string& prop, const char* value,
 {
   this->Properties.AppendProperty(prop, value, asString);
 }
+
+bool cmTest::GetCommandExpandLists() const
+{
+  return this->CommandExpandLists;
+}
+
+void cmTest::SetCommandExpandLists(bool b)
+{
+  this->CommandExpandLists = b;
+}

Source/cmTest.h

@@ -51,10 +51,15 @@ public:
   bool GetOldStyle() const { return this->OldStyle; }
   void SetOldStyle(bool b) { this->OldStyle = b; }
 
+  /** Set/Get whether lists in command lines should be expanded. */
+  bool GetCommandExpandLists() const;
+  void SetCommandExpandLists(bool b);
+
 private:
   cmPropertyMap Properties;
   std::string Name;
   std::vector<std::string> Command;
+  bool CommandExpandLists;
 
   bool OldStyle;
 

Source/cmTestGenerator.cxx

@@ -76,12 +76,22 @@ void cmTestGenerator::GenerateScriptForConfig(std::ostream& os,
   // Start the test command.
   os << indent << "add_test(" << this->Test->GetName() << " ";
 
-  // Get the test command line to be executed.
-  std::vector<std::string> const& command = this->Test->GetCommand();
+  // Evaluate command line arguments
+  std::vector<std::string> argv =
+    EvaluateCommandLineArguments(this->Test->GetCommand(), ge, config);
+
+  // Expand arguments if COMMAND_EXPAND_LISTS is set
+  if (this->Test->GetCommandExpandLists()) {
+    argv = cmSystemTools::ExpandedLists(argv.begin(), argv.end());
+    // Expanding lists on an empty command may have left it empty
+    if (argv.empty()) {
+      argv.emplace_back();
+    }
+  }
 
   // Check whether the command executable is a target whose name is to
   // be translated.
-  std::string exe = command[0];
+  std::string exe = argv[0];
   cmGeneratorTarget* target = this->LG->FindGeneratorTargetToUse(exe);
   if (target && target->GetType() == cmStateEnums::EXECUTABLE) {
     // Use the target file on disk.
@@ -101,16 +111,14 @@ void cmTestGenerator::GenerateScriptForConfig(std::ostream& os,
     }
   } else {
     // Use the command name given.
-    exe = ge.Parse(exe)->Evaluate(this->LG, config);
     cmSystemTools::ConvertToUnixSlashes(exe);
   }
 
   // Generate the command line with full escapes.
   os << cmOutputConverter::EscapeForCMake(exe);
-  for (std::string const& arg : cmMakeRange(command).advance(1)) {
-    os << " "
-       << cmOutputConverter::EscapeForCMake(
-            ge.Parse(arg)->Evaluate(this->LG, config));
+
+  for (auto const& arg : cmMakeRange(argv).advance(1)) {
+    os << " " << cmOutputConverter::EscapeForCMake(arg);
   }
 
   // Finish the test command.
@@ -208,3 +216,16 @@ void cmTestGenerator::GenerateInternalProperties(std::ostream& os)
 
   os << "\"";
 }
+
+std::vector<std::string> cmTestGenerator::EvaluateCommandLineArguments(
+  const std::vector<std::string>& argv, cmGeneratorExpression& ge,
+  const std::string& config) const
+{
+  // Evaluate executable name and arguments
+  auto evaluatedRange =
+    cmMakeRange(argv).transform([&](const std::string& arg) {
+      return ge.Parse(arg)->Evaluate(this->LG, config);
+    });
+
+  return { evaluatedRange.begin(), evaluatedRange.end() };
+}

Source/cmTestGenerator.h

@@ -11,6 +11,7 @@
 #include <string>
 #include <vector>
 
+class cmGeneratorExpression;
 class cmLocalGenerator;
 class cmTest;
 
@@ -38,6 +39,9 @@ public:
 
 private:
   void GenerateInternalProperties(std::ostream& os);
+  std::vector<std::string> EvaluateCommandLineArguments(
+    const std::vector<std::string>& argv, cmGeneratorExpression& ge,
+    const std::string& config) const;
 
 protected:
   void GenerateScriptConfigs(std::ostream& os, Indent indent) override;

Tests/RunCMake/CMakeLists.txt

@@ -565,3 +565,5 @@ if(${CMAKE_GENERATOR} MATCHES "Visual Studio ([^9]|9[0-9])")
   add_RunCMake_test(CSharpCustomCommand)
   add_RunCMake_test(CSharpReferenceImport)
 endif()
+
+add_RunCMake_test("CTestCommandExpandLists")

Tests/RunCMake/CTestCommandExpandLists/CMakeLists.txt

@@ -0,0 +1,3 @@
+cmake_minimum_required(VERSION 3.14)
+project(${RunCMake_TEST} NONE)
+include(${RunCMake_TEST}.cmake)

Tests/RunCMake/CTestCommandExpandLists/CMakeLists.txt.in

@@ -0,0 +1,3 @@
+cmake_minimum_required(VERSION 3.14)
+project(@CASE_NAME@ NONE)
+include("@RunCMake_SOURCE_DIR@/@[email protected]")

Tests/RunCMake/CTestCommandExpandLists/RunCMakeTest.cmake

@@ -0,0 +1,5 @@
+include(RunCTest)
+
+run_ctest(expandGeneratorExpressionResult)
+run_ctest(expandEmptyCommand)
+run_cmake(multipleExpandOptions)

Tests/RunCMake/CTestCommandExpandLists/compare_options.cmake

@@ -0,0 +1,14 @@
+set(range 1 2 3 4 5 6 7 8 9 10)
+set(aargs "")
+set(bargs "")
+foreach(n IN LISTS range)
+  set(aval "${A${n}ARG}")
+  set(bval "${B${n}ARG}")
+  if(aval OR bval)
+    list(APPEND aargs "\"${aval}\"")
+    list(APPEND bargs "\"${bval}\"")
+  endif()
+endforeach()
+if(NOT "${aargs}" STREQUAL "${bargs}")
+  message(FATAL_ERROR "COMPARE_OPTIONS: \n\t${aargs} != \n\t${bargs}")
+endif()

Tests/RunCMake/CTestCommandExpandLists/expandEmptyCommand-result.txt

@@ -0,0 +1 @@
+(-1|255)

Tests/RunCMake/CTestCommandExpandLists/expandEmptyCommand-stderr.txt

@@ -0,0 +1 @@
+Unable to find executable:

Tests/RunCMake/CTestCommandExpandLists/expandEmptyCommand-stdout.txt

@@ -0,0 +1,13 @@
+Test project .*/Tests/RunCMake/CTestCommandExpandLists/expandEmptyCommand-build
+.* +Start 1: CommandExpandEmptyList
+Could not find executable +
+Looked in the following places:
+.*
+1/1 Test #1: CommandExpandEmptyList +\.+\*\*\*Not Run +[0-9.]+ sec
++
+0% tests passed, 1 tests failed out of 1
++
+Total Test time \(real\) = +[0-9.]+ sec
++
+The following tests FAILED:
+.* +1 - CommandExpandEmptyList \(Not Run\)$

Tests/RunCMake/CTestCommandExpandLists/expandEmptyCommand.cmake

@@ -0,0 +1,10 @@
+include(CTest)
+
+set(argv /bin/true)
+list(POP_BACK argv)
+
+add_test(
+  NAME CommandExpandEmptyList
+  COMMAND "$<JOIN:${argv},;>"
+  COMMAND_EXPAND_LISTS
+)

Tests/RunCMake/CTestCommandExpandLists/expandGeneratorExpressionResult-result.txt

@@ -0,0 +1 @@
+0

Tests/RunCMake/CTestCommandExpandLists/expandGeneratorExpressionResult-stdout.txt

@@ -0,0 +1,7 @@
+Test project .*/Tests/RunCMake/CTestCommandExpandLists/expandGeneratorExpressionResult-build
+.* +Start 1: CommandExpandList
+1/1 Test #1: CommandExpandList +\.+ +Passed +[0-9.]+ sec
++
+100% tests passed, 0 tests failed out of 1
++
+Total Test time \(real\) = +[0-9.]+ sec

Tests/RunCMake/CTestCommandExpandLists/expandGeneratorExpressionResult.cmake

@@ -0,0 +1,19 @@
+include(CTest)
+
+
+set(cmp_args "1ARG=COMMAND_EXPAND_LISTS" "2ARG=test" "3ARG=outfile"
+  "4ARG=content")
+set(AARGS "")
+foreach(arg IN LISTS cmp_args)
+  list(APPEND AARGS "-DA${arg}")
+endforeach()
+
+
+
+add_test(
+  NAME CommandExpandList
+  COMMAND ${CMAKE_COMMAND} ${AARGS} -V
+  "-DB$<JOIN:${cmp_args},;-DB>"
+  "-P" "${CMAKE_CURRENT_LIST_DIR}/compare_options.cmake"
+  COMMAND_EXPAND_LISTS
+)

Tests/RunCMake/CTestCommandExpandLists/multipleExpandOptions-result.txt

@@ -0,0 +1 @@
+1

Tests/RunCMake/CTestCommandExpandLists/multipleExpandOptions-stderr.txt

@@ -0,0 +1,2 @@
+CMake Error at multipleExpandOptions\.cmake:3 \(add_test\):
+ +add_test may be given at most one COMMAND_EXPAND_LISTS\.

Tests/RunCMake/CTestCommandExpandLists/multipleExpandOptions-stdout.txt

@@ -0,0 +1,2 @@
+-- Configuring incomplete, errors occurred!
+See also ".*/Tests/RunCMake/CTestCommandExpandLists/multipleExpandOptions-build/CMakeFiles/CMakeOutput\.log".

Tests/RunCMake/CTestCommandExpandLists/multipleExpandOptions.cmake

@@ -0,0 +1,8 @@
+include(CTest)
+
+add_test(
+  NAME MultipleExpandOptions
+  COMMAND /bin/true
+  COMMAND_EXPAND_LISTS
+  COMMAND_EXPAND_LISTS
+)

Tests/RunCMake/CTestCommandExpandLists/test.cmake.in

@@ -0,0 +1,15 @@
+cmake_minimum_required(VERSION 3.14)
+
+set(CTEST_SITE                          "test-site")
+set(CTEST_BUILD_NAME                    "test-build-name")
+set(CTEST_SOURCE_DIRECTORY              "@RunCMake_BINARY_DIR@/@CASE_NAME@")
+set(CTEST_BINARY_DIRECTORY              "@RunCMake_BINARY_DIR@/@CASE_NAME@-build")
+set(CTEST_CMAKE_GENERATOR               "@RunCMake_GENERATOR@")
+set(CTEST_CMAKE_GENERATOR_PLATFORM      "@RunCMake_GENERATOR_PLATFORM@")
+set(CTEST_CMAKE_GENERATOR_TOOLSET       "@RunCMake_GENERATOR_TOOLSET@")
+set(CTEST_BUILD_CONFIGURATION           "$ENV{CMAKE_CONFIG_TYPE}")
+
+ctest_start(Experimental)
+ctest_configure()
+ctest_build()
+ctest_test()