Merge topic 'ctest-tests-from-file'

701029726f ctest_test: add options INCLUDE_FROM_FILE and EXCLUDE_FROM_FILE
dbacc1d5a8 ctest: add command line option to exclude tests listed in a given file
022f20f663 ctest: add command line option to run the tests listed in a given file

Acked-by: Kitware Robot <[email protected]>
Acked-by: Alex <[email protected]>
Merge-request: !9128

Help/command/ctest_test.rst

@@ -13,6 +13,8 @@ Perform the :ref:`CTest Test Step` as a :ref:`Dashboard Client`.
              [INCLUDE <include-regex>]
              [EXCLUDE_LABEL <label-exclude-regex>]
              [INCLUDE_LABEL <label-include-regex>]
+             [EXCLUDE_FROM_FILE <filename>]
+             [INCLUDE_FROM_FILE <filename>]
              [EXCLUDE_FIXTURE <regex>]
              [EXCLUDE_FIXTURE_SETUP <regex>]
              [EXCLUDE_FIXTURE_CLEANUP <regex>]
@@ -72,6 +74,16 @@ The options are:
   Specify a regular expression matching test labels to include.
   Tests not matching this expression are excluded.
 
+``EXCLUDE_FROM_FILE <filename>``
+  .. versionadded:: 3.29
+
+  Do NOT run tests listed with their exact name in the given file.
+
+``INCLUDE_FROM_FILE <filename>``
+  .. versionadded:: 3.29
+
+  Only run the tests listed with their exact name in the given file.
+
 ``EXCLUDE_FIXTURE <regex>``
   .. versionadded:: 3.7
 

Help/manual/ctest.1.rst

@@ -234,6 +234,30 @@ Run Tests
  of the test's labels (i.e. the multiple ``-LE`` labels form an ``AND``
  relationship).  See `Label Matching`_.
 
+.. option:: --tests-from-file <filename>
+
+ .. versionadded:: 3.29
+
+ Run tests listed in the given file.
+
+ This option tells CTest to run the tests which are listed in the given
+ file. The file must contain one exact test name per line.
+ Lines can be commented out using a ``#``.
+ This option can be combined with the other options like
+ ``-R``, ``-E``, ``-L`` or ``-LE``.
+
+.. option:: --exclude-from-file <filename>
+
+ .. versionadded:: 3.29
+
+ Exclude tests listed in the given file.
+
+ This option tells CTest to NOT run the tests which are listed in the given
+ file. The file must contain one exact test name per line.
+ Lines can be commented out using a ``#``.
+ This option can be combined with the other options like
+ ``-R``, ``-E``, ``-L`` or ``-LE``.
+
 .. option:: -FA <regex>, --fixture-exclude-any <regex>
 
  Exclude fixtures matching ``<regex>`` from automatically adding any tests to

Help/release/dev/ctest-tests-from-file.rst

@@ -0,0 +1,10 @@
+ctest-tests-from-file
+---------------------
+
+* :manual:`ctest(1)` gained the :option:`--tests-from-file <ctest
+  --tests-from-file>` and :option:`--exclude-from-file <ctest
+  --exclude-from-file>` options to run or exclude tests named in a file.
+
+* The :command:`ctest_test` command gained options
+  ``INCLUDE_FROM_FILE`` and ``EXCLUDE_FROM_FILE`` to run or exclude
+  tests named in a file.

Source/CTest/cmCTestTestCommand.cxx

@@ -26,6 +26,8 @@ void cmCTestTestCommand::BindArguments()
   this->Bind("INCLUDE"_s, this->Include);
   this->Bind("EXCLUDE_LABEL"_s, this->ExcludeLabel);
   this->Bind("INCLUDE_LABEL"_s, this->IncludeLabel);
+  this->Bind("EXCLUDE_FROM_FILE"_s, this->ExcludeTestsFromFile);
+  this->Bind("INCLUDE_FROM_FILE"_s, this->IncludeTestsFromFile);
   this->Bind("EXCLUDE_FIXTURE"_s, this->ExcludeFixture);
   this->Bind("EXCLUDE_FIXTURE_SETUP"_s, this->ExcludeFixtureSetup);
   this->Bind("EXCLUDE_FIXTURE_CLEANUP"_s, this->ExcludeFixtureCleanup);
@@ -80,6 +82,14 @@ cmCTestGenericHandler* cmCTestTestCommand::InitializeHandler()
   if (!this->IncludeLabel.empty()) {
     handler->AddMultiOption("LabelRegularExpression", this->IncludeLabel);
   }
+
+  if (!this->ExcludeTestsFromFile.empty()) {
+    handler->SetOption("ExcludeTestListFile", this->ExcludeTestsFromFile);
+  }
+  if (!this->IncludeTestsFromFile.empty()) {
+    handler->SetOption("TestListFile", this->IncludeTestsFromFile);
+  }
+
   if (!this->ExcludeFixture.empty()) {
     handler->SetOption("ExcludeFixtureRegularExpression",
                        this->ExcludeFixture);

Source/CTest/cmCTestTestCommand.h

@@ -51,6 +51,8 @@ protected:
   std::string Include;
   std::string ExcludeLabel;
   std::string IncludeLabel;
+  std::string IncludeTestsFromFile;
+  std::string ExcludeTestsFromFile;
   std::string ExcludeFixture;
   std::string ExcludeFixtureSetup;
   std::string ExcludeFixtureCleanup;

Source/CTest/cmCTestTestHandler.cxx

@@ -345,6 +345,8 @@ void cmCTestTestHandler::Initialize()
   this->ExcludeFixtureRegExp.clear();
   this->ExcludeFixtureSetupRegExp.clear();
   this->ExcludeFixtureCleanupRegExp.clear();
+  this->TestListFile.clear();
+  this->ExcludeTestListFile.clear();
 
   this->TestsToRunString.clear();
   this->UseUnion = false;
@@ -585,6 +587,14 @@ bool cmCTestTestHandler::ProcessOptions()
   if (val) {
     this->ResourceSpecFile = *val;
   }
+  val = this->GetOption("TestListFile");
+  if (val) {
+    this->TestListFile = val;
+  }
+  val = this->GetOption("ExcludeTestListFile");
+  if (val) {
+    this->ExcludeTestListFile = val;
+  }
   this->SetRerunFailed(cmIsOn(this->GetOption("RerunFailed")));
 
   return true;
@@ -933,6 +943,21 @@ bool cmCTestTestHandler::ComputeTestList()
         continue;
       }
     }
+
+    if (!this->TestsToRunByName.empty()) {
+      if (this->TestsToRunByName.find(tp.Name) ==
+          this->TestsToRunByName.end()) {
+        continue;
+      }
+    }
+
+    if (!this->TestsToExcludeByName.empty()) {
+      if (this->TestsToExcludeByName.find(tp.Name) !=
+          this->TestsToExcludeByName.end()) {
+        continue;
+      }
+    }
+
     tp.Index = cnt; // save the index into the test list for this test
     finalList.push_back(tp);
   }
@@ -1818,6 +1843,15 @@ bool cmCTestTestHandler::GetListOfTests()
   if (this->ResourceSpecFile.empty() && specFile) {
     this->ResourceSpecFile = *specFile;
   }
+
+  if (!this->TestListFile.empty()) {
+    this->TestsToRunByName = this->ReadTestListFile(this->TestListFile);
+  }
+  if (!this->ExcludeTestListFile.empty()) {
+    this->TestsToExcludeByName =
+      this->ReadTestListFile(this->ExcludeTestListFile);
+  }
+
   cmCTestOptionalLog(this->CTest, HANDLER_VERBOSE_OUTPUT,
                      "Done constructing a list of tests" << std::endl,
                      this->Quiet);
@@ -1986,6 +2020,34 @@ void cmCTestTestHandler::ExpandTestsToRunInformationForRerunFailed()
   }
 }
 
+std::set<std::string> cmCTestTestHandler::ReadTestListFile(
+  const std::string& testListFileName) const
+{
+  std::set<std::string> testNames;
+
+  cmsys::ifstream ifs(testListFileName.c_str());
+  if (ifs) {
+    std::string line;
+    while (cmSystemTools::GetLineFromStream(ifs, line)) {
+      std::string trimmed = cmTrimWhitespace(line);
+      if (trimmed.empty() || (trimmed[0] == '#')) {
+        continue;
+      }
+
+      testNames.insert(trimmed);
+    }
+    ifs.close();
+  } else if (!this->CTest->GetShowOnly() &&
+             !this->CTest->ShouldPrintLabels()) {
+    cmCTestLog(this->CTest, ERROR_MESSAGE,
+               "Problem reading test list file: "
+                 << testListFileName
+                 << " while generating list of tests to run." << std::endl);
+  }
+
+  return testNames;
+}
+
 void cmCTestTestHandler::RecordCustomTestMeasurements(cmXMLWriter& xml,
                                                       std::string content)
 {

Source/CTest/cmCTestTestHandler.h

@@ -341,6 +341,8 @@ private:
   std::string GetTestStatus(cmCTestTestResult const&);
   void ExpandTestsToRunInformation(size_t numPossibleTests);
   void ExpandTestsToRunInformationForRerunFailed();
+  std::set<std::string> ReadTestListFile(
+    const std::string& testListFileName) const;
 
   std::vector<std::string> CustomPreTest;
   std::vector<std::string> CustomPostTest;
@@ -359,6 +361,10 @@ private:
   std::vector<cmsys::RegularExpression> ExcludeLabelRegularExpressions;
   cmsys::RegularExpression IncludeTestsRegularExpression;
   cmsys::RegularExpression ExcludeTestsRegularExpression;
+  std::string TestListFile;
+  std::string ExcludeTestListFile;
+  std::set<std::string> TestsToRunByName;
+  std::set<std::string> TestsToExcludeByName;
 
   std::string ResourceSpecFile;
 

Source/cmCTest.cxx

@@ -2224,6 +2224,22 @@ bool cmCTest::HandleCommandLineArguments(size_t& i,
                                                     args[i]);
   }
 
+  else if (this->CheckArgument(arg, "--tests-from-file"_s) &&
+           i < args.size() - 1) {
+    i++;
+    this->GetTestHandler()->SetPersistentOption("TestListFile", args[i]);
+    this->GetMemCheckHandler()->SetPersistentOption("TestListFile", args[i]);
+  }
+
+  else if (this->CheckArgument(arg, "--exclude-from-file"_s) &&
+           i < args.size() - 1) {
+    i++;
+    this->GetTestHandler()->SetPersistentOption("ExcludeTestListFile",
+                                                args[i]);
+    this->GetMemCheckHandler()->SetPersistentOption("ExcludeTestListFile",
+                                                    args[i]);
+  }
+
   else if (this->CheckArgument(arg, "--rerun-failed"_s)) {
     this->GetTestHandler()->SetPersistentOption("RerunFailed", "true");
     this->GetMemCheckHandler()->SetPersistentOption("RerunFailed", "true");

Source/ctest.cxx

@@ -107,6 +107,9 @@ const cmDocumentationEntry cmDocumentationOptions[] = {
     "Run a specific number of tests by number." },
   { "-U, --union", "Take the Union of -I and -R" },
   { "--rerun-failed", "Run only the tests that failed previously" },
+  { "--tests-from-file <file>", "Run the tests listed in the given file" },
+  { "--exclude-from-file <file>",
+    "Run tests except those listed in the given file" },
   { "--repeat until-fail:<n>, --repeat-until-fail <n>",
     "Require each test to run <n> times without failing in order to pass" },
   { "--repeat until-pass:<n>",

Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake

@@ -207,6 +207,24 @@ set_tests_properties(test1 PROPERTIES SKIP_REGULAR_EXPRESSION \"test1\")
 endfunction()
 run_SkipRegexFoundTest()
 
+
+function(run_TestsFromFileTest arg)
+  set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/TestsFromFile)
+  set(RunCMake_TEST_NO_CLEAN 1)
+  file(REMOVE_RECURSE "${RunCMake_TEST_BINARY_DIR}")
+  file(MAKE_DIRECTORY "${RunCMake_TEST_BINARY_DIR}")
+
+  file(WRITE "${RunCMake_TEST_BINARY_DIR}/CTestTestfile.cmake" "
+add_test(Test1 \"${CMAKE_COMMAND}\" -E echo \"test1\")
+add_test(Test2 \"${CMAKE_COMMAND}\" -E echo \"test2\")
+add_test(Test11 \"${CMAKE_COMMAND}\" -E echo \"test11\")
+")
+  run_cmake_command(TestsFromFile-${arg} ${CMAKE_CTEST_COMMAND} --${arg} ${RunCMake_SOURCE_DIR}/TestsFromFile-TestList.txt )
+endfunction()
+run_TestsFromFileTest(tests-from-file)
+run_TestsFromFileTest(exclude-from-file)
+
+
 function(run_SerialFailed)
   set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/SerialFailed)
   set(RunCMake_TEST_NO_CLEAN 1)

Tests/RunCMake/CTestCommandLine/TestsFromFile-TestList.txt

@@ -0,0 +1,2 @@
+Test1
+# Test11

Tests/RunCMake/CTestCommandLine/TestsFromFile-exclude-from-file-stdout.txt

@@ -0,0 +1,7 @@
+ +Start 2: Test2
+1/2 Test #2: Test2 ............................   Passed +[0-9.]+ sec
+ +Start 3: Test11
+2/2 Test #3: Test11 ...........................   Passed +[0-9.]+ sec
++
+100% tests passed, 0 tests failed out of 2
++

Tests/RunCMake/CTestCommandLine/TestsFromFile-tests-from-file-stdout.txt

@@ -0,0 +1,5 @@
+ +Start 1: Test1
+1/1 Test #1: Test1 ............................   Passed +[0-9.]+ sec
++
+100% tests passed, 0 tests failed out of 1
++

Tests/RunCMake/ctest_test/RunCMakeTest.cmake

@@ -163,6 +163,22 @@ add_test(NAME NotRunTest COMMAND ${CMAKE_COMMAND} -E true)
 endfunction()
 run_stop_on_failure()
 
+
+# test include/exclude tests from file
+function(run_tests_from_file mode)
+  set(CASE_CTEST_TEST_ARGS ${mode} ${RunCMake_SOURCE_DIR}/TestsFromFile-TestList.txt)
+  set(CASE_CMAKELISTS_SUFFIX_CODE [[
+add_test(NAME Test1 COMMAND ${CMAKE_COMMAND} -E true)
+add_test(NAME Test2 COMMAND ${CMAKE_COMMAND} -E true)
+add_test(NAME Test11 COMMAND ${CMAKE_COMMAND} -E true)
+  ]])
+
+  run_ctest(TestsFromFile-${mode})
+endfunction()
+run_tests_from_file(INCLUDE_FROM_FILE)
+run_tests_from_file(EXCLUDE_FROM_FILE)
+
+
 # Make sure environment gets logged
 function(run_environment)
   set(ENV{BAD_ENVIRONMENT_VARIABLE} "Bad environment variable")

Tests/RunCMake/ctest_test/TestsFromFile-EXCLUDE_FROM_FILE-stdout.txt

@@ -0,0 +1,7 @@
+ +Start 3: Test2
+2/3 Test #3: Test2 ............................   Passed +[0-9.]+ sec
+ +Start 4: Test11
+3/3 Test #4: Test11 ...........................   Passed +[0-9.]+ sec
++
+100% tests passed, 0 tests failed out of 3
++

Tests/RunCMake/ctest_test/TestsFromFile-INCLUDE_FROM_FILE-stdout.txt

@@ -0,0 +1,5 @@
+ +Start 2: Test1
+1/1 Test #2: Test1 ............................   Passed +[0-9.]+ sec
++
+100% tests passed, 0 tests failed out of 1
++

Tests/RunCMake/ctest_test/TestsFromFile-TestList.txt

@@ -0,0 +1,2 @@
+Test1
+# Test11