Merge topic 'ctest-tests-from-file'

170ec48601 Help: Improve ctest tests-from-file documentation wording and wrapping
1a4837641e ctest: Remove unnecessary and ambiguous tests-from-file comment syntax
d52c66bfb3 ctest: Honor tests-from-file options with empty input
8673264e25 Tests: Make ctest tests-from-file expected output more precise

Acked-by: Kitware Robot <[email protected]>
Acked-by: buildbot <[email protected]>
Merge-request: !9322

Help/manual/ctest.1.rst

@@ -240,9 +240,9 @@ Run Tests
 
  Run tests listed in the given file.
 
- This option tells CTest to run the tests which are listed in the given
- file. The file must contain one exact test name per line.
- Lines can be commented out using a ``#``.
+ This option tells CTest to run tests that are listed in the given file.
+ The file must contain one exact test name per line.
+ Lines that do not exactly match any test names are ignored.
  This option can be combined with the other options like
  ``-R``, ``-E``, ``-L`` or ``-LE``.
 
@@ -252,9 +252,9 @@ Run Tests
 
  Exclude tests listed in the given file.
 
- This option tells CTest to NOT run the tests which are listed in the given
- file. The file must contain one exact test name per line.
- Lines can be commented out using a ``#``.
+ This option tells CTest to NOT run tests that are listed in the given file.
+ The file must contain one exact test name per line.
+ Lines that do not exactly match any test names are ignored.
  This option can be combined with the other options like
  ``-R``, ``-E``, ``-L`` or ``-LE``.
 

Source/CTest/cmCTestTestHandler.cxx

@@ -347,6 +347,8 @@ void cmCTestTestHandler::Initialize()
   this->ExcludeFixtureCleanupRegExp.clear();
   this->TestListFile.clear();
   this->ExcludeTestListFile.clear();
+  this->TestsToRunByName.reset();
+  this->TestsToExcludeByName.reset();
 
   this->TestsToRunString.clear();
   this->UseUnion = false;
@@ -943,16 +945,16 @@ bool cmCTestTestHandler::ComputeTestList()
       }
     }
 
-    if (!this->TestsToRunByName.empty()) {
-      if (this->TestsToRunByName.find(tp.Name) ==
-          this->TestsToRunByName.end()) {
+    if (this->TestsToRunByName) {
+      if (this->TestsToRunByName->find(tp.Name) ==
+          this->TestsToRunByName->end()) {
         continue;
       }
     }
 
-    if (!this->TestsToExcludeByName.empty()) {
-      if (this->TestsToExcludeByName.find(tp.Name) !=
-          this->TestsToExcludeByName.end()) {
+    if (this->TestsToExcludeByName) {
+      if (this->TestsToExcludeByName->find(tp.Name) !=
+          this->TestsToExcludeByName->end()) {
         continue;
       }
     }
@@ -1846,13 +1848,15 @@ bool cmCTestTestHandler::GetListOfTests()
   }
 
   if (!this->TestListFile.empty()) {
-    if (!this->ReadTestListFile(this->TestListFile, this->TestsToRunByName)) {
+    this->TestsToRunByName = this->ReadTestListFile(this->TestListFile);
+    if (!this->TestsToRunByName) {
       return false;
     }
   }
   if (!this->ExcludeTestListFile.empty()) {
-    if (!this->ReadTestListFile(this->ExcludeTestListFile,
-                                this->TestsToExcludeByName)) {
+    this->TestsToExcludeByName =
+      this->ReadTestListFile(this->ExcludeTestListFile);
+    if (!this->TestsToExcludeByName) {
       return false;
     }
   }
@@ -2025,32 +2029,27 @@ void cmCTestTestHandler::ExpandTestsToRunInformationForRerunFailed()
   }
 }
 
-bool cmCTestTestHandler::ReadTestListFile(
-  std::string const& testListFileName, std::set<std::string>& testNames) const
+cm::optional<std::set<std::string>> cmCTestTestHandler::ReadTestListFile(
+  std::string const& testListFileName) const
 {
-  testNames.clear();
+  cm::optional<std::set<std::string>> result;
   cmsys::ifstream ifs(testListFileName.c_str());
   if (ifs) {
+    std::set<std::string> testNames;
     std::string line;
     while (cmSystemTools::GetLineFromStream(ifs, line)) {
-      std::string trimmed = cmTrimWhitespace(line);
-      if (trimmed.empty() || (trimmed[0] == '#')) {
-        continue;
+      if (!line.empty()) {
+        testNames.insert(line);
       }
-
-      testNames.insert(trimmed);
     }
-    ifs.close();
-  } else if (!this->CTest->GetShowOnly() &&
-             !this->CTest->ShouldPrintLabels()) {
+    result = std::move(testNames);
+  } else {
     cmCTestLog(this->CTest, ERROR_MESSAGE,
                "Problem reading test list file: "
                  << testListFileName
                  << " while generating list of tests to run." << std::endl);
-    return false;
   }
-
-  return true;
+  return result;
 }
 
 void cmCTestTestHandler::RecordCustomTestMeasurements(cmXMLWriter& xml,

Source/CTest/cmCTestTestHandler.h

@@ -342,8 +342,8 @@ private:
   std::string GetTestStatus(cmCTestTestResult const&);
   void ExpandTestsToRunInformation(size_t numPossibleTests);
   void ExpandTestsToRunInformationForRerunFailed();
-  bool ReadTestListFile(std::string const& testListFileName,
-                        std::set<std::string>& testNames) const;
+  cm::optional<std::set<std::string>> ReadTestListFile(
+    std::string const& testListFileName) const;
 
   std::vector<std::string> CustomPreTest;
   std::vector<std::string> CustomPostTest;
@@ -364,8 +364,8 @@ private:
   cmsys::RegularExpression ExcludeTestsRegularExpression;
   std::string TestListFile;
   std::string ExcludeTestListFile;
-  std::set<std::string> TestsToRunByName;
-  std::set<std::string> TestsToExcludeByName;
+  cm::optional<std::set<std::string>> TestsToRunByName;
+  cm::optional<std::set<std::string>> TestsToExcludeByName;
 
   std::string ResourceSpecFile;
 

Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake

@@ -209,7 +209,7 @@ run_SkipRegexFoundTest()
 
 
 function(run_TestsFromFileTest case)
-  set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/TestsFromFile)
+  set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/TestsFromFile-${case})
   set(RunCMake_TEST_NO_CLEAN 1)
   file(REMOVE_RECURSE "${RunCMake_TEST_BINARY_DIR}")
   file(MAKE_DIRECTORY "${RunCMake_TEST_BINARY_DIR}")
@@ -223,6 +223,8 @@ add_test(Test11 \"${CMAKE_COMMAND}\" -E echo \"test11\")
 endfunction()
 run_TestsFromFileTest(include --tests-from-file ${RunCMake_SOURCE_DIR}/TestsFromFile-TestList.txt)
 run_TestsFromFileTest(exclude --exclude-from-file ${RunCMake_SOURCE_DIR}/TestsFromFile-TestList.txt)
+run_TestsFromFileTest(include-empty --tests-from-file ${RunCMake_SOURCE_DIR}/TestsFromFile-TestList-empty.txt)
+run_TestsFromFileTest(exclude-empty --exclude-from-file ${RunCMake_SOURCE_DIR}/TestsFromFile-TestList-empty.txt)
 run_TestsFromFileTest(include-missing --tests-from-file ${RunCMake_SOURCE_DIR}/TestsFromFile-TestList-missing.txt)
 run_TestsFromFileTest(exclude-missing --exclude-from-file ${RunCMake_SOURCE_DIR}/TestsFromFile-TestList-missing.txt)
 

Tests/RunCMake/CTestCommandLine/TestsFromFile-TestList.txt

@@ -1,2 +1,5 @@
 Test1
+
+est
+  Test11
 # Test11

Tests/RunCMake/CTestCommandLine/TestsFromFile-exclude-empty-stdout.txt

@@ -0,0 +1,10 @@
+Test project [^
+]*/Tests/RunCMake/CTestCommandLine/TestsFromFile-exclude-empty
+ +Start 1: Test1
+1/3 Test #1: Test1 ............................   Passed +[0-9.]+ sec
+ +Start 2: Test2
+2/3 Test #2: Test2 ............................   Passed +[0-9.]+ sec
+ +Start 3: Test11
+3/3 Test #3: Test11 ...........................   Passed +[0-9.]+ sec
++
+100% tests passed, 0 tests failed out of 3

Tests/RunCMake/CTestCommandLine/TestsFromFile-exclude-missing-stdout.txt

@@ -1,2 +1,2 @@
 Test project [^
-]*/Tests/RunCMake/CTestCommandLine/TestsFromFile$
+]*/Tests/RunCMake/CTestCommandLine/TestsFromFile-exclude-missing$

Tests/RunCMake/CTestCommandLine/TestsFromFile-exclude-stdout.txt

@@ -1,3 +1,5 @@
+Test project [^
+]*/Tests/RunCMake/CTestCommandLine/TestsFromFile-exclude
  +Start 2: Test2
 1/2 Test #2: Test2 ............................   Passed +[0-9.]+ sec
  +Start 3: Test11

Tests/RunCMake/CTestCommandLine/TestsFromFile-include-empty-stderr.txt

@@ -0,0 +1 @@
+^No tests were found!!!$

Tests/RunCMake/CTestCommandLine/TestsFromFile-include-missing-stdout.txt

@@ -1,2 +1,2 @@
 Test project [^
-]*/Tests/RunCMake/CTestCommandLine/TestsFromFile$
+]*/Tests/RunCMake/CTestCommandLine/TestsFromFile-include-missing$

Tests/RunCMake/CTestCommandLine/TestsFromFile-include-stdout.txt

@@ -1,3 +1,5 @@
+Test project [^
+]*/Tests/RunCMake/CTestCommandLine/TestsFromFile-include
  +Start 1: Test1
 1/1 Test #1: Test1 ............................   Passed +[0-9.]+ sec
 +

Tests/RunCMake/ctest_test/RunCMakeTest.cmake

@@ -177,6 +177,8 @@ add_test(NAME Test11 COMMAND ${CMAKE_COMMAND} -E true)
 endfunction()
 run_tests_from_file(include INCLUDE_FROM_FILE ${RunCMake_SOURCE_DIR}/TestsFromFile-TestList.txt)
 run_tests_from_file(exclude EXCLUDE_FROM_FILE ${RunCMake_SOURCE_DIR}/TestsFromFile-TestList.txt)
+run_tests_from_file(include-empty INCLUDE_FROM_FILE ${RunCMake_SOURCE_DIR}/TestsFromFile-TestList-empty.txt)
+run_tests_from_file(exclude-empty EXCLUDE_FROM_FILE ${RunCMake_SOURCE_DIR}/TestsFromFile-TestList-empty.txt)
 run_tests_from_file(include-missing INCLUDE_FROM_FILE ${RunCMake_SOURCE_DIR}/TestsFromFile-TestList-missing.txt)
 run_tests_from_file(exclude-missing EXCLUDE_FROM_FILE ${RunCMake_SOURCE_DIR}/TestsFromFile-TestList-missing.txt)
 

Tests/RunCMake/ctest_test/TestsFromFile-TestList.txt

@@ -1,2 +1,5 @@
 Test1
+
+est
+  Test11
 # Test11

Tests/RunCMake/ctest_test/TestsFromFile-exclude-empty-stdout.txt

@@ -0,0 +1,12 @@
+Test project [^
+]*/Tests/RunCMake/ctest_test/TestsFromFile-exclude-empty-build
+ +Start 1: RunCMakeVersion
+1/4 Test #1: RunCMakeVersion ..................   Passed +[0-9.]+ sec
+ +Start 2: Test1
+2/4 Test #2: Test1 ............................   Passed +[0-9.]+ sec
+ +Start 3: Test2
+3/4 Test #3: Test2 ............................   Passed +[0-9.]+ sec
+ +Start 4: Test11
+4/4 Test #4: Test11 ...........................   Passed +[0-9.]+ sec
++
+100% tests passed, 0 tests failed out of 4

Tests/RunCMake/ctest_test/TestsFromFile-exclude-stdout.txt

@@ -1,3 +1,7 @@
+Test project [^
+]*/Tests/RunCMake/ctest_test/TestsFromFile-exclude-build
+ +Start 1: RunCMakeVersion
+1/3 Test #1: RunCMakeVersion ..................   Passed +[0-9.]+ sec
  +Start 3: Test2
 2/3 Test #3: Test2 ............................   Passed +[0-9.]+ sec
  +Start 4: Test11

Tests/RunCMake/ctest_test/TestsFromFile-include-empty-result.txt

@@ -0,0 +1 @@
+(-1|255)

Tests/RunCMake/ctest_test/TestsFromFile-include-empty-stderr.txt

@@ -0,0 +1 @@
+^No tests were found!!!$

Tests/RunCMake/ctest_test/TestsFromFile-include-stdout.txt

@@ -1,3 +1,5 @@
+Test project [^
+]*/Tests/RunCMake/ctest_test/TestsFromFile-include-build
  +Start 2: Test1
 1/1 Test #2: Test1 ............................   Passed +[0-9.]+ sec
 +