Merge topic 'skip_regular_expression'

5dbd9c8583 Vim: Add SKIP_REGULAR_EXPRESSION test property to vim syntax file
407dd1a910 Help: Add documentation for SKIP_REGULAR_EXPRESSION test property
d7955d4e5d Tests: Create test for SKIP_REGULAR_EXPRESSION test property
4f1dec86a5 CTest: Add SKIP_REGULAR_EXPRESSION test property

Acked-by: Kitware Robot <[email protected]>
Merge-request: !3570

Auxiliary/vim/syntax/cmake.vim

@@ -288,6 +288,7 @@ syn keyword cmakeProperty contained
             \ SKIP_AUTORCC
             \ SKIP_AUTOUIC
             \ SKIP_BUILD_RPATH
+            \ SKIP_REGULAR_EXPRESSION
             \ SKIP_RETURN_CODE
             \ SOURCES
             \ SOURCE_DIR
@@ -1772,6 +1773,7 @@ syn keyword cmakeKWadd_test contained
             \ NAME
             \ OFF
             \ PASS_REGULAR_EXPRESSION
+            \ SKIP_REGULAR_EXPRESSION
             \ TARGET_FILE
             \ WILL_FAIL
             \ WORKING_DIRECTORY

Help/command/add_test.rst

@@ -38,8 +38,9 @@ The given test command is expected to exit with code ``0`` to pass and
 non-zero to fail, or vice-versa if the :prop_test:`WILL_FAIL` test
 property is set.  Any output written to stdout or stderr will be
 captured by :manual:`ctest(1)` but does not affect the pass/fail status
-unless the :prop_test:`PASS_REGULAR_EXPRESSION` or
-:prop_test:`FAIL_REGULAR_EXPRESSION` test property is used.
+unless the :prop_test:`PASS_REGULAR_EXPRESSION`,
+:prop_test:`FAIL_REGULAR_EXPRESSION` or
+:prop_test:`SKIP_REGULAR_EXPRESSION` test property is used.
 
 The ``COMMAND`` and ``WORKING_DIRECTORY`` options may use "generator
 expressions" with the syntax ``$<...>``.  See the

Help/manual/cmake-properties.7.rst

@@ -407,6 +407,7 @@ Properties on Tests
    /prop_test/REQUIRED_FILES
    /prop_test/RESOURCE_LOCK
    /prop_test/RUN_SERIAL
+   /prop_test/SKIP_REGULAR_EXPRESSION
    /prop_test/SKIP_RETURN_CODE
    /prop_test/TIMEOUT
    /prop_test/TIMEOUT_AFTER_MATCH

Help/prop_test/SKIP_REGULAR_EXPRESSION.rst

@@ -0,0 +1,17 @@
+SKIP_REGULAR_EXPRESSION
+-----------------------
+
+If the output matches this regular expression the test will be marked as skipped.
+
+If set, if the output matches one of specified regular expressions,
+the test will be marked as skipped.  Example:
+
+.. code-block:: cmake
+
+  set_property(TEST mytest PROPERTY
+    SKIP_REGULAR_EXPRESSION "[^a-z]Skip" "SKIP" "Skipped"
+  )
+
+``SKIP_REGULAR_EXPRESSION`` expects a list of regular expressions.
+
+See also the :prop_test:`SKIP_RETURN_CODE` property.

Help/prop_test/SKIP_RETURN_CODE.rst

@@ -8,3 +8,5 @@ test are met. If such a situation should not be considered a hard failure
 a return code of the process can be specified that will mark the test as
 ``Not Run`` if it is encountered. Valid values are in the range of
 0 to 255, inclusive.
+
+See also the :prop_test:`SKIP_REGULAR_EXPRESSION` property.

Help/release/dev/add_skip_regular_expression_test_property.rst

@@ -0,0 +1,10 @@
+add_skip_regular_expression_test_property
+-----------------------------------------
+
+* A new test property, :prop_test:`SKIP_REGULAR_EXPRESSION`, has been added.
+  This property is similar to :prop_test:`FAIL_REGULAR_EXPRESSION` and
+  :prop_test:`PASS_REGULAR_EXPRESSION`, but with the same meaning as
+  :prop_test:`SKIP_RETURN_CODE`. This is useful, for example, in cases where
+  the user has no control over the return code of the test. For example, in
+  Catch2, the return value is the number of assertion failed, therefore it is
+  impossible to use it for :prop_test:`SKIP_RETURN_CODE`.

Source/CTest/cmCTestMultiProcessHandler.cxx

@@ -823,6 +823,11 @@ static Json::Value DumpCTestProperties(
       "FAIL_REGULAR_EXPRESSION",
       DumpRegExToJsonArray(testProperties.ErrorRegularExpressions)));
   }
+  if (!testProperties.SkipRegularExpressions.empty()) {
+    properties.append(DumpCTestProperty(
+      "SKIP_REGULAR_EXPRESSION",
+      DumpRegExToJsonArray(testProperties.SkipRegularExpressions)));
+  }
   if (!testProperties.FixturesCleanup.empty()) {
     properties.append(DumpCTestProperty(
       "FIXTURES_CLEANUP", DumpToJsonArray(testProperties.FixturesCleanup)));

Source/CTest/cmCTestRunTest.cxx

@@ -77,6 +77,7 @@ bool cmCTestRunTest::EndTest(size_t completed, size_t total, bool started)
   }
   std::int64_t retVal = this->TestProcess->GetExitValue();
   bool forceFail = false;
+  bool forceSkip = false;
   bool skipped = false;
   bool outputTestErrorsToConsole = false;
   if (!this->TestProperties->RequiredRegularExpressions.empty() &&
@@ -116,16 +117,34 @@ bool cmCTestRunTest::EndTest(size_t completed, size_t total, bool started)
       }
     }
   }
+  if (!this->TestProperties->SkipRegularExpressions.empty() &&
+      this->FailedDependencies.empty()) {
+    for (auto& skip : this->TestProperties->SkipRegularExpressions) {
+      if (skip.first.find(this->ProcessOutput)) {
+        reason = "Skip regular expression found in output.";
+        reason += " Regex=[";
+        reason += skip.second;
+        reason += "]";
+        forceSkip = true;
+        break;
+      }
+    }
+  }
   std::ostringstream outputStream;
   if (res == cmProcess::State::Exited) {
     bool success = !forceFail &&
       (retVal == 0 ||
        !this->TestProperties->RequiredRegularExpressions.empty());
-    if (this->TestProperties->SkipReturnCode >= 0 &&
-        this->TestProperties->SkipReturnCode == retVal) {
+    if ((this->TestProperties->SkipReturnCode >= 0 &&
+         this->TestProperties->SkipReturnCode == retVal) ||
+        forceSkip) {
       this->TestResult.Status = cmCTestTestHandler::NOT_RUN;
       std::ostringstream s;
-      s << "SKIP_RETURN_CODE=" << this->TestProperties->SkipReturnCode;
+      if (forceSkip) {
+        s << "SKIP_REGULAR_EXPRESSION_MATCHED";
+      } else {
+        s << "SKIP_RETURN_CODE=" << this->TestProperties->SkipReturnCode;
+      }
       this->TestResult.CompletionStatus = s.str();
       cmCTestLog(this->CTest, HANDLER_OUTPUT, "***Skipped ");
       skipped = true;

Source/CTest/cmCTestTestHandler.cxx

@@ -526,7 +526,7 @@ int cmCTestTestHandler::ProcessHandler()
     std::vector<cmCTestTestHandler::cmCTestTestResult> disabledTests;
 
     for (cmCTestTestResult const& ft : resultsSet) {
-      if (cmHasLiteralPrefix(ft.CompletionStatus, "SKIP_RETURN_CODE=") ||
+      if (cmHasLiteralPrefix(ft.CompletionStatus, "SKIP_") ||
           ft.CompletionStatus == "Disabled") {
         disabledTests.push_back(ft);
       }
@@ -599,7 +599,7 @@ int cmCTestTestHandler::ProcessHandler()
 
       for (cmCTestTestResult const& ft : resultsSet) {
         if (ft.Status != cmCTestTestHandler::COMPLETED &&
-            !cmHasLiteralPrefix(ft.CompletionStatus, "SKIP_RETURN_CODE=") &&
+            !cmHasLiteralPrefix(ft.CompletionStatus, "SKIP_") &&
             ft.CompletionStatus != "Disabled") {
           ofs << ft.TestCount << ":" << ft.Name << std::endl;
           auto testColor = cmCTest::Color::RED;
@@ -2229,6 +2229,13 @@ bool cmCTestTestHandler::SetTestsProperties(
               rt.ErrorRegularExpressions.emplace_back(cr, cr);
             }
           }
+          if (key == "SKIP_REGULAR_EXPRESSION") {
+            std::vector<std::string> lval;
+            cmSystemTools::ExpandListArgument(val, lval);
+            for (std::string const& cr : lval) {
+              rt.SkipRegularExpressions.emplace_back(cr, cr);
+            }
+          }
           if (key == "PROCESSORS") {
             rt.Processors = atoi(val.c_str());
             if (rt.Processors < 1) {

Source/CTest/cmCTestTestHandler.h

@@ -117,6 +117,8 @@ public:
       ErrorRegularExpressions;
     std::vector<std::pair<cmsys::RegularExpression, std::string>>
       RequiredRegularExpressions;
+    std::vector<std::pair<cmsys::RegularExpression, std::string>>
+      SkipRegularExpressions;
     std::vector<std::pair<cmsys::RegularExpression, std::string>>
       TimeoutRegularExpressions;
     std::map<std::string, std::string> Measurements;

Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake

@@ -120,6 +120,20 @@ set_tests_properties(test1 PROPERTIES FAIL_REGULAR_EXPRESSION \"foo;test1;bar\"
 endfunction()
 run_FailRegexFoundTest()
 
+function(run_SkipRegexFoundTest)
+  set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/SkipRegexFound)
+  set(RunCMake_TEST_NO_CLEAN 1)
+  file(REMOVE_RECURSE "${RunCMake_TEST_BINARY_DIR}")
+  file(MAKE_DIRECTORY "${RunCMake_TEST_BINARY_DIR}")
+  file(WRITE "${RunCMake_TEST_BINARY_DIR}/CTestTestfile.cmake" "
+add_test(test1 \"${CMAKE_COMMAND}\" -E echo \"test1\")
+set_tests_properties(test1 PROPERTIES SKIP_REGULAR_EXPRESSION \"test1\")
+")
+
+  run_cmake_command(SkipRegexFound ${CMAKE_CTEST_COMMAND} -V)
+endfunction()
+run_SkipRegexFoundTest()
+
 function(run_SerialFailed)
   set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/SerialFailed)
   set(RunCMake_TEST_NO_CLEAN 1)

Tests/RunCMake/CTestCommandLine/SkipRegexFound-check.cmake

@@ -0,0 +1,13 @@
+set(last_test_log "${RunCMake_TEST_BINARY_DIR}/Testing/Temporary/LastTest.log")
+if(EXISTS "${last_test_log}")
+  file(READ "${last_test_log}" last_test_log_content)
+  string(REGEX REPLACE "\n+$" "" last_test_log_content "${last_test_log_content}")
+  if(NOT last_test_log_content MATCHES "
+Test Pass Reason:
+Skip regular expression found in output. Regex=[[]test1]")
+    string(REPLACE "\n" "\n  " last_test_log_content "  ${last_test_log_content}")
+    set(RunCMake_TEST_FAILED "LastTest.log does not have expected content:\n${last_test_log_content}")
+  endif()
+else()
+  set(RunCMake_TEST_FAILED "LastTest.log missing:\n ${last_test_log}")
+endif()