Add 'DISABLED' test property

When this property is set, the test is skipped and its status is
automatically set to 'Not Run'.  A disabled test will not be counted in
the total number of tests and its completion status will be 'Disabled'.

Help/manual/cmake-properties.7.rst

@@ -325,6 +325,7 @@ Properties on Tests
    /prop_test/ATTACHED_FILES
    /prop_test/COST
    /prop_test/DEPENDS
+   /prop_test/DISABLED
    /prop_test/ENVIRONMENT
    /prop_test/FAIL_REGULAR_EXPRESSION
    /prop_test/FIXTURES_CLEANUP

Help/prop_test/DISABLED.rst

@@ -0,0 +1,15 @@
+DISABLED
+--------
+
+If set to true, the test will be skipped and its status will be 'Not Run'. A
+DISABLED test will not be counted in the total number of tests and its
+completion status will be reported to CDash as 'Disabled'.
+
+A DISABLED test does not participate in test fixture dependency resolution.
+If a DISABLED test has fixture requirements defined in its
+:prop_test:`FIXTURES_REQUIRED` property, it will not cause setup or cleanup
+tests for those fixtures to be added to the test set.
+
+If a test with the :prop_test:`FIXTURES_SETUP` property set is DISABLED, the
+fixture behavior will be as though that setup test was passing and any test
+case requiring that fixture will still run.

Help/release/dev/ctest-disable-tests.rst

@@ -0,0 +1,5 @@
+ctest-disable-tests
+-------------------
+
+* A :prop_test:`DISABLED` test property was added to mark tests that
+  are configured but explicitly disabled so they do not run.

Source/CTest/cmCTestMultiProcessHandler.cxx

@@ -163,7 +163,9 @@ void cmCTestMultiProcessHandler::StartTestProcess(int test)
     this->TestRunningMap[test] = false;
     this->RunningCount -= GetProcessorsUsed(test);
     testRun->EndTest(this->Completed, this->Total, false);
-    this->Failed->push_back(this->Properties[test]->Name);
+    if (!this->Properties[test]->Disabled) {
+      this->Failed->push_back(this->Properties[test]->Name);
+    }
     delete testRun;
   }
 }

Source/CTest/cmCTestRunTest.cxx

@@ -256,6 +256,8 @@ bool cmCTestRunTest::EndTest(size_t completed, size_t total, bool started)
         cmCTestLog(this->CTest, HANDLER_OUTPUT, "Other");
         this->TestResult.Status = cmCTestTestHandler::OTHER_FAULT;
     }
+  } else if ("Disabled" == this->TestResult.CompletionStatus) {
+    cmCTestLog(this->CTest, HANDLER_OUTPUT, "***Not Run (Disabled) ");
   } else // cmsysProcess_State_Error
   {
     cmCTestLog(this->CTest, HANDLER_OUTPUT, "***Not Run ");
@@ -416,6 +418,24 @@ bool cmCTestRunTest::StartTest(size_t total)
                << this->TestProperties->Index << ": "
                << this->TestProperties->Name << std::endl);
   this->ProcessOutput.clear();
+
+  // Return immediately if test is disabled
+  if (this->TestProperties->Disabled) {
+    this->TestResult.Properties = this->TestProperties;
+    this->TestResult.ExecutionTime = 0;
+    this->TestResult.CompressOutput = false;
+    this->TestResult.ReturnValue = -1;
+    this->TestResult.CompletionStatus = "Disabled";
+    this->TestResult.Status = cmCTestTestHandler::NOT_RUN;
+    this->TestResult.TestCount = this->TestProperties->Index;
+    this->TestResult.Name = this->TestProperties->Name;
+    this->TestResult.Path = this->TestProperties->Directory;
+    this->TestProcess = new cmProcess;
+    this->TestResult.Output = "Disabled";
+    this->TestResult.FullCommandLine = "";
+    return false;
+  }
+
   this->ComputeArguments();
   std::vector<std::string>& args = this->TestProperties->Args;
   this->TestResult.Properties = this->TestProperties;

Source/CTest/cmCTestTestHandler.cxx

@@ -487,6 +487,19 @@ int cmCTestTestHandler::ProcessHandler()
       }
     }
 
+    typedef std::set<cmCTestTestHandler::cmCTestTestResult,
+                     cmCTestTestResultLess>
+      SetOfTests;
+    SetOfTests resultsSet(this->TestResults.begin(), this->TestResults.end());
+    std::vector<cmCTestTestHandler::cmCTestTestResult> disabledTests;
+
+    for (SetOfTests::iterator ftit = resultsSet.begin();
+         ftit != resultsSet.end(); ++ftit) {
+      if (ftit->CompletionStatus == "Disabled") {
+        disabledTests.push_back(*ftit);
+      }
+    }
+
     float percent = float(passed.size()) * 100.0f / float(total);
     if (!failed.empty() && percent > 99) {
       percent = 99;
@@ -505,21 +518,33 @@ int cmCTestTestHandler::ProcessHandler()
                        "\nTotal Test time (real) = " << realBuf << "\n",
                        this->Quiet);
 
+    if (!disabledTests.empty()) {
+      cmGeneratedFileStream ofs;
+      cmCTestLog(this->CTest, HANDLER_OUTPUT, std::endl
+                   << "The following tests are disabled and did not run:"
+                   << std::endl);
+      this->StartLogFile("TestsDisabled", ofs);
+
+      for (std::vector<cmCTestTestHandler::cmCTestTestResult>::iterator dtit =
+             disabledTests.begin();
+           dtit != disabledTests.end(); ++dtit) {
+        ofs << dtit->TestCount << ":" << dtit->Name << std::endl;
+        cmCTestLog(this->CTest, HANDLER_OUTPUT, "\t"
+                     << std::setw(3) << dtit->TestCount << " - " << dtit->Name
+                     << std::endl);
+      }
+    }
+
     if (!failed.empty()) {
       cmGeneratedFileStream ofs;
       cmCTestLog(this->CTest, HANDLER_OUTPUT, std::endl
                    << "The following tests FAILED:" << std::endl);
       this->StartLogFile("TestsFailed", ofs);
 
-      typedef std::set<cmCTestTestHandler::cmCTestTestResult,
-                       cmCTestTestResultLess>
-        SetOfTests;
-      SetOfTests resultsSet(this->TestResults.begin(),
-                            this->TestResults.end());
-
       for (SetOfTests::iterator ftit = resultsSet.begin();
            ftit != resultsSet.end(); ++ftit) {
-        if (ftit->Status != cmCTestTestHandler::COMPLETED) {
+        if (ftit->Status != cmCTestTestHandler::COMPLETED &&
+            ftit->CompletionStatus != "Disabled") {
           ofs << ftit->TestCount << ":" << ftit->Name << std::endl;
           cmCTestLog(
             this->CTest, HANDLER_OUTPUT, "\t"
@@ -841,6 +866,11 @@ void cmCTestTestHandler::UpdateForFixtures(ListOfTests& tests) const
   size_t fixtureTestsAdded = 0;
   std::set<std::string> addedFixtures;
   for (size_t i = 0; i < tests.size(); ++i) {
+    // Skip disabled tests
+    if (tests[i].Disabled) {
+      continue;
+    }
+
     // There are two things to do for each test:
     //   1. For every fixture required by this test, record that fixture as
     //      being required and create dependencies on that fixture's setup
@@ -1200,6 +1230,7 @@ void cmCTestTestHandler::GenerateDartOutput(cmXMLWriter& xml)
     cmCTestTestResult* result = &this->TestResults[cc];
     this->WriteTestResultHeader(xml, result);
     xml.StartElement("Results");
+
     if (result->Status != cmCTestTestHandler::NOT_RUN) {
       if (result->Status != cmCTestTestHandler::COMPLETED ||
           result->ReturnValue) {
@@ -1208,6 +1239,7 @@ void cmCTestTestHandler::GenerateDartOutput(cmXMLWriter& xml)
         xml.Attribute("name", "Exit Code");
         xml.Element("Value", this->GetTestStatus(result->Status));
         xml.EndElement(); // NamedMeasurement
+
         xml.StartElement("NamedMeasurement");
         xml.Attribute("type", "text/string");
         xml.Attribute("name", "Exit Value");
@@ -2001,6 +2033,9 @@ bool cmCTestTestHandler::SetTestsProperties(
           if (key == "WILL_FAIL") {
             rtit->WillFail = cmSystemTools::IsOn(val.c_str());
           }
+          if (key == "DISABLED") {
+            rtit->Disabled = cmSystemTools::IsOn(val.c_str());
+          }
           if (key == "ATTACHED_FILES") {
             cmSystemTools::ExpandListArgument(val, rtit->AttachedFiles);
           }
@@ -2179,6 +2214,7 @@ bool cmCTestTestHandler::AddTest(const std::vector<std::string>& args)
 
   test.IsInBasedOnREOptions = true;
   test.WillFail = false;
+  test.Disabled = false;
   test.RunSerial = false;
   test.Timeout = 0;
   test.ExplicitTimeout = false;

Source/CTest/cmCTestTestHandler.h

@@ -114,6 +114,7 @@ public:
     std::map<std::string, std::string> Measurements;
     bool IsInBasedOnREOptions;
     bool WillFail;
+    bool Disabled;
     float Cost;
     int PreviousRuns;
     bool RunSerial;

Tests/RunCMake/CMakeLists.txt

@@ -202,6 +202,7 @@ endif()
 add_RunCMake_test(ctest_start)
 add_RunCMake_test(ctest_submit)
 add_RunCMake_test(ctest_test)
+add_RunCMake_test(ctest_disabled_test)
 add_RunCMake_test(ctest_upload)
 add_RunCMake_test(ctest_fixtures)
 add_RunCMake_test(file)

Tests/RunCMake/ctest_disabled_test/CMakeLists.txt.in

@@ -0,0 +1,6 @@
+cmake_minimum_required(VERSION 3.7)
+project(@CASE_NAME@ NONE)
+include(CTest)
+
+add_test(NAME SuccessfulTest COMMAND "${CMAKE_COMMAND}" --version)
+@CASE_CMAKELISTS_SUFFIX_CODE@

Tests/RunCMake/ctest_disabled_test/CTestConfig.cmake.in

@@ -0,0 +1 @@
+set(CTEST_PROJECT_NAME "@CASE_NAME@")

Tests/RunCMake/ctest_disabled_test/DisableAllTests-result.txt

@@ -0,0 +1 @@
+(-1|255)

Tests/RunCMake/ctest_disabled_test/DisableAllTests-stderr.txt

@@ -0,0 +1 @@
+No tests were found!!!

Tests/RunCMake/ctest_disabled_test/DisableAllTests-stdout.txt

@@ -0,0 +1,2 @@
+    Start 1: SuccessfulTest
+1/1 Test #1: SuccessfulTest ...................\*\*\*\Not Run \(Disabled\) +[0-9.]+ sec

Tests/RunCMake/ctest_disabled_test/DisableCleanupTest-stdout.txt

@@ -0,0 +1,11 @@
+    Start 1: SuccessfulTest
+1/2 Test #1: SuccessfulTest ...................   Passed +[0-9.]+ sec
+    Start 2: CleanupTest
+2/2 Test #2: CleanupTest ......................\*\*\*\Not Run \(Disabled\) +[0-9.]+ sec
++
+100% tests passed, 0 tests failed out of 1
++
+Total Test time \(real\) = +[0-9.]+ sec
++
+The following tests are disabled and did not run:
+.*2 \- CleanupTest

Tests/RunCMake/ctest_disabled_test/DisableFailingTest-stdout.txt

@@ -0,0 +1,9 @@
+50% tests passed, 1 tests failed out of 2
++
+Total Test time \(real\) = +[0-9.]+ sec
++
+The following tests are disabled and did not run:
+.*3 \- DisabledFailingTest
++
+The following tests FAILED:
+.*2 \- FailingTest \(Failed\)

Tests/RunCMake/ctest_disabled_test/DisableNotRunTest-result.txt

@@ -0,0 +1 @@
+(-1|255)

Tests/RunCMake/ctest_disabled_test/DisableNotRunTest-stderr.txt

@@ -0,0 +1 @@
+Unable to find executable: invalidCommand

Tests/RunCMake/ctest_disabled_test/DisableNotRunTest-stdout.txt

@@ -0,0 +1,17 @@
+    Start 1: SuccessfulTest
+1/3 Test #1: SuccessfulTest ...................   Passed +[0-9.]+ sec
+    Start 2: DisabledTest
+2/3 Test #2: DisabledTest .....................\*\*\*\Not Run \(Disabled\) +[0-9.]+ sec
+    Start 3: NotRunTest
+.*
+3/3 Test #3: NotRunTest .......................\*\*\*\Not Run +[0-9.]+ sec
++
+50% tests passed, 1 tests failed out of 2
++
+Total Test time \(real\) = +[0-9.]+ sec
++
+The following tests are disabled and did not run:
+.*2 \- DisabledTest
++
+The following tests FAILED:
+.*3 - NotRunTest \(Not Run\)

Tests/RunCMake/ctest_disabled_test/DisableRequiredTest-stdout.txt

@@ -0,0 +1,13 @@
+    Start 1: SuccessfulTest
+1/3 Test #1: SuccessfulTest ...................   Passed +[0-9.]+ sec
+    Start 2: DisabledTest
+2/3 Test #2: DisabledTest .....................\*\*\*\Not Run \(Disabled\) +[0-9.]+ sec
+    Start 3: SuccessfulCleanupTest
+3/3 Test #3: SuccessfulCleanupTest ............   Passed +[0-9.]+ sec
++
+100% tests passed, 0 tests failed out of 2
++
+Total Test time \(real\) = +[0-9.]+ sec
++
+The following tests are disabled and did not run:
+.*2 \- DisabledTest

Tests/RunCMake/ctest_disabled_test/DisableSetupTest-stdout.txt

@@ -0,0 +1,13 @@
+    Start 2: DisabledTest
+1/3 Test #2: DisabledTest .....................\*\*\*\Not Run \(Disabled\) +[0-9.]+ sec
+    Start 1: SuccessfulTest
+2/3 Test #1: SuccessfulTest ...................   Passed +[0-9.]+ sec
+    Start 3: SuccessfulCleanupTest
+3/3 Test #3: SuccessfulCleanupTest ............   Passed +[0-9.]+ sec
++
+100% tests passed, 0 tests failed out of 2
++
+Total Test time \(real\) = +[0-9.]+ sec
++
+The following tests are disabled and did not run:
+.*2 \- DisabledTest

Tests/RunCMake/ctest_disabled_test/DisabledTest-result.txt

@@ -0,0 +1 @@
+(-1|255)

Tests/RunCMake/ctest_disabled_test/DisabledTest-stderr.txt

@@ -0,0 +1 @@
+Unable to find executable: invalidCommand

Tests/RunCMake/ctest_disabled_test/DisabledTest-stdout.txt

@@ -0,0 +1,17 @@
+    Start 1: SuccessfulTest
+1/3 Test #1: SuccessfulTest ...................   Passed +[0-9.]+ sec
+    Start 2: DisabledTest
+2/3 Test #2: DisabledTest .....................\*\*\*\Not Run \(Disabled\) +[0-9.]+ sec
+    Start 3: NotRunTest
+.*
+3/3 Test #3: NotRunTest .......................\*\*\*\Not Run +[0-9.]+ sec
++
+50% tests passed, 1 tests failed out of 2
++
+Total Test time \(real\) = +[0-9.]+ sec
++
+The following tests are disabled and did not run:
+.*2 \- DisabledTest
++
+The following tests FAILED:
+.*3 - NotRunTest \(Not Run\)

Tests/RunCMake/ctest_disabled_test/RunCMakeTest.cmake

@@ -0,0 +1,89 @@
+include(RunCTest)
+
+function(run_DisableNotRunTest)
+  set(CASE_CMAKELISTS_SUFFIX_CODE [[
+add_test(NAME DisabledTest COMMAND notACommand --version)
+add_test(NAME NotRunTest COMMAND invalidCommand --version)
+
+set_tests_properties(SuccessfulTest PROPERTIES DISABLED false)
+set_tests_properties(DisabledTest PROPERTIES DISABLED true)
+  ]])
+  run_ctest(DisableNotRunTest)
+endfunction()
+run_DisableNotRunTest()
+
+function(run_DisableFailingTest)
+  set(CASE_CMAKELISTS_SUFFIX_CODE [[
+set(someFile "${CMAKE_CURRENT_SOURCE_DIR}/test.cmake")
+add_test(NAME FailingTest
+          COMMAND ${CMAKE_COMMAND} -E compare_files "${someFile}" "${someFile}xxx")
+add_test(NAME DisabledFailingTest
+          COMMAND ${CMAKE_COMMAND} -E compare_files "${someFile}" "${someFile}xxx")
+
+set_tests_properties(FailingTest PROPERTIES DISABLED false)
+set_tests_properties(DisabledFailingTest PROPERTIES DISABLED true)
+  ]])
+  run_ctest(DisableFailingTest)
+endfunction()
+run_DisableFailingTest()
+
+function(run_DisableSetupTest)
+  set(CASE_CMAKELISTS_SUFFIX_CODE [[
+add_test(NAME DisabledTest COMMAND "${CMAKE_COMMAND}" --version)
+add_test(NAME SuccessfulCleanupTest COMMAND "${CMAKE_COMMAND}" --version)
+
+set_tests_properties(DisabledTest PROPERTIES DISABLED true
+                                             FIXTURES_SETUP "Foo")
+set_tests_properties(SuccessfulTest PROPERTIES FIXTURES_REQUIRED "Foo")
+set_tests_properties(SuccessfulCleanupTest PROPERTIES FIXTURES_CLEANUP "Foo")
+  ]])
+run_ctest(DisableSetupTest)
+endfunction()
+run_DisableSetupTest()
+
+function(run_DisableRequiredTest)
+  set(CASE_CMAKELISTS_SUFFIX_CODE [[
+add_test(NAME DisabledTest COMMAND "${CMAKE_COMMAND}" --version)
+add_test(NAME SuccessfulCleanupTest COMMAND "${CMAKE_COMMAND}" --version)
+
+set_tests_properties(SuccessfulTest PROPERTIES FIXTURES_SETUP "Foo")
+set_tests_properties(DisabledTest PROPERTIES DISABLED true
+                                             FIXTURES_REQUIRED "Foo")
+set_tests_properties(SuccessfulCleanupTest PROPERTIES FIXTURES_CLEANUP "Foo")
+  ]])
+run_ctest(DisableRequiredTest)
+endfunction()
+run_DisableRequiredTest()
+
+function(run_DisableCleanupTest)
+  set(CASE_CMAKELISTS_SUFFIX_CODE [[
+add_test(NAME CleanupTest COMMAND "${CMAKE_COMMAND}" --version)
+
+set_tests_properties(SuccessfulTest PROPERTIES FIXTURES_REQUIRED "Foo")
+set_tests_properties(CleanupTest PROPERTIES DISABLED true
+                                            FIXTURES_CLEANUP "Foo")
+  ]])
+run_ctest(DisableCleanupTest)
+endfunction()
+run_DisableCleanupTest()
+
+# Consider a fixture that has a setup test, a cleanup test and a disabled test
+# which requires that fixture. Limit the test list with a regular expression
+# that matches the disabled test but not the setup or cleanup tests, so the
+# initial set of tests to be executed contains just the disabled test. Since
+# the only test requiring the fixture is disabled, CTest should not
+# automatically add in the setup and cleanup tests for the fixture, since no
+# enabled test requires them.
+function(run_DisableAllTests)
+  set(CASE_CMAKELISTS_SUFFIX_CODE [[
+add_test(NAME SetupTest COMMAND "${CMAKE_COMMAND}" --version)
+add_test(NAME CleanupTest COMMAND "${CMAKE_COMMAND}" --version)
+
+set_tests_properties(SetupTest PROPERTIES FIXTURES_SETUP "Foo")
+set_tests_properties(SuccessfulTest PROPERTIES DISABLED true
+                                               FIXTURES_REQUIRED "Foo")
+set_tests_properties(CleanupTest PROPERTIES FIXTURES_CLEANUP "Foo")
+  ]])
+run_ctest(DisableAllTests -R Successful)
+endfunction()
+run_DisableAllTests()

Tests/RunCMake/ctest_disabled_test/test.cmake.in

@@ -0,0 +1,16 @@
+cmake_minimum_required(VERSION 3.7)
+
+set(CTEST_SITE                          "test-site")
+set(CTEST_BUILD_NAME                    "test-build-name")
+set(CTEST_SOURCE_DIRECTORY              "@RunCMake_BINARY_DIR@/@CASE_NAME@")
+set(CTEST_BINARY_DIRECTORY              "@RunCMake_BINARY_DIR@/@CASE_NAME@-build")
+set(CTEST_CMAKE_GENERATOR               "@RunCMake_GENERATOR@")
+set(CTEST_CMAKE_GENERATOR_PLATFORM      "@RunCMake_GENERATOR_PLATFORM@")
+set(CTEST_CMAKE_GENERATOR_TOOLSET       "@RunCMake_GENERATOR_TOOLSET@")
+set(CTEST_BUILD_CONFIGURATION           "$ENV{CMAKE_CONFIG_TYPE}")
+
+set(ctest_test_args "@CASE_CTEST_TEST_ARGS@")
+ctest_start(Experimental)
+ctest_configure()
+ctest_build()
+ctest_test(${ctest_test_args})