Add --no-tests=<[error|ignore]> option to CTest

If no tests were found, the default behavior of CTest is to always log an
error message but to return an error code in script mode only. This option
unifies the behavior of CTest by either returning an error code if no tests
were found or by ignoring it.

Signed-off-by: Stefan Dinkelacker <[email protected]>

Help/manual/ctest.1.rst

@@ -370,6 +370,14 @@ See `Build and Test Mode`_.
  This option will not run any tests, it will simply print the list of
  all labels associated with the test set.
 
+``--no-tests=<[error|ignore]>``
+ Regard no tests found either as error or ignore it.
+
+ If no tests were found, the default behavior of CTest is to always log an
+ error message but to return an error code in script mode only.  This option
+ unifies the behavior of CTest by either returning an error code if no tests
+ were found or by ignoring it.
+
 .. include:: OPTIONS_HELP.txt
 
 .. _`Label and Subproject Summary`:

Help/release/dev/ctest-no-tests.rst

@@ -0,0 +1,6 @@
+ctest-no-tests
+--------------
+
+* The :manual:`ctest(1)` tool gained a ``--no-tests=<[error|ignore]>`` option
+  to explicitly set and unify the behavior between direct invocation and
+  script mode if no tests were found.

Source/CTest/cmCTestTestHandler.cxx

@@ -410,10 +410,15 @@ int cmCTestTestHandler::ProcessHandler()
 
   auto clock_finish = std::chrono::steady_clock::now();
 
+  bool noTestsFoundError = false;
   if (passed.size() + failed.size() == 0) {
-    if (!this->CTest->GetShowOnly() && !this->CTest->ShouldPrintLabels()) {
+    if (!this->CTest->GetShowOnly() && !this->CTest->ShouldPrintLabels() &&
+        this->CTest->GetNoTestsMode() != cmCTest::NoTests::Ignore) {
       cmCTestLog(this->CTest, ERROR_MESSAGE,
                  "No tests were found!!!" << std::endl);
+      if (this->CTest->GetNoTestsMode() == cmCTest::NoTests::Error) {
+        noTestsFoundError = true;
+      }
     }
   } else {
     if (this->HandlerVerbose && !passed.empty() &&
@@ -459,6 +464,12 @@ int cmCTestTestHandler::ProcessHandler()
     this->LogFile = nullptr;
     return -1;
   }
+
+  if (noTestsFoundError) {
+    this->LogFile = nullptr;
+    return -1;
+  }
+
   this->LogFile = nullptr;
   return 0;
 }

Source/cmCTest.cxx

@@ -208,6 +208,8 @@ struct cmCTest::Private
   bool OutputColorCode = cmCTest::ColoredOutputSupportedByConsole();
 
   std::map<std::string, std::string> Definitions;
+
+  cmCTest::NoTests NoTestsMode = cmCTest::NoTests::Legacy;
 };
 
 struct tm* cmCTest::GetNightlyTime(std::string const& str, bool tomorrowtag)
@@ -2059,6 +2061,19 @@ bool cmCTest::HandleCommandLineArguments(size_t& i,
     this->SetNotesFiles(args[i].c_str());
   }
 
+  const std::string noTestsPrefix = "--no-tests=";
+  if (cmHasPrefix(arg, noTestsPrefix)) {
+    const std::string noTestsMode = arg.substr(noTestsPrefix.length());
+    if (noTestsMode == "error") {
+      this->Impl->NoTestsMode = cmCTest::NoTests::Error;
+    } else if (noTestsMode != "ignore") {
+      errormsg = "'--no-tests=' given unknown value '" + noTestsMode + "'";
+      return false;
+    } else {
+      this->Impl->NoTestsMode = cmCTest::NoTests::Ignore;
+    }
+  }
+
   // options that control what tests are run
   if (this->CheckArgument(arg, "-I", "--tests-information") &&
       i < args.size() - 1) {
@@ -2896,6 +2911,11 @@ cmCTest::Repeat cmCTest::GetRepeatMode() const
   return this->Impl->RepeatMode;
 }
 
+cmCTest::NoTests cmCTest::GetNoTestsMode() const
+{
+  return this->Impl->NoTestsMode;
+}
+
 void cmCTest::SetBuildID(const std::string& id)
 {
   this->Impl->BuildID = id;

Source/cmCTest.h

@@ -442,6 +442,14 @@ public:
   };
   Repeat GetRepeatMode() const;
 
+  enum class NoTests
+  {
+    Legacy,
+    Error,
+    Ignore
+  };
+  NoTests GetNoTestsMode() const;
+
   void GenerateSubprojectsOutput(cmXMLWriter& xml);
   std::vector<std::string> GetLabelsForSubprojects();
 

Source/ctest.cxx

@@ -144,6 +144,8 @@ static const char* cmDocumentationOptions[][2] = {
   { "--http1.0", "Submit using HTTP 1.0." },
   { "--no-compress-output", "Do not compress test output when submitting." },
   { "--print-labels", "Print all available test labels." },
+  { "--no-tests=<[error|ignore]>",
+    "Regard no tests found either as 'error' or 'ignore' it." },
   { nullptr, nullptr }
 };
 

Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake

@@ -314,5 +314,34 @@ function(run_ShowOnly)
 endfunction()
 run_ShowOnly()
 
+function(run_NoTests)
+  set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/NoTests)
+  set(RunCMake_TEST_NO_CLEAN 1)
+  file(REMOVE_RECURSE "${RunCMake_TEST_BINARY_DIR}")
+  file(MAKE_DIRECTORY "${RunCMake_TEST_BINARY_DIR}")
+  file(WRITE "${RunCMake_TEST_BINARY_DIR}/CTestTestfile.cmake" "")
+  run_cmake_command(no-tests_ignore ${CMAKE_CTEST_COMMAND} --no-tests=ignore)
+  run_cmake_command(no-tests_error ${CMAKE_CTEST_COMMAND} --no-tests=error)
+  run_cmake_command(no-tests_bad ${CMAKE_CTEST_COMMAND} --no-tests=bad)
+  run_cmake_command(no-tests_legacy ${CMAKE_CTEST_COMMAND})
+  file(WRITE "${RunCMake_TEST_BINARY_DIR}/NoTestsScript.cmake" "
+    set(CTEST_COMMAND \"${CMAKE_CTEST_COMMAND}\")
+    set(CTEST_SOURCE_DIRECTORY \"${RunCMake_SOURCE_DIR}\")
+    set(CTEST_BINARY_DIRECTORY \"${RunCMake_TEST_BINARY_DIR}\")
+    ctest_start(Experimental)
+    ctest_test()
+")
+  run_cmake_command(
+    no-tests-script_ignore ${CMAKE_CTEST_COMMAND} --no-tests=ignore
+    -S "${RunCMake_TEST_BINARY_DIR}/NoTestsScript.cmake")
+  run_cmake_command(
+    no-tests-script_error ${CMAKE_CTEST_COMMAND} --no-tests=error
+    -S "${RunCMake_TEST_BINARY_DIR}/NoTestsScript.cmake")
+  run_cmake_command(
+    no-tests-script_legacy ${CMAKE_CTEST_COMMAND}
+    -S "${RunCMake_TEST_BINARY_DIR}/NoTestsScript.cmake")
+endfunction()
+run_NoTests()
+
 # Check the configuration type variable is passed
 run_ctest(check-configuration-type)

Tests/RunCMake/CTestCommandLine/no-tests-script_error-result.txt

@@ -0,0 +1 @@
+(-1|255)

Tests/RunCMake/CTestCommandLine/no-tests-script_error-stderr.txt

@@ -0,0 +1 @@
+^No tests were found!!!$

Tests/RunCMake/CTestCommandLine/no-tests-script_legacy-result.txt

@@ -0,0 +1 @@
+(-1|255)

Tests/RunCMake/CTestCommandLine/no-tests-script_legacy-stderr.txt

@@ -0,0 +1 @@
+^No tests were found!!!$

Tests/RunCMake/CTestCommandLine/no-tests_bad-result.txt

@@ -0,0 +1 @@
+1

Tests/RunCMake/CTestCommandLine/no-tests_bad-stderr.txt

@@ -0,0 +1 @@
+^CMake Error: '--no-tests=' given unknown value 'bad'$

Tests/RunCMake/CTestCommandLine/no-tests_error-result.txt

@@ -0,0 +1 @@
+8

Tests/RunCMake/CTestCommandLine/no-tests_error-stderr.txt

@@ -0,0 +1 @@
+No tests were found!!!

Tests/RunCMake/CTestCommandLine/no-tests_legacy-stderr.txt

@@ -0,0 +1 @@
+^No tests were found!!!$