CTest: Report tests not run due to unknown configuration

When add_test(NAME) is called without the CONFIGURATIONS argument then
the test is intended to run in any configuration.  In multi-config
generators like the VS IDE and Xcode tests created by add_test(NAME) can
only be run when testing a known configuration (otherwise there is no
way to generate the test command line).  If no test command line is
known for a particular configuration, or if no configuration is given to
ctest, report the test as not run instead of silently skipping it.

Also fix CMake's own TestsWorkingDirectory test invocation to correct a
previously silent failure exposed by this change.

Source/CTest/cmCTestRunTest.cxx

@@ -412,6 +412,30 @@ bool cmCTestRunTest::StartTest(size_t total)
   this->TestResult.TestCount = this->TestProperties->Index;  
   this->TestResult.Name = this->TestProperties->Name;
   this->TestResult.Path = this->TestProperties->Directory.c_str();
+
+  if(args.size() >= 2 && args[1] == "NOT_AVAILABLE")
+    {
+    this->TestProcess = new cmProcess;
+    std::string msg;
+    if(this->CTest->GetConfigType().empty())
+      {
+      msg = "Test not available without configuration.";
+      msg += "  (Missing \"-C <config>\"?)";
+      }
+    else
+      {
+      msg = "Test not available in configuration \"";
+      msg += this->CTest->GetConfigType();
+      msg += "\".";
+      }
+    *this->TestHandler->LogFile << msg << std::endl;
+    cmCTestLog(this->CTest, ERROR_MESSAGE, msg << std::endl);
+    this->TestResult.Output = msg;
+    this->TestResult.FullCommandLine = "";
+    this->TestResult.CompletionStatus = "Not Run";
+    this->TestResult.Status = cmCTestTestHandler::NOT_RUN;
+    return false;
+    }
   
   // Check if all required files exist
   for(std::vector<std::string>::iterator i =

Source/CTest/cmCTestTestHandler.cxx

@@ -1320,6 +1320,10 @@ std::string cmCTestTestHandler::FindTheExecutable(const char *exe)
   std::string resConfig;
   std::vector<std::string> extraPaths;
   std::vector<std::string> failedPaths;
+  if(strcmp(exe, "NOT_AVAILABLE") == 0)
+    {
+    return exe;
+    }
   return cmCTestTestHandler::FindExecutable(this->CTest,
                                             exe, resConfig,
                                             extraPaths,

Source/cmTestGenerator.cxx

@@ -129,6 +129,22 @@ void cmTestGenerator::GenerateScriptForConfig(std::ostream& os,
   os << ")\n";
 }
 
+//----------------------------------------------------------------------------
+void cmTestGenerator::GenerateScriptNoConfig(std::ostream& os,
+                                             Indent const& indent)
+{
+  os << indent << "ADD_TEST(" << this->Test->GetName() << " NOT_AVAILABLE)\n";
+}
+
+//----------------------------------------------------------------------------
+bool cmTestGenerator::NeedsScriptNoConfig() const
+{
+  return (this->TestGenerated && // test generated for at least one config
+          this->ActionsPerConfig && // test is config-aware
+          this->Configurations.empty() && // test runs in all configs
+          !this->ConfigurationTypes->empty()); // config-dependent command
+}
+
 //----------------------------------------------------------------------------
 void cmTestGenerator::GenerateOldStyle(std::ostream& fout,
                                        Indent const& indent)

Source/cmTestGenerator.h

@@ -34,6 +34,8 @@ protected:
   virtual void GenerateScriptForConfig(std::ostream& os,
                                        const char* config,
                                        Indent const& indent);
+  virtual void GenerateScriptNoConfig(std::ostream& os, Indent const& indent);
+  virtual bool NeedsScriptNoConfig() const;
   void GenerateOldStyle(std::ostream& os, Indent const& indent);
 
   cmTest* Test;

Tests/CMakeLists.txt

@@ -1310,7 +1310,7 @@ ${CMake_BINARY_DIR}/bin/cmake -DVERSION=master -P ${CMake_SOURCE_DIR}/Utilities/
     --build-makeprogram ${CMAKE_TEST_MAKEPROGRAM}
     --build-exe-dir "${CMake_BINARY_DIR}/Tests/TestsWorkingDirectory"
     --force-new-ctest-process
-    --test-command ${CMAKE_CTEST_COMMAND} -V
+    --test-command ${CMAKE_CTEST_COMMAND} -V -C \${CTEST_CONFIGURATION_TYPE}
     )
   LIST(APPEND TEST_BUILD_DIRS "${CMake_BINARY_DIR}/Tests/TestsWorkingDirectory")