CTest: Add CTEST_RESOURCE_SPEC_FILE variable

Help/manual/cmake-variables.7.rst

@@ -623,6 +623,7 @@ Variables for CTest
    /variable/CTEST_P4_COMMAND
    /variable/CTEST_P4_OPTIONS
    /variable/CTEST_P4_UPDATE_OPTIONS
+   /variable/CTEST_RESOURCE_SPEC_FILE
    /variable/CTEST_RUN_CURRENT_SCRIPT
    /variable/CTEST_SCP_COMMAND
    /variable/CTEST_SITE

Help/manual/ctest.1.rst

@@ -994,8 +994,12 @@ Configuration settings include:
 
 ``ResourceSpecFile``
   Specify a
-  :ref:`resource specification file <ctest-resource-specification-file>`. See
-  :ref:`ctest-resource-allocation` for more information.
+  :ref:`resource specification file <ctest-resource-specification-file>`.
+
+  * `CTest Script`_ variable: :variable:`CTEST_RESOURCE_SPEC_FILE`
+  * :module:`CTest` module variable: ``CTEST_RESOURCE_SPEC_FILE``
+
+  See :ref:`ctest-resource-allocation` for more information.
 
 ``LabelsForSubprojects``
   Specify a semicolon-separated list of labels that will be treated as

Help/release/dev/ctest_resource_spec_file-variable.rst

@@ -0,0 +1,6 @@
+ctest_resource_spec_file-variable
+---------------------------------
+
+* :manual:`ctest(1)` gained a new :variable:`CTEST_RESOURCE_SPEC_FILE`
+  variable, which can be used to specify a
+  :ref:`resource specification file <ctest-resource-specification-file>`.

Help/variable/CTEST_RESOURCE_SPEC_FILE.rst

@@ -0,0 +1,5 @@
+CTEST_RESOURCE_SPEC_FILE
+------------------------
+
+Specify the CTest ``ResourceSpecFile`` setting in a :manual:`ctest(1)`
+dashboard client script.

Source/CTest/cmCTestTestCommand.cxx

@@ -52,6 +52,13 @@ cmCTestGenericHandler* cmCTestTestCommand::InitializeHandler()
     }
   }
   this->CTest->SetTimeOut(timeout);
+
+  const char* resourceSpecFile =
+    this->Makefile->GetDefinition("CTEST_RESOURCE_SPEC_FILE");
+  if (this->ResourceSpecFile.empty() && resourceSpecFile) {
+    this->ResourceSpecFile = resourceSpecFile;
+  }
+
   cmCTestGenericHandler* handler = this->InitializeActualHandler();
   if (!this->Start.empty() || !this->End.empty() || !this->Stride.empty()) {
     handler->SetOption(

Source/CTest/cmCTestTestHandler.cxx

@@ -408,7 +408,9 @@ int cmCTestTestHandler::ProcessHandler()
   // start the real time clock
   auto clock_start = std::chrono::steady_clock::now();
 
-  this->ProcessDirectory(passed, failed);
+  if (!this->ProcessDirectory(passed, failed)) {
+    return -1;
+  }
 
   auto clock_finish = std::chrono::steady_clock::now();
 
@@ -545,22 +547,11 @@ bool cmCTestTestHandler::ProcessOptions()
   if (val) {
     this->ExcludeFixtureCleanupRegExp = val;
   }
-  this->SetRerunFailed(cmIsOn(this->GetOption("RerunFailed")));
-
   val = this->GetOption("ResourceSpecFile");
   if (val) {
-    this->UseResourceSpec = true;
     this->ResourceSpecFile = val;
-    auto result = this->ResourceSpec.ReadFromJSONFile(val);
-    if (result != cmCTestResourceSpec::ReadFileResult::READ_OK) {
-      cmCTestLog(this->CTest, ERROR_MESSAGE,
-                 "Could not read/parse resource spec file "
-                   << val << ": "
-                   << cmCTestResourceSpec::ResultToString(result)
-                   << std::endl);
-      return false;
-    }
   }
+  this->SetRerunFailed(cmIsOn(this->GetOption("RerunFailed")));
 
   return true;
 }
@@ -1261,7 +1252,7 @@ bool cmCTestTestHandler::GetValue(const char* tag, std::string& value,
   return ret;
 }
 
-void cmCTestTestHandler::ProcessDirectory(std::vector<std::string>& passed,
+bool cmCTestTestHandler::ProcessDirectory(std::vector<std::string>& passed,
                                           std::vector<std::string>& failed)
 {
   this->ComputeTestList();
@@ -1285,7 +1276,17 @@ void cmCTestTestHandler::ProcessDirectory(std::vector<std::string>& passed,
   } else {
     parallel->SetTestLoad(this->CTest->GetTestLoad());
   }
-  if (this->UseResourceSpec) {
+  if (!this->ResourceSpecFile.empty()) {
+    this->UseResourceSpec = true;
+    auto result = this->ResourceSpec.ReadFromJSONFile(this->ResourceSpecFile);
+    if (result != cmCTestResourceSpec::ReadFileResult::READ_OK) {
+      cmCTestLog(this->CTest, ERROR_MESSAGE,
+                 "Could not read/parse resource spec file "
+                   << this->ResourceSpecFile << ": "
+                   << cmCTestResourceSpec::ResultToString(result)
+                   << std::endl);
+      return false;
+    }
     parallel->InitResourceAllocator(this->ResourceSpec);
   }
 
@@ -1345,6 +1346,8 @@ void cmCTestTestHandler::ProcessDirectory(std::vector<std::string>& passed,
   this->ElapsedTestingTime =
     std::chrono::steady_clock::now() - elapsed_time_start;
   *this->LogFile << "End testing: " << this->CTest->CurrentTime() << std::endl;
+
+  return true;
 }
 
 void cmCTestTestHandler::GenerateTestCommand(
@@ -1743,6 +1746,10 @@ void cmCTestTestHandler::GetListOfTests()
   if (cmSystemTools::GetErrorOccuredFlag()) {
     return;
   }
+  const char* specFile = mf.GetDefinition("CTEST_RESOURCE_SPEC_FILE");
+  if (this->ResourceSpecFile.empty() && specFile) {
+    this->ResourceSpecFile = specFile;
+  }
   cmCTestOptionalLog(this->CTest, HANDLER_VERBOSE_OUTPUT,
                      "Done constructing a list of tests" << std::endl,
                      this->Quiet);

Source/CTest/cmCTestTestHandler.h

@@ -278,7 +278,7 @@ private:
   /**
    * Run the tests for a directory and any subdirectories
    */
-  void ProcessDirectory(std::vector<std::string>& passed,
+  bool ProcessDirectory(std::vector<std::string>& passed,
                         std::vector<std::string>& failed);
 
   /**

Source/cmLocalGenerator.cxx

@@ -301,6 +301,12 @@ void cmLocalGenerator::GenerateTestFiles()
           "# testing this directory and lists subdirectories to "
           "be tested as well.\n";
 
+  std::string resourceSpecFile =
+    this->Makefile->GetSafeDefinition("CTEST_RESOURCE_SPEC_FILE");
+  if (!resourceSpecFile.empty()) {
+    fout << "set(CTEST_RESOURCE_SPEC_FILE \"" << resourceSpecFile << "\")\n";
+  }
+
   cmProp testIncludeFile = this->Makefile->GetProperty("TEST_INCLUDE_FILE");
   if (testIncludeFile) {
     fout << "include(\"" << *testIncludeFile << "\")\n";

Tests/RunCMake/CTestResourceAllocation/RunCMakeTest.cmake

@@ -140,9 +140,13 @@ run_ctresalloc_verify(ctresalloc-verify-noend "test1")
 # Now test the resource allocation feature of CTest
 ###############################################################################
 
-function(run_ctest_resource name parallel random)
-  run_ctest("${name}-ctest-s-res" "-DCTEST_RESOURCE_ALLOC_ENABLED=1" "-DCTRESALLOC_COMMAND=${CTRESALLOC_COMMAND}" "-DCTEST_PARALLEL=${parallel}" "-DCTEST_RANDOM=${random}")
-  run_ctest("${name}-ctest-s-nores" "-DCTEST_RESOURCE_ALLOC_ENABLED=0" "-DCTRESALLOC_COMMAND=${CTRESALLOC_COMMAND}" "-DCTEST_PARALLEL=${parallel}" "-DCTEST_RANDOM=${random}")
+function(run_ctest_resource name parallel random extra)
+  run_ctest("${name}-ctest-s-res" "-DCTEST_RESOURCE_ALLOC_ENABLED=1" "-DCTEST_RESOURCE_SPEC_SOURCE=ARG" "-DCTRESALLOC_COMMAND=${CTRESALLOC_COMMAND}" "-DCTEST_PARALLEL=${parallel}" "-DCTEST_RANDOM=${random}")
+  run_ctest("${name}-ctest-s-nores" "-DCTEST_RESOURCE_ALLOC_ENABLED=0" "-DCTEST_RESOURCE_SPEC_SOURCE=NONE" "-DCTRESALLOC_COMMAND=${CTRESALLOC_COMMAND}" "-DCTEST_PARALLEL=${parallel}" "-DCTEST_RANDOM=${random}")
+  if(extra)
+    run_ctest("${name}-ctest-s-res-variable" "-DCTEST_RESOURCE_ALLOC_ENABLED=1" "-DCTEST_RESOURCE_SPEC_SOURCE=VARIABLE" "-DCTRESALLOC_COMMAND=${CTRESALLOC_COMMAND}" "-DCTEST_PARALLEL=${parallel}" "-DCTEST_RANDOM=${random}")
+    run_ctest("${name}-ctest-s-res-cache" "-DCTEST_RESOURCE_ALLOC_ENABLED=1" "-DCTEST_RESOURCE_SPEC_SOURCE=CACHE" "-DCTRESALLOC_COMMAND=${CTRESALLOC_COMMAND}" "-DCTEST_PARALLEL=${parallel}" "-DCTEST_RANDOM=${random}")
+  endif()
 endfunction()
 
 function(verify_ctest_resources)
@@ -155,15 +159,15 @@ function(verify_ctest_resources)
   endif()
 endfunction()
 
-run_ctest_resource(lotsoftests 10 1)
-run_ctest_resource(checkfree1 2 0)
-run_ctest_resource(checkfree2 1 0)
-run_ctest_resource(notenough1 1 0)
-run_ctest_resource(notenough2 1 0)
-run_ctest_resource(notenough3 1 0)
-run_ctest_resource(combine 1 0)
-run_ctest_resource(ensure_parallel 2 0)
+run_ctest_resource(lotsoftests 10 1 0)
+run_ctest_resource(checkfree1 2 0 1)
+run_ctest_resource(checkfree2 1 0 0)
+run_ctest_resource(notenough1 1 0 1)
+run_ctest_resource(notenough2 1 0 0)
+run_ctest_resource(notenough3 1 0 0)
+run_ctest_resource(combine 1 0 0)
+run_ctest_resource(ensure_parallel 2 0 0)
 
 set(ENV{CTEST_RESOURCE_GROUP_COUNT} 2)
-run_ctest_resource(process_count 1 0)
+run_ctest_resource(process_count 1 0 0)
 unset(ENV{CTEST_RESOURCE_GROUP_COUNT})

Tests/RunCMake/CTestResourceAllocation/checkfree1-ctest-s-res-cache-check.cmake

@@ -0,0 +1 @@
+verify_ctest_resources()

Tests/RunCMake/CTestResourceAllocation/checkfree1-ctest-s-res-variable-check.cmake

@@ -0,0 +1 @@
+verify_ctest_resources()

Tests/RunCMake/CTestResourceAllocation/notenough1-ctest-s-res-cache-check.cmake

@@ -0,0 +1,3 @@
+if(EXISTS "${RunCMake_TEST_BINARY_DIR}/ctresalloc.log")
+  set(RunCMake_TEST_FAILED "ctresalloc.log should not exist")
+endif()

Tests/RunCMake/CTestResourceAllocation/notenough1-ctest-s-res-cache-result.txt

@@ -0,0 +1 @@
+(-1|255)

Tests/RunCMake/CTestResourceAllocation/notenough1-ctest-s-res-cache-stderr.txt

@@ -0,0 +1,14 @@
+^Insufficient resources for test Test1:
+
+  Test requested resources of type 'fluxcapacitors' in the following amounts:
+    200 slots
+  but only the following units were available:
+    'outatime': 121 slots
+
+Resource spec file:
+
+  [^
+]*/Tests/RunCMake/CTestResourceAllocation/resspec.json
+CMake Error at [^
+]*/Tests/RunCMake/CTestResourceAllocation/notenough1-ctest-s-res-cache/test\.cmake:[0-9]+ \(message\):
+  Tests did not pass$

Tests/RunCMake/CTestResourceAllocation/notenough1-ctest-s-res-variable-check.cmake

@@ -0,0 +1,3 @@
+if(EXISTS "${RunCMake_TEST_BINARY_DIR}/ctresalloc.log")
+  set(RunCMake_TEST_FAILED "ctresalloc.log should not exist")
+endif()

Tests/RunCMake/CTestResourceAllocation/notenough1-ctest-s-res-variable-result.txt

@@ -0,0 +1 @@
+(-1|255)

Tests/RunCMake/CTestResourceAllocation/notenough1-ctest-s-res-variable-stderr.txt

@@ -0,0 +1,14 @@
+^Insufficient resources for test Test1:
+
+  Test requested resources of type 'fluxcapacitors' in the following amounts:
+    200 slots
+  but only the following units were available:
+    'outatime': 121 slots
+
+Resource spec file:
+
+  [^
+]*/Tests/RunCMake/CTestResourceAllocation/resspec.json
+CMake Error at [^
+]*/Tests/RunCMake/CTestResourceAllocation/notenough1-ctest-s-res-variable/test\.cmake:[0-9]+ \(message\):
+  Tests did not pass$

Tests/RunCMake/CTestResourceAllocation/test.cmake.in

@@ -8,15 +8,21 @@ set(CTEST_CMAKE_GENERATOR_TOOLSET "@RunCMake_GENERATOR_TOOLSET@")
 set(CTEST_BUILD_CONFIGURATION "$ENV{CMAKE_CONFIG_TYPE}")
 set(CTEST_NIGHTLY_START_TIME "01:00:00 UTC")
 
-ctest_start(Experimental QUIET)
-ctest_configure(OPTIONS
+set(config_options
   "-DCTEST_RESOURCE_ALLOC_ENABLED=${CTEST_RESOURCE_ALLOC_ENABLED};-DCTRESALLOC_COMMAND=${CTRESALLOC_COMMAND}"
   )
-ctest_build()
 
-if(CTEST_RESOURCE_ALLOC_ENABLED)
+if(CTEST_RESOURCE_SPEC_SOURCE STREQUAL "ARG")
   set(resspec RESOURCE_SPEC_FILE "@RunCMake_SOURCE_DIR@/resspec.json")
+elseif(CTEST_RESOURCE_SPEC_SOURCE STREQUAL "VARIABLE")
+  set(CTEST_RESOURCE_SPEC_FILE "@RunCMake_SOURCE_DIR@/resspec.json")
+elseif(CTEST_RESOURCE_SPEC_SOURCE STREQUAL "CACHE")
+  list(APPEND config_options "-DCTEST_RESOURCE_SPEC_FILE=@RunCMake_SOURCE_DIR@/resspec.json")
 endif()
+
+ctest_start(Experimental QUIET)
+ctest_configure(OPTIONS "${config_options}")
+ctest_build()
 ctest_test(${resspec} RETURN_VALUE retval PARALLEL_LEVEL ${CTEST_PARALLEL} SCHEDULE_RANDOM ${CTEST_RANDOM})
 if(retval)
   message(FATAL_ERROR "Tests did not pass")