CTest: Add ability to dynamically generate resource spec file

Issue: #25106

Help/manual/cmake-properties.7.rst

@@ -503,6 +503,7 @@ Properties on Tests
    /prop_test/FIXTURES_CLEANUP
    /prop_test/FIXTURES_REQUIRED
    /prop_test/FIXTURES_SETUP
+   /prop_test/GENERATED_RESOURCE_SPEC_FILE
    /prop_test/LABELS
    /prop_test/MEASUREMENT
    /prop_test/PASS_REGULAR_EXPRESSION

Help/manual/ctest.1.rst

@@ -1817,6 +1817,25 @@ The following variables are passed to the test process:
   uppercase in the ``CTEST_RESOURCE_GROUP_<num>_<resource-type>`` environment
   variable.
 
+.. _`ctest-resource-dynamically-generated-spec-file`:
+
+Dynamically-Generated Resource Specification File
+-------------------------------------------------
+
+.. versionadded:: 3.28
+
+A project may optionally specify a single test which will be used to
+dynamically generate the resource specification file that CTest will use for
+scheduling tests that use resources. The test that generates the file must
+have the :prop_test:`GENERATED_RESOURCE_SPEC_FILE` property set, and must have
+exactly one fixture in its :prop_test:`FIXTURES_SETUP` property. This fixture
+is considered by CTest to have special meaning: it's the fixture that generates
+the resource spec file. The fixture may have any name. If such a fixture
+exists, all tests that have :prop_test:`RESOURCE_GROUPS` set must have the
+fixture in their :prop_test:`FIXTURES_REQUIRED`, and a resource spec file may
+not be specified with the ``--resource-spec-file`` argument or the
+:variable:`CTEST_RESOURCE_SPEC_FILE` variable.
+
 See Also
 ========
 

Help/prop_test/GENERATED_RESOURCE_SPEC_FILE.rst

@@ -0,0 +1,7 @@
+GENERATED_RESOURCE_SPEC_FILE
+----------------------------
+
+.. versionadded:: 3.28
+
+Path to the :ref:`dynamically-generated resource spec file
+<ctest-resource-dynamically-generated-spec-file>` generated by this test.

Help/release/dev/dynamically-generated-resource-spec-file.rst

@@ -0,0 +1,6 @@
+dynamically-generated-resource-spec-file
+----------------------------------------
+
+* CTest may now take a :ref:`dynamically-generated resource spec file
+  <ctest-resource-dynamically-generated-spec-file>`, which can be specified by the
+  :prop_test:`GENERATED_RESOURCE_SPEC_FILE` test property.

Source/CTest/cmCTestMultiProcessHandler.cxx

@@ -35,6 +35,7 @@
 #include "cmCTestRunTest.h"
 #include "cmCTestTestHandler.h"
 #include "cmDuration.h"
+#include "cmJSONState.h"
 #include "cmListFileCache.h"
 #include "cmRange.h"
 #include "cmStringAlgorithms.h"
@@ -75,6 +76,7 @@ cmCTestMultiProcessHandler::cmCTestMultiProcessHandler()
   this->ProcessorsAvailable = cmAffinity::GetProcessorsAvailable();
   this->HaveAffinity = this->ProcessorsAvailable.size();
   this->HasCycles = false;
+  this->HasInvalidGeneratedResourceSpec = false;
   this->SerialTestRunning = false;
 }
 
@@ -95,7 +97,9 @@ void cmCTestMultiProcessHandler::SetTests(TestMap& tests,
   if (!this->CTest->GetShowOnly()) {
     this->ReadCostData();
     this->HasCycles = !this->CheckCycles();
-    if (this->HasCycles) {
+    this->HasInvalidGeneratedResourceSpec =
+      !this->CheckGeneratedResourceSpec();
+    if (this->HasCycles || this->HasInvalidGeneratedResourceSpec) {
       return;
     }
     this->CreateTestCostList();
@@ -125,7 +129,7 @@ void cmCTestMultiProcessHandler::SetTestLoad(unsigned long load)
 void cmCTestMultiProcessHandler::RunTests()
 {
   this->CheckResume();
-  if (this->HasCycles) {
+  if (this->HasCycles || this->HasInvalidGeneratedResourceSpec) {
     return;
   }
 #ifdef CMAKE_UV_SIGNAL_HACK
@@ -180,7 +184,7 @@ bool cmCTestMultiProcessHandler::StartTestProcess(int test)
   }
   testRun->SetIndex(test);
   testRun->SetTestProperties(this->Properties[test]);
-  if (this->TestHandler->UseResourceSpec) {
+  if (this->UseResourceSpec) {
     testRun->SetUseAllocatedResources(true);
     testRun->SetAllocatedResources(this->AllocatedResources[test]);
   }
@@ -229,7 +233,7 @@ bool cmCTestMultiProcessHandler::StartTestProcess(int test)
       }
       e << "\n";
     }
-    e << "Resource spec file:\n\n  " << this->TestHandler->ResourceSpecFile;
+    e << "Resource spec file:\n\n  " << this->ResourceSpecFile;
     cmCTestRunTest::StartFailure(std::move(testRun), this->Total, e.str(),
                                  "Insufficient resources");
     return false;
@@ -253,7 +257,7 @@ bool cmCTestMultiProcessHandler::StartTestProcess(int test)
 
 bool cmCTestMultiProcessHandler::AllocateResources(int index)
 {
-  if (!this->TestHandler->UseResourceSpec) {
+  if (!this->UseResourceSpec) {
     return true;
   }
 
@@ -322,7 +326,7 @@ bool cmCTestMultiProcessHandler::TryAllocateResources(
 
 void cmCTestMultiProcessHandler::DeallocateResources(int index)
 {
-  if (!this->TestHandler->UseResourceSpec) {
+  if (!this->UseResourceSpec) {
     return;
   }
 
@@ -358,7 +362,7 @@ bool cmCTestMultiProcessHandler::AllResourcesAvailable()
 
 void cmCTestMultiProcessHandler::CheckResourcesAvailable()
 {
-  if (this->TestHandler->UseResourceSpec) {
+  if (this->UseResourceSpec) {
     for (auto test : this->SortedTests) {
       std::map<std::string, std::vector<cmCTestBinPackerAllocation>>
         allocations;
@@ -1445,3 +1449,81 @@ bool cmCTestMultiProcessHandler::CheckCycles()
                      this->Quiet);
   return true;
 }
+
+bool cmCTestMultiProcessHandler::CheckGeneratedResourceSpec()
+{
+  for (auto& test : this->Properties) {
+    if (!test.second->GeneratedResourceSpecFile.empty()) {
+      if (this->ResourceSpecSetupTest) {
+        cmCTestLog(
+          this->CTest, ERROR_MESSAGE,
+          "Only one test may define the GENERATED_RESOURCE_SPEC_FILE property"
+            << std::endl);
+        return false;
+      }
+
+      if (test.second->FixturesSetup.size() != 1) {
+        cmCTestLog(this->CTest, ERROR_MESSAGE,
+                   "Test that defines GENERATED_RESOURCE_SPEC_FILE must have "
+                   "exactly one FIXTURES_SETUP"
+                     << std::endl);
+        return false;
+      }
+
+      if (!cmSystemTools::FileIsFullPath(
+            test.second->GeneratedResourceSpecFile)) {
+        cmCTestLog(this->CTest, ERROR_MESSAGE,
+                   "GENERATED_RESOURCE_SPEC_FILE must be an absolute path"
+                     << std::endl);
+        return false;
+      }
+
+      this->ResourceSpecSetupTest = test.first;
+      this->ResourceSpecSetupFixture = *test.second->FixturesSetup.begin();
+    }
+  }
+
+  if (!this->ResourceSpecSetupFixture.empty()) {
+    for (auto& test : this->Properties) {
+      if (!test.second->ResourceGroups.empty() &&
+          !test.second->FixturesRequired.count(
+            this->ResourceSpecSetupFixture)) {
+        cmCTestLog(this->CTest, ERROR_MESSAGE,
+                   "All tests that have RESOURCE_GROUPS must include the "
+                   "resource spec generator fixture in their FIXTURES_REQUIRED"
+                     << std::endl);
+        return false;
+      }
+    }
+  }
+
+  if (!this->ResourceSpecFile.empty()) {
+    if (this->ResourceSpecSetupTest) {
+      cmCTestLog(this->CTest, ERROR_MESSAGE,
+                 "GENERATED_RESOURCE_SPEC_FILE test property cannot be used "
+                 "in conjunction with ResourceSpecFile option"
+                   << std::endl);
+      return false;
+    }
+    std::string error;
+    if (!this->InitResourceAllocator(error)) {
+      cmCTestLog(this->CTest, ERROR_MESSAGE, error << std::endl);
+      return false;
+    }
+  }
+
+  return true;
+}
+
+bool cmCTestMultiProcessHandler::InitResourceAllocator(std::string& error)
+{
+  if (!this->ResourceSpec.ReadFromJSONFile(this->ResourceSpecFile)) {
+    error = cmStrCat("Could not read/parse resource spec file ",
+                     this->ResourceSpecFile, ": ",
+                     this->ResourceSpec.parseState.GetErrorMessage());
+    return false;
+  }
+  this->UseResourceSpec = true;
+  this->ResourceAllocator.InitializeFromResourceSpec(this->ResourceSpec);
+  return true;
+}

Source/CTest/cmCTestMultiProcessHandler.h

@@ -11,15 +11,17 @@
 #include <string>
 #include <vector>
 
+#include <cm/optional>
+
 #include <cm3p/uv.h>
 
 #include "cmCTest.h"
 #include "cmCTestResourceAllocator.h"
+#include "cmCTestResourceSpec.h"
 #include "cmCTestTestHandler.h"
 #include "cmUVHandlePtr.h"
 
 struct cmCTestBinPackerAllocation;
-class cmCTestResourceSpec;
 class cmCTestRunTest;
 
 /** \class cmCTestMultiProcessHandler
@@ -90,13 +92,13 @@ public:
     this->RepeatCount = count;
   }
 
-  void SetQuiet(bool b) { this->Quiet = b; }
-
-  void InitResourceAllocator(const cmCTestResourceSpec& spec)
+  void SetResourceSpecFile(const std::string& resourceSpecFile)
   {
-    this->ResourceAllocator.InitializeFromResourceSpec(spec);
+    this->ResourceSpecFile = resourceSpecFile;
   }
 
+  void SetQuiet(bool b) { this->Quiet = b; }
+
   void CheckResourcesAvailable();
 
 protected:
@@ -158,6 +160,15 @@ protected:
     std::map<std::string, ResourceAllocationError>* errors = nullptr);
   void DeallocateResources(int index);
   bool AllResourcesAvailable();
+  bool InitResourceAllocator(std::string& error);
+  bool CheckGeneratedResourceSpec();
+
+  bool UseResourceSpec = false;
+  cmCTestResourceSpec ResourceSpec;
+  std::string ResourceSpecFile;
+  std::string ResourceSpecSetupFixture;
+  cm::optional<std::size_t> ResourceSpecSetupTest;
+  bool HasInvalidGeneratedResourceSpec;
 
   // map from test number to set of depend tests
   TestMap Tests;

Source/CTest/cmCTestRunTest.cxx

@@ -152,6 +152,18 @@ cmCTestRunTest::EndTestResult cmCTestRunTest::EndTest(size_t completed,
       }
     }
   }
+  std::string resourceSpecParseError;
+  if (!this->TestProperties->GeneratedResourceSpecFile.empty()) {
+    this->MultiTestHandler.ResourceSpecFile =
+      this->TestProperties->GeneratedResourceSpecFile;
+    if (!this->MultiTestHandler.InitResourceAllocator(
+          resourceSpecParseError)) {
+      reason = "Invalid resource spec file";
+      forceFail = true;
+    } else {
+      this->MultiTestHandler.CheckResourcesAvailable();
+    }
+  }
   std::ostringstream outputStream;
   if (res == cmProcess::State::Exited) {
     bool success = !forceFail &&
@@ -260,6 +272,16 @@ cmCTestRunTest::EndTestResult cmCTestRunTest::EndTest(size_t completed,
     cmCTestLog(this->CTest, HANDLER_OUTPUT, this->ProcessOutput << std::endl);
   }
 
+  if (!resourceSpecParseError.empty()) {
+    cmCTestLog(this->CTest, ERROR_MESSAGE,
+               resourceSpecParseError << std::endl);
+  } else if (!this->TestProperties->GeneratedResourceSpecFile.empty()) {
+    cmCTestLog(this->CTest, HANDLER_VERBOSE_OUTPUT,
+               "Using generated resource spec file "
+                 << this->TestProperties->GeneratedResourceSpecFile
+                 << std::endl);
+  }
+
   if (this->TestHandler->LogFile) {
     *this->TestHandler->LogFile << "Test time = " << buf << std::endl;
   }

Source/CTest/cmCTestTestHandler.cxx

@@ -42,7 +42,6 @@
 #include "cmExecutionStatus.h"
 #include "cmGeneratedFileStream.h"
 #include "cmGlobalGenerator.h"
-#include "cmJSONState.h"
 #include "cmList.h"
 #include "cmMakefile.h"
 #include "cmState.h"
@@ -284,7 +283,6 @@ cmCTestTestHandler::cmCTestTestHandler()
   this->UseIncludeRegExpFlag = false;
   this->UseExcludeRegExpFlag = false;
   this->UseExcludeRegExpFirst = false;
-  this->UseResourceSpec = false;
 
   this->CustomMaximumPassedTestOutputSize = 1 * 1024;
   this->CustomMaximumFailedTestOutputSize = 300 * 1024;
@@ -891,8 +889,7 @@ bool cmCTestTestHandler::ComputeTestList()
   }
 
   if (this->RerunFailed) {
-    this->ComputeTestListForRerunFailed();
-    return true;
+    return this->ComputeTestListForRerunFailed();
   }
 
   cmCTestTestHandler::ListOfTests::size_type tmsize = this->TestList.size();
@@ -951,7 +948,7 @@ bool cmCTestTestHandler::ComputeTestList()
   return true;
 }
 
-void cmCTestTestHandler::ComputeTestListForRerunFailed()
+bool cmCTestTestHandler::ComputeTestListForRerunFailed()
 {
   this->ExpandTestsToRunInformationForRerunFailed();
 
@@ -978,6 +975,8 @@ void cmCTestTestHandler::ComputeTestListForRerunFailed()
   this->TestList = finalList;
 
   this->UpdateMaxTestNameWidth();
+
+  return true;
 }
 
 void cmCTestTestHandler::UpdateForFixtures(ListOfTests& tests) const
@@ -1351,18 +1350,6 @@ bool cmCTestTestHandler::ProcessDirectory(std::vector<std::string>& passed,
   } else {
     parallel->SetTestLoad(this->CTest->GetTestLoad());
   }
-  if (!this->ResourceSpecFile.empty()) {
-    this->UseResourceSpec = true;
-    if (!this->ResourceSpec.ReadFromJSONFile(this->ResourceSpecFile)) {
-      cmCTestLog(this->CTest, ERROR_MESSAGE,
-                 "Could not read/parse resource spec file "
-                   << this->ResourceSpecFile << ": "
-                   << this->ResourceSpec.parseState.GetErrorMessage()
-                   << std::endl);
-      return false;
-    }
-    parallel->InitResourceAllocator(this->ResourceSpec);
-  }
 
   *this->LogFile
     << "Start testing: " << this->CTest->CurrentTime() << std::endl
@@ -1397,6 +1384,7 @@ bool cmCTestTestHandler::ProcessDirectory(std::vector<std::string>& passed,
     tests[p.Index] = depends;
     properties[p.Index] = &p;
   }
+  parallel->SetResourceSpecFile(this->ResourceSpecFile);
   parallel->SetTests(tests, properties);
   parallel->SetPassFailVectors(&passed, &failed);
   this->TestResults.clear();
@@ -2336,6 +2324,8 @@ bool cmCTestTestHandler::SetTestsProperties(
             if (!ParseResourceGroupsProperty(val, rt.ResourceGroups)) {
               return false;
             }
+          } else if (key == "GENERATED_RESOURCE_SPEC_FILE"_s) {
+            rt.GeneratedResourceSpecFile = val;
           } else if (key == "SKIP_RETURN_CODE"_s) {
             rt.SkipReturnCode = atoi(val.c_str());
             if (rt.SkipReturnCode < 0 || rt.SkipReturnCode > 255) {

Source/CTest/cmCTestTestHandler.h

@@ -21,7 +21,6 @@
 
 #include "cmCTest.h"
 #include "cmCTestGenericHandler.h"
-#include "cmCTestResourceSpec.h"
 #include "cmCTestTypes.h" // IWYU pragma: keep
 #include "cmDuration.h"
 #include "cmListFileCache.h"
@@ -172,6 +171,7 @@ public:
     std::set<std::string> FixturesRequired;
     std::set<std::string> RequireSuccessDepends;
     std::vector<std::vector<cmCTestTestResourceRequirement>> ResourceGroups;
+    std::string GeneratedResourceSpecFile;
     // Private test generator properties used to track backtraces
     cmListFileBacktrace Backtrace;
   };
@@ -319,7 +319,7 @@ private:
 
   // compute the lists of tests that will actually run
   // based on LastTestFailed.log
-  void ComputeTestListForRerunFailed();
+  bool ComputeTestListForRerunFailed();
 
   // add required setup/cleanup tests not already in the
   // list of tests to be run and update dependencies between
@@ -360,8 +360,6 @@ private:
   cmsys::RegularExpression IncludeTestsRegularExpression;
   cmsys::RegularExpression ExcludeTestsRegularExpression;
 
-  bool UseResourceSpec;
-  cmCTestResourceSpec ResourceSpec;
   std::string ResourceSpecFile;
 
   void RecordCustomTestMeasurements(cmXMLWriter& xml, std::string content);

Tests/RunCMake/CTestResourceAllocation/CMakeLists.txt.in

@@ -4,6 +4,8 @@ if(CASE_NAME MATCHES "^(.*)-ctest-s")
   set(projname "${CMAKE_MATCH_1}")
   project(${projname} NONE)
   include(CTest)
-  include("@RunCMake_SOURCE_DIR@/ResourceCommon.cmake")
+  if(NOT CASE_NAME MATCHES "^dynamic-resource-")
+    include("@RunCMake_SOURCE_DIR@/resource-common.cmake")
+  endif()
   include("@RunCMake_SOURCE_DIR@/${projname}.cmake")
 endif()

Tests/RunCMake/CTestResourceAllocation/RunCMakeTest.cmake

@@ -179,3 +179,19 @@ run_ctest_resource(ensure_parallel 2 0 0)
 set(ENV{CTEST_RESOURCE_GROUP_COUNT} 2)
 run_ctest_resource(process_count 1 0 0)
 unset(ENV{CTEST_RESOURCE_GROUP_COUNT})
+
+function(run_ctest_resource_dynamic name)
+  run_ctest("${name}-ctest-s" ${ARGN})
+endfunction()
+
+run_ctest_resource_dynamic(dynamic-resource -VV)
+run_ctest_resource_dynamic(dynamic-resource-notenough)
+run_ctest_resource_dynamic(dynamic-resource-nofile)
+run_ctest_resource_dynamic(dynamic-resource-multiple-generators)
+run_ctest_resource_dynamic(dynamic-resource-no-setup-fixture)
+run_ctest_resource_dynamic(dynamic-resource-multiple-setup-fixtures)
+run_ctest_resource_dynamic(dynamic-resource-no-required-fixture)
+run_ctest_resource_dynamic(dynamic-resource-conflicting-spec -DCTEST_RESOURCE_SPEC_SOURCE=CACHE)
+run_ctest_resource_dynamic(dynamic-resource-circular)
+run_ctest_resource_dynamic(dynamic-resource-circular-no-required-fixtures)
+run_ctest_resource_dynamic(dynamic-resource-relative-path)

Tests/RunCMake/CTestResourceAllocation/dynamic-resource-circular-ctest-s-result.txt

@@ -0,0 +1 @@
+(-1|255)

Tests/RunCMake/CTestResourceAllocation/dynamic-resource-circular-ctest-s-stderr.txt

@@ -0,0 +1,3 @@
+^Error: a cycle exists in the test dependency graph for the test "GenerateSpecFile"\.
+Please fix the cycle and run ctest again.
+No tests were found!!!$

Tests/RunCMake/CTestResourceAllocation/dynamic-resource-circular-no-required-fixtures-ctest-s-result.txt

@@ -0,0 +1 @@
+(-1|255)

Tests/RunCMake/CTestResourceAllocation/dynamic-resource-circular-no-required-fixtures-ctest-s-stderr.txt

@@ -0,0 +1,2 @@
+^All tests that have RESOURCE_GROUPS must include the resource spec generator fixture in their FIXTURES_REQUIRED
+No tests were found!!!$

Tests/RunCMake/CTestResourceAllocation/dynamic-resource-circular-no-required-fixtures.cmake

@@ -0,0 +1,11 @@
+add_test(NAME RealTest COMMAND "${CMAKE_COMMAND}" -E true)
+add_test(NAME GenerateSpecFile COMMAND "${CMAKE_COMMAND}" -E copy "${CTEST_DYNAMIC_RESOURCE_SPEC_FILE}" "${CMAKE_BINARY_DIR}")
+set_tests_properties(GenerateSpecFile PROPERTIES
+  GENERATED_RESOURCE_SPEC_FILE "${CMAKE_BINARY_DIR}/dynamic-resspec.json"
+  FIXTURES_SETUP "ResourceSpec"
+  RESOURCE_GROUPS "widgets:1"
+  )
+set_tests_properties(RealTest PROPERTIES
+  FIXTURES_REQUIRED "ResourceSpec"
+  RESOURCE_GROUPS "widgets:1"
+  )

Tests/RunCMake/CTestResourceAllocation/dynamic-resource-circular.cmake

@@ -0,0 +1,12 @@
+add_test(NAME RealTest COMMAND "${CMAKE_COMMAND}" -E true)
+add_test(NAME GenerateSpecFile COMMAND "${CMAKE_COMMAND}" -E copy "${CTEST_DYNAMIC_RESOURCE_SPEC_FILE}" "${CMAKE_BINARY_DIR}")
+set_tests_properties(GenerateSpecFile PROPERTIES
+  GENERATED_RESOURCE_SPEC_FILE "${CMAKE_BINARY_DIR}/dynamic-resspec.json"
+  FIXTURES_SETUP "ResourceSpec"
+  FIXTURES_REQUIRED "ResourceSpec"
+  RESOURCE_GROUPS "widgets:1"
+  )
+set_tests_properties(RealTest PROPERTIES
+  FIXTURES_REQUIRED "ResourceSpec"
+  RESOURCE_GROUPS "widgets:1"
+  )

Tests/RunCMake/CTestResourceAllocation/dynamic-resource-conflicting-spec-ctest-s-result.txt

@@ -0,0 +1 @@
+(-1|255)

Tests/RunCMake/CTestResourceAllocation/dynamic-resource-conflicting-spec-ctest-s-stderr.txt

@@ -0,0 +1,2 @@
+^GENERATED_RESOURCE_SPEC_FILE test property cannot be used in conjunction with ResourceSpecFile option
+No tests were found!!!$

Tests/RunCMake/CTestResourceAllocation/dynamic-resource-conflicting-spec.cmake

@@ -0,0 +1,10 @@
+add_test(NAME RealTest COMMAND "${CMAKE_COMMAND}" -E true)
+add_test(NAME GenerateSpecFile COMMAND "${CMAKE_COMMAND}" -E copy "${CTEST_DYNAMIC_RESOURCE_SPEC_FILE}" "${CMAKE_BINARY_DIR}")
+set_tests_properties(GenerateSpecFile PROPERTIES
+  GENERATED_RESOURCE_SPEC_FILE "${CMAKE_BINARY_DIR}/dynamic-resspec.json"
+  FIXTURES_SETUP "ResourceSpec"
+  )
+set_tests_properties(RealTest PROPERTIES
+  FIXTURES_REQUIRED "ResourceSpec"
+  RESOURCE_GROUPS "widgets:1"
+  )

Tests/RunCMake/CTestResourceAllocation/dynamic-resource-ctest-s-stdout.txt

@@ -0,0 +1,24 @@
+test 2
+    Start 2: GenerateSpecFile
+
+2: Test command: "?[^
+]*[\\/]bin([\\/][^\\/
+]+)?[\\/]cmake(\.exe)?"? "-E" "copy" "[^
+]*/Tests/RunCMake/CTestResourceAllocation/dynamic-resspec\.json" "[^
+]*/Tests/RunCMake/CTestResourceAllocation/dynamic-resource-ctest-s-build"
+2: Working Directory: [^
+]*/Tests/RunCMake/CTestResourceAllocation/dynamic-resource-ctest-s-build
+2: Test timeout computed to be: 600
+1/2 Test #2: GenerateSpecFile .................   Passed +[0-9]+\.[0-9]+ sec
+Using generated resource spec file [^
+]*/Tests/RunCMake/CTestResourceAllocation/dynamic-resource-ctest-s-build/dynamic-resspec\.json
+test 1
+    Start 1: RealTest
+
+1: Test command: "?[^
+]*[\\/]bin([\\/][^\\/
+]+)?[\\/]cmake(\.exe)?"? "-E" "true"
+1: Working Directory: [^
+]*/Tests/RunCMake/CTestResourceAllocation/dynamic-resource-ctest-s-build
+1: Test timeout computed to be: 600
+2/2 Test #1: RealTest .........................   Passed +[0-9]+\.[0-9]+ sec

Tests/RunCMake/CTestResourceAllocation/dynamic-resource-multiple-generators-ctest-s-result.txt

@@ -0,0 +1 @@
+(-1|255)

Tests/RunCMake/CTestResourceAllocation/dynamic-resource-multiple-generators-ctest-s-stderr.txt

@@ -0,0 +1,2 @@
+^Only one test may define the GENERATED_RESOURCE_SPEC_FILE property
+No tests were found!!!$

Tests/RunCMake/CTestResourceAllocation/dynamic-resource-multiple-generators.cmake

@@ -0,0 +1,11 @@
+add_test(NAME RealTest COMMAND "${CMAKE_COMMAND}" -E true)
+add_test(NAME GenerateSpecFile1 COMMAND "${CMAKE_COMMAND}" -E true)
+add_test(NAME GenerateSpecFile2 COMMAND "${CMAKE_COMMAND}" -E true)
+set_tests_properties(GenerateSpecFile1 GenerateSpecFile2 PROPERTIES
+  GENERATED_RESOURCE_SPEC_FILE "${CMAKE_BINARY_DIR}/dynamic-resspec.json"
+  FIXTURES_SETUP "ResourceSpec"
+  )
+set_tests_properties(RealTest PROPERTIES
+  FIXTURES_REQUIRED "ResourceSpec"
+  RESOURCE_GROUPS "widgets:1"
+  )

Tests/RunCMake/CTestResourceAllocation/dynamic-resource-multiple-setup-fixtures-ctest-s-result.txt

@@ -0,0 +1 @@
+(-1|255)

Tests/RunCMake/CTestResourceAllocation/dynamic-resource-multiple-setup-fixtures-ctest-s-stderr.txt

@@ -0,0 +1,2 @@
+^Test that defines GENERATED_RESOURCE_SPEC_FILE must have exactly one FIXTURES_SETUP
+No tests were found!!!$

Tests/RunCMake/CTestResourceAllocation/dynamic-resource-multiple-setup-fixtures.cmake

@@ -0,0 +1,10 @@
+add_test(NAME RealTest COMMAND "${CMAKE_COMMAND}" -E true)
+add_test(NAME GenerateSpecFile COMMAND "${CMAKE_COMMAND}" -E copy "${CTEST_DYNAMIC_RESOURCE_SPEC_FILE}" "${CMAKE_BINARY_DIR}")
+set_tests_properties(GenerateSpecFile PROPERTIES
+  GENERATED_RESOURCE_SPEC_FILE "${CMAKE_BINARY_DIR}/dynamic-resspec.json"
+  FIXTURES_SETUP "ResourceSpec;InvalidResourceSpec"
+  )
+set_tests_properties(RealTest PROPERTIES
+  FIXTURES_REQUIRED "ResourceSpec"
+  RESOURCE_GROUPS "widgets:1"
+  )

Tests/RunCMake/CTestResourceAllocation/dynamic-resource-no-required-fixture-ctest-s-result.txt

@@ -0,0 +1 @@
+(-1|255)

Tests/RunCMake/CTestResourceAllocation/dynamic-resource-no-required-fixture-ctest-s-stderr.txt

@@ -0,0 +1,2 @@
+^All tests that have RESOURCE_GROUPS must include the resource spec generator fixture in their FIXTURES_REQUIRED
+No tests were found!!!$

Tests/RunCMake/CTestResourceAllocation/dynamic-resource-no-required-fixture.cmake

@@ -0,0 +1,9 @@
+add_test(NAME RealTest COMMAND "${CMAKE_COMMAND}" -E true)
+add_test(NAME GenerateSpecFile COMMAND "${CMAKE_COMMAND}" -E copy "${CTEST_DYNAMIC_RESOURCE_SPEC_FILE}" "${CMAKE_BINARY_DIR}")
+set_tests_properties(GenerateSpecFile PROPERTIES
+  GENERATED_RESOURCE_SPEC_FILE "${CMAKE_BINARY_DIR}/dynamic-resspec.json"
+  FIXTURES_SETUP "ResourceSpec"
+  )
+set_tests_properties(RealTest PROPERTIES
+  RESOURCE_GROUPS "widgets:1"
+  )

Tests/RunCMake/CTestResourceAllocation/dynamic-resource-no-setup-fixture-ctest-s-result.txt

@@ -0,0 +1 @@
+(-1|255)

Tests/RunCMake/CTestResourceAllocation/dynamic-resource-no-setup-fixture-ctest-s-stderr.txt

@@ -0,0 +1,2 @@
+^Test that defines GENERATED_RESOURCE_SPEC_FILE must have exactly one FIXTURES_SETUP
+No tests were found!!!$

Tests/RunCMake/CTestResourceAllocation/dynamic-resource-no-setup-fixture.cmake

@@ -0,0 +1,9 @@
+add_test(NAME RealTest COMMAND "${CMAKE_COMMAND}" -E true)
+add_test(NAME GenerateSpecFile COMMAND "${CMAKE_COMMAND}" -E copy "${CTEST_DYNAMIC_RESOURCE_SPEC_FILE}" "${CMAKE_BINARY_DIR}")
+set_tests_properties(GenerateSpecFile PROPERTIES
+  GENERATED_RESOURCE_SPEC_FILE "${CMAKE_BINARY_DIR}/dynamic-resspec.json"
+  )
+set_tests_properties(RealTest PROPERTIES
+  FIXTURES_REQUIRED "ResourceSpec"
+  RESOURCE_GROUPS "widgets:1"
+  )

Tests/RunCMake/CTestResourceAllocation/dynamic-resource-nofile-ctest-s-result.txt

@@ -0,0 +1 @@
+(-1|255)

Tests/RunCMake/CTestResourceAllocation/dynamic-resource-nofile-ctest-s-stderr.txt

@@ -0,0 +1,7 @@
+^Could not read/parse resource spec file [^
+]*/Tests/RunCMake/CTestResourceAllocation/dynamic-resource-nofile-ctest-s-build/dynamic-resspec\.json:[ ]
+File not found: [^
+]*/Tests/RunCMake/CTestResourceAllocation/dynamic-resource-nofile-ctest-s-build/dynamic-resspec\.json
+CMake Error at [^
+]*/Tests/RunCMake/CTestResourceAllocation/dynamic-resource-nofile-ctest-s/test\.cmake:[0-9]+ \(message\):
+  Tests did not pass$

Tests/RunCMake/CTestResourceAllocation/dynamic-resource-nofile-ctest-s-stdout.txt

@@ -0,0 +1,5 @@
+    Start 2: GenerateSpecFile
+1/2 Test #2: GenerateSpecFile .................\*\*\*Failed  Invalid resource spec file +[0-9]+\.[0-9]+ sec
+    Start 1: RealTest
+Failed test dependencies: GenerateSpecFile
+2/2 Test #1: RealTest .........................\*\*\*Not Run +[0-9]+\.[0-9]+ sec

Tests/RunCMake/CTestResourceAllocation/dynamic-resource-nofile.cmake

@@ -0,0 +1,10 @@
+add_test(NAME RealTest COMMAND "${CMAKE_COMMAND}" -E true)
+add_test(NAME GenerateSpecFile COMMAND "${CMAKE_COMMAND}" -E true)
+set_tests_properties(GenerateSpecFile PROPERTIES
+  GENERATED_RESOURCE_SPEC_FILE "${CMAKE_BINARY_DIR}/dynamic-resspec.json"
+  FIXTURES_SETUP "ResourceSpec"
+  )
+set_tests_properties(RealTest PROPERTIES
+  FIXTURES_REQUIRED "ResourceSpec"
+  RESOURCE_GROUPS "widgets:1"
+  )

Tests/RunCMake/CTestResourceAllocation/dynamic-resource-notenough-ctest-s-result.txt

@@ -0,0 +1 @@
+(-1|255)

Tests/RunCMake/CTestResourceAllocation/dynamic-resource-notenough-ctest-s-stderr.txt

@@ -0,0 +1,14 @@
+^Insufficient resources for test RealTest:
+
+  Test requested resources of type 'widgets' in the following amounts:
+    2 slots
+  but only the following units were available:
+    '0': 1 slot
+
+Resource spec file:
+
+  [^
+]*/Tests/RunCMake/CTestResourceAllocation/dynamic-resource-notenough-ctest-s-build/dynamic-resspec\.json
+CMake Error at [^
+]*/Tests/RunCMake/CTestResourceAllocation/dynamic-resource-notenough-ctest-s/test\.cmake:[0-9]+ \(message\):
+  Tests did not pass$

Tests/RunCMake/CTestResourceAllocation/dynamic-resource-notenough-ctest-s-stdout.txt

@@ -0,0 +1,4 @@
+    Start 2: GenerateSpecFile
+1/2 Test #2: GenerateSpecFile .................   Passed +[0-9]+\.[0-9]+ sec
+    Start 1: RealTest
+2/2 Test #1: RealTest .........................\*\*\*Not Run +[0-9]+\.[0-9]+ sec

Tests/RunCMake/CTestResourceAllocation/dynamic-resource-notenough.cmake

@@ -0,0 +1,10 @@
+add_test(NAME RealTest COMMAND "${CMAKE_COMMAND}" -E true)
+add_test(NAME GenerateSpecFile COMMAND "${CMAKE_COMMAND}" -E copy "${CTEST_DYNAMIC_RESOURCE_SPEC_FILE}" "${CMAKE_BINARY_DIR}")
+set_tests_properties(GenerateSpecFile PROPERTIES
+  GENERATED_RESOURCE_SPEC_FILE "${CMAKE_BINARY_DIR}/dynamic-resspec.json"
+  FIXTURES_SETUP "ResourceSpec"
+  )
+set_tests_properties(RealTest PROPERTIES
+  FIXTURES_REQUIRED "ResourceSpec"
+  RESOURCE_GROUPS "widgets:2"
+  )

Tests/RunCMake/CTestResourceAllocation/dynamic-resource-relative-path-ctest-s-result.txt

@@ -0,0 +1 @@
+(-1|255)

Tests/RunCMake/CTestResourceAllocation/dynamic-resource-relative-path-ctest-s-stderr.txt

@@ -0,0 +1,2 @@
+^GENERATED_RESOURCE_SPEC_FILE must be an absolute path
+No tests were found!!!$

Tests/RunCMake/CTestResourceAllocation/dynamic-resource-relative-path.cmake

@@ -0,0 +1,10 @@
+add_test(NAME RealTest COMMAND "${CMAKE_COMMAND}" -E true)
+add_test(NAME GenerateSpecFile COMMAND "${CMAKE_COMMAND}" -E copy "${CTEST_DYNAMIC_RESOURCE_SPEC_FILE}" "${CMAKE_BINARY_DIR}")
+set_tests_properties(GenerateSpecFile PROPERTIES
+  GENERATED_RESOURCE_SPEC_FILE "dynamic-resspec.json"
+  FIXTURES_SETUP "ResourceSpec"
+  )
+set_tests_properties(RealTest PROPERTIES
+  FIXTURES_REQUIRED "ResourceSpec"
+  RESOURCE_GROUPS "widgets:1"
+  )

Tests/RunCMake/CTestResourceAllocation/dynamic-resource.cmake

@@ -0,0 +1,10 @@
+add_test(NAME RealTest COMMAND "${CMAKE_COMMAND}" -E true)
+add_test(NAME GenerateSpecFile COMMAND "${CMAKE_COMMAND}" -E copy "${CTEST_DYNAMIC_RESOURCE_SPEC_FILE}" "${CMAKE_BINARY_DIR}")
+set_tests_properties(GenerateSpecFile PROPERTIES
+  GENERATED_RESOURCE_SPEC_FILE "${CMAKE_BINARY_DIR}/dynamic-resspec.json"
+  FIXTURES_SETUP "ResourceSpec"
+  )
+set_tests_properties(RealTest PROPERTIES
+  FIXTURES_REQUIRED "ResourceSpec"
+  RESOURCE_GROUPS "widgets:1"
+  )

Tests/RunCMake/CTestResourceAllocation/dynamic-resspec.json

@@ -0,0 +1,16 @@
+{
+  "version": {
+    "major": 1,
+    "minor": 0
+  },
+  "local": [
+    {
+      "widgets": [
+        {
+          "id": "0",
+          "slots": 1
+        }
+      ]
+    }
+  ]
+}

Tests/RunCMake/CTestResourceAllocation/test.cmake.in

@@ -8,9 +8,15 @@ set(CTEST_CMAKE_GENERATOR_TOOLSET "@RunCMake_GENERATOR_TOOLSET@")
 set(CTEST_BUILD_CONFIGURATION "$ENV{CMAKE_CONFIG_TYPE}")
 set(CTEST_NIGHTLY_START_TIME "01:00:00 UTC")
 
-set(config_options
-  "-DCTEST_RESOURCE_ALLOC_ENABLED=${CTEST_RESOURCE_ALLOC_ENABLED};-DCTRESALLOC_COMMAND=${CTRESALLOC_COMMAND}"
-  )
+if("@CASE_NAME@" MATCHES "^dynamic-resource-")
+  set(config_options
+    "-DCTEST_DYNAMIC_RESOURCE_SPEC_FILE=@RunCMake_SOURCE_DIR@/dynamic-resspec.json"
+    )
+else()
+  set(config_options
+    "-DCTEST_RESOURCE_ALLOC_ENABLED=${CTEST_RESOURCE_ALLOC_ENABLED};-DCTRESALLOC_COMMAND=${CTRESALLOC_COMMAND}"
+    )
+endif()
 
 if(CTEST_RESOURCE_SPEC_SOURCE STREQUAL "CMDLINE")
   list(APPEND config_options "-DCTEST_RESOURCE_SPEC_FILE=@RunCMake_SOURCE_DIR@/noexist.json")