ctest: Add option to specify the --schedule-random seed

When `--schedule-random` is used in automated CI jobs, failures may
occur due to test order.  We now log the seed.  Provide a way for
developers to re-run the same order by specifying the seed.

Fixes: #26760
Co-authored-by: Brad King <[email protected]>

Help/manual/ctest.1.rst

@@ -445,6 +445,15 @@ Run Tests
  This option will run the tests in a random order.  It is commonly
  used to detect implicit dependencies in a test suite.
 
+.. option:: --schedule-random-seed
+
+ .. versionadded:: 4.1
+
+ Override the random order seed
+
+ This option is used to allow recreating failures owing to
+ random order of execution by ``--schedule-random``.
+
 .. option:: --submit-index
 
  Legacy option for old Dart2 dashboard server feature.

Help/release/dev/ctest-schedule-random-seed.rst

@@ -0,0 +1,7 @@
+ctest-schedule-random-seed
+--------------------------
+
+* :manual:`ctest(1)` gained a
+  :option:`--schedule-random-seed <ctest --schedule-random-seed>`
+  option to specify a numeric random seed to make
+  :option:`ctest --schedule-random` deterministic for reproduction.

Source/CTest/cmCTestTestCommand.h

@@ -43,6 +43,7 @@ protected:
     cm::optional<ArgumentParser::Maybe<std::string>> ParallelLevel;
     std::string Repeat;
     std::string ScheduleRandom;
+    std::string ScheduleRandomSeed;
     std::string StopTime;
     std::string TestLoad;
     std::string ResourceSpecFile;
@@ -70,6 +71,7 @@ protected:
       .Bind("PARALLEL_LEVEL"_s, &TestArguments::ParallelLevel)
       .Bind("REPEAT"_s, &TestArguments::Repeat)
       .Bind("SCHEDULE_RANDOM"_s, &TestArguments::ScheduleRandom)
+      .Bind("SCHEDULE_RANDOM_SEED"_s, &TestArguments::ScheduleRandomSeed)
       .Bind("STOP_TIME"_s, &TestArguments::StopTime)
       .Bind("TEST_LOAD"_s, &TestArguments::TestLoad)
       .Bind("RESOURCE_SPEC_FILE"_s, &TestArguments::ResourceSpecFile)

Source/CTest/cmCTestTestHandler.cxx

@@ -1324,12 +1324,14 @@ bool cmCTestTestHandler::ProcessDirectory(std::vector<std::string>& passed,
 
   bool randomSchedule = this->CTest->GetScheduleType() == "Random";
   if (randomSchedule) {
-    unsigned int seed = static_cast<unsigned>(time(nullptr));
-    srand(seed);
-    *this->LogFile
-      << "Test order random seed: " << seed << std::endl
-      << "----------------------------------------------------------"
-      << std::endl;
+    cm::optional<unsigned int> scheduleRandomSeed =
+      this->CTest->GetRandomSeed();
+    if (!scheduleRandomSeed.has_value()) {
+      scheduleRandomSeed = static_cast<unsigned int>(time(nullptr));
+    }
+    srand(*scheduleRandomSeed);
+    *this->LogFile << "Test order random seed: " << *scheduleRandomSeed
+                   << std::endl;
   }
 
   for (cmCTestTestProperties& p : this->TestList) {

Source/CTest/cmCTestTestHandler.h

@@ -34,6 +34,7 @@ struct cmCTestTestOptions
   bool ScheduleRandom = false;
   bool StopOnFailure = false;
   bool UseUnion = false;
+  cm::optional<unsigned int> ScheduleRandomSeed;
 
   int OutputSizePassed = 1 * 1024;
   int OutputSizeFailed = 300 * 1024;

Source/cmCTest.cxx

@@ -2527,6 +2527,20 @@ int cmCTest::Run(std::vector<std::string> const& args)
                        this->Impl->TestOptions.ScheduleRandom = true;
                        return true;
                      } },
+    CommandArgument{
+      "--schedule-random-seed", CommandArgument::Values::One,
+      [this](std::string const& sz) -> bool {
+        unsigned long seed_value;
+        if (cmStrToULong(sz, &seed_value)) {
+          this->Impl->TestOptions.ScheduleRandomSeed =
+            static_cast<unsigned int>(seed_value);
+        } else {
+          cmCTestLog(this, WARNING,
+                     "Invalid value for '--schedule-random-seed': " << sz
+                                                                    << "\n");
+        }
+        return true;
+      } },
     CommandArgument{ "--rerun-failed", CommandArgument::Values::Zero,
                      [this](std::string const&) -> bool {
                        this->Impl->TestOptions.RerunFailed = true;
@@ -2795,6 +2809,11 @@ void cmCTest::SetStopTime(std::string const& time_str)
   }
 }
 
+cm::optional<unsigned int> cmCTest::GetRandomSeed() const
+{
+  return this->Impl->TestOptions.ScheduleRandomSeed;
+}
+
 std::string cmCTest::GetScheduleType() const
 {
   return this->Impl->ScheduleType;

Source/cmCTest.h

@@ -198,6 +198,8 @@ public:
   std::string GetScheduleType() const;
   void SetScheduleType(std::string const& type);
 
+  cm::optional<unsigned int> GetRandomSeed() const;
+
   /** The max output width */
   int GetMaxTestNameWidth() const;
   void SetMaxTestNameWidth(int w);

Source/ctest.cxx

@@ -149,6 +149,7 @@ cmDocumentationEntry const cmDocumentationOptions[] = {
   { "--extra-submit <file>[;<file>]", "Submit extra files to the dashboard." },
   { "--http-header <header>", "Append HTTP header when submitting" },
   { "--schedule-random", "Use a random order for scheduling tests" },
+  { "--schedule-random-seed", "Override seed for random order of tests" },
   { "--submit-index",
     "Submit individual dashboard tests with specific index" },
   { "--timeout <seconds>", "Set the default test timeout." },

Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake

@@ -656,3 +656,17 @@ set_tests_properties(test1 PROPERTIES TIMEOUT_SIGNAL_GRACE_PERIOD 1000)
     run_cmake_command(TimeoutSignalBad ${CMAKE_CTEST_COMMAND})
   endblock()
 endif()
+
+block()
+  set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/ScheduleRandomSeed)
+  set(RunCMake_TEST_NO_CLEAN 1)
+  file(REMOVE_RECURSE "${RunCMake_TEST_BINARY_DIR}")
+  file(MAKE_DIRECTORY "${RunCMake_TEST_BINARY_DIR}")
+  file(WRITE "${RunCMake_TEST_BINARY_DIR}/CTestTestfile.cmake" "
+foreach(i RANGE 1 5)
+  add_test(test\${i} \"${CMAKE_COMMAND}\" -E true)
+endforeach()
+")
+  run_cmake_command(ScheduleRandomSeed1 ${CMAKE_CTEST_COMMAND} --schedule-random --schedule-random-seed 42)
+  run_cmake_command(ScheduleRandomSeed2 ${CMAKE_CTEST_COMMAND} --schedule-random --schedule-random-seed 42)
+endblock()

Tests/RunCMake/CTestCommandLine/ScheduleRandomSeed1-check.cmake

@@ -0,0 +1,2 @@
+string(REGEX MATCHALL "Start [1-5]" ScheduleRandomSeed1_ORDER "${actual_stdout}")
+set_property(DIRECTORY PROPERTY ScheduleRandomSeed1_ORDER "${ScheduleRandomSeed1_ORDER}")

Tests/RunCMake/CTestCommandLine/ScheduleRandomSeed2-check.cmake

@@ -0,0 +1,10 @@
+string(REGEX MATCHALL "Start [1-5]" ScheduleRandomSeed2_ORDER "${actual_stdout}")
+get_property(ScheduleRandomSeed1_ORDER DIRECTORY PROPERTY ScheduleRandomSeed1_ORDER)
+if(NOT "${ScheduleRandomSeed1_ORDER}" STREQUAL "${ScheduleRandomSeed2_ORDER}")
+  string(CONCAT RunCMake_TEST_FAILED
+    "ScheduleRandomSeed1 order:\n"
+    " ${ScheduleRandomSeed1_ORDER}\n"
+    "does not match ScheduleRandomSeed2 order:\n"
+    " ${ScheduleRandomSeed2_ORDER}\n"
+    )
+endif()