ctest: Add support for running under a make job server on POSIX systems

Share job slots with the job server by acquiring a token before running
each test, and releasing the token when the test finishes.

Help/manual/CTEST_EXAMPLE_MAKEFILE_JOB_SERVER.make

@@ -0,0 +1,2 @@
+test:
+	+ctest -j 8

Help/manual/ctest.1.rst

@@ -1841,6 +1841,31 @@ fixture in their :prop_test:`FIXTURES_REQUIRED`, and a resource spec file may
 not be specified with the ``--resource-spec-file`` argument or the
 :variable:`CTEST_RESOURCE_SPEC_FILE` variable.
 
+.. _`ctest-job-server-integration`:
+
+Job Server Integration
+======================
+
+.. versionadded:: 3.29
+
+On POSIX systems, when running under the context of a `Job Server`_,
+CTest shares its job slots.  This is independent of the :prop_test:`PROCESSORS`
+test property, which still counts against CTest's :option:`-j <ctest -j>`
+parallel level.  CTest acquires exactly one token from the job server before
+running each test, and returns it when the test finishes.
+
+For example, consider the ``Makefile``:
+
+.. literalinclude:: CTEST_EXAMPLE_MAKEFILE_JOB_SERVER.make
+  :language: make
+
+When invoked via ``make -j 2 test``, ``ctest`` connects to the job server,
+acquires a token for each test, and runs at most 2 tests concurrently.
+
+On Windows systems, job server integration is not yet implemented.
+
+.. _`Job Server`: https://www.gnu.org/software/make/manual/html_node/Job-Slots.html
+
 See Also
 ========
 

Help/release/dev/ctest-jobserver-client.rst

@@ -0,0 +1,5 @@
+ctest-jobserver-client
+----------------------
+
+* :manual:`ctest(1)` now supports :ref:`job server integration
+  <ctest-job-server-integration>` on POSIX systems.

Source/CTest/cmCTestMultiProcessHandler.cxx

@@ -40,6 +40,7 @@
 #include "cmRange.h"
 #include "cmStringAlgorithms.h"
 #include "cmSystemTools.h"
+#include "cmUVJobServerClient.h"
 #include "cmWorkingDirectory.h"
 
 namespace cmsys {
@@ -130,10 +131,19 @@ void cmCTestMultiProcessHandler::InitializeLoop()
   this->Loop.init();
   this->StartNextTestsOnIdle_.init(*this->Loop, this);
   this->StartNextTestsOnTimer_.init(*this->Loop, this);
+
+  this->JobServerClient = cmUVJobServerClient::Connect(
+    *this->Loop, /*onToken=*/[this]() { this->JobServerReceivedToken(); },
+    /*onDisconnect=*/nullptr);
+  if (this->JobServerClient) {
+    cmCTestLog(this->CTest, OUTPUT,
+               "Connected to MAKE jobserver" << std::endl);
+  }
 }
 
 void cmCTestMultiProcessHandler::FinalizeLoop()
 {
+  this->JobServerClient.reset();
   this->StartNextTestsOnTimer_.reset();
   this->StartNextTestsOnIdle_.reset();
   this->Loop.reset();
@@ -461,6 +471,26 @@ std::string cmCTestMultiProcessHandler::GetName(int test)
 
 void cmCTestMultiProcessHandler::StartTest(int test)
 {
+  if (this->JobServerClient) {
+    // There is a job server.  Request a token and queue the test to run
+    // when a token is received.  Note that if we do not get a token right
+    // away it's possible that the system load will be higher when the
+    // token is received and we may violate the test-load limit.  However,
+    // this is unlikely because if we do not get a token right away, some
+    // other job that's currently running must finish before we get one.
+    this->JobServerClient->RequestToken();
+    this->JobServerQueuedTests.emplace_back(test);
+  } else {
+    // There is no job server.  Start the test now.
+    this->StartTestProcess(test);
+  }
+}
+
+void cmCTestMultiProcessHandler::JobServerReceivedToken()
+{
+  assert(!this->JobServerQueuedTests.empty());
+  int test = this->JobServerQueuedTests.front();
+  this->JobServerQueuedTests.pop_front();
   this->StartTestProcess(test);
 }
 
@@ -692,6 +722,9 @@ void cmCTestMultiProcessHandler::FinishTestProcess(
 
   runner.reset();
 
+  if (this->JobServerClient) {
+    this->JobServerClient->ReleaseToken();
+  }
   this->StartNextTestsOnIdle();
 }
 

Source/CTest/cmCTestMultiProcessHandler.h

@@ -19,6 +19,7 @@
 #include "cmCTestResourceSpec.h"
 #include "cmCTestTestHandler.h"
 #include "cmUVHandlePtr.h"
+#include "cmUVJobServerClient.h"
 
 struct cmCTestBinPackerAllocation;
 class cmCTestRunTest;
@@ -204,6 +205,15 @@ protected:
   cmCTestResourceAllocator ResourceAllocator;
   std::vector<cmCTestTestHandler::cmCTestTestResult>* TestResults;
   size_t ParallelLevel; // max number of process that can be run at once
+
+  // 'make' jobserver client.  If connected, we acquire a token
+  // for each test before running its process.
+  cm::optional<cmUVJobServerClient> JobServerClient;
+  // List of tests that are queued to run when a token is available.
+  std::list<int> JobServerQueuedTests;
+  // Callback invoked when a token is received.
+  void JobServerReceivedToken();
+
   unsigned long TestLoad;
   unsigned long FakeLoadForTesting;
   cm::uv_loop_ptr Loop;

Tests/RunCMake/Make/CTestJobServer-NoPipe-j2-stdout.txt

@@ -0,0 +1,9 @@
+Test project [^
+]*/Tests/RunCMake/Make/CTestJobServer-build
+    Start [0-9]+: test[0-9]+
+    Start [0-9]+: test[0-9]+
+    Start [0-9]+: test[0-9]+
+    Start [0-9]+: test[0-9]+
+    Start [0-9]+: test[0-9]+
+    Start [0-9]+: test[0-9]+
+1/6 Test #[0-9]+: test[0-9]+ ............................   Passed +[0-9.]+ sec

Tests/RunCMake/Make/CTestJobServer-NoTests-j2-stderr.txt

@@ -0,0 +1 @@
+No tests were found!!!

Tests/RunCMake/Make/CTestJobServer-NoTests-j2-stdout.txt

@@ -0,0 +1,3 @@
+Test project [^
+]*/Tests/RunCMake/Make/CTestJobServer-build
+Connected to MAKE jobserver

Tests/RunCMake/Make/CTestJobServer-Tests-j2-stdout.txt

@@ -0,0 +1,6 @@
+Test project [^
+]*/Tests/RunCMake/Make/CTestJobServer-build
+Connected to MAKE jobserver
+    Start [0-9]+: test[0-9]+
+    Start [0-9]+: test[0-9]+
+1/6 Test #[0-9]+: test[0-9]+ ............................   Passed +[0-9.]+ sec

Tests/RunCMake/Make/CTestJobServer-Tests-j3-stdout.txt

@@ -0,0 +1,7 @@
+Test project [^
+]*/Tests/RunCMake/Make/CTestJobServer-build
+Connected to MAKE jobserver
+    Start [0-9]+: test[0-9]+
+    Start [0-9]+: test[0-9]+
+    Start [0-9]+: test[0-9]+
+1/6 Test #[0-9]+: test[0-9]+ ............................   Passed +[0-9.]+ sec

Tests/RunCMake/Make/CTestJobServer.cmake

@@ -0,0 +1,4 @@
+enable_testing()
+foreach(i RANGE 1 6)
+  add_test(NAME test${i} COMMAND ${CMAKE_COMMAND} -E true)
+endforeach()

Tests/RunCMake/Make/CTestJobServer.make

@@ -0,0 +1,11 @@
+NoPipe:
+	env MAKEFLAGS= $(CMAKE_CTEST_COMMAND) -j6
+.PHONY: NoPipe
+
+NoTests:
+	+$(CMAKE_CTEST_COMMAND) -j6 -R NoTests
+.PHONY: NoTests
+
+Tests:
+	+$(CMAKE_CTEST_COMMAND) -j6
+.PHONY: Tests

Tests/RunCMake/Make/RunCMakeTest.cmake

@@ -79,9 +79,29 @@ function(detect_jobserver_present)
   run_cmake_command(DetectJobServer-present-parallel-build ${CMAKE_COMMAND} --build . -j4)
 endfunction()
 
+function(run_make_rule case rule job_count)
+  run_cmake_command(${case}-${rule}-j${job_count}
+    ${RunCMake_MAKE_PROGRAM} -f "${RunCMake_SOURCE_DIR}/${case}.make" ${rule} -j${job_count}
+    CMAKE_COMMAND="${CMAKE_COMMAND}" CMAKE_CTEST_COMMAND="${CMAKE_CTEST_COMMAND}"
+    )
+endfunction()
+
+function(run_CTestJobServer)
+  set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/CTestJobServer-build)
+  run_cmake(CTestJobServer)
+  set(RunCMake_TEST_NO_CLEAN 1)
+  run_make_rule(CTestJobServer NoPipe 2)
+  run_make_rule(CTestJobServer NoTests 2)
+  run_make_rule(CTestJobServer Tests 2)
+  run_make_rule(CTestJobServer Tests 3)
+endfunction()
+
 # Jobservers are currently only supported by GNU makes, except MSYS2 make
 if(MAKE_IS_GNU AND NOT RunCMake_GENERATOR MATCHES "MSYS Makefiles")
   detect_jobserver_present()
+  if(UNIX)
+    run_CTestJobServer()
+  endif()
 endif()
 
 if(MAKE_IS_GNU)