ExternalProject: Enable Make Job Server with Explicit Build Command

Introduces `BUILD_JOB_SERVER_AWARE` option to `ExternalProject_Add` and
`JOB_SERVER_AWARE` to `ExternalProject_Add_Step`. When using an explicit
`BUILD_COMMAND` or `COMMAND`, the generated commands won't use `$(MAKE)`
thus failing to connect to the outer make's job server. These new
options enable explicit job server integration.

Co-authored-by: Brad King <[email protected]>

Fixes: #16273

Help/release/dev/ExternalProject-build-jobserver.rst

@@ -0,0 +1,10 @@
+ExternalProject-build-jobserver
+-------------------------------
+
+* The :module:`ExternalProject` module now includes the
+  ``BUILD_JOB_SERVER_AWARE`` option for the
+  :command:`ExternalProject_Add` command. This option enables
+  the integration of the GNU Make job server when using an
+  explicit ``BUILD_COMMAND`` with certain :ref:`Makefile Generators`.
+  Additionally, the :command:`ExternalProject_Add_Step` command
+  has been updated to support the new ``JOB_SERVER_AWARE`` option.

Modules/ExternalProject.cmake

@@ -684,6 +684,14 @@ pass ``-v`` to the external project's build step, even if it also uses
   build step's own underlying call to :command:`add_custom_command`, which
   has additional documentation.
 
+``BUILD_JOB_SERVER_AWARE <bool>``
+  .. versionadded:: 3.28
+
+  Specifies that the build step is aware of the GNU Make job server.
+  See the :command:`add_custom_command` documentation of its
+  ``JOB_SERVER_AWARE`` option for details.  This option is relevant
+  only when an explicit ``BUILD_COMMAND`` is specified.
+
 Install Step Options
 """"""""""""""""""""
 
@@ -1021,6 +1029,13 @@ control needed to implement such step-level capabilities.
     When enabled, this option specifies that the custom step should always be
     run (i.e. that it is always considered out of date).
 
+  ``JOB_SERVER_AWARE <bool>``
+    .. versionadded:: 3.28
+
+    Specifies that the custom step is aware of the GNU Make job server.
+    See the :command:`add_custom_command` documentation of its
+    ``JOB_SERVER_AWARE`` option for details.
+
   ``EXCLUDE_FROM_MAIN <bool>``
     When enabled, this option specifies that the external project's main target
     does not depend on the custom step.
@@ -2366,6 +2381,7 @@ function(ExternalProject_Add_Step name step)
     INDEPENDENT
     BYPRODUCTS
     ALWAYS
+    JOB_SERVER_AWARE
     EXCLUDE_FROM_MAIN
     WORKING_DIRECTORY
     LOG
@@ -2545,6 +2561,16 @@ function(ExternalProject_Add_Step name step)
     set(maybe_COMMAND_touch "COMMAND \${CMAKE_COMMAND} -E touch \${stamp_file}")
   endif()
 
+  get_property(job_server_aware
+    TARGET ${name}
+    PROPERTY _EP_${step}_JOB_SERVER_AWARE
+  )
+  if(job_server_aware)
+    set(maybe_JOB_SERVER_AWARE "JOB_SERVER_AWARE 1")
+  else()
+    set(maybe_JOB_SERVER_AWARE "")
+  endif()
+
   # Wrap with log script?
   get_property(log TARGET ${name} PROPERTY _EP_${step}_LOG)
   if(command AND log)
@@ -2571,6 +2597,7 @@ function(ExternalProject_Add_Step name step)
       COMMENT \${comment}
       COMMAND ${__cmdQuoted}
       ${maybe_COMMAND_touch}
+      ${maybe_JOB_SERVER_AWARE}
       DEPENDS \${depends}
       WORKING_DIRECTORY \${work_dir}
       VERBATIM
@@ -3945,6 +3972,17 @@ function(_ep_add_build_command name)
     PROPERTY _EP_BUILD_BYPRODUCTS
   )
 
+  get_property(build_job_server_aware
+    TARGET ${name}
+    PROPERTY _EP_BUILD_JOB_SERVER_AWARE
+  )
+  if(build_job_server_aware)
+    set(maybe_JOB_SERVER_AWARE "JOB_SERVER_AWARE 1")
+  else()
+    set(maybe_JOB_SERVER_AWARE "")
+  endif()
+
+
   set(__cmdQuoted)
   foreach(__item IN LISTS cmd)
     string(APPEND __cmdQuoted " [==[${__item}]==]")
@@ -3958,6 +3996,7 @@ function(_ep_add_build_command name)
       DEPENDEES configure
       DEPENDS \${file_deps}
       ALWAYS \${always}
+      ${maybe_JOB_SERVER_AWARE}
       ${log}
       ${uses_terminal}
     )"
@@ -4252,6 +4291,7 @@ function(ExternalProject_Add name)
     BUILD_IN_SOURCE
     BUILD_ALWAYS
     BUILD_BYPRODUCTS
+    BUILD_JOB_SERVER_AWARE
     #
     # Install step options
     #

Tests/RunCMake/CMakeLists.txt

@@ -851,7 +851,7 @@ endif()
 if(CMake_TEST_RunCMake_ExternalProject_DOWNLOAD_SERVER_TIMEOUT)
   list(APPEND ExternalProject_ARGS -DDOWNLOAD_SERVER_TIMEOUT=${CMake_TEST_RunCMake_ExternalProject_DOWNLOAD_SERVER_TIMEOUT})
 endif()
-add_RunCMake_test(ExternalProject)
+add_RunCMake_test(ExternalProject -DDETECT_JOBSERVER=$<TARGET_FILE:detect_jobserver>)
 add_RunCMake_test(FetchContent)
 add_RunCMake_test(FetchContent_find_package)
 set(CTestCommandLine_ARGS -DPython_EXECUTABLE=${Python_EXECUTABLE})

Tests/RunCMake/ExternalProject/DetectJobServer.cmake

@@ -0,0 +1,16 @@
+include(ExternalProject)
+ExternalProject_Add(Foo
+  SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/Foo
+  BUILD_COMMAND ${DETECT_JOBSERVER} "ep.txt"
+  BUILD_JOB_SERVER_AWARE 1
+  INSTALL_COMMAND ""
+)
+
+# Add a second step to test JOB_SERVER_AWARE
+ExternalProject_Add_Step(Foo
+  second_step
+  COMMAND ${DETECT_JOBSERVER} "ep_second_step.txt"
+  DEPENDEES build
+  ALWAYS 1
+  JOB_SERVER_AWARE 1
+)

Tests/RunCMake/ExternalProject/Foo/CMakeLists.txt

@@ -0,0 +1,4 @@
+cmake_minimum_required(VERSION 3.27)
+project(Foo NONE)
+
+add_custom_target(drive ALL COMMAND ${CMAKE_COMMAND} -E true)

Tests/RunCMake/ExternalProject/GNUMakeJobServerAware-check.cmake

@@ -0,0 +1,16 @@
+set(BUILD_DIR "${RunCMake_BINARY_DIR}/GNUMakeJobServerAware-build")
+
+function(check target regex)
+  file(STRINGS ${BUILD_DIR}/${target} lines
+    REGEX ${regex}
+  )
+
+  list(LENGTH lines len)
+  if(len EQUAL 0)
+    message(FATAL_ERROR "Could not find matching lines '${regex}' in ${BUILD_DIR}/${target}")
+  endif()
+endfunction()
+
+check("/CMakeFiles/Foo.dir/build.make" [[\+cd (/d )?"?.*"? && "?.*"? --build "?.*"?]])
+check("/CMakeFiles/Foo.dir/build.make" [[\+cd (/d )?"?.*"? && "?.*"? -E touch "?.*"?]])
+check("/CMakeFiles/Foo.dir/build.make" [[\+"?.*"? -E true]])

Tests/RunCMake/ExternalProject/GNUMakeJobServerAware.cmake

@@ -0,0 +1,16 @@
+include(ExternalProject)
+ExternalProject_Add(Foo
+  SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/Foo
+  BUILD_COMMAND ${CMAKE_COMMAND} --build <BINARY_DIR>
+  BUILD_JOB_SERVER_AWARE 1
+  INSTALL_COMMAND ""
+)
+
+# Add a second step to test JOB_SERVER_AWARE
+ExternalProject_Add_Step(Foo
+  second_step
+  COMMAND ${CMAKE_COMMAND} -E true
+  DEPENDEES build
+  ALWAYS 1
+  JOB_SERVER_AWARE 1
+)

Tests/RunCMake/ExternalProject/RunCMakeTest.cmake

@@ -144,6 +144,24 @@ function(__ep_test_with_build_with_server testName)
   run_cmake_command(${testName}-build ${CMAKE_COMMAND} --build .)
 endfunction()
 
+if(RunCMake_GENERATOR MATCHES "(MSYS|MinGW|Unix) Makefiles")
+  __ep_test_with_build(GNUMakeJobServerAware)
+endif()
+
+function(__ep_test_jobserver)
+  set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/DetectJobServer-build)
+  set(RunCMake_TEST_NO_CLEAN 1)
+  file(REMOVE_RECURSE "${RunCMake_TEST_BINARY_DIR}")
+  file(MAKE_DIRECTORY "${RunCMake_TEST_BINARY_DIR}")
+  run_cmake_with_options(DetectJobServer -DDETECT_JOBSERVER=${DETECT_JOBSERVER})
+  run_cmake_command(DetectJobServer-clean ${CMAKE_COMMAND} --build . --target clean)
+  run_cmake_command(DetectJobServer-build ${CMAKE_COMMAND} --build . -j4)
+endfunction()
+
+if(RunCMake_GENERATOR MATCHES "(MinGW|Unix) Makefiles")
+  __ep_test_jobserver()
+endif()
+
 __ep_test_with_build(MultiCommand)
 
 set(RunCMake_TEST_OUTPUT_MERGE 1)