Merge topic 'ExternalProject-build-jobserver'

bc43398e72 ExternalProject: Enable Make Job Server with Explicit Build Command
99be022428 Tests/RunCMake/Make: simplify GNUMakeJobSeverAware check function

Acked-by: Kitware Robot <[email protected]>
Tested-by: buildbot <[email protected]>
Merge-request: !8667

Help/release/dev/ExternalProject-build-jobserver.rst

@@ -0,0 +1,10 @@
+ExternalProject-build-jobserver
+-------------------------------
+
+* The :module:`ExternalProject` module now includes the
+  ``BUILD_JOB_SERVER_AWARE`` option for the
+  :command:`ExternalProject_Add` command. This option enables
+  the integration of the GNU Make job server when using an
+  explicit ``BUILD_COMMAND`` with certain :ref:`Makefile Generators`.
+  Additionally, the :command:`ExternalProject_Add_Step` command
+  has been updated to support the new ``JOB_SERVER_AWARE`` option.

Modules/ExternalProject.cmake

@@ -684,6 +684,14 @@ pass ``-v`` to the external project's build step, even if it also uses
   build step's own underlying call to :command:`add_custom_command`, which
   has additional documentation.
 
+``BUILD_JOB_SERVER_AWARE <bool>``
+  .. versionadded:: 3.28
+
+  Specifies that the build step is aware of the GNU Make job server.
+  See the :command:`add_custom_command` documentation of its
+  ``JOB_SERVER_AWARE`` option for details.  This option is relevant
+  only when an explicit ``BUILD_COMMAND`` is specified.
+
 Install Step Options
 """"""""""""""""""""
 
@@ -1021,6 +1029,13 @@ control needed to implement such step-level capabilities.
     When enabled, this option specifies that the custom step should always be
     run (i.e. that it is always considered out of date).
 
+  ``JOB_SERVER_AWARE <bool>``
+    .. versionadded:: 3.28
+
+    Specifies that the custom step is aware of the GNU Make job server.
+    See the :command:`add_custom_command` documentation of its
+    ``JOB_SERVER_AWARE`` option for details.
+
   ``EXCLUDE_FROM_MAIN <bool>``
     When enabled, this option specifies that the external project's main target
     does not depend on the custom step.
@@ -2366,6 +2381,7 @@ function(ExternalProject_Add_Step name step)
     INDEPENDENT
     BYPRODUCTS
     ALWAYS
+    JOB_SERVER_AWARE
     EXCLUDE_FROM_MAIN
     WORKING_DIRECTORY
     LOG
@@ -2545,6 +2561,16 @@ function(ExternalProject_Add_Step name step)
     set(maybe_COMMAND_touch "COMMAND \${CMAKE_COMMAND} -E touch \${stamp_file}")
   endif()
 
+  get_property(job_server_aware
+    TARGET ${name}
+    PROPERTY _EP_${step}_JOB_SERVER_AWARE
+  )
+  if(job_server_aware)
+    set(maybe_JOB_SERVER_AWARE "JOB_SERVER_AWARE 1")
+  else()
+    set(maybe_JOB_SERVER_AWARE "")
+  endif()
+
   # Wrap with log script?
   get_property(log TARGET ${name} PROPERTY _EP_${step}_LOG)
   if(command AND log)
@@ -2571,6 +2597,7 @@ function(ExternalProject_Add_Step name step)
       COMMENT \${comment}
       COMMAND ${__cmdQuoted}
       ${maybe_COMMAND_touch}
+      ${maybe_JOB_SERVER_AWARE}
       DEPENDS \${depends}
       WORKING_DIRECTORY \${work_dir}
       VERBATIM
@@ -3945,6 +3972,17 @@ function(_ep_add_build_command name)
     PROPERTY _EP_BUILD_BYPRODUCTS
   )
 
+  get_property(build_job_server_aware
+    TARGET ${name}
+    PROPERTY _EP_BUILD_JOB_SERVER_AWARE
+  )
+  if(build_job_server_aware)
+    set(maybe_JOB_SERVER_AWARE "JOB_SERVER_AWARE 1")
+  else()
+    set(maybe_JOB_SERVER_AWARE "")
+  endif()
+
+
   set(__cmdQuoted)
   foreach(__item IN LISTS cmd)
     string(APPEND __cmdQuoted " [==[${__item}]==]")
@@ -3958,6 +3996,7 @@ function(_ep_add_build_command name)
       DEPENDEES configure
       DEPENDS \${file_deps}
       ALWAYS \${always}
+      ${maybe_JOB_SERVER_AWARE}
       ${log}
       ${uses_terminal}
     )"
@@ -4252,6 +4291,7 @@ function(ExternalProject_Add name)
     BUILD_IN_SOURCE
     BUILD_ALWAYS
     BUILD_BYPRODUCTS
+    BUILD_JOB_SERVER_AWARE
     #
     # Install step options
     #

Tests/RunCMake/CMakeLists.txt

@@ -851,7 +851,7 @@ endif()
 if(CMake_TEST_RunCMake_ExternalProject_DOWNLOAD_SERVER_TIMEOUT)
   list(APPEND ExternalProject_ARGS -DDOWNLOAD_SERVER_TIMEOUT=${CMake_TEST_RunCMake_ExternalProject_DOWNLOAD_SERVER_TIMEOUT})
 endif()
-add_RunCMake_test(ExternalProject)
+add_RunCMake_test(ExternalProject -DDETECT_JOBSERVER=$<TARGET_FILE:detect_jobserver>)
 add_RunCMake_test(FetchContent)
 add_RunCMake_test(FetchContent_find_package)
 set(CTestCommandLine_ARGS -DPython_EXECUTABLE=${Python_EXECUTABLE})

Tests/RunCMake/ExternalProject/DetectJobServer.cmake

@@ -0,0 +1,16 @@
+include(ExternalProject)
+ExternalProject_Add(Foo
+  SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/Foo
+  BUILD_COMMAND ${DETECT_JOBSERVER} "ep.txt"
+  BUILD_JOB_SERVER_AWARE 1
+  INSTALL_COMMAND ""
+)
+
+# Add a second step to test JOB_SERVER_AWARE
+ExternalProject_Add_Step(Foo
+  second_step
+  COMMAND ${DETECT_JOBSERVER} "ep_second_step.txt"
+  DEPENDEES build
+  ALWAYS 1
+  JOB_SERVER_AWARE 1
+)

Tests/RunCMake/ExternalProject/Foo/CMakeLists.txt

@@ -0,0 +1,4 @@
+cmake_minimum_required(VERSION 3.27)
+project(Foo NONE)
+
+add_custom_target(drive ALL COMMAND ${CMAKE_COMMAND} -E true)

Tests/RunCMake/ExternalProject/GNUMakeJobServerAware-check.cmake

@@ -0,0 +1,16 @@
+set(BUILD_DIR "${RunCMake_BINARY_DIR}/GNUMakeJobServerAware-build")
+
+function(check target regex)
+  file(STRINGS ${BUILD_DIR}/${target} lines
+    REGEX ${regex}
+  )
+
+  list(LENGTH lines len)
+  if(len EQUAL 0)
+    message(FATAL_ERROR "Could not find matching lines '${regex}' in ${BUILD_DIR}/${target}")
+  endif()
+endfunction()
+
+check("/CMakeFiles/Foo.dir/build.make" [[\+cd (/d )?"?.*"? && "?.*"? --build "?.*"?]])
+check("/CMakeFiles/Foo.dir/build.make" [[\+cd (/d )?"?.*"? && "?.*"? -E touch "?.*"?]])
+check("/CMakeFiles/Foo.dir/build.make" [[\+"?.*"? -E true]])

Tests/RunCMake/ExternalProject/GNUMakeJobServerAware.cmake

@@ -0,0 +1,16 @@
+include(ExternalProject)
+ExternalProject_Add(Foo
+  SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/Foo
+  BUILD_COMMAND ${CMAKE_COMMAND} --build <BINARY_DIR>
+  BUILD_JOB_SERVER_AWARE 1
+  INSTALL_COMMAND ""
+)
+
+# Add a second step to test JOB_SERVER_AWARE
+ExternalProject_Add_Step(Foo
+  second_step
+  COMMAND ${CMAKE_COMMAND} -E true
+  DEPENDEES build
+  ALWAYS 1
+  JOB_SERVER_AWARE 1
+)

Tests/RunCMake/ExternalProject/RunCMakeTest.cmake

@@ -144,6 +144,24 @@ function(__ep_test_with_build_with_server testName)
   run_cmake_command(${testName}-build ${CMAKE_COMMAND} --build .)
 endfunction()
 
+if(RunCMake_GENERATOR MATCHES "(MSYS|MinGW|Unix) Makefiles")
+  __ep_test_with_build(GNUMakeJobServerAware)
+endif()
+
+function(__ep_test_jobserver)
+  set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/DetectJobServer-build)
+  set(RunCMake_TEST_NO_CLEAN 1)
+  file(REMOVE_RECURSE "${RunCMake_TEST_BINARY_DIR}")
+  file(MAKE_DIRECTORY "${RunCMake_TEST_BINARY_DIR}")
+  run_cmake_with_options(DetectJobServer -DDETECT_JOBSERVER=${DETECT_JOBSERVER})
+  run_cmake_command(DetectJobServer-clean ${CMAKE_COMMAND} --build . --target clean)
+  run_cmake_command(DetectJobServer-build ${CMAKE_COMMAND} --build . -j4)
+endfunction()
+
+if(RunCMake_GENERATOR MATCHES "(MinGW|Unix) Makefiles")
+  __ep_test_jobserver()
+endif()
+
 __ep_test_with_build(MultiCommand)
 
 set(RunCMake_TEST_OUTPUT_MERGE 1)

Tests/RunCMake/Make/DetectJobServer-absent-parallel-build-stderr.txt

@@ -1 +0,0 @@
-^(Warning: (Borland's make|NMake|Watcom's WMake) does not support parallel builds\. Ignoring parallel build command line option\.)?$

Tests/RunCMake/Make/DetectJobServer-absent.cmake

@@ -1,13 +0,0 @@
-# Verifies that the jobserver connection is absent
-add_custom_command(OUTPUT custom_command.txt
-  JOB_SERVER_AWARE OFF
-  COMMENT "Should not detect jobserver"
-  COMMAND ${DETECT_JOBSERVER} --absent "custom_command.txt"
-)
-
-# trigger the custom command to run
-add_custom_target(dummy ALL
-  JOB_SERVER_AWARE OFF
-  DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/custom_command.txt
-  COMMAND ${DETECT_JOBSERVER} --absent "custom_target.txt"
-)

Tests/RunCMake/Make/DetectJobServer-present.cmake

@@ -2,12 +2,12 @@
 add_custom_command(OUTPUT custom_command.txt
   JOB_SERVER_AWARE ON
   COMMENT "Should detect jobserver support"
-  COMMAND ${DETECT_JOBSERVER} --present "custom_command.txt"
+  COMMAND ${DETECT_JOBSERVER} "custom_command.txt"
 )
 
 # trigger the custom command to run
 add_custom_target(dummy ALL
   JOB_SERVER_AWARE ON
   DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/custom_command.txt
-  COMMAND ${DETECT_JOBSERVER} --present "custom_target.txt"
+  COMMAND ${DETECT_JOBSERVER} "custom_target.txt"
 )

Tests/RunCMake/Make/GNUMakeJobServerAware-check.cmake

@@ -1,21 +1,13 @@
 set(BUILD_DIR "${RunCMake_BINARY_DIR}/GNUMakeJobServerAware-build")
 
-function(check target line)
-  # Read the file and split it into a list of lines
-  file(READ ${BUILD_DIR}/${target} contents)
-  STRING(REGEX REPLACE ";" "\\\\;" contents "${contents}")
-  STRING(REGEX REPLACE "\n" ";" contents "${contents}")
+function(check target regex)
+  file(STRINGS ${BUILD_DIR}/${target} lines
+    REGEX ${regex}
+  )
 
-  set(found FALSE)
-  foreach(entry ${contents})
-    if("${entry}" MATCHES "${line}")
-      set(found TRUE)
-      break()
-    endif()
-  endforeach()
-
-  if(NOT found)
-    message(FATAL_ERROR "Could not find '${line}' in ${BUILD_DIR}/${target}\n${contents}")
+  list(LENGTH lines len)
+  if(len EQUAL 0)
+    message(FATAL_ERROR "Could not find matching lines '${regex}' in ${BUILD_DIR}/${target}")
   endif()
 endfunction()
 

Tests/RunCMake/Make/RunCMakeTest.cmake

@@ -71,39 +71,18 @@ if(NOT RunCMake_GENERATOR STREQUAL "Watcom WMake")
   run_CMP0113(NEW)
 endif()
 
-function(detect_jobserver_present is_parallel)
+function(detect_jobserver_present)
   set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/DetectJobServer-present-build)
   set(RunCMake_TEST_NO_CLEAN 1)
   set(RunCMake_TEST_OPTIONS "-DDETECT_JOBSERVER=${DETECT_JOBSERVER}")
   run_cmake(DetectJobServer-present)
-  if (is_parallel)
-    run_cmake_command(DetectJobServer-present-parallel-build ${CMAKE_COMMAND} --build . -j4)
-  else()
-    run_cmake_command(DetectJobServer-present-build ${CMAKE_COMMAND} --build .)
-  endif()
-endfunction()
-
-function(detect_jobserver_absent is_parallel)
-  set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/DetectJobServer-absent-build)
-  set(RunCMake_TEST_NO_CLEAN 1)
-  set(RunCMake_TEST_OPTIONS "-DDETECT_JOBSERVER=${DETECT_JOBSERVER}")
-  run_cmake(DetectJobServer-absent)
-  if (is_parallel)
-    run_cmake_command(DetectJobServer-absent-parallel-build ${CMAKE_COMMAND} --build . -j4)
-  else()
-    run_cmake_command(DetectJobServer-absent-build ${CMAKE_COMMAND} --build .)
-  endif()
+  run_cmake_command(DetectJobServer-present-parallel-build ${CMAKE_COMMAND} --build . -j4)
 endfunction()
 
 # Jobservers are currently only supported by GNU makes, except MSYS2 make
 if(MAKE_IS_GNU AND NOT RunCMake_GENERATOR MATCHES "MSYS Makefiles")
-  detect_jobserver_present(ON)
-else()
-  detect_jobserver_absent(ON)
+  detect_jobserver_present()
 endif()
-# No matter which generator is used, the jobserver should not be present if a
-# parallel build is not requested
-detect_jobserver_absent(OFF)
 
 if(MAKE_IS_GNU)
   # In GNU makes, `JOB_SERVER_AWARE` support is implemented by prefixing

Tests/RunCMake/detect_jobserver.c

@@ -11,22 +11,24 @@
 #include <string.h>
 
 #define MAX_MESSAGE_LENGTH 1023
-#define USAGE "Usage: %s [--present|--absent] <output_file>\n"
+#define USAGE "Usage: %s <output_file>\n"
 
-// Extracts --jobserver-auth=<string> or --jobserver-fds=<string> from
-// MAKEFLAGS. The returned pointer points to the start of <string> Returns NULL
-// if MAKEFLAGS is not set or does not contain --jobserver-auth or
-// --jobserver-fds
+// Extracts the jobserver details from the MAKEFLAGS environment variable.
+//
+// Returns a pointer to either a string of the form "R,W" where R and W are fds
+// or "fifo:PATH".
+//
+// Returns NULL if MAKEFLAGS is not set or does not contain recognized
+// jobserver flags.
 char* jobserver_auth(char* message)
 {
-  const char* jobserver_auth = "--jobserver-auth=";
-  const char* jobserver_fds = "--jobserver-fds=";
-  char* auth;
-  char* fds;
-  char* start;
+  const char* jobserver_flags[3] = { "--jobserver-auth=", "--jobserver-fds=",
+                                     "-J" };
+  char* start = NULL;
   char* end;
   char* result;
   size_t len;
+  int i;
 
   char* makeflags = getenv("MAKEFLAGS");
   if (makeflags == NULL) {
@@ -34,18 +36,24 @@ char* jobserver_auth(char* message)
     return NULL;
   }
 
-  // write MAKEFLAGS to stdout for debugging
   fprintf(stdout, "MAKEFLAGS: %s\n", makeflags);
 
-  auth = strstr(makeflags, jobserver_auth);
-  fds = strstr(makeflags, jobserver_fds);
-  if (auth == NULL && fds == NULL) {
-    strncpy(message, "No jobserver found", MAX_MESSAGE_LENGTH);
+  for (i = 0; i < 3; i++) {
+    start = strstr(makeflags, jobserver_flags[i]);
+    if (start != NULL) {
+      start += strlen(jobserver_flags[i]);
+      break;
+    }
+  }
+
+  if (start == NULL) {
+    strncpy(message, "No jobserver flags found", MAX_MESSAGE_LENGTH);
     return NULL;
-  } else if (auth != NULL) {
-    start = auth + strlen(jobserver_auth);
-  } else {
-    start = fds + strlen(jobserver_fds);
+  }
+
+  // Skip leading white space
+  while (*start == ' ' || *start == '\t') {
+    start++;
   }
 
   end = strchr(start, ' ');
@@ -132,38 +140,21 @@ int posix(const char* jobserver, char* message)
 }
 #endif
 
-// Takes 2 arguments:
-// Either --present or --absent to indicate we expect the jobserver to be
-// "present and valid", or "absent or invalid"
-//
-// if `--present` is passed, the exit code will be 0 if the jobserver is
-// present, 1 if it is absent if `--absent` is passed, the exit code will be 0
-// if the jobserver is absent, 1 if it is present in either case, if there is
-// some fatal error (e.g the output file cannot be opened), the exit code will
-// be 2
+// Takes 1 argument: an outfile to write results to.
 int main(int argc, char** argv)
 {
   char message[MAX_MESSAGE_LENGTH + 1];
   char* output_file;
   FILE* fp;
-  int expecting_present;
-  int expecting_absent;
   char* jobserver;
   int result;
 
-  if (argc != 3) {
-    fprintf(stderr, USAGE, argv[0]);
-    return 2;
-  }
-
-  expecting_present = strcmp(argv[1], "--present") == 0;
-  expecting_absent = strcmp(argv[1], "--absent") == 0;
-  if (!expecting_present && !expecting_absent) {
+  if (argc != 2) {
     fprintf(stderr, USAGE, argv[0]);
     return 2;
   }
 
-  output_file = argv[2];
+  output_file = argv[1];
   fp = fopen(output_file, "w");
   if (fp == NULL) {
     fprintf(stderr, "Error opening output file: %s\n", output_file);
@@ -172,11 +163,6 @@ int main(int argc, char** argv)
 
   jobserver = jobserver_auth(message);
   if (jobserver == NULL) {
-    if (expecting_absent) {
-      fprintf(stdout, "Success\n");
-      return 0;
-    }
-
     fprintf(stderr, "%s\n", message);
     return 1;
   }
@@ -187,18 +173,7 @@ int main(int argc, char** argv)
   result = posix(jobserver, message);
 #endif
   free(jobserver);
-  message[MAX_MESSAGE_LENGTH] = 0;
-
-  if (result == 0 && expecting_present) {
-    fprintf(stdout, "Success\n");
-    return 0;
-  }
-
-  if (result == 1 && expecting_absent) {
-    fprintf(stdout, "Success\n");
-    return 0;
-  }
+  message[MAX_MESSAGE_LENGTH] = '\0';
 
-  fprintf(stderr, "%s\n", message);
-  return 1;
+  return result;
 }