Merge topic 'command-job-server-aware'

95941fd990 add_custom_{target,command}: Add argument JOB_SERVER_AWARE

Acked-by: Kitware Robot <[email protected]>
Acked-by: buildbot <[email protected]>
Merge-request: !8547

Help/command/add_custom_command.rst

@@ -24,6 +24,7 @@ The first signature is for adding a custom command to produce an output:
                      [COMMENT comment]
                      [DEPFILE depfile]
                      [JOB_POOL job_pool]
+                     [JOB_SERVER_AWARE <bool>]
                      [VERBATIM] [APPEND] [USES_TERMINAL]
                      [COMMAND_EXPAND_LISTS]
                      [DEPENDS_EXPLICIT_ONLY])
@@ -221,6 +222,19 @@ The options are:
   Using a pool that is not defined by :prop_gbl:`JOB_POOLS` causes
   an error by ninja at build time.
 
+``JOB_SERVER_AWARE``
+  .. versionadded:: 3.28
+
+  Specify that the command is GNU Make job server aware.
+
+  For the :generator:`Unix Makefiles`, :generator:`MSYS Makefiles`, and
+  :generator:`MinGW Makefiles` generators this will add the ``+`` prefix to the
+  recipe line. See the `GNU Make Documentation`_ for more information.
+
+  This option is silently ignored by other generators.
+
+.. _`GNU Make Documentation`: https://www.gnu.org/software/make/manual/html_node/MAKE-Variable.html
+
 ``MAIN_DEPENDENCY``
   Specify the primary input source file to the command.  This is
   treated just like any value given to the ``DEPENDS`` option

Help/command/add_custom_target.rst

@@ -12,6 +12,7 @@ Add a target with no output so it will always be built.
                     [WORKING_DIRECTORY dir]
                     [COMMENT comment]
                     [JOB_POOL job_pool]
+                    [JOB_SERVER_AWARE <bool>]
                     [VERBATIM] [USES_TERMINAL]
                     [COMMAND_EXPAND_LISTS]
                     [SOURCES src1 [src2...]])
@@ -146,6 +147,19 @@ The options are:
   Using a pool that is not defined by :prop_gbl:`JOB_POOLS` causes
   an error by ninja at build time.
 
+``JOB_SERVER_AWARE``
+  .. versionadded:: 3.28
+
+  Specify that the command is GNU Make job server aware.
+
+  For the :generator:`Unix Makefiles`, :generator:`MSYS Makefiles`, and
+  :generator:`MinGW Makefiles` generators this will add the ``+`` prefix to the
+  recipe line. See the `GNU Make Documentation`_ for more information.
+
+  This option is silently ignored by other generators.
+
+.. _`GNU Make Documentation`: https://www.gnu.org/software/make/manual/html_node/MAKE-Variable.html
+
 ``SOURCES``
   Specify additional source files to be included in the custom target.
   Specified source files will be added to IDE project files for

Help/release/dev/command-job-server-aware.rst

@@ -0,0 +1,5 @@
+command-job-server-aware
+------------------------
+
+* The :command:`add_custom_command` and :command:`add_custom_target`
+  commands gained a ``JOB_SERVER_AWARE`` option.

Source/cmAddCustomCommandCommand.cxx

@@ -19,6 +19,7 @@
 #include "cmPolicies.h"
 #include "cmStringAlgorithms.h"
 #include "cmSystemTools.h"
+#include "cmValue.h"
 
 bool cmAddCustomCommandCommand(std::vector<std::string> const& args,
                                cmExecutionStatus& status)
@@ -39,6 +40,7 @@ bool cmAddCustomCommandCommand(std::vector<std::string> const& args,
   std::string working;
   std::string depfile;
   std::string job_pool;
+  std::string job_server_aware;
   std::string comment_buffer;
   const char* comment = nullptr;
   std::vector<std::string> depends;
@@ -78,6 +80,7 @@ bool cmAddCustomCommandCommand(std::vector<std::string> const& args,
     doing_working_directory,
     doing_depfile,
     doing_job_pool,
+    doing_job_server_aware,
     doing_nothing
   };
 
@@ -95,6 +98,7 @@ bool cmAddCustomCommandCommand(std::vector<std::string> const& args,
   MAKE_STATIC_KEYWORD(DEPFILE);
   MAKE_STATIC_KEYWORD(IMPLICIT_DEPENDS);
   MAKE_STATIC_KEYWORD(JOB_POOL);
+  MAKE_STATIC_KEYWORD(JOB_SERVER_AWARE);
   MAKE_STATIC_KEYWORD(MAIN_DEPENDENCY);
   MAKE_STATIC_KEYWORD(OUTPUT);
   MAKE_STATIC_KEYWORD(OUTPUTS);
@@ -126,6 +130,7 @@ bool cmAddCustomCommandCommand(std::vector<std::string> const& args,
     keyPRE_BUILD,
     keyPRE_LINK,
     keySOURCE,
+    keyJOB_SERVER_AWARE,
     keyTARGET,
     keyUSES_TERMINAL,
     keyVERBATIM,
@@ -190,6 +195,8 @@ bool cmAddCustomCommandCommand(std::vector<std::string> const& args,
         }
       } else if (copy == keyJOB_POOL) {
         doing = doing_job_pool;
+      } else if (copy == keyJOB_SERVER_AWARE) {
+        doing = doing_job_server_aware;
       }
     } else {
       std::string filename;
@@ -226,6 +233,9 @@ bool cmAddCustomCommandCommand(std::vector<std::string> const& args,
         case doing_job_pool:
           job_pool = copy;
           break;
+        case doing_job_server_aware:
+          job_server_aware = copy;
+          break;
         case doing_working_directory:
           working = copy;
           break;
@@ -324,6 +334,15 @@ bool cmAddCustomCommandCommand(std::vector<std::string> const& args,
     return false;
   }
 
+  // If using a GNU Make generator and `JOB_SERVER_AWARE` is set then
+  // prefix all commands with '+'.
+  if (cmIsOn(job_server_aware) &&
+      mf.GetGlobalGenerator()->IsGNUMakeJobServerAware()) {
+    for (auto& commandLine : commandLines) {
+      commandLine.insert(commandLine.begin(), "+");
+    }
+  }
+
   // Choose which mode of the command to use.
   auto cc = cm::make_unique<cmCustomCommand>();
   cc->SetByproducts(byproducts);

Source/cmAddCustomTargetCommand.cxx

@@ -17,6 +17,7 @@
 #include "cmStringAlgorithms.h"
 #include "cmSystemTools.h"
 #include "cmTarget.h"
+#include "cmValue.h"
 
 bool cmAddCustomTargetCommand(std::vector<std::string> const& args,
                               cmExecutionStatus& status)
@@ -54,6 +55,7 @@ bool cmAddCustomTargetCommand(std::vector<std::string> const& args,
   const char* comment = nullptr;
   std::vector<std::string> sources;
   std::string job_pool;
+  std::string JOB_SERVER_AWARE;
 
   // Keep track of parser state.
   enum tdoing
@@ -65,6 +67,7 @@ bool cmAddCustomTargetCommand(std::vector<std::string> const& args,
     doing_comment,
     doing_source,
     doing_job_pool,
+    doing_JOB_SERVER_AWARE,
     doing_nothing
   };
   tdoing doing = doing_command;
@@ -102,6 +105,8 @@ bool cmAddCustomTargetCommand(std::vector<std::string> const& args,
       doing = doing_comment;
     } else if (copy == "JOB_POOL") {
       doing = doing_job_pool;
+    } else if (copy == "JOB_SERVER_AWARE") {
+      doing = doing_JOB_SERVER_AWARE;
     } else if (copy == "COMMAND") {
       doing = doing_command;
 
@@ -148,6 +153,9 @@ bool cmAddCustomTargetCommand(std::vector<std::string> const& args,
         case doing_job_pool:
           job_pool = copy;
           break;
+        case doing_JOB_SERVER_AWARE:
+          JOB_SERVER_AWARE = copy;
+          break;
         default:
           status.SetError("Wrong syntax. Unknown type of argument.");
           return false;
@@ -212,6 +220,15 @@ bool cmAddCustomTargetCommand(std::vector<std::string> const& args,
     return false;
   }
 
+  // If using a GNU Make generator and `JOB_SERVER_AWARE` is set then
+  // prefix all commands with '+'.
+  if (cmIsOn(JOB_SERVER_AWARE) &&
+      mf.GetGlobalGenerator()->IsGNUMakeJobServerAware()) {
+    for (auto& commandLine : commandLines) {
+      commandLine.insert(commandLine.begin(), "+");
+    }
+  }
+
   // Add the utility target to the makefile.
   auto cc = cm::make_unique<cmCustomCommand>();
   cc->SetWorkingDirectory(working_directory.c_str());

Source/cmGlobalBorlandMakefileGenerator.h

@@ -54,6 +54,8 @@ public:
   bool AllowDeleteOnError() const override { return false; }
   bool CanEscapeOctothorpe() const override { return true; }
 
+  bool IsGNUMakeJobServerAware() const override { return false; }
+
 protected:
   std::vector<GeneratedMakeCommand> GenerateBuildCommand(
     const std::string& makeProgram, const std::string& projectName,

Source/cmGlobalGenerator.h

@@ -158,6 +158,8 @@ public:
 
   virtual bool CheckCxxModuleSupport() { return false; }
 
+  virtual bool IsGNUMakeJobServerAware() const { return false; }
+
   bool Compute();
   virtual void AddExtraIDETargets() {}
 

Source/cmGlobalJOMMakefileGenerator.h

@@ -47,6 +47,8 @@ public:
   void EnableLanguage(std::vector<std::string> const& languages, cmMakefile*,
                       bool optional) override;
 
+  bool IsGNUMakeJobServerAware() const override { return false; }
+
 protected:
   std::vector<GeneratedMakeCommand> GenerateBuildCommand(
     const std::string& makeProgram, const std::string& projectName,

Source/cmGlobalNMakeMakefileGenerator.h

@@ -54,6 +54,8 @@ public:
   void EnableLanguage(std::vector<std::string> const& languages, cmMakefile*,
                       bool optional) override;
 
+  bool IsGNUMakeJobServerAware() const override { return false; }
+
 protected:
   std::vector<GeneratedMakeCommand> GenerateBuildCommand(
     const std::string& makeProgram, const std::string& projectName,

Source/cmGlobalUnixMakefileGenerator3.h

@@ -120,6 +120,8 @@ public:
 
   void Configure() override;
 
+  bool IsGNUMakeJobServerAware() const override { return true; }
+
   /**
    * Generate the all required files for building this project/tree. This
    * basically creates a series of LocalGenerators for each directory and

Source/cmGlobalWatcomWMakeGenerator.h

@@ -53,6 +53,8 @@ public:
   bool AllowNotParallel() const override { return false; }
   bool AllowDeleteOnError() const override { return false; }
 
+  bool IsGNUMakeJobServerAware() const override { return false; }
+
 protected:
   std::vector<GeneratedMakeCommand> GenerateBuildCommand(
     const std::string& makeProgram, const std::string& projectName,

Tests/RunCMake/CMakeLists.txt

@@ -171,8 +171,9 @@ endif()
 if(NOT CMAKE_GENERATOR MATCHES "Visual Studio|Xcode")
   add_RunCMake_test(CMP0065 -DCMAKE_SYSTEM_NAME=${CMAKE_SYSTEM_NAME})
 endif()
+add_executable(detect_jobserver detect_jobserver.c)
 if(CMAKE_GENERATOR MATCHES "Make")
-  add_RunCMake_test(Make -DMAKE_IS_GNU=${MAKE_IS_GNU})
+  add_RunCMake_test(Make -DMAKE_IS_GNU=${MAKE_IS_GNU} -DDETECT_JOBSERVER=$<TARGET_FILE:detect_jobserver>)
 endif()
 unset(ninja_test_with_qt_version)
 unset(ninja_qt_args)

Tests/RunCMake/Make/DetectJobServer-absent-parallel-build-stderr.txt

@@ -0,0 +1 @@
+^(Warning: (Borland's make|NMake|Watcom's WMake) does not support parallel builds\. Ignoring parallel build command line option\.)?$

Tests/RunCMake/Make/DetectJobServer-absent.cmake

@@ -0,0 +1,13 @@
+# Verifies that the jobserver connection is absent
+add_custom_command(OUTPUT custom_command.txt
+  JOB_SERVER_AWARE OFF
+  COMMENT "Should not detect jobserver"
+  COMMAND ${DETECT_JOBSERVER} --absent "custom_command.txt"
+)
+
+# trigger the custom command to run
+add_custom_target(dummy ALL
+  JOB_SERVER_AWARE OFF
+  DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/custom_command.txt
+  COMMAND ${DETECT_JOBSERVER} --absent "custom_target.txt"
+)

Tests/RunCMake/Make/DetectJobServer-present.cmake

@@ -0,0 +1,13 @@
+# Verifies that the jobserver is present
+add_custom_command(OUTPUT custom_command.txt
+  JOB_SERVER_AWARE ON
+  COMMENT "Should detect jobserver support"
+  COMMAND ${DETECT_JOBSERVER} --present "custom_command.txt"
+)
+
+# trigger the custom command to run
+add_custom_target(dummy ALL
+  JOB_SERVER_AWARE ON
+  DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/custom_command.txt
+  COMMAND ${DETECT_JOBSERVER} --present "custom_target.txt"
+)

Tests/RunCMake/Make/GNUMakeJobServerAware-check.cmake

@@ -0,0 +1,12 @@
+# This test verifies that the commands in the generated Makefiles contain the
+# `+` prefix
+function(check_for_plus_prefix target)
+  set(file "${RunCMake_BINARY_DIR}/GNUMakeJobServerAware-build/${target}")
+  file(READ "${file}" build_file)
+  if(NOT "${build_file}" MATCHES [[\+]])
+    message(FATAL_ERROR "The file ${file} does not contain the expected prefix in the custom command.")
+  endif()
+endfunction()
+
+check_for_plus_prefix("CMakeFiles/dummy.dir/build.make")
+check_for_plus_prefix("CMakeFiles/dummy2.dir/build.make")

Tests/RunCMake/Make/GNUMakeJobServerAware.cmake

@@ -0,0 +1,12 @@
+add_custom_command(
+  OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/custom-command"
+  JOB_SERVER_AWARE ON
+  COMMAND $(CMAKE_COMMAND) -E touch "${CMAKE_CURRENT_BINARY_DIR}/custom-command"
+)
+add_custom_target(dummy ALL DEPENDS "${CMAKE_CURRENT_BINARY_DIR}/custom-command")
+
+add_custom_target(
+  dummy2 ALL
+  JOB_SERVER_AWARE ON
+  COMMAND ${CMAKE_COMMAND} -E true
+)

Tests/RunCMake/Make/RunCMakeTest.cmake

@@ -70,3 +70,43 @@ if(NOT RunCMake_GENERATOR STREQUAL "Watcom WMake")
   run_CMP0113(OLD)
   run_CMP0113(NEW)
 endif()
+
+function(detect_jobserver_present is_parallel)
+  set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/DetectJobServer-present-build)
+  set(RunCMake_TEST_NO_CLEAN 1)
+  set(RunCMake_TEST_OPTIONS "-DDETECT_JOBSERVER=${DETECT_JOBSERVER}")
+  run_cmake(DetectJobServer-present)
+  if (is_parallel)
+    run_cmake_command(DetectJobServer-present-parallel-build ${CMAKE_COMMAND} --build . -j4)
+  else()
+    run_cmake_command(DetectJobServer-present-build ${CMAKE_COMMAND} --build .)
+  endif()
+endfunction()
+
+function(detect_jobserver_absent is_parallel)
+  set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/DetectJobServer-absent-build)
+  set(RunCMake_TEST_NO_CLEAN 1)
+  set(RunCMake_TEST_OPTIONS "-DDETECT_JOBSERVER=${DETECT_JOBSERVER}")
+  run_cmake(DetectJobServer-absent)
+  if (is_parallel)
+    run_cmake_command(DetectJobServer-absent-parallel-build ${CMAKE_COMMAND} --build . -j4)
+  else()
+    run_cmake_command(DetectJobServer-absent-build ${CMAKE_COMMAND} --build .)
+  endif()
+endfunction()
+
+# Jobservers are currently only supported by GNU makes, except MSYS2 make
+if(MAKE_IS_GNU AND NOT RunCMake_GENERATOR MATCHES "MSYS Makefiles")
+  detect_jobserver_present(ON)
+else()
+  detect_jobserver_absent(ON)
+endif()
+# No matter which generator is used, the jobserver should not be present if a
+# parallel build is not requested
+detect_jobserver_absent(OFF)
+
+if(MAKE_IS_GNU)
+  # In GNU makes, `JOB_SERVER_AWARE` support is implemented by prefixing
+  # commands with the '+' operator.
+  run_cmake(GNUMakeJobServerAware)
+endif()

Tests/RunCMake/detect_jobserver.c

@@ -0,0 +1,204 @@
+#ifndef _CRT_SECURE_NO_WARNINGS
+#  define _CRT_SECURE_NO_WARNINGS
+#endif
+
+#if defined(_MSC_VER) && _MSC_VER >= 1928
+#  pragma warning(disable : 5105) /* macro expansion warning in windows.h */
+#endif
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#define MAX_MESSAGE_LENGTH 1023
+#define USAGE "Usage: %s [--present|--absent] <output_file>\n"
+
+// Extracts --jobserver-auth=<string> or --jobserver-fds=<string> from
+// MAKEFLAGS. The returned pointer points to the start of <string> Returns NULL
+// if MAKEFLAGS is not set or does not contain --jobserver-auth or
+// --jobserver-fds
+char* jobserver_auth(char* message)
+{
+  const char* jobserver_auth = "--jobserver-auth=";
+  const char* jobserver_fds = "--jobserver-fds=";
+  char* auth;
+  char* fds;
+  char* start;
+  char* end;
+  char* result;
+  size_t len;
+
+  char* makeflags = getenv("MAKEFLAGS");
+  if (makeflags == NULL) {
+    strncpy(message, "MAKEFLAGS not set", MAX_MESSAGE_LENGTH);
+    return NULL;
+  }
+
+  // write MAKEFLAGS to stdout for debugging
+  fprintf(stdout, "MAKEFLAGS: %s\n", makeflags);
+
+  auth = strstr(makeflags, jobserver_auth);
+  fds = strstr(makeflags, jobserver_fds);
+  if (auth == NULL && fds == NULL) {
+    strncpy(message, "No jobserver found", MAX_MESSAGE_LENGTH);
+    return NULL;
+  } else if (auth != NULL) {
+    start = auth + strlen(jobserver_auth);
+  } else {
+    start = fds + strlen(jobserver_fds);
+  }
+
+  end = strchr(start, ' ');
+  if (end == NULL) {
+    end = start + strlen(start);
+  }
+  len = (size_t)(end - start);
+  result = (char*)malloc(len + 1);
+  strncpy(result, start, len);
+  result[len] = '\0';
+
+  return result;
+}
+
+#if defined(_WIN32)
+#  include <windows.h>
+
+int windows_semaphore(const char* semaphore, char* message)
+{
+  // Open the semaphore
+  HANDLE hSemaphore = OpenSemaphoreA(SEMAPHORE_ALL_ACCESS, FALSE, semaphore);
+
+  if (hSemaphore == NULL) {
+#  if defined(_MSC_VER) && _MSC_VER < 1900
+    sprintf(message, "Error opening semaphore: %s (%ld)\n", semaphore,
+            GetLastError());
+#  else
+    snprintf(message, MAX_MESSAGE_LENGTH,
+             "Error opening semaphore: %s (%ld)\n", semaphore, GetLastError());
+#  endif
+    return 1;
+  }
+
+  strncpy(message, "Success", MAX_MESSAGE_LENGTH);
+  return 0;
+}
+#else
+#  include <errno.h>
+#  include <fcntl.h>
+
+int test_fd(int read_fd, int write_fd, char* message)
+{
+  // Detect if the file descriptors are valid
+  int read_good = fcntl(read_fd, F_GETFD) != -1;
+  int read_error = errno;
+
+  int write_good = fcntl(write_fd, F_GETFD) != -1;
+  int write_error = errno;
+
+  if (!read_good || !write_good) {
+    snprintf(message, MAX_MESSAGE_LENGTH,
+             "Error opening file descriptors: %d (%s), %d (%s)\n", read_fd,
+             strerror(read_error), write_fd, strerror(write_error));
+    return 1;
+  }
+
+  snprintf(message, MAX_MESSAGE_LENGTH, "Success\n");
+  return 0;
+}
+
+int posix(const char* jobserver, char* message)
+{
+  int read_fd;
+  int write_fd;
+  const char* path;
+
+  // First try to parse as "R,W" file descriptors
+  if (sscanf(jobserver, "%d,%d", &read_fd, &write_fd) == 2) {
+    return test_fd(read_fd, write_fd, message);
+  }
+
+  // Then try to parse as "fifo:PATH"
+  if (strncmp(jobserver, "fifo:", 5) == 0) {
+    path = jobserver + 5;
+    read_fd = open(path, O_RDONLY);
+    write_fd = open(path, O_WRONLY);
+    return test_fd(read_fd, write_fd, message);
+  }
+
+  // We don't understand the format
+  snprintf(message, MAX_MESSAGE_LENGTH, "Unrecognized jobserver format: %s\n",
+           jobserver);
+  return 1;
+}
+#endif
+
+// Takes 2 arguments:
+// Either --present or --absent to indicate we expect the jobserver to be
+// "present and valid", or "absent or invalid"
+//
+// if `--present` is passed, the exit code will be 0 if the jobserver is
+// present, 1 if it is absent if `--absent` is passed, the exit code will be 0
+// if the jobserver is absent, 1 if it is present in either case, if there is
+// some fatal error (e.g the output file cannot be opened), the exit code will
+// be 2
+int main(int argc, char** argv)
+{
+  char message[MAX_MESSAGE_LENGTH + 1];
+  char* output_file;
+  FILE* fp;
+  int expecting_present;
+  int expecting_absent;
+  char* jobserver;
+  int result;
+
+  if (argc != 3) {
+    fprintf(stderr, USAGE, argv[0]);
+    return 2;
+  }
+
+  expecting_present = strcmp(argv[1], "--present") == 0;
+  expecting_absent = strcmp(argv[1], "--absent") == 0;
+  if (!expecting_present && !expecting_absent) {
+    fprintf(stderr, USAGE, argv[0]);
+    return 2;
+  }
+
+  output_file = argv[2];
+  fp = fopen(output_file, "w");
+  if (fp == NULL) {
+    fprintf(stderr, "Error opening output file: %s\n", output_file);
+    return 2;
+  }
+
+  jobserver = jobserver_auth(message);
+  if (jobserver == NULL) {
+    if (expecting_absent) {
+      fprintf(stdout, "Success\n");
+      return 0;
+    }
+
+    fprintf(stderr, "%s\n", message);
+    return 1;
+  }
+
+#if defined(_WIN32)
+  result = windows_semaphore(jobserver, message);
+#else
+  result = posix(jobserver, message);
+#endif
+  free(jobserver);
+  message[MAX_MESSAGE_LENGTH] = 0;
+
+  if (result == 0 && expecting_present) {
+    fprintf(stdout, "Success\n");
+    return 0;
+  }
+
+  if (result == 1 && expecting_absent) {
+    fprintf(stdout, "Success\n");
+    return 0;
+  }
+
+  fprintf(stderr, "%s\n", message);
+  return 1;
+}