ExternalProject: Added new USES_TERMINAL options

Added new USES_TERMINAL option to the ExternalProject_Add_Step
function.  This option passes USES_TERMINAL to the underlying
add_custom_command call so that the Ninja console pool is used.
Also, corresponding new USES_TERMINAL_<step> options were added
to the ExternalProject_Add function.

Justification: if using Ninja with a CMake superbuild, it's often
desirable to limit the superbuild to ONE sub-Ninja process at a
time to avoid oversubscribing the CPU.  Using the console pool also
makes it easy to monitor the progress of the sub-Ninja process.

Independent USES_TERMINAL_<step> arguments are passed to
ExternalProject_Add instead of one USES_TERMINAL argument that
controls everything.  Users may wish to run some steps in parallel
but not others (e.g. parallelize configure but not build).

Help/release/dev/ExternalProject-USES_TERMINAL.rst

@@ -0,0 +1,7 @@
+ExternalProject-USES_TERMINAL
+-----------------------------
+
+* The :module:`ExternalProject` module learned new ``USES_TERMINAL``
+  arguments for giving steps exclusive terminal access.  Especially
+  useful with the :generator:`Ninja` generator to monitor CMake
+  superbuild progress and prevent CPU oversubscription.

Modules/ExternalProject.cmake

@@ -175,6 +175,23 @@ Create custom targets to build projects in external trees
   ``LOG_INSTALL 1``
     Wrap install in script to log output
 
+  Steps can be given direct access to the terminal if possible.  With
+  the :generator:`Ninja` generator, this places the steps in the
+  ``console`` :prop_gbl:`pool <JOB_POOLS>`.  Options are:
+
+  ``USES_TERMINAL_DOWNLOAD 1``
+    Give download terminal access.
+  ``USES_TERMINAL_UPDATE 1``
+    Give update terminal access.
+  ``USES_TERMINAL_CONFIGURE 1``
+    Give configure terminal access.
+  ``USES_TERMINAL_BUILD 1``
+    Give build terminal access.
+  ``USES_TERMINAL_TEST 1``
+    Give test terminal access.
+  ``USES_TERMINAL_INSTALL 1``
+    Give install terminal access.
+
   Other options are:
 
   ``STEP_TARGETS <step-target>...``
@@ -256,6 +273,8 @@ Create custom targets to build projects in external trees
     Working directory for command
   ``LOG 1``
     Wrap step in script to log output
+  ``USES_TERMINAL 1``
+    Give the step direct access to the terminal if possible.
 
   The command line, comment, working directory, and byproducts of every
   standard and custom step are processed to replace tokens ``<SOURCE_DIR>``,
@@ -1463,6 +1482,14 @@ function(ExternalProject_Add_Step name step)
     get_property(comment TARGET ${name} PROPERTY _EP_${step}_COMMENT)
   endif()
 
+  # Uses terminal?
+  get_property(uses_terminal TARGET ${name} PROPERTY _EP_${step}_USES_TERMINAL)
+  if(uses_terminal)
+    set(uses_terminal USES_TERMINAL)
+  else()
+    set(uses_terminal "")
+  endif()
+
   # Run every time?
   get_property(always TARGET ${name} PROPERTY _EP_${step}_ALWAYS)
   if(always)
@@ -1505,6 +1532,7 @@ function(ExternalProject_Add_Step name step)
     DEPENDS ${depends}
     WORKING_DIRECTORY ${work_dir}
     VERBATIM
+    ${uses_terminal}
     )
   set_property(TARGET ${name} APPEND PROPERTY _EP_STEPS ${step})
 
@@ -1890,6 +1918,14 @@ function(_ep_add_download_command name)
     set(log "")
   endif()
 
+  get_property(uses_terminal TARGET ${name} PROPERTY
+    _EP_USES_TERMINAL_DOWNLOAD)
+  if(uses_terminal)
+    set(uses_terminal USES_TERMINAL 1)
+  else()
+    set(uses_terminal "")
+  endif()
+
   ExternalProject_Add_Step(${name} download
     COMMENT ${comment}
     COMMAND ${cmd}
@@ -1897,6 +1933,7 @@ function(_ep_add_download_command name)
     DEPENDS ${depends}
     DEPENDEES mkdir
     ${log}
+    ${uses_terminal}
     )
 endfunction()
 
@@ -2001,6 +2038,14 @@ Update to Mercurial >= 2.1.1.
     set(log "")
   endif()
 
+  get_property(uses_terminal TARGET ${name} PROPERTY
+    _EP_USES_TERMINAL_UPDATE)
+  if(uses_terminal)
+    set(uses_terminal USES_TERMINAL 1)
+  else()
+    set(uses_terminal "")
+  endif()
+
   ExternalProject_Add_Step(${name} update
     COMMENT ${comment}
     COMMAND ${cmd}
@@ -2009,6 +2054,7 @@ Update to Mercurial >= 2.1.1.
     WORKING_DIRECTORY ${work_dir}
     DEPENDEES download
     ${log}
+    ${uses_terminal}
     )
 
   if(always AND update_disconnected)
@@ -2021,6 +2067,7 @@ Update to Mercurial >= 2.1.1.
       WORKING_DIRECTORY ${work_dir}
       DEPENDEES download
       ${log}
+      ${uses_terminal}
     )
     set_property(SOURCE ${skip-update_stamp_file} PROPERTY SYMBOLIC 1)
   endif()
@@ -2149,6 +2196,14 @@ function(_ep_add_configure_command name)
     set(log "")
   endif()
 
+  get_property(uses_terminal TARGET ${name} PROPERTY
+    _EP_USES_TERMINAL_CONFIGURE)
+  if(uses_terminal)
+    set(uses_terminal USES_TERMINAL 1)
+  else()
+    set(uses_terminal "")
+  endif()
+
   get_property(update_disconnected_set TARGET ${name} PROPERTY _EP_UPDATE_DISCONNECTED SET)
   if(update_disconnected_set)
     get_property(update_disconnected TARGET ${name} PROPERTY _EP_UPDATE_DISCONNECTED)
@@ -2167,6 +2222,7 @@ function(_ep_add_configure_command name)
     DEPENDEES ${update_dep} patch
     DEPENDS ${file_deps}
     ${log}
+    ${uses_terminal}
     )
 endfunction()
 
@@ -2188,6 +2244,14 @@ function(_ep_add_build_command name)
     set(log "")
   endif()
 
+  get_property(uses_terminal TARGET ${name} PROPERTY
+    _EP_USES_TERMINAL_BUILD)
+  if(uses_terminal)
+    set(uses_terminal USES_TERMINAL 1)
+  else()
+    set(uses_terminal "")
+  endif()
+
   get_property(build_always TARGET ${name} PROPERTY _EP_BUILD_ALWAYS)
   if(build_always)
     set(always 1)
@@ -2204,6 +2268,7 @@ function(_ep_add_build_command name)
     DEPENDEES configure
     ALWAYS ${always}
     ${log}
+    ${uses_terminal}
     )
 endfunction()
 
@@ -2225,11 +2290,20 @@ function(_ep_add_install_command name)
     set(log "")
   endif()
 
+  get_property(uses_terminal TARGET ${name} PROPERTY
+    _EP_USES_TERMINAL_INSTALL)
+  if(uses_terminal)
+    set(uses_terminal USES_TERMINAL 1)
+  else()
+    set(uses_terminal "")
+  endif()
+
   ExternalProject_Add_Step(${name} install
     COMMAND ${cmd}
     WORKING_DIRECTORY ${binary_dir}
     DEPENDEES build
     ${log}
+    ${uses_terminal}
     )
 endfunction()
 
@@ -2277,6 +2351,14 @@ function(_ep_add_test_command name)
       set(log "")
     endif()
 
+    get_property(uses_terminal TARGET ${name} PROPERTY
+      _EP_USES_TERMINAL_TEST)
+    if(uses_terminal)
+      set(uses_terminal USES_TERMINAL 1)
+    else()
+      set(uses_terminal "")
+    endif()
+
     ExternalProject_Add_Step(${name} test
       COMMAND ${cmd}
       WORKING_DIRECTORY ${binary_dir}
@@ -2284,6 +2366,7 @@ function(_ep_add_test_command name)
       ${dependers_args}
       ${exclude_args}
       ${log}
+      ${uses_terminal}
       )
   endif()
 endfunction()

Tests/RunCMake/ExternalProject/RunCMakeTest.cmake

@@ -11,3 +11,4 @@ run_cmake(Add_StepDependencies)
 run_cmake(Add_StepDependencies_iface)
 run_cmake(Add_StepDependencies_iface_step)
 run_cmake(Add_StepDependencies_no_target)
+run_cmake(UsesTerminal)

Tests/RunCMake/ExternalProject/UsesTerminal-check.cmake

@@ -0,0 +1,97 @@
+cmake_minimum_required(VERSION 3.3)
+
+# If we are using the Ninja generator, we can check and verify that the
+# USES_TERMINAL option actually works by examining the Ninja build file.
+# This is the only way, since CMake doesn't offer a way to examine the
+# options on a custom command after it has been added.  Furthermore,
+# there isn't an easy way to test for this by actually running Ninja.
+#
+# Other generators don't currently support USES_TERMINAL at this time.
+# This file can be improved to support them if they do.  Until then, we
+# simply assume success for new generator types.
+#
+# For Ninja, there is a complication.  If the Ninja generator detects a
+# version of Ninja < 1.5, it won't actually emit the console pool command,
+# because those Ninja versions don't yet support the console pool.  In
+# that case, we also have to assume success.
+
+# Check Ninja build output to verify whether or not a target step is in the
+# console pool.
+macro(CheckNinjaStep _target _step _require)
+  if("${_build}" MATCHES
+"  DESC = Performing ${_step} step for '${_target}'
+  pool = console"
+  )
+    if(NOT ${_require})
+      set(RunCMake_TEST_FAILED "${_target} ${_step} step is in console pool")
+      return()
+    endif()
+  else()
+    if(${_require})
+      set(RunCMake_TEST_FAILED "${_target} ${_step} step not in console pool")
+      return()
+    endif()
+  endif()
+endmacro()
+
+# Check Ninja build output to verify whether each target step is in the
+# console pool.
+macro(CheckNinjaTarget _target
+  _download _update _configure _build _test _install
+  )
+  CheckNinjaStep(${_target} download ${_download})
+  CheckNinjaStep(${_target} update ${_update})
+  CheckNinjaStep(${_target} configure ${_configure})
+  CheckNinjaStep(${_target} build ${_build})
+  CheckNinjaStep(${_target} test ${_test})
+  CheckNinjaStep(${_target} install ${_install})
+endmacro()
+
+# Load build/make file, depending on generator
+if(RunCMake_GENERATOR STREQUAL Ninja)
+  # Check the Ninja version.  If < 1.5, console pool isn't supported and
+  # so the generator would not emit console pool usage.  That would cause
+  # this test to fail.
+  execute_process(COMMAND ${RunCMake_MAKE_PROGRAM} --version
+    RESULT_VARIABLE _version_result
+    OUTPUT_VARIABLE _version
+    ERROR_QUIET
+    OUTPUT_STRIP_TRAILING_WHITESPACE
+    )
+  if(_version_result OR _version VERSION_EQUAL "0")
+    set(RunCMake_TEST_FAILED "Failed to get Ninja version")
+    return()
+  endif()
+  if(_version VERSION_LESS "1.5")
+    return() # console pool not supported on Ninja < 1.5
+  endif()
+
+  # Read the Ninja build file
+  set(_build_file "${RunCMake_TEST_BINARY_DIR}/build.ninja")
+
+  if(NOT EXISTS "${_build_file}")
+    set(RunCMake_TEST_FAILED "Ninja build file not created")
+    return()
+  endif()
+
+  file(READ "${_build_file}" _build)
+
+  set(_target_check_macro CheckNinjaTarget)
+elseif((RunCMake_GENERATOR STREQUAL "") OR NOT DEFINED RunCMake_GENERATOR)
+  # protection in case somebody renamed RunCMake_GENERATOR
+  set(RunCMake_TEST_FAILED "Unknown generator")
+  return()
+else()
+  # We don't yet know how to test USES_TERMINAL on this generator.
+  return()
+endif()
+
+# Actual tests:
+CheckNinjaTarget(TerminalTest1
+  true  true  true  true  true  true )
+CheckNinjaTarget(TerminalTest2
+  true  false true  false true  false)
+CheckNinjaTarget(TerminalTest3
+  false true  false true  false true )
+CheckNinjaTarget(TerminalTest4
+  false false false false false false)

Tests/RunCMake/ExternalProject/UsesTerminal.cmake

@@ -0,0 +1,45 @@
+if(NOT CMAKE_CONFIGURATION_TYPES)
+  set(CMAKE_BUILD_TYPE Debug)
+endif()
+include(ExternalProject)
+
+# Test various combinations of USES_TERMINAL with ExternalProject_Add.
+
+macro(DoTerminalTest _target)
+  ExternalProject_Add(${_target}
+    DOWNLOAD_COMMAND "${CMAKE_COMMAND}" -E echo "download"
+    UPDATE_COMMAND "${CMAKE_COMMAND}" -E echo "update"
+    CONFIGURE_COMMAND "${CMAKE_COMMAND}" -E echo "configure"
+    BUILD_COMMAND "${CMAKE_COMMAND}" -E echo "build"
+    TEST_COMMAND "${CMAKE_COMMAND}" -E echo "test"
+    INSTALL_COMMAND "${CMAKE_COMMAND}" -E echo "install"
+    ${ARGN}
+    )
+endmacro()
+
+# USES_TERMINAL on all steps
+DoTerminalTest(TerminalTest1
+  USES_TERMINAL_DOWNLOAD 1
+  USES_TERMINAL_UPDATE 1
+  USES_TERMINAL_CONFIGURE 1
+  USES_TERMINAL_BUILD 1
+  USES_TERMINAL_TEST 1
+  USES_TERMINAL_INSTALL 1
+  )
+
+# USES_TERMINAL on every other step, starting with download
+DoTerminalTest(TerminalTest2
+  USES_TERMINAL_DOWNLOAD 1
+  USES_TERMINAL_CONFIGURE 1
+  USES_TERMINAL_TEST 1
+  )
+
+# USES_TERMINAL on every other step, starting with update
+DoTerminalTest(TerminalTest3
+  USES_TERMINAL_UPDATE 1
+  USES_TERMINAL_BUILD 1
+  USES_TERMINAL_INSTALL 1
+  )
+
+# USES_TERMINAL on no step
+DoTerminalTest(TerminalTest4)