ExternalProject: Set environment variables

Add the ability to modify the environment of the various steps running
as part of an external project build. This specifically adds the ability
to set them on the configure , build , install, and test steps, as well
as exposing the `ENVIRONMENT_MODIFICATION` keyword arguments to
`ExternalProject_Add_Step`, allowing customization of the environment of
custom steps.

The values of the environment variable respect the `LIST_SEPARATOR`.

Fixes: #26963

Auxiliary/vim/syntax/cmake.vim

@@ -2287,6 +2287,7 @@ syn keyword cmakeKWExternalProject contained
             \ BUILD_ALWAYS
             \ BUILD_BYPRODUCTS
             \ BUILD_COMMAND
+            \ BUILD_ENVIRONMENT_MODIFICATION
             \ BUILD_IN_SOURCE
             \ CHECKOUT
             \ CMAKE_ARGS
@@ -2296,6 +2297,7 @@ syn keyword cmakeKWExternalProject contained
             \ CMAKE_INSTALL_MODE
             \ COMMENT
             \ CONFIGURE_COMMAND
+            \ CONFIGURE_ENVIRONMENT_MODIFICATION
             \ CONFIGURE_HANDLED_BY_BUILD
             \ CVS
             \ CVSROOT
@@ -2312,6 +2314,7 @@ syn keyword cmakeKWExternalProject contained
             \ DOWNLOAD_NAME
             \ DOWNLOAD_NO_EXTRACT
             \ DOWNLOAD_NO_PROGRESS
+            \ ENVIRONMENT_MODIFICATION
             \ EP_BASE
             \ EP_INDEPENDENT_STEP_TARGETS
             \ EP_PREFIX
@@ -2341,6 +2344,7 @@ syn keyword cmakeKWExternalProject contained
             \ INSTALL_BYPRODUCTS
             \ INSTALL_COMMAND
             \ INSTALL_DIR
+            \ INSTALL_ENVIRONMENT_MODIFICATION
             \ JOB_POOLS
             \ LIST_SEPARATOR
             \ LOG_BUILD
@@ -2380,6 +2384,7 @@ syn keyword cmakeKWExternalProject contained
             \ TEST_AFTER_INSTALL
             \ TEST_BEFORE_INSTALL
             \ TEST_COMMAND
+            \ TEST_ENVIRONMENT_MODIFICATION
             \ TEST_EXCLUDE_FROM_MAIN
             \ TIMEOUT
             \ TLS_CAINFO

Modules/ExternalProject.cmake

@@ -654,6 +654,48 @@ overridden if required.
   examples of build systems whose build step is smart enough to know if the
   configure step needs to be rerun.
 
+``CONFIGURE_ENVIRONMENT_MODIFICATION <modification>...``
+  .. versionadded: 4.2
+
+  Specify environment variables that should be modified for the configure step.
+
+  Set a :ref:`semicolon-separated list <CMake Language Lists>` of environment
+  variables and values of the form ``MYVAR=OP:VALUE``, where ``MYVAR`` is the
+  case-sensitive name of an environment variable to be modified.  Entries are
+  considered in the order specified in the property's value. The ``OP`` may be
+  one of:
+
+  .. include:: ../include/ENVIRONMENT_MODIFICATION_OPS.rst
+
+  .. code-block:: cmake
+
+    ExternalProject_Add(example
+      ... # Download options, etc...
+      CONFIGURE_ENVIRONMENT_MODIFICATION
+        SDKROOT=set:macosx
+        PKG_CONFIG_PATH=set:$ENV{PKG_CONFIG_PATH}
+    )
+
+  This snippet defines two environment variables when configuring the example
+  project. The ``SDKROOT`` environment variable is set to ``macosx`, while
+  the value of ``PKG_CONFIG_PATH`` is forwarded to the external project.
+
+  Environment modifications work with ``LIST_SEPARATOR`` to replace the
+  separator with a ``;`` in the environment variable.
+
+  .. code-block:: cmake
+
+    ExternalProject_Add(example
+      ... # Download options, etc...
+      LIST_SEPARATOR ,
+      CONFIGURE_ENVIRONMENT_MODIFICATION
+        LIST_VAR=set:a,b,c
+    )
+
+  This snippet
+  and the environment variable ``LIST_VAR`` is passed to the configure command
+  invocation with the value ``a;b;c``.
+
 Build Step Options
 """"""""""""""""""
 
@@ -719,6 +761,19 @@ pass ``-v`` to the external project's build step, even if it also uses
   ``JOB_SERVER_AWARE`` option for details.  This option is relevant
   only when an explicit ``BUILD_COMMAND`` is specified.
 
+``BUILD_ENVIRONMENT_MODIFICATION <modification>...``
+  .. versionadded: 4.2
+
+  Specify environment variables that should be modified for the build step.
+
+  Set a :ref:`semicolon-separated list <CMake Language Lists>` of environment
+  variables and values of the form ``MYVAR=OP:VALUE``, where ``MYVAR`` is the
+  case-sensitive name of an environment variable to be modified.  Entries are
+  considered in the order specified in the property's value. The ``OP`` may be
+  one of:
+
+  .. include:: ../include/ENVIRONMENT_MODIFICATION_OPS.rst
+
 Install Step Options
 """"""""""""""""""""
 
@@ -777,6 +832,19 @@ step. This can be overridden with custom install commands if required.
   :envvar:`CMAKE_INSTALL_MODE` environment variable changes from one run
   to another.
 
+``INSTALL_ENVIRONMENT_MODIFICATION <modification>...``
+  .. versionadded: 4.2
+
+  Specify environment variables that should be modified for the install step.
+
+  Set a :ref:`semicolon-separated list <CMake Language Lists>` of environment
+  variables and values of the form ``MYVAR=OP:VALUE``, where ``MYVAR`` is the
+  case-sensitive name of an environment variable to be modified.  Entries are
+  considered in the order specified in the property's value. The ``OP`` may be
+  one of:
+
+  .. include:: ../include/ENVIRONMENT_MODIFICATION_OPS.rst
+
 Test Step Options
 """""""""""""""""
 
@@ -815,6 +883,19 @@ options are provided.
   This may cause a step target to be created automatically for either
   the ``install`` or ``build`` step.  See policy :policy:`CMP0114`.
 
+``TEST_ENVIRONMENT_MODIFICATION <modification>...``
+  .. versionadded: 4.2
+
+  Specify environment variables that should be modified for the test step.
+
+  Set a :ref:`semicolon-separated list <CMake Language Lists>` of environment
+  variables and values of the form ``MYVAR=OP:VALUE``, where ``MYVAR`` is the
+  case-sensitive name of an environment variable to be modified.  Entries are
+  considered in the order specified in the property's value. The ``OP`` may be
+  one of:
+
+  .. include:: ../include/ENVIRONMENT_MODIFICATION_OPS.rst
+
 Output Logging Options
 """"""""""""""""""""""
 
@@ -940,7 +1021,8 @@ Miscellaneous Options
 """""""""""""""""""""
 
 ``LIST_SEPARATOR <sep>``
-  For any of the various ``..._COMMAND`` options, and ``CMAKE_ARGS``,
+  For any of the various ``..._COMMAND`` options, ``CMAKE_ARGS``, and
+  `..._ENVIRONMENT_MODIFICATION`` operations,
   ``ExternalProject`` will replace ``<sep>`` with ``;`` in the specified
   command lines. This can be used to ensure a command has a literal ``;`` in it
   where direct usage would otherwise be interpreted as argument separators to
@@ -1043,6 +1125,20 @@ control needed to implement such step-level capabilities.
   ``DEPENDS <file>...``
     Files on which this custom step depends.
 
+  ``ENVIRONMENT_MODIFICATION <modification>...``
+    .. versionadded: 4.2
+
+    Specify environment variables that should be modified while running the
+    commands in the external project step.
+
+    Set a :ref:`semicolon-separated list <CMake Language Lists>` of environment
+    variables and values of the form ``MYVAR=OP:VALUE``, where ``MYVAR`` is the
+    case-sensitive name of an environment variable to be modified.  Entries are
+    considered in the order specified in the property's value. The ``OP`` may be
+    one of:
+
+    .. include:: ../include/ENVIRONMENT_MODIFICATION_OPS.rst
+
   ``INDEPENDENT <bool>``
     .. versionadded:: 3.19
 
@@ -2057,6 +2153,7 @@ function(ExternalProject_Add_Step name step)
     DEPENDEES
     DEPENDERS
     DEPENDS
+    ENVIRONMENT_MODIFICATION
     INDEPENDENT
     BYPRODUCTS
     ALWAYS
@@ -2176,6 +2273,50 @@ function(ExternalProject_Add_Step name step)
     string(REPLACE "${sep}" "\\;" command "${command}")
   endif()
 
+  # Add environment here!
+  get_property(environment
+    TARGET ${name}
+    PROPERTY _EP_${step}_ENVIRONMENT_MODIFICATION)
+  if(environment AND command)
+    if("${sep}" STREQUAL ":")
+      # The environment modification operation and value is separated by a
+      # colon. We should not replace that colon with a semicolon, allowing
+      # colons to act as a valid list separator.
+      # <name>=<op>:<value>
+      # Note: Environment variable names can contain `:` on Windows
+      set(result "")
+      foreach(env_modification IN LISTS environment)
+        if("${env_modification}" MATCHES "(.*)=([a-z]*):(.*)?")
+          if(CMAKE_MATCH_COUNT EQUAL 3)
+            string(REPLACE "${sep}" "\\\\\\\\\\;" _escapedMod "${CMAKE_MATCH_3}")
+          endif()
+          list(APPEND result "${CMAKE_MATCH_1}=${CMAKE_MATCH_2}:${_escapedMod}")
+        else()
+          message(SEND_ERROR "Malformed environment modification specifier:"
+          " '${env_modification}'\n"
+          "Expected MYVAR=OP:VALUE")
+        endif()
+      endforeach()
+      set(environment ${result})
+    elseif(sep)
+      # if the separator is not a colon, we don't have to worry about
+      # accidentally replacing the separator between the modification and value
+      string(REPLACE "${sep}" "\\\\\\\\;" environment "${environment}")
+    endif()
+    list(JOIN environment ";--modify;" environment)
+    list(PREPEND environment "--modify")
+
+    set(_result "")
+    list(APPEND _result "${CMAKE_COMMAND}" -E env ${environment} --)
+    foreach(element IN LISTS command)
+      list(APPEND _result "${element}")
+      if("${element}" STREQUAL COMMAND)
+        list(APPEND _result "${CMAKE_COMMAND}" -E env ${environment} --)
+      endif()
+    endforeach()
+    set(command ${_result})
+  endif()
+
   # Replace location tags.
   _ep_replace_location_tags(
     ${name}
@@ -2660,6 +2801,14 @@ function(_ep_add_configure_command name)
     set(dependees patch)
   endif()
 
+  get_property(environment
+    TARGET ${name}
+    PROPERTY _EP_CONFIGURE_ENVIRONMENT_MODIFICATION
+  )
+  if(environment)
+    set(environment "ENVIRONMENT_MODIFICATION" ${environment})
+  endif()
+
   get_property(log
     TARGET ${name}
     PROPERTY _EP_LOG_CONFIGURE
@@ -2691,6 +2840,7 @@ function(_ep_add_configure_command name)
       WORKING_DIRECTORY \${binary_dir}
       DEPENDEES \${dependees}
       DEPENDS \${file_deps}
+      ${environment}
       ${log}
       ${uses_terminal}
     )"
@@ -2770,6 +2920,13 @@ function(_ep_add_build_command name)
     set(maybe_JOB_SERVER_AWARE "")
   endif()
 
+  get_property(environment
+    TARGET ${name}
+    PROPERTY _EP_BUILD_ENVIRONMENT_MODIFICATION
+  )
+  if(environment)
+    set(environment ENVIRONMENT_MODIFICATION ${environment})
+  endif()
 
   set(__cmdQuoted)
   foreach(__item IN LISTS cmd)
@@ -2785,6 +2942,7 @@ function(_ep_add_build_command name)
       DEPENDS \${file_deps}
       ALWAYS \${always}
       ${maybe_JOB_SERVER_AWARE}
+      ${environment}
       ${log}
       ${uses_terminal}
     )"
@@ -2857,6 +3015,14 @@ function(_ep_add_install_command name)
     set(maybe_JOB_SERVER_AWARE "")
   endif()
 
+  get_property(environment
+    TARGET ${name}
+    PROPERTY _EP_INSTALL_ENVIRONMENT_MODIFICATION
+  )
+  if(environment)
+    set(environment ENVIRONMENT_MODIFICATION ${environment})
+  endif()
+
   set(__cmdQuoted)
   foreach(__item IN LISTS cmd)
     string(APPEND __cmdQuoted " [==[${__item}]==]")
@@ -2870,6 +3036,7 @@ function(_ep_add_install_command name)
       DEPENDEES build
       ALWAYS \${always}
       ${maybe_JOB_SERVER_AWARE}
+      ${environment}
       ${log}
       ${uses_terminal}
     )"
@@ -2936,6 +3103,14 @@ function(_ep_add_test_command name)
       set(uses_terminal "")
     endif()
 
+    get_property(environment
+      TARGET ${name}
+      PROPERTY _EP_TEST_ENVIRONMENT_MODIFICATION
+    )
+    if(environment)
+      set(environment ENVIRONMENT_MODIFICATION ${environment})
+    endif()
+
     set(__cmdQuoted)
     foreach(__item IN LISTS cmd)
       string(APPEND __cmdQuoted " [==[${__item}]==]")
@@ -2948,6 +3123,7 @@ function(_ep_add_test_command name)
         ${dependees_args}
         ${dependers_args}
         ${exclude_args}
+        ${environment}
         ${log}
         ${uses_terminal}
       )"

Modules/ExternalProject/shared_internal_commands.cmake

@@ -1896,6 +1896,7 @@ macro(_ep_get_add_keywords out_var)
     # Configure step options
     #
     CONFIGURE_COMMAND
+    CONFIGURE_ENVIRONMENT_MODIFICATION
     CMAKE_COMMAND
     CMAKE_GENERATOR
     CMAKE_GENERATOR_PLATFORM
@@ -1910,6 +1911,7 @@ macro(_ep_get_add_keywords out_var)
     # Build step options
     #
     BUILD_COMMAND
+    BUILD_ENVIRONMENT_MODIFICATION
     BUILD_IN_SOURCE
     BUILD_ALWAYS
     BUILD_BYPRODUCTS
@@ -1918,12 +1920,14 @@ macro(_ep_get_add_keywords out_var)
     # Install step options
     #
     INSTALL_COMMAND
+    INSTALL_ENVIRONMENT_MODIFICATION
     INSTALL_BYPRODUCTS
     INSTALL_JOB_SERVER_AWARE
     #
     # Test step options
     #
     TEST_COMMAND
+    TEST_ENVIRONMENT_MODIFICATION
     TEST_BEFORE_INSTALL
     TEST_AFTER_INSTALL
     TEST_EXCLUDE_FROM_MAIN

Tests/RunCMake/ExternalProject/EnvVars-build-stdout.txt

@@ -0,0 +1,32 @@
+.*(Performing custom step for 'CustomCommandEnvVars'|CustomCommandEnvVars-custom).*
+ *-- Variable - CustomVar: custom.*
+ *-- Variable - CustomVar2: custom2.*
+ *-- Stage: custom
+ *-- Separator: ;
+ *-- List: 1;2;3
+.*(Performing configure step for 'CustomCommandEnvVars'|CustomCommandEnvVars-configure).*
+ *-- Stage: config
+ *-- Separator: ;
+ *-- List: 4;5;6.*
+ *-- Variable - Stage: config.*
+ *-- Variable - ListVar: 4;5;6
+.*(Performing build step for 'CustomCommandEnvVars'|CustomCommandEnvVars-build).*
+ *-- Stage: build
+ *-- Separator: ;
+ *-- List: 4;5;6
+.*(Performing install step for 'CustomCommandEnvVars'|CustomCommandEnvVars-install).*
+ *-- Stage: install
+ *-- Separator: ;
+ *-- List: 4;5;6
+.*(Performing test step for 'CustomCommandEnvVars'|CustomCommandEnvVars-test).*
+ *-- Stage: test
+ *-- Separator: ;
+ *-- List: 4;5;6
+.*(Performing configure step for 'DefaultCommandEnvVars'|DefaultCommandEnvVars-configure).*
+ *-- ConfigVar: config
+ *-- Separator: ,
+ *-- List: 7,8,9
+.*(Performing build step for 'DefaultCommandEnvVars'|DefaultCommandEnvVars-build).*
+ *-- Stage: build
+ *-- Separator: ,
+ *-- List: 7,8,9,10

Tests/RunCMake/ExternalProject/EnvVars.cmake

@@ -0,0 +1,102 @@
+include(ExternalProject)
+
+#
+## Set environment variables on custom commands
+#
+
+# Comma list-separator
+set(ScriptPath "${CMAKE_CURRENT_SOURCE_DIR}/EnvVars/EchoVar.cmake")
+ExternalProject_Add(CustomCommandEnvVars
+  DOWNLOAD_COMMAND ""
+  UPDATE_COMMAND ""
+  PATCH_COMMAND ""
+  LIST_SEPARATOR ,
+  CONFIGURE_COMMAND "${CMAKE_COMMAND}" -P ${ScriptPath}
+  COMMAND "${CMAKE_COMMAND}" -DVARNAME=Stage -P ${ScriptPath}
+  COMMAND "${CMAKE_COMMAND}" -DVARNAME=ListVar -P ${ScriptPath}
+  CONFIGURE_ENVIRONMENT_MODIFICATION
+    Stage=set:config
+    ListVar=set:4,5,6
+    ListSeparator=set:,
+  BUILD_COMMAND "${CMAKE_COMMAND}" -P ${ScriptPath}
+  BUILD_ENVIRONMENT_MODIFICATION
+    Stage=set:build
+    ListVar=set:4,5,6
+    ListSeparator=set:,
+  INSTALL_COMMAND "${CMAKE_COMMAND}" -P ${ScriptPath}
+  INSTALL_ENVIRONMENT_MODIFICATION
+    InstallVar=set:install
+    Stage=set:install
+    ListVar=set:4,5,6
+    ListSeparator=set:,
+  TEST_COMMAND "${CMAKE_COMMAND}" -P ${ScriptPath}
+  TEST_ENVIRONMENT_MODIFICATION
+    Stage=set:test
+    ListVar=set:4,5,6
+    ListSeparator=set:,)
+
+ExternalProject_Add_Step(CustomCommandEnvVars custom
+  DEPENDERS configure
+  COMMAND "${CMAKE_COMMAND}" -DVARNAME=CustomVar -P ${ScriptPath}
+  COMMAND "${CMAKE_COMMAND}" -DVARNAME=CustomVar2 -P ${ScriptPath}
+  COMMAND "${CMAKE_COMMAND}" -P ${ScriptPath}
+  ENVIRONMENT_MODIFICATION
+    CustomVar=set:custom
+    CustomVar2=set:custom2
+    Stage=set:custom
+    ListVar=set:1,2,3
+    ListSeparator=set:,)
+
+#
+## Set environment variables on the default commands
+#
+
+# No list separator
+ExternalProject_Add(DefaultCommandEnvVars
+  SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/EnvVars"
+  DOWNLOAD_COMMAND ""
+  UPDATE_COMMAND ""
+  PATCH_COMMAND ""
+  DEPENDS CustomCommandEnvVars
+  CMAKE_ARGS
+    -DVARIABLE=ConfigVar
+  CONFIGURE_ENVIRONMENT_MODIFICATION
+    ConfigVar=set:config
+    ListVar=set:7,8,9
+    ListSeparator=set:,
+  BUILD_ENVIRONMENT_MODIFICATION
+    Stage=set:build
+    ListVar=set:7,8,9,10
+    ListSeparator=set:,
+  INSTALL_COMMAND ""                 # empty install command should not show up
+  INSTALL_ENVIRONMENT_MODIFICATION
+    Stage=set:install
+    Separator=set:,)
+
+# Using `:` as a list separator on Windows does not work as it replaces the `:`
+# between the drive letter and the filepath with `;`.
+if(NOT WIN32)
+  # Ensure that using `:` as a list-separator does not break setting environment
+  # variables
+  ExternalProject_Add(DefaultCommandListSepEnvVars
+    SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/EnvVars"
+    DOWNLOAD_COMMAND ""
+    UPDATE_COMMAND ""
+    PATCH_COMMAND ""
+    DEPENDS DefaultCommandEnvVars
+    LIST_SEPARATOR :
+    CMAKE_ARGS
+      -DVARIABLE=ConfigVar
+    CONFIGURE_ENVIRONMENT_MODIFICATION
+      ConfigVar=set:config
+      ListVar=set:10:11:12
+      ListSeparator=set::
+    BUILD_ENVIRONMENT_MODIFICATION
+      Stage=set:build
+      ListVar=set:10:11:12
+      ListSeparator=set::
+    INSTALL_ENVIRONMENT_MODIFICATION
+      Stage=set:install
+      ListSeparator=set::
+      ListVar=set:10:11:12:13)
+endif()

Tests/RunCMake/ExternalProject/EnvVars/CMakeLists.txt

@@ -0,0 +1,12 @@
+cmake_minimum_required(VERSION 4.0)
+
+project(EnvVars LANGUAGES NONE)
+
+message(STATUS "${VARIABLE}: $ENV{${VARIABLE}}")
+message(STATUS "Separator: $ENV{ListSeparator}")
+message(STATUS "List: $ENV{ListVar}")
+
+add_custom_target(EchoEnvVars ALL COMMAND
+  "${CMAKE_COMMAND}" -P "${CMAKE_CURRENT_SOURCE_DIR}/EchoVar.cmake")
+
+install(SCRIPT "${CMAKE_CURRENT_SOURCE_DIR}/EchoVar.cmake")

Tests/RunCMake/ExternalProject/EnvVars/EchoVar.cmake

@@ -0,0 +1,7 @@
+if(VARNAME)
+  message(STATUS "Variable - ${VARNAME}: $ENV{${VARNAME}}")
+else()
+  message(STATUS "Stage: $ENV{Stage}")
+  message(STATUS "Separator: $ENV{ListSeparator}")
+  message(STATUS "List: $ENV{ListVar}")
+endif()

Tests/RunCMake/ExternalProject/InvalidEnvModification-result.txt

@@ -0,0 +1 @@
+1

Tests/RunCMake/ExternalProject/InvalidEnvModification-stderr.txt

@@ -0,0 +1,6 @@
+.* Malformed environment modification specifier: 'HI'
+.* Expected MYVAR=OP:VALUE
+.* Malformed environment modification specifier: 'INVALID=SETTING'
+.* Expected MYVAR=OP:VALUE
+.* Malformed environment modification specifier: 'INVALID=OP1:operation:10'
+.* Expected MYVAR=OP:VALUE

Tests/RunCMake/ExternalProject/InvalidEnvModification.cmake

@@ -0,0 +1,12 @@
+include(ExternalProject)
+
+ExternalProject_Add(EnvModification
+  SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/EnvVars"
+  DOWNLOAD_COMMAND ""
+  UPDATE_COMMAND ""
+  PATCH_COMMAND ""
+  LIST_SEPARATOR :
+  CONFIGURE_ENVIRONMENT_MODIFICATION
+    HI
+    INVALID=SETTING
+    INVALID=OP1:operation:10)

Tests/RunCMake/ExternalProject/RunCMakeTest.cmake

@@ -259,3 +259,8 @@ if(GIT_EXECUTABLE)
   run_cmake(TLSVersionBadVar)
   run_cmake(TLSVersionBadEnv)
 endif()
+
+set(RunCMake_TEST_OUTPUT_MERGE 1)
+__ep_test_with_build(EnvVars)
+unset(RunCMake_TEST_OUTPUT_MERGE)
+run_cmake(InvalidEnvModification)