Merge topic 'ctest-environment-modifications'

de4f1f26b0 CTest: add an ENVIRONMENT_MODIFICATION property
4c757fa3c8 Help/prop_test/ENVIRONMENT: clarify the scope of the changes

Acked-by: Kitware Robot <[email protected]>
Merge-request: !6299

Auxiliary/vim/syntax/cmake.vim

@@ -160,6 +160,7 @@ syn keyword cmakeProperty contained
             \ ENABLED_LANGUAGES
             \ ENABLE_EXPORTS
             \ ENVIRONMENT
+            \ ENVIRONMENT_MODIFICATION
             \ EXCLUDE_FROM_ALL
             \ EXCLUDE_FROM_DEFAULT_BUILD
             \ EXPORT_NAME

Help/manual/cmake-properties.7.rst

@@ -451,6 +451,7 @@ Properties on Tests
    /prop_test/DEPENDS
    /prop_test/DISABLED
    /prop_test/ENVIRONMENT
+   /prop_test/ENVIRONMENT_MODIFICATION
    /prop_test/FAIL_REGULAR_EXPRESSION
    /prop_test/FIXTURES_CLEANUP
    /prop_test/FIXTURES_REQUIRED

Help/prop_test/ENVIRONMENT.rst

@@ -5,5 +5,5 @@ Specify environment variables that should be defined for running a test.
 
 If set to a list of environment variables and values of the form
 ``MYVAR=value`` those environment variables will be defined while running
-the test.  The environment is restored to its previous state after the
-test is done.
+the test.  The environment changes from this property do not affect other
+tests.

Help/prop_test/ENVIRONMENT_MODIFICATION.rst

@@ -0,0 +1,33 @@
+ENVIRONMENT_MODIFICATION
+------------------------
+
+Specify environment variables that should be modified for running a test. Note
+that the operations performed by this property are performed after the
+:prop_test:`ENVIRONMENT` property is already applied.
+
+If set to a list of environment variables and values of the form
+``MYVAR=OP:VALUE``. Entries are considered in the order specified in the
+property's value. The ``OP`` may be one of:
+
+  - ``reset``: Reset to the unmodified value, ignoring all modifications to
+    ``MYVAR`` prior to this entry. Note that this will reset the variable to
+    the value set by :prop_test:`ENVIRONMENT`, if it was set, and otherwise
+    to its state from the rest of the CTest execution.
+  - ``set``: Replaces the current value of ``MYVAR`` with ``VALUE``.
+  - ``unset``: Unsets the current value of ``MYVAR``.
+  - ``string_append``: Appends ``VALUE`` to the current value of ``MYVAR``.
+  - ``string_prepend``: Prepends ``VALUE`` to the current value of ``MYVAR``.
+  - ``path_list_append``: Appends ``VALUE`` to the current value of ``MYVAR``
+    using the platform-specific list separator.
+  - ``path_list_prepend``: Prepends ``VALUE`` to the current value of
+    ``MYVAR`` using the platform-specific list separator.
+  - ``cmake_list_append``: Appends ``VALUE`` to the current value of ``MYVAR``
+    using ``;`` as the separator.
+  - ``cmake_list_prepend``: Prepends ``VALUE`` to the current value of
+    ``MYVAR`` using ``;`` as the separator.
+
+Unrecognized ``OP`` values will result in the test failing before it is
+executed. This is so that future operations may be added without changing
+valid behavior of existing tests.
+
+The environment changes from this property do not affect other tests.

Help/release/dev/ctest-environment-modifications.rst

@@ -0,0 +1,7 @@
+ctest-environment-modifications
+-------------------------------
+
+* :manual:`ctest(1)` learned to be able to modify the environment for a test
+  through the :prop_test:`ENVIRONMENT_MODIFICATION` property. This is allows
+  for updates to environment variables based on the environment present at
+  test time.

Source/CTest/cmCTestMultiProcessHandler.cxx

@@ -1026,6 +1026,11 @@ static Json::Value DumpCTestProperties(
     properties.append(DumpCTestProperty(
       "ENVIRONMENT", DumpToJsonArray(testProperties.Environment)));
   }
+  if (!testProperties.EnvironmentModification.empty()) {
+    properties.append(DumpCTestProperty(
+      "ENVIRONMENT_MODIFICATION",
+      DumpToJsonArray(testProperties.EnvironmentModification)));
+  }
   if (!testProperties.ErrorRegularExpressions.empty()) {
     properties.append(DumpCTestProperty(
       "FAIL_REGULAR_EXPRESSION",

Source/CTest/cmCTestRunTest.cxx

@@ -8,12 +8,16 @@
 #include <cstdint>
 #include <cstdio>
 #include <cstring>
+#include <functional>
 #include <iomanip>
 #include <ratio>
 #include <sstream>
 #include <utility>
 
 #include <cm/memory>
+#include <cm/optional>
+#include <cm/string_view>
+#include <cmext/string_view>
 
 #include "cmsys/RegularExpression.hxx"
 
@@ -640,6 +644,7 @@ bool cmCTestRunTest::StartTest(size_t completed, size_t total)
 
   return this->ForkProcess(timeout, this->TestProperties->ExplicitTimeout,
                            &this->TestProperties->Environment,
+                           &this->TestProperties->EnvironmentModification,
                            &this->TestProperties->Affinity);
 }
 
@@ -696,6 +701,17 @@ void cmCTestRunTest::ComputeArguments()
     cmCTestLog(this->CTest, HANDLER_VERBOSE_OUTPUT,
                this->Index << ":  " << env << std::endl);
   }
+  if (!this->TestProperties->EnvironmentModification.empty()) {
+    cmCTestLog(this->CTest, HANDLER_VERBOSE_OUTPUT,
+               this->Index << ": "
+                           << "Environment variable modifications: "
+                           << std::endl);
+  }
+  for (std::string const& envmod :
+       this->TestProperties->EnvironmentModification) {
+    cmCTestLog(this->CTest, HANDLER_VERBOSE_OUTPUT,
+               this->Index << ":  " << envmod << std::endl);
+  }
 }
 
 void cmCTestRunTest::ParseOutputForMeasurements()
@@ -719,9 +735,11 @@ void cmCTestRunTest::ParseOutputForMeasurements()
   }
 }
 
-bool cmCTestRunTest::ForkProcess(cmDuration testTimeOut, bool explicitTimeout,
-                                 std::vector<std::string>* environment,
-                                 std::vector<size_t>* affinity)
+bool cmCTestRunTest::ForkProcess(
+  cmDuration testTimeOut, bool explicitTimeout,
+  std::vector<std::string>* environment,
+  std::vector<std::string>* environment_modification,
+  std::vector<size_t>* affinity)
 {
   this->TestProcess->SetId(this->Index);
   this->TestProcess->SetWorkingDirectory(this->TestProperties->Directory);
@@ -770,6 +788,127 @@ bool cmCTestRunTest::ForkProcess(cmDuration testTimeOut, bool explicitTimeout,
     }
   }
 
+  if (environment_modification && !environment_modification->empty()) {
+    std::map<std::string, cm::optional<std::string>> env_application;
+
+#ifdef _WIN32
+    char path_sep = ';';
+#else
+    char path_sep = ':';
+#endif
+
+    auto apply_diff =
+      [&env_application](const std::string& name,
+                         std::function<void(std::string&)> const& apply) {
+        auto entry = env_application.find(name);
+        std::string output;
+        if (entry != env_application.end() && entry->second) {
+          output = *entry->second;
+        }
+        apply(output);
+        entry->second = output;
+      };
+
+    bool err_occurred = false;
+
+    for (auto const& envmod : *environment_modification) {
+      // Split on `=`
+      auto const eq_loc = envmod.find_first_of('=');
+      if (eq_loc == std::string::npos) {
+        cmCTestLog(this->CTest, ERROR_MESSAGE,
+                   "Error: Missing `=` after the variable name in: "
+                     << envmod << std::endl);
+        err_occurred = true;
+        continue;
+      }
+      auto const name = envmod.substr(0, eq_loc);
+
+      // Split value on `:`
+      auto const op_value_start = eq_loc + 1;
+      auto const colon_loc = envmod.find_first_of(':', op_value_start);
+      if (colon_loc == std::string::npos) {
+        cmCTestLog(this->CTest, ERROR_MESSAGE,
+                   "Error: Missing `:` after the operation in: " << envmod
+                                                                 << std::endl);
+        err_occurred = true;
+        continue;
+      }
+      auto const op =
+        envmod.substr(op_value_start, colon_loc - op_value_start);
+
+      auto const value_start = colon_loc + 1;
+      auto const value = envmod.substr(value_start);
+
+      // Determine what to do with the operation.
+      if (op == "reset"_s) {
+        auto entry = env_application.find(name);
+        if (entry != env_application.end()) {
+          env_application.erase(entry);
+        }
+      } else if (op == "set"_s) {
+        env_application[name] = value;
+      } else if (op == "unset"_s) {
+        env_application[name] = {};
+      } else if (op == "string_append"_s) {
+        apply_diff(name, [&value](std::string& output) { output += value; });
+      } else if (op == "string_prepend"_s) {
+        apply_diff(name,
+                   [&value](std::string& output) { output.insert(0, value); });
+      } else if (op == "path_list_append"_s) {
+        apply_diff(name, [&value, path_sep](std::string& output) {
+          if (!output.empty()) {
+            output += path_sep;
+          }
+          output += value;
+        });
+      } else if (op == "path_list_prepend"_s) {
+        apply_diff(name, [&value, path_sep](std::string& output) {
+          if (!output.empty()) {
+            output.insert(output.begin(), path_sep);
+          }
+          output.insert(0, value);
+        });
+      } else if (op == "cmake_list_append"_s) {
+        apply_diff(name, [&value](std::string& output) {
+          if (!output.empty()) {
+            output += ';';
+          }
+          output += value;
+        });
+      } else if (op == "cmake_list_prepend"_s) {
+        apply_diff(name, [&value](std::string& output) {
+          if (!output.empty()) {
+            output.insert(output.begin(), ';');
+          }
+          output.insert(0, value);
+        });
+      } else {
+        cmCTestLog(this->CTest, ERROR_MESSAGE,
+                   "Error: Unrecognized environment manipulation argument: "
+                     << op << std::endl);
+        err_occurred = true;
+        continue;
+      }
+    }
+
+    if (err_occurred) {
+      return false;
+    }
+
+    for (auto const& env_apply : env_application) {
+      if (env_apply.second) {
+        auto const env_update =
+          cmStrCat(env_apply.first, '=', *env_apply.second);
+        cmSystemTools::PutEnv(env_update);
+        envMeasurement << env_update << std::endl;
+      } else {
+        cmSystemTools::UnsetEnv(env_apply.first.c_str());
+        // Signify that this variable is being actively unset
+        envMeasurement << "#" << env_apply.first << "=" << std::endl;
+      }
+    }
+  }
+
   if (this->UseAllocatedResources) {
     std::vector<std::string> envLog;
     this->SetupResourcesEnvironment(&envLog);

Source/CTest/cmCTestRunTest.h

@@ -113,6 +113,7 @@ private:
   void ExeNotFound(std::string exe);
   bool ForkProcess(cmDuration testTimeOut, bool explicitTimeout,
                    std::vector<std::string>* environment,
+                   std::vector<std::string>* environment_modification,
                    std::vector<size_t>* affinity);
   void WriteLogOutputTop(size_t completed, size_t total);
   // Run post processing of the process output for MemCheck

Source/CTest/cmCTestTestHandler.cxx

@@ -2245,6 +2245,8 @@ bool cmCTestTestHandler::SetTestsProperties(
             cmExpandList(val, rt.Depends);
           } else if (key == "ENVIRONMENT"_s) {
             cmExpandList(val, rt.Environment);
+          } else if (key == "ENVIRONMENT_MODIFICATION"_s) {
+            cmExpandList(val, rt.EnvironmentModification);
           } else if (key == "LABELS"_s) {
             std::vector<std::string> Labels = cmExpandedList(val);
             rt.Labels.insert(rt.Labels.end(), Labels.begin(), Labels.end());

Source/CTest/cmCTestTestHandler.h

@@ -151,6 +151,7 @@ public:
     // return code of test which will mark test as "not run"
     int SkipReturnCode;
     std::vector<std::string> Environment;
+    std::vector<std::string> EnvironmentModification;
     std::vector<std::string> Labels;
     std::set<std::string> LockedResources;
     std::set<std::string> FixturesSetup;

Tests/Environment/CMakeLists.txt

@@ -9,6 +9,7 @@ add_test(Environment1 Environment)
 add_test(Environment2 Environment)
 add_test(EchoEnvironment1 ${CMAKE_COMMAND} -E environment)
 add_test(EchoEnvironment2 ${CMAKE_COMMAND} -E environment)
+add_test(EchoEnvironment3 ${CMAKE_COMMAND} -P "${CMAKE_CURRENT_SOURCE_DIR}/check_mod.cmake")
 
 # Make sure "CMAKE_ENV.*Happy Thanksgiving" is in the output of
 # the "1" tests:
@@ -24,3 +25,37 @@ set_tests_properties(Environment1 EchoEnvironment1 PROPERTIES
 set_tests_properties(Environment2 EchoEnvironment2 PROPERTIES
   FAIL_REGULAR_EXPRESSION "CMAKE_ENV.*Happy Thanksgiving"
 )
+
+set_property(TEST EchoEnvironment3
+  PROPERTY ENVIRONMENT_MODIFICATION
+    # Variables expected to be unset.
+    "UNSET_EXPLICIT=set:value"
+    "UNSET_EXPLICIT=unset:"
+    "UNSET_VIA_RESET=set:value"
+    "UNSET_VIA_RESET=reset:"
+
+    # Direct settings.
+    "DIRECT=set:old"
+    "DIRECT=set:new"
+
+    # String manipulation.
+    "STRING_MANIP=set:-core-"
+    "STRING_MANIP=string_append:post-"
+    "STRING_MANIP=string_prepend:-pre"
+    "STRING_MANIP=string_append:suffix"
+    "STRING_MANIP=string_prepend:prefix"
+
+    # Path manipulation.
+    "PATH_MANIP=set:core"
+    "PATH_MANIP=path_list_append:post"
+    "PATH_MANIP=path_list_prepend:pre"
+    "PATH_MANIP=path_list_append:suffix"
+    "PATH_MANIP=path_list_prepend:prefix"
+
+    # CMake list manipulation.
+    "CMAKE_LIST_MANIP=set:core"
+    "CMAKE_LIST_MANIP=cmake_list_append:post"
+    "CMAKE_LIST_MANIP=cmake_list_prepend:pre"
+    "CMAKE_LIST_MANIP=cmake_list_append:suffix"
+    "CMAKE_LIST_MANIP=cmake_list_prepend:prefix"
+)

Tests/Environment/check_mod.cmake

@@ -0,0 +1,55 @@
+execute_process(
+  COMMAND ${CMAKE_COMMAND} -E environment
+  OUTPUT_VARIABLE out
+  ERROR_VARIABLE err
+  RESULT_VARIABLE res)
+
+if (res)
+  message(FATAL_ERROR "Failed with exit code ${res}: ${err}")
+endif ()
+
+if (CMAKE_HOST_WIN32)
+  set(path_sep ";")
+else ()
+  set(path_sep ":")
+endif ()
+
+set(unexpect_UNSET_EXPLICIT "")
+set(unexpect_UNSET_VIA_RESET "")
+set(expect_DIRECT "new")
+set(expect_STRING_MANIP "prefix-pre-core-post-suffix")
+set(expect_PATH_MANIP "prefix${path_sep}pre${path_sep}core${path_sep}post${path_sep}suffix")
+set(expect_CMAKE_LIST_MANIP "prefix;pre;core;post;suffix")
+
+set(expected_vars
+  DIRECT
+  STRING_MANIP
+  PATH_MANIP
+  CMAKE_LIST_MANIP)
+
+while (out)
+  string(FIND "${out}" "\n" nl_pos)
+  string(SUBSTRING "${out}" 0 "${nl_pos}" line)
+  math(EXPR line_next "${nl_pos} + 1")
+  string(SUBSTRING "${out}" "${line_next}" -1 out)
+
+  string(FIND "${line}" "=" eq_pos)
+  string(SUBSTRING "${line}" 0 "${eq_pos}" name)
+  math(EXPR value_start "${eq_pos} + 1")
+  string(SUBSTRING "${line}" "${value_start}" -1 value)
+
+  if (DEFINED "unexpect_${name}")
+    message(SEND_ERROR "Found `${name}=${value}` when it should have been unset")
+  elseif (DEFINED "expect_${name}")
+    list(REMOVE_ITEM expected_vars "${name}")
+    if (expect_${name} STREQUAL value)
+      message(STATUS "Found `${name}=${value}` as expected")
+    else ()
+      message(SEND_ERROR "Found `${name}=${value}` when it should have been ${expect_${name}}")
+    endif ()
+  endif ()
+endwhile ()
+
+if (expected_vars)
+  message(SEND_ERROR "Did not test expected variables: ${expected_vars}")
+endif ()

Tests/RunCMake/CMakeLists.txt

@@ -374,6 +374,7 @@ add_RunCMake_test(ctest_disabled_test)
 add_RunCMake_test(ctest_skipped_test)
 add_RunCMake_test(ctest_update)
 add_RunCMake_test(ctest_upload)
+add_RunCMake_test(ctest_environment)
 add_RunCMake_test(ctest_fixtures)
 add_RunCMake_test(file -DMSYS=${MSYS})
 add_RunCMake_test(file-CHMOD -DMSYS=${MSYS})

Tests/RunCMake/ctest_environment/CMakeLists.txt.in

@@ -0,0 +1,3 @@
+cmake_minimum_required(VERSION 3.21.0)
+project("@CASE_NAME@" NONE)
+include("@CASE_SOURCE_DIR@/@[email protected]")

Tests/RunCMake/ctest_environment/ENVIRONMENT_MODIFICATION-invalid-op-result.txt

@@ -0,0 +1 @@
+(-1|255)

Tests/RunCMake/ctest_environment/ENVIRONMENT_MODIFICATION-invalid-op-stderr.txt

@@ -0,0 +1 @@
+Error: Unrecognized environment manipulation argument: unknown

Tests/RunCMake/ctest_environment/ENVIRONMENT_MODIFICATION-invalid-op.cmake

@@ -0,0 +1,6 @@
+include(CTest)
+add_test(NAME cmake_version COMMAND "${CMAKE_COMMAND}" --version)
+
+set_property(TEST cmake_version
+  PROPERTY ENVIRONMENT_MODIFICATION
+    INVALID_OP=unknown:)

Tests/RunCMake/ctest_environment/ENVIRONMENT_MODIFICATION-no-colon-result.txt

@@ -0,0 +1 @@
+(-1|255)

Tests/RunCMake/ctest_environment/ENVIRONMENT_MODIFICATION-no-colon-stderr.txt

@@ -0,0 +1 @@
+Error: Missing `:` after the operation in: MISSING_COLON=unset

Tests/RunCMake/ctest_environment/ENVIRONMENT_MODIFICATION-no-colon.cmake

@@ -0,0 +1,6 @@
+include(CTest)
+
+add_test(NAME cmake_version COMMAND "${CMAKE_COMMAND}" --version)
+set_property(TEST cmake_version
+  PROPERTY ENVIRONMENT_MODIFICATION
+    MISSING_COLON=unset)

Tests/RunCMake/ctest_environment/ENVIRONMENT_MODIFICATION-no-equals-result.txt

@@ -0,0 +1 @@
+(-1|255)

Tests/RunCMake/ctest_environment/ENVIRONMENT_MODIFICATION-no-equals-stderr.txt

@@ -0,0 +1 @@
+Error: Missing `=` after the variable name in: MISSING_EQUAL

Tests/RunCMake/ctest_environment/ENVIRONMENT_MODIFICATION-no-equals.cmake

@@ -0,0 +1,6 @@
+include(CTest)
+
+add_test(NAME cmake_version COMMAND "${CMAKE_COMMAND}" --version)
+set_property(TEST cmake_version
+  PROPERTY ENVIRONMENT_MODIFICATION
+    MISSING_EQUAL)

Tests/RunCMake/ctest_environment/RunCMakeTest.cmake

@@ -0,0 +1,12 @@
+include(RunCTest)
+
+# Isolate our ctest runs from external environment.
+unset(ENV{CTEST_PARALLEL_LEVEL})
+unset(ENV{CTEST_OUTPUT_ON_FAILURE})
+
+set(CASE_SOURCE_DIR "${RunCMake_SOURCE_DIR}")
+set(RunCTest_VERBOSE_FLAG "-VV")
+
+run_ctest(ENVIRONMENT_MODIFICATION-invalid-op)
+run_ctest(ENVIRONMENT_MODIFICATION-no-colon)
+run_ctest(ENVIRONMENT_MODIFICATION-no-equals)

Tests/RunCMake/ctest_environment/test.cmake.in

@@ -0,0 +1,16 @@
+cmake_minimum_required(VERSION 3.7)
+
+set(CTEST_SITE                          "test-site")
+set(CTEST_BUILD_NAME                    "test-build-name")
+set(CTEST_SOURCE_DIRECTORY              "@RunCMake_BINARY_DIR@/@CASE_NAME@")
+set(CTEST_BINARY_DIRECTORY              "@RunCMake_BINARY_DIR@/@CASE_NAME@-build")
+set(CTEST_CMAKE_GENERATOR               "@RunCMake_GENERATOR@")
+set(CTEST_CMAKE_GENERATOR_PLATFORM      "@RunCMake_GENERATOR_PLATFORM@")
+set(CTEST_CMAKE_GENERATOR_TOOLSET       "@RunCMake_GENERATOR_TOOLSET@")
+set(CTEST_BUILD_CONFIGURATION           "$ENV{CMAKE_CONFIG_TYPE}")
+
+set(ctest_test_args "@CASE_CTEST_TEST_ARGS@")
+ctest_start(Experimental)
+ctest_configure()
+ctest_build()
+ctest_test(${ctest_test_args})