CTest: Log environment variables as a test measurement

Help/release/dev/ctest-log-environment.rst

@@ -0,0 +1,8 @@
+ctest-log-environment
+---------------------
+
+* :manual:`ctest(1)` now logs environment variables that it sets for each test,
+  either due to the :prop_test:`ENVIRONMENT` property or the
+  :ref:`resource allocation <ctest-resource-allocation>` feature, and submits
+  this log to CDash. It does not log environment variables that were set
+  outside of CTest.

Source/CTest/cmCTestRunTest.cxx

@@ -429,6 +429,7 @@ void cmCTestRunTest::StartFailure(std::string const& output,
   this->TestResult.Path = this->TestProperties->Directory;
   this->TestResult.Output = output;
   this->TestResult.FullCommandLine.clear();
+  this->TestResult.Environment.clear();
 }
 
 std::string cmCTestRunTest::GetTestPrefix(size_t completed, size_t total) const
@@ -500,6 +501,7 @@ bool cmCTestRunTest::StartTest(size_t completed, size_t total)
     this->TestResult.Status = cmCTestTestHandler::NOT_RUN;
     this->TestResult.Output = "Disabled";
     this->TestResult.FullCommandLine.clear();
+    this->TestResult.Environment.clear();
     return false;
   }
 
@@ -519,6 +521,7 @@ bool cmCTestRunTest::StartTest(size_t completed, size_t total)
     cmCTestLog(this->CTest, HANDLER_OUTPUT, msg << std::endl);
     this->TestResult.Output = msg;
     this->TestResult.FullCommandLine.clear();
+    this->TestResult.Environment.clear();
     this->TestResult.CompletionStatus = "Fixture dependency failed";
     this->TestResult.Status = cmCTestTestHandler::NOT_RUN;
     return false;
@@ -539,6 +542,7 @@ bool cmCTestRunTest::StartTest(size_t completed, size_t total)
     cmCTestLog(this->CTest, ERROR_MESSAGE, msg << std::endl);
     this->TestResult.Output = msg;
     this->TestResult.FullCommandLine.clear();
+    this->TestResult.Environment.clear();
     this->TestResult.CompletionStatus = "Missing Configuration";
     this->TestResult.Status = cmCTestTestHandler::NOT_RUN;
     return false;
@@ -554,6 +558,7 @@ bool cmCTestRunTest::StartTest(size_t completed, size_t total)
                  "Unable to find required file: " << file << std::endl);
       this->TestResult.Output = "Unable to find required file: " + file;
       this->TestResult.FullCommandLine.clear();
+      this->TestResult.Environment.clear();
       this->TestResult.CompletionStatus = "Required Files Missing";
       this->TestResult.Status = cmCTestTestHandler::NOT_RUN;
       return false;
@@ -569,6 +574,7 @@ bool cmCTestRunTest::StartTest(size_t completed, size_t total)
                "Unable to find executable: " << args[1] << std::endl);
     this->TestResult.Output = "Unable to find executable: " + args[1];
     this->TestResult.FullCommandLine.clear();
+    this->TestResult.Environment.clear();
     this->TestResult.CompletionStatus = "Unable to find executable";
     this->TestResult.Status = cmCTestTestHandler::NOT_RUN;
     return false;
@@ -713,25 +719,43 @@ bool cmCTestRunTest::ForkProcess(cmDuration testTimeOut, bool explicitTimeout,
   cmSystemTools::SaveRestoreEnvironment sre;
 #endif
 
+  std::ostringstream envMeasurement;
   if (environment && !environment->empty()) {
     cmSystemTools::AppendEnv(*environment);
+    for (auto const& var : *environment) {
+      envMeasurement << var << std::endl;
+    }
   }
 
   if (this->UseAllocatedResources) {
-    this->SetupResourcesEnvironment();
+    std::vector<std::string> envLog;
+    this->SetupResourcesEnvironment(&envLog);
+    for (auto const& var : envLog) {
+      envMeasurement << var << std::endl;
+    }
   } else {
     cmSystemTools::UnsetEnv("CTEST_RESOURCE_GROUP_COUNT");
+    // Signify that this variable is being actively unset
+    envMeasurement << "#CTEST_RESOURCE_GROUP_COUNT=" << std::endl;
   }
 
+  this->TestResult.Environment = envMeasurement.str();
+  // Remove last newline
+  this->TestResult.Environment.erase(this->TestResult.Environment.length() -
+                                     1);
+
   return this->TestProcess->StartProcess(this->MultiTestHandler.Loop,
                                          affinity);
 }
 
-void cmCTestRunTest::SetupResourcesEnvironment()
+void cmCTestRunTest::SetupResourcesEnvironment(std::vector<std::string>* log)
 {
   std::string processCount = "CTEST_RESOURCE_GROUP_COUNT=";
   processCount += std::to_string(this->AllocatedResources.size());
   cmSystemTools::PutEnv(processCount);
+  if (log) {
+    log->push_back(processCount);
+  }
 
   std::size_t i = 0;
   for (auto const& process : this->AllocatedResources) {
@@ -757,8 +781,14 @@ void cmCTestRunTest::SetupResourcesEnvironment()
         var += "id:" + it2.Id + ",slots:" + std::to_string(it2.Slots);
       }
       cmSystemTools::PutEnv(var);
+      if (log) {
+        log->push_back(var);
+      }
     }
     cmSystemTools::PutEnv(resourceList);
+    if (log) {
+      log->push_back(resourceList);
+    }
     ++i;
   }
 }

Source/CTest/cmCTestRunTest.h

@@ -119,7 +119,7 @@ private:
   // Run post processing of the process output for MemCheck
   void MemCheckPostProcess();
 
-  void SetupResourcesEnvironment();
+  void SetupResourcesEnvironment(std::vector<std::string>* log = nullptr);
 
   // Returns "completed/total Test #Index: "
   std::string GetTestPrefix(size_t completed, size_t total) const;

Source/CTest/cmCTestTestHandler.cxx

@@ -1432,6 +1432,12 @@ void cmCTestTestHandler::GenerateDartOutput(cmXMLWriter& xml)
     xml.Attribute("name", "Command Line");
     xml.Element("Value", result.FullCommandLine);
     xml.EndElement(); // NamedMeasurement
+
+    xml.StartElement("NamedMeasurement");
+    xml.Attribute("type", "text/string");
+    xml.Attribute("name", "Environment");
+    xml.Element("Value", result.Environment);
+    xml.EndElement(); // NamedMeasurement
     for (auto const& measure : result.Properties->Measurements) {
       xml.StartElement("NamedMeasurement");
       xml.Attribute("type", "text/string");

Source/CTest/cmCTestTestHandler.h

@@ -169,6 +169,7 @@ public:
     std::string Path;
     std::string Reason;
     std::string FullCommandLine;
+    std::string Environment;
     cmDuration ExecutionTime;
     std::int64_t ReturnValue;
     int Status;

Tests/RunCMake/CTestResourceAllocation/RunCMakeTest.cmake

@@ -38,6 +38,13 @@ function(run_ctresalloc_verify name tests)
   run_cmake_command(${name} "${CTRESALLOC_COMMAND}" verify "${RunCMake_SOURCE_DIR}/${name}.log" "${CMAKE_CURRENT_LIST_DIR}/resspec.json" "${tests}")
 endfunction()
 
+function(read_testing_file filename variable)
+  file(READ "${RunCMake_TEST_BINARY_DIR}/Testing/TAG" _tag)
+  string(REGEX REPLACE "^([^\n]*)\n.*$" "\\1" _date "${_tag}")
+  file(READ "${RunCMake_TEST_BINARY_DIR}/Testing/${_date}/${filename}" _contents)
+  set("${variable}" "${_contents}" PARENT_SCOPE)
+endfunction()
+
 unset(ENV{CTEST_RESOURCE_GROUP_COUNT})
 set(RunCMake_TEST_NO_CLEAN 1)
 file(REMOVE_RECURSE "${RunCMake_BINARY_DIR}/ctresalloc-write-proc-good1-build")

Tests/RunCMake/CTestResourceAllocation/process_count-ctest-s-nores-check.cmake

@@ -0,0 +1,4 @@
+read_testing_file("Test.xml" _test_contents)
+if(NOT _test_contents MATCHES "#CTEST_RESOURCE_GROUP_COUNT=")
+  string(APPEND RunCMake_TEST_FAILED "Could not find unset variable CTEST_RESOURCE_GROUP_COUNT in test measurements\n")
+endif()

Tests/RunCMake/CTestResourceAllocation/process_count-ctest-s-res-check.cmake

@@ -1 +1,9 @@
 verify_ctest_resources()
+
+read_testing_file("Test.xml" _test_contents)
+if(NOT _test_contents MATCHES "\nCTEST_RESOURCE_GROUP_0=widgets")
+  string(APPEND RunCMake_TEST_FAILED "Could not find variable CTEST_RESOURCE_GROUP_0 in test measurements\n")
+endif()
+if(NOT _test_contents MATCHES "\nCTEST_RESOURCE_GROUP_0_WIDGETS=id:")
+  string(APPEND RunCMake_TEST_FAILED "Could not find variable CTEST_RESOURCE_GROUP_0_WIDGETS in test measurements\n")
+endif()

Tests/RunCMake/ctest_test/RunCMakeTest.cmake

@@ -107,3 +107,14 @@ add_test(NAME NotRunTest COMMAND ${CMAKE_COMMAND} -E true)
   run_ctest_test(stop-on-failure STOP_ON_FAILURE)
 endfunction()
 run_stop_on_failure()
+
+# Make sure environment gets logged
+function(run_environment)
+  set(ENV{BAD_ENVIRONMENT_VARIABLE} "Bad environment variable")
+  set(CASE_CMAKELISTS_SUFFIX_CODE [[
+set_property(TEST RunCMakeVersion PROPERTY ENVIRONMENT "ENV1=env1;ENV2=env2")
+  ]])
+
+  run_ctest(TestEnvironment)
+endfunction()
+run_environment()

Tests/RunCMake/ctest_test/TestEnvironment-check.cmake

@@ -0,0 +1,10 @@
+file(READ "${RunCMake_TEST_BINARY_DIR}/Testing/TAG" _tag)
+string(REGEX REPLACE "^([^\n]*)\n.*$" "\\1" _date "${_tag}")
+file(READ "${RunCMake_TEST_BINARY_DIR}/Testing/${_date}/Test.xml" _test_contents)
+
+if(NOT _test_contents MATCHES "<Value>ENV1=env1\nENV2=env2\n#CTEST_RESOURCE_GROUP_COUNT=</Value>")
+  string(APPEND RunCMake_TEST_FAILED "Could not find expected environment variables in Test.xml")
+endif()
+if(_test_contents MATCHES "BAD_ENVIRONMENT_VARIABLE")
+  string(APPEND RunCMake_TEST_FAILED "Found BAD_ENVIRONMENT_VARIABLE in Test.xml")
+endif()