Merge topic 'ctest_measurement_file'

cbcb92d1cb ctest: add support for attaching files to tests at run time

Acked-by: Kitware Robot <[email protected]>
Merge-request: !6192

Help/command/ctest_test.rst

@@ -248,5 +248,14 @@ separate from the interactive comparison UI.
 Attached Files
 """"""""""""""
 
-To associate other types of files with a test, use the
-:prop_test:`ATTACHED_FILES` or :prop_test:`ATTACHED_FILES_ON_FAIL` test properties.
+The following example demonstrates how to upload non-image files to CDash.
+
+.. code-block:: c++
+
+   std::cout <<
+     "<DartMeasurementFile type=\"file\" name=\"MyTestInputData\">" <<
+     "/dir/to/data.csv</DartMeasurementFile>" << std::endl;
+
+If the name of the file to upload is known at configure time, you can use the
+:prop_test:`ATTACHED_FILES` or :prop_test:`ATTACHED_FILES_ON_FAIL` test
+properties instead.

Help/release/dev/ctest-measurements-docs.rst

@@ -0,0 +1,5 @@
+ctest-measurements-docs
+-----------------------
+
+* :manual:`ctest(1)` gained documentation for its ability to capture
+  :ref:`Additional Test Measurements`.

Help/release/dev/ctest-runtime-files.rst

@@ -0,0 +1,8 @@
+ctest-runtime-files
+-------------------
+
+* :manual:`ctest(1)` learned to recognize files attached to a test at run time.
+  Previously it was only possible to attach files to tests at configure time
+  by using the :prop_test:`ATTACHED_FILES` or
+  :prop_test:`ATTACHED_FILES_ON_FAIL` test properties.
+  See :ref:`Additional Test Measurements` for more information.

Source/CTest/cmCTestTestHandler.cxx

@@ -1550,19 +1550,29 @@ void cmCTestTestHandler::AttachFiles(cmXMLWriter& xml,
       result.Properties->AttachOnFail.end());
   }
   for (std::string const& file : result.Properties->AttachedFiles) {
-    const std::string& base64 = this->CTest->Base64GzipEncodeFile(file);
-    std::string const fname = cmSystemTools::GetFilenameName(file);
-    xml.StartElement("NamedMeasurement");
-    xml.Attribute("name", "Attached File");
-    xml.Attribute("encoding", "base64");
-    xml.Attribute("compression", "tar/gzip");
-    xml.Attribute("filename", fname);
-    xml.Attribute("type", "file");
-    xml.Element("Value", base64);
-    xml.EndElement(); // NamedMeasurement
+    this->AttachFile(xml, file, "");
   }
 }
 
+void cmCTestTestHandler::AttachFile(cmXMLWriter& xml, std::string const& file,
+                                    std::string const& name)
+{
+  const std::string& base64 = this->CTest->Base64GzipEncodeFile(file);
+  std::string const fname = cmSystemTools::GetFilenameName(file);
+  xml.StartElement("NamedMeasurement");
+  std::string measurement_name = name;
+  if (measurement_name.empty()) {
+    measurement_name = "Attached File";
+  }
+  xml.Attribute("name", measurement_name);
+  xml.Attribute("encoding", "base64");
+  xml.Attribute("compression", "tar/gzip");
+  xml.Attribute("filename", fname);
+  xml.Attribute("type", "file");
+  xml.Element("Value", base64);
+  xml.EndElement(); // NamedMeasurement
+}
+
 int cmCTestTestHandler::ExecuteCommands(std::vector<std::string>& vec)
 {
   for (std::string const& it : vec) {
@@ -2041,11 +2051,11 @@ void cmCTestTestHandler::GenerateRegressionImages(cmXMLWriter& xml,
         cmCTest::CleanString(measurementfile.match(5));
       if (cmSystemTools::FileExists(filename)) {
         long len = cmSystemTools::FileLength(filename);
+        std::string k1 = measurementfile.match(1);
+        std::string v1 = measurementfile.match(2);
+        std::string k2 = measurementfile.match(3);
+        std::string v2 = measurementfile.match(4);
         if (len == 0) {
-          std::string k1 = measurementfile.match(1);
-          std::string v1 = measurementfile.match(2);
-          std::string k2 = measurementfile.match(3);
-          std::string v2 = measurementfile.match(4);
           if (cmSystemTools::LowerCase(k1) == "type") {
             v1 = "text/string";
           }
@@ -2060,35 +2070,53 @@ void cmCTestTestHandler::GenerateRegressionImages(cmXMLWriter& xml,
           xml.Element("Value", "Image " + filename + " is empty");
           xml.EndElement();
         } else {
-          cmsys::ifstream ifs(filename.c_str(),
-                              std::ios::in
+          std::string type;
+          std::string name;
+          if (cmSystemTools::LowerCase(k1) == "type") {
+            type = v1;
+          } else if (cmSystemTools::LowerCase(k2) == "type") {
+            type = v2;
+          }
+          if (cmSystemTools::LowerCase(k1) == "name") {
+            name = v1;
+          } else if (cmSystemTools::LowerCase(k2) == "name") {
+            name = v2;
+          }
+          if (type == "file") {
+            // Treat this measurement like an "ATTACHED_FILE" when the type
+            // is explicitly "file" (not an image).
+            this->AttachFile(xml, filename, name);
+          } else {
+            cmsys::ifstream ifs(filename.c_str(),
+                                std::ios::in
 #ifdef _WIN32
-                                | std::ios::binary
+                                  | std::ios::binary
 #endif
-          );
-          auto file_buffer = cm::make_unique<unsigned char[]>(len + 1);
-          ifs.read(reinterpret_cast<char*>(file_buffer.get()), len);
-          auto encoded_buffer = cm::make_unique<unsigned char[]>(
-            static_cast<int>(static_cast<double>(len) * 1.5 + 5.0));
-
-          size_t rlen = cmsysBase64_Encode(file_buffer.get(), len,
-                                           encoded_buffer.get(), 1);
-
-          xml.StartElement("NamedMeasurement");
-          xml.Attribute(measurementfile.match(1).c_str(),
-                        measurementfile.match(2));
-          xml.Attribute(measurementfile.match(3).c_str(),
-                        measurementfile.match(4));
-          xml.Attribute("encoding", "base64");
-          std::ostringstream ostr;
-          for (size_t cc = 0; cc < rlen; cc++) {
-            ostr << encoded_buffer[cc];
-            if (cc % 60 == 0 && cc) {
-              ostr << std::endl;
+            );
+            auto file_buffer = cm::make_unique<unsigned char[]>(len + 1);
+            ifs.read(reinterpret_cast<char*>(file_buffer.get()), len);
+            auto encoded_buffer = cm::make_unique<unsigned char[]>(
+              static_cast<int>(static_cast<double>(len) * 1.5 + 5.0));
+
+            size_t rlen = cmsysBase64_Encode(file_buffer.get(), len,
+                                             encoded_buffer.get(), 1);
+
+            xml.StartElement("NamedMeasurement");
+            xml.Attribute(measurementfile.match(1).c_str(),
+                          measurementfile.match(2));
+            xml.Attribute(measurementfile.match(3).c_str(),
+                          measurementfile.match(4));
+            xml.Attribute("encoding", "base64");
+            std::ostringstream ostr;
+            for (size_t cc = 0; cc < rlen; cc++) {
+              ostr << encoded_buffer[cc];
+              if (cc % 60 == 0 && cc) {
+                ostr << std::endl;
+              }
             }
+            xml.Element("Value", ostr.str());
+            xml.EndElement(); // NamedMeasurement
           }
-          xml.Element("Value", ostr.str());
-          xml.EndElement(); // NamedMeasurement
         }
       } else {
         int idx = 4;

Source/CTest/cmCTestTestHandler.h

@@ -237,6 +237,8 @@ protected:
                              cmCTestTestResult const& result);
   // Write attached test files into the xml
   void AttachFiles(cmXMLWriter& xml, cmCTestTestResult& result);
+  void AttachFile(cmXMLWriter& xml, std::string const& file,
+                  std::string const& name);
 
   //! Clean test output to specified length
   void CleanTestOutput(std::string& output, size_t length);

Source/cmCTest.cxx

@@ -1612,8 +1612,33 @@ int cmCTest::GenerateDoneFile()
   return 0;
 }
 
+bool cmCTest::TryToChangeDirectory(std::string const& dir)
+{
+  cmCTestLog(this, OUTPUT,
+             "Internal ctest changing into directory: " << dir << std::endl);
+  cmsys::Status status = cmSystemTools::ChangeDirectory(dir);
+  if (!status) {
+    auto msg = "Failed to change working directory to \"" + dir +
+      "\" : " + status.GetString() + "\n";
+    cmCTestLog(this, ERROR_MESSAGE, msg);
+    return false;
+  }
+  return true;
+}
+
 std::string cmCTest::Base64GzipEncodeFile(std::string const& file)
 {
+  const std::string currDir = cmSystemTools::GetCurrentWorkingDirectory();
+  std::string parentDir = cmSystemTools::GetParentDirectory(file);
+
+  // Temporarily change to the file's directory so the tar gets created
+  // with a flat directory structure.
+  if (currDir != parentDir) {
+    if (!this->TryToChangeDirectory(parentDir)) {
+      return "";
+    }
+  }
+
   std::string tarFile = file + "_temp.tar.gz";
   std::vector<std::string> files;
   files.push_back(file);
@@ -1628,6 +1653,12 @@ std::string cmCTest::Base64GzipEncodeFile(std::string const& file)
   }
   std::string base64 = this->Base64EncodeFile(tarFile);
   cmSystemTools::RemoveFile(tarFile);
+
+  // Change back to the directory we started in.
+  if (currDir != parentDir) {
+    cmSystemTools::ChangeDirectory(currDir);
+  }
+
   return base64;
 }
 
@@ -2853,14 +2884,7 @@ int cmCTest::ExecuteTests()
     }
 
     if (currDir != workDir) {
-      cmCTestLog(this, OUTPUT,
-                 "Internal ctest changing into directory: " << workDir
-                                                            << std::endl);
-      cmsys::Status status = cmSystemTools::ChangeDirectory(workDir);
-      if (!status) {
-        auto msg = "Failed to change working directory to \"" + workDir +
-          "\" : " + status.GetString() + "\n";
-        cmCTestLog(this, ERROR_MESSAGE, msg);
+      if (!this->TryToChangeDirectory(workDir)) {
         return 1;
       }
     }

Source/cmCTest.h

@@ -536,6 +536,9 @@ private:
   int RunCMakeAndTest(std::string* output);
   int ExecuteTests();
 
+  /** return true iff change directory was successful */
+  bool TryToChangeDirectory(std::string const& dir);
+
   struct Private;
   std::unique_ptr<Private> Impl;
 };

Tests/RunCMake/ctest_test/RunCMakeTest.cmake

@@ -161,6 +161,10 @@ add_test(
   NAME img_measurement
   COMMAND ${CMAKE_COMMAND} -E
   echo <DartMeasurementFile name="TestImage" type="image/png">]] ${IMAGE_DIR}/cmake-logo-16.png [[</DartMeasurementFile>)
+add_test(
+  NAME file_measurement
+  COMMAND ${CMAKE_COMMAND} -E
+  echo <DartMeasurementFile name="my_test_input_data" type="file">]] ${IMAGE_DIR}/cmake-logo-16.png [[</DartMeasurementFile>)
   ]])
   run_ctest(TestMeasurements)
 endfunction()

Tests/RunCMake/ctest_test/TestMeasurements-check.cmake

@@ -15,3 +15,8 @@ if(NOT _test_contents MATCHES [[NamedMeasurement name="TestImage" type="image/pn
   string(APPEND RunCMake_TEST_FAILED
     "Could not find expected <NamedMeasurement> tag for type='image/png' in Test.xml")
 endif()
+# Check file measurement.
+if(NOT _test_contents MATCHES [[NamedMeasurement name="my_test_input_data" encoding="base64" compression="tar/gzip" filename="cmake-logo-16.png" type="file"]])
+  string(APPEND RunCMake_TEST_FAILED
+    "Could not find expected <NamedMeasurement> tag for type='file' in Test.xml")
+endif()