Merge topic 'ctest_test_measurements_docs'

b60789a758 Help: Document CTest custom test measurements
63b5ddcce2 Tests: Add cases for CTest extra measurements from tests
52eac4573d Help: Fix link to cdash.org from CTest manual

Acked-by: Kitware Robot <[email protected]>
Acked-by: Zack Galbreath <[email protected]>
Merge-request: !6029

Help/command/ctest_test.rst

@@ -170,3 +170,83 @@ The options are:
 
 See also the :variable:`CTEST_CUSTOM_MAXIMUM_PASSED_TEST_OUTPUT_SIZE`
 and :variable:`CTEST_CUSTOM_MAXIMUM_FAILED_TEST_OUTPUT_SIZE` variables.
+
+.. _`Additional Test Measurements`:
+
+Additional Test Measurements
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+CTest can parse the output of your tests for extra measurements to report
+to CDash.
+
+When run as a :ref:`Dashboard Client`, CTest will include these custom
+measurements in the ``Test.xml`` file that gets uploaded to CDash.
+
+Check the `CDash test measurement documentation
+<https://github.com/Kitware/CDash/blob/master/docs/test_measurements.md>`_
+for more information on the types of test measurements that CDash recognizes.
+
+The following example demonstrates how to output a variety of custom test
+measurements.
+
+.. code-block:: c++
+
+   std::cout <<
+     "<DartMeasurement type=\"numeric/double\" name=\"score\">28.3</DartMeasurement>"
+     << std::endl;
+
+   std::cout <<
+     "<DartMeasurement type=\"text/string\" name=\"color\">red</DartMeasurement>"
+     << std::endl;
+
+   std::cout <<
+     "<DartMeasurement type=\"text/link\" name=\"CMake URL\">https://cmake.org</DartMeasurement>"
+     << std::endl;
+
+   std::cout <<
+     "<DartMeasurement type=\"text/preformatted\" name=\"Console Output\">" <<
+     "line 1.\n" <<
+     "  \033[31;1m line 2. Bold red, and indented!\033[0;0ml\n" <<
+     "line 3. Not bold or indented...\n" <<
+     "</DartMeasurement>" << std::endl;
+
+Image Measurements
+""""""""""""""""""
+
+The following example demonstrates how to upload test images to CDash.
+
+.. code-block:: c++
+
+   std::cout <<
+     "<DartMeasurementFile type=\"image/jpg\" name=\"TestImage\">" <<
+     "/dir/to/test_img.jpg</DartMeasurementFile>" << std::endl;
+
+   std::cout <<
+     "<DartMeasurementFile type=\"image/gif\" name=\"ValidImage\">" <<
+     "/dir/to/valid_img.gif</DartMeasurementFile>" << std::endl;
+
+   std::cout <<
+     "<DartMeasurementFile type=\"image/png\" name=\"AlgoResult\"> <<
+     "/dir/to/img.png</DartMeasurementFile>"
+     << std::endl;
+
+Images will be displayed together in an interactive comparison mode on CDash
+if they are provided with two or more of the following names.
+
+* ``TestImage``
+* ``ValidImage``
+* ``BaselineImage``
+* ``DifferenceImage2``
+
+By convention, ``TestImage`` is the image generated by your test, and
+``ValidImage`` (or ``BaselineImage``) is basis of comparison used to determine
+if the test passed or failed.
+
+If another image name is used it will be displayed by CDash as a static image
+separate from the interactive comparison UI.
+
+Attached Files
+""""""""""""""
+
+To associate other types of files with a test, use the
+:prop_test:`ATTACHED_FILES` or :prop_test:`ATTACHED_FILES_ON_FAIL` test properties.

Help/manual/ctest.1.rst

@@ -1095,6 +1095,8 @@ Configuration settings include:
   * `CTest Script`_ variable: :variable:`CTEST_TEST_TIMEOUT`
   * :module:`CTest` module variable: ``DART_TESTING_TIMEOUT``
 
+To report extra test values to CDash, see :ref:`Additional Test Measurements`.
+
 .. _`CTest Coverage Step`:
 
 CTest Coverage Step
@@ -1671,4 +1673,4 @@ See Also
 
 .. include:: LINKS.txt
 
-.. _`CDash`: http://cdash.org/
+_`CDash`: https://cdash.org

Tests/RunCMake/CMakeLists.txt

@@ -361,7 +361,9 @@ if(COVERAGE_COMMAND)
 endif()
 add_RunCMake_test(ctest_start)
 add_RunCMake_test(ctest_submit)
-add_RunCMake_test(ctest_test)
+add_RunCMake_test(ctest_test
+  -DIMAGE_DIR=${CMAKE_SOURCE_DIR}/Utilities/Sphinx/static
+)
 add_RunCMake_test(ctest_disabled_test)
 add_RunCMake_test(ctest_skipped_test)
 add_RunCMake_test(ctest_update)

Tests/RunCMake/ctest_test/RunCMakeTest.cmake

@@ -149,3 +149,19 @@ run_environment()
 
 # test for OUTPUT_JUNIT
 run_ctest_test(OutputJUnit OUTPUT_JUNIT junit.xml REPEAT UNTIL_FAIL:2)
+
+# Verify that extra measurements get reported.
+function(run_measurements)
+  set(CASE_CMAKELISTS_SUFFIX_CODE [[
+add_test(
+  NAME double_measurement
+  COMMAND ${CMAKE_COMMAND} -E
+  echo <DartMeasurement type="numeric/double" name="my_custom_value">1.4847</DartMeasurement>)
+add_test(
+  NAME img_measurement
+  COMMAND ${CMAKE_COMMAND} -E
+  echo <DartMeasurementFile name="TestImage" type="image/png">]] ${IMAGE_DIR}/cmake-logo-16.png [[</DartMeasurementFile>)
+  ]])
+  run_ctest(TestMeasurements)
+endfunction()
+run_measurements()

Tests/RunCMake/ctest_test/TestMeasurements-check.cmake

@@ -0,0 +1,17 @@
+file(READ "${RunCMake_TEST_BINARY_DIR}/Testing/TAG" _tag)
+string(REGEX REPLACE "^([^\n]*)\n.*$" "\\1" _date "${_tag}")
+file(READ "${RunCMake_TEST_BINARY_DIR}/Testing/${_date}/Test.xml" _test_contents)
+
+# Check double measurement.
+if(NOT _test_contents MATCHES [[NamedMeasurement type="numeric/double" name="my_custom_value"]])
+  string(APPEND RunCMake_TEST_FAILED
+    "Could not find expected <NamedMeasurement> tag for type='numeric/double' in Test.xml")
+endif()
+if(NOT _test_contents MATCHES "<Value>1.4847</Value>")
+  string(APPEND RunCMake_TEST_FAILED "Could not find expected measurement value in Test.xml")
+endif()
+# Check img measurement.
+if(NOT _test_contents MATCHES [[NamedMeasurement name="TestImage" type="image/png" encoding="base64"]])
+  string(APPEND RunCMake_TEST_FAILED
+    "Could not find expected <NamedMeasurement> tag for type='image/png' in Test.xml")
+endif()