Merge topic 'cmake-presets-output-junit'

bea4ed5430 CTest: Add support for outputJUnitFile in presets
9270a02003 CMakePresets.json: Add outputJUnitFile to test presets schema
757786bb73 Tests: Add test for outputLogFile in CMakePresets.json
b68c3596e7 CMakePresets.json: Disallow extra properties in test output schema

Acked-by: Kitware Robot <[email protected]>
Acked-by: scivision <[email protected]>
Merge-request: !7806

Help/manual/cmake-presets.7.rst

@@ -657,6 +657,12 @@ that may contain the following fields:
     passing :option:`--output-log <ctest --output-log>` on the command line.
     This field supports macro expansion.
 
+  ``outputJUnitFile``
+    An optional string specifying a path to a JUnit file. Equivalent to
+    passing :option:`--output-junit <ctest --output-junit>` on the command line.
+    This field supports macro expansion. This is allowed in preset files
+    specifying version ``6`` or above.
+
   ``labelSummary``
     An optional bool. If false, equivalent to passing
     :option:`--no-label-summary <ctest --no-label-summary>` on the command

Help/manual/presets/schema.json

@@ -83,7 +83,7 @@
         "vendor": { "$ref": "#/definitions/vendor" },
         "configurePresets": { "$ref": "#/definitions/configurePresetsV3"},
         "buildPresets": { "$ref": "#/definitions/buildPresetsV4"},
-        "testPresets": { "$ref": "#/definitions/testPresetsV5"},
+        "testPresets": { "$ref": "#/definitions/testPresetsV6"},
         "packagePresets": { "$ref": "#/definitions/packagePresetsV6"},
         "workflowPresets": { "$ref": "#/definitions/workflowPresetsV6" },
         "include": { "$ref": "#/definitions/include"}
@@ -705,6 +705,25 @@
         "additionalProperties": false
       }
     },
+    "testPresetsItemsV6": {
+      "type": "array",
+      "description": "An optional array of test preset objects. Used to specify arguments to ctest. Available in version 6 and higher.",
+      "items": {
+        "type": "object",
+        "properties": {
+          "output": {
+            "type": "object",
+            "description": "An optional object specifying output options.",
+            "properties": {
+              "outputJUnitFile": {
+                "type": "string",
+                "description": "An optional string specifying a path to a JUnit file. Equivalent to passing --output-junit on the command line."
+              }
+            }
+          }
+        }
+      }
+    },
     "testPresetsItemsV5": {
       "type": "array",
       "description": "An optional array of test preset objects. Used to specify arguments to ctest. Available in version 5 and higher.",
@@ -1051,6 +1070,58 @@
         ]
       }
     },
+    "testPresetsV6": {
+      "type": "array",
+      "description": "An optional array of test preset objects. Used to specify arguments to ctest. Available in version 6 and higher.",
+      "allOf": [
+        { "$ref": "#/definitions/testPresetsItemsV2" },
+        { "$ref": "#/definitions/testPresetsItemsV3" },
+        { "$ref": "#/definitions/testPresetsItemsV5" },
+        { "$ref": "#/definitions/testPresetsItemsV6" }
+      ],
+      "items": {
+        "type": "object",
+        "properties": {
+          "name": {},
+          "hidden": {},
+          "inherits": {},
+          "configurePreset": {},
+          "vendor": {},
+          "displayName": {},
+          "description": {},
+          "inheritConfigureEnvironment": {},
+          "environment": {},
+          "configuration": {},
+          "overwriteConfigurationFile": {},
+          "output": {
+            "type": "object",
+            "properties": {
+              "shortProgress": {},
+              "verbosity": {},
+              "debug": {},
+              "outputOnFailure": {},
+              "quiet": {},
+              "outputLogFile": {},
+              "outputJUnitFile": {},
+              "labelSummary": {},
+              "subprojectSummary": {},
+              "maxPassedTestOutputSize": {},
+              "maxFailedTestOutputSize": {},
+              "maxTestNameWidth": {},
+              "testOutputTruncation": {}
+            },
+            "additionalProperties": false
+          },
+          "filter": {},
+          "execution": {},
+          "condition": {}
+        },
+        "required": [
+          "name"
+        ],
+        "additionalProperties": false
+      }
+    },
     "testPresetsV5": {
       "type": "array",
       "description": "An optional array of test preset objects. Used to specify arguments to ctest. Available in version 5 and higher.",
@@ -1073,7 +1144,24 @@
           "environment": {},
           "configuration": {},
           "overwriteConfigurationFile": {},
-          "output": {},
+          "output": {
+            "type": "object",
+            "properties": {
+              "shortProgress": {},
+              "verbosity": {},
+              "debug": {},
+              "outputOnFailure": {},
+              "quiet": {},
+              "outputLogFile": {},
+              "labelSummary": {},
+              "subprojectSummary": {},
+              "maxPassedTestOutputSize": {},
+              "maxFailedTestOutputSize": {},
+              "maxTestNameWidth": {},
+              "testOutputTruncation": {}
+            },
+            "additionalProperties": false
+          },
           "filter": {},
           "execution": {},
           "condition": {}
@@ -1105,7 +1193,23 @@
           "environment": {},
           "configuration": {},
           "overwriteConfigurationFile": {},
-          "output": {},
+          "output": {
+            "type": "object",
+            "properties": {
+              "shortProgress": {},
+              "verbosity": {},
+              "debug": {},
+              "outputOnFailure": {},
+              "quiet": {},
+              "outputLogFile": {},
+              "labelSummary": {},
+              "subprojectSummary": {},
+              "maxPassedTestOutputSize": {},
+              "maxFailedTestOutputSize": {},
+              "maxTestNameWidth": {}
+            },
+            "additionalProperties": false
+          },
           "filter": {},
           "execution": {},
           "condition": {}
@@ -1136,7 +1240,23 @@
           "environment": {},
           "configuration": {},
           "overwriteConfigurationFile": {},
-          "output": {},
+          "output": {
+            "type": "object",
+            "properties": {
+              "shortProgress": {},
+              "verbosity": {},
+              "debug": {},
+              "outputOnFailure": {},
+              "quiet": {},
+              "outputLogFile": {},
+              "labelSummary": {},
+              "subprojectSummary": {},
+              "maxPassedTestOutputSize": {},
+              "maxFailedTestOutputSize": {},
+              "maxTestNameWidth": {}
+            },
+            "additionalProperties": false
+          },
           "filter": {},
           "execution": {}
         },

Help/release/3.25.rst

@@ -21,6 +21,9 @@ Presets
 * The :manual:`cmake-presets(7)` format now supports a
   ``workflowPresets`` field to specify presets for :option:`cmake --workflow`.
 
+* The :manual:`cmake-presets(7)` format now supports a
+  ``outputJUnitFile`` field to specify JUnit output in test presets.
+
 Languages
 ---------
 

Source/cmCMakePresetsGraph.cxx

@@ -265,6 +265,8 @@ bool ExpandMacros(const cmCMakePresetsGraph& graph, const TestPreset& preset,
   if (out->Output) {
     CHECK_EXPAND(out, out->Output->OutputLogFile, macroExpanders,
                  graph.GetVersion(preset));
+    CHECK_EXPAND(out, out->Output->OutputJUnitFile, macroExpanders,
+                 graph.GetVersion(preset));
   }
 
   if (out->Filter) {
@@ -851,6 +853,7 @@ cmCMakePresetsGraph::TestPreset::VisitPresetInherit(
                            parentOutput.OutputOnFailure);
       InheritOptionalValue(output.Quiet, parentOutput.Quiet);
       InheritString(output.OutputLogFile, parentOutput.OutputLogFile);
+      InheritString(output.OutputJUnitFile, parentOutput.OutputJUnitFile);
       InheritOptionalValue(output.LabelSummary, parentOutput.LabelSummary);
       InheritOptionalValue(output.SubprojectSummary,
                            parentOutput.SubprojectSummary);
@@ -1253,6 +1256,8 @@ const char* cmCMakePresetsGraph::ResultToString(ReadFileResult result)
       return "Invalid workflow steps";
     case ReadFileResult::WORKFLOW_STEP_UNREACHABLE_FROM_FILE:
       return "Workflow step is unreachable from preset's file";
+    case ReadFileResult::CTEST_JUNIT_UNSUPPORTED:
+      return "File version must be 6 or higher for CTest JUnit output support";
   }
 
   return "Unknown error";

Source/cmCMakePresetsGraph.h

@@ -54,6 +54,7 @@ public:
     TEST_OUTPUT_TRUNCATION_UNSUPPORTED,
     INVALID_WORKFLOW_STEPS,
     WORKFLOW_STEP_UNREACHABLE_FROM_FILE,
+    CTEST_JUNIT_UNSUPPORTED,
   };
 
   std::string errors;
@@ -230,6 +231,7 @@ public:
       cm::optional<bool> OutputOnFailure;
       cm::optional<bool> Quiet;
       std::string OutputLogFile;
+      std::string OutputJUnitFile;
       cm::optional<bool> LabelSummary;
       cm::optional<bool> SubprojectSummary;
       cm::optional<int> MaxPassedTestOutputSize;

Source/cmCMakePresetsGraphReadJSON.cxx

@@ -588,6 +588,11 @@ cmCMakePresetsGraph::ReadFileResult cmCMakePresetsGraph::ReadJSONFile(
       return ReadFileResult::TEST_OUTPUT_TRUNCATION_UNSUPPORTED;
     }
 
+    // Support for outputJUnitFile added in version 6.
+    if (v < 6 && preset.Output && !preset.Output->OutputJUnitFile.empty()) {
+      return ReadFileResult::CTEST_JUNIT_UNSUPPORTED;
+    }
+
     this->TestPresetOrder.push_back(preset.Name);
   }
 

Source/cmCMakePresetsGraphReadJSONTestPresets.cxx

@@ -104,6 +104,8 @@ auto const TestPresetOptionalOutputHelper =
             cmCMakePresetsGraphInternal::PresetOptionalBoolHelper, false)
       .Bind("outputLogFile"_s, &TestPreset::OutputOptions::OutputLogFile,
             cmCMakePresetsGraphInternal::PresetStringHelper, false)
+      .Bind("outputJUnitFile"_s, &TestPreset::OutputOptions::OutputJUnitFile,
+            cmCMakePresetsGraphInternal::PresetStringHelper, false)
       .Bind("labelSummary"_s, &TestPreset::OutputOptions::LabelSummary,
             cmCMakePresetsGraphInternal::PresetOptionalBoolHelper, false)
       .Bind("subprojectSummary"_s,

Source/cmCTest.cxx

@@ -2116,11 +2116,7 @@ bool cmCTest::HandleCommandLineArguments(size_t& i,
       return false;
     }
     i++;
-    this->Impl->TestHandler.SetJUnitXMLFileName(std::string(args[i]));
-    // Turn test output compression off.
-    // This makes it easier to include test output in the resulting
-    // JUnit XML report.
-    this->Impl->CompressTestOutput = false;
+    this->SetOutputJUnitFileName(std::string(args[i]));
   }
 
   cm::string_view noTestsPrefix = "--no-tests=";
@@ -2458,6 +2454,9 @@ bool cmCTest::SetArgsFromPreset(const std::string& presetName,
     if (!expandedPreset->Output->OutputLogFile.empty()) {
       this->SetOutputLogFileName(expandedPreset->Output->OutputLogFile);
     }
+    if (!expandedPreset->Output->OutputJUnitFile.empty()) {
+      this->SetOutputJUnitFileName(expandedPreset->Output->OutputJUnitFile);
+    }
 
     this->Impl->LabelSummary =
       expandedPreset->Output->LabelSummary.value_or(true);
@@ -3541,6 +3540,15 @@ void cmCTest::SetOutputLogFileName(const std::string& name)
   }
 }
 
+void cmCTest::SetOutputJUnitFileName(const std::string& name)
+{
+  this->Impl->TestHandler.SetJUnitXMLFileName(name);
+  // Turn test output compression off.
+  // This makes it easier to include test output in the resulting
+  // JUnit XML report.
+  this->Impl->CompressTestOutput = false;
+}
+
 static const char* cmCTestStringLogType[] = { "DEBUG",
                                               "OUTPUT",
                                               "HANDLER_OUTPUT",

Source/cmCTest.h

@@ -359,6 +359,9 @@ public:
   /** Set the output log file name */
   void SetOutputLogFileName(const std::string& name);
 
+  /** Set the output JUnit file name */
+  void SetOutputJUnitFileName(const std::string& name);
+
   /** Set the visual studio or Xcode config type */
   void SetConfigType(const std::string& ct);
 

Tests/RunCMake/CMakePresetsTest/Good-test-outputJUnit-check.cmake

@@ -0,0 +1,4 @@
+include("${CMAKE_CURRENT_LIST_DIR}/check.cmake")
+if(NOT EXISTS "${RunCMake_TEST_BINARY_DIR}/default/output.xml")
+  string(APPEND RunCMake_TEST_FAILED "Expected ${RunCMake_TEST_BINARY_DIR}/default/output.xml to exist but it does not\n")
+endif()

Tests/RunCMake/CMakePresetsTest/Good-test-outputLog-check.cmake

@@ -0,0 +1,4 @@
+include("${CMAKE_CURRENT_LIST_DIR}/check.cmake")
+if(NOT EXISTS "${RunCMake_TEST_BINARY_DIR}/default/output.log")
+  string(APPEND RunCMake_TEST_FAILED "Expected ${RunCMake_TEST_BINARY_DIR}/default/output.log to exist but it does not\n")
+endif()

Tests/RunCMake/CMakePresetsTest/Good.json.in

@@ -1,5 +1,5 @@
 {
-    "version": 5,
+    "version": 6,
     "configurePresets": [
         {
             "name": "default",
@@ -171,6 +171,20 @@
             "execution": {
                 "showOnly": "human"
             }
+        },
+        {
+            "name": "outputLog",
+            "inherits": "minimal",
+            "output": {
+                "outputLogFile": "${sourceDir}/build/default/output.log"
+            }
+        },
+        {
+            "name": "outputJUnit",
+            "inherits": "minimal",
+            "output": {
+                "outputJUnitFile": "${sourceDir}/build/default/output.xml"
+            }
         }
     ]
 }

Tests/RunCMake/CMakePresetsTest/OutputJUnitUnsupported-test-x-result.txt

@@ -0,0 +1 @@
+1

Tests/RunCMake/CMakePresetsTest/OutputJUnitUnsupported-test-x-stderr.txt

@@ -0,0 +1,2 @@
+^CMake Error: Could not read presets from [^
+]*/Tests/RunCMake/CMakePresetsTest/OutputJUnitUnsupported: File version must be 6 or higher for CTest JUnit output support$

Tests/RunCMake/CMakePresetsTest/OutputJUnitUnsupported.json.in

@@ -0,0 +1,17 @@
+{
+  "version": 5,
+  "configurePresets": [
+    {
+      "name": "default"
+    }
+  ],
+  "testPresets": [
+    {
+      "name": "unsupported",
+      "configurePreset": "default",
+      "output": {
+        "outputJUnitFile": "junit.xml"
+      }
+    }
+  ]
+}

Tests/RunCMake/CMakePresetsTest/RunCMakeTest.cmake

@@ -78,7 +78,7 @@ set(CMakePresetsTest_ASSETS "Good-indexFile.txt")
 set(GoodTestPresets
   "minimal;defaults;noEnvironment;withEnvironment"
   "config-debug;config-release"
-  "exclude;index;indexFile;showOnly")
+  "exclude;index;indexFile;showOnly;outputLog;outputJUnit")
 run_cmake_test_presets(Good
                        "default"
                        ""
@@ -105,6 +105,8 @@ run_cmake_test_presets(Invalid "" "" "hidden;vendorMacro")
 set(CMakePresets_SCHEMA_EXPECTED_RESULT 1)
 run_cmake_test_presets(PresetsUnsupported "" "" "x")
 run_cmake_test_presets(ConditionFuture "" "" "x")
+run_cmake_test_presets(TestOutputTruncationUnsupported "" "" "x")
+run_cmake_test_presets(OutputJUnitUnsupported "" "" "x")
 set(CMakePresets_SCHEMA_EXPECTED_RESULT 0)
 run_cmake_test_presets(ConfigurePresetUnreachable "" "" "x")
 set(CMakePresetsTest_NO_CONFIGURE 0)

Tests/RunCMake/CMakePresetsTest/TestOutputTruncationUnsupported-test-x-result.txt

@@ -0,0 +1 @@
+1

Tests/RunCMake/CMakePresetsTest/TestOutputTruncationUnsupported-test-x-stderr.txt

@@ -0,0 +1,2 @@
+^CMake Error: Could not read presets from [^
+]*/Tests/RunCMake/CMakePresetsTest/TestOutputTruncationUnsupported: File version must be 5 or higher for testOutputTruncation preset support\.$

Tests/RunCMake/CMakePresetsTest/TestOutputTruncationUnsupported.json.in

@@ -0,0 +1,17 @@
+{
+  "version": 4,
+  "configurePresets": [
+    {
+      "name": "default"
+    }
+  ],
+  "testPresets": [
+    {
+      "name": "default",
+      "configurePreset": "default",
+      "output": {
+        "testOutputTruncation": "tail"
+      }
+    }
+  ]
+}