Merge topic 'ctest-show-only-json-schema'

2e7bca5f05 ctest: Add JSON schema for --show-only=json-v1 output
fe1cd4e157 Help: Make docs for ctest --show-only=json-v1 reflect the implementation

Acked-by: Kitware Robot <[email protected]>
Merge-request: !10865

Help/manual/ctest.1.rst

@@ -1619,12 +1619,14 @@ model is defined as follows:
   The string "ctestInfo".
 
 ``version``
-  A JSON object specifying the version components.  Its members are
+  A JSON object specifying the version components.  Its members are:
 
   ``major``
-    A non-negative integer specifying the major version component.
+    A positive integer specifying the major version component
+    of the JSON object model.
   ``minor``
-    A non-negative integer specifying the minor version component.
+    A non-negative integer specifying the minor version component
+    of the JSON object model.
 
 ``backtraceGraph``
     JSON object representing backtrace information with the
@@ -1638,32 +1640,54 @@ model is defined as follows:
       List of node JSON objects with members:
 
       ``command``
-        Index into the ``commands`` member of the ``backtraceGraph``.
+        An optional member present when the node represents a command
+        invocation within the file.  The value is an unsigned integer 0-based
+        index into the ``commands`` member of the ``backtraceGraph``.
       ``file``
-        Index into the ``files`` member of the ``backtraceGraph``.
+        An unsigned integer 0-based index into the ``files`` member of the
+        ``backtraceGraph``.
       ``line``
-        Line number in the file where the backtrace was added.
+        An optional member present when the node represents a line within
+        the file.  The value is an unsigned integer 1-based line number
+        in the file where the backtrace was added.
       ``parent``
-        Index into the ``nodes`` member of the ``backtraceGraph``
-        representing the parent in the graph.
+        An optional member present when the node is not the bottom of the
+        call stack.  The value is an unsigned integer 0-based index into the
+        ``nodes`` member of the ``backtraceGraph`` representing the parent
+        in the graph.
 
 ``tests``
   A JSON array listing information about each test.  Each entry
   is a JSON object with members:
 
   ``name``
-    Test name.
+    Test name. This cannot be empty.
   ``config``
-    Configuration that the test can run on.
-    Empty string means any config.
+    Optional field specifying the configuration for which the test will run.
+    This will always match the :option:`-C <ctest -C>` option specified on the
+    ``ctest`` command line.  If no such option was given, this field will not
+    be present.
   ``command``
-    List where the first element is the test command and the
-    remaining elements are the command arguments.
+    Optional array where the first element is the test command and the
+    remaining elements are the command arguments.  Normally, this field should
+    be present and non-empty, but in certain corner cases involving generator
+    expressions, it is possible for a test to have no command and therefore
+    this field can be missing.
   ``backtrace``
     Index into the ``nodes`` member of the ``backtraceGraph``.
   ``properties``
-    Test properties.
-    Can contain keys for each of the supported test properties.
+    Optional array of test properties.
+    Each array item will be a JSON object with the following members:
+
+    ``name``
+      The name of the test property. This cannot be empty.
+    ``value``
+      The property value, which can be a string, a number, a boolean, or an
+      array of strings.
+
+.. versionadded:: 4.1
+  The JSON output format is described in machine-readable form by
+  :download:`this JSON schema </manual/ctest/show-only-schema.json>`.
 
 .. _`ctest-resource-allocation`:
 

Help/manual/ctest/show-only-schema.json

@@ -0,0 +1,121 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "type": "object",
+  "required": ["kind", "version", "backtraceGraph", "tests"],
+  "properties": {
+    "kind": {
+      "type": "string",
+      "const": "ctestInfo"
+    },
+    "version": {
+      "type": "object",
+      "required": ["major", "minor"],
+      "properties": {
+        "major": {
+          "const": 1,
+          "description": "A positive integer specifying the major version component of the JSON object model."
+        },
+        "minor": {
+          "const": 0,
+          "description": "A non-negative integer specifying the minor version component of the JSON object model."
+        }
+      }
+    },
+    "backtraceGraph": {
+      "type": "object",
+      "required": ["commands", "files", "nodes"],
+      "properties": {
+        "commands": {
+          "type": "array",
+          "description": "List of CMake command names.",
+          "items": {
+            "type": "string"
+          }
+        },
+        "files": {
+          "type": "array",
+          "description": "List of file paths, which may be relative or absolute. Relative paths are relative to the top-level source directory.",
+          "items": {
+            "type": "string"
+          }
+        },
+        "nodes": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "required": ["file"],
+            "properties": {
+              "command": {
+                "type": "integer",
+                "minimum": 0,
+                "description": "An optional member present when the node represents a command invocation within the file. The value is an unsigned integer 0-based index into the commands member of the backtraceGraph."
+              },
+              "file": {
+                "type": "integer",
+                "minimum": 0,
+                "description": "An unsigned integer 0-based index into the files member of the backtraceGraph."
+              },
+              "line": {
+                "type": "integer",
+                "minimum": 1,
+                "description": "An optional member present when the node represents a line within the file. The value is an unsigned integer 1-based line number in the file where the backtrace was added."
+              },
+              "parent": {
+                "type": "integer",
+                "minimum": 0,
+                "description": "An optional member present when the node is not the bottom of the call stack. The value is an unsigned integer 0-based index into the nodes member of the backtraceGraph representing the parent in the graph."
+              }
+            }
+          }
+        }
+      }
+    },
+    "tests": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "required": ["name", "backtrace"],
+        "properties": {
+          "name": {
+            "type": "string",
+            "description": "Test name",
+            "minLength": 1
+          },
+          "config": {
+            "type": "string",
+            "description": "Optional field specifying the configuration for which the test will run. This will always match the -C option specified on the ctest command line. If no such option was given, this field will not be present."
+          },
+          "command": {
+            "type": "array",
+            "items": {
+              "type": "string"
+            },
+            "minItems": 1,
+            "description": "Optional array where the first element is the test command and the remaining elements are the command arguments. Normally, this field should be present and non-empty, but in certain corner cases involving generator expressions, it is possible for a test to have no command and therefore this field can be missing."
+          },
+          "backtrace": {
+            "type": "integer",
+            "description": "Index into the nodes member of the backtraceGraph."
+          },
+          "properties": {
+            "type": "array",
+            "description": "Optional list of test properties associated with the test.",
+            "items": {
+              "type": "object",
+              "properties": {
+                "name": {
+                  "type": "string",
+                  "description": "Test property name.",
+                  "minLength": 1
+                },
+                "value": {
+                  "description": "Value of the test property. Any valid JSON type might be present."
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+}

Tests/RunCMake/CMakeLists.txt

@@ -1084,7 +1084,10 @@ if(CMake_TEST_RunCMake_ExternalProject_RUN_SERIAL)
 endif()
 add_RunCMake_test(FetchContent)
 add_RunCMake_test(FetchContent_find_package)
-set(CTestCommandLine_ARGS -DPython_EXECUTABLE=${Python_EXECUTABLE})
+set(CTestCommandLine_ARGS
+  -DPython_EXECUTABLE=${Python_EXECUTABLE}
+  -DCMake_TEST_JSON_SCHEMA=${CMake_TEST_JSON_SCHEMA}
+)
 if(NOT CMake_TEST_EXTERNAL_CMAKE)
   list(APPEND CTestCommandLine_ARGS -DTEST_AFFINITY=$<TARGET_FILE:testAffinity>)
 endif()

Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake

@@ -1,5 +1,6 @@
 include(RunCMake)
 include(RunCTest)
+cmake_policy(SET CMP0140 NEW)
 
 # Do not use any proxy for lookup of an invalid site.
 # DNS failure by proxy looks different than DNS failure without proxy.
@@ -449,6 +450,20 @@ function(show_only_json_check_python v)
   set(json_file "${RunCMake_TEST_BINARY_DIR}/ctest.json")
   file(WRITE "${json_file}" "${actual_stdout}")
   set(actual_stdout "" PARENT_SCOPE)
+
+  if(CMake_TEST_JSON_SCHEMA)
+    execute_process(
+      COMMAND ${Python_EXECUTABLE} "${RunCMake_SOURCE_DIR}/show-only_json_validate_schema.py" "${json_file}"
+      RESULT_VARIABLE result
+      OUTPUT_VARIABLE output
+      ERROR_VARIABLE output
+    )
+    if(NOT result STREQUAL 0)
+      string(REPLACE "\n" "\n  " output "${output}")
+      string(APPEND RunCMake_TEST_FAILED "Failed to validate version ${v} JSON schema for file: ${file}\nOutput:\n${output}\n")
+    endif()
+  endif()
+
   execute_process(
     COMMAND ${Python_EXECUTABLE} "${RunCMake_SOURCE_DIR}/show-only_json-v${v}_check.py" "${json_file}"
     RESULT_VARIABLE result
@@ -457,8 +472,9 @@ function(show_only_json_check_python v)
     )
   if(NOT result EQUAL 0)
     string(REPLACE "\n" "\n  " output "  ${output}")
-    set(RunCMake_TEST_FAILED "Unexpected output:\n${output}" PARENT_SCOPE)
+    string(APPEND RunCMake_TEST_FAILED "Unexpected output:\n${output}" PARENT_SCOPE)
   endif()
+  return(PROPAGATE RunCMake_TEST_FAILED)
 endfunction()
 
 function(run_ShowOnly)

Tests/RunCMake/CTestCommandLine/show-only_json_validate_schema.py

@@ -0,0 +1,16 @@
+import json
+import jsonschema
+import os.path
+import sys
+
+
+with open(sys.argv[1], "r", encoding="utf-8-sig") as f:
+    contents = json.load(f)
+
+schema_file = os.path.join(
+    os.path.dirname(__file__),
+    "..", "..", "..", "Help", "manual", "ctest", "show-only-schema.json")
+with open(schema_file, "r", encoding="utf-8") as f:
+    schema = json.load(f)
+
+jsonschema.validate(contents, schema)