CTest: Add --show-only[=format] option to print test info

format can be 'human' to print the current text format or 'json-v1' to
print the test object model in json format and is useful for IDEs who
want to gather information about the tests. Defaults to 'human' format.

Help/manual/ctest.1.rst

@@ -109,13 +109,23 @@ Options
 
  This option tells CTest to write all its output to a log file.
 
-``-N,--show-only``
+``-N,--show-only[=<format>]``
  Disable actual execution of tests.
 
  This option tells CTest to list the tests that would be run but not
  actually run them.  Useful in conjunction with the ``-R`` and ``-E``
  options.
 
+ ``<format>`` can be one of the following values.
+
+   ``human``
+     Human-friendly output.  This is not guaranteed to be stable.
+     This is the default.
+
+   ``json-v1``
+     Dump the test information in JSON format.
+     See `Show as JSON Object Model`_.
+
 ``-L <regex>, --label-regex <regex>``
  Run tests with labels matching regular expression.
 
@@ -1163,6 +1173,65 @@ Configuration settings include:
   * :module:`CTest` module variable: ``TRIGGER_SITE`` if set,
     else ``CTEST_TRIGGER_SITE``
 
+.. _`Show as JSON Object Model`:
+
+Show as JSON Object Model
+=========================
+
+When the ``--show-only=json-v1`` command line option is given, the test
+information is output in JSON format.  Version 1.0 of the JSON object
+model is defined as follows:
+
+``kind``
+  The string "ctestInfo".
+
+``version``
+  A JSON object specifying the version components.  Its members are
+
+  ``major``
+    A non-negative integer specifying the major version component.
+  ``minor``
+    A non-negative integer specifying the minor version component.
+
+``backtraceGraph``
+    JSON object representing backtrace information with the
+    following members:
+
+    ``commands``
+      List of command names.
+    ``files``
+      List of file names.
+    ``nodes``
+      List of node JSON objects with members:
+
+      ``command``
+        Index into the ``commands`` member of the ``backtraceGraph``.
+      ``file``
+        Index into the ``files`` member of the ``backtraceGraph``.
+      ``line``
+        Line number in the file where the backtrace was added.
+      ``parent``
+        Index into the ``nodes`` member of the ``backtraceGraph``
+        representing the parent in the graph.
+
+``tests``
+  A JSON array listing information about each test.  Each entry
+  is a JSON object with members:
+
+  ``name``
+    Test name.
+  ``config``
+    Configuration that the test can run on.
+    Empty string means any config.
+  ``command``
+    List where the first element is the test command and the
+    remaining elements are the command arguments.
+  ``backtrace``
+    Index into the ``nodes`` member of the ``backtraceGraph``.
+  ``properties``
+    Test properties.
+    Can contain keys for each of the supported test properties.
+
 See Also
 ========
 

Help/release/dev/ctest-show-only-json-v1.rst

@@ -0,0 +1,6 @@
+ctest-show-only-json-v1
+-----------------------
+
+* :manual:`ctest(1)` gained a ``--show-only=json-v1`` option to show the
+  list of tests in a machine-readable JSON format.
+  See the :ref:`Show as JSON Object Model` section of the manual.

Source/CTest/cmCTestMultiProcessHandler.cxx

@@ -6,9 +6,13 @@
 #include "cmCTest.h"
 #include "cmCTestRunTest.h"
 #include "cmCTestTestHandler.h"
+#include "cmDuration.h"
+#include "cmListFileCache.h"
 #include "cmSystemTools.h"
 #include "cmWorkingDirectory.h"
 
+#include "cm_jsoncpp_value.h"
+#include "cm_jsoncpp_writer.h"
 #include "cm_uv.h"
 
 #include "cmUVSignalHackRAII.h" // IWYU pragma: keep
@@ -20,13 +24,19 @@
 #include <chrono>
 #include <cstring>
 #include <iomanip>
+#include <iostream>
 #include <list>
 #include <math.h>
 #include <sstream>
 #include <stack>
 #include <stdlib.h>
+#include <unordered_map>
 #include <utility>
 
+namespace cmsys {
+class RegularExpression;
+}
+
 class TestComparator
 {
 public:
@@ -725,9 +735,330 @@ void cmCTestMultiProcessHandler::MarkFinished()
   cmSystemTools::RemoveFile(fname);
 }
 
+static Json::Value DumpToJsonArray(const std::set<std::string>& values)
+{
+  Json::Value jsonArray = Json::arrayValue;
+  for (auto& it : values) {
+    jsonArray.append(it);
+  }
+  return jsonArray;
+}
+
+static Json::Value DumpToJsonArray(const std::vector<std::string>& values)
+{
+  Json::Value jsonArray = Json::arrayValue;
+  for (auto& it : values) {
+    jsonArray.append(it);
+  }
+  return jsonArray;
+}
+
+static Json::Value DumpRegExToJsonArray(
+  const std::vector<std::pair<cmsys::RegularExpression, std::string>>& values)
+{
+  Json::Value jsonArray = Json::arrayValue;
+  for (auto& it : values) {
+    jsonArray.append(it.second);
+  }
+  return jsonArray;
+}
+
+static Json::Value DumpMeasurementToJsonArray(
+  const std::map<std::string, std::string>& values)
+{
+  Json::Value jsonArray = Json::arrayValue;
+  for (auto& it : values) {
+    Json::Value measurement = Json::objectValue;
+    measurement["measurement"] = it.first;
+    measurement["value"] = it.second;
+    jsonArray.append(measurement);
+  }
+  return jsonArray;
+}
+
+static Json::Value DumpTimeoutAfterMatch(
+  cmCTestTestHandler::cmCTestTestProperties& testProperties)
+{
+  Json::Value timeoutAfterMatch = Json::objectValue;
+  timeoutAfterMatch["timeout"] = testProperties.AlternateTimeout.count();
+  timeoutAfterMatch["regex"] =
+    DumpRegExToJsonArray(testProperties.TimeoutRegularExpressions);
+  return timeoutAfterMatch;
+}
+
+static Json::Value DumpCTestProperty(std::string const& name,
+                                     Json::Value value)
+{
+  Json::Value property = Json::objectValue;
+  property["name"] = name;
+  property["value"] = std::move(value);
+  return property;
+}
+
+static Json::Value DumpCTestProperties(
+  cmCTestTestHandler::cmCTestTestProperties& testProperties)
+{
+  Json::Value properties = Json::arrayValue;
+  if (!testProperties.AttachOnFail.empty()) {
+    properties.append(DumpCTestProperty(
+      "ATTACHED_FILES_ON_FAIL", DumpToJsonArray(testProperties.AttachOnFail)));
+  }
+  if (!testProperties.AttachedFiles.empty()) {
+    properties.append(DumpCTestProperty(
+      "ATTACHED_FILES", DumpToJsonArray(testProperties.AttachedFiles)));
+  }
+  if (testProperties.Cost != 0.0f) {
+    properties.append(
+      DumpCTestProperty("COST", static_cast<double>(testProperties.Cost)));
+  }
+  if (!testProperties.Depends.empty()) {
+    properties.append(
+      DumpCTestProperty("DEPENDS", DumpToJsonArray(testProperties.Depends)));
+  }
+  if (testProperties.Disabled) {
+    properties.append(DumpCTestProperty("DISABLED", testProperties.Disabled));
+  }
+  if (!testProperties.Environment.empty()) {
+    properties.append(DumpCTestProperty(
+      "ENVIRONMENT", DumpToJsonArray(testProperties.Environment)));
+  }
+  if (!testProperties.ErrorRegularExpressions.empty()) {
+    properties.append(DumpCTestProperty(
+      "FAIL_REGULAR_EXPRESSION",
+      DumpRegExToJsonArray(testProperties.ErrorRegularExpressions)));
+  }
+  if (!testProperties.FixturesCleanup.empty()) {
+    properties.append(DumpCTestProperty(
+      "FIXTURES_CLEANUP", DumpToJsonArray(testProperties.FixturesCleanup)));
+  }
+  if (!testProperties.FixturesRequired.empty()) {
+    properties.append(DumpCTestProperty(
+      "FIXTURES_REQUIRED", DumpToJsonArray(testProperties.FixturesRequired)));
+  }
+  if (!testProperties.FixturesSetup.empty()) {
+    properties.append(DumpCTestProperty(
+      "FIXTURES_SETUP", DumpToJsonArray(testProperties.FixturesSetup)));
+  }
+  if (!testProperties.Labels.empty()) {
+    properties.append(
+      DumpCTestProperty("LABELS", DumpToJsonArray(testProperties.Labels)));
+  }
+  if (!testProperties.Measurements.empty()) {
+    properties.append(DumpCTestProperty(
+      "MEASUREMENT", DumpMeasurementToJsonArray(testProperties.Measurements)));
+  }
+  if (!testProperties.RequiredRegularExpressions.empty()) {
+    properties.append(DumpCTestProperty(
+      "PASS_REGULAR_EXPRESSION",
+      DumpRegExToJsonArray(testProperties.RequiredRegularExpressions)));
+  }
+  if (testProperties.WantAffinity) {
+    properties.append(
+      DumpCTestProperty("PROCESSOR_AFFINITY", testProperties.WantAffinity));
+  }
+  if (testProperties.Processors != 1) {
+    properties.append(
+      DumpCTestProperty("PROCESSORS", testProperties.Processors));
+  }
+  if (!testProperties.RequiredFiles.empty()) {
+    properties["REQUIRED_FILES"] =
+      DumpToJsonArray(testProperties.RequiredFiles);
+  }
+  if (!testProperties.LockedResources.empty()) {
+    properties.append(DumpCTestProperty(
+      "RESOURCE_LOCK", DumpToJsonArray(testProperties.LockedResources)));
+  }
+  if (testProperties.RunSerial) {
+    properties.append(
+      DumpCTestProperty("RUN_SERIAL", testProperties.RunSerial));
+  }
+  if (testProperties.SkipReturnCode != -1) {
+    properties.append(
+      DumpCTestProperty("SKIP_RETURN_CODE", testProperties.SkipReturnCode));
+  }
+  if (testProperties.ExplicitTimeout) {
+    properties.append(
+      DumpCTestProperty("TIMEOUT", testProperties.Timeout.count()));
+  }
+  if (!testProperties.TimeoutRegularExpressions.empty()) {
+    properties.append(DumpCTestProperty(
+      "TIMEOUT_AFTER_MATCH", DumpTimeoutAfterMatch(testProperties)));
+  }
+  if (testProperties.WillFail) {
+    properties.append(DumpCTestProperty("WILL_FAIL", testProperties.WillFail));
+  }
+  if (!testProperties.Directory.empty()) {
+    properties.append(
+      DumpCTestProperty("WORKING_DIRECTORY", testProperties.Directory));
+  }
+  return properties;
+}
+
+class BacktraceData
+{
+  std::unordered_map<std::string, Json::ArrayIndex> CommandMap;
+  std::unordered_map<std::string, Json::ArrayIndex> FileMap;
+  std::unordered_map<cmListFileContext const*, Json::ArrayIndex> NodeMap;
+  Json::Value Commands = Json::arrayValue;
+  Json::Value Files = Json::arrayValue;
+  Json::Value Nodes = Json::arrayValue;
+
+  Json::ArrayIndex AddCommand(std::string const& command)
+  {
+    auto i = this->CommandMap.find(command);
+    if (i == this->CommandMap.end()) {
+      i = this->CommandMap.emplace(command, this->Commands.size()).first;
+      this->Commands.append(command);
+    }
+    return i->second;
+  }
+
+  Json::ArrayIndex AddFile(std::string const& file)
+  {
+    auto i = this->FileMap.find(file);
+    if (i == this->FileMap.end()) {
+      i = this->FileMap.emplace(file, this->Files.size()).first;
+      this->Files.append(file);
+    }
+    return i->second;
+  }
+
+public:
+  bool Add(cmListFileBacktrace const& bt, Json::ArrayIndex& index);
+  Json::Value Dump();
+};
+
+bool BacktraceData::Add(cmListFileBacktrace const& bt, Json::ArrayIndex& index)
+{
+  if (bt.Empty()) {
+    return false;
+  }
+  cmListFileContext const* top = &bt.Top();
+  auto found = this->NodeMap.find(top);
+  if (found != this->NodeMap.end()) {
+    index = found->second;
+    return true;
+  }
+  Json::Value entry = Json::objectValue;
+  entry["file"] = this->AddFile(top->FilePath);
+  if (top->Line) {
+    entry["line"] = static_cast<int>(top->Line);
+  }
+  if (!top->Name.empty()) {
+    entry["command"] = this->AddCommand(top->Name);
+  }
+  Json::ArrayIndex parent;
+  if (this->Add(bt.Pop(), parent)) {
+    entry["parent"] = parent;
+  }
+  index = this->NodeMap[top] = this->Nodes.size();
+  this->Nodes.append(std::move(entry)); // NOLINT(*)
+  return true;
+}
+
+Json::Value BacktraceData::Dump()
+{
+  Json::Value backtraceGraph;
+  this->CommandMap.clear();
+  this->FileMap.clear();
+  this->NodeMap.clear();
+  backtraceGraph["commands"] = std::move(this->Commands);
+  backtraceGraph["files"] = std::move(this->Files);
+  backtraceGraph["nodes"] = std::move(this->Nodes);
+  return backtraceGraph;
+}
+
+static void AddBacktrace(BacktraceData& backtraceGraph, Json::Value& object,
+                         cmListFileBacktrace const& bt)
+{
+  Json::ArrayIndex backtrace;
+  if (backtraceGraph.Add(bt, backtrace)) {
+    object["backtrace"] = backtrace;
+  }
+}
+
+static Json::Value DumpCTestInfo(
+  cmCTestRunTest& testRun,
+  cmCTestTestHandler::cmCTestTestProperties& testProperties,
+  BacktraceData& backtraceGraph)
+{
+  Json::Value testInfo = Json::objectValue;
+  // test name should always be present
+  testInfo["name"] = testProperties.Name;
+  std::string const& config = testRun.GetCTest()->GetConfigType();
+  if (!config.empty()) {
+    testInfo["config"] = config;
+  }
+  std::string const& command = testRun.GetActualCommand();
+  if (!command.empty()) {
+    std::vector<std::string> commandAndArgs;
+    commandAndArgs.push_back(command);
+    const std::vector<std::string>& args = testRun.GetArguments();
+    if (!args.empty()) {
+      commandAndArgs.reserve(args.size() + 1);
+      commandAndArgs.insert(commandAndArgs.end(), args.begin(), args.end());
+    }
+    testInfo["command"] = DumpToJsonArray(commandAndArgs);
+  }
+  Json::Value properties = DumpCTestProperties(testProperties);
+  if (!properties.empty()) {
+    testInfo["properties"] = properties;
+  }
+  if (!testProperties.Backtrace.Empty()) {
+    AddBacktrace(backtraceGraph, testInfo, testProperties.Backtrace);
+  }
+  return testInfo;
+}
+
+static Json::Value DumpVersion(int major, int minor)
+{
+  Json::Value version = Json::objectValue;
+  version["major"] = major;
+  version["minor"] = minor;
+  return version;
+}
+
+void cmCTestMultiProcessHandler::PrintOutputAsJson()
+{
+  this->TestHandler->SetMaxIndex(this->FindMaxIndex());
+
+  Json::Value result = Json::objectValue;
+  result["kind"] = "ctestInfo";
+  result["version"] = DumpVersion(1, 0);
+
+  BacktraceData backtraceGraph;
+  Json::Value tests = Json::arrayValue;
+  for (auto& it : this->Properties) {
+    cmCTestTestHandler::cmCTestTestProperties& p = *it.second;
+
+    // Don't worry if this fails, we are only showing the test list, not
+    // running the tests
+    cmWorkingDirectory workdir(p.Directory);
+    cmCTestRunTest testRun(*this);
+    testRun.SetIndex(p.Index);
+    testRun.SetTestProperties(&p);
+    testRun.ComputeArguments();
+
+    Json::Value testInfo = DumpCTestInfo(testRun, p, backtraceGraph);
+    tests.append(testInfo);
+  }
+  result["backtraceGraph"] = backtraceGraph.Dump();
+  result["tests"] = std::move(tests);
+
+  Json::StreamWriterBuilder builder;
+  builder["indentation"] = "  ";
+  std::unique_ptr<Json::StreamWriter> jout(builder.newStreamWriter());
+  jout->write(result, &std::cout);
+}
+
 // For ShowOnly mode
 void cmCTestMultiProcessHandler::PrintTestList()
 {
+  if (this->CTest->GetOutputAsJson()) {
+    PrintOutputAsJson();
+    return;
+  }
+
   this->TestHandler->SetMaxIndex(this->FindMaxIndex());
   int count = 0;
 

Source/CTest/cmCTestMultiProcessHandler.h

@@ -51,6 +51,7 @@ public:
   void SetParallelLevel(size_t);
   void SetTestLoad(unsigned long load);
   virtual void RunTests();
+  void PrintOutputAsJson();
   void PrintTestList();
   void PrintLabels();
 

Source/CTest/cmCTestRunTest.h

@@ -78,6 +78,10 @@ public:
 
   cmCTest* GetCTest() const { return this->CTest; }
 
+  std::string& GetActualCommand() { return this->ActualCommand; }
+
+  const std::vector<std::string>& GetArguments() { return this->Arguments; }
+
   void FinalizeTest();
 
   bool TimedOutForStopTime() const { return this->TimeoutIsForStopTime; }

Source/cmCTest.cxx

@@ -278,6 +278,8 @@ cmCTest::cmCTest()
   this->ExtraVerbose = false;
   this->ProduceXML = false;
   this->ShowOnly = false;
+  this->OutputAsJson = false;
+  this->OutputAsJsonVersion = 1;
   this->RunConfigurationScript = false;
   this->UseHTTP10 = false;
   this->PrintLabels = false;
@@ -1930,6 +1932,20 @@ bool cmCTest::HandleCommandLineArguments(size_t& i,
   if (this->CheckArgument(arg, "-N", "--show-only")) {
     this->ShowOnly = true;
   }
+  if (cmSystemTools::StringStartsWith(arg.c_str(), "--show-only=")) {
+    this->ShowOnly = true;
+
+    // Check if a specific format is requested. Defaults to human readable
+    // text.
+    std::string argWithFormat = "--show-only=";
+    std::string format = arg.substr(argWithFormat.length());
+    if (format == "json-v1") {
+      // Force quiet mode so the only output is the json object model.
+      this->Quiet = true;
+      this->OutputAsJson = true;
+      this->OutputAsJsonVersion = 1;
+    }
+  }
 
   if (this->CheckArgument(arg, "-O", "--output-log") && i < args.size() - 1) {
     i++;
@@ -2630,6 +2646,16 @@ bool cmCTest::GetShowOnly()
   return this->ShowOnly;
 }
 
+bool cmCTest::GetOutputAsJson()
+{
+  return this->OutputAsJson;
+}
+
+int cmCTest::GetOutputAsJsonVersion()
+{
+  return this->OutputAsJsonVersion;
+}
+
 int cmCTest::GetMaxTestNameWidth() const
 {
   return this->MaxTestNameWidth;

Source/cmCTest.h

@@ -215,6 +215,10 @@ public:
   /** Should we only show what we would do? */
   bool GetShowOnly();
 
+  bool GetOutputAsJson();
+
+  int GetOutputAsJsonVersion();
+
   bool ShouldUseHTTP10() { return this->UseHTTP10; }
 
   bool ShouldPrintLabels() { return this->PrintLabels; }
@@ -507,6 +511,8 @@ private:
   t_TestingHandlers TestingHandlers;
 
   bool ShowOnly;
+  bool OutputAsJson;
+  int OutputAsJsonVersion;
 
   /** Map of configuration properties */
   typedef std::map<std::string, std::string> CTestConfigurationMap;

Source/ctest.cxx

@@ -46,7 +46,10 @@ static const char* cmDocumentationOptions[][2] = {
     "given number of jobs." },
   { "-Q,--quiet", "Make ctest quiet." },
   { "-O <file>, --output-log <file>", "Output to log file" },
-  { "-N,--show-only", "Disable actual execution of tests." },
+  { "-N,--show-only[=format]",
+    "Disable actual execution of tests. The optional 'format' defines the "
+    "format of the test information and can be 'human' for the current text "
+    "format or 'json-v1' for json format. Defaults to 'human'." },
   { "-L <regex>, --label-regex <regex>",
     "Run tests with labels matching "
     "regular expression." },