ctest: Include cmake instrumentation data in XML files

Help/dev/experimental.rst

@@ -129,7 +129,13 @@ set
 * variable ``CMAKE_EXPERIMENTAL_INSTRUMENTATION`` to
 * value ``a37d1069-1972-4901-b9c9-f194aaf2b6e0``.
 
-To enable instrumentation at the user-level, files should be blaced under
+To enable instrumentation at the user-level, files should be placed under
 either
 ``<CMAKE_CONFIG_DIR>/instrumentation-a37d1069-1972-4901-b9c9-f194aaf2b6e0`` or
 ``<CMAKE_BINARY_DIR>/.cmake/instrumentation-a37d1069-1972-4901-b9c9-f194aaf2b6e0``.
+
+To include instrumentation data in CTest XML files (for submission to CDash),
+you need to set the following environment variables:
+
+* ``CTEST_USE_INSTRUMENTATION=1``
+* ``CTEST_EXPERIMENTAL_INSTRUMENTATION=a37d1069-1972-4901-b9c9-f194aaf2b6e0``

Help/envvar/CTEST_USE_INSTRUMENTATION.rst

@@ -0,0 +1,15 @@
+CTEST_USE_INSTRUMENTATION
+-------------------------
+
+.. versionadded:: 4.0
+
+.. include:: ENV_VAR.txt
+
+.. note::
+
+   This feature is only available when experimental support for instrumentation
+   has been enabled by the ``CMAKE_EXPERIMENTAL_INSTRUMENTATION`` gate.
+
+Setting this environment variable enables
+:manual:`instrumentation <cmake-instrumentation(7)>` for CTest in
+:ref:`Dashboard Client` mode.

Help/envvar/CTEST_USE_VERBOSE_INSTRUMENTATION.rst

@@ -0,0 +1,17 @@
+CTEST_USE_VERBOSE_INSTRUMENTATION
+---------------------------------
+
+.. versionadded:: 4.0
+
+.. include:: ENV_VAR.txt
+
+.. note::
+
+   This feature is only available when experimental support for instrumentation
+   has been enabled by the ``CMAKE_EXPERIMENTAL_INSTRUMENTATION`` gate.
+
+Setting this environment variable causes CTest to report the full
+command line (including arguments) to CDash for each instrumented command.
+By default, CTest truncates the command line at the first space.
+
+See also :envvar:`CTEST_USE_INSTRUMENTATION`

Help/manual/cmake-env-variables.7.rst

@@ -116,7 +116,9 @@ Environment Variables for CTest
    /envvar/CTEST_OUTPUT_ON_FAILURE
    /envvar/CTEST_PARALLEL_LEVEL
    /envvar/CTEST_PROGRESS_OUTPUT
+   /envvar/CTEST_USE_INSTRUMENTATION
    /envvar/CTEST_USE_LAUNCHERS_DEFAULT
+   /envvar/CTEST_USE_VERBOSE_INSTRUMENTATION
    /envvar/DASHBOARD_TEST_FROM_CTEST
 
 Environment Variables for the CMake curses interface

Help/manual/cmake-instrumentation.7.rst

@@ -94,6 +94,37 @@ Instrumentation can be configured at the user-level by placing query files in
 the :envvar:`CMAKE_CONFIG_DIR` under
 ``<config_dir>/instrumentation/<version>/query/``.
 
+Enabling Instrumentation for CDash Submissions
+----------------------------------------------
+
+You can enable instrumentation when using CTest in :ref:`Dashboard Client`
+mode by setting the :envvar:`CTEST_USE_INSTRUMENTATION` environment variable
+to the current UUID for the ``CMAKE_EXPERIMENTAL_INSTRUMENTATION`` feature.
+Doing so automatically enables the ``dynamicSystemInformation`` query.
+
+The following table shows how each type of instrumented command gets mapped
+to a corresponding type of CTest XML file.
+
+=================================================== ==================
+:ref:`Snippet Role <cmake-instrumentation Data v1>` CTest XML File
+=================================================== ==================
+``configure``                                       ``Configure.xml``
+``generate``                                        ``Configure.xml``
+``compile``                                         ``Build.xml``
+``link``                                            ``Build.xml``
+``custom``                                          ``Build.xml``
+``build``                                           unused!
+``cmakeBuild``                                      ``Build.xml``
+``cmakeInstall``                                    ``Build.xml``
+``install``                                         ``Build.xml``
+``ctest``                                           ``Build.xml``
+``test``                                            ``Test.xml``
+=================================================== ==================
+
+By default the command line reported to CDash is truncated at the first space.
+You can instead choose to report the full command line (including arguments)
+by setting :envvar:`CTEST_USE_VERBOSE_INSTRUMENTATION` to 1.
+
 .. _`cmake-instrumentation API v1`:
 
 API v1
@@ -123,6 +154,10 @@ subdirectories:
   files, they should never be removed by other processes. Data collected here
   remains until after `Indexing`_ occurs and all `Callbacks`_ are executed.
 
+``cdash/``
+  Holds temporary files used internally to generate XML content to be submitted
+  to CDash.
+
 .. _`cmake-instrumentation v1 Query Files`:
 
 v1 Query Files

Source/CTest/cmCTestBuildHandler.cxx

@@ -11,6 +11,7 @@
 #include <cm/string_view>
 #include <cmext/algorithm>
 
+#include <cm3p/json/value.h>
 #include <cm3p/uv.h>
 
 #include "cmsys/Directory.hxx"
@@ -21,6 +22,9 @@
 #include "cmDuration.h"
 #include "cmFileTimeCache.h"
 #include "cmGeneratedFileStream.h"
+#include "cmInstrumentation.h"
+#include "cmInstrumentationQuery.h"
+#include "cmJSONState.h"
 #include "cmList.h"
 #include "cmMakefile.h"
 #include "cmProcessOutput.h"
@@ -429,6 +433,11 @@ int cmCTestBuildHandler::ProcessHandler()
   } else {
     this->GenerateXMLLogScraped(xml);
   }
+
+  this->CTest->GetInstrumentation().CollectTimingData(
+    cmInstrumentationQuery::Hook::PrepareForCDash);
+  this->GenerateInstrumentationXML(xml);
+
   this->GenerateXMLFooter(xml, elapsed_build_time);
 
   if (!res || retVal || this->TotalErrors > 0) {
@@ -595,6 +604,88 @@ void cmCTestBuildHandler::GenerateXMLLogScraped(cmXMLWriter& xml)
   }
 }
 
+void cmCTestBuildHandler::GenerateInstrumentationXML(cmXMLWriter& xml)
+{
+  // Record instrumentation data on a per-target basis.
+  cmsys::Directory targets_dir;
+  std::string targets_snippet_dir = cmStrCat(
+    this->CTest->GetInstrumentation().GetCDashDir(), "/build/targets");
+  if (targets_dir.Load(targets_snippet_dir) &&
+      targets_dir.GetNumberOfFiles() > 0) {
+    xml.StartElement("Targets");
+    for (unsigned int i = 0; i < targets_dir.GetNumberOfFiles(); i++) {
+      if (!targets_dir.FileIsDirectory(i)) {
+        continue;
+      }
+      std::string target_name = targets_dir.GetFile(i);
+      if (target_name == "." || target_name == "..") {
+        continue;
+      }
+      std::string target_type = "UNKNOWN";
+
+      xml.StartElement("Target");
+      xml.Attribute("name", target_name);
+
+      // Check if we have a link snippet for this target.
+      cmsys::Directory target_dir;
+      if (!target_dir.Load(targets_dir.GetFilePath(i))) {
+        cmSystemTools::Error(
+          cmStrCat("Error loading directory ", targets_dir.GetFilePath(i)));
+      }
+      Json::Value link_item;
+      for (unsigned int j = 0; j < target_dir.GetNumberOfFiles(); j++) {
+        std::string fname = target_dir.GetFile(j);
+        if (fname.rfind("link-", 0) == 0) {
+          std::string fpath = target_dir.GetFilePath(j);
+          cmJSONState parseState = cmJSONState(fpath, &link_item);
+          if (!parseState.errors.empty()) {
+            cmSystemTools::Error(parseState.GetErrorMessage(true));
+            break;
+          }
+
+          if (!link_item.isObject()) {
+            std::string error_msg =
+              cmStrCat("Expected snippet ", fpath, " to contain an object");
+            cmSystemTools::Error(error_msg);
+            break;
+          }
+          break;
+        }
+      }
+
+      // If so, parse targetType and targetLabels (optional) from it.
+      if (link_item.isMember("targetType")) {
+        target_type = link_item["targetType"].asString();
+      }
+
+      xml.Attribute("type", target_type);
+
+      if (link_item.isMember("targetLabels") &&
+          !link_item["targetLabels"].empty()) {
+        xml.StartElement("Labels");
+        for (auto const& json_label_item : link_item["targetLabels"]) {
+          xml.Element("Label", json_label_item.asString());
+        }
+        xml.EndElement(); // Labels
+      }
+
+      // Write instrumendation data for this target.
+      std::string target_subdir = cmStrCat("build/targets/", target_name);
+      this->CTest->ConvertInstrumentationSnippetsToXML(xml, target_subdir);
+      std::string target_dir_fullpath = cmStrCat(
+        this->CTest->GetInstrumentation().GetCDashDir(), '/', target_subdir);
+      if (cmSystemTools::FileIsDirectory(target_dir_fullpath)) {
+        cmSystemTools::RemoveADirectory(target_dir_fullpath);
+      }
+      xml.EndElement(); // Target
+    }
+    xml.EndElement(); // Targets
+  }
+
+  // Also record instrumentation data for custom commands (no target).
+  this->CTest->ConvertInstrumentationSnippetsToXML(xml, "build/commands");
+}
+
 void cmCTestBuildHandler::GenerateXMLFooter(cmXMLWriter& xml,
                                             cmDuration elapsed_build_time)
 {

Source/CTest/cmCTestBuildHandler.h

@@ -85,6 +85,7 @@ private:
   void GenerateXMLHeader(cmXMLWriter& xml);
   void GenerateXMLLaunched(cmXMLWriter& xml);
   void GenerateXMLLogScraped(cmXMLWriter& xml);
+  void GenerateInstrumentationXML(cmXMLWriter& xml);
   void GenerateXMLFooter(cmXMLWriter& xml, cmDuration elapsed_build_time);
   bool IsLaunchedErrorFile(char const* fname);
   bool IsLaunchedWarningFile(char const* fname);

Source/CTest/cmCTestConfigureCommand.cxx

@@ -17,6 +17,8 @@
 #include "cmExecutionStatus.h"
 #include "cmGeneratedFileStream.h"
 #include "cmGlobalGenerator.h"
+#include "cmInstrumentation.h"
+#include "cmInstrumentationQuery.h"
 #include "cmList.h"
 #include "cmMakefile.h"
 #include "cmStringAlgorithms.h"
@@ -203,6 +205,11 @@ bool cmCTestConfigureCommand::ExecuteConfigure(ConfigureArguments const& args,
   xml.Element("EndDateTime", endDateTime);
   xml.Element("EndConfigureTime", endTime);
   xml.Element("ElapsedMinutes", elapsedMinutes.count());
+
+  this->CTest->GetInstrumentation().CollectTimingData(
+    cmInstrumentationQuery::Hook::PrepareForCDash);
+  this->CTest->ConvertInstrumentationSnippetsToXML(xml, "configure");
+
   xml.EndElement(); // Configure
   this->CTest->EndXML(xml);
 

Source/CTest/cmCTestRunTest.cxx

@@ -21,6 +21,7 @@
 #include "cmCTestMemCheckHandler.h"
 #include "cmCTestMultiProcessHandler.h"
 #include "cmDuration.h"
+#include "cmInstrumentation.h"
 #include "cmProcess.h"
 #include "cmStringAlgorithms.h"
 #include "cmSystemTools.h"
@@ -34,7 +35,6 @@ cmCTestRunTest::cmCTestRunTest(cmCTestMultiProcessHandler& multiHandler,
   , CTest(MultiTestHandler.CTest)
   , TestHandler(MultiTestHandler.TestHandler)
   , TestProperties(MultiTestHandler.Properties[Index])
-  , Instrumentation(cmSystemTools::GetLogicalWorkingDirectory())
 {
 }
 
@@ -664,8 +664,8 @@ bool cmCTestRunTest::StartTest(size_t completed, size_t total)
     return false;
   }
   this->StartTime = this->CTest->CurrentTime();
-  if (this->Instrumentation.HasQuery()) {
-    this->Instrumentation.GetPreTestStats();
+  if (this->CTest->GetInstrumentation().HasQuery()) {
+    this->CTest->GetInstrumentation().GetPreTestStats();
   }
 
   return this->ForkProcess();
@@ -1016,12 +1016,13 @@ void cmCTestRunTest::WriteLogOutputTop(size_t completed, size_t total)
 
 void cmCTestRunTest::FinalizeTest(bool started)
 {
-  if (this->Instrumentation.HasQuery()) {
-    this->Instrumentation.InstrumentTest(
+  if (this->CTest->GetInstrumentation().HasQuery()) {
+    std::string data_file = this->CTest->GetInstrumentation().InstrumentTest(
       this->TestProperties->Name, this->ActualCommand, this->Arguments,
       this->TestProcess->GetExitValue(), this->TestProcess->GetStartTime(),
       this->TestProcess->GetSystemStartTime(),
       this->GetCTest()->GetConfigType());
+    this->TestResult.InstrumentationFile = data_file;
   }
   this->MultiTestHandler.FinishTestProcess(this->TestProcess->GetRunner(),
                                            started);

Source/CTest/cmCTestRunTest.h

@@ -14,7 +14,6 @@
 #include "cmCTest.h"
 #include "cmCTestMultiProcessHandler.h"
 #include "cmCTestTestHandler.h"
-#include "cmInstrumentation.h"
 #include "cmProcess.h"
 
 /** \class cmRunTest
@@ -141,7 +140,6 @@ private:
   int NumberOfRunsTotal = 1; // default to 1 run of the test
   bool RunAgain = false;     // default to not having to run again
   size_t TotalNumberOfTests;
-  cmInstrumentation Instrumentation;
 };
 
 inline int getNumWidth(size_t n)

Source/CTest/cmCTestTestHandler.cxx

@@ -41,6 +41,8 @@
 #include "cmExecutionStatus.h"
 #include "cmGeneratedFileStream.h"
 #include "cmGlobalGenerator.h"
+#include "cmInstrumentation.h"
+#include "cmInstrumentationQuery.h"
 #include "cmList.h"
 #include "cmMakefile.h"
 #include "cmState.h"
@@ -1381,6 +1383,9 @@ void cmCTestTestHandler::GenerateCTestXML(cmXMLWriter& xml)
     return;
   }
 
+  this->CTest->GetInstrumentation().CollectTimingData(
+    cmInstrumentationQuery::Hook::PrepareForCDash);
+
   this->CTest->StartXML(xml, this->CMake, this->AppendXML);
   this->CTest->GenerateSubprojectsOutput(xml);
   xml.StartElement("Testing");
@@ -1395,7 +1400,6 @@ void cmCTestTestHandler::GenerateCTestXML(cmXMLWriter& xml)
   for (cmCTestTestResult& result : this->TestResults) {
     this->WriteTestResultHeader(xml, result);
     xml.StartElement("Results");
-
     if (result.Status != cmCTestTestHandler::NOT_RUN) {
       if (result.Status != cmCTestTestHandler::COMPLETED ||
           result.ReturnValue) {
@@ -1473,6 +1477,15 @@ void cmCTestTestHandler::GenerateCTestXML(cmXMLWriter& xml)
     xml.Content(result.Output);
     xml.EndElement(); // Value
     xml.EndElement(); // Measurement
+
+    if (!result.InstrumentationFile.empty()) {
+      std::string instrument_file_path =
+        cmStrCat(this->CTest->GetInstrumentation().GetCDashDir(), "/test/",
+                 result.InstrumentationFile);
+      this->CTest->ConvertInstrumentationJSONFileToXML(instrument_file_path,
+                                                       xml);
+    }
+
     xml.EndElement(); // Results
 
     this->AttachFiles(xml, result);

Source/CTest/cmCTestTestHandler.h

@@ -192,6 +192,7 @@ public:
     std::string CustomCompletionStatus;
     std::string Output;
     std::string TestMeasurementsOutput;
+    std::string InstrumentationFile;
     int TestCount = 0;
     cmCTestTestProperties* Properties = nullptr;
   };
@@ -250,6 +251,7 @@ protected:
                              cmCTestTestResult const& result);
   void WriteTestResultFooter(cmXMLWriter& xml,
                              cmCTestTestResult const& result);
+
   // Write attached test files into the xml
   void AttachFiles(cmXMLWriter& xml, cmCTestTestResult& result);
   void AttachFile(cmXMLWriter& xml, std::string const& file,

Source/cmCTest.cxx

@@ -27,6 +27,7 @@
 #include <cmext/string_view>
 
 #include <cm3p/curl/curl.h>
+#include <cm3p/json/value.h>
 #include <cm3p/uv.h>
 #include <cm3p/zlib.h>
 
@@ -115,6 +116,8 @@ struct cmCTest::Private
   bool UseHTTP10 = false;
   bool PrintLabels = false;
   bool Failover = false;
+  bool UseVerboseInstrumentation = false;
+  cmJSONState parseState;
 
   bool FlushTestProgressLine = false;
 
@@ -195,6 +198,8 @@ struct cmCTest::Private
 
   cmCTestTestOptions TestOptions;
   std::vector<std::string> CommandLineHttpHeaders;
+
+  std::unique_ptr<cmInstrumentation> Instrumentation;
 };
 
 struct tm* cmCTest::GetNightlyTime(std::string const& str, bool tomorrowtag)
@@ -320,6 +325,11 @@ cmCTest::cmCTest()
   if (cmSystemTools::GetEnv("CTEST_PROGRESS_OUTPUT", envValue)) {
     this->Impl->TestProgressOutput = !cmIsOff(envValue);
   }
+  envValue.clear();
+  if (cmSystemTools::GetEnv("CTEST_USE_VERBOSE_INSTRUMENTATION", envValue)) {
+    this->Impl->UseVerboseInstrumentation = !cmIsOff(envValue);
+  }
+  envValue.clear();
 
   this->Impl->Parts[PartStart].SetName("Start");
   this->Impl->Parts[PartUpdate].SetName("Update");
@@ -2628,8 +2638,6 @@ int cmCTest::Run(std::vector<std::string> const& args)
   }
 #endif
 
-  cmInstrumentation instrumentation(
-    cmSystemTools::GetCurrentWorkingDirectory());
   std::function<int()> doTest = [this, &cmakeAndTest, &runScripts,
                                  &processSteps]() -> int {
     // now what should cmake do? if --build-and-test was specified then
@@ -2650,6 +2658,8 @@ int cmCTest::Run(std::vector<std::string> const& args)
 
     return this->ExecuteTests();
   };
+  cmInstrumentation instrumentation(
+    cmSystemTools::GetCurrentWorkingDirectory());
   int ret = instrumentation.InstrumentCommand("ctest", args,
                                               [doTest]() { return doTest(); });
   instrumentation.CollectTimingData(cmInstrumentationQuery::Hook::PostTest);
@@ -3673,3 +3683,128 @@ bool cmCTest::StartLogFile(char const* name, int submitIndex,
   }
   return true;
 }
+
+cmInstrumentation& cmCTest::GetInstrumentation()
+{
+  if (!this->Impl->Instrumentation) {
+    this->Impl->Instrumentation =
+      cm::make_unique<cmInstrumentation>(this->GetBinaryDir());
+  }
+  return *this->Impl->Instrumentation;
+}
+
+bool cmCTest::GetUseVerboseInstrumentation() const
+{
+  return this->Impl->UseVerboseInstrumentation;
+}
+
+void cmCTest::ConvertInstrumentationSnippetsToXML(cmXMLWriter& xml,
+                                                  std::string const& subdir)
+{
+  std::string data_dir =
+    cmStrCat(this->GetInstrumentation().GetCDashDir(), '/', subdir);
+
+  cmsys::Directory d;
+  if (!d.Load(data_dir) || d.GetNumberOfFiles() == 0) {
+    return;
+  }
+
+  xml.StartElement("Commands");
+
+  for (unsigned int i = 0; i < d.GetNumberOfFiles(); i++) {
+    std::string fpath = d.GetFilePath(i);
+    std::string fname = d.GetFile(i);
+    if (fname.rfind('.', 0) == 0) {
+      continue;
+    }
+    this->ConvertInstrumentationJSONFileToXML(fpath, xml);
+  }
+
+  xml.EndElement(); // Commands
+}
+
+bool cmCTest::ConvertInstrumentationJSONFileToXML(std::string const& fpath,
+                                                  cmXMLWriter& xml)
+{
+  Json::Value root;
+  this->Impl->parseState = cmJSONState(fpath, &root);
+  if (!this->Impl->parseState.errors.empty()) {
+    cmCTestLog(this, ERROR_MESSAGE,
+               this->Impl->parseState.GetErrorMessage(true) << std::endl);
+    return false;
+  }
+
+  if (root.type() != Json::objectValue) {
+    cmCTestLog(this, ERROR_MESSAGE,
+               "Expected object, found " << root.type() << " for "
+                                         << root.asString() << std::endl);
+    return false;
+  }
+
+  std::vector<std::string> required_members = {
+    "command",
+    "role",
+    "dynamicSystemInformation",
+  };
+  for (std::string const& required_member : required_members) {
+    if (!root.isMember(required_member)) {
+      cmCTestLog(this, ERROR_MESSAGE,
+                 fpath << " is missing the '" << required_member << "' key"
+                       << std::endl);
+      return false;
+    }
+  }
+
+  // Do not record command-level data for Test.xml files because
+  // it is redundant with information actually captured by CTest.
+  bool generating_test_xml = root["role"] == "test";
+  if (!generating_test_xml) {
+    std::string element_name = root["role"].asString();
+    element_name[0] = static_cast<char>(std::toupper(element_name[0]));
+    xml.StartElement(element_name);
+    std::vector<std::string> keys = root.getMemberNames();
+    for (auto const& key : keys) {
+      auto key_type = root[key].type();
+      if (key_type == Json::objectValue || key_type == Json::arrayValue) {
+        continue;
+      }
+      if (key == "role" || key == "target" || key == "targetType" ||
+          key == "targetLabels") {
+        continue;
+      }
+      // Truncate the full command line if verbose instrumentation
+      // was not requested.
+      if (key == "command" && !this->GetUseVerboseInstrumentation()) {
+        std::string command_str = root[key].asString();
+        std::string truncated = command_str.substr(0, command_str.find(' '));
+        if (command_str != truncated) {
+          truncated = cmStrCat(truncated, " (truncated)");
+        }
+        xml.Attribute(key.c_str(), truncated);
+        continue;
+      }
+      xml.Attribute(key.c_str(), root[key].asString());
+    }
+  }
+
+  // Record dynamicSystemInformation section as XML.
+  auto dynamic_information = root["dynamicSystemInformation"];
+  std::vector<std::string> keys = dynamic_information.getMemberNames();
+  for (auto const& key : keys) {
+    std::string measurement_name = key;
+    measurement_name[0] = static_cast<char>(std::toupper(measurement_name[0]));
+
+    xml.StartElement("NamedMeasurement");
+    xml.Attribute("type", "numeric/double");
+    xml.Attribute("name", measurement_name);
+    xml.Element("Value", dynamic_information[key].asString());
+    xml.EndElement(); // NamedMeasurement
+  }
+
+  if (!generating_test_xml) {
+    xml.EndElement(); // role
+  }
+
+  cmSystemTools::RemoveFile(fpath);
+  return true;
+}

Source/cmCTest.h

@@ -21,6 +21,7 @@
 
 class cmake;
 class cmGeneratedFileStream;
+class cmInstrumentation;
 class cmMakefile;
 class cmValue;
 class cmXMLWriter;
@@ -391,6 +392,11 @@ public:
   bool StartLogFile(char const* name, int submitIndex,
                     cmGeneratedFileStream& xofs);
 
+  void ConvertInstrumentationSnippetsToXML(cmXMLWriter& xml,
+                                           std::string const& subdir);
+  bool ConvertInstrumentationJSONFileToXML(std::string const& fpath,
+                                           cmXMLWriter& xml);
+
   void AddSiteProperties(cmXMLWriter& xml, cmake* cm);
 
   bool GetInteractiveDebugMode() const;
@@ -433,6 +439,9 @@ public:
   cmCTestTestOptions const& GetTestOptions() const;
   std::vector<std::string> GetCommandLineHttpHeaders() const;
 
+  cmInstrumentation& GetInstrumentation();
+  bool GetUseVerboseInstrumentation() const;
+
 private:
   int GenerateNotesFile(cmake* cm, std::string const& files);
 

Source/cmInstrumentation.cxx

@@ -20,10 +20,12 @@
 #include "cmCryptoHash.h"
 #include "cmExperimental.h"
 #include "cmInstrumentationQuery.h"
+#include "cmJSONState.h"
 #include "cmStringAlgorithms.h"
 #include "cmSystemTools.h"
 #include "cmTimestamp.h"
 #include "cmUVProcessChain.h"
+#include "cmValue.h"
 
 cmInstrumentation::cmInstrumentation(std::string const& binary_dir)
 {
@@ -53,6 +55,75 @@ void cmInstrumentation::LoadQueries()
     this->hasQuery = this->hasQuery ||
       this->ReadJSONQueries(cmStrCat(this->userTimingDirv1, "/query"));
   }
+
+  std::string envVal;
+  if (cmSystemTools::GetEnv("CTEST_USE_INSTRUMENTATION", envVal) &&
+      !cmIsOff(envVal)) {
+    if (cmSystemTools::GetEnv("CTEST_EXPERIMENTAL_INSTRUMENTATION", envVal)) {
+      std::string const uuid = cmExperimental::DataForFeature(
+                                 cmExperimental::Feature::Instrumentation)
+                                 .Uuid;
+      if (envVal == uuid) {
+        this->AddHook(cmInstrumentationQuery::Hook::PrepareForCDash);
+        this->AddQuery(
+          cmInstrumentationQuery::Query::DynamicSystemInformation);
+        this->cdashDir = cmStrCat(this->timingDirv1, "/cdash");
+        cmSystemTools::MakeDirectory(this->cdashDir);
+        cmSystemTools::MakeDirectory(cmStrCat(this->cdashDir, "/configure"));
+        cmSystemTools::MakeDirectory(cmStrCat(this->cdashDir, "/build"));
+        cmSystemTools::MakeDirectory(
+          cmStrCat(this->cdashDir, "/build/commands"));
+        cmSystemTools::MakeDirectory(
+          cmStrCat(this->cdashDir, "/build/targets"));
+        cmSystemTools::MakeDirectory(cmStrCat(this->cdashDir, "/test"));
+        this->cdashSnippetsMap = { {
+                                     "configure",
+                                     "configure",
+                                   },
+                                   {
+                                     "generate",
+                                     "configure",
+                                   },
+                                   {
+                                     "compile",
+                                     "build",
+                                   },
+                                   {
+                                     "link",
+                                     "build",
+                                   },
+                                   {
+                                     "custom",
+                                     "build",
+                                   },
+                                   {
+                                     "build",
+                                     "skip",
+                                   },
+                                   {
+                                     "cmakeBuild",
+                                     "build",
+                                   },
+                                   {
+                                     "cmakeInstall",
+                                     "build",
+                                   },
+                                   {
+                                     "install",
+                                     "build",
+                                   },
+                                   {
+                                     "ctest",
+                                     "build",
+                                   },
+                                   {
+                                     "test",
+                                     "test",
+                                   } };
+        this->hasQuery = true;
+      }
+    }
+  }
 }
 
 bool cmInstrumentation::ReadJSONQueries(std::string const& directory)
@@ -211,6 +282,11 @@ int cmInstrumentation::CollectTimingData(cmInstrumentationQuery::Hook hook)
                                     cmSystemTools::OUTPUT_PASSTHROUGH);
   }
 
+  // Special case for CDash collation
+  if (this->HasHook(cmInstrumentationQuery::Hook::PrepareForCDash)) {
+    this->PrepareDataForCDash(directory, index_path);
+  }
+
   // Delete files
   for (auto const& f : index["snippets"]) {
     cmSystemTools::RemoveFile(cmStrCat(directory, "/", f.asString()));
@@ -308,7 +384,7 @@ void cmInstrumentation::WriteInstrumentationJson(Json::Value& root,
   ftmp.close();
 }
 
-int cmInstrumentation::InstrumentTest(
+std::string cmInstrumentation::InstrumentTest(
   std::string const& name, std::string const& command,
   std::vector<std::string> const& args, int64_t result,
   std::chrono::steady_clock::time_point steadyStart,
@@ -331,11 +407,11 @@ int cmInstrumentation::InstrumentTest(
     this->InsertDynamicSystemInformation(root, "after");
   }
 
-  std::string const& file_name =
+  std::string file_name =
     cmStrCat("test-", this->ComputeSuffixHash(command_str),
              this->ComputeSuffixTime(), ".json");
   this->WriteInstrumentationJson(root, "data", file_name);
-  return 1;
+  return file_name;
 }
 
 void cmInstrumentation::GetPreTestStats()
@@ -547,3 +623,107 @@ int cmInstrumentation::CollectTimingAfterBuild(int ppid)
   this->CollectTimingData(cmInstrumentationQuery::Hook::PostBuild);
   return ret;
 }
+
+void cmInstrumentation::AddHook(cmInstrumentationQuery::Hook hook)
+{
+  this->hooks.insert(hook);
+}
+
+void cmInstrumentation::AddQuery(cmInstrumentationQuery::Query query)
+{
+  this->queries.insert(query);
+}
+
+std::string const& cmInstrumentation::GetCDashDir()
+{
+  return this->cdashDir;
+}
+
+/** Copy the snippets referred to by an index file to a separate
+ * directory where they will be parsed for submission to CDash.
+ **/
+void cmInstrumentation::PrepareDataForCDash(std::string const& data_dir,
+                                            std::string const& index_path)
+{
+  Json::Value root;
+  std::string error_msg;
+  cmJSONState parseState = cmJSONState(index_path, &root);
+  if (!parseState.errors.empty()) {
+    cmSystemTools::Error(parseState.GetErrorMessage(true));
+    return;
+  }
+
+  if (!root.isObject()) {
+    error_msg =
+      cmStrCat("Expected index file ", index_path, " to contain an object");
+    cmSystemTools::Error(error_msg);
+    return;
+  }
+
+  if (!root.isMember("snippets")) {
+    error_msg = cmStrCat("Expected index file ", index_path,
+                         " to have a key 'snippets'");
+    cmSystemTools::Error(error_msg);
+    return;
+  }
+
+  std::string dst_dir;
+  Json::Value snippets = root["snippets"];
+  for (auto const& snippet : snippets) {
+    // Parse the role of this snippet.
+    std::string snippet_str = snippet.asString();
+    std::string snippet_path = cmStrCat(data_dir, '/', snippet_str);
+    Json::Value snippet_root;
+    parseState = cmJSONState(snippet_path, &snippet_root);
+    if (!parseState.errors.empty()) {
+      cmSystemTools::Error(parseState.GetErrorMessage(true));
+      continue;
+    }
+    if (!snippet_root.isObject()) {
+      error_msg = cmStrCat("Expected snippet file ", snippet_path,
+                           " to contain an object");
+      cmSystemTools::Error(error_msg);
+      continue;
+    }
+    if (!snippet_root.isMember("role")) {
+      error_msg = cmStrCat("Expected snippet file ", snippet_path,
+                           " to have a key 'role'");
+      cmSystemTools::Error(error_msg);
+      continue;
+    }
+
+    std::string snippet_role = snippet_root["role"].asString();
+    auto map_element = this->cdashSnippetsMap.find(snippet_role);
+    if (map_element == this->cdashSnippetsMap.end()) {
+      std::string message =
+        "Unexpected snippet type encountered: " + snippet_role;
+      cmSystemTools::Message(message, "Warning");
+      continue;
+    }
+
+    if (map_element->second == "skip") {
+      continue;
+    }
+
+    if (map_element->second == "build") {
+      // We organize snippets on a per-target basis (when possible)
+      // for Build.xml.
+      if (snippet_root.isMember("target")) {
+        dst_dir = cmStrCat(this->cdashDir, "/build/targets/",
+                           snippet_root["target"].asString());
+        cmSystemTools::MakeDirectory(dst_dir);
+      } else {
+        dst_dir = cmStrCat(this->cdashDir, "/build/commands");
+      }
+    } else {
+      dst_dir = cmStrCat(this->cdashDir, '/', map_element->second);
+    }
+
+    std::string dst = cmStrCat(dst_dir, '/', snippet_str);
+    cmsys::Status copied = cmSystemTools::CopyFileAlways(snippet_path, dst);
+    if (!copied) {
+      error_msg = cmStrCat("Failed to copy ", snippet_path, " to ", dst);
+      cmSystemTools::Error(error_msg);
+    }
+  }
+}

Source/cmInstrumentation.h

@@ -29,11 +29,13 @@ public:
     cm::optional<std::map<std::string, std::string>> arrayOptions =
       cm::nullopt,
     bool reloadQueriesAfterCommand = false);
-  int InstrumentTest(std::string const& name, std::string const& command,
-                     std::vector<std::string> const& args, int64_t result,
-                     std::chrono::steady_clock::time_point steadyStart,
-                     std::chrono::system_clock::time_point systemStart,
-                     std::string config);
+  std::string InstrumentTest(std::string const& name,
+                             std::string const& command,
+                             std::vector<std::string> const& args,
+                             int64_t result,
+                             std::chrono::steady_clock::time_point steadyStart,
+                             std::chrono::system_clock::time_point systemStart,
+                             std::string config);
   void GetPreTestStats();
   void LoadQueries();
   bool HasQuery() const;
@@ -49,7 +51,10 @@ public:
   int CollectTimingData(cmInstrumentationQuery::Hook hook);
   int SpawnBuildDaemon();
   int CollectTimingAfterBuild(int ppid);
+  void AddHook(cmInstrumentationQuery::Hook hook);
+  void AddQuery(cmInstrumentationQuery::Query query);
   std::string errorMsg;
+  std::string const& GetCDashDir();
 
 private:
   void WriteInstrumentationJson(Json::Value& index,
@@ -66,13 +71,17 @@ private:
   static std::string GetCommandStr(std::vector<std::string> const& args);
   static std::string ComputeSuffixHash(std::string const& command_str);
   static std::string ComputeSuffixTime();
+  void PrepareDataForCDash(std::string const& data_dir,
+                           std::string const& index_path);
   std::string binaryDir;
   std::string timingDirv1;
   std::string userTimingDirv1;
+  std::string cdashDir;
   std::set<cmInstrumentationQuery::Query> queries;
   std::set<cmInstrumentationQuery::Hook> hooks;
   std::vector<std::string> callbacks;
   std::vector<std::string> queryFiles;
+  std::map<std::string, std::string> cdashSnippetsMap;
   Json::Value preTestStats;
   bool hasQuery = false;
 };

Source/cmInstrumentationQuery.cxx

@@ -19,8 +19,9 @@ std::vector<std::string> const cmInstrumentationQuery::QueryString{
   "staticSystemInformation", "dynamicSystemInformation"
 };
 std::vector<std::string> const cmInstrumentationQuery::HookString{
-  "postGenerate",   "preBuild", "postBuild",   "preCMakeBuild",
-  "postCMakeBuild", "postTest", "postInstall", "manual"
+  "postGenerate",  "preBuild",        "postBuild",
+  "preCMakeBuild", "postCMakeBuild",  "postTest",
+  "postInstall",   "prepareForCDash", "manual"
 };
 
 namespace ErrorMessages {

Source/cmInstrumentationQuery.h

@@ -28,6 +28,7 @@ public:
     PostCMakeBuild,
     PostTest,
     PostInstall,
+    PrepareForCDash,
     Manual
   };
   static std::vector<std::string> const HookString;

Tests/RunCMake/CMakeLists.txt

@@ -605,6 +605,9 @@ add_RunCMake_test(ctest_upload)
 add_RunCMake_test(ctest_environment)
 add_RunCMake_test(ctest_empty_binary_directory)
 add_RunCMake_test(ctest_fixtures)
+if(CMAKE_GENERATOR MATCHES "Make|Ninja")
+  add_RunCMake_test(ctest_instrumentation)
+endif()
 add_RunCMake_test(define_property)
 add_RunCMake_test(file -DCYGWIN=${CYGWIN} -DMSYS=${MSYS})
 add_RunCMake_test(file-CHMOD -DMSYS=${MSYS})

Tests/RunCMake/ctest_instrumentation/CMakeLists.txt.in

@@ -0,0 +1,10 @@
+cmake_minimum_required(VERSION 3.10)
+@CASE_CMAKELISTS_PREFIX_CODE@
+project(CTestInstrumentation@CASE_NAME@)
+if(USE_INSTRUMENTATION)
+  set(CMAKE_EXPERIMENTAL_INSTRUMENTATION "a37d1069-1972-4901-b9c9-f194aaf2b6e0")
+endif()
+include(CTest)
+add_executable(main main.c)
+add_test(NAME main COMMAND main)
+@CASE_CMAKELISTS_SUFFIX_CODE@

Tests/RunCMake/ctest_instrumentation/InstrumentationInCTestXML-check.cmake

@@ -0,0 +1,41 @@
+foreach(xml_type Configure Build Test)
+  file(GLOB xml_file "${RunCMake_TEST_BINARY_DIR}/Testing/*/${xml_type}.xml")
+  if(xml_file)
+    file(READ "${xml_file}" xml_content)
+    if(NOT xml_content MATCHES "AfterHostMemoryUsed")
+      set(RunCMake_TEST_FAILED "'AfterHostMemoryUsed' not found in ${xml_type}.xml")
+    endif()
+    if(NOT xml_type STREQUAL "Test")
+      if(NOT xml_content MATCHES "<Commands>")
+        set(RunCMake_TEST_FAILED "<Commands> element not found in ${xml_type}.xml")
+      endif()
+    endif()
+    if (xml_type STREQUAL "Build")
+      if(NOT xml_content MATCHES "<Targets>")
+        set(RunCMake_TEST_FAILED "<Targets> element not found in Build.xml")
+      endif()
+      if(NOT xml_content MATCHES "<Target name=\"main\" type=\"EXECUTABLE\">")
+        set(RunCMake_TEST_FAILED "<Target> element for 'main' not found in Build.xml")
+      endif()
+      if(NOT xml_content MATCHES "<Compile")
+        set(RunCMake_TEST_FAILED "<Compile> element not found in Build.xml")
+      endif()
+      if(NOT xml_content MATCHES "<Link")
+        set(RunCMake_TEST_FAILED "<Link> element not found in Build.xml")
+      endif()
+      if(NOT xml_content MATCHES "<CmakeBuild")
+        set(RunCMake_TEST_FAILED "<CmakeBuild> element not found in Build.xml")
+      endif()
+    endif()
+  else()
+    set(RunCMake_TEST_FAILED "${xml_type}.xml not found")
+  endif()
+endforeach()
+
+foreach(dir_to_check "configure" "test" "build/targets" "build/commands")
+  file(GLOB leftover_cdash_snippets
+    "${RunCMake_TEST_BINARY_DIR}/.cmake/instrumentation-a37d1069-1972-4901-b9c9-f194aaf2b6e0/v1/cdash/${dir_to_check}/*")
+  if(leftover_cdash_snippets)
+    set(RunCMake_TEST_FAILED "Leftover snippets found in cdash dir: ${leftover_cdash_snippets}")
+  endif()
+endforeach()

Tests/RunCMake/ctest_instrumentation/NoInstrumentationInCTestXML-check.cmake

@@ -0,0 +1,11 @@
+foreach(xml_type Configure Build Test)
+  file(GLOB xml_file "${RunCMake_TEST_BINARY_DIR}/Testing/*/${xml_type}.xml")
+  if(xml_file)
+    file(READ "${xml_file}" xml_content)
+    if(xml_content MATCHES "AfterHostMemoryUsed")
+      set(RunCMake_TEST_FAILED "'AfterHostMemoryUsed' found in ${xml_type}.xml")
+    endif()
+  else()
+    set(RunCMake_TEST_FAILED "${xml_type}.xml not found")
+  endif()
+endforeach()

Tests/RunCMake/ctest_instrumentation/RunCMakeTest.cmake

@@ -0,0 +1,22 @@
+include(RunCTest)
+
+function(run_InstrumentationInCTestXML USE_INSTRUMENTATION)
+  if(USE_INSTRUMENTATION)
+    set(ENV{CTEST_USE_INSTRUMENTATION} "1")
+    set(ENV{CTEST_EXPERIMENTAL_INSTRUMENTATION} "a37d1069-1972-4901-b9c9-f194aaf2b6e0")
+    set(RunCMake_USE_INSTRUMENTATION TRUE)
+    set(CASE_NAME InstrumentationInCTestXML)
+  else()
+    set(ENV{CTEST_USE_INSTRUMENTATION} "0")
+    set(ENV{CTEST_EXPERIMENTAL_INSTRUMENTATION} "0")
+    set(RunCMake_USE_INSTRUMENTATION FALSE)
+    set(CASE_NAME NoInstrumentationInCTestXML)
+  endif()
+  configure_file(${RunCMake_SOURCE_DIR}/main.c
+                 ${RunCMake_BINARY_DIR}/${CASE_NAME}/main.c COPYONLY)
+  run_ctest("${CASE_NAME}")
+  unset(RunCMake_USE_LAUNCHERS)
+  unset(RunCMake_USE_INSTRUMENTATION)
+endfunction()
+run_InstrumentationInCTestXML(ON)
+run_InstrumentationInCTestXML(OFF)

Tests/RunCMake/ctest_instrumentation/main.c

@@ -0,0 +1,4 @@
+int main(void)
+{
+  return 0;
+}

Tests/RunCMake/ctest_instrumentation/test.cmake.in

@@ -0,0 +1,17 @@
+cmake_minimum_required(VERSION 3.10)
+
+set(CTEST_SITE                          "test-site")
+set(CTEST_BUILD_NAME                    "test-build-name")
+set(CTEST_SOURCE_DIRECTORY              "@RunCMake_BINARY_DIR@/@CASE_NAME@")
+set(CTEST_BINARY_DIRECTORY              "@RunCMake_BINARY_DIR@/@CASE_NAME@-build")
+set(CTEST_CMAKE_GENERATOR               "@RunCMake_GENERATOR@")
+set(CTEST_CMAKE_GENERATOR_PLATFORM      "@RunCMake_GENERATOR_PLATFORM@")
+set(CTEST_CMAKE_GENERATOR_TOOLSET       "@RunCMake_GENERATOR_TOOLSET@")
+set(CTEST_BUILD_CONFIGURATION           "$ENV{CMAKE_CONFIG_TYPE}")
+set(CTEST_USE_LAUNCHERS                 TRUE)
+set(CTEST_USE_INSTRUMENTATION           "@RunCMake_USE_INSTRUMENTATION@")
+
+ctest_start(Experimental)
+ctest_configure(OPTIONS "-DUSE_INSTRUMENTATION=${CTEST_USE_INSTRUMENTATION}")
+ctest_build()
+ctest_test()