Merge topic 'instrumentation-ctest-interface'

21ac285cd8 Experimental: Update the Instrumentation UUID
9dec460c8c instrumentation: Store CDash settings in query files
afa94bae1e instrumentation: Rename queries field to options

Acked-by: Kitware Robot <[email protected]>
Tested-by: buildbot <[email protected]>
Merge-request: !10934

Help/command/cmake_instrumentation.rst

@@ -19,7 +19,7 @@ This allows for configuring instrumentation at the project-level.
     API_VERSION <version>
     DATA_VERSION <version>
     [HOOKS <hooks>...]
-    [QUERIES <queries>...]
+    [OPTIONS <options>...]
     [CALLBACK <callback>]
   )
 
@@ -28,7 +28,7 @@ only supported value for both fields is 1.  See :ref:`cmake-instrumentation API
 for details of the ``API_VERSION`` and :ref:`cmake-instrumentation Data v1` for details
 of the ``DATA_VERSION``.
 
-Each of the optional keywords ``HOOKS``, ``QUERIES``, and ``CALLBACK``
+Each of the optional keywords ``HOOKS``, ``OPTIONS``, and ``CALLBACK``
 correspond to one of the parameters to the :ref:`cmake-instrumentation v1 Query Files`.
 The ``CALLBACK`` keyword can be provided multiple times to create multiple callbacks.
 
@@ -48,7 +48,7 @@ equivalent JSON query file.
     API_VERSION 1
     DATA_VERSION 1
     HOOKS postGenerate preCMakeBuild postCMakeBuild
-    QUERIES staticSystemInformation dynamicSystemInformation
+    OPTIONS staticSystemInformation dynamicSystemInformation
     CALLBACK ${CMAKE_COMMAND} -P /path/to/handle_data.cmake
     CALLBACK ${CMAKE_COMMAND} -P /path/to/handle_data_2.cmake
   )
@@ -60,7 +60,7 @@ equivalent JSON query file.
     "hooks": [
       "postGenerate", "preCMakeBuild", "postCMakeBuild"
     ],
-    "queries": [
+    "options": [
       "staticSystemInformation", "dynamicSystemInformation"
     ],
     "callbacks": [

Help/dev/experimental.rst

@@ -127,15 +127,15 @@ In order to activate support for the :command:`cmake_instrumentation` command,
 set
 
 * variable ``CMAKE_EXPERIMENTAL_INSTRUMENTATION`` to
-* value ``a37d1069-1972-4901-b9c9-f194aaf2b6e0``.
+* value ``d16a3082-c4e1-489b-b90c-55750a334f27``.
 
 To enable instrumentation at the user-level, files should be placed under
 either
-``<CMAKE_CONFIG_DIR>/instrumentation-a37d1069-1972-4901-b9c9-f194aaf2b6e0`` or
-``<CMAKE_BINARY_DIR>/.cmake/instrumentation-a37d1069-1972-4901-b9c9-f194aaf2b6e0``.
+``<CMAKE_CONFIG_DIR>/instrumentation-d16a3082-c4e1-489b-b90c-55750a334f27`` or
+``<CMAKE_BINARY_DIR>/.cmake/instrumentation-d16a3082-c4e1-489b-b90c-55750a334f27``.
 
 To include instrumentation data in CTest XML files (for submission to CDash),
 you need to set the following environment variables:
 
 * ``CTEST_USE_INSTRUMENTATION=1``
-* ``CTEST_EXPERIMENTAL_INSTRUMENTATION=a37d1069-1972-4901-b9c9-f194aaf2b6e0``
+* ``CTEST_EXPERIMENTAL_INSTRUMENTATION=d16a3082-c4e1-489b-b90c-55750a334f27``

Help/manual/cmake-instrumentation.7.rst

@@ -100,7 +100,7 @@ Enabling Instrumentation for CDash Submissions
 You can enable instrumentation when using CTest in :ref:`Dashboard Client`
 mode by setting the :envvar:`CTEST_USE_INSTRUMENTATION` environment variable
 to the current UUID for the ``CMAKE_EXPERIMENTAL_INSTRUMENTATION`` feature.
-Doing so automatically enables the ``dynamicSystemInformation`` query.
+Doing so automatically enables the ``dynamicSystemInformation`` option.
 
 The following table shows how each type of instrumented command gets mapped
 to a corresponding type of CTest XML file.
@@ -125,6 +125,9 @@ By default the command line reported to CDash is truncated at the first space.
 You can instead choose to report the full command line (including arguments)
 by setting :envvar:`CTEST_USE_VERBOSE_INSTRUMENTATION` to 1.
 
+Alternatively, you can use the `v1 Query Files`_ to enable instrumentation for
+CDash using the ``cdashSubmit`` and ``cdashVerbose`` options.
+
 .. _`cmake-instrumentation API v1`:
 
 API v1
@@ -146,8 +149,9 @@ subdirectories:
 
 ``query/generated/``
   Holds query files generated by a CMake project with the
-  :command:`cmake_instrumentation` command. These files are owned by CMake and
-  are deleted and regenerated automatically during the CMake configure step.
+  :command:`cmake_instrumentation` command or the
+  :envvar:`CTEST_USE_INSTRUMENTATION` variable. These files are owned by CMake
+  and are deleted and regenerated automatically during the CMake configure step.
 
 ``data/``
   Holds instrumentation data collected on the project. CMake owns all data
@@ -192,9 +196,10 @@ key is required, but all other fields are optional.
   * ``postInstall``
   * ``postTest``
 
-``queries``
-  A list of strings specifying additional optional data to collect during
-  instrumentation. Elements in this list should be one of the following:
+``options``
+  A list of strings used to enable certain optional behavior, including the
+  collection of certain additional data. Elements in this list should be one of
+  the following:
 
     ``staticSystemInformation``
       Enables collection of the static information about the host machine CMake
@@ -207,13 +212,26 @@ key is required, but all other fields are optional.
       generated by CMake, and includes information from immediately before and
       after the command is executed.
 
+    ``cdashSubmit``
+      Enables including instrumentation data in CDash. This does not
+      automatically enable ``dynamicSystemInformation``, but is otherwise
+      equivalent to having the :envvar:`CTEST_USE_INSTRUMENTATION` environment
+      variable enabled.
+
+    ``cdashVerbose``
+      Enables including the full untruncated commands in data submitted to
+      CDash. Equivalent to having the
+      :envvar:`CTEST_USE_VERBOSE_INSTRUMENTATION` environment variable enabled.
+
 The ``callbacks`` listed will be invoked during the specified hooks
 *at a minimum*. When there are multiple query files, the ``callbacks``,
-``hooks`` and ``queries`` between them will be merged. Therefore, if any query
+``hooks`` and ``options`` between them will be merged. Therefore, if any query
 file includes any ``hooks``, every ``callback`` across all query files will be
 executed at every ``hook`` across all query files. Additionally, if any query
-file includes any optional ``queries``, the optional query data will be present
-in all data files.
+file requests optional data using the ``options`` field, any related data will
+be present in all snippet files. User written ``callbacks`` should be able to
+handle the presence of this optional data, since it may be requested by an
+unrelated query.
 
 Example:
 
@@ -229,9 +247,10 @@ Example:
       "postCMakeBuild",
       "postInstall"
     ],
-    "queries": [
+    "options": [
       "staticSystemInformation",
-      "dynamicSystemInformation"
+      "dynamicSystemInformation",
+      "cdashSubmit"
     ]
   }
 
@@ -240,12 +259,14 @@ invocation, an index file ``index-<timestamp>.json`` will be generated in
 ``<build>/.cmake/instrumentation/v1/data`` containing a list of data snippet
 files created since the previous indexing. The commands
 ``/usr/bin/python callback.py index-<timestamp>.json`` and
-``/usr/bin/cmake -P callback.cmake arg index-<timestamp>.json`` will be executed in
-that order. The index file will contain the ``staticSystemInformation`` data and
-each snippet file listed in the index will contain the
+``/usr/bin/cmake -P callback.cmake arg index-<timestamp>.json`` will be executed
+in that order. The index file will contain the ``staticSystemInformation`` data
+and each snippet file listed in the index will contain the
 ``dynamicSystemInformation`` data. Once both callbacks have completed, the index
 file and all snippet files listed by it will be deleted from the project build
-tree.
+tree. The instrumentation data will be present in the XML files submitted to
+CDash, but with truncated command strings because ``cdashVerbose`` was not
+enabled.
 
 .. _`cmake-instrumentation Data v1`:
 

Source/cmCTest.cxx

@@ -114,7 +114,6 @@ struct cmCTest::Private
   bool UseHTTP10 = false;
   bool PrintLabels = false;
   bool Failover = false;
-  bool UseVerboseInstrumentation = false;
   cmJSONState parseState;
 
   bool FlushTestProgressLine = false;
@@ -320,10 +319,6 @@ cmCTest::cmCTest()
     this->Impl->TestProgressOutput = !cmIsOff(envValue);
   }
   envValue.clear();
-  if (cmSystemTools::GetEnv("CTEST_USE_VERBOSE_INSTRUMENTATION", envValue)) {
-    this->Impl->UseVerboseInstrumentation = !cmIsOff(envValue);
-  }
-  envValue.clear();
 
   this->Impl->Parts[PartStart].SetName("Start");
   this->Impl->Parts[PartUpdate].SetName("Update");
@@ -3629,11 +3624,6 @@ cmInstrumentation& cmCTest::GetInstrumentation()
   return *this->Impl->Instrumentation;
 }
 
-bool cmCTest::GetUseVerboseInstrumentation() const
-{
-  return this->Impl->UseVerboseInstrumentation;
-}
-
 void cmCTest::ConvertInstrumentationSnippetsToXML(cmXMLWriter& xml,
                                                   std::string const& subdir)
 {
@@ -3662,6 +3652,8 @@ void cmCTest::ConvertInstrumentationSnippetsToXML(cmXMLWriter& xml,
 bool cmCTest::ConvertInstrumentationJSONFileToXML(std::string const& fpath,
                                                   cmXMLWriter& xml)
 {
+  bool verboseCommands = this->GetInstrumentation().HasOption(
+    cmInstrumentationQuery::Option::CDashVerbose);
   Json::Value root;
   this->Impl->parseState = cmJSONState(fpath, &root);
   if (!this->Impl->parseState.errors.empty()) {
@@ -3710,7 +3702,7 @@ bool cmCTest::ConvertInstrumentationJSONFileToXML(std::string const& fpath,
       }
       // Truncate the full command line if verbose instrumentation
       // was not requested.
-      if (key == "command" && !this->GetUseVerboseInstrumentation()) {
+      if (key == "command" && !verboseCommands) {
         std::string command_str = root[key].asString();
         std::string truncated = command_str.substr(0, command_str.find(' '));
         if (command_str != truncated) {

Source/cmCTest.h

@@ -436,7 +436,6 @@ public:
   std::vector<std::string> GetCommandLineHttpHeaders() const;
 
   cmInstrumentation& GetInstrumentation();
-  bool GetUseVerboseInstrumentation() const;
 
 private:
   int GenerateNotesFile(cmake* cm, std::string const& files);

Source/cmExperimental.cxx

@@ -65,7 +65,7 @@ cmExperimental::FeatureData const LookupTable[] = {
     cmExperimental::TryCompileCondition::Never },
   // Instrumentation
   { "Instrumentation",
-    "a37d1069-1972-4901-b9c9-f194aaf2b6e0",
+    "d16a3082-c4e1-489b-b90c-55750a334f27",
     "CMAKE_EXPERIMENTAL_INSTRUMENTATION",
     "CMake's support for collecting instrumentation data is experimental. It "
     "is meant only for experimentation and feedback to CMake developers.",

Source/cmInstrumentation.cxx

@@ -29,6 +29,53 @@
 
 using LoadQueriesAfter = cmInstrumentation::LoadQueriesAfter;
 
+std::map<std::string, std::string> cmInstrumentation::cdashSnippetsMap = {
+  {
+    "configure",
+    "configure",
+  },
+  {
+    "generate",
+    "configure",
+  },
+  {
+    "compile",
+    "build",
+  },
+  {
+    "link",
+    "build",
+  },
+  {
+    "custom",
+    "build",
+  },
+  {
+    "build",
+    "skip",
+  },
+  {
+    "cmakeBuild",
+    "build",
+  },
+  {
+    "cmakeInstall",
+    "build",
+  },
+  {
+    "install",
+    "build",
+  },
+  {
+    "ctest",
+    "build",
+  },
+  {
+    "test",
+    "test",
+  }
+};
+
 cmInstrumentation::cmInstrumentation(std::string const& binary_dir,
                                      LoadQueriesAfter loadQueries)
 {
@@ -38,6 +85,7 @@ cmInstrumentation::cmInstrumentation(std::string const& binary_dir,
   this->binaryDir = binary_dir;
   this->timingDirv1 =
     cmStrCat(this->binaryDir, "/.cmake/instrumentation-", uuid, "/v1");
+  this->cdashDir = cmStrCat(this->timingDirv1, "/cdash");
   if (cm::optional<std::string> configDir =
         cmSystemTools::GetCMakeConfigDirectory()) {
     this->userTimingDirv1 =
@@ -60,7 +108,10 @@ void cmInstrumentation::LoadQueries()
     this->hasQuery = this->hasQuery ||
       this->ReadJSONQueries(cmStrCat(this->userTimingDirv1, "/query"));
   }
+}
 
+void cmInstrumentation::CheckCDashVariable()
+{
   std::string envVal;
   if (cmSystemTools::GetEnv("CTEST_USE_INSTRUMENTATION", envVal) &&
       !cmIsOff(envVal)) {
@@ -69,63 +120,23 @@ void cmInstrumentation::LoadQueries()
                                  cmExperimental::Feature::Instrumentation)
                                  .Uuid;
       if (envVal == uuid) {
+        std::set<cmInstrumentationQuery::Option> options_ = {
+          cmInstrumentationQuery::Option::CDashSubmit,
+          cmInstrumentationQuery::Option::DynamicSystemInformation
+        };
+        if (cmSystemTools::GetEnv("CTEST_USE_VERBOSE_INSTRUMENTATION",
+                                  envVal) &&
+            !cmIsOff(envVal)) {
+          options_.insert(cmInstrumentationQuery::Option::CDashVerbose);
+        }
+        for (auto const& option : options_) {
+          this->AddOption(option);
+        }
+        std::set<cmInstrumentationQuery::Hook> hooks_ = {
+          cmInstrumentationQuery::Hook::PrepareForCDash
+        };
         this->AddHook(cmInstrumentationQuery::Hook::PrepareForCDash);
-        this->AddQuery(
-          cmInstrumentationQuery::Query::DynamicSystemInformation);
-        this->cdashDir = cmStrCat(this->timingDirv1, "/cdash");
-        cmSystemTools::MakeDirectory(this->cdashDir);
-        cmSystemTools::MakeDirectory(cmStrCat(this->cdashDir, "/configure"));
-        cmSystemTools::MakeDirectory(cmStrCat(this->cdashDir, "/build"));
-        cmSystemTools::MakeDirectory(
-          cmStrCat(this->cdashDir, "/build/commands"));
-        cmSystemTools::MakeDirectory(
-          cmStrCat(this->cdashDir, "/build/targets"));
-        cmSystemTools::MakeDirectory(cmStrCat(this->cdashDir, "/test"));
-        this->cdashSnippetsMap = { {
-                                     "configure",
-                                     "configure",
-                                   },
-                                   {
-                                     "generate",
-                                     "configure",
-                                   },
-                                   {
-                                     "compile",
-                                     "build",
-                                   },
-                                   {
-                                     "link",
-                                     "build",
-                                   },
-                                   {
-                                     "custom",
-                                     "build",
-                                   },
-                                   {
-                                     "build",
-                                     "skip",
-                                   },
-                                   {
-                                     "cmakeBuild",
-                                     "build",
-                                   },
-                                   {
-                                     "cmakeInstall",
-                                     "build",
-                                   },
-                                   {
-                                     "install",
-                                     "build",
-                                   },
-                                   {
-                                     "ctest",
-                                     "build",
-                                   },
-                                   {
-                                     "test",
-                                     "test",
-                                   } };
-        this->hasQuery = true;
+        this->WriteJSONQuery(options_, hooks_, {});
       }
     }
   }
@@ -159,7 +170,7 @@ bool cmInstrumentation::ReadJSONQueries(std::string const& directory)
 void cmInstrumentation::ReadJSONQuery(std::string const& file)
 {
   auto query = cmInstrumentationQuery();
-  query.ReadJSON(file, this->errorMsg, this->queries, this->hooks,
+  query.ReadJSON(file, this->errorMsg, this->options, this->hooks,
                  this->callbacks);
   if (!this->errorMsg.empty()) {
     cmSystemTools::Error(cmStrCat(
@@ -174,15 +185,15 @@ bool cmInstrumentation::HasErrors() const
 }
 
 void cmInstrumentation::WriteJSONQuery(
-  std::set<cmInstrumentationQuery::Query> const& queries_,
+  std::set<cmInstrumentationQuery::Option> const& options_,
   std::set<cmInstrumentationQuery::Hook> const& hooks_,
   std::vector<std::vector<std::string>> const& callbacks_)
 {
   Json::Value root;
   root["version"] = 1;
-  root["queries"] = Json::arrayValue;
-  for (auto const& query : queries_) {
-    root["queries"].append(cmInstrumentationQuery::QueryString[query]);
+  root["options"] = Json::arrayValue;
+  for (auto const& option : options_) {
+    root["options"].append(cmInstrumentationQuery::OptionString[option]);
   }
   root["hooks"] = Json::arrayValue;
   for (auto const& hook : hooks_) {
@@ -192,13 +203,9 @@ void cmInstrumentation::WriteJSONQuery(
   for (auto const& callback : callbacks_) {
     root["callbacks"].append(cmInstrumentation::GetCommandStr(callback));
   }
-  cmsys::Directory d;
-  int n = 0;
-  if (d.Load(cmStrCat(this->timingDirv1, "/query/generated"))) {
-    n = (int)d.GetNumberOfFiles() - 2; // Don't count '.' or '..'
-  }
-  this->WriteInstrumentationJson(root, "query/generated",
-                                 cmStrCat("query-", n, ".json"));
+  this->WriteInstrumentationJson(
+    root, "query/generated",
+    cmStrCat("query-", this->writtenJsonQueries++, ".json"));
 }
 
 void cmInstrumentation::ClearGeneratedQueries()
@@ -214,9 +221,9 @@ bool cmInstrumentation::HasQuery() const
   return this->hasQuery;
 }
 
-bool cmInstrumentation::HasQuery(cmInstrumentationQuery::Query query) const
+bool cmInstrumentation::HasOption(cmInstrumentationQuery::Option option) const
 {
-  return (this->queries.find(query) != this->queries.end());
+  return (this->options.find(option) != this->options.end());
 }
 
 bool cmInstrumentation::HasHook(cmInstrumentationQuery::Hook hook) const
@@ -281,7 +288,8 @@ int cmInstrumentation::CollectTimingData(cmInstrumentationQuery::Hook hook)
   index["dataDir"] = directory;
   index["buildDir"] = this->binaryDir;
   index["version"] = 1;
-  if (this->HasQuery(cmInstrumentationQuery::Query::StaticSystemInformation)) {
+  if (this->HasOption(
+        cmInstrumentationQuery::Option::StaticSystemInformation)) {
     this->InsertStaticSystemInformation(index);
   }
   for (auto const& file : files) {
@@ -305,7 +313,7 @@ int cmInstrumentation::CollectTimingData(cmInstrumentationQuery::Hook hook)
   }
 
   // Special case for CDash collation
-  if (this->HasHook(cmInstrumentationQuery::Hook::PrepareForCDash)) {
+  if (this->HasOption(cmInstrumentationQuery::Option::CDashSubmit)) {
     this->PrepareDataForCDash(directory, index_path);
   }
 
@@ -427,8 +435,8 @@ std::string cmInstrumentation::InstrumentTest(
 
   // Post-Command
   this->InsertTimingData(root, steadyStart, systemStart);
-  if (this->HasQuery(
-        cmInstrumentationQuery::Query::DynamicSystemInformation)) {
+  if (this->HasOption(
+        cmInstrumentationQuery::Option::DynamicSystemInformation)) {
     this->InsertDynamicSystemInformation(root, "after");
   }
 
@@ -443,8 +451,8 @@ std::string cmInstrumentation::InstrumentTest(
 
 void cmInstrumentation::GetPreTestStats()
 {
-  if (this->HasQuery(
-        cmInstrumentationQuery::Query::DynamicSystemInformation)) {
+  if (this->HasOption(
+        cmInstrumentationQuery::Option::DynamicSystemInformation)) {
     this->InsertDynamicSystemInformation(this->preTestStats, "before");
   }
 }
@@ -452,8 +460,8 @@ void cmInstrumentation::GetPreTestStats()
 int cmInstrumentation::InstrumentCommand(
   std::string command_type, std::vector<std::string> const& command,
   std::function<int()> const& callback,
-  cm::optional<std::map<std::string, std::string>> options,
-  cm::optional<std::map<std::string, std::string>> arrayOptions,
+  cm::optional<std::map<std::string, std::string>> data,
+  cm::optional<std::map<std::string, std::string>> arrayData,
   LoadQueriesAfter reloadQueriesAfterCommand)
 {
 
@@ -478,8 +486,8 @@ int cmInstrumentation::InstrumentCommand(
   auto system_start = std::chrono::system_clock::now();
   double preConfigureMemory = 0;
   double preConfigureLoad = 0;
-  if (this->HasQuery(
-        cmInstrumentationQuery::Query::DynamicSystemInformation)) {
+  if (this->HasOption(
+        cmInstrumentationQuery::Option::DynamicSystemInformation)) {
     this->InsertDynamicSystemInformation(root, "before");
   } else if (reloadQueriesAfterCommand == LoadQueriesAfter::Yes) {
     this->GetDynamicSystemInformation(preConfigureMemory, preConfigureLoad);
@@ -495,8 +503,8 @@ int cmInstrumentation::InstrumentCommand(
     if (!this->HasQuery()) {
       return ret;
     }
-    if (this->HasQuery(
-          cmInstrumentationQuery::Query::DynamicSystemInformation)) {
+    if (this->HasOption(
+          cmInstrumentationQuery::Option::DynamicSystemInformation)) {
       root["dynamicSystemInformation"] = Json::objectValue;
       root["dynamicSystemInformation"]["beforeHostMemoryUsed"] =
         preConfigureMemory;
@@ -507,14 +515,14 @@ int cmInstrumentation::InstrumentCommand(
 
   // Post-Command
   this->InsertTimingData(root, steady_start, system_start);
-  if (this->HasQuery(
-        cmInstrumentationQuery::Query::DynamicSystemInformation)) {
+  if (this->HasOption(
+        cmInstrumentationQuery::Option::DynamicSystemInformation)) {
     this->InsertDynamicSystemInformation(root, "after");
   }
 
   // Gather additional data
-  if (options.has_value()) {
-    for (auto const& item : options.value()) {
+  if (data.has_value()) {
+    for (auto const& item : data.value()) {
       if (item.first == "role" && !item.second.empty()) {
         command_type = item.second;
       } else if (!item.second.empty()) {
@@ -529,8 +537,8 @@ int cmInstrumentation::InstrumentCommand(
     root["config"] = "";
   }
 
-  if (arrayOptions.has_value()) {
-    for (auto const& item : arrayOptions.value()) {
+  if (arrayData.has_value()) {
+    for (auto const& item : arrayData.value()) {
       if (item.first == "targetLabels" && command_type != "link") {
         continue;
       }
@@ -659,9 +667,9 @@ void cmInstrumentation::AddHook(cmInstrumentationQuery::Hook hook)
   this->hooks.insert(hook);
 }
 
-void cmInstrumentation::AddQuery(cmInstrumentationQuery::Query query)
+void cmInstrumentation::AddOption(cmInstrumentationQuery::Option option)
 {
-  this->queries.insert(query);
+  this->options.insert(option);
 }
 
 std::string const& cmInstrumentation::GetCDashDir()
@@ -675,6 +683,13 @@ std::string const& cmInstrumentation::GetCDashDir()
 void cmInstrumentation::PrepareDataForCDash(std::string const& data_dir,
                                             std::string const& index_path)
 {
+  cmSystemTools::MakeDirectory(this->cdashDir);
+  cmSystemTools::MakeDirectory(cmStrCat(this->cdashDir, "/configure"));
+  cmSystemTools::MakeDirectory(cmStrCat(this->cdashDir, "/build"));
+  cmSystemTools::MakeDirectory(cmStrCat(this->cdashDir, "/build/commands"));
+  cmSystemTools::MakeDirectory(cmStrCat(this->cdashDir, "/build/targets"));
+  cmSystemTools::MakeDirectory(cmStrCat(this->cdashDir, "/test"));
+
   Json::Value root;
   std::string error_msg;
   cmJSONState parseState = cmJSONState(index_path, &root);

Source/cmInstrumentation.h

@@ -33,6 +33,7 @@ public:
   cmInstrumentation(std::string const& binary_dir,
                     LoadQueriesAfter loadQueries = LoadQueriesAfter::Yes);
   void LoadQueries();
+  void CheckCDashVariable();
   int InstrumentCommand(
     std::string command_type, std::vector<std::string> const& command,
     std::function<int()> const& callback,
@@ -49,12 +50,12 @@ public:
                              std::string config);
   void GetPreTestStats();
   bool HasQuery() const;
-  bool HasQuery(cmInstrumentationQuery::Query) const;
-  bool HasHook(cmInstrumentationQuery::Hook) const;
+  bool HasOption(cmInstrumentationQuery::Option option) const;
+  bool HasHook(cmInstrumentationQuery::Hook hook) const;
   bool HasPreOrPostBuildHook() const;
   bool ReadJSONQueries(std::string const& directory);
   void ReadJSONQuery(std::string const& file);
-  void WriteJSONQuery(std::set<cmInstrumentationQuery::Query> const& queries,
+  void WriteJSONQuery(std::set<cmInstrumentationQuery::Option> const& options,
                       std::set<cmInstrumentationQuery::Hook> const& hooks,
                       std::vector<std::vector<std::string>> const& callback);
   void ClearGeneratedQueries();
@@ -62,7 +63,7 @@ public:
   int SpawnBuildDaemon();
   int CollectTimingAfterBuild(int ppid);
   void AddHook(cmInstrumentationQuery::Hook hook);
-  void AddQuery(cmInstrumentationQuery::Query query);
+  void AddOption(cmInstrumentationQuery::Option option);
   bool HasErrors() const;
   std::string const& GetCDashDir();
 
@@ -87,11 +88,11 @@ private:
   std::string timingDirv1;
   std::string userTimingDirv1;
   std::string cdashDir;
-  std::set<cmInstrumentationQuery::Query> queries;
+  std::set<cmInstrumentationQuery::Option> options;
   std::set<cmInstrumentationQuery::Hook> hooks;
   std::vector<std::string> callbacks;
   std::vector<std::string> queryFiles;
-  std::map<std::string, std::string> cdashSnippetsMap;
+  static std::map<std::string, std::string> cdashSnippetsMap;
   Json::Value preTestStats;
   std::string errorMsg;
   bool hasQuery = false;
@@ -101,4 +102,5 @@ private:
   std::unique_ptr<cmsys::SystemInformation> systemInformation;
   cmsys::SystemInformation& GetSystemInformation();
 #endif
+  int writtenJsonQueries = 0;
 };

Source/cmInstrumentationCommand.cxx

@@ -35,9 +35,9 @@ bool validateVersion(std::string const& key, std::string const& versionString,
   }
   version = std::atoi(versionString.c_str());
   if (version != 1) {
-    status.SetError(cmStrCat(
-      "QUERY subcommand given an unsupported ", key, " \"", versionString,
-      "\" (the only currently supported version is 1)."));
+    status.SetError(
+      cmStrCat("given an unsupported ", key, " \"", versionString,
+               "\" (the only currently supported version is 1)."));
     return false;
   }
   return true;
@@ -79,7 +79,7 @@ bool cmInstrumentationCommand(std::vector<std::string> const& args,
   {
     ArgumentParser::NonEmpty<std::string> ApiVersion;
     ArgumentParser::NonEmpty<std::string> DataVersion;
-    ArgumentParser::NonEmpty<std::vector<std::string>> Queries;
+    ArgumentParser::NonEmpty<std::vector<std::string>> Options;
     ArgumentParser::NonEmpty<std::vector<std::string>> Hooks;
     ArgumentParser::NonEmpty<std::vector<std::vector<std::string>>> Callbacks;
   };
@@ -87,7 +87,7 @@ bool cmInstrumentationCommand(std::vector<std::string> const& args,
   static auto const parser = cmArgumentParser<Arguments>{}
                                .Bind("API_VERSION"_s, &Arguments::ApiVersion)
                                .Bind("DATA_VERSION"_s, &Arguments::DataVersion)
-                               .Bind("QUERIES"_s, &Arguments::Queries)
+                               .Bind("OPTIONS"_s, &Arguments::Options)
                                .Bind("HOOKS"_s, &Arguments::Hooks)
                                .Bind("CALLBACK"_s, &Arguments::Callbacks);
 
@@ -111,17 +111,17 @@ bool cmInstrumentationCommand(std::vector<std::string> const& args,
     return false;
   }
 
-  std::set<cmInstrumentationQuery::Query> queries;
-  auto queryParser = EnumParser<cmInstrumentationQuery::Query>(
-    cmInstrumentationQuery::QueryString);
-  for (auto const& arg : arguments.Queries) {
-    cmInstrumentationQuery::Query query;
-    if (!queryParser(arg, query)) {
+  std::set<cmInstrumentationQuery::Option> options;
+  auto optionParser = EnumParser<cmInstrumentationQuery::Option>(
+    cmInstrumentationQuery::OptionString);
+  for (auto const& arg : arguments.Options) {
+    cmInstrumentationQuery::Option option;
+    if (!optionParser(arg, option)) {
       status.SetError(
-        cmStrCat("given invalid argument to QUERIES \"", arg, '"'));
+        cmStrCat("given invalid argument to OPTIONS \"", arg, '"'));
       return false;
     }
-    queries.insert(query);
+    options.insert(option);
   }
 
   std::set<cmInstrumentationQuery::Hook> hooks;
@@ -140,7 +140,7 @@ bool cmInstrumentationCommand(std::vector<std::string> const& args,
   status.GetMakefile()
     .GetCMakeInstance()
     ->GetInstrumentation()
-    ->WriteJSONQuery(queries, hooks, arguments.Callbacks);
+    ->WriteJSONQuery(options, hooks, arguments.Callbacks);
 
   return true;
 }

Source/cmInstrumentationQuery.cxx

@@ -15,8 +15,9 @@
 #include "cmJSONHelpers.h"
 #include "cmStringAlgorithms.h"
 
-std::vector<std::string> const cmInstrumentationQuery::QueryString{
-  "staticSystemInformation", "dynamicSystemInformation"
+std::vector<std::string> const cmInstrumentationQuery::OptionString{
+  "staticSystemInformation", "dynamicSystemInformation", "cdashSubmit",
+  "cdashVerbose"
 };
 std::vector<std::string> const cmInstrumentationQuery::HookString{
   "postGenerate",  "preBuild",        "postBuild",
@@ -64,11 +65,11 @@ static std::function<bool(E&, Json::Value const*, cmJSONState*)> EnumHelper(
     return false;
   };
 }
-static auto const QueryHelper = EnumHelper<cmInstrumentationQuery::Query>(
-  cmInstrumentationQuery::QueryString, "query");
+static auto const OptionHelper = EnumHelper<cmInstrumentationQuery::Option>(
+  cmInstrumentationQuery::OptionString, "option");
 static auto const QueryListHelper =
-  JSONHelperBuilder::Vector<cmInstrumentationQuery::Query>(
-    ErrorMessages::InvalidArray, QueryHelper);
+  JSONHelperBuilder::Vector<cmInstrumentationQuery::Option>(
+    ErrorMessages::InvalidArray, OptionHelper);
 static auto const HookHelper = EnumHelper<cmInstrumentationQuery::Hook>(
   cmInstrumentationQuery::HookString, "hook");
 static auto const HookListHelper =
@@ -85,13 +86,13 @@ static auto const QueryRootHelper =
   JSONHelperBuilder::Object<QueryRoot>(ErrorMessages::InvalidRootQueryObject,
                                        false)
     .Bind("version"_s, &QueryRoot::version, VersionHelper, true)
-    .Bind("queries"_s, &QueryRoot::queries, QueryListHelper, false)
+    .Bind("options"_s, &QueryRoot::options, QueryListHelper, false)
     .Bind("hooks"_s, &QueryRoot::hooks, HookListHelper, false)
     .Bind("callbacks"_s, &QueryRoot::callbacks, CallbackListHelper, false);
 
 bool cmInstrumentationQuery::ReadJSON(std::string const& filename,
                                       std::string& errorMessage,
-                                      std::set<Query>& queries,
+                                      std::set<Option>& options,
                                       std::set<Hook>& hooks,
                                       std::vector<std::string>& callbacks)
 {
@@ -105,8 +106,8 @@ bool cmInstrumentationQuery::ReadJSON(std::string const& filename,
     errorMessage = this->parseState.GetErrorMessage(true);
     return false;
   }
-  std::move(this->queryRoot.queries.begin(), this->queryRoot.queries.end(),
-            std::inserter(queries, queries.end()));
+  std::move(this->queryRoot.options.begin(), this->queryRoot.options.end(),
+            std::inserter(options, options.end()));
   std::move(this->queryRoot.hooks.begin(), this->queryRoot.hooks.end(),
             std::inserter(hooks, hooks.end()));
   std::move(this->queryRoot.callbacks.begin(), this->queryRoot.callbacks.end(),

Source/cmInstrumentationQuery.h

@@ -12,12 +12,14 @@ class cmInstrumentationQuery
 {
 
 public:
-  enum Query
+  enum Option
   {
     StaticSystemInformation,
-    DynamicSystemInformation
+    DynamicSystemInformation,
+    CDashSubmit,
+    CDashVerbose
   };
-  static std::vector<std::string> const QueryString;
+  static std::vector<std::string> const OptionString;
 
   enum Hook
   {
@@ -35,7 +37,7 @@ public:
 
   struct QueryJSONRoot
   {
-    std::vector<cmInstrumentationQuery::Query> queries;
+    std::vector<cmInstrumentationQuery::Option> options;
     std::vector<cmInstrumentationQuery::Hook> hooks;
     std::vector<std::string> callbacks;
     int version;
@@ -43,7 +45,7 @@ public:
 
   cmInstrumentationQuery() = default;
   bool ReadJSON(std::string const& file, std::string& errorMessage,
-                std::set<Query>& queries, std::set<Hook>& hooks,
+                std::set<Option>& options, std::set<Hook>& hooks,
                 std::vector<std::string>& callbacks);
   QueryJSONRoot queryRoot;
   cmJSONState parseState;

Source/cmake.cxx

@@ -2644,6 +2644,7 @@ int cmake::ActualConfigure()
       cmStrCat(this->GetHomeOutputDirectory(), "/CMakeFiles"_s),
       this->FileAPI->GetConfigureLogVersions());
     this->Instrumentation->LoadQueries();
+    this->Instrumentation->CheckCDashVariable();
   }
 #endif
 

Tests/RunCMake/ConfigDir/check-reply.cmake

@@ -1,6 +1,6 @@
 if (NOT EXISTS ${RunCMake_TEST_BINARY_DIR}/.cmake/api/v1/reply)
   set(RunCMake_TEST_FAILED "Failed to read FileAPI query from user config directory")
 endif()
-if (NOT EXISTS ${RunCMake_TEST_BINARY_DIR}/.cmake/instrumentation-a37d1069-1972-4901-b9c9-f194aaf2b6e0/v1/data)
+if (NOT EXISTS ${RunCMake_TEST_BINARY_DIR}/.cmake/instrumentation-d16a3082-c4e1-489b-b90c-55750a334f27/v1/data)
   set(RunCMake_TEST_FAILED "Failed to read Instrumentation query from user config directory")
 endif()

Tests/RunCMake/Instrumentation/RunCMakeTest.cmake

@@ -9,7 +9,7 @@ function(instrument test)
     "BUILD;BUILD_MAKE_PROGRAM;INSTALL;TEST;COPY_QUERIES;COPY_QUERIES_GENERATED;NO_WARN;STATIC_QUERY;DYNAMIC_QUERY;INSTALL_PARALLEL;MANUAL_HOOK"
     "CHECK_SCRIPT;CONFIGURE_ARG" "" ${ARGN})
   set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/${test})
-  set(uuid "a37d1069-1972-4901-b9c9-f194aaf2b6e0")
+  set(uuid "d16a3082-c4e1-489b-b90c-55750a334f27")
   set(v1 ${RunCMake_TEST_BINARY_DIR}/.cmake/instrumentation-${uuid}/v1)
   set(query_dir ${CMAKE_CURRENT_LIST_DIR}/query)
 
@@ -90,7 +90,7 @@ function(instrument test)
 endfunction()
 
 # Bad Queries
-instrument(bad-query)
+instrument(bad-option)
 instrument(bad-hook)
 instrument(empty)
 instrument(bad-version)
@@ -100,7 +100,7 @@ instrument(hooks-1 BUILD INSTALL TEST STATIC_QUERY)
 instrument(hooks-2 BUILD INSTALL TEST)
 instrument(hooks-no-callbacks MANUAL_HOOK)
 
-# Check data file contents
+# Check data file contents for optional query data
 instrument(no-query BUILD INSTALL TEST
   CHECK_SCRIPT check-data-dir.cmake)
 instrument(dynamic-query BUILD INSTALL TEST DYNAMIC_QUERY

Tests/RunCMake/Instrumentation/bad-option-stderr.txt

@@ -0,0 +1,5 @@
+^CMake Error: Could not load instrumentation queries from [^
+]+:
+bad-option.json:[0-9]+: Not a valid option: "bad option"
+  "options": \["staticSystemInformation", "bad option"\]
+                                         \^$

Tests/RunCMake/Instrumentation/bad-query-stderr.txt

@@ -1,5 +0,0 @@
-^CMake Error: Could not load instrumentation queries from [^
-]+:
-bad-query.json:[0-9]+: Not a valid query: "bad query"
-  "queries": \["staticSystemInformation", "bad query"\]
-                                         \^$

Tests/RunCMake/Instrumentation/cmake-command-bad-api-version-stderr.txt

@@ -1,6 +1,6 @@
 CMake Error at [^
 ]*\(cmake_instrumentation\):
-  cmake_instrumentation QUERY subcommand given an unsupported API_VERSION "0"
-  \(the only currently supported version is 1\).
+  cmake_instrumentation given an unsupported API_VERSION "0" \(the only
+  currently supported version is 1\).
 Call Stack \(most recent call first\):
   CMakeLists.txt:6 \(include\)

Tests/RunCMake/Instrumentation/cmake-command-missing-version-stderr.txt

@@ -1,6 +1,6 @@
 CMake Error at [^
 ]*\(cmake_instrumentation\):
-  cmake_instrumentation QUERY subcommand given an unsupported DATA_VERSION ""
-  \(the only currently supported version is 1\).
+  cmake_instrumentation given an unsupported DATA_VERSION "" \(the only
+  currently supported version is 1\).
 Call Stack \(most recent call first\):
   CMakeLists.txt:6 \(include\)

Tests/RunCMake/Instrumentation/project/CMakeLists.txt

@@ -2,7 +2,7 @@ cmake_minimum_required(VERSION 3.30)
 project(instrumentation)
 enable_testing()
 if (EXISTS ${INSTRUMENT_COMMAND_FILE})
-  set(CMAKE_EXPERIMENTAL_INSTRUMENTATION "a37d1069-1972-4901-b9c9-f194aaf2b6e0")
+  set(CMAKE_EXPERIMENTAL_INSTRUMENTATION "d16a3082-c4e1-489b-b90c-55750a334f27")
   include(${INSTRUMENT_COMMAND_FILE})
 endif()
 

Tests/RunCMake/Instrumentation/query/bad-option.json.in

@@ -0,0 +1,4 @@
+{
+  "version": 1,
+  "options": ["staticSystemInformation", "bad option"]
+}

Tests/RunCMake/Instrumentation/query/bad-query.json.in

@@ -1,4 +0,0 @@
-{
-  "version": 1,
-  "queries": ["staticSystemInformation", "bad query"]
-}

Tests/RunCMake/Instrumentation/query/both-query.json.in

@@ -1,6 +1,6 @@
 {
   "version": 1,
-  "queries": [
+  "options": [
     "staticSystemInformation",
     "dynamicSystemInformation"
   ]

Tests/RunCMake/Instrumentation/query/cmake-command-data.cmake

@@ -1,5 +1,5 @@
 cmake_instrumentation(
   API_VERSION 1
   DATA_VERSION 1
-  QUERIES dynamicSystemInformation
+  OPTIONS dynamicSystemInformation
 )

Tests/RunCMake/Instrumentation/query/cmake-command-parallel-install.cmake

@@ -2,5 +2,5 @@ set_property(GLOBAL PROPERTY INSTALL_PARALLEL ON)
 cmake_instrumentation(
   API_VERSION 1
   DATA_VERSION 1
-  QUERIES dynamicSystemInformation
+  OPTIONS dynamicSystemInformation
 )

Tests/RunCMake/Instrumentation/query/cmake-command.cmake

@@ -8,6 +8,7 @@
     API_VERSION 1
     DATA_VERSION 1
     HOOKS postGenerate
+    OPTIONS cdashSubmit cdashVerbose
     CALLBACK ${CMAKE_COMMAND} -E echo callback1
   )
   # Query 2
@@ -15,7 +16,7 @@
     API_VERSION 1
     DATA_VERSION 1
     HOOKS postCMakeBuild
-    QUERIES staticSystemInformation dynamicSystemInformation
+    OPTIONS staticSystemInformation dynamicSystemInformation
     CALLBACK ${CMAKE_COMMAND} -E echo callback2
     CALLBACK ${CMAKE_COMMAND} -E echo callback3
   )

Tests/RunCMake/Instrumentation/query/dynamic-query.json.in

@@ -1,6 +1,6 @@
 {
   "version": 1,
-  "queries": [
+  "options": [
     "staticSystemInformation",
     "dynamicSystemInformation"
   ]

Tests/RunCMake/Instrumentation/query/generated/query-0.json.in

@@ -1,6 +1,6 @@
 {
   "callbacks" : [],
   "hooks" : [],
-  "queries" : [],
+  "options" : [],
   "version": 1
 }

Tests/RunCMake/Instrumentation/query/generated/query-1.json.in

@@ -7,6 +7,9 @@
   [
     "postGenerate"
   ],
-  "queries" : [],
+  "options" : [
+    "cdashSubmit",
+    "cdashVerbose"
+  ],
   "version" : 1
 }

Tests/RunCMake/Instrumentation/query/generated/query-2.json.in

@@ -8,7 +8,7 @@
   [
     "postCMakeBuild"
   ],
-  "queries" :
+  "options" :
   [
     "staticSystemInformation",
     "dynamicSystemInformation"

Tests/RunCMake/Instrumentation/query/hooks-1.json.in

@@ -2,5 +2,5 @@
   "version": 1,
   "hooks": ["preCMakeBuild", "postInstall"],
   "callbacks": ["@GET_HOOK@"],
-  "queries": ["staticSystemInformation"]
+  "options": ["staticSystemInformation"]
 }

Tests/RunCMake/ctest_instrumentation/CMakeLists.txt.in

@@ -2,7 +2,7 @@ cmake_minimum_required(VERSION 3.10)
 @CASE_CMAKELISTS_PREFIX_CODE@
 project(CTestInstrumentation@CASE_NAME@)
 if(USE_INSTRUMENTATION)
-  set(CMAKE_EXPERIMENTAL_INSTRUMENTATION "a37d1069-1972-4901-b9c9-f194aaf2b6e0")
+  set(CMAKE_EXPERIMENTAL_INSTRUMENTATION "d16a3082-c4e1-489b-b90c-55750a334f27")
 endif()
 include(CTest)
 add_executable(main main.c)

Tests/RunCMake/ctest_instrumentation/InstrumentationInCTestXML-check.cmake

@@ -1,3 +1,12 @@
+set(timingDir "${RunCMake_TEST_BINARY_DIR}/.cmake/instrumentation-d16a3082-c4e1-489b-b90c-55750a334f27/v1")
+file(READ "${timingDir}/query/generated/query-0.json" jsonData)
+string(JSON options GET "${jsonData}" options)
+if (options MATCHES cdashVerbose AND NOT ${RunCMake_USE_VERBOSE_INSTRUMENTATION})
+  set(RunCMake_TEST_FAILED "cdashVerbose option not found in generated query despite environment variable")
+elseif (NOT options MATCHES cdashVerbose AND ${RunCMake_USE_VERBOSE_INSTRUMENTATION})
+  set(RunCMake_TEST_FAILED "cdashVerbose option found in generated query despite environment variable")
+endif()
+
 foreach(xml_type Configure Build Test)
   file(GLOB xml_file "${RunCMake_TEST_BINARY_DIR}/Testing/*/${xml_type}.xml")
   if(xml_file)
@@ -29,6 +38,12 @@ foreach(xml_type Configure Build Test)
       if(NOT xml_content MATCHES "<CmakeBuild")
         set(RunCMake_TEST_FAILED "<CmakeBuild> element not found in Build.xml")
       endif()
+      if(NOT RunCMake_USE_VERBOSE_INSTRUMENTATION AND NOT xml_content MATCHES "(truncated)")
+        set(RunCMake_TEST_FAILED "Commands not truncated despite cdashVerbose option")
+      endif()
+      if(verbose AND xml_content MATCHES "(truncated)")
+        set(RunCMake_TEST_FAILED "Commands truncated despite cdashVerbose option")
+      endif()
     endif()
   else()
     set(RunCMake_TEST_FAILED "${xml_type}.xml not found")
@@ -37,7 +52,7 @@ endforeach()
 
 foreach(dir_to_check "configure" "test" "build/targets" "build/commands")
   file(GLOB leftover_cdash_snippets
-    "${RunCMake_TEST_BINARY_DIR}/.cmake/instrumentation-a37d1069-1972-4901-b9c9-f194aaf2b6e0/v1/cdash/${dir_to_check}/*")
+    "${timingDir}/cdash/${dir_to_check}/*")
   if(leftover_cdash_snippets)
     set(RunCMake_TEST_FAILED "Leftover snippets found in cdash dir: ${leftover_cdash_snippets}")
   endif()

Tests/RunCMake/ctest_instrumentation/RunCMakeTest.cmake

@@ -1,9 +1,16 @@
 include(RunCTest)
 
-function(run_InstrumentationInCTestXML USE_INSTRUMENTATION)
+function(run_InstrumentationInCTestXML CASE_NAME USE_INSTRUMENTATION USE_VERBOSE_INSTRUMENTATION)
+  if(USE_VERBOSE_INSTRUMENTATION)
+    set(ENV{CTEST_USE_VERBOSE_INSTRUMENTATION} "1")
+    set(RunCMake_USE_VERBOSE_INSTRUMENTATION TRUE)
+  else()
+    set(ENV{CTEST_USE_VERBOSE_INSTRUMENTATION} "0")
+    set(RunCMake_USE_VERBOSE_INSTRUMENTATION FALSE)
+  endif()
   if(USE_INSTRUMENTATION)
     set(ENV{CTEST_USE_INSTRUMENTATION} "1")
-    set(ENV{CTEST_EXPERIMENTAL_INSTRUMENTATION} "a37d1069-1972-4901-b9c9-f194aaf2b6e0")
+    set(ENV{CTEST_EXPERIMENTAL_INSTRUMENTATION} "d16a3082-c4e1-489b-b90c-55750a334f27")
     set(RunCMake_USE_INSTRUMENTATION TRUE)
     set(CASE_NAME InstrumentationInCTestXML)
   else()
@@ -18,5 +25,6 @@ function(run_InstrumentationInCTestXML USE_INSTRUMENTATION)
   unset(RunCMake_USE_LAUNCHERS)
   unset(RunCMake_USE_INSTRUMENTATION)
 endfunction()
-run_InstrumentationInCTestXML(ON)
-run_InstrumentationInCTestXML(OFF)
+run_InstrumentationInCTestXML(InstrumentationInCTestXML ON OFF)
+run_InstrumentationInCTestXML(VerboseInstrumentationInCTestXML ON ON)
+run_InstrumentationInCTestXML(NoInstrumentationInCTestXML OFF OFF)

Tests/RunCMake/ctest_labels_for_subprojects/RunCMakeTest.cmake

@@ -38,7 +38,7 @@ file(COPY "${CTEST_RUNCMAKE_SOURCE_DIRECTORY}/MyThirdPartyDependency"
   if(USE_INSTRUMENTATION)
     set(CASE_CMAKELISTS_SUFFIX_CODE [[
   add_subdirectory(MyThirdPartyDependency)
-  set(CMAKE_EXPERIMENTAL_INSTRUMENTATION "a37d1069-1972-4901-b9c9-f194aaf2b6e0")
+  set(CMAKE_EXPERIMENTAL_INSTRUMENTATION "d16a3082-c4e1-489b-b90c-55750a334f27")
   cmake_instrumentation(DATA_VERSION 1 API_VERSION 1)
     ]])
     set(RunCMake-check-file CTestScriptVariableCommandLine-check.cmake)