Merge topic 'multiple-L-labels'

44ad3f0b7f ctest: Support multiple -L and -LE options to mean "AND"

Acked-by: Kitware Robot <[email protected]>
Acked-by: Lars Bilke <[email protected]>
Merge-request: !5329

Help/manual/ctest.1.rst

@@ -155,7 +155,10 @@ Options
  Run tests with labels matching regular expression.
 
  This option tells CTest to run only the tests whose labels match the
- given regular expression.
+ given regular expression.  When more than one ``-L`` option is given,
+ a test will only be run if each regular expression matches at least one
+ of the test's labels (i.e. the multiple ``-L`` labels form an ``AND``
+ relationship).  See `Label Matching`_.
 
 ``-R <regex>, --tests-regex <regex>``
  Run tests matching regular expression.
@@ -173,7 +176,10 @@ Options
  Exclude tests with labels matching regular expression.
 
  This option tells CTest to NOT run the tests whose labels match the
- given regular expression.
+ given regular expression.  When more than one ``-LE`` option is given,
+ a test will only be excluded if each regular expression matches at least one
+ of the test's labels (i.e. the multiple ``-LE`` labels form an ``AND``
+ relationship).  See `Label Matching`_.
 
 ``-FA <regex>, --fixture-exclude-any <regex>``
  Exclude fixtures matching ``<regex>`` from automatically adding any tests to
@@ -398,6 +404,46 @@ Specify the directory in which to look for tests.
 
 .. include:: OPTIONS_HELP.txt
 
+.. _`Label Matching`:
+
+Label Matching
+==============
+
+Tests may have labels attached to them. Tests may be included
+or excluded from a test run by filtering on the labels.
+Each individual filter is a regular expression applied to
+the labels attached to a test.
+
+When ``-L`` is used, in order for a test to be included in a
+test run, each regular expression must match at least one
+label.  Using more than one ``-L`` option means "match **all**
+of these".
+
+The ``-LE`` option works just like ``-L``, but excludes tests
+rather than including them. A test is excluded if each regular
+expression matches at least one label.
+
+If a test has no labels attached to it, then ``-L`` will never
+include that test, and ``-LE`` will never exclude that test.
+As an example of tests with labels, consider five tests,
+with the following labels:
+
+* *test1* has labels *tuesday* and *production*
+* *test2* has labels *tuesday* and *test*
+* *test3* has labels *wednesday* and *production*
+* *test4* has label *wednesday*
+* *test5* has labels *friday* and *test*
+
+Running ``ctest`` with ``-L tuesday -L test`` will select *test2*, which has
+both labels. Running CTest with ``-L test`` will select *test2* and
+*test5*, because both of them have a label that matches that regular
+expression.
+
+Because the matching works with regular expressions, take note that
+running CTest with ``-L es`` will match all five tests.
+To select the *tuesday* and *wednesday* tests together, use a single
+regular expression that matches either of them, like ``-L "tue|wed"``.
+
 .. _`Label and Subproject Summary`:
 
 Label and Subproject Summary

Source/CTest/cmCTestGenericHandler.cxx

@@ -21,32 +21,47 @@ cmCTestGenericHandler::cmCTestGenericHandler()
 
 cmCTestGenericHandler::~cmCTestGenericHandler() = default;
 
-void cmCTestGenericHandler::SetOption(const std::string& op, const char* value)
+/* Modify the given `map`, setting key `op` to `value` if `value`
+ * is non-null, otherwise removing key `op` (if it exists).
+ */
+static void SetMapValue(cmCTestGenericHandler::t_StringToString& map,
+                        const std::string& op, const char* value)
 {
   if (!value) {
-    auto remit = this->Options.find(op);
-    if (remit != this->Options.end()) {
-      this->Options.erase(remit);
-    }
+    map.erase(op);
     return;
   }
 
-  this->Options[op] = value;
+  map[op] = value;
+}
+
+void cmCTestGenericHandler::SetOption(const std::string& op, const char* value)
+{
+  SetMapValue(this->Options, op, value);
 }
 
 void cmCTestGenericHandler::SetPersistentOption(const std::string& op,
                                                 const char* value)
 {
   this->SetOption(op, value);
-  if (!value) {
-    auto remit = this->PersistentOptions.find(op);
-    if (remit != this->PersistentOptions.end()) {
-      this->PersistentOptions.erase(remit);
-    }
-    return;
+  SetMapValue(this->PersistentOptions, op, value);
+}
+
+void cmCTestGenericHandler::AddMultiOption(const std::string& op,
+                                           const std::string& value)
+{
+  if (!value.empty()) {
+    this->MultiOptions[op].emplace_back(value);
   }
+}
 
-  this->PersistentOptions[op] = value;
+void cmCTestGenericHandler::AddPersistentMultiOption(const std::string& op,
+                                                     const std::string& value)
+{
+  if (!value.empty()) {
+    this->MultiOptions[op].emplace_back(value);
+    this->PersistentMultiOptions[op].emplace_back(value);
+  }
 }
 
 void cmCTestGenericHandler::Initialize()
@@ -68,6 +83,17 @@ const char* cmCTestGenericHandler::GetOption(const std::string& op)
   return remit->second.c_str();
 }
 
+std::vector<std::string> cmCTestGenericHandler::GetMultiOption(
+  const std::string& optionName) const
+{
+  // Avoid inserting a key, which MultiOptions[op] would do.
+  auto remit = this->MultiOptions.find(optionName);
+  if (remit == this->MultiOptions.end()) {
+    return {};
+  }
+  return remit->second;
+}
+
 bool cmCTestGenericHandler::StartResultingXML(cmCTest::Part part,
                                               const char* name,
                                               cmGeneratedFileStream& xofs)

Source/CTest/cmCTestGenericHandler.h

@@ -72,11 +72,41 @@ public:
   virtual ~cmCTestGenericHandler();
 
   using t_StringToString = std::map<std::string, std::string>;
+  using t_StringToMultiString =
+    std::map<std::string, std::vector<std::string>>;
 
+  /**
+   * Options collect a single value from flags; passing the
+   * flag multiple times on the command-line *overwrites* values,
+   * and only the last one specified counts. Set an option to
+   * nullptr to "unset" it.
+   *
+   * The value is stored as a string. The values set for single
+   * and multi-options (see below) live in different spaces,
+   * so calling a single-getter for a key that has only been set
+   * as a multi-value will return nullptr.
+   */
   void SetPersistentOption(const std::string& op, const char* value);
   void SetOption(const std::string& op, const char* value);
   const char* GetOption(const std::string& op);
 
+  /**
+   * Multi-Options collect one or more values from flags; passing
+   * the flag multiple times on the command-line *adds* values,
+   * rather than overwriting the previous values.
+   *
+   * Adding an empty value does nothing.
+   *
+   * The value is stored as a vector of strings. The values set for single
+   * (see above) and multi-options live in different spaces,
+   * so calling a multi-getter for a key that has only been set
+   * as a single-value will return an empty vector.
+   */
+  void AddPersistentMultiOption(const std::string& optionName,
+                                const std::string& value);
+  void AddMultiOption(const std::string& optionName, const std::string& value);
+  std::vector<std::string> GetMultiOption(const std::string& op) const;
+
   void SetCommand(cmCTestCommand* command) { this->Command = command; }
 
   void SetSubmitIndex(int idx) { this->SubmitIndex = idx; }
@@ -100,6 +130,8 @@ protected:
   cmCTest* CTest;
   t_StringToString Options;
   t_StringToString PersistentOptions;
+  t_StringToMultiString MultiOptions;
+  t_StringToMultiString PersistentMultiOptions;
   t_StringToString LogFileNames;
 
   cmCTestCommand* Command;

Source/CTest/cmCTestTestCommand.cxx

@@ -73,11 +73,11 @@ cmCTestGenericHandler* cmCTestTestCommand::InitializeHandler()
     handler->SetOption("IncludeRegularExpression", this->Include.c_str());
   }
   if (!this->ExcludeLabel.empty()) {
-    handler->SetOption("ExcludeLabelRegularExpression",
-                       this->ExcludeLabel.c_str());
+    handler->AddMultiOption("ExcludeLabelRegularExpression",
+                            this->ExcludeLabel);
   }
   if (!this->IncludeLabel.empty()) {
-    handler->SetOption("LabelRegularExpression", this->IncludeLabel.c_str());
+    handler->AddMultiOption("LabelRegularExpression", this->IncludeLabel);
   }
   if (!this->ExcludeFixture.empty()) {
     handler->SetOption("ExcludeFixtureRegularExpression",

Source/CTest/cmCTestTestHandler.cxx

@@ -287,8 +287,6 @@ cmCTestTestHandler::cmCTestTestHandler()
 {
   this->UseUnion = false;
 
-  this->UseIncludeLabelRegExpFlag = false;
-  this->UseExcludeLabelRegExpFlag = false;
   this->UseIncludeRegExpFlag = false;
   this->UseExcludeRegExpFlag = false;
   this->UseExcludeRegExpFirst = false;
@@ -327,13 +325,11 @@ void cmCTestTestHandler::Initialize()
 
   this->TestsToRun.clear();
 
-  this->UseIncludeLabelRegExpFlag = false;
-  this->UseExcludeLabelRegExpFlag = false;
   this->UseIncludeRegExpFlag = false;
   this->UseExcludeRegExpFlag = false;
   this->UseExcludeRegExpFirst = false;
-  this->IncludeLabelRegularExpression = "";
-  this->ExcludeLabelRegularExpression = "";
+  this->IncludeLabelRegularExpressions.clear();
+  this->ExcludeLabelRegularExpressions.clear();
   this->IncludeRegExp.clear();
   this->ExcludeRegExp.clear();
   this->ExcludeFixtureRegExp.clear();
@@ -479,6 +475,22 @@ int cmCTestTestHandler::ProcessHandler()
   return 0;
 }
 
+/* Given a multi-option value `parts`, compile those parts into
+ * regular expressions in `expressions`. Skip empty values.
+ * Returns true if there were any expressions.
+ */
+static bool BuildLabelRE(const std::vector<std::string>& parts,
+                         std::vector<cmsys::RegularExpression>& expressions)
+{
+  expressions.clear();
+  for (const auto& p : parts) {
+    if (!p.empty()) {
+      expressions.emplace_back(p);
+    }
+  }
+  return !expressions.empty();
+}
+
 bool cmCTestTestHandler::ProcessOptions()
 {
   // Update internal data structure from generic one
@@ -519,18 +531,11 @@ bool cmCTestTestHandler::ProcessOptions()
     this->CTest->SetStopOnFailure(true);
   }
 
-  const char* val;
-  val = this->GetOption("LabelRegularExpression");
-  if (val) {
-    this->UseIncludeLabelRegExpFlag = true;
-    this->IncludeLabelRegExp = val;
-  }
-  val = this->GetOption("ExcludeLabelRegularExpression");
-  if (val) {
-    this->UseExcludeLabelRegExpFlag = true;
-    this->ExcludeLabelRegExp = val;
-  }
-  val = this->GetOption("IncludeRegularExpression");
+  BuildLabelRE(this->GetMultiOption("LabelRegularExpression"),
+               this->IncludeLabelRegularExpressions);
+  BuildLabelRE(this->GetMultiOption("ExcludeLabelRegularExpression"),
+               this->ExcludeLabelRegularExpressions);
+  const char* val = this->GetOption("IncludeRegularExpression");
   if (val) {
     this->UseIncludeRegExp();
     this->SetIncludeRegExp(val);
@@ -763,10 +768,40 @@ void cmCTestTestHandler::PrintLabelOrSubprojectSummary(bool doSubProject)
   cmCTestOptionalLog(this->CTest, HANDLER_OUTPUT, "\n", this->Quiet);
 }
 
+/**
+ * Check if the labels (from a test) match all the expressions.
+ *
+ * Each of the RE's must match at least one label
+ * (e.g. all of the REs must match **some** label,
+ * in order for the filter to apply to the test).
+ */
+static bool MatchLabelsAgainstFilterRE(
+  const std::vector<std::string>& labels,
+  const std::vector<cmsys::RegularExpression>& expressions)
+{
+  for (const auto& re : expressions) {
+    // check to see if the label regular expression matches
+    bool found = false; // assume it does not match
+    cmsys::RegularExpressionMatch match;
+    // loop over all labels and look for match
+    for (std::string const& l : labels) {
+      if (re.find(l.c_str(), match)) {
+        found = true;
+        break;
+      }
+    }
+    // if no match was found, exclude the test
+    if (!found) {
+      return false;
+    }
+  }
+  return true;
+}
+
 void cmCTestTestHandler::CheckLabelFilterInclude(cmCTestTestProperties& it)
 {
   // if not using Labels to filter then return
-  if (!this->UseIncludeLabelRegExpFlag) {
+  if (this->IncludeLabelRegularExpressions.empty()) {
     return;
   }
   // if there are no labels and we are filtering by labels
@@ -775,16 +810,9 @@ void cmCTestTestHandler::CheckLabelFilterInclude(cmCTestTestProperties& it)
     it.IsInBasedOnREOptions = false;
     return;
   }
-  // check to see if the label regular expression matches
-  bool found = false; // assume it does not match
-  // loop over all labels and look for match
-  for (std::string const& l : it.Labels) {
-    if (this->IncludeLabelRegularExpression.find(l)) {
-      found = true;
-    }
-  }
   // if no match was found, exclude the test
-  if (!found) {
+  if (!MatchLabelsAgainstFilterRE(it.Labels,
+                                  this->IncludeLabelRegularExpressions)) {
     it.IsInBasedOnREOptions = false;
   }
 }
@@ -792,7 +820,7 @@ void cmCTestTestHandler::CheckLabelFilterInclude(cmCTestTestProperties& it)
 void cmCTestTestHandler::CheckLabelFilterExclude(cmCTestTestProperties& it)
 {
   // if not using Labels to filter then return
-  if (!this->UseExcludeLabelRegExpFlag) {
+  if (this->ExcludeLabelRegularExpressions.empty()) {
     return;
   }
   // if there are no labels and we are excluding by labels
@@ -800,16 +828,9 @@ void cmCTestTestHandler::CheckLabelFilterExclude(cmCTestTestProperties& it)
   if (it.Labels.empty()) {
     return;
   }
-  // check to see if the label regular expression matches
-  bool found = false; // assume it does not match
-  // loop over all labels and look for match
-  for (std::string const& l : it.Labels) {
-    if (this->ExcludeLabelRegularExpression.find(l)) {
-      found = true;
-    }
-  }
   // if match was found, exclude the test
-  if (found) {
+  if (MatchLabelsAgainstFilterRE(it.Labels,
+                                 this->ExcludeLabelRegularExpressions)) {
     it.IsInBasedOnREOptions = false;
   }
 }
@@ -1704,12 +1725,6 @@ bool cmCTestTestHandler::ParseResourceGroupsProperty(
 
 bool cmCTestTestHandler::GetListOfTests()
 {
-  if (!this->IncludeLabelRegExp.empty()) {
-    this->IncludeLabelRegularExpression.compile(this->IncludeLabelRegExp);
-  }
-  if (!this->ExcludeLabelRegExp.empty()) {
-    this->ExcludeLabelRegularExpression.compile(this->ExcludeLabelRegExp);
-  }
   if (!this->IncludeRegExp.empty()) {
     this->IncludeTestsRegularExpression.compile(this->IncludeRegExp);
   }

Source/CTest/cmCTestTestHandler.h

@@ -320,20 +320,16 @@ private:
 
   std::vector<int> TestsToRun;
 
-  bool UseIncludeLabelRegExpFlag;
-  bool UseExcludeLabelRegExpFlag;
   bool UseIncludeRegExpFlag;
   bool UseExcludeRegExpFlag;
   bool UseExcludeRegExpFirst;
-  std::string IncludeLabelRegExp;
-  std::string ExcludeLabelRegExp;
   std::string IncludeRegExp;
   std::string ExcludeRegExp;
   std::string ExcludeFixtureRegExp;
   std::string ExcludeFixtureSetupRegExp;
   std::string ExcludeFixtureCleanupRegExp;
-  cmsys::RegularExpression IncludeLabelRegularExpression;
-  cmsys::RegularExpression ExcludeLabelRegularExpression;
+  std::vector<cmsys::RegularExpression> IncludeLabelRegularExpressions;
+  std::vector<cmsys::RegularExpression> ExcludeLabelRegularExpressions;
   cmsys::RegularExpression IncludeTestsRegularExpression;
   cmsys::RegularExpression ExcludeTestsRegularExpression;
 

Source/cmCTest.cxx

@@ -2108,17 +2108,17 @@ bool cmCTest::HandleCommandLineArguments(size_t& i,
   } else if (this->CheckArgument(arg, "-L"_s, "--label-regex") &&
              i < args.size() - 1) {
     i++;
-    this->GetTestHandler()->SetPersistentOption("LabelRegularExpression",
-                                                args[i].c_str());
-    this->GetMemCheckHandler()->SetPersistentOption("LabelRegularExpression",
-                                                    args[i].c_str());
+    this->GetTestHandler()->AddPersistentMultiOption("LabelRegularExpression",
+                                                     args[i]);
+    this->GetMemCheckHandler()->AddPersistentMultiOption(
+      "LabelRegularExpression", args[i]);
   } else if (this->CheckArgument(arg, "-LE"_s, "--label-exclude") &&
              i < args.size() - 1) {
     i++;
-    this->GetTestHandler()->SetPersistentOption(
-      "ExcludeLabelRegularExpression", args[i].c_str());
-    this->GetMemCheckHandler()->SetPersistentOption(
-      "ExcludeLabelRegularExpression", args[i].c_str());
+    this->GetTestHandler()->AddPersistentMultiOption(
+      "ExcludeLabelRegularExpression", args[i]);
+    this->GetMemCheckHandler()->AddPersistentMultiOption(
+      "ExcludeLabelRegularExpression", args[i]);
   }
 
   else if (this->CheckArgument(arg, "-E"_s, "--exclude-regex") &&
@@ -2268,6 +2268,15 @@ void cmCTest::SetPersistentOptionIfNotEmpty(const std::string& value,
   }
 }
 
+void cmCTest::AddPersistentMultiOptionIfNotEmpty(const std::string& value,
+                                                 const std::string& optionName)
+{
+  if (!value.empty()) {
+    this->GetTestHandler()->AddPersistentMultiOption(optionName, value);
+    this->GetMemCheckHandler()->AddPersistentMultiOption(optionName, value);
+  }
+}
+
 bool cmCTest::SetArgsFromPreset(const std::string& presetName,
                                 bool listPresets)
 {
@@ -2419,7 +2428,7 @@ bool cmCTest::SetArgsFromPreset(const std::string& presetName,
     if (expandedPreset->Filter->Include) {
       this->SetPersistentOptionIfNotEmpty(
         expandedPreset->Filter->Include->Name, "IncludeRegularExpression");
-      this->SetPersistentOptionIfNotEmpty(
+      this->AddPersistentMultiOptionIfNotEmpty(
         expandedPreset->Filter->Include->Label, "LabelRegularExpression");
 
       if (expandedPreset->Filter->Include->Index) {
@@ -2452,7 +2461,7 @@ bool cmCTest::SetArgsFromPreset(const std::string& presetName,
     if (expandedPreset->Filter->Exclude) {
       this->SetPersistentOptionIfNotEmpty(
         expandedPreset->Filter->Exclude->Name, "ExcludeRegularExpression");
-      this->SetPersistentOptionIfNotEmpty(
+      this->AddPersistentMultiOptionIfNotEmpty(
         expandedPreset->Filter->Exclude->Label,
         "ExcludeLabelRegularExpression");
 

Source/cmCTest.h

@@ -463,6 +463,8 @@ public:
 private:
   void SetPersistentOptionIfNotEmpty(const std::string& value,
                                      const std::string& optionName);
+  void AddPersistentMultiOptionIfNotEmpty(const std::string& value,
+                                          const std::string& optionName);
 
   int GenerateNotesFile(const std::string& files);
 

Source/ctest.cxx

@@ -55,8 +55,9 @@ static const char* cmDocumentationOptions[][2] = {
     "format of the test information and can be 'human' for the current text "
     "format or 'json-v1' for json format. Defaults to 'human'." },
   { "-L <regex>, --label-regex <regex>",
-    "Run tests with labels matching "
-    "regular expression." },
+    "Run tests with labels matching regular expression. "
+    "With multiple -L, run tests where each "
+    "regular expression matches at least one label." },
   { "-R <regex>, --tests-regex <regex>",
     "Run tests matching regular "
     "expression." },
@@ -64,8 +65,9 @@ static const char* cmDocumentationOptions[][2] = {
     "Exclude tests matching regular "
     "expression." },
   { "-LE <regex>, --label-exclude <regex>",
-    "Exclude tests with labels "
-    "matching regular expression." },
+    "Exclude tests with labels matching regular expression. "
+    "With multiple -LE, exclude tests where each "
+    "regular expression matches at least one label." },
   { "-FA <regex>, --fixture-exclude-any <regex>",
     "Do not automatically "
     "add any tests for "

Tests/CMakeTests/CMakeLists.txt

@@ -7,6 +7,7 @@ macro(AddCMakeTest TestName PreArgs)
   add_test(NAME CMake.${TestName}
     COMMAND ${CMAKE_EXECUTABLE} ${PreArgs}
     -P "${CMAKE_CURRENT_BINARY_DIR}/${TestName}Test.cmake" ${ARGN})
+  set_tests_properties("CMake.${TestName}" PROPERTIES LABELS "CMake;command")
 endmacro()
 
 

Tests/CTestTestLabelRegExp/test.cmake.in

@@ -27,11 +27,15 @@ expect_test_list("test1.*test3.*Total Tests: 2" --label-regex foo)
 expect_test_list("test2.*test3.*Total Tests: 2" --label-regex bar)
 expect_test_list("test1.*test2.*test3.*Total Tests: 3" --label-regex foo|bar)
 expect_test_list("Total Tests: 0" --label-regex baz)
+expect_test_list("Total Tests: 0" --label-regex foo --label-regex baz)
+expect_test_list("test3.*Total Tests: 1" --label-regex foo --label-regex bar)
 
 expect_test_list("test2.*Total Tests: 1" --label-exclude foo)
 expect_test_list("test1.*Total Tests: 1" --label-exclude bar)
 expect_test_list("Total Tests: 0" --label-exclude foo|bar)
 expect_test_list("test1.*test2.*test3.*Total Tests: 3" --label-exclude baz)
+expect_test_list("test1.*test2.*Total Tests: 2" --label-exclude foo --label-exclude bar)
+expect_test_list("test1.*test2.*test3.*Total Tests: 3" --label-exclude foo --label-exclude baz)
 
 expect_test_list("test1.*Total Tests: 1" --label-regex foo --label-exclude bar)
 expect_test_list("test2.*Total Tests: 1" --label-regex bar --label-exclude foo)

Tests/QtAutogen/TestMacros.cmake

@@ -43,6 +43,7 @@ macro(ADD_AUTOGEN_TEST NAME)
     --build-options ${build_options} ${Autogen_BUILD_OPTIONS}
     ${_TestCommand}
   )
+  set_tests_properties("${_QtXAutogen}.${NAME}" PROPERTIES LABELS "Qt${QT_TEST_VERSION}")
   list(APPEND TEST_BUILD_DIRS "${_BuildDir}")
   unset(_TestCommand)
   unset(_QtXAutogen)

Tests/QtAutogen/Tests.cmake

@@ -36,7 +36,9 @@ ADD_AUTOGEN_TEST(UnityMocSource)
 
 if(QT_TEST_ALLOW_QT_MACROS)
   ADD_AUTOGEN_TEST(MocCMP0071)
+  set_property(TEST "Qt${QT_TEST_VERSION}Autogen.MocCMP0071" APPEND PROPERTY LABELS "policy")
   ADD_AUTOGEN_TEST(MocCMP0100)
+  set_property(TEST "Qt${QT_TEST_VERSION}Autogen.MocCMP0100" APPEND PROPERTY LABELS "policy")
   ADD_AUTOGEN_TEST(MocInclude)
   ADD_AUTOGEN_TEST(MocIncludeSymlink)
   ADD_AUTOGEN_TEST(MocSkipSource)

Tests/RunCMake/CMakeLists.txt

@@ -29,6 +29,10 @@ macro(add_RunCMake_test test)
     ${TEST_ARGS}
     -P "${CMAKE_CURRENT_SOURCE_DIR}/${Test_Dir}/RunCMakeTest.cmake"
     )
+  set_tests_properties("RunCMake.${test}" PROPERTIES LABELS "CMake;run")
+  if(${test} MATCHES ^CMP)
+    set_property(TEST "RunCMake.${test}" APPEND PROPERTY LABELS "policy")
+  endif()
 endmacro()
 
 function(add_RunCMake_test_group test types)