Merge topic 'ci-valgrind'

e981fc4f4d gitlab-ci: add a job to test with Valgrind on Linux with Ninja
f1e8762a8b ci: support valgrind memcheck runs
f22d8a3f36 ci: support memcheck-testing external test processes
35337bcc06 gitlab-ci: report JUnit results for memcheck runs
8459ff022b ci: factor out memcheck "prep" logic

Acked-by: Kitware Robot <[email protected]>
Merge-request: !11257

.gitlab-ci.yml

@@ -1023,12 +1023,26 @@ t:fedora42-asan:
         - .fedora42_asan
         - .cmake_memcheck_linux
         - .linux_x86_64_tags
+        - .cmake_junit_artifacts
         - .run_dependent
     needs:
         - b:fedora42-asan
     variables:
         CMAKE_CI_JOB_NIGHTLY: "true"
 
+t:fedora42-ninja-valgrind:
+    extends:
+        - .fedora42_ninja_valgrind
+        - .cmake_test_linux_release
+        # Use a fast CPU so that timeouts are less of a concern.
+        - .linux_x86_64_tags_fast_x11
+        - .cmake_valgrind_artifacts
+        - .run_dependent
+        - .needs_centos7_x86_64
+    variables:
+        CMAKE_CI_JOB_NIGHTLY: "true"
+    timeout: 2 hours
+
 # macOS builds
 
 b:macos-x86_64-ninja:

.gitlab/artifacts.yml

@@ -146,6 +146,19 @@
             junit:
                 - ${CMAKE_CI_BUILD_DIR}/junit.xml
 
+.cmake_valgrind_artifacts:
+    artifacts:
+        expire_in: 1d
+        when: always
+        reports:
+            annotations:
+                - ${CMAKE_CI_BUILD_DIR}/annotations.json
+            junit:
+                - ${CMAKE_CI_BUILD_DIR}/junit.xml
+        paths:
+            # Valgrind logs.
+            - ${CMAKE_CI_BUILD_DIR}/Testing/Temporary/MemoryChecker.*.log
+
 .cmake_sphinx_artifacts:
     artifacts:
         expire_in: 1d

.gitlab/ci/.gitattributes

@@ -0,0 +1,2 @@
+# Do not apply a size limit to suppression files.
+*.valgrind.supp  -hooks-max-size

.gitlab/ci/configure_fedora42_ninja_valgrind.cmake

@@ -0,0 +1,6 @@
+set(CMake_TEST_Qt5 ON CACHE BOOL "")
+set(CMake_TEST_Qt6 ON CACHE BOOL "")
+
+include("${CMAKE_CURRENT_LIST_DIR}/configure_fedora42_valgrind.cmake")
+include("${CMAKE_CURRENT_LIST_DIR}/configure_fedora42_ninja.cmake")
+include("${CMAKE_CURRENT_LIST_DIR}/configure_external_test.cmake")

.gitlab/ci/configure_fedora42_valgrind.cmake

@@ -0,0 +1,2 @@
+# Disable bootstrap testing for Valgrind testing.
+set(CMAKE_SKIP_BOOTSTRAP_TEST OFF CACHE BOOL "")

.gitlab/ci/ctest_exclusions.cmake

@@ -27,6 +27,20 @@ if ("$ENV{CMAKE_CONFIGURATION}" MATCHES "_jom")
     )
 endif()
 
+if ("$ENV{CMAKE_CONFIGURATION}" MATCHES "_valgrind")
+  list(APPEND test_exclusions
+    # Tests that timeout under valgrind.
+    "^RunCMake.NinjaMultiConfig$"
+    "^RunCMake.Autogen_Qt6_1$"
+    "^RunCMake.GoogleTest$"
+    "^RunCMake.CXXModules$"
+    "^RunCMake.CommandLine$"
+
+    # Too spurious under Valgrind.
+    "^RunCMake.testUVProcessChain$"
+    )
+endif()
+
 string(REPLACE ";" "|" test_exclusions "${test_exclusions}")
 if (test_exclusions)
   set(test_exclusions "(${test_exclusions})")

.gitlab/ci/ctest_memcheck.cmake

@@ -20,18 +20,13 @@ if (NOT "$ENV{CMAKE_CI_TEST_TIMEOUT}" STREQUAL "")
   set(CTEST_TEST_TIMEOUT "$ENV{CMAKE_CI_TEST_TIMEOUT}")
 endif ()
 
-set(CTEST_MEMORYCHECK_TYPE "$ENV{CTEST_MEMORYCHECK_TYPE}")
-set(CTEST_MEMORYCHECK_SANITIZER_OPTIONS "$ENV{CTEST_MEMORYCHECK_SANITIZER_OPTIONS}")
-
-set(lsan_suppressions "${CMAKE_CURRENT_LIST_DIR}/ctest_memcheck_$ENV{CMAKE_CONFIGURATION}.lsan.supp")
-if (EXISTS "${lsan_suppressions}")
-  set(ENV{LSAN_OPTIONS} "suppressions='${lsan_suppressions}'")
-endif ()
+include("${CMAKE_CURRENT_LIST_DIR}/ctest_memcheck_prep.cmake")
 
 include("${CMAKE_CURRENT_LIST_DIR}/ctest_exclusions.cmake")
 ctest_memcheck(
   PARALLEL_LEVEL "${nproc}"
   TEST_LOAD "${nproc}"
+  OUTPUT_JUNIT "${CTEST_BINARY_DIRECTORY}/junit.xml"
   RETURN_VALUE test_result
   EXCLUDE "${test_exclusions}"
   DEFECT_COUNT defects)

.gitlab/ci/ctest_memcheck_fedora42.valgrind.supp

@@ -0,0 +1,91 @@
+# Add Valgrind suppressions here.
+{
+   libc_start_main-malloc
+   Memcheck:Leak
+   match-leak-kinds: reachable
+   fun:malloc
+   ...
+   fun:__libc_start_main@@GLIBC_2.34
+   ...
+}
+{
+   dl_init-malloc
+   Memcheck:Leak
+   match-leak-kinds: reachable
+   fun:malloc
+   ...
+   fun:_dl_init
+   ...
+}
+{
+   dl_init-calloc
+   Memcheck:Leak
+   match-leak-kinds: reachable
+   fun:calloc
+   ...
+   fun:_dl_init
+   ...
+}
+{
+   dl_init-realloc
+   Memcheck:Leak
+   match-leak-kinds: reachable
+   fun:realloc
+   ...
+   fun:_dl_init
+   ...
+}
+{
+   dl_open-malloc
+   Memcheck:Leak
+   match-leak-kinds: reachable
+   fun:malloc
+   ...
+   fun:_dl_open
+   ...
+}
+{
+   dl_open-calloc
+   Memcheck:Leak
+   match-leak-kinds: reachable
+   fun:calloc
+   ...
+   fun:_dl_open
+   ...
+}
+{
+   qt5-object-new
+   Memcheck:Leak
+   match-leak-kinds: reachable
+   fun:_Znwm
+   ...
+   fun:_ZN7QObjectC1ER14QObjectPrivatePS_
+   ...
+}
+{
+   qt5-logger-malloc
+   Memcheck:Leak
+   match-leak-kinds: reachable
+   fun:malloc
+   ...
+   fun:_ZNK14QMessageLogger5debugEv
+   ...
+}
+{
+   strerror_l-malloc
+   Memcheck:Leak
+   match-leak-kinds: reachable
+   fun:malloc
+   fun:__vasprintf_internal
+   fun:__asprintf_chk
+   fun:strerror_l
+   ...
+}
+{
+   bash
+   Memcheck:Leak
+   match-leak-kinds: reachable
+   ...
+   obj:/usr/bin/bash
+   ...
+}

.gitlab/ci/ctest_memcheck_prep.cmake

@@ -0,0 +1,50 @@
+set(CTEST_MEMORYCHECK_TYPE "$ENV{CTEST_MEMORYCHECK_TYPE}")
+set(CTEST_MEMORYCHECK_SANITIZER_OPTIONS "$ENV{CTEST_MEMORYCHECK_SANITIZER_OPTIONS}")
+
+set(lsan_suppressions "${CMAKE_CURRENT_LIST_DIR}/ctest_memcheck_$ENV{CMAKE_CONFIGURATION}.lsan.supp")
+if (EXISTS "${lsan_suppressions}")
+  set(ENV{LSAN_OPTIONS} "suppressions='${lsan_suppressions}'")
+endif ()
+
+if (CTEST_MEMORYCHECK_TYPE STREQUAL "Valgrind")
+  find_program(valgrind_exe NAMES valgrind)
+  set(CTEST_MEMORYCHECK_COMMAND "${valgrind_exe}")
+
+  set(valgrind_suppressions "${CMAKE_CURRENT_LIST_DIR}/ctest_memcheck_$ENV{CMAKE_CONFIGURATION}.valgrind.supp")
+  set(common_valgrind_suppressions "${CMAKE_CURRENT_LIST_DIR}/ctest_memcheck_$ENV{CMAKE_VALGRIND_CONFIGURATION}.valgrind.supp")
+  if (EXISTS "${valgrind_suppressions}")
+    set(CTEST_MEMORYCHECK_SUPPRESSIONS_FILE "${valgrind_suppressions}")
+  elseif (EXISTS "${common_valgrind_suppressions}")
+    set(CTEST_MEMORYCHECK_SUPPRESSIONS_FILE "${common_valgrind_suppressions}")
+  endif ()
+
+  set(valgrind_skip
+    /bin/*
+    /sbin/*
+    /usr/bin/*
+    /usr/lib64/qt5/bin/*
+    /usr/lib64/qt6/bin/*
+    /usr/lib64/qt6/libexec/*
+    bootstrap
+    sample_script
+    */Tests/CTestTest2/kwsysBin/*
+    */Tests/CTestTestCrash/Crash
+    *QtAutogen
+    # Ignore ISPC files which may contain unimplemented instructions.
+    */build/Tests/ISPC/TryCompile/ISPCTryCompile
+    # Ignore anything CI downloads.
+    */.gitlab/*)
+  list(JOIN valgrind_skip "," valgrind_skip)
+  string(CONCAT valgrind_options
+    "--gen-suppressions=all "
+    "--child-silent-after-fork=yes "
+    "--trace-children=yes "
+    "--trace-children-skip=${valgrind_skip} "
+    "--track-origins=yes "
+    "-q "
+    "--leak-check=yes "
+    "--show-reachable=yes "
+    "--num-callers=50 "
+    "-v ")
+  set(CTEST_MEMORYCHECK_COMMAND_OPTIONS "${valgrind_options}")
+endif ()

.gitlab/ci/ctest_standalone.cmake

@@ -33,6 +33,10 @@ elseif("$ENV{CMAKE_CONFIGURATION}" MATCHES "extdeps")
   set(CTEST_CONFIGURE_COMMAND "/opt/extdeps/bin/cmake -C \"${initial_cache}\" -G \"${CTEST_CMAKE_GENERATOR}\" \"${CTEST_SOURCE_DIRECTORY}\"")
 endif()
 
+if ("$ENV{CMAKE_CI_RUN_MEMCHECK}" STREQUAL "true")
+  include("${CMAKE_CURRENT_LIST_DIR}/ctest_memcheck_prep.cmake")
+endif ()
+
 # Configure the project.
 ctest_configure(
   OPTIONS "${cmake_args}"
@@ -93,20 +97,37 @@ if (NOT "$ENV{CTEST_LABELS}" STREQUAL "")
 endif ()
 
 include("${CMAKE_CURRENT_LIST_DIR}/ctest_exclusions.cmake")
-ctest_test(
-  PARALLEL_LEVEL "${nproc}"
-  TEST_LOAD "${nproc}"
-  OUTPUT_JUNIT "${CTEST_BINARY_DIRECTORY}/junit.xml"
-  RETURN_VALUE test_result
-  ${ctest_label_args}
-  EXCLUDE "${test_exclusions}")
-ctest_submit(PARTS Test)
+set(extra_annotations)
+if ("$ENV{CMAKE_CI_RUN_MEMCHECK}" STREQUAL "true")
+  ctest_memcheck(
+    PARALLEL_LEVEL "${nproc}"
+    TEST_LOAD "${nproc}"
+    OUTPUT_JUNIT "${CTEST_BINARY_DIRECTORY}/junit.xml"
+    RETURN_VALUE test_result
+    ${ctest_label_args}
+    EXCLUDE "${test_exclusions}")
+  ctest_submit(PARTS Test)
+  ctest_submit(PARTS Memcheck)
+  list(APPEND extra_annotations
+    "Dynamic Analysis"  "https://open.cdash.org/viewDynamicAnalysis.php?buildid=${build_id}"
+  )
+else ()
+  ctest_test(
+    PARALLEL_LEVEL "${nproc}"
+    TEST_LOAD "${nproc}"
+    OUTPUT_JUNIT "${CTEST_BINARY_DIRECTORY}/junit.xml"
+    RETURN_VALUE test_result
+    ${ctest_label_args}
+    EXCLUDE "${test_exclusions}")
+  ctest_submit(PARTS Test)
+endif ()
 
 ctest_annotation_report("${CTEST_BINARY_DIRECTORY}/annotations.json"
   "All Tests"     "https://open.cdash.org/viewTest.php?buildid=${build_id}"
   "Test Failures" "https://open.cdash.org/viewTest.php?onlyfailed&buildid=${build_id}"
   "Tests Not Run" "https://open.cdash.org/viewTest.php?onlynotrun&buildid=${build_id}"
-  "Test Passes"   "https://open.cdash.org/viewTest.php?onlypassed&buildid=${build_id}")
+  "Test Passes"   "https://open.cdash.org/viewTest.php?onlypassed&buildid=${build_id}"
+  ${extra_annotations})
 
 if (test_result)
   ctest_submit(PARTS Done)

.gitlab/ci/env_fedora42_ninja_valgrind.sh

@@ -0,0 +1 @@
+source .gitlab/ci/env_fedora42_ninja.sh

.gitlab/os-linux.yml

@@ -337,6 +337,15 @@
         CTEST_MEMORYCHECK_TYPE: AddressSanitizer
         CTEST_MEMORYCHECK_SANITIZER_OPTIONS: ""
 
+.fedora_valgrind_addon:
+    extends: .fedora_memcheck
+
+    variables:
+        CTEST_MEMORYCHECK_TYPE: Valgrind
+        CMAKE_CI_RUN_MEMCHECK: "true"
+        CMAKE_CI_TEST_TIMEOUT: "1500"
+        CMAKE_VALGRIND_CONFIGURATION: fedora42
+
 .fedora42_asan:
     extends:
         - .fedora42
@@ -345,6 +354,15 @@
     variables:
         CMAKE_CONFIGURATION: fedora42_asan
 
+
+.fedora42_ninja_valgrind:
+    extends:
+        - .fedora42
+        - .fedora_valgrind_addon
+
+    variables:
+        CMAKE_CONFIGURATION: fedora42_ninja_valgrind
+
 ### Intel Compiler
 
 .intelcompiler:
@@ -616,6 +634,13 @@
         - linux-x86_64
         - x11
 
+.linux_x86_64_tags_fast_x11:
+    tags:
+        - cmake
+        - docker
+        - linux-x86_64-v4
+        - x11
+
 .linux_x86_64_tags_cuda_arch_30:
     tags:
         - cmake