Merge topic 'ci-gcov'

f7b0ddbe92 gitlab-ci: add a series of jobs to perform coverage
cedd32892e ci: support setting a different test timeout for CI
da4a1ec2ff gitlab-ci: add support for running gcov-based coverage
61aed5e5f2 CTestCustom: ignore coverage results from coverage tests
7191d72acd Tests/EnforceConfig: clear the `CMAKE_BUILD_TYPE` environment variable

Acked-by: Kitware Robot <[email protected]>
Merge-request: !11245

.gitlab-ci.yml

@@ -40,6 +40,7 @@ variables:
 ################################################################################
 
 # Job prefixes:
+#   - `a:` analysis
 #   - `b:` build
 #   - `k:` package
 #   - `l:` lint
@@ -147,6 +148,44 @@ l:clazy-fedora42:
         - .linux_x86_64_tags
         - .run_automatically
 
+# Coverage builds
+
+b:fedora42-gcc-gcov:
+    extends:
+        - .fedora42_gcc_gcov
+        - .cmake_build_linux
+        - .cmake_build_artifacts
+        - .linux_x86_64_tags
+        - .run_manually
+    variables:
+        CMAKE_CI_JOB_NIGHTLY: "true"
+
+t:fedora42-gcc-gcov:
+    extends:
+        - .fedora42_gcc_gcov
+        - .cmake_test_linux
+        - .cmake_coverage_artifacts
+        - .linux_x86_64_tags_x11
+        - .run_dependent
+    # Run coverage analysis even if the tests fail.
+    allow_failure: true
+    needs:
+        - b:fedora42-gcc-gcov
+    variables:
+        CMAKE_CI_JOB_NIGHTLY: "true"
+
+a:fedora42-gcc-gcov:
+    extends:
+        - .fedora42_gcc_gcov
+        - .cmake_coverage_linux
+        - .cmake_cdash_artifacts
+        - .linux_x86_64_tags
+        - .run_dependent
+    needs:
+        - t:fedora42-gcc-gcov
+    variables:
+        CMAKE_CI_JOB_NIGHTLY: "true"
+
 # Linux builds
 
 b:centos7-x86_64:

.gitlab/artifacts.yml

@@ -62,6 +62,12 @@
             - ${CMAKE_CI_BUILD_DIR}/DartConfiguration.tcl
             - ${CMAKE_CI_BUILD_DIR}/CTestCustom.cmake
             - ${CMAKE_CI_BUILD_DIR}/cdash-build-id
+
+            # CTest coverage files.
+            - ${CMAKE_CI_BUILD_DIR}/CMakeFiles/TargetDirectories.txt
+            - ${CMAKE_CI_BUILD_DIR}/**/*.gcno
+            - ${CMAKE_CI_BUILD_DIR}/Source/QtDialog/moc_*.cpp
+            - ${CMAKE_CI_BUILD_DIR}/Source/QtDialog/ui_*.h
         reports:
             annotations:
                 - ${CMAKE_CI_BUILD_DIR}/annotations.json
@@ -167,6 +173,32 @@
             # Take the install tree.
             - ${CMAKE_CI_BUILD_DIR}/install/
 
+.cmake_coverage_artifacts:
+    artifacts:
+        expire_in: 1d
+        # External testing can be useful even if test jobs fail.
+        when: always
+        reports:
+            junit:
+                - ${CMAKE_CI_BUILD_DIR}/junit.xml
+            annotations:
+                - ${CMAKE_CI_BUILD_DIR}/annotations.json
+        paths:
+            # Generated sources.
+            - ${CMAKE_CI_BUILD_DIR}/Source/QtDialog/moc_*.cpp
+            - ${CMAKE_CI_BUILD_DIR}/Source/QtDialog/ui_*.h
+
+            # CTest coverage files.
+            - ${CMAKE_CI_BUILD_DIR}/CMakeFiles/TargetDirectories.txt
+            - ${CMAKE_CI_BUILD_DIR}/**/*.gcno
+            - ${CMAKE_CI_BUILD_DIR}/**/*.gcda
+
+            # CTest/CDash information.
+            - ${CMAKE_CI_BUILD_DIR}/Testing/
+            - ${CMAKE_CI_BUILD_DIR}/DartConfiguration.tcl
+            - ${CMAKE_CI_BUILD_DIR}/CTestCustom.cmake
+            - ${CMAKE_CI_BUILD_DIR}/cdash-build-id
+
 .cmake_doc_artifacts:
     artifacts:
         expire_in: 1d

.gitlab/ci/configure_fedora42_gcc_gcov.cmake

@@ -0,0 +1,46 @@
+set(CMake_TEST_GUI "ON" CACHE BOOL "")
+set(CMake_TEST_MODULE_COMPILATION "named,compile_commands,collation,partitions,internal_partitions,export_bmi,install_bmi,shared,bmionly,build_database" CACHE STRING "")
+set(CMake_TEST_TLS_VERIFY_URL "https://gitlab.kitware.com" CACHE STRING "")
+set(CMake_TEST_TLS_VERIFY_URL_BAD "https://badtls-expired.kitware.com" CACHE STRING "")
+set(CMake_TEST_TLS_VERSION "1.3" CACHE STRING "")
+set(CMake_TEST_TLS_VERSION_URL_BAD "https://badtls-v1-1.kitware.com:8011" CACHE STRING "")
+
+string(CONCAT coverage_flags
+  "--coverage "
+  "-fprofile-abs-path "
+  # These files are committed generated sources and contain relative path
+  # `#line` directives. CI coverage then cannot find the source files reliably.
+  # See related issue #20001.
+  "-fprofile-exclude-files=cmExprParser[.].*\\;cmFortranParser[.].* "
+  "-fdiagnostics-show-option "
+  "-Wall "
+  "-Wextra "
+  "-Wshadow "
+  "-Wpointer-arith "
+  "-Winvalid-pch "
+  "-Wcast-align "
+  "-Wdisabled-optimization "
+  "-Wwrite-strings "
+  "-fstack-protector-all "
+  "-Wconversion "
+  "-Wno-error=sign-conversion "
+  "-Wno-error=conversion ")
+string(CONCAT link_flags
+  "--coverage ")
+
+set(CMAKE_C_FLAGS "${coverage_flags}" CACHE STRING "")
+set(CMAKE_CXX_FLAGS "${coverage_flags} -Woverloaded-virtual -Wstrict-null-sentinel" CACHE STRING "")
+# Apply `LDFLAGS`.
+set(CMAKE_EXE_LINKER_FLAGS_INIT "${link_flags}" CACHE STRING "")
+set(CMAKE_SHARED_LINKER_FLAGS_INIT "${link_flags}" CACHE STRING "")
+set(CMAKE_MODULE_LINKER_FLAGS_INIT "${link_flags}" CACHE STRING "")
+
+# Do not bootstrap for the coverage test suite.
+set(CMAKE_SKIP_BOOTSTRAP_TEST TRUE CACHE BOOL "")
+
+# Cover compilation with C++11 only and not higher standards.
+set(CMAKE_CXX_STANDARD "11" CACHE STRING "")
+# Qt 6 requires C++17, so use Qt 5.
+set(CMake_QT_MAJOR_VERSION "5" CACHE STRING "")
+
+include("${CMAKE_CURRENT_LIST_DIR}/configure_fedora42_common.cmake")

.gitlab/ci/ctest_coverage.cmake

@@ -0,0 +1,29 @@
+cmake_minimum_required(VERSION 3.29)
+
+include("${CMAKE_CURRENT_LIST_DIR}/gitlab_ci.cmake")
+
+# Read the files from the build directory.
+ctest_read_custom_files("${CTEST_BINARY_DIRECTORY}")
+
+# Pick up from where the configure left off.
+ctest_start(APPEND)
+
+find_program(GCOV NAMES gcov)
+set(CTEST_COVERAGE_COMMAND "${GCOV}")
+set(CTEST_COVERAGE_EXTRA_FLAGS
+  --hash-filenames
+  --long-file-names)
+string(REPLACE ";" " " CTEST_COVERAGE_EXTRA_FLAGS "${CTEST_COVERAGE_EXTRA_FLAGS}")
+ctest_coverage(
+  RETURN_VALUE coverage_result)
+ctest_submit(PARTS Coverage)
+
+include("${CMAKE_CURRENT_LIST_DIR}/ctest_annotation.cmake")
+ctest_annotation_report("${CTEST_BINARY_DIRECTORY}/annotations.json"
+  "Coverage Report" "https://open.cdash.org/viewCoverage.php?buildid=${build_id}"
+)
+
+if (coverage_result)
+  message(FATAL_ERROR
+    "Failed to gather coverage")
+endif ()

.gitlab/ci/ctest_memcheck.cmake

@@ -16,6 +16,10 @@ if (NOT "$ENV{CTEST_MAX_PARALLELISM}" STREQUAL "")
   endif ()
 endif ()
 
+if (NOT "$ENV{CMAKE_CI_TEST_TIMEOUT}" STREQUAL "")
+  set(CTEST_TEST_TIMEOUT "$ENV{CMAKE_CI_TEST_TIMEOUT}")
+endif ()
+
 set(CTEST_MEMORYCHECK_TYPE "$ENV{CTEST_MEMORYCHECK_TYPE}")
 set(CTEST_MEMORYCHECK_SANITIZER_OPTIONS "$ENV{CTEST_MEMORYCHECK_SANITIZER_OPTIONS}")
 

.gitlab/ci/ctest_standalone.cmake

@@ -14,6 +14,10 @@ if (NOT "$ENV{CTEST_MAX_PARALLELISM}" STREQUAL "")
   endif ()
 endif ()
 
+if (NOT "$ENV{CMAKE_CI_TEST_TIMEOUT}" STREQUAL "")
+  set(CTEST_TEST_TIMEOUT "$ENV{CMAKE_CI_TEST_TIMEOUT}")
+endif ()
+
 # Create an entry in CDash.
 ctest_start("${ctest_model}" GROUP "${ctest_group}")
 

.gitlab/ci/ctest_test.cmake

@@ -16,6 +16,10 @@ if (NOT "$ENV{CTEST_MAX_PARALLELISM}" STREQUAL "")
   endif ()
 endif ()
 
+if (NOT "$ENV{CMAKE_CI_TEST_TIMEOUT}" STREQUAL "")
+  set(CTEST_TEST_TIMEOUT "$ENV{CMAKE_CI_TEST_TIMEOUT}")
+endif ()
+
 include("${CMAKE_CURRENT_LIST_DIR}/ctest_exclusions.cmake")
 ctest_test(
   PARALLEL_LEVEL "${nproc}"

.gitlab/os-linux.yml

@@ -97,6 +97,20 @@
         GIT_CLONE_PATH: "$CI_BUILDS_DIR/cmake-ci"
         CMAKE_ARCH: x86_64
 
+#### Coverage builds
+
+.fedora42_gcc_gcov:
+    extends: .fedora42
+
+    variables:
+        # See issue #20001
+        CMAKE_GENERATOR: "Unix Makefiles"
+        CMAKE_CONFIGURATION: fedora42_gcc_gcov
+        CTEST_NO_WARNINGS_ALLOWED: 1
+        CMAKE_CI_BUILD_TYPE: Debug
+        CMAKE_CI_TEST_TIMEOUT: "1500"
+        CMAKE_CI_NO_INSTALL: 1
+
 #### Lint builds
 
 .fedora42_tidy:
@@ -727,6 +741,16 @@
 
     interruptible: true
 
+.cmake_coverage_linux:
+    stage: test
+
+    script:
+        - *before_script_linux
+        - "$LAUNCHER ctest --output-on-failure -V -S .gitlab/ci/ctest_coverage.cmake"
+    coverage: '/Percentage Coverage: \d+.\d+%/'
+
+    interruptible: true
+
 .cmake_build_linux_release:
     stage: build
 

CTestCustom.cmake.in

@@ -126,6 +126,9 @@ list(APPEND CTEST_CUSTOM_COVERAGE_EXCLUDE
 
   # Exclude Qt source files from coverage results:
   "[A-Za-z]./[Qq]t/qt-.+-opensource-src"
+
+  # Exclude coverage results from coverage test directories.
+  "/Testing/.*Coverage/"
   )
 
 list(APPEND CTEST_CUSTOM_MEMCHECK_IGNORE

Tests/EnforceConfig.cmake.in

@@ -32,6 +32,7 @@ endif()
 unset(ENV{NO_COLOR})
 unset(ENV{CLICOLOR})
 unset(ENV{CLICOLOR_FORCE})
+unset(ENV{CMAKE_BUILD_TYPE})
 unset(ENV{CMAKE_GENERATOR})
 unset(ENV{CMAKE_GENERATOR_INSTANCE})
 unset(ENV{CMAKE_GENERATOR_PLATFORM})