Merge topic 'ci-package-pipeline'

7c60a2625d gitlab-ci: Optionally upload release packages to custom URL
fc8a981893 gitlab-ci: Generalize package and help upload URLs
f73649823b gitlab-ci: Stage unsigned release package artifacts in a dedicated directory
0929221ca3 gitlab-ci: Simplify Windows packaging pipeline
ce2e825306 gitlab-ci: Simplify macOS packaging jobs
05531a7c4e gitlab-ci: add pre-build and post-build steps to Windows
5ab92d8e71 gitlab-ci: add pre-build and post-build steps to macOS

Acked-by: Kitware Robot <[email protected]>
Acked-by: Ben Boeckel <[email protected]>
Merge-request: !8266

.gitlab-ci.yml

@@ -16,7 +16,6 @@ stages:
     - build
     - test
     - test-ext
-    - package
     - upload
 
 ################################################################################
@@ -37,7 +36,6 @@ stages:
 
 # Job prefixes:
 #   - `b:` build
-#   - `k:` package
 #   - `l:` lint
 #   - `p:` prep
 #   - `t:` test
@@ -943,7 +941,7 @@ t:macos-x86_64-xcode-ub:
 b:macos-package:
     extends:
         - .macos_package
-        - .cmake_build_macos_package
+        - .cmake_build_macos
         - .cmake_release_artifacts
         - .macos_x86_64_tags_package
         - .run_only_for_package
@@ -966,7 +964,7 @@ u:macos-package:
 b:macos10.10-package:
     extends:
         - .macos10.10_package
-        - .cmake_build_macos_package
+        - .cmake_build_macos
         - .cmake_release_artifacts
         - .macos_x86_64_tags_package
         - .run_only_for_package
@@ -1268,27 +1266,13 @@ b:windows-x86_64-package:
     extends:
         - .windows_x86_64_package
         - .cmake_build_windows
-        - .cmake_build_package_artifacts
+        - .cmake_release_artifacts
         - .windows_x86_64_tags_nonconcurrent_vs2022
         - .run_only_for_package
     dependencies:
         - p:doc-package
     needs:
         - p:doc-package
-    variables:
-        CMAKE_CI_ARTIFACTS_NAME: "artifacts-windows-x86_64-build"
-
-k:windows-x86_64-package:
-    extends:
-        - .windows_x86_64_package
-        - .cmake_package_windows
-        - .cmake_release_artifacts
-        - .windows_x86_64_tags_nonconcurrent_vs2022
-        - .run_only_for_package
-    dependencies:
-        - b:windows-x86_64-package
-    needs:
-        - b:windows-x86_64-package
     variables:
         CMAKE_CI_ARTIFACTS_NAME: "artifacts-windows-x86_64"
 
@@ -1297,35 +1281,21 @@ u:windows-x86_64-package:
         - .rsync_upload_package
         - .run_only_for_package
     dependencies:
-        - k:windows-x86_64-package
+        - b:windows-x86_64-package
     needs:
-        - k:windows-x86_64-package
+        - b:windows-x86_64-package
 
 b:windows-i386-package:
     extends:
         - .windows_i386_package
         - .cmake_build_windows
-        - .cmake_build_package_artifacts
+        - .cmake_release_artifacts
         - .windows_x86_64_tags_nonconcurrent_vs2022
         - .run_only_for_package
     dependencies:
         - p:doc-package
     needs:
         - p:doc-package
-    variables:
-        CMAKE_CI_ARTIFACTS_NAME: "artifacts-windows-i386-build"
-
-k:windows-i386-package:
-    extends:
-        - .windows_i386_package
-        - .cmake_package_windows
-        - .cmake_release_artifacts
-        - .windows_x86_64_tags_nonconcurrent_vs2022
-        - .run_only_for_package
-    dependencies:
-        - b:windows-i386-package
-    needs:
-        - b:windows-i386-package
     variables:
         CMAKE_CI_ARTIFACTS_NAME: "artifacts-windows-i386"
 
@@ -1334,35 +1304,21 @@ u:windows-i386-package:
         - .rsync_upload_package
         - .run_only_for_package
     dependencies:
-        - k:windows-i386-package
+        - b:windows-i386-package
     needs:
-        - k:windows-i386-package
+        - b:windows-i386-package
 
 b:windows-arm64-package:
     extends:
         - .windows_arm64_package
         - .cmake_build_windows
-        - .cmake_build_package_artifacts
+        - .cmake_release_artifacts
         - .windows_x86_64_tags_nonconcurrent_vs2022_arm64
         - .run_only_for_package
     dependencies:
         - p:doc-package
     needs:
         - p:doc-package
-    variables:
-        CMAKE_CI_ARTIFACTS_NAME: "artifacts-windows-arm64-build"
-
-k:windows-arm64-package:
-    extends:
-        - .windows_arm64_package
-        - .cmake_package_windows
-        - .cmake_release_artifacts
-        - .windows_x86_64_tags_nonconcurrent_vs2022_arm64
-        - .run_only_for_package
-    dependencies:
-        - b:windows-arm64-package
-    needs:
-        - b:windows-arm64-package
     variables:
         CMAKE_CI_ARTIFACTS_NAME: "artifacts-windows-arm64"
 
@@ -1371,6 +1327,6 @@ u:windows-arm64-package:
         - .rsync_upload_package
         - .run_only_for_package
     dependencies:
-        - k:windows-arm64-package
+        - b:windows-arm64-package
     needs:
-        - k:windows-arm64-package
+        - b:windows-arm64-package

.gitlab/artifacts.yml

@@ -63,35 +63,6 @@
             - build/DartConfiguation.tcl
             - build/CTestCustom.cmake
 
-.cmake_build_package_artifacts:
-    artifacts:
-        expire_in: 1d
-        name: "$CMAKE_CI_ARTIFACTS_NAME"
-        paths:
-            # Allow CPack to find CMAKE_ROOT.
-            - build/CMakeFiles/CMakeSourceDir.txt
-
-            # Install rules.
-            - build/**/cmake_install.cmake
-
-            # We need the main binaries.
-            - build/bin/
-
-            # Pass through the documentation.
-            - build/install-doc/
-
-            # CPack configuration.
-            - build/CPackConfig.cmake
-            - build/CMakeCPackOptions.cmake
-            - build/Source/QtDialog/QtDialogCPack.cmake
-
-            # CPack/IFW packaging files.
-            - build/CMake*.qs
-
-            # CPack/WIX packaging files.
-            - build/Utilities/Release/WiX/custom_action_dll*.wxs
-            - build/Utilities/Release/WiX/CustomAction/CMakeWiXCustomActions.*
-
 .cmake_release_artifacts:
     artifacts:
         expire_in: 5d
@@ -109,6 +80,8 @@
             # Any source packages made.
             - build/cmake-*.tar.gz
             - build/cmake-*.zip
+            # Any unsigned packages made.
+            - build/unsigned/cmake-*
 
 .cmake_junit_artifacts:
     artifacts:

.gitlab/ci/CMakeCPack.cmake

@@ -0,0 +1,3 @@
+if(NOT "$ENV{CMAKE_CI_PACKAGE}" MATCHES "^(dev)?$")
+  configure_file(${CMAKE_CURRENT_LIST_DIR}/package_info.cmake.in ${CMake_BINARY_DIR}/ci_package_info.cmake @ONLY)
+endif()

.gitlab/ci/configure_windows_package_common.cmake

@@ -19,4 +19,6 @@ set(CMake_TEST_Qt5 OFF CACHE BOOL "")
 set(CMake_TEST_Qt6 OFF CACHE BOOL "")
 set(Python_FIND_REGISTRY NEVER CACHE STRING "")
 
+set(CMake_CPACK_CUSTOM_SCRIPT "${CMAKE_CURRENT_LIST_DIR}/CMakeCPack.cmake" CACHE FILEPATH "")
+
 include("${CMAKE_CURRENT_LIST_DIR}/configure_common.cmake")

.gitlab/ci/package_info.cmake.in

@@ -0,0 +1 @@
+set(CPACK_PACKAGE_FILE_NAME "@CPACK_PACKAGE_FILE_NAME@")

.gitlab/ci/package_macos.sh

@@ -0,0 +1,12 @@
+cd build
+cpack -G TGZ
+cpack -G DragNDrop
+
+case "$CMAKE_CI_PACKAGE" in
+    dev)
+        ;;
+    *)
+        mkdir -p unsigned
+        mv cmake-*-macos*-universal.* unsigned/
+        ;;
+esac

.gitlab/ci/package_windows.ps1

@@ -0,0 +1,7 @@
+if (Test-Path -Path "build/ci_package_info.cmake" -PathType Leaf) {
+    cmake -P .gitlab/ci/package_windows_build.cmake
+} else {
+    cd build
+    cpack -G ZIP
+    cpack -G WIX
+}

.gitlab/ci/package_windows_build.cmake

@@ -0,0 +1,41 @@
+cmake_minimum_required(VERSION 3.24)
+include(build/ci_package_info.cmake)
+
+set(build "${CMAKE_CURRENT_BINARY_DIR}/build")
+
+file(GLOB paths RELATIVE "${CMAKE_CURRENT_BINARY_DIR}"
+  # Allow CPack to find CMAKE_ROOT.
+  "${build}/CMakeFiles/CMakeSourceDir.txt"
+
+  # We need the main binaries.
+  "${build}/bin"
+
+  # Pass through the documentation.
+  "${build}/install-doc"
+
+  # CPack configuration.
+  "${build}/CPackConfig.cmake"
+  "${build}/CMakeCPackOptions.cmake"
+  "${build}/Source/QtDialog/QtDialogCPack.cmake"
+
+  # CPack/IFW packaging files.
+  "${build}/CMake*.qs"
+
+  # CPack/WIX packaging files.
+  "${build}/Utilities/Release/WiX/custom_action_dll*.wxs"
+  "${build}/Utilities/Release/WiX/CustomAction/CMakeWiXCustomActions.*"
+  )
+
+file(GLOB_RECURSE paths_recurse RELATIVE "${CMAKE_CURRENT_BINARY_DIR}"
+  # Install rules.
+  "${build}/cmake_install.cmake"
+  "${build}/*/cmake_install.cmake"
+  )
+
+# Create a "package" containing the build-tree files needed to build a package.
+file(MAKE_DIRECTORY build/unsigned)
+file(ARCHIVE_CREATE
+  OUTPUT build/unsigned/${CPACK_PACKAGE_FILE_NAME}.build.zip
+  PATHS ${paths} ${paths_recurse}
+  FORMAT zip
+  )

.gitlab/ci/post_build.ps1

@@ -0,0 +1,4 @@
+$pwsh = [System.Diagnostics.Process]::GetCurrentProcess().MainModule.FileName
+if (Test-Path -Path ".gitlab/ci/post_build_$env:CMAKE_CONFIGURATION.ps1" -PathType Leaf) {
+  . ".gitlab/ci/post_build_$env:CMAKE_CONFIGURATION.ps1"
+}

.gitlab/ci/post_build_macos10.10_package.sh

@@ -0,0 +1 @@
+. .gitlab/ci/package_macos.sh

.gitlab/ci/post_build_macos_package.sh

@@ -0,0 +1 @@
+. .gitlab/ci/package_macos.sh

.gitlab/ci/post_build_windows_arm64_package.ps1

@@ -0,0 +1 @@
+. .gitlab/ci/package_windows.ps1

.gitlab/ci/post_build_windows_i386_package.ps1

@@ -0,0 +1 @@
+. .gitlab/ci/package_windows.ps1

.gitlab/ci/post_build_windows_x86_64_package.ps1

@@ -0,0 +1 @@
+. .gitlab/ci/package_windows.ps1

.gitlab/ci/pre_build.ps1

@@ -0,0 +1,4 @@
+$pwsh = [System.Diagnostics.Process]::GetCurrentProcess().MainModule.FileName
+if (Test-Path -Path ".gitlab/ci/pre_build_$env:CMAKE_CONFIGURATION.ps1" -PathType Leaf) {
+  . ".gitlab/ci/pre_build_$env:CMAKE_CONFIGURATION.ps1"
+}

.gitlab/os-macos.yml

@@ -169,25 +169,14 @@
         # Allow the server to already be running.
         - "sccache --start-server || :"
         - sccache --show-stats
+        - .gitlab/ci/pre_build.sh
         - ctest -VV -S .gitlab/ci/ctest_configure.cmake
         - ctest -VV -S .gitlab/ci/ctest_build.cmake
+        - .gitlab/ci/post_build.sh
         - sccache --show-stats
 
     interruptible: true
 
-.cmake_build_macos_package:
-    stage: build
-
-    script:
-        - *before_script_macos
-        - ctest -VV -S .gitlab/ci/ctest_configure.cmake
-        - ctest -VV -S .gitlab/ci/ctest_build.cmake
-        - cd build
-        - cpack -G TGZ
-        - cpack -G DragNDrop
-
-    interruptible: true
-
 .cmake_test_macos:
     stage: test
 

.gitlab/os-windows.yml

@@ -358,25 +358,15 @@
         - Invoke-Expression -Command .gitlab/ci/vcvarsall.ps1
         - sccache --start-server
         - sccache --show-stats
+        - (& "$pwsh" -File ".gitlab/ci/pre_build.ps1")
         - ctest -VV -S .gitlab/ci/ctest_configure.cmake
         - ctest -VV -S .gitlab/ci/ctest_build.cmake
+        - (& "$pwsh" -File ".gitlab/ci/post_build.ps1")
         - sccache --show-stats
         - sccache --stop-server
 
     interruptible: true
 
-.cmake_package_windows:
-    stage: package
-
-    script:
-        - *before_script_windows
-        - Invoke-Expression -Command .gitlab/ci/vcvarsall.ps1
-        - cd build
-        - cpack -G ZIP
-        - cpack -G WIX
-
-    interruptible: true
-
 .cmake_test_windows:
     stage: test
 

.gitlab/rules.yml

@@ -66,11 +66,15 @@
     rules:
         - if: '$CMAKE_CI_PACKAGE == "dev"'
           variables:
-              RSYNC_DESTINATION: "dev"
+              RSYNC_DESTINATION: "[email protected]:dev/"
           when: on_success
         - if: '$CMAKE_CI_PACKAGE != null && $CI_JOB_STAGE == "prep"'
           when: manual
-        - if: '$CMAKE_CI_PACKAGE != null && $CI_JOB_STAGE != "package" && $CI_JOB_STAGE != "upload"'
+        - if: '$CMAKE_CI_PACKAGE != null && $CI_JOB_STAGE != "upload"'
+          when: on_success
+        - if: '$CMAKE_CI_PACKAGE != null && $CI_JOB_STAGE == "upload" && $CMAKE_CI_PACKAGE_URL != null'
+          variables:
+              RSYNC_DESTINATION: "$CMAKE_CI_PACKAGE_URL"
           when: on_success
         - when: never
 
@@ -78,19 +82,19 @@
     rules:
         - if: '$CMAKE_CI_PACKAGE =~ /v[0-9]+\.[0-9]+/'
           variables:
-              RSYNC_DESTINATION: "$CMAKE_CI_PACKAGE"
+              RSYNC_DESTINATION: "[email protected]:$CMAKE_CI_PACKAGE/"
               CMAKE_CI_SPHINX_QTHELP: "ON"
               CMAKE_CI_VERSION_NO_GIT: "ON"
           when: manual
         - if: '$CMAKE_CI_PROJECT_MAIN_BRANCH != null && $CI_COMMIT_BRANCH != null && $CMAKE_CI_PROJECT_MAIN_BRANCH == $CI_COMMIT_BRANCH'
           variables:
-              RSYNC_DESTINATION: "git-master"
+              RSYNC_DESTINATION: "[email protected]:git-master/"
               CMAKE_CI_SPHINX_QTHELP: "OFF"
               CMAKE_CI_VERSION_NO_GIT: "OFF"
           when: on_success
         - if: '$CMAKE_CI_PROJECT_CONTINUOUS_BRANCH != null && $CI_COMMIT_BRANCH != null && $CMAKE_CI_PROJECT_CONTINUOUS_BRANCH == $CI_COMMIT_BRANCH'
           variables:
-              RSYNC_DESTINATION: "git-stage"
+              RSYNC_DESTINATION: "[email protected]:git-stage/"
               CMAKE_CI_SPHINX_QTHELP: "OFF"
               CMAKE_CI_VERSION_NO_GIT: "OFF"
           when: on_success

.gitlab/upload.yml

@@ -17,7 +17,7 @@
         - chmod 400 $RSYNC_BINARY_KEY
         - ssh-keygen -y -f $RSYNC_BINARY_KEY > $RSYNC_BINARY_KEY.pub
         - test -n "$RSYNC_DESTINATION"
-        - rsync -tv --recursive -e "ssh -i $RSYNC_BINARY_KEY -o StrictHostKeyChecking=no -o LogLevel=ERROR" build/ [email protected]:$RSYNC_DESTINATION/
+        - rsync -tv --recursive -e "ssh -i $RSYNC_BINARY_KEY -o StrictHostKeyChecking=no -o LogLevel=ERROR" build/ "$RSYNC_DESTINATION"
 
 .rsync_upload_help:
     stage: upload
@@ -36,4 +36,4 @@
         - chmod 400 $RSYNC_HELP_KEY
         - ssh-keygen -y -f $RSYNC_HELP_KEY > $RSYNC_HELP_KEY.pub
         - test -n "$RSYNC_DESTINATION"
-        - rsync -tv --recursive --delete -e "ssh -i $RSYNC_HELP_KEY -o StrictHostKeyChecking=no -o LogLevel=ERROR" build/html/ [email protected]:$RSYNC_DESTINATION/
+        - rsync -tv --recursive --delete -e "ssh -i $RSYNC_HELP_KEY -o StrictHostKeyChecking=no -o LogLevel=ERROR" build/html/ "$RSYNC_DESTINATION"

CMakeCPack.cmake

@@ -264,5 +264,9 @@ set(CPACK_SOURCE_IGNORE_FILES
   "~$"
   )
 
+if(CMake_CPACK_CUSTOM_SCRIPT)
+  include(${CMake_CPACK_CUSTOM_SCRIPT})
+endif()
+
 # include CPack model once all variables are set
 include(CPack)