Merge topic 'ci-macos-package'

ab2276e6b9 Utilities/Release: remove old macOS release script
9671282bcc gitlab-ci: add a job to upload macOS binaries
5138be9558 ci: use a pre-built Qt for macOS packages
8ff3ce730d gitlab-ci: add a job to make macOS packages
7d4ec05c6f gitlab-ci: add a job to make a doc "bundle"
f0aaf24f08 gitlab-ci: always place the build directory at the top
3a96ea903f ci: remove extra newline
758a8592ed Utilities: also support an already extracted doc tree

Acked-by: Kitware Robot <[email protected]>
Merge-request: !5227

.gitlab-ci.yml

@@ -8,10 +8,15 @@ include:
     - local: .gitlab/os-macos.yml
     - local: .gitlab/os-windows.yml
 
+    # Post-build steps
+    - local: .gitlab/upload.yml
+
 stages:
     - build
     - test
+    - package
     - test-ext
+    - upload
 
 ################################################################################
 # Job declarations
@@ -52,6 +57,14 @@ build:fedora31-sphinx:
         - .linux_builder_tags_qt
         - .run_automatically
 
+build:fedora31-sphinx-package:
+    extends:
+        - .fedora31_sphinx_package
+        - .cmake_build_linux
+        - .linux_builder_tags_qt
+        - .cmake_doc_artifacts
+        - .run_only_for_package
+
 # Linux builds
 
 build:centos6-release:
@@ -167,6 +180,29 @@ test:macos-xcode:
     needs:
         - test:macos-ninja
 
+package:macos:
+    extends:
+        - .macos_package
+        - .cmake_build_macos_package
+        - .cmake_release_artifacts
+        - .macos_builder_tags_package
+        - .run_only_for_package
+    dependencies:
+        - build:fedora31-sphinx-package
+    needs:
+        - build:fedora31-sphinx-package
+
+upload:macos:
+    extends:
+        - .rsync_upload
+        - .run_only_for_package
+    dependencies:
+        - package:macos
+    needs:
+        - package:macos
+    variables:
+        RSYNC_DESTINATION: dev
+
 # Windows builds
 
 build:windows-vs2019-x64-ninja:

.gitlab/artifacts.yml

@@ -72,6 +72,7 @@
         paths:
             # Any packages made.
             - build/cmake-*-Linux-x86_64.*
+            - build/cmake-*-Darwin-x86_64.*
 
 .cmake_test_artifacts:
     artifacts:
@@ -79,3 +80,10 @@
         paths:
             # Take the install tree.
             - build/install/
+
+.cmake_doc_artifacts:
+    artifacts:
+        expire_in: 1d
+        paths:
+            # Take the install tree.
+            - build/install-doc/

.gitlab/ci/configure_fedora31_common.cmake

@@ -1,5 +1,4 @@
 set(BUILD_CursesDialog ON CACHE BOOL "")
 set(BUILD_QtDialog ON CACHE BOOL "")
 
-
 include("${CMAKE_CURRENT_LIST_DIR}/configure_common.cmake")

.gitlab/ci/configure_fedora31_sphinx.cmake

@@ -1,8 +1,2 @@
-set(SPHINX_INFO ON CACHE BOOL "")
-set(SPHINX_MAN ON CACHE BOOL "")
-set(SPHINX_HTML ON CACHE BOOL "")
-set(SPHINX_SINGLEHTML ON CACHE BOOL "")
-set(SPHINX_QTHELP ON CACHE BOOL "")
-set(SPHINX_TEXT ON CACHE BOOL "")
-
+include("${CMAKE_CURRENT_LIST_DIR}/configure_sphinx.cmake")
 include("${CMAKE_CURRENT_LIST_DIR}/configure_common.cmake")

.gitlab/ci/configure_fedora31_sphinx_package.cmake

@@ -0,0 +1,13 @@
+# Disable formats not wanted in the package's documentation.
+set(SPHINX_INFO OFF CACHE BOOL "")
+set(SPHINX_SINGLEHTML OFF CACHE BOOL "")
+set(SPHINX_TEXT OFF CACHE BOOL "")
+
+# Set the destination directory for docs that packages expect.
+set(CMAKE_DOC_DIR "doc/cmake" CACHE STRING "")
+
+# Use a custom prefix to avoid conflicting with other builds.
+set(CMAKE_INSTALL_PREFIX "${CMAKE_BINARY_DIR}/install-doc" CACHE PATH "")
+
+include("${CMAKE_CURRENT_LIST_DIR}/configure_sphinx.cmake")
+include("${CMAKE_CURRENT_LIST_DIR}/configure_common.cmake")

.gitlab/ci/configure_macos_package.cmake

@@ -0,0 +1,24 @@
+set(CMake_DOC_ARTIFACT_PREFIX "$ENV{CI_PROJECT_DIR}/build/install-doc" CACHE PATH "")
+
+# Set up install destinations as expected by the packaging scripts.
+set(CMAKE_INSTALL_PREFIX "/" CACHE PATH "")
+set(CMAKE_DOC_DIR "doc/cmake" CACHE STRING "")
+
+# Settings for CMake packages for macOS.
+set(CPACK_DMG_FORMAT "UDBZ" CACHE STRING "")
+set(CMAKE_CXX_FLAGS "-stdlib=libc++" CACHE STRING "")
+set(CMAKE_C_STANDARD "11" CACHE STRING "")
+set(CMAKE_CXX_STANDARD "14" CACHE STRING "")
+set(CMAKE_OSX_ARCHITECTURES "x86_64" CACHE STRING "")
+set(CMAKE_OSX_DEPLOYMENT_TARGET "10.7" CACHE STRING "")
+set(CMAKE_SKIP_BOOTSTRAP_TEST "TRUE" CACHE STRING "")
+set(CPACK_SYSTEM_NAME "Darwin-x86_64" CACHE STRING "")
+set(BUILD_CursesDialog "ON" CACHE BOOL "")
+set(BUILD_QtDialog "TRUE" CACHE BOOL "")
+set(CMake_GUI_DISTRIBUTE_WITH_Qt_LGPL "3" CACHE STRING "")
+set(CMake_INSTALL_DEPENDENCIES "ON" CACHE BOOL "")
+set(CMAKE_SKIP_RPATH "TRUE" CACHE BOOL "")
+set(CMake_TEST_NO_FindPackageModeMakefileTest "TRUE" CACHE BOOL "")
+
+include("${CMAKE_CURRENT_LIST_DIR}/configure_macos_common.cmake")
+include("${CMAKE_CURRENT_LIST_DIR}/configure_common.cmake")

.gitlab/ci/configure_sphinx.cmake

@@ -0,0 +1,6 @@
+set(SPHINX_INFO ON CACHE BOOL "")
+set(SPHINX_MAN ON CACHE BOOL "")
+set(SPHINX_HTML ON CACHE BOOL "")
+set(SPHINX_SINGLEHTML ON CACHE BOOL "")
+set(SPHINX_QTHELP ON CACHE BOOL "")
+set(SPHINX_TEXT ON CACHE BOOL "")

.gitlab/ci/download_qt.cmake

@@ -6,7 +6,11 @@ set(qt_version_minor "15")
 set(qt_version_patch "0")
 # This URL is only visible inside of Kitware's network. Please use your own Qt
 # Account to obtain these files.
-set(qt_url_root "https://paraview.org/files/dependencies/internal/qt")
+if ("$ENV{CMAKE_CONFIGURATION}" MATCHES "package")
+  set(qt_url_root "https://cmake.org/files/dependencies")
+else ()
+  set(qt_url_root "https://paraview.org/files/dependencies/internal/qt")
+endif ()
 
 # Determine the ABI to fetch for Qt.
 if ("$ENV{CMAKE_CONFIGURATION}" MATCHES "vs2015")
@@ -51,15 +55,21 @@ if (qt_platform STREQUAL "windows_x86")
 
   set(qt_subdir "${qt_version}/msvc${msvc_year}_64")
 elseif (qt_platform STREQUAL "mac_x64")
-  set(qt_build_stamp "202005140805")
-  set(qt_file_name_prefix "${qt_version}-0-${qt_build_stamp}")
-
-  foreach (qt_component IN ITEMS qtbase)
+  if ("$ENV{CMAKE_CONFIGURATION}" MATCHES "package")
     list(APPEND qt_files
-      "${qt_file_name_prefix}${qt_component}-MacOS-MacOS_10_13-Clang-MacOS-MacOS_10_13-X86_64.7z")
-  endforeach ()
+      "qt-5.6.2-macosx10.7-x86_64.tar.xz")
+    set(qt_subdir "qt-5.6.2-macosx10.7-x86_64")
+  else ()
+    set(qt_build_stamp "202005140805")
+    set(qt_file_name_prefix "${qt_version}-0-${qt_build_stamp}")
+
+    foreach (qt_component IN ITEMS qtbase)
+      list(APPEND qt_files
+        "${qt_file_name_prefix}${qt_component}-MacOS-MacOS_10_13-Clang-MacOS-MacOS_10_13-X86_64.7z")
+    endforeach ()
 
-  set(qt_subdir "${qt_version}/clang_64")
+    set(qt_subdir "${qt_version}/clang_64")
+  endif ()
 else ()
   message(FATAL_ERROR
     "Unknown files for ${qt_platform}")
@@ -72,7 +82,11 @@ if (NOT qt_subdir)
 endif ()
 
 # Build up the path to the file to download.
-set(qt_url_path "${qt_platform}/desktop/qt5_${qt_version_nodot}/qt.qt5.${qt_version_nodot}.${qt_abi}")
+if ("$ENV{CMAKE_CONFIGURATION}" MATCHES "package")
+  set(qt_url_path "")
+else ()
+  set(qt_url_path "${qt_platform}/desktop/qt5_${qt_version_nodot}/qt.qt5.${qt_version_nodot}.${qt_abi}")
+endif ()
 set(qt_url_prefix "${qt_url_root}/${qt_url_path}")
 
 # Include the file containing the hashes of the files that matter.

.gitlab/ci/download_qt_hashes.cmake

@@ -9,3 +9,5 @@ set("5.15.0-0-202005150700qtbase-Windows-Windows_10-MSVC2015-Windows-Windows_10-
 set("5.15.0-0-202005150700qtwinextras-Windows-Windows_10-MSVC2015-Windows-Windows_10-X86_64.7z_hash" 4bca3a8d8c7611e211a82d86b3396f8a622abe7859d5052452414642ec191844)
 
 set("5.15.0-0-202005140805qtbase-MacOS-MacOS_10_13-Clang-MacOS-MacOS_10_13-X86_64.7z_hash" 04d867c81d2431f288c42c9752642759460b9468477de349368dcc8de0c8ddc4)
+
+set("qt-5.6.2-macosx10.7-x86_64.tar.xz_hash" 2b60373ea60037ce356d4c9f5a8c1df9854127a2c55118252e1a2f5a5f4e0010)

.gitlab/ci/gitlab_ci.cmake

@@ -5,10 +5,10 @@ endif ()
 
 # Set up the source and build paths.
 set(CTEST_SOURCE_DIRECTORY "$ENV{CI_PROJECT_DIR}")
+set(CTEST_BINARY_DIRECTORY "${CTEST_SOURCE_DIRECTORY}/build")
 if (NOT "$ENV{CTEST_SOURCE_SUBDIRECTORY}" STREQUAL "")
   string(APPEND CTEST_SOURCE_DIRECTORY "/$ENV{CTEST_SOURCE_SUBDIRECTORY}")
 endif ()
-set(CTEST_BINARY_DIRECTORY "${CTEST_SOURCE_DIRECTORY}/build")
 
 if ("$ENV{CMAKE_CONFIGURATION}" STREQUAL "")
   message(FATAL_ERROR

.gitlab/os-linux.yml

@@ -54,6 +54,13 @@
         CTEST_SOURCE_SUBDIRECTORY: "Utilities/Sphinx"
         CMake_SKIP_INSTALL: 1
 
+.fedora31_sphinx_package:
+    extends: .fedora31
+
+    variables:
+        CMAKE_CONFIGURATION: fedora31_sphinx_package
+        CTEST_SOURCE_SUBDIRECTORY: "Utilities/Sphinx"
+
 #### Build and test
 
 .fedora31_ninja:

.gitlab/os-macos.yml

@@ -37,6 +37,14 @@
         CTEST_NO_WARNINGS_ALLOWED: 1
         CMAKE_GENERATOR: "Unix Makefiles"
 
+.macos_package:
+    extends: .macos_build
+
+    variables:
+        CMAKE_CONFIGURATION: macos_package
+        CTEST_NO_WARNINGS_ALLOWED: 1
+        CMake_SKIP_INSTALL: 1
+
 ### External testing
 
 .macos_xcode:
@@ -56,6 +64,15 @@
         - xcode-11.5
         - nonconcurrent
 
+.macos_builder_tags_package:
+    tags:
+        - cmake # Since this is a bare runner, pin to a project.
+        - macos
+        - shell
+        - xcode-11.5
+        - nonconcurrent
+        - finder
+
 .macos_builder_ext_tags:
     tags:
         - cmake # Since this is a bare runner, pin to a project.
@@ -91,6 +108,24 @@
 
     interruptible: true
 
+.cmake_build_macos_package:
+    stage: package
+
+    script:
+        - *before_script_macos
+        - .gitlab/ci/sccache.sh
+        # Allow the server to already be running.
+        - "sccache --start-server || :"
+        - sccache --show-stats
+        - ctest -VV -S .gitlab/ci/ctest_configure.cmake
+        - ctest -VV -S .gitlab/ci/ctest_build.cmake
+        - sccache --show-stats
+        - cd build
+        - cpack -G TGZ
+        - cpack -G DragNDrop
+
+    interruptible: true
+
 .cmake_test_macos:
     stage: test
 

.gitlab/rules.yml

@@ -16,3 +16,9 @@
         - if: '$CI_PROJECT_PATH == "cmake/cmake"'
           when: on_success
         - when: never
+
+.run_only_for_package:
+    rules:
+        - if: '$CMAKE_PACKAGE == "true"'
+          when: on_success
+        - when: never

.gitlab/upload.yml

@@ -0,0 +1,18 @@
+# Steps for uploading artifacts
+
+.rsync_upload:
+    image: "fedora:32"
+    stage: upload
+    tags:
+        - docker
+        - linux
+        - build
+    environment:
+        name: rsync-upload
+
+    script:
+        - ls build/
+        - dnf install -y --setopt=install_weak_deps=False rsync openssh-clients
+        - chmod 400 $RSYNC_BINARY_KEY
+        - ssh-keygen -y -f $RSYNC_BINARY_KEY > $RSYNC_BINARY_KEY.pub
+        - rsync -tv --recursive -e "ssh -i $RSYNC_BINARY_KEY -o StrictHostKeyChecking=no" build/ [email protected]:$RSYNC_DESTINATION/

Tests/CMakeLists.txt

@@ -804,10 +804,6 @@ ${CMake_SOURCE_DIR}/Utilities/Release/push.bash --dir dev -- '${CMake_BUILD_NIGH
       set_tests_properties (${name} PROPERTIES TIMEOUT ${CMAKE_LONG_TEST_TIMEOUT})
     endif()
   endmacro()
-  if(CMake_BUILD_NIGHTLY_RELEASES)
-    ADD_NIGHTLY_BUILD_TEST(CMakeNightlyOSX
-      osx_release.cmake)
-  endif()
 
   # add tests with more complex invocations
   add_test(Framework ${CMAKE_CTEST_COMMAND}

Utilities/CMakeLists.txt

@@ -21,6 +21,11 @@ if(CMAKE_DOC_TARBALL)
   add_custom_target(documentation ALL DEPENDS ${dir}.stamp)
   install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/${dir}/
           DESTINATION . USE_SOURCE_PERMISSIONS)
+elseif (CMake_DOC_ARTIFACT_PREFIX)
+  # Undocumented option for CI usage to reuse already
+  # built documentation.
+  install(DIRECTORY ${CMake_DOC_ARTIFACT_PREFIX}/
+          DESTINATION . USE_SOURCE_PERMISSIONS)
 else()
   # Normal documentation build.
   add_subdirectory(Sphinx)

Utilities/Release/create-cmake-release.cmake

@@ -45,10 +45,8 @@ echo 'Failed to create \${name}.tar.gz'
 endfunction()
 
 write_docs_shell_script("create-${CMAKE_CREATE_VERSION}-docs.sh")
-write_rel_shell_script("create-${CMAKE_CREATE_VERSION}-macos.sh"   osx_release    ) # macOS x86_64
 
 message("Build docs first and then build for each platform:
  ./create-${CMAKE_CREATE_VERSION}-docs.sh    &&
- ./create-${CMAKE_CREATE_VERSION}-macos.sh   &&
  echo done
 ")

Utilities/Release/osx_release.cmake

@@ -1,34 +0,0 @@
-set(PROCESSORS 4)
-set(CMAKE_RELEASE_DIRECTORY /Users/kitware/CMakeReleaseDirectory)
-set(BOOTSTRAP_ARGS "--prefix=/ --docdir=doc/cmake")
-set(HOST dragnipur)
-set(MAKE_PROGRAM "make")
-set(MAKE "${MAKE_PROGRAM} -j5")
-set(CPACK_BINARY_GENERATORS "DragNDrop TGZ")
-set(CPACK_SOURCE_GENERATORS "")
-set(CPACK_DMG_FORMAT "UDBZ") #build using bzip2 for smaller package size
-set(CC clang)
-set(CXX clang++)
-set(CFLAGS   "")
-set(CXXFLAGS "-stdlib=libc++")
-set(INITIAL_CACHE "
-CMAKE_BUILD_TYPE:STRING=Release
-CMAKE_C_STANDARD:STRING=11
-CMAKE_CXX_STANDARD:STRING=14
-CMAKE_OSX_ARCHITECTURES:STRING=x86_64
-CMAKE_OSX_DEPLOYMENT_TARGET:STRING=10.7
-CMAKE_SKIP_BOOTSTRAP_TEST:STRING=TRUE
-CPACK_SYSTEM_NAME:STRING=Darwin-x86_64
-BUILD_CursesDialog:BOOL=ON
-BUILD_QtDialog:BOOL=TRUE
-CMake_GUI_DISTRIBUTE_WITH_Qt_LGPL:STRING=3
-CMake_INSTALL_DEPENDENCIES:BOOL=ON
-CMAKE_SKIP_RPATH:BOOL=TRUE
-CMake_TEST_NO_FindPackageModeMakefileTest:BOOL=TRUE
-")
-set(ENV [[
-export CMAKE_PREFIX_PATH='/Users/kitware/SDKs/qt-5.6.2-clang-x64'
-]])
-set(SIGN "")
-get_filename_component(path "${CMAKE_CURRENT_LIST_FILE}" PATH)
-include(${path}/release_cmake.cmake)