Merge topic 'parallel-lzma-compression'

b3bacf0152 cmCPackArchiveGenerator: support multithreaded compression
b71d385ed4 cmCPackArchiveGenerator: support setting archive options
948aa8bd1c cmArchiveWrite: support setting archive filter options
b9c17de023 cmArchiveWrite: split out opening the file

Acked-by: Kitware Robot <[email protected]>
Acked-by: Cristian Adam <[email protected]>
Merge-request: !3195

Help/cpack_gen/archive.rst

@@ -51,3 +51,25 @@ Variables specific to CPack Archive generator
   Enable component packaging. If enabled (ON), then the archive generator
   creates  multiple packages. The default is OFF, which means that a single
   package containing files of all components is generated.
+
+Variables used by CPack Archive generator
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+These variables are used by the Archive generator, but are also available to
+CPack generators which are essentially archives at their core. These include:
+
+  - :cpack_gen:`CPack Cygwin Generator`
+  - :cpack_gen:`CPack FreeBSD Generator`
+
+.. variable:: CPACK_ARCHIVE_THREADS
+
+  The number of threads to use when performing the compression. If set to
+  ``0``, the number of available cores on the machine will be used instead.
+  The default is ``1`` which limits compression to a single thread. Note that
+  not all compression modes support threading in all environments. Currently,
+  only the XZ compression may support it.
+
+.. note::
+
+    Official CMake binaries available on ``cmake.org`` ship with a ``liblzma``
+    that does not support parallel compression.

Help/cpack_gen/cygwin.rst

@@ -3,6 +3,11 @@ CPack Cygwin Generator
 
 Cygwin CPack generator (Cygwin).
 
+Variables affecting the CPack Cygwin generator
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+  - :variable:`CPACK_ARCHIVE_THREADS`
+
 Variables specific to CPack Cygwin generator
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 

Help/cpack_gen/freebsd.rst

@@ -3,6 +3,11 @@ CPack FreeBSD Generator
 
 The built in (binary) CPack FreeBSD (pkg) generator (Unix only)
 
+Variables affecting the CPack FreeBSD (pkg) generator
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+  - :variable:`CPACK_ARCHIVE_THREADS`
+
 Variables specific to CPack FreeBSD (pkg) generator
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 

Help/release/dev/parallel-lzma-compression.rst

@@ -0,0 +1,6 @@
+parallel-lzma-compression
+-------------------------
+
+* The :cpack_gen:`CPack Archive Generator`'s ``TXZ`` format learned the
+  :variable:`CPACK_ARCHIVE_THREADS` variable to enable parallel compression.
+  Requires support in the ``liblzma`` used by CMake.

Source/CPack/cmCPackArchiveGenerator.cxx

@@ -8,6 +8,8 @@
 #include <utility>
 #include <vector>
 
+#include "cm_libarchive.h"
+
 #include "cmCPackComponentGroup.h"
 #include "cmCPackGenerator.h"
 #include "cmCPackLog.h"
@@ -154,6 +156,20 @@ int cmCPackArchiveGenerator::addOneComponentToArchive(
   }                                                                           \
   cmArchiveWrite archive(gf, this->Compress, this->ArchiveFormat);            \
   do {                                                                        \
+    if (!this->SetArchiveOptions(&archive)) {                                 \
+      cmCPackLogger(cmCPackLog::LOG_ERROR,                                    \
+                    "Problem to set archive options <"                        \
+                      << (filename) << ">, ERROR = " << (archive).GetError()  \
+                      << std::endl);                                          \
+      return 0;                                                               \
+    }                                                                         \
+    if (!archive.Open()) {                                                    \
+      cmCPackLogger(cmCPackLog::LOG_ERROR,                                    \
+                    "Problem to open archive <"                               \
+                      << (filename) << ">, ERROR = " << (archive).GetError()  \
+                      << std::endl);                                          \
+      return 0;                                                               \
+    }                                                                         \
     if (!(archive)) {                                                         \
       cmCPackLogger(cmCPackLog::LOG_ERROR,                                    \
                     "Problem to create archive <"                             \
@@ -328,3 +344,23 @@ bool cmCPackArchiveGenerator::SupportsComponentInstallation() const
   // (for backward compatibility reason)
   return IsOn("CPACK_ARCHIVE_COMPONENT_INSTALL");
 }
+
+bool cmCPackArchiveGenerator::SetArchiveOptions(cmArchiveWrite* archive)
+{
+#if ARCHIVE_VERSION_NUMBER >= 3004000
+  // Upstream fixed an issue with their integer parsing in 3.4.0 which would
+  // cause spurious errors to be raised from `strtoull`.
+  if (this->Compress == cmArchiveWrite::CompressXZ) {
+    const char* threads = "1";
+    if (this->IsSet("CPACK_ARCHIVE_THREADS")) {
+      threads = this->GetOption("CPACK_ARCHIVE_THREADS");
+    }
+
+    if (!archive->SetFilterOption("xz", "threads", threads)) {
+      return false;
+    }
+  }
+#endif
+
+  return true;
+}

Source/CPack/cmCPackArchiveGenerator.h

@@ -86,6 +86,8 @@ private:
     return this->OutputExtension.c_str();
   }
 
+  bool SetArchiveOptions(cmArchiveWrite* archive);
+
 private:
   cmArchiveWrite::Compress Compress;
   std::string ArchiveFormat;

Source/CPack/cmCPackDebGenerator.cxx

@@ -173,6 +173,7 @@ bool DebGenerator::generateDataTar() const
   }
   cmArchiveWrite data_tar(fileStream_data_tar, TarCompressionType,
                           DebianArchiveType);
+  data_tar.Open();
 
   // uid/gid should be the one of the root user, and this root user has
   // always uid/gid equal to 0.
@@ -291,6 +292,7 @@ bool DebGenerator::generateControlTar(std::string const& md5Filename) const
   }
   cmArchiveWrite control_tar(fileStream_control_tar,
                              cmArchiveWrite::CompressGZip, DebianArchiveType);
+  control_tar.Open();
 
   // sets permissions and uid/gid for the files
   control_tar.SetUIDAndGID(0u, 0u);
@@ -410,6 +412,7 @@ bool DebGenerator::generateDeb() const
   cmGeneratedFileStream debStream;
   debStream.Open(outputPath, false, true);
   cmArchiveWrite deb(debStream, cmArchiveWrite::CompressNone, "arbsd");
+  deb.Open();
 
   // uid/gid should be the one of the root user, and this root user has
   // always uid/gid equal to 0.

Source/cmArchiveWrite.cxx

@@ -170,15 +170,19 @@ cmArchiveWrite::cmArchiveWrite(std::ostream& os, Compress c,
                            cm_archive_error_string(this->Archive));
     return;
   }
+}
 
+bool cmArchiveWrite::Open()
+{
   if (archive_write_open(
         this->Archive, this, nullptr,
         reinterpret_cast<archive_write_callback*>(&Callback::Write),
         nullptr) != ARCHIVE_OK) {
     this->Error =
       cmStrCat("archive_write_open: ", cm_archive_error_string(this->Archive));
-    return;
+    return false;
   }
+  return true;
 }
 
 cmArchiveWrite::~cmArchiveWrite()
@@ -365,3 +369,16 @@ bool cmArchiveWrite::AddData(const char* file, size_t size)
   }
   return true;
 }
+
+bool cmArchiveWrite::SetFilterOption(const char* module, const char* key,
+                                     const char* value)
+{
+  if (archive_write_set_filter_option(this->Archive, module, key, value) !=
+      ARCHIVE_OK) {
+    this->Error = "archive_write_set_filter_option: ";
+    this->Error += cm_archive_error_string(this->Archive);
+    return false;
+  }
+
+  return true;
+}

Source/cmArchiveWrite.h

@@ -62,6 +62,8 @@ public:
   cmArchiveWrite(const cmArchiveWrite&) = delete;
   cmArchiveWrite& operator=(const cmArchiveWrite&) = delete;
 
+  bool Open();
+
   /**
    * Add a path (file or directory) to the archive.  Directories are
    * added recursively.  The "path" must be readable on disk, either
@@ -139,6 +141,9 @@ public:
     this->Gname = "";
   }
 
+  //! Set an option on a filter;
+  bool SetFilterOption(const char* module, const char* key, const char* value);
+
 private:
   bool Okay() const { return this->Error.empty(); }
   bool AddPath(const char* path, size_t skip, const char* prefix,

Source/cmSystemTools.cxx

@@ -1321,6 +1321,7 @@ bool cmSystemTools::CreateTar(const std::string& outFileName,
 
   cmArchiveWrite a(fout, compress, format.empty() ? "paxr" : format);
 
+  a.Open();
   a.SetMTime(mtime);
   a.SetVerbose(verbose);
   bool tarCreatedSuccessfully = true;

Tests/RunCMake/CPack/RunCMakeTest.cmake

@@ -20,6 +20,8 @@ run_cpack_test(LONG_FILENAMES "DEB.LONG_FILENAMES" false "MONOLITHIC")
 run_cpack_test_subtests(MAIN_COMPONENT "invalid;found" "RPM.MAIN_COMPONENT" false "COMPONENT")
 run_cpack_test(MINIMAL "RPM.MINIMAL;DEB.MINIMAL;7Z;TBZ2;TGZ;TXZ;TZ;ZIP;STGZ;External" false "MONOLITHIC;COMPONENT")
 run_cpack_test_package_target(MINIMAL "RPM.MINIMAL;DEB.MINIMAL;7Z;TBZ2;TGZ;TXZ;TZ;ZIP;STGZ;External" false "MONOLITHIC;COMPONENT")
+run_cpack_test_package_target(THREADED_ALL "TXZ" false "MONOLITHIC;COMPONENT")
+run_cpack_test_package_target(THREADED "TXZ" false "MONOLITHIC;COMPONENT")
 run_cpack_test_subtests(PACKAGE_CHECKSUM "invalid;MD5;SHA1;SHA224;SHA256;SHA384;SHA512" "TGZ" false "MONOLITHIC")
 run_cpack_test(PARTIALLY_RELOCATABLE_WARNING "RPM.PARTIALLY_RELOCATABLE_WARNING" false "COMPONENT")
 run_cpack_test(PER_COMPONENT_FIELDS "RPM.PER_COMPONENT_FIELDS;DEB.PER_COMPONENT_FIELDS" false "COMPONENT")

Tests/RunCMake/CPack/tests/THREADED/ExpectedFiles.cmake

@@ -0,0 +1,2 @@
+set(EXPECTED_FILES_COUNT "1")
+set(EXPECTED_FILE_CONTENT_1_LIST "/foo;/foo/CMakeLists.txt")

Tests/RunCMake/CPack/tests/THREADED/test.cmake

@@ -0,0 +1,7 @@
+install(FILES CMakeLists.txt DESTINATION foo COMPONENT test)
+
+set(CPACK_ARCHIVE_THREADS 2)
+
+if(PACKAGING_TYPE STREQUAL "COMPONENT")
+  set(CPACK_COMPONENTS_ALL test)
+endif()

Tests/RunCMake/CPack/tests/THREADED_ALL/ExpectedFiles.cmake

@@ -0,0 +1,2 @@
+set(EXPECTED_FILES_COUNT "1")
+set(EXPECTED_FILE_CONTENT_1_LIST "/foo;/foo/CMakeLists.txt")

Tests/RunCMake/CPack/tests/THREADED_ALL/test.cmake

@@ -0,0 +1,7 @@
+install(FILES CMakeLists.txt DESTINATION foo COMPONENT test)
+
+set(CPACK_ARCHIVE_THREADS 0)
+
+if(PACKAGING_TYPE STREQUAL "COMPONENT")
+  set(CPACK_COMPONENTS_ALL test)
+endif()