Merge topic 'file-archive'

c7e1198a23 file: Add ARCHIVE_{CREATE|EXTRACT} subcommands

Acked-by: Kitware Robot <[email protected]>
Merge-request: !4475

Help/command/file.rst

@@ -42,6 +42,10 @@ Synopsis
   `Locking`_
     file(`LOCK`_ <path> [...])
 
+   `Archiving`_
+     file(`ARCHIVE_CREATE`_ OUTPUT <archive> FILES <files> [...])
+     file(`ARCHIVE_EXTRACT`_ INPUT <archive> DESTINATION <dir> [...])
+
 Reading
 ^^^^^^^
 
@@ -888,3 +892,62 @@ child directory or file.
 Trying to lock file twice is not allowed.  Any intermediate directories and
 file itself will be created if they not exist.  ``GUARD`` and ``TIMEOUT``
 options ignored on ``RELEASE`` operation.
+
+Archiving
+^^^^^^^^^
+
+.. _ARCHIVE_CREATE:
+
+.. code-block:: cmake
+
+  file(ARCHIVE_CREATE OUTPUT <archive>
+    [FILES <files>]
+    [DIRECTORY <dirs>]
+    [FORMAT <format>]
+    [TYPE <type>]
+    [MTIME <mtime>]
+    [VERBOSE])
+
+Creates an archive specifed by ``OUTPUT`` with the content of ``FILES`` and
+``DIRECTORY``.
+
+To specify the format of the archive set the ``FORMAT`` option.
+Supported formats are: ``7zip``, ``gnutar``, ``pax``, ``paxr``, ``raw``,
+(restricted pax, default), and ``zip``.
+
+To specify the type of compression set the ``TYPE`` option.
+Supported compression types are: ``None``, ``BZip2``, ``GZip``, ``XZ``,
+and ``Zstd``.
+
+.. note::
+  With ``FORMAT`` set to ``raw`` only one file will be compressed with the
+  compression type specified by ``TYPE``.
+
+With ``VERBOSE`` the command will produce verbose output.
+
+To specify the modification time recorded in tarball entries use
+the ``MTIME`` option.
+
+.. _ARCHIVE_EXTRACT:
+
+.. code-block:: cmake
+
+  file(ARCHIVE_EXTRACT INPUT <archive>
+    [FILES <files>]
+    [DIRECTORY <dirs>]
+    [DESTINATION <dir>]
+    [LIST_ONLY]
+    [VERBOSE])
+
+Extracts or lists the content of an archive specified by ``INPUT``.
+
+The directory where the content of the archive will be extracted can
+be specified via ``DESTINATION``. If the directory does not exit, it
+will be created.
+
+To select which files and directories will be extracted or listed
+use  ``FILES`` and ``DIRECTORY`` options.
+
+``LIST_ONLY`` will only list the files in the archive.
+
+With ``VERBOSE`` the command will produce verbose output.

Help/release/dev/file_archive.rst

@@ -0,0 +1,7 @@
+file_archive
+------------
+
+* The :command:`file` command gained the ``ARCHIVE_{CREATE|EXTRACT}`` subcommands.
+
+  These subcommands will replicate the :manual:`cmake(1)` ``-E tar`` functionality in
+  CMake scripting code.

Source/cmFileCommand.cxx

@@ -8,6 +8,7 @@
 #include <cmath>
 #include <cstdio>
 #include <cstdlib>
+#include <iterator>
 #include <map>
 #include <set>
 #include <sstream>
@@ -49,6 +50,7 @@
 #include "cmSubcommandTable.h"
 #include "cmSystemTools.h"
 #include "cmTimestamp.h"
+#include "cmWorkingDirectory.h"
 #include "cmake.h"
 
 #if !defined(CMAKE_BOOTSTRAP)
@@ -2893,6 +2895,210 @@ bool HandleConfigureCommand(std::vector<std::string> const& args,
   return true;
 }
 
+bool HandleArchiveCreateCommand(std::vector<std::string> const& args,
+                                cmExecutionStatus& status)
+{
+  struct Arguments
+  {
+    std::string Output;
+    std::string Format;
+    std::string Type;
+    std::string MTime;
+    bool Verbose = false;
+    std::vector<std::string> Files;
+    std::vector<std::string> Directories;
+  };
+
+  static auto const parser = cmArgumentParser<Arguments>{}
+                               .Bind("OUTPUT"_s, &Arguments::Output)
+                               .Bind("FORMAT"_s, &Arguments::Format)
+                               .Bind("TYPE"_s, &Arguments::Type)
+                               .Bind("MTIME"_s, &Arguments::MTime)
+                               .Bind("VERBOSE"_s, &Arguments::Verbose)
+                               .Bind("FILES"_s, &Arguments::Files)
+                               .Bind("DIRECTORY"_s, &Arguments::Directories);
+
+  std::vector<std::string> unrecognizedArguments;
+  std::vector<std::string> keywordsMissingValues;
+  auto parsedArgs =
+    parser.Parse(cmMakeRange(args).advance(1), &unrecognizedArguments,
+                 &keywordsMissingValues);
+  auto argIt = unrecognizedArguments.begin();
+  if (argIt != unrecognizedArguments.end()) {
+    status.SetError(cmStrCat("Unrecognized argument: \"", *argIt, "\""));
+    cmSystemTools::SetFatalErrorOccured();
+    return false;
+  }
+
+  const std::vector<std::string> LIST_ARGS = {
+    "OUTPUT", "FORMAT", "TYPE", "MTIME", "FILES", "DIRECTORY",
+  };
+  auto kwbegin = keywordsMissingValues.cbegin();
+  auto kwend = cmRemoveMatching(keywordsMissingValues, LIST_ARGS);
+  if (kwend != kwbegin) {
+    status.SetError(cmStrCat("Keywords missing values:\n  ",
+                             cmJoin(cmMakeRange(kwbegin, kwend), "\n  ")));
+    cmSystemTools::SetFatalErrorOccured();
+    return false;
+  }
+
+  const char* knownFormats[] = {
+    "7zip", "gnutar", "pax", "paxr", "raw", "zip"
+  };
+
+  if (!parsedArgs.Format.empty() &&
+      !cmContains(knownFormats, parsedArgs.Format)) {
+    status.SetError(
+      cmStrCat("archive format ", parsedArgs.Format, " not supported"));
+    cmSystemTools::SetFatalErrorOccured();
+    return false;
+  }
+
+  const char* zipFileFormats[] = { "7zip", "zip" };
+  if (!parsedArgs.Type.empty() &&
+      cmContains(zipFileFormats, parsedArgs.Format)) {
+    status.SetError(cmStrCat("archive format ", parsedArgs.Format,
+                             " does not support TYPE arguments"));
+    cmSystemTools::SetFatalErrorOccured();
+    return false;
+  }
+
+  static std::map<std::string, cmSystemTools::cmTarCompression>
+    compressionTypeMap = { { "None", cmSystemTools::TarCompressNone },
+                           { "BZip2", cmSystemTools::TarCompressBZip2 },
+                           { "GZip", cmSystemTools::TarCompressGZip },
+                           { "XZ", cmSystemTools::TarCompressXZ },
+                           { "Zstd", cmSystemTools::TarCompressZstd } };
+
+  std::string const& outFile = parsedArgs.Output;
+  std::vector<std::string> files = parsedArgs.Files;
+  std::copy(parsedArgs.Directories.begin(), parsedArgs.Directories.end(),
+            std::back_inserter(files));
+
+  cmSystemTools::cmTarCompression compress = cmSystemTools::TarCompressNone;
+  auto typeIt = compressionTypeMap.find(parsedArgs.Type);
+  if (typeIt != compressionTypeMap.end()) {
+    compress = typeIt->second;
+  } else if (!parsedArgs.Type.empty()) {
+    status.SetError(
+      cmStrCat("compression type ", parsedArgs.Type, " is not supported"));
+    cmSystemTools::SetFatalErrorOccured();
+    return false;
+  }
+
+  if (files.empty()) {
+    status.GetMakefile().IssueMessage(MessageType::AUTHOR_WARNING,
+                                      "No files or directories specified");
+  }
+
+  if (!cmSystemTools::CreateTar(outFile, files, compress, parsedArgs.Verbose,
+                                parsedArgs.MTime, parsedArgs.Format)) {
+    status.SetError(cmStrCat("failed to compress: ", outFile));
+    cmSystemTools::SetFatalErrorOccured();
+    return false;
+  }
+
+  return true;
+}
+
+bool HandleArchiveExtractCommand(std::vector<std::string> const& args,
+                                 cmExecutionStatus& status)
+{
+  struct Arguments
+  {
+    std::string Input;
+    bool Verbose = false;
+    bool ListOnly = false;
+    std::string Destination;
+    std::vector<std::string> Files;
+    std::vector<std::string> Directories;
+  };
+
+  static auto const parser = cmArgumentParser<Arguments>{}
+                               .Bind("INPUT"_s, &Arguments::Input)
+                               .Bind("VERBOSE"_s, &Arguments::Verbose)
+                               .Bind("LIST_ONLY"_s, &Arguments::ListOnly)
+                               .Bind("DESTINATION"_s, &Arguments::Destination)
+                               .Bind("FILES"_s, &Arguments::Files)
+                               .Bind("DIRECTORY"_s, &Arguments::Directories);
+
+  std::vector<std::string> unrecognizedArguments;
+  std::vector<std::string> keywordsMissingValues;
+  auto parsedArgs =
+    parser.Parse(cmMakeRange(args).advance(1), &unrecognizedArguments,
+                 &keywordsMissingValues);
+  auto argIt = unrecognizedArguments.begin();
+  if (argIt != unrecognizedArguments.end()) {
+    status.SetError(cmStrCat("Unrecognized argument: \"", *argIt, "\""));
+    cmSystemTools::SetFatalErrorOccured();
+    return false;
+  }
+
+  const std::vector<std::string> LIST_ARGS = {
+    "INPUT",
+    "DESTINATION",
+    "FILES",
+    "DIRECTORY",
+  };
+  auto kwbegin = keywordsMissingValues.cbegin();
+  auto kwend = cmRemoveMatching(keywordsMissingValues, LIST_ARGS);
+  if (kwend != kwbegin) {
+    status.SetError(cmStrCat("Keywords missing values:\n  ",
+                             cmJoin(cmMakeRange(kwbegin, kwend), "\n  ")));
+    cmSystemTools::SetFatalErrorOccured();
+    return false;
+  }
+
+  std::string inFile = parsedArgs.Input;
+  std::vector<std::string> files = parsedArgs.Files;
+  std::copy(parsedArgs.Directories.begin(), parsedArgs.Directories.end(),
+            std::back_inserter(files));
+
+  if (parsedArgs.ListOnly) {
+    if (!cmSystemTools::ListTar(inFile, files, parsedArgs.Verbose)) {
+      status.SetError(cmStrCat("failed to list: ", inFile));
+      cmSystemTools::SetFatalErrorOccured();
+      return false;
+    }
+  } else {
+    std::string destDir = cmSystemTools::GetCurrentWorkingDirectory();
+    if (!parsedArgs.Destination.empty()) {
+      if (cmSystemTools::FileIsFullPath(parsedArgs.Destination)) {
+        destDir = parsedArgs.Destination;
+      } else {
+        destDir = cmStrCat(destDir, "/", parsedArgs.Destination);
+      }
+
+      if (!cmSystemTools::MakeDirectory(destDir)) {
+        status.SetError(cmStrCat("failed to create directory: ", destDir));
+        cmSystemTools::SetFatalErrorOccured();
+        return false;
+      }
+
+      if (!cmSystemTools::FileIsFullPath(inFile)) {
+        inFile =
+          cmStrCat(cmSystemTools::GetCurrentWorkingDirectory(), "/", inFile);
+      }
+    }
+
+    cmWorkingDirectory workdir(destDir);
+    if (workdir.Failed()) {
+      status.SetError(
+        cmStrCat("failed to change working directory to: ", destDir));
+      cmSystemTools::SetFatalErrorOccured();
+      return false;
+    }
+
+    if (!cmSystemTools::ExtractTar(inFile, files, parsedArgs.Verbose)) {
+      status.SetError(cmStrCat("failed to extract: ", inFile));
+      cmSystemTools::SetFatalErrorOccured();
+      return false;
+    }
+  }
+
+  return true;
+}
+
 } // namespace
 
 bool cmFileCommand(std::vector<std::string> const& args,
@@ -2947,6 +3153,8 @@ bool cmFileCommand(std::vector<std::string> const& args,
     { "CREATE_LINK"_s, HandleCreateLinkCommand },
     { "GET_RUNTIME_DEPENDENCIES"_s, HandleGetRuntimeDependenciesCommand },
     { "CONFIGURE"_s, HandleConfigureCommand },
+    { "ARCHIVE_CREATE"_s, HandleArchiveCreateCommand },
+    { "ARCHIVE_EXTRACT"_s, HandleArchiveExtractCommand },
   };
 
   return subcommand(args[0], args, status);

Tests/RunCMake/CMakeLists.txt

@@ -461,6 +461,7 @@ if(CMAKE_C_COMPILER_ID STREQUAL "AppleClang"
   add_RunCMake_test(Framework)
 endif()
 
+add_RunCMake_test(File_Archive)
 add_RunCMake_test(File_Configure)
 add_RunCMake_test(File_Generate)
 add_RunCMake_test(ExportWithoutLanguage)

Tests/RunCMake/File_Archive/7zip.cmake

@@ -0,0 +1,7 @@
+set(OUTPUT_NAME "test.7z")
+
+set(COMPRESSION_FORMAT 7zip)
+
+include(${CMAKE_CURRENT_LIST_DIR}/roundtrip.cmake)
+
+check_magic("377abcaf271c" LIMIT 6 HEX)

Tests/RunCMake/File_Archive/CMakeLists.txt

@@ -0,0 +1,3 @@
+cmake_minimum_required(VERSION 3.0)
+project(${RunCMake_TEST} NONE)
+include(${RunCMake_TEST}.cmake)

Tests/RunCMake/File_Archive/RunCMakeTest.cmake

@@ -0,0 +1,17 @@
+include(RunCMake)
+
+run_cmake(7zip)
+run_cmake(gnutar)
+run_cmake(gnutar-gz)
+run_cmake(pax)
+run_cmake(pax-xz)
+run_cmake(pax-zstd)
+run_cmake(paxr)
+run_cmake(paxr-bz2)
+run_cmake(zip)
+
+# Extracting only selected files or directories
+run_cmake(zip-filtered)
+
+run_cmake(unsupported-format)
+run_cmake(zip-with-bad-type)

Tests/RunCMake/File_Archive/gnutar-gz.cmake

@@ -0,0 +1,8 @@
+set(OUTPUT_NAME "test.tar.gz")
+
+set(COMPRESSION_FORMAT gnutar)
+set(COMPRESSION_TYPE GZip)
+
+include(${CMAKE_CURRENT_LIST_DIR}/roundtrip.cmake)
+
+check_magic("1f8b" LIMIT 2 HEX)

Tests/RunCMake/File_Archive/gnutar.cmake

@@ -0,0 +1,7 @@
+set(OUTPUT_NAME "test.tar")
+
+set(COMPRESSION_FORMAT gnutar)
+
+include(${CMAKE_CURRENT_LIST_DIR}/roundtrip.cmake)
+
+check_magic("7573746172202000" OFFSET 257 LIMIT 8 HEX)

Tests/RunCMake/File_Archive/pax-xz.cmake

@@ -0,0 +1,8 @@
+set(OUTPUT_NAME "test.tar.xz")
+
+set(COMPRESSION_FORMAT pax)
+set(COMPRESSION_TYPE XZ)
+
+include(${CMAKE_CURRENT_LIST_DIR}/roundtrip.cmake)
+
+check_magic("fd377a585a00" LIMIT 6 HEX)

Tests/RunCMake/File_Archive/pax-zstd.cmake

@@ -0,0 +1,8 @@
+set(OUTPUT_NAME "test.tar.zstd")
+
+set(COMPRESSION_FORMAT pax)
+set(COMPRESSION_TYPE Zstd)
+
+include(${CMAKE_CURRENT_LIST_DIR}/roundtrip.cmake)
+
+check_magic("28b52ffd0058" LIMIT 6 HEX)

Tests/RunCMake/File_Archive/pax.cmake

@@ -0,0 +1,7 @@
+set(OUTPUT_NAME "test.tar")
+
+set(COMPRESSION_FORMAT pax)
+
+include(${CMAKE_CURRENT_LIST_DIR}/roundtrip.cmake)
+
+check_magic("7573746172003030" OFFSET 257 LIMIT 8 HEX)

Tests/RunCMake/File_Archive/paxr-bz2.cmake

@@ -0,0 +1,8 @@
+set(OUTPUT_NAME "test.tar.bz2")
+
+set(COMPRESSION_FORMAT paxr)
+set(COMPRESSION_TYPE BZip2)
+
+include(${CMAKE_CURRENT_LIST_DIR}/roundtrip.cmake)
+
+check_magic("425a68" LIMIT 3 HEX)

Tests/RunCMake/File_Archive/paxr.cmake

@@ -0,0 +1,7 @@
+set(OUTPUT_NAME "test.tar")
+
+set(COMPRESSION_FORMAT paxr)
+
+include(${CMAKE_CURRENT_LIST_DIR}/roundtrip.cmake)
+
+check_magic("7573746172003030" OFFSET 257 LIMIT 8 HEX)

Tests/RunCMake/File_Archive/roundtrip.cmake

@@ -0,0 +1,92 @@
+foreach(parameter OUTPUT_NAME COMPRESSION_FORMAT)
+  if(NOT DEFINED ${parameter})
+    message(FATAL_ERROR "missing required parameter ${parameter}")
+  endif()
+endforeach()
+
+set(COMPRESS_DIR compress_dir)
+set(FULL_COMPRESS_DIR ${CMAKE_CURRENT_BINARY_DIR}/${COMPRESS_DIR})
+
+set(DECOMPRESS_DIR decompress_dir)
+set(FULL_DECOMPRESS_DIR ${CMAKE_CURRENT_BINARY_DIR}/${DECOMPRESS_DIR})
+
+set(FULL_OUTPUT_NAME ${CMAKE_CURRENT_BINARY_DIR}/${OUTPUT_NAME})
+
+set(CHECK_FILES
+  "f1.txt"
+  "d1/f1.txt"
+  "d 2/f1.txt"
+  "d + 3/f1.txt"
+  "d_4/f1.txt"
+  "d-4/f1.txt"
+  "My Special Directory/f1.txt"
+)
+
+foreach(file ${CHECK_FILES})
+  configure_file(${CMAKE_CURRENT_LIST_FILE} ${FULL_COMPRESS_DIR}/${file} COPYONLY)
+endforeach()
+
+if(UNIX)
+  execute_process(COMMAND ln -sf f1.txt ${FULL_COMPRESS_DIR}/d1/f2.txt)
+  list(APPEND CHECK_FILES "d1/f2.txt")
+endif()
+
+file(REMOVE ${FULL_OUTPUT_NAME})
+file(REMOVE_RECURSE ${FULL_DECOMPRESS_DIR})
+file(MAKE_DIRECTORY ${FULL_DECOMPRESS_DIR})
+
+file(ARCHIVE_CREATE
+    OUTPUT ${FULL_OUTPUT_NAME}
+    FORMAT "${COMPRESSION_FORMAT}"
+    TYPE "${COMPRESSION_TYPE}"
+    VERBOSE
+    DIRECTORY ${COMPRESS_DIR})
+
+file(ARCHIVE_EXTRACT
+    INPUT ${FULL_OUTPUT_NAME}
+    ${DECOMPRESSION_OPTIONS}
+    DESTINATION ${FULL_DECOMPRESS_DIR}
+    VERBOSE)
+
+if(CUSTOM_CHECK_FILES)
+  set(CHECK_FILES ${CUSTOM_CHECK_FILES})
+endif()
+
+foreach(file ${CHECK_FILES})
+  set(input ${FULL_COMPRESS_DIR}/${file})
+  set(output ${FULL_DECOMPRESS_DIR}/${COMPRESS_DIR}/${file})
+
+  if(NOT EXISTS ${input})
+     message(SEND_ERROR "Cannot find input file ${output}")
+  endif()
+
+  if(NOT EXISTS ${output})
+     message(SEND_ERROR "Cannot find output file ${output}")
+  endif()
+
+  file(MD5 ${input} input_md5)
+  file(MD5 ${output} output_md5)
+
+  if(NOT input_md5 STREQUAL output_md5)
+    message(SEND_ERROR "Files \"${input}\" and \"${output}\" are different")
+  endif()
+endforeach()
+
+foreach(file ${NOT_EXISTING_FILES_CHECK})
+  set(output ${FULL_DECOMPRESS_DIR}/${COMPRESS_DIR}/${file})
+
+  if(EXISTS ${output})
+     message(SEND_ERROR "File ${output} exists but it shouldn't")
+  endif()
+endforeach()
+
+function(check_magic EXPECTED)
+  file(READ ${FULL_OUTPUT_NAME} ACTUAL
+    ${ARGN}
+  )
+
+  if(NOT ACTUAL STREQUAL EXPECTED)
+    message(FATAL_ERROR
+      "Actual [${ACTUAL}] does not match expected [${EXPECTED}]")
+  endif()
+endfunction()

Tests/RunCMake/File_Archive/unsupported-format-result.txt

@@ -0,0 +1 @@
+1

Tests/RunCMake/File_Archive/unsupported-format-stderr.txt

@@ -0,0 +1,5 @@
+CMake Error at roundtrip.cmake:38 \(file\):
+  file archive format rar not supported
+Call Stack \(most recent call first\):
+  unsupported-format.cmake:5 \(include\)
+  CMakeLists.txt:3 \(include\)

Tests/RunCMake/File_Archive/unsupported-format.cmake

@@ -0,0 +1,5 @@
+set(OUTPUT_NAME "test.rar")
+
+set(COMPRESSION_FORMAT rar)
+
+include(${CMAKE_CURRENT_LIST_DIR}/roundtrip.cmake)

Tests/RunCMake/File_Archive/zip-filtered.cmake

@@ -0,0 +1,26 @@
+set(OUTPUT_NAME "test.zip")
+
+set(COMPRESSION_FORMAT zip)
+
+set(DECOMPRESSION_OPTIONS
+  FILES
+    compress_dir/f1.txt # Decompress only file
+    compress_dir/d1     # and whole directory
+)
+
+set(CUSTOM_CHECK_FILES
+  "f1.txt"
+  "d1/f1.txt"
+)
+
+# This files shouldn't exists
+set(NOT_EXISTING_FILES_CHECK
+  "d 2/f1.txt"
+  "d + 3/f1.txt"
+  "d_4/f1.txt"
+  "d-4/f1.txt"
+)
+
+include(${CMAKE_CURRENT_LIST_DIR}/roundtrip.cmake)
+
+check_magic("504b0304" LIMIT 4 HEX)

Tests/RunCMake/File_Archive/zip-with-bad-type-result.txt

@@ -0,0 +1 @@
+1

Tests/RunCMake/File_Archive/zip-with-bad-type-stderr.txt

@@ -0,0 +1,5 @@
+CMake Error at roundtrip.cmake:38 \(file\):
+  file archive format zip does not support TYPE arguments
+Call Stack \(most recent call first\):
+  zip-with-bad-type.cmake:6 \(include\)
+  CMakeLists.txt:3 \(include\)

Tests/RunCMake/File_Archive/zip-with-bad-type.cmake

@@ -0,0 +1,6 @@
+set(OUTPUT_NAME "test.zip")
+
+set(COMPRESSION_FORMAT zip)
+set(COMPRESSION_TYPE BZip2)
+
+include(${CMAKE_CURRENT_LIST_DIR}/roundtrip.cmake)

Tests/RunCMake/File_Archive/zip.cmake

@@ -0,0 +1,7 @@
+set(OUTPUT_NAME "test.zip")
+
+set(COMPRESSION_FORMAT zip)
+
+include(${CMAKE_CURRENT_LIST_DIR}/roundtrip.cmake)
+
+check_magic("504b0304" LIMIT 4 HEX)