CMakeDetermineCompilerABI: Detect byte order as part of check

We already detect `sizeof(void*)`.  Detect the byte order as part of the
same check.

Issue: #21392

Help/manual/cmake-variables.7.rst

@@ -521,6 +521,7 @@ Variables for Languages
    /variable/CMAKE_LANG_ARCHIVE_APPEND
    /variable/CMAKE_LANG_ARCHIVE_CREATE
    /variable/CMAKE_LANG_ARCHIVE_FINISH
+   /variable/CMAKE_LANG_BYTE_ORDER
    /variable/CMAKE_LANG_COMPILER
    /variable/CMAKE_LANG_COMPILER_EXTERNAL_TOOLCHAIN
    /variable/CMAKE_LANG_COMPILER_ID

Help/release/dev/abi-byte-order.rst

@@ -0,0 +1,5 @@
+abi-byte-order
+--------------
+
+* The :variable:`CMAKE_<LANG>_BYTE_ORDER` variable was added to provide the
+  target architecture byte order detected from the toolchain.

Help/variable/CMAKE_LANG_BYTE_ORDER.rst

@@ -0,0 +1,20 @@
+CMAKE_<LANG>_BYTE_ORDER
+-----------------------
+
+.. versionadded:: 3.20
+
+Byte order of ``<LANG>`` compiler target architecture, if known.
+If defined and not empty, the value is one of:
+
+``BIG_ENDIAN``
+  The target architecture is Big Endian.
+
+``LITTLE_ENDIAN``
+  The target architecture is Little Endian.
+
+This is defined for languages ``C``, ``CXX``, ``OBJC``, ``OBJCXX``,
+and ``CUDA``.
+
+If :variable:`CMAKE_OSX_ARCHITECTURES` specifies multiple architectures, the
+value of ``CMAKE_<LANG>_BYTE_ORDER`` is non-empty only if all architectures
+share the same byte order.

Modules/CMakeCCompiler.cmake.in

@@ -48,6 +48,7 @@ set(CMAKE_C_LINKER_PREFERENCE 10)
 # Save compiler ABI information.
 set(CMAKE_C_SIZEOF_DATA_PTR "@CMAKE_C_SIZEOF_DATA_PTR@")
 set(CMAKE_C_COMPILER_ABI "@CMAKE_C_COMPILER_ABI@")
+set(CMAKE_C_BYTE_ORDER "@CMAKE_C_BYTE_ORDER@")
 set(CMAKE_C_LIBRARY_ARCHITECTURE "@CMAKE_C_LIBRARY_ARCHITECTURE@")
 
 if(CMAKE_C_SIZEOF_DATA_PTR)

Modules/CMakeCCompilerABI.c

@@ -17,6 +17,8 @@ int main(int argc, char* argv[])
 {
   int require = 0;
   require += info_sizeof_dptr[argc];
+  require += info_byte_order_big_endian[argc];
+  require += info_byte_order_little_endian[argc];
 #if defined(ABI_ID)
   require += info_abi[argc];
 #endif

Modules/CMakeCUDACompiler.cmake.in

@@ -31,6 +31,7 @@ set(CMAKE_CUDA_LINKER_PREFERENCE_PROPAGATES 1)
 
 set(CMAKE_CUDA_SIZEOF_DATA_PTR "@CMAKE_CUDA_SIZEOF_DATA_PTR@")
 set(CMAKE_CUDA_COMPILER_ABI "@CMAKE_CUDA_COMPILER_ABI@")
+set(CMAKE_CUDA_BYTE_ORDER "@CMAKE_CUDA_BYTE_ORDER@")
 set(CMAKE_CUDA_LIBRARY_ARCHITECTURE "@CMAKE_CUDA_LIBRARY_ARCHITECTURE@")
 
 if(CMAKE_CUDA_SIZEOF_DATA_PTR)

Modules/CMakeCUDACompilerABI.cu

@@ -8,6 +8,8 @@ int main(int argc, char* argv[])
 {
   int require = 0;
   require += info_sizeof_dptr[argc];
+  require += info_byte_order_big_endian[argc];
+  require += info_byte_order_little_endian[argc];
 #if defined(ABI_ID)
   require += info_abi[argc];
 #endif

Modules/CMakeCXXCompiler.cmake.in

@@ -60,6 +60,7 @@ set(CMAKE_CXX_LINKER_PREFERENCE_PROPAGATES 1)
 # Save compiler ABI information.
 set(CMAKE_CXX_SIZEOF_DATA_PTR "@CMAKE_CXX_SIZEOF_DATA_PTR@")
 set(CMAKE_CXX_COMPILER_ABI "@CMAKE_CXX_COMPILER_ABI@")
+set(CMAKE_CXX_BYTE_ORDER "@CMAKE_CXX_BYTE_ORDER@")
 set(CMAKE_CXX_LIBRARY_ARCHITECTURE "@CMAKE_CXX_LIBRARY_ARCHITECTURE@")
 
 if(CMAKE_CXX_SIZEOF_DATA_PTR)

Modules/CMakeCXXCompilerABI.cpp

@@ -8,6 +8,8 @@ int main(int argc, char* argv[])
 {
   int require = 0;
   require += info_sizeof_dptr[argc];
+  require += info_byte_order_big_endian[argc];
+  require += info_byte_order_little_endian[argc];
 #if defined(ABI_ID)
   require += info_abi[argc];
 #endif

Modules/CMakeCompilerABI.h

@@ -9,6 +9,18 @@ const char info_sizeof_dptr[] = {
   /* clang-format on */
 };
 
+/* Byte order.  Only one of these will have bytes in the right order.  */
+static unsigned short const info_byte_order_big_endian[] = {
+  /* INFO:byte_order string for BIG_ENDIAN */
+  0x494E, 0x464F, 0x3A62, 0x7974, 0x655F, 0x6F72, 0x6465, 0x725B,
+  0x4249, 0x475F, 0x454E, 0x4449, 0x414E, 0x5D00, 0x0000
+};
+static unsigned short const info_byte_order_little_endian[] = {
+  /* INFO:byte_order string for LITTLE_ENDIAN */
+  0x4E49, 0x4F46, 0x623A, 0x7479, 0x5F65, 0x726F, 0x6564, 0x5B72,
+  0x494C, 0x5454, 0x454C, 0x455F, 0x444E, 0x4149, 0x5D4E, 0x0000
+};
+
 /* Application Binary Interface.  */
 
 /* Check for (some) ARM ABIs.

Modules/CMakeDetermineCompilerABI.cmake

@@ -77,11 +77,22 @@ function(CMAKE_DETERMINE_COMPILER_ABI lang src)
         "Detecting ${lang} compiler ABI info compiled with the following output:\n${OUTPUT}\n\n")
       file(STRINGS "${BIN}" ABI_STRINGS LIMIT_COUNT 32 REGEX "INFO:[A-Za-z0-9_]+\\[[^]]*\\]")
       set(ABI_SIZEOF_DPTR "NOTFOUND")
+      set(ABI_BYTE_ORDER "NOTFOUND")
       set(ABI_NAME "NOTFOUND")
       foreach(info ${ABI_STRINGS})
         if("${info}" MATCHES "INFO:sizeof_dptr\\[0*([^]]*)\\]" AND NOT ABI_SIZEOF_DPTR)
           set(ABI_SIZEOF_DPTR "${CMAKE_MATCH_1}")
         endif()
+        if("${info}" MATCHES "INFO:byte_order\\[(BIG_ENDIAN|LITTLE_ENDIAN)\\]")
+          set(byte_order "${CMAKE_MATCH_1}")
+          if(ABI_BYTE_ORDER STREQUAL "NOTFOUND")
+            # Tentatively use the value because this is the first occurrence.
+            set(ABI_BYTE_ORDER "${byte_order}")
+          elseif(NOT ABI_BYTE_ORDER STREQUAL "${byte_order}")
+            # Drop value because multiple occurrences do not match.
+            set(ABI_BYTE_ORDER "")
+          endif()
+        endif()
         if("${info}" MATCHES "INFO:abi\\[([^]]*)\\]" AND NOT ABI_NAME)
           set(ABI_NAME "${CMAKE_MATCH_1}")
         endif()
@@ -93,6 +104,10 @@ function(CMAKE_DETERMINE_COMPILER_ABI lang src)
         set(CMAKE_${lang}_SIZEOF_DATA_PTR "${CMAKE_${lang}_SIZEOF_DATA_PTR_DEFAULT}" PARENT_SCOPE)
       endif()
 
+      if(ABI_BYTE_ORDER)
+        set(CMAKE_${lang}_BYTE_ORDER "${ABI_BYTE_ORDER}" PARENT_SCOPE)
+      endif()
+
       if(ABI_NAME)
         set(CMAKE_${lang}_COMPILER_ABI "${ABI_NAME}" PARENT_SCOPE)
       endif()

Modules/CMakeOBJCCompiler.cmake.in

@@ -45,6 +45,7 @@ endforeach()
 # Save compiler ABI information.
 set(CMAKE_OBJC_SIZEOF_DATA_PTR "@CMAKE_OBJC_SIZEOF_DATA_PTR@")
 set(CMAKE_OBJC_COMPILER_ABI "@CMAKE_OBJC_COMPILER_ABI@")
+set(CMAKE_OBJC_BYTE_ORDER "@CMAKE_OBJC_BYTE_ORDER@")
 set(CMAKE_OBJC_LIBRARY_ARCHITECTURE "@CMAKE_OBJC_LIBRARY_ARCHITECTURE@")
 
 if(CMAKE_OBJC_SIZEOF_DATA_PTR)

Modules/CMakeOBJCCompilerABI.m

@@ -12,6 +12,8 @@ int main(int argc, char *argv[])
 {
   int require = 0;
   require += info_sizeof_dptr[argc];
+  require += info_byte_order_big_endian[argc];
+  require += info_byte_order_little_endian[argc];
 #if defined(ABI_ID)
   require += info_abi[argc];
 #endif

Modules/CMakeOBJCXXCompiler.cmake.in

@@ -55,6 +55,7 @@ set(CMAKE_OBJCXX_LINKER_PREFERENCE_PROPAGATES 1)
 # Save compiler ABI information.
 set(CMAKE_OBJCXX_SIZEOF_DATA_PTR "@CMAKE_OBJCXX_SIZEOF_DATA_PTR@")
 set(CMAKE_OBJCXX_COMPILER_ABI "@CMAKE_OBJCXX_COMPILER_ABI@")
+set(CMAKE_OBJCXX_BYTE_ORDER "@CMAKE_OBJCXX_BYTE_ORDER@")
 set(CMAKE_OBJCXX_LIBRARY_ARCHITECTURE "@CMAKE_OBJCXX_LIBRARY_ARCHITECTURE@")
 
 if(CMAKE_OBJCXX_SIZEOF_DATA_PTR)

Modules/CMakeOBJCXXCompilerABI.mm

@@ -12,6 +12,8 @@ int main(int argc, char *argv[])
 {
   int require = 0;
   require += info_sizeof_dptr[argc];
+  require += info_byte_order_big_endian[argc];
+  require += info_byte_order_little_endian[argc];
 #if defined(ABI_ID)
   require += info_abi[argc];
 #endif

Tests/RunCMake/ABI/C.cmake

@@ -0,0 +1,4 @@
+enable_language(C)
+if(NOT CMAKE_C_BYTE_ORDER MATCHES "^(BIG_ENDIAN|LITTLE_ENDIAN)$" AND NOT CMAKE_OSX_ARCHITECTURES MATCHES ";ppc|ppc;")
+  message(FATAL_ERROR "CMAKE_C_BYTE_ORDER has unexpected value '${CMAKE_C_BYTE_ORDER}'")
+endif()

Tests/RunCMake/ABI/CMakeLists.txt

@@ -0,0 +1,3 @@
+cmake_minimum_required(VERSION 3.19)
+project(${RunCMake_TEST} NONE)
+include(${RunCMake_TEST}.cmake)

Tests/RunCMake/ABI/CUDA.cmake

@@ -0,0 +1,4 @@
+enable_language(CUDA)
+if(NOT CMAKE_CUDA_BYTE_ORDER MATCHES "^(BIG_ENDIAN|LITTLE_ENDIAN)$" AND NOT CMAKE_OSX_ARCHITECTURES MATCHES ";ppc|ppc;")
+  message(FATAL_ERROR "CMAKE_CUDA_BYTE_ORDER has unexpected value '${CMAKE_CUDA_BYTE_ORDER}'")
+endif()

Tests/RunCMake/ABI/CXX.cmake

@@ -0,0 +1,4 @@
+enable_language(CXX)
+if(NOT CMAKE_CXX_BYTE_ORDER MATCHES "^(BIG_ENDIAN|LITTLE_ENDIAN)$" AND NOT CMAKE_OSX_ARCHITECTURES MATCHES ";ppc|ppc;")
+  message(FATAL_ERROR "CMAKE_CXX_BYTE_ORDER has unexpected value '${CMAKE_CXX_BYTE_ORDER}'")
+endif()

Tests/RunCMake/ABI/OBJC.cmake

@@ -0,0 +1,4 @@
+enable_language(OBJC)
+if(NOT CMAKE_OBJC_BYTE_ORDER MATCHES "^(BIG_ENDIAN|LITTLE_ENDIAN)$" AND NOT CMAKE_OSX_ARCHITECTURES MATCHES ";ppc|ppc;")
+  message(FATAL_ERROR "CMAKE_OBJC_BYTE_ORDER has unexpected value '${CMAKE_OBJC_BYTE_ORDER}'")
+endif()

Tests/RunCMake/ABI/OBJCXX.cmake

@@ -0,0 +1,4 @@
+enable_language(OBJCXX)
+if(NOT CMAKE_OBJCXX_BYTE_ORDER MATCHES "^(BIG_ENDIAN|LITTLE_ENDIAN)$" AND NOT CMAKE_OSX_ARCHITECTURES MATCHES ";ppc|ppc;")
+  message(FATAL_ERROR "CMAKE_OBJCXX_BYTE_ORDER has unexpected value '${CMAKE_OBJCXX_BYTE_ORDER}'")
+endif()

Tests/RunCMake/ABI/RunCMakeTest.cmake

@@ -0,0 +1,13 @@
+include(RunCMake)
+
+run_cmake(C)
+run_cmake(CXX)
+
+if(APPLE)
+  run_cmake(OBJC)
+  run_cmake(OBJCXX)
+endif()
+
+if(CMake_TEST_CUDA)
+  run_cmake(CUDA)
+endif()

Tests/RunCMake/CMakeLists.txt

@@ -179,6 +179,10 @@ if(NOT CMake_TEST_EXTERNAL_CMAKE)
     )
 endif()
 
+add_RunCMake_test(ABI -DCMake_TEST_CUDA=${CMake_TEST_CUDA})
+set_property(TEST RunCMake.ABI APPEND
+  PROPERTY LABELS "CUDA")
+
 add_RunCMake_test(AndroidTestUtilities)
 set(autogen_with_qt5 FALSE)
 if(CMake_TEST_Qt5)