cmake: Update use of MbedTLS to support update to version 3.6.0

MbedTLS changed a lot of internals with their LTS version 3.6.0, which
are incompatible with the find module currently shipped with
OBS Studio.

The solution requires several changes to be applied at once:

* Rename the generated target name to MbedTLS::mbedtls to match the
  name used by MbedTLS' own CMake package
* Update find module to use the updated target name(s)
* Set CMAKE_FIND_PACKAGE_PREFER_CONFIG to TRUE before trying to find
  MbedTLS to ensure that CMake package files are used with priority
  (Those are shipped only with MbedTLS 3.6.0 in obs-deps).
* A deprecation warning is emitted if the find module is used with
  MbedTLS 3.6.0 available

UI/cmake/feature-whatsnew.cmake

@@ -6,14 +6,16 @@ if(ENABLE_WHATSNEW AND TARGET OBS::browser-panels)
   if(OS_MACOS)
     include(cmake/feature-macos-update.cmake)
   elseif(OS_LINUX)
+    set(CMAKE_FIND_PACKAGE_PREFER_CONFIG TRUE)
     find_package(MbedTLS REQUIRED)
+    set(CMAKE_FIND_PACKAGE_PREFER_CONFIG FALSE)
     find_package(nlohmann_json REQUIRED)
 
     if(NOT TARGET OBS::blake2)
       add_subdirectory("${CMAKE_SOURCE_DIR}/deps/blake2" "${CMAKE_BINARY_DIR}/deps/blake2")
     endif()
 
-    target_link_libraries(obs-studio PRIVATE MbedTLS::MbedTLS nlohmann_json::nlohmann_json OBS::blake2)
+    target_link_libraries(obs-studio PRIVATE MbedTLS::mbedtls nlohmann_json::nlohmann_json OBS::blake2)
 
     target_sources(
       obs-studio

UI/cmake/os-windows.cmake

@@ -6,7 +6,9 @@ if(NOT TARGET OBS::w32-pthreads)
   add_subdirectory("${CMAKE_SOURCE_DIR}/deps/w32-pthreads" "${CMAKE_BINARY_DIR}/deps/w32-pthreads")
 endif()
 
-find_package(MbedTLS)
+set(CMAKE_FIND_PACKAGE_PREFER_CONFIG TRUE)
+find_package(MbedTLS REQUIRED)
+set(CMAKE_FIND_PACKAGE_PREFER_CONFIG FALSE)
 find_package(Detours REQUIRED)
 find_package(nlohmann_json REQUIRED)
 
@@ -38,12 +40,12 @@ target_sources(obs-studio PRIVATE system-info-windows.cpp)
 
 target_link_libraries(
   obs-studio
-  PRIVATE crypt32 OBS::blake2 OBS::w32-pthreads MbedTLS::MbedTLS nlohmann_json::nlohmann_json Detours::Detours
+  PRIVATE crypt32 OBS::blake2 OBS::w32-pthreads MbedTLS::mbedtls nlohmann_json::nlohmann_json Detours::Detours
 )
 
 target_compile_definitions(obs-studio PRIVATE PSAPI_VERSION=2)
 
-target_link_options(obs-studio PRIVATE /IGNORE:4099)
+target_link_options(obs-studio PRIVATE /IGNORE:4099 $<$<CONFIG:DEBUG>:/NODEFAULTLIB:MSVCRT>)
 
 add_library(obs-update-helpers INTERFACE)
 add_library(OBS::update-helpers ALIAS obs-update-helpers)

cmake/finders/FindMbedTLS.cmake

@@ -14,9 +14,9 @@ Components
 
 This module contains provides several components:
 
-``MbedCrypto``
-``MbedTLS``
-``MbedX509``
+``mbedcrypto``
+``mbedtls``
+``mbedx509``
 
 Import targets exist for each component.
 
@@ -27,13 +27,13 @@ Imported Targets
 
 This module defines the :prop_tgt:`IMPORTED` targets:
 
-``MbedTLS::MbedCrypto``
+``MbedTLS::mbedcrypto``
   Crypto component
 
-``MbedTLS::MbedTLS``
+``MbedTLS::mbedtls``
   TLS component
 
-``MbedTLS::MbedX509``
+``MbedTLS::mbedX509``
   X509 component
 
 Result Variables
@@ -78,7 +78,7 @@ macro(MbedTLS_set_soname component)
     )
 
     if(_result EQUAL 0 AND _output MATCHES "^@rpath/")
-      set_property(TARGET MbedTLS::Mbed${component} PROPERTY IMPORTED_SONAME "${_output}")
+      set_property(TARGET MbedTLS::mbed${component} PROPERTY IMPORTED_SONAME "${_output}")
     endif()
   elseif(CMAKE_HOST_SYSTEM_NAME MATCHES "Linux|FreeBSD")
     execute_process(
@@ -89,7 +89,7 @@ macro(MbedTLS_set_soname component)
 
     if(_result EQUAL 0)
       string(REGEX REPLACE "[ \t]+SONAME[ \t]+([^ \t]+)" "\\1" _soname "${_output}")
-      set_property(TARGET MbedTLS::Mbed${component} PROPERTY IMPORTED_SONAME "${_soname}")
+      set_property(TARGET MbedTLS::mbed${component} PROPERTY IMPORTED_SONAME "${_soname}")
       unset(_soname)
     endif()
   endif()
@@ -142,8 +142,15 @@ else()
   set(MbedTLS_VERSION 0.0.0)
 endif()
 
+if(MbedTLS_VERSION VERSION_GREATER_EQUAL 3.6.0)
+  message(
+    DEPRECATION
+    "Use of the custom CMake find module for MbedTLS versions >= 3.6.0 is not supported - build errors might occur!"
+  )
+endif()
+
 find_library(
-  MbedTLS_LIBRARY
+  Mbedtls_LIBRARY
   NAMES libmbedtls mbedtls
   HINTS "${PC_MbedTLS_LIBRARY_DIRS}"
   PATHS /usr/lib /usr/local/lib
@@ -151,7 +158,7 @@ find_library(
 )
 
 find_library(
-  MbedCrypto_LIBRARY
+  Mbedcrypto_LIBRARY
   NAMES libmbedcrypto mbedcrypto
   HINTS "${PC_MbedTLS_LIBRARY_DIRS}"
   PATHS /usr/lib /usr/local/lib
@@ -159,14 +166,14 @@ find_library(
 )
 
 find_library(
-  MbedX509_LIBRARY
+  Mbedx509_LIBRARY
   NAMES libmbedx509 mbedx509
   HINTS "${PC_MbedTLS_LIBRARY_DIRS}"
   PATHS /usr/lib /usr/local/lib
   DOC "MbedX509 location"
 )
 
-if(MbedTLS_LIBRARY AND NOT MbedCrypto_LIBRARY AND NOT MbedX509_LIBRARY)
+if(Mbedtls_LIBRARY AND NOT Mbedcrypto_LIBRARY AND NOT Mbedx509_LIBRARY)
   set(CMAKE_REQUIRED_LIBRARIES "${MbedTLS_LIBRARY}")
   set(CMAKE_REQUIRED_INCLUDES "${MbedTLS_INCLUDE_DIR}")
 
@@ -185,38 +192,38 @@ endif()
 if(MbedTLS_INCLUDES_X509 AND MbedTLS_INCLUDES_CRYPTO)
   find_package_handle_standard_args(
     MbedTLS
-    REQUIRED_VARS MbedTLS_LIBRARY MbedTLS_INCLUDE_DIR
+    REQUIRED_VARS Mbedtls_LIBRARY MbedTLS_INCLUDE_DIR
     VERSION_VAR MbedTLS_VERSION
     REASON_FAILURE_MESSAGE "${MbedTLS_ERROR_REASON}"
   )
-  mark_as_advanced(MbedTLS_LIBRARY MbedTLS_INCLUDE_DIR)
-  list(APPEND _COMPONENTS TLS)
+  mark_as_advanced(Mbedtls_LIBRARY MbedTLS_INCLUDE_DIR)
+  list(APPEND _COMPONENTS tls)
 else()
   find_package_handle_standard_args(
     MbedTLS
-    REQUIRED_VARS MbedTLS_LIBRARY MbedCrypto_LIBRARY MbedX509_LIBRARY MbedTLS_INCLUDE_DIR
+    REQUIRED_VARS Mbedtls_LIBRARY Mbedcrypto_LIBRARY Mbedx509_LIBRARY MbedTLS_INCLUDE_DIR
     VERSION_VAR MbedTLS_VERSION
     REASON_FAILURE_MESSAGE "${MbedTLS_ERROR_REASON}"
   )
-  mark_as_advanced(MbedTLS_LIBRARY MbedCrypto_LIBRARY MbedX509_LIBRARY MbedTLS_INCLUDE_DIR)
-  list(APPEND _COMPONENTS TLS Crypto X509)
+  mark_as_advanced(Mbedtls_LIBRARY Mbedcrypto_LIBRARY Mbedx509_LIBRARY MbedTLS_INCLUDE_DIR)
+  list(APPEND _COMPONENTS tls crypto x509)
 endif()
 unset(MbedTLS_ERROR_REASON)
 
 if(MbedTLS_FOUND)
   foreach(component IN LISTS _COMPONENTS)
-    if(NOT TARGET MbedTLS::Mbed${component})
+    if(NOT TARGET MbedTLS::mbed${component})
       if(IS_ABSOLUTE "${Mbed${component}_LIBRARY}")
-        add_library(MbedTLS::Mbed${component} UNKNOWN IMPORTED)
-        set_property(TARGET MbedTLS::Mbed${component} PROPERTY IMPORTED_LOCATION "${Mbed${component}_LIBRARY}")
+        add_library(MbedTLS::mbed${component} UNKNOWN IMPORTED)
+        set_property(TARGET MbedTLS::mbed${component} PROPERTY IMPORTED_LOCATION "${Mbed${component}_LIBRARY}")
       else()
-        add_library(MbedTLS::Mbed${component} INTERFACE IMPORTED)
-        set_property(TARGET MbedTLS::Mbed${component} PROPERTY IMPORTED_LIBNAME "${Mbed${component}_LIBRARY}")
+        add_library(MbedTLS::mbed${component} INTERFACE IMPORTED)
+        set_property(TARGET MbedTLS::mbed${component} PROPERTY IMPORTED_LIBNAME "${Mbed${component}_LIBRARY}")
       endif()
 
       mbedtls_set_soname(${component})
       set_target_properties(
-        MbedTLS::MbedTLS
+        MbedTLS::mbedtls
         PROPERTIES
           INTERFACE_COMPILE_OPTIONS "${PC_MbedTLS_CFLAGS_OTHER}"
           INTERFACE_INCLUDE_DIRECTORIES "${MbedTLS_INCLUDE_DIR}"
@@ -230,7 +237,7 @@ if(MbedTLS_FOUND)
     set(MbedTLS_LIBRARIES ${MbedTLS_LIBRARY})
   else()
     set(MbedTLS_LIBRARIES ${MbedTLS_LIBRARY} ${MbedCrypto_LIBRARY} ${MbedX509_LIBRARY})
-    set_property(TARGET MbedTLS::MbedTLS PROPERTY INTERFACE_LINK_LIBRARIES MbedTLS::MbedCrypto MbedTLS::MbedX509)
+    set_property(TARGET MbedTLS::mbedtls PROPERTY INTERFACE_LINK_LIBRARIES MbedTLS::mbedcrypto MbedTLS::mbedx509)
   endif()
 endif()
 

plugins/obs-outputs/CMakeLists.txt

@@ -1,6 +1,8 @@
 cmake_minimum_required(VERSION 3.28...3.30)
 
+set(CMAKE_FIND_PACKAGE_PREFER_CONFIG TRUE)
 find_package(MbedTLS REQUIRED)
+set(CMAKE_FIND_PACKAGE_PREFER_CONFIG FALSE)
 find_package(ZLIB REQUIRED)
 
 if(NOT TARGET happy-eyeballs)
@@ -71,7 +73,7 @@ target_link_libraries(
     OBS::libobs
     OBS::happy-eyeballs
     OBS::opts-parser
-    MbedTLS::MbedTLS
+    MbedTLS::mbedtls
     ZLIB::ZLIB
     $<$<PLATFORM_ID:Windows>:OBS::w32-pthreads>
     $<$<PLATFORM_ID:Windows>:crypt32>
@@ -83,7 +85,10 @@ target_link_libraries(
 )
 
 # Remove once jansson has been fixed on obs-deps
-target_link_options(obs-outputs PRIVATE $<$<PLATFORM_ID:Windows>:/IGNORE:4098>)
+target_link_options(
+  obs-outputs
+  PRIVATE $<$<PLATFORM_ID:Windows>:/IGNORE:4098> $<$<AND:$<PLATFORM_ID:Windows>,$<CONFIG:DEBUG>>:/NODEFAULTLIB:MSVCRT>
+)
 
 if(OS_WINDOWS)
   configure_file(cmake/windows/obs-module.rc.in obs-outputs.rc)