obs-qsv11: Fix VPL initialization on intel-mediaSDK

VPL does not have a consistent initialization process for the various
backends it dispatches to. So this resolves the new VPL dispatcher
failing to initialize the video session when it dispatches to
intel-mediaSDK backend. On the oneVPL-intel-gpu backend VA-API sessions
are correctly initialized so this wasnt noticed when tested on newer
hardware.

cmake/Modules/FindLibva.cmake

@@ -15,6 +15,8 @@ find_package(PkgConfig QUIET)
 if(PKG_CONFIG_FOUND)
   pkg_check_modules(_LIBVA libva)
   pkg_check_modules(_LIBVA_DRM libva-drm)
+  pkg_check_modules(_LIBVA_WAYLAND libva-wayland)
+  pkg_check_modules(_LIBVA_X11 libva-x11)
 endif()
 
 find_path(
@@ -35,14 +37,29 @@ find_library(
   HINTS ${_LIBVA_DRM_LIBRARY_DIRS}
   PATHS /usr/lib /usr/local/lib /opt/local/lib)
 
+find_library(
+  LIBVA_WAYLAND_LIB
+  NAMES ${_LIBVA_WAYLAND_LIBRARIES} libva-wayland
+  HINTS ${_LIBVA_WAYLAND_LIBRARY_DIRS}
+  PATHS /usr/lib /usr/local/lib /opt/local/lib)
+
+find_library(
+  LIBVA_X11_LIB
+  NAMES ${_LIBVA_X11_LIBRARIES} libva-x11
+  HINTS ${_LIBVA_X11_LIBRARY_DIRS}
+  PATHS /usr/lib /usr/local/lib /opt/local/lib)
+
 include(FindPackageHandleStandardArgs)
-find_package_handle_standard_args(Libva REQUIRED_VARS LIBVA_INCLUDE_DIR LIBVA_LIB LIBVA_DRM_LIB)
-mark_as_advanced(LIBVA_INCLUDE_DIR LIBVA_LIB LIBVA_DRM_LIB)
+find_package_handle_standard_args(Libva REQUIRED_VARS LIBVA_INCLUDE_DIR LIBVA_LIB LIBVA_DRM_LIB LIBVA_WAYLAND_LIB
+                                                      LIBVA_X11_LIB)
+mark_as_advanced(LIBVA_INCLUDE_DIR LIBVA_LIB LIBVA_DRM_LIB LIBVA_WAYLAND_LIB LIBVA_X11_LIB)
 
 if(LIBVA_FOUND)
   set(LIBVA_INCLUDE_DIRS ${LIBVA_INCLUDE_DIR})
   set(LIBVA_LIBRARIES ${LIBVA_LIB})
   set(LIBVA_DRM_LIBRARIES ${LIBVA_DRM_LIB})
+  set(LIBVA_WAYLAND_LIBRARIES ${LIBVA_WAYLAND_LIB})
+  set(LIBVA_X11_LIBRARIES ${LIBVA_X11_LIB})
 
   if(NOT TARGET Libva::va)
     if(IS_ABSOLUTE "${LIBVA_LIBRARIES}")
@@ -67,4 +84,28 @@ if(LIBVA_FOUND)
 
     set_target_properties(Libva::drm PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${LIBVA_INCLUDE_DIRS}")
   endif()
+
+  if(NOT TARGET Libva::wayland)
+    if(IS_ABSOLUTE "${LIBVA_WAYLAND_LIBRARIES}")
+      add_library(Libva::wayland UNKNOWN IMPORTED)
+      set_target_properties(Libva::wayland PROPERTIES IMPORTED_LOCATION "${LIBVA_WAYLAND_LIBRARIES}")
+    else()
+      add_library(Libva::wayland INTERFACE IMPORTED)
+      set_target_properties(Libva::wayland PROPERTIES IMPORTED_LIBNAME "${LIBVA_WAYLAND_LIBRARIES}")
+    endif()
+
+    set_target_properties(Libva::wayland PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${LIBVA_INCLUDE_DIRS}")
+  endif()
+
+  if(NOT TARGET Libva::x11)
+    if(IS_ABSOLUTE "${LIBVA_X11_LIBRARIES}")
+      add_library(Libva::x11 UNKNOWN IMPORTED)
+      set_target_properties(Libva::x11 PROPERTIES IMPORTED_LOCATION "${LIBVA_X11_LIBRARIES}")
+    else()
+      add_library(Libva::x11 INTERFACE IMPORTED)
+      set_target_properties(Libva::x11 PROPERTIES IMPORTED_LIBNAME "${LIBVA_X11_LIBRARIES}")
+    endif()
+
+    set_target_properties(Libva::x11 PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${LIBVA_INCLUDE_DIRS}")
+  endif()
 endif()

plugins/obs-qsv11/CMakeLists.txt

@@ -58,5 +58,5 @@ elseif(OS_LINUX)
 
   target_sources(obs-qsv11 PRIVATE common_utils_linux.cpp)
 
-  target_link_libraries(obs-qsv11 PRIVATE Libva::va Libva::drm)
+  target_link_libraries(obs-qsv11 PRIVATE Libva::va Libva::drm Libva::wayland Libva::x11)
 endif()

plugins/obs-qsv11/cmake/legacy.cmake

@@ -55,7 +55,7 @@ elseif(OS_LINUX)
 
   target_sources(obs-qsv11 PRIVATE common_utils_linux.cpp)
 
-  target_link_libraries(obs-qsv11 PRIVATE Libva::va Libva::drm)
+  target_link_libraries(obs-qsv11 PRIVATE Libva::va Libva::drm Libva::wayland Libva::x11)
 endif()
 
 set_target_properties(obs-qsv11 PROPERTIES FOLDER "plugins/obs-qsv11")

plugins/obs-qsv11/common_utils_linux.cpp

@@ -5,6 +5,8 @@
 #include <util/c99defs.h>
 #include <util/dstr.h>
 #include <va/va_drm.h>
+#include <va/va_x11.h>
+#include <va/va_wayland.h>
 #include <va/va_str.h>
 #include <stdio.h>
 #include <stdlib.h>
@@ -13,6 +15,9 @@
 #include <string.h>
 #include <dirent.h>
 
+#include <obs.h>
+#include <obs-nix-platform.h>
+
 mfxStatus simple_alloc(mfxHDL pthis, mfxFrameAllocRequest *request,
 		       mfxFrameAllocResponse *response)
 {
@@ -102,9 +107,29 @@ mfxStatus Initialize(mfxVersion ver, mfxSession *pSession,
 		cfg, (const mfxU8 *)"mfxImplDescription.AccelerationMode",
 		impl);
 
+	mfxHDL vaDisplay = nullptr;
+	if (obs_get_nix_platform() == OBS_NIX_PLATFORM_X11_EGL) {
+		vaDisplay =
+			vaGetDisplay((Display *)obs_get_nix_platform_display());
+	} else if (obs_get_nix_platform() == OBS_NIX_PLATFORM_WAYLAND) {
+		vaDisplay = vaGetDisplayWl(
+			(wl_display *)obs_get_nix_platform_display());
+	}
+
 	sts = MFXCreateSession(loader, 0, pSession);
 	MSDK_CHECK_RESULT(sts, MFX_ERR_NONE, sts);
 
+	// VPL expects the VADisplay to be initialized.
+	int major;
+	int minor;
+	if (vaInitialize(vaDisplay, &major, &minor) != VA_STATUS_SUCCESS) {
+		vaTerminate(vaDisplay);
+		return MFX_ERR_DEVICE_FAILED;
+	}
+	sts = MFXVideoCORE_SetHandle(*pSession, MFX_HANDLE_VA_DISPLAY,
+				     vaDisplay);
+	MSDK_CHECK_RESULT(sts, MFX_ERR_NONE, sts);
+
 	return sts;
 }