UI: Retrieve Wayland surface from QWindow

On Wayland, we want to query the window's underlying
platform for the Wayland surface, instead of foolishly
retrieving the X11 display.

Pass QWindow instead of WId directly, and set the surface
as the platform data on Wayland systems.

UI/qt-display.cpp

@@ -89,7 +89,7 @@ void OBSQTDisplay::CreateDisplay()
 	info.format = GS_BGRA;
 	info.zsformat = GS_ZS_NONE;
 
-	QTToGSWindow(winId(), info.window);
+	QTToGSWindow(windowHandle(), info.window);
 
 	display = obs_display_create(&info, backgroundColor);
 

UI/qt-wrappers.cpp

@@ -30,9 +30,14 @@
 #include <QStandardItemModel>
 
 #if !defined(_WIN32) && !defined(__APPLE__)
+#include <obs-nix-platform.h>
 #include <QX11Info>
 #endif
 
+#ifdef ENABLE_WAYLAND
+#include <qpa/qplatformnativeinterface.h>
+#endif
+
 static inline void OBSErrorBoxva(QWidget *parent, const char *msg, va_list args)
 {
 	char full_message[4096];
@@ -108,15 +113,28 @@ void OBSMessageBox::critical(QWidget *parent, const QString &title,
 	mb.exec();
 }
 
-void QTToGSWindow(WId windowId, gs_window &gswindow)
+void QTToGSWindow(QWindow *window, gs_window &gswindow)
 {
 #ifdef _WIN32
-	gswindow.hwnd = (HWND)windowId;
+	gswindow.hwnd = (HWND)window->winId();
 #elif __APPLE__
-	gswindow.view = (id)windowId;
+	gswindow.view = (id)window->winId();
 #else
-	gswindow.id = windowId;
-	gswindow.display = QX11Info::display();
+	switch (obs_get_nix_platform()) {
+	case OBS_NIX_PLATFORM_X11_GLX:
+	case OBS_NIX_PLATFORM_X11_EGL:
+		gswindow.id = window->winId();
+		gswindow.display = obs_get_nix_platform_display();
+		break;
+#ifdef ENABLE_WAYLAND
+	case OBS_NIX_PLATFORM_WAYLAND:
+		QPlatformNativeInterface *native =
+			QGuiApplication::platformNativeInterface();
+		gswindow.display =
+			native->nativeResourceForWindow("surface", window);
+		break;
+#endif
+	}
 #endif
 }
 

UI/qt-wrappers.hpp

@@ -20,6 +20,7 @@
 #include <QApplication>
 #include <QMessageBox>
 #include <QWidget>
+#include <QWindow>
 #include <QThread>
 #include <obs.hpp>
 
@@ -56,7 +57,7 @@ public:
 
 void OBSErrorBox(QWidget *parent, const char *msg, ...);
 
-void QTToGSWindow(WId windowId, gs_window &gswindow);
+void QTToGSWindow(QWindow *window, gs_window &gswindow);
 
 uint32_t TranslateQtKeyboardEventModifiers(Qt::KeyboardModifiers mods);