linux-capture: Log warning when unable to query windows

Checks whether window manager supports ewmh. Logs warning if ewmh is not
supported.

Closes jp9000/obs-studio#488

plugins/linux-capture/xcompcap-helper.cpp

@@ -47,10 +47,82 @@ namespace XCompcap
 		return res;
 	}
 
+	// Specification for checking for ewmh support at
+	// http://standards.freedesktop.org/wm-spec/wm-spec-latest.html#idm140200472693600
+
+	bool ewmhIsSupported()
+	{
+		Display *display = disp();
+		Atom netSupportingWmCheck = XInternAtom(display,
+				"_NET_SUPPORTING_WM_CHECK", true);
+		Atom actualType;
+		int format = 0;
+		unsigned long num = 0, bytes = 0;
+		unsigned char *data = NULL;
+		Window ewmh_window = 0;
+
+		int status = XGetWindowProperty(
+				display,
+				DefaultRootWindow(display),
+				netSupportingWmCheck,
+				0L,
+				1L,
+				false,
+				XA_WINDOW,
+				&actualType,
+				&format,
+				&num,
+				&bytes,
+				&data);
+
+		if (status == Success) {
+			if (num > 0) {
+				ewmh_window = ((Window*)data)[0];
+			}
+			if (data) {
+				XFree(data);
+				data = NULL;
+			}
+		}
+
+		if (ewmh_window) {
+			status = XGetWindowProperty(
+					display,
+					ewmh_window,
+					netSupportingWmCheck,
+					0L,
+					1L,
+					false,
+					XA_WINDOW,
+					&actualType,
+					&format,
+					&num,
+					&bytes,
+					&data);
+			if (status != Success || num == 0 ||
+					ewmh_window != ((Window*)data)[0]) {
+				ewmh_window = 0;
+			}
+			if (status == Success && data) {
+				XFree(data);
+			}
+		}
+
+		return ewmh_window != 0;
+	}
+
 	std::list<Window> getTopLevelWindows()
 	{
 		std::list<Window> res;
 
+		if (!ewmhIsSupported()) {
+			blog(LOG_WARNING, "Unable to query window list "
+					"because window manager "
+					"does not support extended "
+					"window manager Hints");
+			return res;
+		}
+
 		Atom netClList = XInternAtom(disp(), "_NET_CLIENT_LIST", true);
 		Atom actualType;
 		int format;

plugins/linux-capture/xcompcap-helper.hpp

@@ -67,6 +67,7 @@ namespace XCompcap
 	int getRootWindowScreen(Window root);
 	std::string getWindowName(Window win);
 	int getWindowPid(Window win);
+	bool ewmhIsSupported();
 	std::list<Window> getTopLevelWindows();
 	std::list<Window> getAllWindows();