linux-capture: Fix window capture crashes

The xcomposite window capture crashes were due to a few factors:
-------------------------------
1.) The source's X error handler was possibly being overwritten by
another part of the program despite us locking the display, presumably
something in Qt which isn't locking the display when pushing/popping its
own error handler (though this is not yet certain).  The source's calls
to X functions happen in the graphics thread, which is separate from the
UI thread, and it was noticed that somehow the error handler would be
overwritten almost seemingly at random, indicating that something else
in the program outside of OBS code was not locking the display while
pushing/popping the error handler.

To replicate this, make it so that the source cannot find the target
window and so it continually searches for it each video_tick call, then
resize the main OBS window continually (which causes Qt to push/pop its
own error handlers).  A crash will almost always occur due to BadWindow
despite our error handling.

2.) Calling X functions with a window ID that no longer exists,
particularly XGetWindowAttributes, in conjunction the unknown error
handler set in case #1 would cause the program to outright crash because
that error handler is programmed to crash on BadWindow for whatever
reason.  The source would call X functions without even checking if
'win' was 0.

3.) The source stored window IDs (in JSON, even if they've long since
become invalid/pointless, such as system restarts).  This is a bad
practice and will result in more cases of BadWindow.

Fixing the problem (reducing the possibility of getting BadWindow):
-------------------------------
Step 1.) Deprecate and ignore window IDs in stored settings.  Instead of
using window IDs to find the window, we now must always search the
windows and find the target window via the window name exclusively.
This helps ensure that we actually consistently have a working window
ID.

Step 2.) Do not call any X functions if the window ID is 0.

Step 3.) Reset the window ID to 0 any time the window has updated, and
make the source find the window again to ensure it still exists before
attempting to use any X functions on the window ID again.

plugins/linux-capture/xcompcap-helper.cpp

@@ -409,6 +409,39 @@ void XErrorLock::resetError()
 	curErrorText[0] = 0;
 }
 
+XDisplayLock::XDisplayLock()
+{
+	islock = false;
+	lock();
+}
+
+XDisplayLock::~XDisplayLock()
+{
+	unlock();
+}
+
+bool XDisplayLock::isLocked()
+{
+	return islock;
+}
+
+void XDisplayLock::lock()
+{
+	if (!islock) {
+		XLockDisplay(XCompcap::disp());
+		islock = true;
+	}
+}
+
+void XDisplayLock::unlock()
+{
+	if (islock) {
+		XSync(XCompcap::disp(), 0);
+		XUnlockDisplay(XCompcap::disp());
+		islock = false;
+	}
+}
+
 
 ObsGsContextHolder::ObsGsContextHolder()
 {

plugins/linux-capture/xcompcap-helper.hpp

@@ -48,6 +48,23 @@ class XErrorLock
 	void resetError();
 };
 
+class XDisplayLock
+{
+	bool islock;
+
+	public:
+	XDisplayLock(const XDisplayLock&) = delete;
+	XDisplayLock& operator=(const XDisplayLock&) = delete;
+
+	XDisplayLock();
+	~XDisplayLock();
+
+	bool isLocked();
+
+	void unlock();
+	void lock();
+};
+
 class ObsGsContextHolder
 {
 	public:

plugins/linux-capture/xcompcap-main.cpp

@@ -106,6 +106,7 @@ void XCompcapMain::defaults(obs_data_t *settings)
 	obs_data_set_default_bool(settings, "exclude_alpha", false);
 }
 
+#define FIND_WINDOW_INTERVAL 2.0
 
 struct XCompcapMain_private
 {
@@ -137,7 +138,7 @@ struct XCompcapMain_private
 	obs_source_t *source;
 
 	std::string windowName;
-	Window win;
+	Window win = 0;
 	int cut_top, cur_cut_top;
 	int cut_left, cur_cut_left;
 	int cut_right, cur_cut_right;
@@ -148,6 +149,8 @@ struct XCompcapMain_private
 	bool include_border;
 	bool exclude_alpha;
 
+	double window_check_time = 0.0;
+
 	uint32_t width;
 	uint32_t height;
 	uint32_t border;
@@ -201,6 +204,8 @@ XCompcapMain::~XCompcapMain()
 
 static Window getWindowFromString(std::string wstr)
 {
+	XErrorLock xlock;
+
 	if (wstr == "") {
 		return XCompcap::getTopLevelWindows().front();
 	}
@@ -215,8 +220,7 @@ static Window getWindowFromString(std::string wstr)
 	if (firstMark == std::string::npos)
 		return (Window)std::stol(wstr);
 
-	std::string widstr = wstr.substr(0, firstMark);
-	Window wid = (Window)std::stol(widstr);
+	Window wid = 0;
 
 	wstr = wstr.substr(firstMark + strlen(WIN_STRING_DIV));
 
@@ -240,7 +244,7 @@ static Window getWindowFromString(std::string wstr)
 static void xcc_cleanup(XCompcapMain_private *p)
 {
 	PLock lock(&p->lock);
-	XErrorLock xlock;
+	XDisplayLock xlock;
 
 	if (p->gltex) {
 		gs_texture_destroy(p->gltex);
@@ -299,7 +303,9 @@ void XCompcapMain::updateSettings(obs_data_t *settings)
 
 	xlock.resetError();
 
-	XCompositeRedirectWindow(xdisp, p->win, CompositeRedirectAutomatic);
+	if (p->win)
+		XCompositeRedirectWindow(xdisp, p->win,
+				CompositeRedirectAutomatic);
 
 	if (xlock.gotError()) {
 		blog(LOG_ERROR, "XCompositeRedirectWindow failed: %s",
@@ -307,18 +313,19 @@ void XCompcapMain::updateSettings(obs_data_t *settings)
 		return;
 	}
 
-	XSelectInput(xdisp, p->win, StructureNotifyMask | ExposureMask);
+	if (p->win)
+		XSelectInput(xdisp, p->win, StructureNotifyMask | ExposureMask);
 	XSync(xdisp, 0);
 
 	XWindowAttributes attr;
-	if (!XGetWindowAttributes(xdisp, p->win, &attr)) {
+	if (!p->win || !XGetWindowAttributes(xdisp, p->win, &attr)) {
 		p->win = 0;
 		p->width = 0;
 		p->height = 0;
 		return;
 	}
 
-	if (p->cursor && p->show_cursor) {
+	if (p->win && p->cursor && p->show_cursor) {
 		Window child;
 		int x, y;
 
@@ -451,8 +458,6 @@ void XCompcapMain::updateSettings(obs_data_t *settings)
 
 void XCompcapMain::tick(float seconds)
 {
-	UNUSED_PARAMETER(seconds);
-
 	if (!obs_source_showing(p->source))
 		return;
 
@@ -463,21 +468,30 @@ void XCompcapMain::tick(float seconds)
 
 	XCompcap::processEvents();
 
-	if (XCompcap::windowWasReconfigured(p->win))
-		updateSettings(0);
+	if (p->win && XCompcap::windowWasReconfigured(p->win)) {
+		p->window_check_time = FIND_WINDOW_INTERVAL;
+		p->win = 0;
+	}
 
-	XErrorLock xlock;
-	xlock.resetError();
+	XDisplayLock xlock;
 	XWindowAttributes attr;
 
-	if (!XGetWindowAttributes(xdisp, p->win, &attr)) {
+	if (!p->win || !XGetWindowAttributes(xdisp, p->win, &attr)) {
+		p->window_check_time += (double)seconds;
+
+		if (p->window_check_time < FIND_WINDOW_INTERVAL)
+			return;
+
 		Window newWin = getWindowFromString(p->windowName);
 
-		if (XGetWindowAttributes(xdisp, newWin, &attr)) {
+		p->window_check_time = 0.0;
+
+		if (newWin && XGetWindowAttributes(xdisp, newWin, &attr)) {
 			p->win = newWin;
 			updateSettings(0);
+		} else {
+			return;
 		}
-		return;
 	}
 
 	if (!p->tex || !p->gltex)
@@ -524,11 +538,11 @@ void XCompcapMain::tick(float seconds)
 
 void XCompcapMain::render(gs_effect_t *effect)
 {
-	PLock lock(&p->lock, true);
-
 	if (!p->win)
 		return;
 
+	PLock lock(&p->lock, true);
+
 	effect = obs_get_base_effect(OBS_EFFECT_OPAQUE);
 
 	if (!lock.isLocked() || !p->tex)
@@ -552,10 +566,16 @@ void XCompcapMain::render(gs_effect_t *effect)
 
 uint32_t XCompcapMain::width()
 {
+	if (!p->win)
+		return 0;
+
 	return p->width - p->cur_cut_left - p->cur_cut_right;
 }
 
 uint32_t XCompcapMain::height()
 {
+	if (!p->win)
+		return 0;
+
 	return p->height - p->cur_cut_bot - p->cur_cut_top;
 }