libobs-winrt,win-capture: Cursor toggle exceptions

A user has reported that toggling the cursor can cause exceptions.

Propagate that information upward, so we can reset the capture.

Display capture never needed to call the function in the first place.

Shuffle around window capture code to make a common reset function.

Builds on PR #4663.

libobs-winrt/winrt-capture.cpp

@@ -599,15 +599,32 @@ extern "C" EXPORT BOOL winrt_capture_active(const struct winrt_capture *capture)
 	return capture->active;
 }
 
-extern "C" EXPORT void winrt_capture_show_cursor(struct winrt_capture *capture,
+extern "C" EXPORT BOOL winrt_capture_show_cursor(struct winrt_capture *capture,
 						 BOOL visible)
 {
-	if (capture->capture_cursor) {
-		if (capture->cursor_visible != visible) {
-			capture->session.IsCursorCaptureEnabled(visible);
-			capture->cursor_visible = visible;
+	BOOL success = FALSE;
+
+	try {
+		if (capture->capture_cursor) {
+			if (capture->cursor_visible != visible) {
+				capture->session.IsCursorCaptureEnabled(
+					visible);
+				capture->cursor_visible = visible;
+			}
 		}
+
+		success = TRUE;
+	} catch (winrt::hresult_error &err) {
+		blog(LOG_ERROR,
+		     "GraphicsCaptureSession::IsCursorCaptureEnabled (0x%08X): %ls",
+		     err.to_abi(), err.message().c_str());
+	} catch (...) {
+		blog(LOG_ERROR,
+		     "GraphicsCaptureSession::IsCursorCaptureEnabled (0x%08X)",
+		     winrt::to_hresult());
 	}
+
+	return success;
 }
 
 extern "C" EXPORT void winrt_capture_render(struct winrt_capture *capture,

libobs-winrt/winrt-capture.h

@@ -18,7 +18,7 @@ EXPORT struct winrt_capture *winrt_capture_init_monitor(BOOL cursor,
 EXPORT void winrt_capture_free(struct winrt_capture *capture);
 
 EXPORT BOOL winrt_capture_active(const struct winrt_capture *capture);
-EXPORT void winrt_capture_show_cursor(struct winrt_capture *capture,
+EXPORT BOOL winrt_capture_show_cursor(struct winrt_capture *capture,
 				      BOOL visible);
 EXPORT void winrt_capture_render(struct winrt_capture *capture,
 				 gs_effect_t *effect);

plugins/win-capture/duplicator-monitor-capture.c

@@ -39,8 +39,6 @@ typedef struct winrt_capture *(*PFN_winrt_capture_init_monitor)(
 typedef void (*PFN_winrt_capture_free)(struct winrt_capture *capture);
 
 typedef BOOL (*PFN_winrt_capture_active)(const struct winrt_capture *capture);
-typedef void (*PFN_winrt_capture_show_cursor)(struct winrt_capture *capture,
-					      BOOL visible);
 typedef void (*PFN_winrt_capture_render)(struct winrt_capture *capture,
 					 gs_effect_t *effect);
 typedef uint32_t (*PFN_winrt_capture_width)(const struct winrt_capture *capture);
@@ -54,7 +52,6 @@ struct winrt_exports {
 	PFN_winrt_capture_init_monitor winrt_capture_init_monitor;
 	PFN_winrt_capture_free winrt_capture_free;
 	PFN_winrt_capture_active winrt_capture_active;
-	PFN_winrt_capture_show_cursor winrt_capture_show_cursor;
 	PFN_winrt_capture_render winrt_capture_render;
 	PFN_winrt_capture_width winrt_capture_width;
 	PFN_winrt_capture_height winrt_capture_height;
@@ -302,7 +299,6 @@ static bool load_winrt_imports(struct winrt_exports *exports, void *module,
 	WINRT_IMPORT(winrt_capture_init_monitor);
 	WINRT_IMPORT(winrt_capture_free);
 	WINRT_IMPORT(winrt_capture_active);
-	WINRT_IMPORT(winrt_capture_show_cursor);
 	WINRT_IMPORT(winrt_capture_render);
 	WINRT_IMPORT(winrt_capture_width);
 	WINRT_IMPORT(winrt_capture_height);
@@ -397,10 +393,11 @@ static void duplicator_capture_tick(void *data, float seconds)
 			capture->showing = false;
 		}
 		return;
+	}
 
-		/* always try to load the capture immediately when the source is first
+	/* always try to load the capture immediately when the source is first
 	 * shown */
-	} else if (!capture->showing) {
+	if (!capture->showing) {
 		capture->reset_timeout = RESET_INTERVAL_SEC;
 	}
 
@@ -428,12 +425,6 @@ static void duplicator_capture_tick(void *data, float seconds)
 				capture->reset_timeout = 0.0f;
 			}
 		}
-
-		if (capture->capture_winrt) {
-			capture->exports.winrt_capture_show_cursor(
-				capture->capture_winrt,
-				capture->capture_cursor);
-		}
 	} else {
 		if (capture->capture_winrt) {
 			capture->exports.winrt_capture_free(

plugins/win-capture/window-capture.c

@@ -25,6 +25,8 @@
 /* clang-format on */
 
 #define WC_CHECK_TIMER 1.0f
+#define RESIZE_CHECK_TIME 0.2f
+#define CURSOR_CHECK_TIME 0.2f
 
 typedef BOOL (*PFN_winrt_capture_supported)();
 typedef BOOL (*PFN_winrt_capture_cursor_toggle_supported)();
@@ -33,7 +35,7 @@ typedef struct winrt_capture *(*PFN_winrt_capture_init_window)(
 typedef void (*PFN_winrt_capture_free)(struct winrt_capture *capture);
 
 typedef BOOL (*PFN_winrt_capture_active)(const struct winrt_capture *capture);
-typedef void (*PFN_winrt_capture_show_cursor)(struct winrt_capture *capture,
+typedef BOOL (*PFN_winrt_capture_show_cursor)(struct winrt_capture *capture,
 					      BOOL visible);
 typedef void (*PFN_winrt_capture_render)(struct winrt_capture *capture,
 					 gs_effect_t *effect);
@@ -290,17 +292,23 @@ static void wc_destroy(void *data)
 	obs_queue_task(OBS_TASK_GRAPHICS, wc_actual_destroy, data, false);
 }
 
+static void force_reset(struct window_capture *wc)
+{
+	wc->window = NULL;
+	wc->resize_timer = RESIZE_CHECK_TIME;
+	wc->check_window_timer = WC_CHECK_TIMER;
+	wc->cursor_check_time = CURSOR_CHECK_TIME;
+
+	wc->previously_failed = false;
+}
+
 static void wc_update(void *data, obs_data_t *settings)
 {
 	struct window_capture *wc = data;
 	update_settings(wc, settings);
 	log_settings(wc, settings);
 
-	/* forces a reset */
-	wc->window = NULL;
-	wc->check_window_timer = WC_CHECK_TIMER;
-
-	wc->previously_failed = false;
+	force_reset(wc);
 }
 
 static uint32_t wc_width(void *data)
@@ -432,9 +440,6 @@ static void wc_hide(void *data)
 	memset(&wc->last_rect, 0, sizeof(wc->last_rect));
 }
 
-#define RESIZE_CHECK_TIME 0.2f
-#define CURSOR_CHECK_TIME 0.2f
-
 static void wc_tick(void *data, float seconds)
 {
 	struct window_capture *wc = data;
@@ -489,7 +494,7 @@ static void wc_tick(void *data, float seconds)
 	}
 
 	wc->cursor_check_time += seconds;
-	if (wc->cursor_check_time > CURSOR_CHECK_TIME) {
+	if (wc->cursor_check_time >= CURSOR_CHECK_TIME) {
 		DWORD foreground_pid, target_pid;
 
 		// Can't just compare the window handle in case of app with child windows
@@ -503,9 +508,12 @@ static void wc_tick(void *data, float seconds)
 		const bool cursor_hidden = foreground_pid && target_pid &&
 					   foreground_pid != target_pid;
 		wc->capture.cursor_hidden = cursor_hidden;
-		if (wc->capture_winrt)
-			wc->exports.winrt_capture_show_cursor(wc->capture_winrt,
-							      !cursor_hidden);
+		if (wc->capture_winrt &&
+		    !wc->exports.winrt_capture_show_cursor(wc->capture_winrt,
+							   !cursor_hidden)) {
+			force_reset(wc);
+			return;
+		}
 
 		wc->cursor_check_time = 0.0f;
 	}