win-capture: Don't use FindWindow for game capture keepalive

Using and creating a window can use issues in game capture if multiple
game captures are active, so revert back to using a mutex, and just
ignore the keepalive check failure if injected inside a UWP program
(only check to see if GetLastError reports that it's not found -- if it
returns access denied or any other error, assume it's in a UWP program,
and ignore the keepalive check).

plugins/win-capture/game-capture.c

@@ -136,8 +136,7 @@ struct game_capture {
 	ipc_pipe_server_t             pipe;
 	gs_texture_t                  *texture;
 	struct hook_info              *global_hook_info;
-	HANDLE                        keepalive_thread;
-	DWORD                         keepalive_thread_id;
+	HANDLE                        keepalive_mutex;
 	HANDLE                        hook_init;
 	HANDLE                        hook_restart;
 	HANDLE                        hook_stop;
@@ -281,12 +280,6 @@ static void stop_capture(struct game_capture *gc)
 		gc->data = NULL;
 	}
 
-	if (gc->keepalive_thread) {
-		PostThreadMessage(gc->keepalive_thread_id, WM_QUIT, 0, 0);
-		WaitForSingleObject(gc->keepalive_thread, 300);
-		close_handle(&gc->keepalive_thread);
-	}
-
 	if (gc->app_sid) {
 		LocalFree(gc->app_sid);
 		gc->app_sid = NULL;
@@ -298,6 +291,7 @@ static void stop_capture(struct game_capture *gc)
 	close_handle(&gc->hook_exit);
 	close_handle(&gc->hook_init);
 	close_handle(&gc->hook_data_map);
+	close_handle(&gc->keepalive_mutex);
 	close_handle(&gc->global_hook_info_map);
 	close_handle(&gc->target_process);
 	close_handle(&gc->texture_mutexes[0]);
@@ -641,78 +635,18 @@ static inline bool open_target_process(struct game_capture *gc)
 	return true;
 }
 
-struct keepalive_data {
-	struct game_capture *gc;
-	HANDLE initialized;
-};
-
-#define DEF_FLAGS (WS_POPUP | WS_CLIPCHILDREN | WS_CLIPSIBLINGS)
-
-static DWORD WINAPI keepalive_window_thread(struct keepalive_data *data)
-{
-	HANDLE initialized = data->initialized;
-	struct game_capture *gc = data->gc;
-	wchar_t new_name[64];
-	WNDCLASSW wc;
-	HWND window;
-	MSG msg;
-
-	_snwprintf(new_name, sizeof(new_name), L"%s%lu",
-			WINDOW_HOOK_KEEPALIVE, gc->process_id);
-
-	memset(&wc, 0, sizeof(wc));
-	wc.style = CS_OWNDC;
-	wc.hInstance = GetModuleHandleW(NULL);
-	wc.lpfnWndProc = (WNDPROC)DefWindowProc;
-	wc.lpszClassName = new_name;
-
-	if (!RegisterClass(&wc)) {
-		warn("Failed to create keepalive window class: %lu",
-				GetLastError());
-		return 0;
-	}
-
-	window = CreateWindowExW(0, new_name, NULL, DEF_FLAGS, 0, 0, 1, 1,
-			NULL, NULL, wc.hInstance, NULL);
-	if (!window) {
-		warn("Failed to create keepalive window: %lu",
-				GetLastError());
-		return 0;
-	}
-
-	SetEvent(initialized);
-
-	while (GetMessage(&msg, NULL, 0, 0)) {
-		TranslateMessage(&msg);
-		DispatchMessage(&msg);
-	}
-
-	DestroyWindow(window);
-	UnregisterClassW(new_name, wc.hInstance);
-
-	return 0;
-}
-
 static inline bool init_keepalive(struct game_capture *gc)
 {
-	struct keepalive_data data;
-	HANDLE initialized = CreateEvent(NULL, false, false, NULL);
-
-	data.gc = gc;
-	data.initialized = initialized;
+	wchar_t new_name[64];
+	_snwprintf(new_name, 64, L"%s%lu", WINDOW_HOOK_KEEPALIVE,
+			gc->process_id);
 
-	gc->keepalive_thread = CreateThread(NULL, 0,
-			(LPTHREAD_START_ROUTINE)keepalive_window_thread,
-			&data, 0, &gc->keepalive_thread_id);
-	if (!gc->keepalive_thread) {
-		warn("Failed to create keepalive window thread: %lu",
-				GetLastError());
+	gc->keepalive_mutex = CreateMutexW(NULL, false, new_name);
+	if (!gc->keepalive_mutex) {
+		warn("Failed to create keepalive mutex: %lu", GetLastError());
 		return false;
 	}
 
-	WaitForSingleObject(initialized, INFINITE);
-	CloseHandle(initialized);
-
 	return true;
 }
 

plugins/win-capture/graphics-hook/graphics-hook.h

@@ -143,7 +143,13 @@ static inline HMODULE load_system_library(const char *name)
 
 static inline bool capture_alive(void)
 {
-	return !!FindWindowW(keepalive_name, NULL);
+	HANDLE handle = OpenMutexW(SYNCHRONIZE, false, keepalive_name);
+	CloseHandle(handle);
+
+	if (handle)
+		return true;
+
+	return GetLastError() != ERROR_FILE_NOT_FOUND;
 }
 
 static inline bool capture_active(void)