win-capture: Create all named objects within hook

All named objects (including file mapped shared memory) need to be
created within the hook itself due to the fact that UWP programs cannot
access named objects outside of the UWP process.

Because shared memory needs to be created within the hook, the capture
loop cannot start until the shared memory has been filled with valid
data.  Creating an additional "initialize" event fixes this issue.

Additionally, changed the way that named kernel objects are
opened/created.  Before, there were functions that would first try to
open named objects and then implicitly create them if opening failed
(assuming that if the hook didn't create it first, game capture would),
now it's been changed so that you can only either explicitly open or
create.

plugins/win-capture/game-capture.c

@@ -135,6 +135,7 @@ struct game_capture {
 	struct hook_info              *global_hook_info;
 	HANDLE                        keepalive_thread;
 	DWORD                         keepalive_thread_id;
+	HANDLE                        hook_init;
 	HANDLE                        hook_restart;
 	HANDLE                        hook_stop;
 	HANDLE                        hook_ready;
@@ -230,6 +231,7 @@ static void stop_capture(struct game_capture *gc)
 	close_handle(&gc->hook_stop);
 	close_handle(&gc->hook_ready);
 	close_handle(&gc->hook_exit);
+	close_handle(&gc->hook_init);
 	close_handle(&gc->hook_data_map);
 	close_handle(&gc->global_hook_info_map);
 	close_handle(&gc->target_process);
@@ -648,13 +650,13 @@ static inline bool init_keepalive(struct game_capture *gc)
 
 static inline bool init_texture_mutexes(struct game_capture *gc)
 {
-	gc->texture_mutexes[0] = get_mutex_plus_id(MUTEX_TEXTURE1,
+	gc->texture_mutexes[0] = open_mutex_plus_id(MUTEX_TEXTURE1,
 			gc->process_id);
-	gc->texture_mutexes[1] = get_mutex_plus_id(MUTEX_TEXTURE2,
+	gc->texture_mutexes[1] = open_mutex_plus_id(MUTEX_TEXTURE2,
 			gc->process_id);
 
 	if (!gc->texture_mutexes[0] || !gc->texture_mutexes[1]) {
-		warn("failed to create texture mutexes: %lu", GetLastError());
+		warn("failed to open texture mutexes: %lu", GetLastError());
 		return false;
 	}
 
@@ -696,7 +698,7 @@ static inline void reset_frame_interval(struct game_capture *gc)
 
 static inline bool init_hook_info(struct game_capture *gc)
 {
-	gc->global_hook_info_map = get_hook_info(gc->process_id);
+	gc->global_hook_info_map = open_hook_info(gc->process_id);
 	if (!gc->global_hook_info_map) {
 		warn("init_hook_info: get_hook_info failed: %lu",
 				GetLastError());
@@ -921,6 +923,8 @@ static bool is_blacklisted_exe(const char *exe)
 	return false;
 }
 
+static bool init_events(struct game_capture *gc);
+
 static bool init_hook(struct game_capture *gc)
 {
 	struct dstr exe = {0};
@@ -950,12 +954,6 @@ static bool init_hook(struct game_capture *gc)
 	if (!init_keepalive(gc)) {
 		return false;
 	}
-	if (!init_texture_mutexes(gc)) {
-		return false;
-	}
-	if (!init_hook_info(gc)) {
-		return false;
-	}
 	if (!init_pipe(gc)) {
 		return false;
 	}
@@ -964,6 +962,17 @@ static bool init_hook(struct game_capture *gc)
 			return false;
 		}
 	}
+	if (!init_texture_mutexes(gc)) {
+		return false;
+	}
+	if (!init_hook_info(gc)) {
+		return false;
+	}
+	if (!init_events(gc)) {
+		return false;
+	}
+
+	SetEvent(gc->hook_init);
 
 	gc->window = gc->next_window;
 	gc->next_window = NULL;
@@ -1099,7 +1108,7 @@ static void try_hook(struct game_capture *gc)
 static inline bool init_events(struct game_capture *gc)
 {
 	if (!gc->hook_restart) {
-		gc->hook_restart = get_event_plus_id(EVENT_CAPTURE_RESTART,
+		gc->hook_restart = open_event_plus_id(EVENT_CAPTURE_RESTART,
 				gc->process_id);
 		if (!gc->hook_restart) {
 			warn("init_events: failed to get hook_restart "
@@ -1109,7 +1118,7 @@ static inline bool init_events(struct game_capture *gc)
 	}
 
 	if (!gc->hook_stop) {
-		gc->hook_stop = get_event_plus_id(EVENT_CAPTURE_STOP,
+		gc->hook_stop = open_event_plus_id(EVENT_CAPTURE_STOP,
 				gc->process_id);
 		if (!gc->hook_stop) {
 			warn("init_events: failed to get hook_stop event: %lu",
@@ -1118,8 +1127,18 @@ static inline bool init_events(struct game_capture *gc)
 		}
 	}
 
+	if (!gc->hook_init) {
+		gc->hook_init = open_event_plus_id(EVENT_HOOK_INIT,
+				gc->process_id);
+		if (!gc->hook_init) {
+			warn("init_events: failed to get hook_init event: %lu",
+					GetLastError());
+			return false;
+		}
+	}
+
 	if (!gc->hook_ready) {
-		gc->hook_ready = get_event_plus_id(EVENT_HOOK_READY,
+		gc->hook_ready = open_event_plus_id(EVENT_HOOK_READY,
 				gc->process_id);
 		if (!gc->hook_ready) {
 			warn("init_events: failed to get hook_ready event: %lu",
@@ -1129,7 +1148,7 @@ static inline bool init_events(struct game_capture *gc)
 	}
 
 	if (!gc->hook_exit) {
-		gc->hook_exit = get_event_plus_id(EVENT_HOOK_EXIT,
+		gc->hook_exit = open_event_plus_id(EVENT_HOOK_EXIT,
 				gc->process_id);
 		if (!gc->hook_exit) {
 			warn("init_events: failed to get hook_exit event: %lu",
@@ -1471,9 +1490,6 @@ static inline bool init_shtex_capture(struct game_capture *gc)
 
 static bool start_capture(struct game_capture *gc)
 {
-	if (!init_events(gc)) {
-		return false;
-	}
 	if (gc->global_hook_info->type == CAPTURE_TYPE_MEMORY) {
 		if (!init_shmem_capture(gc)) {
 			return false;
@@ -1542,7 +1558,7 @@ static void game_capture_tick(void *data, float seconds)
 	}
 
 	if (gc->active && !gc->hook_ready && gc->process_id) {
-		gc->hook_ready = get_event_plus_id(EVENT_HOOK_READY,
+		gc->hook_ready = open_event_plus_id(EVENT_HOOK_READY,
 				gc->process_id);
 	}
 

plugins/win-capture/graphics-hook-info.h

@@ -13,6 +13,8 @@
 #define EVENT_HOOK_READY      "CaptureHook_HookReady"
 #define EVENT_HOOK_EXIT       "CaptureHook_Exit"
 
+#define EVENT_HOOK_INIT       "CaptureHook_Initialize"
+
 #define WINDOW_HOOK_KEEPALIVE L"CaptureHook_KeepAlive"
 
 #define MUTEX_TEXTURE1        "CaptureHook_TextureMutex1"
@@ -103,19 +105,19 @@ struct hook_info {
 
 #define GC_MAPPING_FLAGS (FILE_MAP_READ | FILE_MAP_WRITE)
 
-static inline HANDLE get_hook_info(DWORD id)
+static inline HANDLE create_hook_info(DWORD id)
 {
-	HANDLE handle;
 	char new_name[64];
 	sprintf(new_name, "%s%lu", SHMEM_HOOK_INFO, id);
 
-	handle = CreateFileMappingA(INVALID_HANDLE_VALUE, NULL,
-			PAGE_READWRITE, 0, sizeof(struct hook_info), new_name);
+	return CreateFileMappingA(INVALID_HANDLE_VALUE, NULL, PAGE_READWRITE,
+			0, sizeof(struct hook_info), new_name);
+}
 
-	if (!handle && GetLastError() == ERROR_ALREADY_EXISTS) {
-		handle = OpenFileMappingA(GC_MAPPING_FLAGS, false,
-				new_name);
-	}
+static inline HANDLE open_hook_info(DWORD id)
+{
+	char new_name[64];
+	sprintf(new_name, "%s%lu", SHMEM_HOOK_INFO, id);
 
-	return handle;
+	return OpenFileMappingA(GC_MAPPING_FLAGS, false, new_name);
 }

plugins/win-capture/graphics-hook/graphics-hook.c

@@ -31,6 +31,7 @@ HANDLE                         signal_restart                  = NULL;
 HANDLE                         signal_stop                     = NULL;
 HANDLE                         signal_ready                    = NULL;
 HANDLE                         signal_exit                     = NULL;
+static HANDLE                  signal_init                     = NULL;
 HANDLE                         tex_mutexes[2]                  = {NULL, NULL};
 static HANDLE                  filemap_hook_info               = NULL;
 
@@ -75,7 +76,7 @@ bool init_pipe(void)
 
 static HANDLE init_event(const char *name, DWORD pid)
 {
-	HANDLE handle = get_event_plus_id(name, pid);
+	HANDLE handle = create_event_plus_id(name, pid);
 	if (!handle)
 		hlog("Failed to get event '%s': %lu", name, GetLastError());
 	return handle;
@@ -83,12 +84,7 @@ static HANDLE init_event(const char *name, DWORD pid)
 
 static HANDLE init_mutex(const char *name, DWORD pid)
 {
-	char new_name[64];
-	HANDLE handle;
-
-	sprintf(new_name, "%s%lu", name, pid);
-
-	handle = OpenMutexA(SYNCHRONIZE, false, new_name);
+	HANDLE handle = create_mutex_plus_id(name, pid);
 	if (!handle)
 		hlog("Failed to open mutex '%s': %lu", name, GetLastError());
 	return handle;
@@ -118,6 +114,11 @@ static inline bool init_signals(void)
 		return false;
 	}
 
+	signal_init = init_event(EVENT_HOOK_INIT, pid);
+	if (!signal_init) {
+		return false;
+	}
+
 	return true;
 }
 
@@ -163,7 +164,7 @@ static inline void log_current_process(void)
 
 static inline bool init_hook_info(void)
 {
-	filemap_hook_info = get_hook_info(GetCurrentProcessId());
+	filemap_hook_info = create_hook_info(GetCurrentProcessId());
 	if (!filemap_hook_info) {
 		hlog("Failed to create hook info file mapping: %lu",
 				GetLastError());
@@ -238,16 +239,6 @@ static inline bool init_hook(HANDLE thread_handle)
 			WINDOW_HOOK_KEEPALIVE, GetCurrentProcessId());
 
 	init_pipe();
-	if (!init_signals()) {
-		return false;
-	}
-	init_mutexes();
-	if (!init_system_path()) {
-		return false;
-	}
-	if (!init_hook_info()) {
-		return false;
-	}
 
 	init_dummy_window_thread();
 	log_current_process();
@@ -380,6 +371,8 @@ static inline bool attempt_hook(void)
 
 static inline void capture_loop(void)
 {
+	WaitForSingleObject(signal_init, INFINITE);
+
 	while (!attempt_hook())
 		Sleep(40);
 
@@ -796,6 +789,19 @@ BOOL WINAPI DllMain(HINSTANCE hinst, DWORD reason, LPVOID unused1)
 		if (!success)
 			DbgOut("Failed to get current thread handle");
 
+		if (!init_signals()) {
+			return false;
+		}
+		if (!init_system_path()) {
+			return false;
+		}
+		if (!init_hook_info()) {
+			return false;
+		}
+		if (!init_mutexes()) {
+			return false;
+		}
+
 		/* this prevents the library from being automatically unloaded
 		 * by the next FreeLibrary call */
 		GetModuleFileNameW(hinst, name, MAX_PATH);

plugins/win-capture/hook-helpers.h

@@ -7,36 +7,52 @@
 #define GC_EVENT_FLAGS (EVENT_MODIFY_STATE | SYNCHRONIZE)
 #define GC_MUTEX_FLAGS (SYNCHRONIZE)
 
-static inline HANDLE get_event(const char *name)
+static inline HANDLE create_event(const char *name)
 {
-	HANDLE event = CreateEventA(NULL, false, false, name);
-	if (!event)
-		event = OpenEventA(GC_EVENT_FLAGS, false, name);
+	return CreateEventA(NULL, false, false, name);
+}
 
-	return event;
+static inline HANDLE open_event(const char *name)
+{
+	return OpenEventA(GC_EVENT_FLAGS, false, name);
 }
 
-static inline HANDLE get_mutex(const char *name)
+static inline HANDLE create_mutex(const char *name)
 {
-	HANDLE event = CreateMutexA(NULL, false, name);
-	if (!event)
-		event = OpenMutexA(GC_MUTEX_FLAGS, false, name);
+	return CreateMutexA(NULL, false, name);
+}
+
+static inline HANDLE open_mutex(const char *name)
+{
+	return OpenMutexA(GC_MUTEX_FLAGS, false, name);
+}
 
-	return event;
+static inline HANDLE create_event_plus_id(const char *name, DWORD id)
+{
+	char new_name[64];
+	sprintf(new_name, "%s%lu", name, id);
+	return create_event(new_name);
+}
+
+static inline HANDLE open_event_plus_id(const char *name, DWORD id)
+{
+	char new_name[64];
+	sprintf(new_name, "%s%lu", name, id);
+	return open_event(new_name);
 }
 
-static inline HANDLE get_event_plus_id(const char *name, DWORD id)
+static inline HANDLE create_mutex_plus_id(const char *name, DWORD id)
 {
 	char new_name[64];
 	sprintf(new_name, "%s%lu", name, id);
-	return get_event(new_name);
+	return create_mutex(new_name);
 }
 
-static inline HANDLE get_mutex_plus_id(const char *name, DWORD id)
+static inline HANDLE open_mutex_plus_id(const char *name, DWORD id)
 {
 	char new_name[64];
 	sprintf(new_name, "%s%lu", name, id);
-	return get_mutex(new_name);
+	return open_mutex(new_name);
 }
 
 static inline bool object_signalled(HANDLE event)