win-capture: Blacklist specific executables from game capture

Prevents the common problem of injecting in to certain processes and
getting the hook DLL "stuck":
- windows explorer
- steam
- battle.net
- gog galaxy
- skype
- uplay
- origin
- microsoft visual studio
- task manager
- league of legends lobby window
- windows 10 system settings window

plugins/win-capture/game-capture.c

@@ -822,19 +822,63 @@ cleanup:
 	return success;
 }
 
+static const char *blacklisted_exes[] = {
+	"explorer",
+	"steam",
+	"battle.net",
+	"galaxyclient",
+	"skype",
+	"uplay",
+	"origin",
+	"devenv",
+	"taskmgr",
+	"systemsettings",
+	"lolclient",
+	"cmd",
+	NULL
+};
+
+static bool is_blacklisted_exe(const char *exe)
+{
+	char cur_exe[MAX_PATH];
+
+	if (!exe)
+		return false;
+
+	for (const char **vals = blacklisted_exes; *vals; vals++) {
+		strcpy(cur_exe, *vals);
+		strcat(cur_exe, ".exe");
+
+		if (strcmpi(cur_exe, exe) == 0)
+			return true;
+	}
+
+	return false;
+}
+
 static bool init_hook(struct game_capture *gc)
 {
+	struct dstr exe = {0};
+	bool blacklisted_process = false;
+
 	if (gc->config.mode == CAPTURE_MODE_ANY) {
-		struct dstr name = {0};
-		if (get_window_exe(&name, gc->next_window)) {
+		if (get_window_exe(&exe, gc->next_window)) {
 			info("attempting to hook fullscreen process: %s",
-					name.array);
-			dstr_free(&name);
+					exe.array);
 		}
 	} else {
 		info("attempting to hook process: %s", gc->executable.array);
+		dstr_copy_dstr(&exe, &gc->executable);
 	}
 
+	blacklisted_process = is_blacklisted_exe(exe.array);
+	if (blacklisted_process)
+		info("cannot capture %s due to being blacklisted", exe.array);
+	dstr_free(&exe);
+
+	if (blacklisted_process) {
+		return false;
+	}
 	if (!open_target_process(gc)) {
 		return false;
 	}
@@ -1688,6 +1732,15 @@ static BOOL CALLBACK EnumFirstMonitor(HMONITOR monitor, HDC hdc,
 	return false;
 }
 
+static bool window_not_blacklisted(const char *title, const char *class,
+		const char *exe)
+{
+	UNUSED_PARAMETER(title);
+	UNUSED_PARAMETER(class);
+
+	return !is_blacklisted_exe(exe);
+}
+
 static obs_properties_t *game_capture_properties(void *data)
 {
 	HMONITOR monitor;
@@ -1738,7 +1791,7 @@ static obs_properties_t *game_capture_properties(void *data)
 	p = obs_properties_add_list(ppts, SETTING_CAPTURE_WINDOW, TEXT_WINDOW,
 			OBS_COMBO_TYPE_LIST, OBS_COMBO_FORMAT_STRING);
 	obs_property_list_add_string(p, "", "");
-	fill_window_list(p, INCLUDE_MINIMIZED);
+	fill_window_list(p, INCLUDE_MINIMIZED, window_not_blacklisted);
 
 	obs_property_set_modified_callback(p, window_changed_callback);
 

plugins/win-capture/window-capture.c

@@ -118,7 +118,7 @@ static obs_properties_t *wc_properties(void *unused)
 
 	p = obs_properties_add_list(ppts, "window", TEXT_WINDOW,
 			OBS_COMBO_TYPE_LIST, OBS_COMBO_FORMAT_STRING);
-	fill_window_list(p, EXCLUDE_MINIMIZED);
+	fill_window_list(p, EXCLUDE_MINIMIZED, NULL);
 
 	p = obs_properties_add_list(ppts, "priority", TEXT_MATCH_PRIORITY,
 			OBS_COMBO_TYPE_LIST, OBS_COMBO_FORMAT_INT);

plugins/win-capture/window-helpers.c

@@ -128,7 +128,7 @@ void get_window_class(struct dstr *class, HWND hwnd)
 		dstr_from_wcs(class, temp);
 }
 
-static void add_window(obs_property_t *p, HWND hwnd)
+static void add_window(obs_property_t *p, HWND hwnd, add_window_cb callback)
 {
 	struct dstr class   = {0};
 	struct dstr title   = {0};
@@ -141,6 +141,13 @@ static void add_window(obs_property_t *p, HWND hwnd)
 	get_window_title(&title, hwnd);
 	get_window_class(&class, hwnd);
 
+	if (callback && !callback(title.array, class.array, exe.array)) {
+		dstr_free(&title);
+		dstr_free(&class);
+		dstr_free(&exe);
+		return;
+	}
+
 	dstr_printf(&desc, "[%s]: %s", exe.array, title.array);
 
 	encode_dstr(&title);
@@ -204,12 +211,13 @@ static inline HWND first_window(enum window_search_mode mode)
 	return window;
 }
 
-void fill_window_list(obs_property_t *p, enum window_search_mode mode)
+void fill_window_list(obs_property_t *p, enum window_search_mode mode,
+		add_window_cb callback)
 {
 	HWND window = first_window(mode);
 
 	while (window) {
-		add_window(p, window);
+		add_window(p, window, callback);
 		window = next_window(window, mode);
 	}
 }

plugins/win-capture/window-helpers.h

@@ -17,7 +17,11 @@ extern bool get_window_exe(struct dstr *name, HWND window);
 extern void get_window_title(struct dstr *name, HWND hwnd);
 extern void get_window_class(struct dstr *class, HWND hwnd);
 
-extern void fill_window_list(obs_property_t *p, enum window_search_mode mode);
+typedef bool (*add_window_cb)(const char *title, const char *class,
+		const char *exe);
+
+extern void fill_window_list(obs_property_t *p, enum window_search_mode mode,
+		add_window_cb callback);
 
 extern void build_window_strings(const char *str,
 		char **class,