libobs/util: Put module load detection in its own func

Puts the detection of obs_module_load in Windows DLLs for the "is this
an obs plugin" check into its own function

libobs/util/platform-windows.c

@@ -136,71 +136,18 @@ void os_dlclose(void *module)
 	FreeLibrary(module);
 }
 
-bool os_is_obs_plugin(const char *path)
+static bool has_obs_export(VOID *base, PIMAGE_NT_HEADERS nt_headers)
 {
-	struct dstr dll_name;
-	wchar_t *wpath;
-
-	HANDLE hFile = INVALID_HANDLE_VALUE;
-	HANDLE hFileMapping = NULL;
-	VOID *base = NULL;
-
-	PIMAGE_DOS_HEADER dos_header;
-	PIMAGE_NT_HEADERS nt_headers;
-	PIMAGE_SECTION_HEADER section, last_section;
-
-	bool ret = false;
-
-	if (!path)
-		return false;
-
-	dstr_init_copy(&dll_name, path);
-	dstr_replace(&dll_name, "\\", "/");
-	if (!dstr_find(&dll_name, ".dll"))
-		dstr_cat(&dll_name, ".dll");
-	os_utf8_to_wcs_ptr(dll_name.array, 0, &wpath);
-
-	dstr_free(&dll_name);
-
-	hFile = CreateFileW(wpath, GENERIC_READ, FILE_SHARE_READ, NULL,
-			    OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, 0);
-
-	bfree(wpath);
-
-	if (hFile == INVALID_HANDLE_VALUE)
-		goto cleanup;
-
-	hFileMapping =
-		CreateFileMapping(hFile, NULL, PAGE_READONLY, 0, 0, NULL);
-	if (hFileMapping == NULL)
-		goto cleanup;
-
-	base = MapViewOfFile(hFileMapping, FILE_MAP_READ, 0, 0, 0);
-	if (!base)
-		goto cleanup;
-
-	/* all mapped file i/o must be prepared to handle exceptions */
 	__try {
-
-		dos_header = (PIMAGE_DOS_HEADER)base;
-
-		if (dos_header->e_magic != IMAGE_DOS_SIGNATURE)
-			goto cleanup;
-
-		nt_headers = (PIMAGE_NT_HEADERS)((byte *)dos_header +
-						 dos_header->e_lfanew);
-
-		if (nt_headers->Signature != IMAGE_NT_SIGNATURE)
-			goto cleanup;
-
 		PIMAGE_DATA_DIRECTORY data_dir;
 		data_dir =
 			&nt_headers->OptionalHeader
 				 .DataDirectory[IMAGE_DIRECTORY_ENTRY_EXPORT];
 
 		if (data_dir->Size == 0)
-			goto cleanup;
+			return false;
 
+		PIMAGE_SECTION_HEADER section, last_section;
 		section = IMAGE_FIRST_SECTION(nt_headers);
 		last_section = section;
 
@@ -220,7 +167,7 @@ bool os_is_obs_plugin(const char *path)
 		/* double check in case we exited early */
 		if (last_section->VirtualAddress > data_dir->VirtualAddress ||
 		    section->VirtualAddress <= data_dir->VirtualAddress)
-			goto cleanup;
+			return false;
 
 		section = last_section;
 
@@ -232,7 +179,7 @@ bool os_is_obs_plugin(const char *path)
 						   section->PointerToRawData);
 
 		if (export->NumberOfNames == 0)
-			goto cleanup;
+			return false;
 
 		/* get a pointer to the export directory names */
 		DWORD *names_ptr;
@@ -250,10 +197,75 @@ bool os_is_obs_plugin(const char *path)
 			       section->PointerToRawData;
 
 			if (!strcmp(name, "obs_module_load")) {
-				ret = true;
-				goto cleanup;
+				return true;
 			}
 		}
+	} __except (EXCEPTION_EXECUTE_HANDLER) {
+		/* we failed somehow, for compatibility let's assume it
+		 * was a valid plugin and let the loader deal with it */
+		return true;
+	}
+
+	return false;
+}
+
+bool os_is_obs_plugin(const char *path)
+{
+	struct dstr dll_name;
+	wchar_t *wpath;
+
+	HANDLE hFile = INVALID_HANDLE_VALUE;
+	HANDLE hFileMapping = NULL;
+	VOID *base = NULL;
+
+	PIMAGE_DOS_HEADER dos_header;
+	PIMAGE_NT_HEADERS nt_headers;
+
+	bool ret = false;
+
+	if (!path)
+		return false;
+
+	dstr_init_copy(&dll_name, path);
+	dstr_replace(&dll_name, "\\", "/");
+	if (!dstr_find(&dll_name, ".dll"))
+		dstr_cat(&dll_name, ".dll");
+	os_utf8_to_wcs_ptr(dll_name.array, 0, &wpath);
+
+	dstr_free(&dll_name);
+
+	hFile = CreateFileW(wpath, GENERIC_READ, FILE_SHARE_READ, NULL,
+			    OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, 0);
+
+	bfree(wpath);
+
+	if (hFile == INVALID_HANDLE_VALUE)
+		goto cleanup;
+
+	hFileMapping =
+		CreateFileMapping(hFile, NULL, PAGE_READONLY, 0, 0, NULL);
+	if (hFileMapping == NULL)
+		goto cleanup;
+
+	base = MapViewOfFile(hFileMapping, FILE_MAP_READ, 0, 0, 0);
+	if (!base)
+		goto cleanup;
+
+	/* all mapped file i/o must be prepared to handle exceptions */
+	__try {
+
+		dos_header = (PIMAGE_DOS_HEADER)base;
+
+		if (dos_header->e_magic != IMAGE_DOS_SIGNATURE)
+			goto cleanup;
+
+		nt_headers = (PIMAGE_NT_HEADERS)((byte *)dos_header +
+						 dos_header->e_lfanew);
+
+		if (nt_headers->Signature != IMAGE_NT_SIGNATURE)
+			goto cleanup;
+
+		ret = has_obs_export(base, nt_headers);
 
 	} __except (EXCEPTION_EXECUTE_HANDLER) {
 		/* we failed somehow, for compatibility let's assume it