libobs: Add os_is_obs_plugin function

This function determines if something is an OBS plugin before attempting
to load it. On Windows, many plugins ship their dependent DLLs alongside
the plugin DLL, so OBS would load things like libcef.dll on startup only
to immediately free it. For other platforms, this is less of a concern
so this function is a no-op for now.

This improves startup time and reduces risk from dependent DLLs
potentially running code with unwanted side effects in DllMain.

docs/sphinx/reference-libobs-util-platform.rst

@@ -154,6 +154,14 @@ These functions are roughly equivalent to dlopen/dlsym/dlclose.
 
 ---------------------
 
+.. function:: bool os_is_obs_plugin(const char *path)
+
+   Returns true if the path is a dynamic library that looks like an OBS plugin.
+
+   Currently only needed on Windows for performance reasons.
+
+---------------------
+
 
 CPU Usage Functions
 -------------------

libobs/obs-module.c

@@ -273,6 +273,10 @@ static void load_all_callback(void *param, const struct obs_module_info *info)
 {
 	obs_module_t *module;
 
+	if (!os_is_obs_plugin(info->bin_path))
+		blog(LOG_WARNING, "Skipping module '%s', not an OBS plugin",
+		     info->bin_path);
+
 	int code = obs_open_module(&module, info->bin_path, info->data_path);
 	if (code != MODULE_SUCCESS) {
 		blog(LOG_DEBUG, "Failed to load module file '%s': %d",

libobs/util/platform-nix.c

@@ -94,6 +94,12 @@ void os_dlclose(void *module)
 		dlclose(module);
 }
 
+bool os_is_obs_plugin(const char *path)
+{
+	/* not necessary on this platform */
+	return true;
+}
+
 #if !defined(__APPLE__)
 
 struct os_cpu_usage_info {

libobs/util/platform-windows.c

@@ -134,6 +134,143 @@ void os_dlclose(void *module)
 	FreeLibrary(module);
 }
 
+bool os_is_obs_plugin(const char *path)
+{
+	struct dstr dll_name;
+	wchar_t *wpath;
+
+	HANDLE hFile = INVALID_HANDLE_VALUE;
+	HANDLE hFileMapping = NULL;
+	VOID *base = NULL;
+
+	PIMAGE_DOS_HEADER dos_header;
+	PIMAGE_NT_HEADERS nt_headers;
+	PIMAGE_SECTION_HEADER section, last_section;
+
+	bool ret = false;
+
+	if (!path)
+		return false;
+
+	dstr_init_copy(&dll_name, path);
+	dstr_replace(&dll_name, "\\", "/");
+	if (!dstr_find(&dll_name, ".dll"))
+		dstr_cat(&dll_name, ".dll");
+
+	os_utf8_to_wcs_ptr(dll_name.array, 0, &wpath);
+
+	hFile = CreateFileW(wpath, GENERIC_READ, FILE_SHARE_READ, NULL,
+			    OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, 0);
+
+	bfree(wpath);
+
+	if (hFile == INVALID_HANDLE_VALUE)
+		goto cleanup;
+
+	hFileMapping =
+		CreateFileMapping(hFile, NULL, PAGE_READONLY, 0, 0, NULL);
+	if (hFileMapping == NULL)
+		goto cleanup;
+
+	base = MapViewOfFile(hFileMapping, FILE_MAP_READ, 0, 0, 0);
+	if (!base)
+		goto cleanup;
+
+	/* all mapped file i/o must be prepared to handle exceptions */
+	__try {
+
+		dos_header = (PIMAGE_DOS_HEADER)base;
+
+		if (dos_header->e_magic != IMAGE_DOS_SIGNATURE)
+			goto cleanup;
+
+		nt_headers = (PIMAGE_NT_HEADERS)((byte *)dos_header +
+						 dos_header->e_lfanew);
+
+		if (nt_headers->Signature != IMAGE_NT_SIGNATURE)
+			goto cleanup;
+
+		PIMAGE_DATA_DIRECTORY data_dir;
+		data_dir =
+			&nt_headers->OptionalHeader
+				 .DataDirectory[IMAGE_DIRECTORY_ENTRY_EXPORT];
+
+		if (data_dir->Size == 0)
+			goto cleanup;
+
+		section = IMAGE_FIRST_SECTION(nt_headers);
+		last_section = section;
+
+		/* find the section that contains the export directory */
+		int i;
+		for (i = 0; i < nt_headers->FileHeader.NumberOfSections; i++) {
+			if (section->VirtualAddress <=
+			    data_dir->VirtualAddress) {
+				last_section = section;
+				section++;
+				continue;
+			} else {
+				break;
+			}
+		}
+
+		/* double check in case we exited early */
+		if (last_section->VirtualAddress > data_dir->VirtualAddress ||
+		    section->VirtualAddress <= data_dir->VirtualAddress)
+			goto cleanup;
+
+		section = last_section;
+
+		/* get a pointer to the export directory */
+		PIMAGE_EXPORT_DIRECTORY export;
+		export = (PIMAGE_EXPORT_DIRECTORY)(
+			(byte *)base + data_dir->VirtualAddress -
+			section->VirtualAddress + section->PointerToRawData);
+
+		if (export->NumberOfNames == 0)
+			goto cleanup;
+
+		/* get a pointer to the export directory names */
+		DWORD *names_ptr;
+		names_ptr = (DWORD *)((byte *)base + export->AddressOfNames -
+				      section->VirtualAddress +
+				      section->PointerToRawData);
+
+		/* iterate through each name and see if its an obs plugin */
+		CHAR *name;
+		size_t j;
+		for (j = 0; j < export->NumberOfNames; j++) {
+
+			name = (CHAR *)base + names_ptr[j] -
+			       section->VirtualAddress +
+			       section->PointerToRawData;
+
+			if (!strcmp(name, "obs_module_load")) {
+				ret = true;
+				goto cleanup;
+			}
+		}
+
+	} __except (EXCEPTION_EXECUTE_HANDLER) {
+		/* we failed somehow, for compatibility let's assume it
+		 * was a valid plugin and let the loader deal with it */
+		ret = true;
+		goto cleanup;
+	}
+
+cleanup:
+	if (base)
+		UnmapViewOfFile(base);
+
+	if (hFileMapping != NULL)
+		CloseHandle(hFileMapping);
+
+	if (hFile != INVALID_HANDLE_VALUE)
+		CloseHandle(hFile);
+
+	return ret;
+}
+
 union time_data {
 	FILETIME ft;
 	unsigned long long val;

libobs/util/platform.h

@@ -84,6 +84,7 @@ EXPORT int os_dtostr(double value, char *dst, size_t size);
 EXPORT void *os_dlopen(const char *path);
 EXPORT void *os_dlsym(void *module, const char *func);
 EXPORT void os_dlclose(void *module);
+EXPORT bool os_is_obs_plugin(const char *path);
 
 struct os_cpu_usage_info;
 typedef struct os_cpu_usage_info os_cpu_usage_info_t;