mac-capture: Filter non-trivial windows

Filter the window list provided by the OS to prevent listing icons and
system windows as possible window capture targets.

Fixes #3146.

plugins/mac-capture/window-utils.h

@@ -18,6 +18,8 @@ typedef struct cocoa_window *cocoa_window_t;
 
 NSArray *enumerate_cocoa_windows(void);
 
+NSArray *filter_nonzero_kcgwindowlayer_sources(NSArray *windows_arr);
+
 bool find_window(cocoa_window_t cw, obs_data_t *settings, bool force);
 
 void init_window(cocoa_window_t cw, obs_data_t *settings);

plugins/mac-capture/window-utils.m

@@ -23,14 +23,25 @@ static NSComparator win_info_cmp = ^(NSDictionary *o1, NSDictionary *o2) {
 	return [o1[WINDOW_NUMBER] compare:o2[WINDOW_NUMBER]];
 };
 
+NSArray *filter_nonzero_kcgwindowlayer_sources(NSArray *windows_arr)
+{
+	NSPredicate *pred =
+		[NSPredicate predicateWithFormat:@"(kCGWindowLayer == 0)"];
+	NSArray *new_windows_arr =
+		[windows_arr filteredArrayUsingPredicate:pred];
+
+	return new_windows_arr;
+}
+
 NSArray *enumerate_windows(void)
 {
 	NSArray *arr = (NSArray *)CGWindowListCopyWindowInfo(
 		kCGWindowListOptionOnScreenOnly, kCGNullWindowID);
+	NSArray *filtered_arr = filter_nonzero_kcgwindowlayer_sources(arr);
 
 	[arr autorelease];
 
-	return [arr sortedArrayUsingComparator:win_info_cmp];
+	return [filtered_arr sortedArrayUsingComparator:win_info_cmp];
 }
 
 #define WAIT_TIME_MS 500