UI: Add option to hide OBS windows on Windows

This uses the SetWindowDisplayAffinity API to hide windows from capture
applications (including OBS). This is not perfect - internal windows
such as context menus, combo box dropdowns, etc will still be displayed.
Even with these limitations, it should help people with single monitors
capture content with less interference from the OBS window.

This implementation is for Windows only but the code is generic enough
that adding other platforms should be straightforward.

UI/data/locale/en-US.ini

@@ -778,6 +778,7 @@ Basic.Settings.General.Theme="Theme"
 Basic.Settings.General.Language="Language"
 Basic.Settings.General.EnableAutoUpdates="Automatically check for updates on startup"
 Basic.Settings.General.OpenStatsOnStartup="Open stats dialog on startup"
+Basic.Settings.General.HideOBSWindowsFromCapture="Hide OBS windows from display capture"
 Basic.Settings.General.WarnBeforeStartingStream="Show confirmation dialog when starting streams"
 Basic.Settings.General.WarnBeforeStoppingStream="Show confirmation dialog when stopping streams"
 Basic.Settings.General.WarnBeforeStoppingRecord="Show confirmation dialog when stopping recording"

UI/forms/OBSBasicSettings.ui

@@ -245,6 +245,13 @@
                      </property>
                     </widget>
                    </item>
+                   <item row="4" column="1">
+                    <widget class="QCheckBox" name="hideOBSFromCapture">
+                     <property name="text">
+                      <string>Basic.Settings.General.HideOBSWindowsFromCapture</string>
+                     </property>
+                    </widget>
+                   </item>
                   </layout>
                  </widget>
                 </item>

UI/obs-app.cpp

@@ -1571,6 +1571,44 @@ bool OBSApp::TranslateString(const char *lookupVal, const char **out) const
 	return text_lookup_getstr(App()->GetTextLookup(), lookupVal, out);
 }
 
+// Global handler to receive all QEvent::Show events so we can apply
+// display affinity on any newly created windows and dialogs without
+// caring where they are coming from (e.g. plugins).
+bool OBSApp::notify(QObject *receiver, QEvent *e)
+{
+	QWidget *w;
+	QWindow *window;
+	int windowType;
+
+	if (!receiver->isWidgetType())
+		goto skip;
+
+	if (e->type() != QEvent::Show)
+		goto skip;
+
+	w = qobject_cast<QWidget *>(receiver);
+
+	if (!w->isWindow())
+		goto skip;
+
+	window = w->windowHandle();
+	if (!window)
+		goto skip;
+
+	windowType = window->flags() & Qt::WindowType::WindowType_Mask;
+
+	if (windowType == Qt::WindowType::Dialog ||
+	    windowType == Qt::WindowType::Window ||
+	    windowType == Qt::WindowType::Tool) {
+		OBSBasic *main = reinterpret_cast<OBSBasic *>(GetMainWindow());
+		if (main)
+			main->SetDisplayAffinity(window);
+	}
+
+skip:
+	return QApplication::notify(receiver, e);
+}
+
 QString OBSTranslator::translate(const char *context, const char *sourceText,
 				 const char *disambiguation, int n) const
 {

UI/obs-app.hpp

@@ -105,6 +105,8 @@ private:
 	void AddExtraThemeColor(QPalette &pal, int group, const char *name,
 				uint32_t color);
 
+	bool notify(QObject *receiver, QEvent *e) override;
+
 public:
 	OBSApp(int &argc, char **argv, profiler_name_store_t *store);
 	~OBSApp();

UI/platform-osx.mm

@@ -196,6 +196,12 @@ void SetAlwaysOnTop(QWidget *window, bool enable)
 	window->show();
 }
 
+bool SetDisplayAffinitySupported(void)
+{
+	// Not implemented yet
+	return false;
+}
+
 typedef void (*set_int_t)(int);
 
 void EnableOSXVSync(bool enable)

UI/platform-windows.cpp

@@ -162,6 +162,20 @@ uint32_t GetWindowsVersion()
 	return ver;
 }
 
+uint32_t GetWindowsBuild()
+{
+	static uint32_t build = 0;
+
+	if (build == 0) {
+		struct win_version_info ver_info;
+
+		get_win_ver(&ver_info);
+		build = ver_info.build;
+	}
+
+	return build;
+}
+
 void SetAeroEnabled(bool enable)
 {
 	static HRESULT(WINAPI * func)(UINT) = nullptr;
@@ -229,6 +243,27 @@ void SetWin32DropStyle(QWidget *window)
 	SetWindowLongPtr(hwnd, GWL_EXSTYLE, ex_style);
 }
 
+bool SetDisplayAffinitySupported(void)
+{
+	static bool checked = false;
+	static bool supported;
+
+	/* this has to be version gated as setting WDA_EXCLUDEFROMCAPTURE on
+	   older Windows builds behaves like WDA_MONITOR (black box) */
+
+	if (!checked) {
+		if (GetWindowsVersion() > 0x0A00 ||
+		    GetWindowsVersion() == 0x0A00 && GetWindowsBuild() > 19041)
+			supported = true;
+		else
+			supported = false;
+
+		checked = true;
+	}
+
+	return supported;
+}
+
 bool DisableAudioDucking(bool disable)
 {
 	ComPtr<IMMDeviceEnumerator> devEmum;

UI/platform-x11.cpp

@@ -251,3 +251,9 @@ void SetAlwaysOnTop(QWidget *window, bool enable)
 	window->setWindowFlags(flags);
 	window->show();
 }
+
+bool SetDisplayAffinitySupported(void)
+{
+	// Not implemented yet
+	return false;
+}

UI/platform.hpp

@@ -37,8 +37,11 @@ std::vector<std::string> GetPreferredLocales();
 bool IsAlwaysOnTop(QWidget *window);
 void SetAlwaysOnTop(QWidget *window, bool enable);
 
+bool SetDisplayAffinitySupported(void);
+
 #ifdef _WIN32
 uint32_t GetWindowsVersion();
+uint32_t GetWindowsBuild();
 void SetAeroEnabled(bool enable);
 void SetProcessPriority(const char *priority);
 void SetWin32DropStyle(QWidget *window);

UI/window-basic-main.cpp

@@ -9947,3 +9947,29 @@ void OBSBasic::UpdatePreviewSafeAreas()
 	drawSafeAreas = config_get_bool(App()->GlobalConfig(), "BasicWindow",
 					"ShowSafeAreas");
 }
+
+void OBSBasic::SetDisplayAffinity(QWindow *window)
+{
+	if (!SetDisplayAffinitySupported())
+		return;
+
+	bool hideFromCapture = config_get_bool(App()->GlobalConfig(),
+					       "BasicWindow",
+					       "HideOBSWindowsFromCapture");
+
+	// Don't hide projectors, those are designed to be visible / captured
+	if (window->property("isOBSProjectorWindow") == true)
+		return;
+
+#ifdef _WIN32
+	HWND hwnd = (HWND)window->winId();
+
+	if (hideFromCapture)
+		SetWindowDisplayAffinity(hwnd, WDA_EXCLUDEFROMCAPTURE);
+	else
+		SetWindowDisplayAffinity(hwnd, WDA_NONE);
+#else
+// TODO: Implement for other platforms if possible. Don't forget to
+// implement SetDisplayAffinitySupported too!
+#endif
+}

UI/window-basic-main.hpp

@@ -943,6 +943,8 @@ public:
 
 	void UpdateEditMenu();
 
+	void SetDisplayAffinity(QWindow *window);
+
 protected:
 	virtual void closeEvent(QCloseEvent *event) override;
 	virtual void changeEvent(QEvent *event) override;

UI/window-basic-settings.cpp

@@ -383,6 +383,7 @@ OBSBasicSettings::OBSBasicSettings(QWidget *parent)
 	HookWidget(ui->theme, 		     COMBO_CHANGED,  GENERAL_CHANGED);
 	HookWidget(ui->enableAutoUpdates,    CHECK_CHANGED,  GENERAL_CHANGED);
 	HookWidget(ui->openStatsOnStartup,   CHECK_CHANGED,  GENERAL_CHANGED);
+	HookWidget(ui->hideOBSFromCapture,   CHECK_CHANGED,  GENERAL_CHANGED);
 	HookWidget(ui->warnBeforeStreamStart,CHECK_CHANGED,  GENERAL_CHANGED);
 	HookWidget(ui->warnBeforeStreamStop, CHECK_CHANGED,  GENERAL_CHANGED);
 	HookWidget(ui->warnBeforeRecordStop, CHECK_CHANGED,  GENERAL_CHANGED);
@@ -589,6 +590,7 @@ OBSBasicSettings::OBSBasicSettings(QWidget *parent)
 #ifdef _WIN32
 	uint32_t winVer = GetWindowsVersion();
 	if (winVer > 0 && winVer < 0x602) {
+		// Older than Windows 8
 		toggleAero = new QCheckBox(
 			QTStr("Basic.Settings.Video.DisableAero"), this);
 		QFormLayout *videoLayout = reinterpret_cast<QFormLayout *>(
@@ -600,6 +602,11 @@ OBSBasicSettings::OBSBasicSettings(QWidget *parent)
 			&OBSBasicSettings::ToggleDisableAero);
 	}
 
+	if (!SetDisplayAffinitySupported()) {
+		delete ui->hideOBSFromCapture;
+		ui->hideOBSFromCapture = nullptr;
+	}
+
 #define PROCESS_PRIORITY(val)                                                \
 	{                                                                    \
 		"Basic.Settings.Advanced.General.ProcessPriority."##val, val \
@@ -627,6 +634,7 @@ OBSBasicSettings::OBSBasicSettings(QWidget *parent)
 	delete ui->processPriority;
 	delete ui->enableNewSocketLoop;
 	delete ui->enableLowLatencyMode;
+	delete ui->hideOBSFromCapture;
 #ifdef __linux__
 	delete ui->browserHWAccel;
 	delete ui->sourcesGroup;
@@ -642,6 +650,7 @@ OBSBasicSettings::OBSBasicSettings(QWidget *parent)
 	ui->processPriority = nullptr;
 	ui->enableNewSocketLoop = nullptr;
 	ui->enableLowLatencyMode = nullptr;
+	ui->hideOBSFromCapture = nullptr;
 #ifdef __linux__
 	ui->browserHWAccel = nullptr;
 	ui->sourcesGroup = nullptr;
@@ -1226,6 +1235,15 @@ void OBSBasicSettings::LoadGeneralSettings()
 						  "OpenStatsOnStartup");
 	ui->openStatsOnStartup->setChecked(openStatsOnStartup);
 
+#if defined(_WIN32)
+	if (ui->hideOBSFromCapture) {
+		bool hideWindowFromCapture =
+			config_get_bool(GetGlobalConfig(), "BasicWindow",
+					"HideOBSWindowsFromCapture");
+		ui->hideOBSFromCapture->setChecked(hideWindowFromCapture);
+	}
+#endif
+
 	bool recordWhenStreaming = config_get_bool(
 		GetGlobalConfig(), "BasicWindow", "RecordWhenStreaming");
 	ui->recordWhenStreaming->setChecked(recordWhenStreaming);
@@ -2974,6 +2992,20 @@ void OBSBasicSettings::SaveGeneralSettings()
 		config_set_bool(GetGlobalConfig(), "General",
 				"EnableAutoUpdates",
 				ui->enableAutoUpdates->isChecked());
+#endif
+#ifdef _WIN32
+	if (WidgetChanged(ui->hideOBSFromCapture)) {
+		bool hide_window = ui->hideOBSFromCapture->isChecked();
+		config_set_bool(GetGlobalConfig(), "BasicWindow",
+				"HideOBSWindowsFromCapture", hide_window);
+
+		QWindowList windows = QGuiApplication::allWindows();
+		for (auto window : windows) {
+			if (window->isVisible()) {
+				main->SetDisplayAffinity(window);
+			}
+		}
+	}
 #endif
 	if (WidgetChanged(ui->openStatsOnStartup))
 		config_set_bool(main->Config(), "General", "OpenStatsOnStartup",

UI/window-projector.cpp

@@ -30,6 +30,10 @@ OBSProjector::OBSProjector(QWidget *widget, obs_source_t *source_, int monitor,
 	if (isAlwaysOnTop)
 		setWindowFlags(Qt::WindowStaysOnTopHint);
 
+	// Mark the window as a projector so SetDisplayAffinity
+	// can skip it
+	windowHandle()->setProperty("isOBSProjectorWindow", true);
+
 #if defined(__linux__) || defined(__FreeBSD__) || defined(__DragonFly__)
 	// Prevents resizing of projector windows
 	setAttribute(Qt::WA_PaintOnScreen, false);