graphics-hook: Avoid conflict between Vulkan and DXGI Present

plugins/win-capture/graphics-hook/dxgi-capture.cpp

@@ -198,6 +198,13 @@ static void update_mismatch_count(bool match)
 static HRESULT STDMETHODCALLTYPE hook_present(IDXGISwapChain *swap,
 					      UINT sync_interval, UINT flags)
 {
+	if (should_passthrough()) {
+		dxgi_presenting = true;
+		const HRESULT hr = RealPresent(swap, sync_interval, flags);
+		dxgi_presenting = false;
+		return hr;
+	}
+
 	const bool capture_overlay = global_hook_info->capture_overlay;
 	const bool test_draw = (flags & DXGI_PRESENT_TEST) != 0;
 
@@ -255,6 +262,14 @@ static HRESULT STDMETHODCALLTYPE
 hook_present1(IDXGISwapChain1 *swap, UINT sync_interval, UINT flags,
 	      const DXGI_PRESENT_PARAMETERS *params)
 {
+	if (should_passthrough()) {
+		dxgi_presenting = true;
+		const HRESULT hr =
+			RealPresent1(swap, sync_interval, flags, params);
+		dxgi_presenting = false;
+		return hr;
+	}
+
 	const bool capture_overlay = global_hook_info->capture_overlay;
 	const bool test_draw = (flags & DXGI_PRESENT_TEST) != 0;
 

plugins/win-capture/graphics-hook/graphics-hook.h

@@ -248,6 +248,19 @@ static inline bool capture_should_init(void)
 	return should_init;
 }
 
+#if COMPILE_VULKAN_HOOK
+extern __declspec(thread) int vk_presenting;
+#endif
+
+static inline bool should_passthrough()
+{
+#if COMPILE_VULKAN_HOOK
+	return vk_presenting > 0;
+#else
+	return false;
+#endif
+}
+
 #ifdef __cplusplus
 }
 #endif

plugins/win-capture/graphics-hook/vulkan-capture.c

@@ -115,6 +115,8 @@ struct vk_data {
 	ID3D11DeviceContext *d3d11_context;
 };
 
+__declspec(thread) int vk_presenting = 0;
+
 /* ------------------------------------------------------------------------- */
 
 static void *vk_alloc(const VkAllocationCallbacks *ac, size_t size,
@@ -1216,7 +1218,14 @@ static VkResult VKAPI_CALL OBS_QueuePresentKHR(VkQueue queue,
 		vk_capture(data, queue, info);
 	}
 
-	return funcs->QueuePresentKHR(queue, info);
+	if (vk_presenting != 0) {
+		flog("non-zero vk_presenting: %d", vk_presenting);
+	}
+
+	vk_presenting++;
+	VkResult res = funcs->QueuePresentKHR(queue, info);
+	vk_presenting--;
+	return res;
 }
 
 /* ======================================================================== */