win-capture: Fail on unsupported Vulkan formats

Don't allow unsupported Vulkan formats to fall back to B8G8R8A8.
Probably better to fail completely than do an illegal copy.

Also remove bad conversion for VK_FORMAT_A2R10G10B10_UNORM_PACK32.
Red and blue channels were reversed, and there's no DXGI equivalent.

Addresses #2796. We can do more later if justified.

plugins/win-capture/graphics-hook/vulkan-capture.c

@@ -457,16 +457,24 @@ static inline bool vk_shtex_init_d3d11_tex(struct vk_data *data,
 	IDXGIResource *dxgi_res;
 	HRESULT hr;
 
+	const UINT width = swap->image_extent.width;
+	const UINT height = swap->image_extent.height;
+
+	flog("OBS requesting %s texture format. capture dimensions: %ux%u",
+	     vk_format_to_str(swap->format), width, height);
+
+	const DXGI_FORMAT format = vk_format_to_dxgi(swap->format);
+	if (format == DXGI_FORMAT_UNKNOWN) {
+		flog("cannot convert to DXGI format");
+		return false;
+	}
+
 	D3D11_TEXTURE2D_DESC desc = {0};
-	desc.Width = swap->image_extent.width;
-	desc.Height = swap->image_extent.height;
+	desc.Width = width;
+	desc.Height = height;
 	desc.MipLevels = 1;
 	desc.ArraySize = 1;
-
-	flog("OBS requesting %s texture format.  capture dimensions: %dx%d",
-	     vk_format_to_str(swap->format), (int)desc.Width, (int)desc.Height);
-
-	desc.Format = vk_format_to_dxgi(swap->format);
+	desc.Format = format;
 	desc.SampleDesc.Count = 1;
 	desc.SampleDesc.Quality = 0;
 	desc.Usage = D3D11_USAGE_DEFAULT;

plugins/win-capture/graphics-hook/vulkan-capture.h

@@ -473,7 +473,6 @@ DXGI_FORMAT vk_format_to_dxgi(VkFormat format)
 	case VK_FORMAT_A8B8G8R8_SRGB_PACK32:
 		break;
 	case VK_FORMAT_A2R10G10B10_UNORM_PACK32:
-		dxgi_format = DXGI_FORMAT_R10G10B10A2_UNORM;
 		break;
 	case VK_FORMAT_A2R10G10B10_SNORM_PACK32:
 		break;
@@ -841,11 +840,6 @@ DXGI_FORMAT vk_format_to_dxgi(VkFormat format)
 	case VK_FORMAT_PVRTC2_4BPP_SRGB_BLOCK_IMG:
 		break;
 	}
-	if (dxgi_format == DXGI_FORMAT_UNKNOWN) {
-		flog("unknown swapchain format, "
-		     "defaulting to B8G8R8A8_UNORM");
-		dxgi_format = DXGI_FORMAT_B8G8R8A8_UNORM;
-	}
 	return dxgi_format;
 }