libobs/graphics: Add color space and WIC support

Only add support for PQ and CCCS JXR images, e.g. Xbox Series X, and
Xbox Game Bar screenshots on Windows.

libobs/graphics/graphics-ffmpeg.c

@@ -1,12 +1,20 @@
 #include "graphics.h"
 
+#include "half.h"
+#include "srgb.h"
+#include <obs-ffmpeg-compat.h>
+#include <util/dstr.h>
+#include <util/platform.h>
+
 #include <libavcodec/avcodec.h>
 #include <libavformat/avformat.h>
 #include <libavutil/imgutils.h>
 #include <libswscale/swscale.h>
 
-#include "../obs-ffmpeg-compat.h"
-#include "srgb.h"
+#ifdef _WIN32
+#include <wincodec.h>
+#pragma comment(lib, "windowscodecs.lib")
+#endif
 
 struct ffmpeg_image {
 	const char *file;
@@ -644,10 +652,230 @@ uint8_t *gs_create_texture_file_data(const char *file,
 	return data;
 }
 
+#ifdef _WIN32
+static float pq_to_linear(float u)
+{
+	const float common = powf(u, 1.f / 78.84375f);
+	return powf(fabsf(max(common - 0.8359375f, 0.f) /
+			  (18.8515625f - 18.6875f * common)),
+		    1.f / 0.1593017578f);
+}
+
+static void convert_pq_to_cccs(const BYTE *intermediate,
+			       const UINT intermediate_size, BYTE *bytes)
+{
+	const BYTE *src_cursor = intermediate;
+	const BYTE *src_cursor_end = src_cursor + intermediate_size;
+	BYTE *dst_cursor = bytes;
+	uint32_t rgb10;
+	struct half rgba16[4];
+	rgba16[3].u = 0x3c00;
+	while (src_cursor < src_cursor_end) {
+		memcpy(&rgb10, src_cursor, sizeof(rgb10));
+		const float blue = (float)(rgb10 & 0x3ff) / 1023.f;
+		const float green = (float)((rgb10 >> 10) & 0x3ff) / 1023.f;
+		const float red = (float)((rgb10 >> 20) & 0x3ff) / 1023.f;
+		const float red2020 = pq_to_linear(red);
+		const float green2020 = pq_to_linear(green);
+		const float blue2020 = pq_to_linear(blue);
+		const float red709 = 1.6604910f * red2020 -
+				     0.5876411f * green2020 -
+				     0.0728499f * blue2020;
+		const float green709 = -0.1245505f * red2020 +
+				       1.1328999f * green2020 -
+				       0.0083494f * blue2020;
+		const float blue709 = -0.0181508f * red2020 -
+				      0.1005789f * green2020 +
+				      1.1187297f * blue2020;
+		rgba16[0] = half_from_float(red709 * 125.f);
+		rgba16[1] = half_from_float(green709 * 125.f);
+		rgba16[2] = half_from_float(blue709 * 125.f);
+		memcpy(dst_cursor, &rgba16, sizeof(rgba16));
+		src_cursor += 4;
+		dst_cursor += 8;
+	}
+}
+
+static void *wic_image_init_internal(const char *file,
+				     IWICBitmapFrameDecode *pFrame,
+				     enum gs_color_format *format,
+				     uint32_t *cx_out, uint32_t *cy_out,
+				     enum gs_color_space *space)
+{
+	BYTE *bytes = NULL;
+
+	WICPixelFormatGUID pixelFormat;
+	HRESULT hr = pFrame->lpVtbl->GetPixelFormat(pFrame, &pixelFormat);
+	if (SUCCEEDED(hr)) {
+		const bool scrgb = memcmp(&pixelFormat,
+					  &GUID_WICPixelFormat64bppRGBAHalf,
+					  sizeof(pixelFormat)) == 0;
+		const bool pq10 = memcmp(&pixelFormat,
+					 &GUID_WICPixelFormat32bppBGR101010,
+					 sizeof(pixelFormat)) == 0;
+		if (scrgb || pq10) {
+			UINT width, height;
+			hr = pFrame->lpVtbl->GetSize(pFrame, &width, &height);
+			if (SUCCEEDED(hr)) {
+				const UINT pitch = 8 * width;
+				const UINT size = pitch * height;
+				bytes = bmalloc(size);
+				if (bytes) {
+					bool success = false;
+					if (pq10) {
+						const UINT intermediate_pitch =
+							4 * width;
+						const UINT intermediate_size =
+							intermediate_pitch *
+							height;
+						BYTE *intermediate = bmalloc(
+							intermediate_size);
+						if (intermediate) {
+							hr = pFrame->lpVtbl->CopyPixels(
+								pFrame, NULL,
+								intermediate_pitch,
+								intermediate_size,
+								intermediate);
+							success = SUCCEEDED(hr);
+							if (success) {
+								convert_pq_to_cccs(
+									intermediate,
+									intermediate_size,
+									bytes);
+							} else {
+								blog(LOG_WARNING,
+								     "WIC: Failed to CopyPixels intermediate for file: %s",
+								     file);
+							}
+
+							bfree(intermediate);
+						} else {
+							blog(LOG_WARNING,
+							     "WIC: Failed to allocate intermediate for file: %s",
+							     file);
+						}
+					} else {
+						hr = pFrame->lpVtbl->CopyPixels(
+							pFrame, NULL, pitch,
+							size, bytes);
+						success = SUCCEEDED(hr);
+						if (!success) {
+							blog(LOG_WARNING,
+							     "WIC: Failed to CopyPixels for file: %s",
+							     file);
+						}
+					}
+
+					if (success) {
+						*format = GS_RGBA16F;
+						*cx_out = width;
+						*cy_out = height;
+						*space = GS_CS_709_SCRGB;
+					} else {
+						bfree(bytes);
+						bytes = NULL;
+					}
+				} else {
+					blog(LOG_WARNING,
+					     "WIC: Failed to allocate for file: %s",
+					     file);
+				}
+			} else {
+				blog(LOG_WARNING,
+				     "WIC: Failed to GetSize of frame for file: %s",
+				     file);
+			}
+		} else {
+			blog(LOG_WARNING,
+			     "WIC: Only handle GUID_WICPixelFormat32bppBGR101010 and GUID_WICPixelFormat64bppRGBAHalf for now");
+		}
+	} else {
+		blog(LOG_WARNING, "WIC: Failed to GetPixelFormat for file: %s",
+		     file);
+	}
+
+	return bytes;
+}
+
+static void *wic_image_init(const struct ffmpeg_image *info, const char *file,
+			    enum gs_color_format *format, uint32_t *cx_out,
+			    uint32_t *cy_out, enum gs_color_space *space)
+{
+	const size_t len = strlen(file);
+	if (len <= 4 && astrcmpi(file + len - 4, ".jxr") != 0) {
+		blog(LOG_WARNING,
+		     "WIC: Only handle JXR for WIC images for now");
+		return NULL;
+	}
+
+	BYTE *bytes = NULL;
+
+	wchar_t *file_w = NULL;
+	os_utf8_to_wcs_ptr(file, 0, &file_w);
+	if (file_w) {
+		IWICImagingFactory *pFactory = NULL;
+		HRESULT hr = CoCreateInstance(&CLSID_WICImagingFactory, NULL,
+					      CLSCTX_INPROC_SERVER,
+					      &IID_IWICImagingFactory,
+					      &pFactory);
+		if (SUCCEEDED(hr)) {
+			IWICBitmapDecoder *pDecoder = NULL;
+			hr = pFactory->lpVtbl->CreateDecoderFromFilename(
+				pFactory, file_w, NULL, GENERIC_READ,
+				WICDecodeMetadataCacheOnDemand, &pDecoder);
+			if (SUCCEEDED(hr)) {
+				IWICBitmapFrameDecode *pFrame = NULL;
+				hr = pDecoder->lpVtbl->GetFrame(pDecoder, 0,
+								&pFrame);
+				if (SUCCEEDED(hr)) {
+					bytes = wic_image_init_internal(
+						file, pFrame, format, cx_out,
+						cy_out, space);
+
+					pFrame->lpVtbl->Release(pFrame);
+				} else {
+					blog(LOG_WARNING,
+					     "WIC: Failed to create IWICBitmapFrameDecode from file: %s",
+					     file);
+				}
+
+				pDecoder->lpVtbl->Release(pDecoder);
+			} else {
+				blog(LOG_WARNING,
+				     "WIC: Failed to create IWICBitmapDecoder from file: %s",
+				     file);
+			}
+
+			pFactory->lpVtbl->Release(pFactory);
+		} else {
+			blog(LOG_WARNING,
+			     "WIC: Failed to create IWICImagingFactory");
+		}
+
+		bfree(file_w);
+	} else {
+		blog(LOG_WARNING, "WIC: Failed to widen file name: %s", file);
+	}
+
+	return bytes;
+}
+#endif
+
 uint8_t *gs_create_texture_file_data2(const char *file,
 				      enum gs_image_alpha_mode alpha_mode,
 				      enum gs_color_format *format,
 				      uint32_t *cx_out, uint32_t *cy_out)
+{
+	enum gs_color_space unused;
+	return gs_create_texture_file_data3(file, alpha_mode, format, cx_out,
+					    cy_out, &unused);
+}
+
+uint8_t *gs_create_texture_file_data3(const char *file,
+				      enum gs_image_alpha_mode alpha_mode,
+				      enum gs_color_format *format,
+				      uint32_t *cx_out, uint32_t *cy_out,
+				      enum gs_color_space *space)
 {
 	struct ffmpeg_image image;
 	uint8_t *data = NULL;
@@ -658,10 +886,18 @@ uint8_t *gs_create_texture_file_data2(const char *file,
 			*format = convert_format(image.format);
 			*cx_out = (uint32_t)image.cx;
 			*cy_out = (uint32_t)image.cy;
+			*space = GS_CS_SRGB;
 		}
 
 		ffmpeg_image_free(&image);
 	}
 
+#ifdef _WIN32
+	if (data == NULL) {
+		data = wic_image_init(&image, file, format, cx_out, cy_out,
+				      space);
+	}
+#endif
+
 	return data;
 }

libobs/graphics/graphics.h

@@ -593,6 +593,11 @@ EXPORT uint8_t *gs_create_texture_file_data(const char *file,
 EXPORT uint8_t *gs_create_texture_file_data2(
 	const char *file, enum gs_image_alpha_mode alpha_mode,
 	enum gs_color_format *format, uint32_t *cx, uint32_t *cy);
+EXPORT uint8_t *
+gs_create_texture_file_data3(const char *file,
+			     enum gs_image_alpha_mode alpha_mode,
+			     enum gs_color_format *format, uint32_t *cx,
+			     uint32_t *cy, enum gs_color_space *space);
 
 #define GS_FLIP_U (1 << 0)
 #define GS_FLIP_V (1 << 1)

libobs/graphics/image-file.c

@@ -193,6 +193,7 @@ not_animated:
 
 static void gs_image_file_init_internal(gs_image_file_t *image,
 					const char *file, uint64_t *mem_usage,
+					enum gs_color_space *space,
 					enum gs_image_alpha_mode alpha_mode)
 {
 	size_t len;
@@ -213,8 +214,9 @@ static void gs_image_file_init_internal(gs_image_file_t *image,
 		}
 	}
 
-	image->texture_data = gs_create_texture_file_data2(
-		file, alpha_mode, &image->format, &image->cx, &image->cy);
+	image->texture_data =
+		gs_create_texture_file_data3(file, alpha_mode, &image->format,
+					     &image->cx, &image->cy, space);
 
 	if (mem_usage) {
 		*mem_usage += image->cx * image->cy *
@@ -230,7 +232,9 @@ static void gs_image_file_init_internal(gs_image_file_t *image,
 
 void gs_image_file_init(gs_image_file_t *image, const char *file)
 {
-	gs_image_file_init_internal(image, file, NULL, GS_IMAGE_ALPHA_STRAIGHT);
+	enum gs_color_space unused;
+	gs_image_file_init_internal(image, file, NULL, &unused,
+				    GS_IMAGE_ALPHA_STRAIGHT);
 }
 
 void gs_image_file_free(gs_image_file_t *image)
@@ -255,18 +259,30 @@ void gs_image_file_free(gs_image_file_t *image)
 
 void gs_image_file2_init(gs_image_file2_t *if2, const char *file)
 {
-	gs_image_file_init_internal(&if2->image, file, &if2->mem_usage,
+	enum gs_color_space unused;
+	gs_image_file_init_internal(&if2->image, file, &if2->mem_usage, &unused,
 				    GS_IMAGE_ALPHA_STRAIGHT);
 }
 
 void gs_image_file3_init(gs_image_file3_t *if3, const char *file,
 			 enum gs_image_alpha_mode alpha_mode)
 {
+	enum gs_color_space unused;
 	gs_image_file_init_internal(&if3->image2.image, file,
-				    &if3->image2.mem_usage, alpha_mode);
+				    &if3->image2.mem_usage, &unused,
+				    alpha_mode);
 	if3->alpha_mode = alpha_mode;
 }
 
+void gs_image_file4_init(gs_image_file4_t *if4, const char *file,
+			 enum gs_image_alpha_mode alpha_mode)
+{
+	gs_image_file_init_internal(&if4->image3.image2.image, file,
+				    &if4->image3.image2.mem_usage, &if4->space,
+				    alpha_mode);
+	if4->image3.alpha_mode = alpha_mode;
+}
+
 void gs_image_file_init_texture(gs_image_file_t *image)
 {
 	if (!image->loaded)
@@ -404,6 +420,13 @@ bool gs_image_file3_tick(gs_image_file3_t *if3, uint64_t elapsed_time_ns)
 					   if3->alpha_mode);
 }
 
+bool gs_image_file4_tick(gs_image_file4_t *if4, uint64_t elapsed_time_ns)
+{
+	return gs_image_file_tick_internal(&if4->image3.image2.image,
+					   elapsed_time_ns,
+					   if4->image3.alpha_mode);
+}
+
 static void
 gs_image_file_update_texture_internal(gs_image_file_t *image,
 				      enum gs_image_alpha_mode alpha_mode)
@@ -434,3 +457,9 @@ void gs_image_file3_update_texture(gs_image_file3_t *if3)
 	gs_image_file_update_texture_internal(&if3->image2.image,
 					      if3->alpha_mode);
 }
+
+void gs_image_file4_update_texture(gs_image_file4_t *if4)
+{
+	gs_image_file_update_texture_internal(&if4->image3.image2.image,
+					      if4->image3.alpha_mode);
+}

libobs/graphics/image-file.h

@@ -56,9 +56,15 @@ struct gs_image_file3 {
 	enum gs_image_alpha_mode alpha_mode;
 };
 
+struct gs_image_file4 {
+	struct gs_image_file3 image3;
+	enum gs_color_space space;
+};
+
 typedef struct gs_image_file gs_image_file_t;
 typedef struct gs_image_file2 gs_image_file2_t;
 typedef struct gs_image_file3 gs_image_file3_t;
+typedef struct gs_image_file4 gs_image_file4_t;
 
 EXPORT void gs_image_file_init(gs_image_file_t *image, const char *file);
 EXPORT void gs_image_file_free(gs_image_file_t *image);
@@ -81,6 +87,13 @@ EXPORT bool gs_image_file3_tick(gs_image_file3_t *if3,
 				uint64_t elapsed_time_ns);
 EXPORT void gs_image_file3_update_texture(gs_image_file3_t *if3);
 
+EXPORT void gs_image_file4_init(gs_image_file4_t *if4, const char *file,
+				enum gs_image_alpha_mode alpha_mode);
+
+EXPORT bool gs_image_file4_tick(gs_image_file4_t *if4,
+				uint64_t elapsed_time_ns);
+EXPORT void gs_image_file4_update_texture(gs_image_file4_t *if4);
+
 static void gs_image_file2_free(gs_image_file2_t *if2)
 {
 	gs_image_file_free(&if2->image);
@@ -102,6 +115,16 @@ static void gs_image_file3_init_texture(gs_image_file3_t *if3)
 	gs_image_file2_init_texture(&if3->image2);
 }
 
+static void gs_image_file4_free(gs_image_file4_t *if4)
+{
+	gs_image_file3_free(&if4->image3);
+}
+
+static void gs_image_file4_init_texture(gs_image_file4_t *if4)
+{
+	gs_image_file3_init_texture(&if4->image3);
+}
+
 #ifdef __cplusplus
 }
 #endif