libobs: Add YUV alpha formats

This will allow YUV alpha formats to be converted to RGBA on the GPU.

libobs/data/format_conversion.effect

@@ -31,6 +31,7 @@ uniform float3    color_range_max = {1.0, 1.0, 1.0};
 uniform texture2d image;
 uniform texture2d image1;
 uniform texture2d image2;
+uniform texture2d image3;
 
 sampler_state def_sampler {
 	Filter   = Linear;
@@ -256,6 +257,19 @@ float3 PSPlanar420_Reverse(VertTexPos frag_in) : TARGET
 	return rgb;
 }
 
+float4 PSPlanar420A_Reverse(VertTexPos frag_in) : TARGET
+{
+	int3 xy0_luma = int3(frag_in.pos.xy, 0);
+	float y = image.Load(xy0_luma).x;
+	int3 xy0_chroma = int3(frag_in.uv, 0);
+	float cb = image1.Load(xy0_chroma).x;
+	float cr = image2.Load(xy0_chroma).x;
+	float alpha = image3.Load(xy0_luma).x;
+	float3 yuv = float3(y, cb, cr);
+	float4 rgba = float4(YUV_to_RGB(yuv), alpha);
+	return rgba;
+}
+
 float3 PSPlanar422_Reverse(FragPosWide frag_in) : TARGET
 {
 	float y = image.Load(int3(frag_in.pos_wide.xz, 0)).x;
@@ -267,6 +281,19 @@ float3 PSPlanar422_Reverse(FragPosWide frag_in) : TARGET
 	return rgb;
 }
 
+float4 PSPlanar422A_Reverse(FragPosWide frag_in) : TARGET
+{
+	int3 xy0_luma = int3(frag_in.pos_wide.xz, 0);
+	float y = image.Load(xy0_luma).x;
+	int3 xy0_chroma = int3(frag_in.pos_wide.yz, 0);
+	float cb = image1.Load(xy0_chroma).x;
+	float cr = image2.Load(xy0_chroma).x;
+	float alpha = image3.Load(xy0_luma).x;
+	float3 yuv = float3(y, cb, cr);
+	float4 rgba = float4(YUV_to_RGB(yuv), alpha);
+	return rgba;
+}
+
 float3 PSPlanar444_Reverse(FragPos frag_in) : TARGET
 {
 	int3 xy0 = int3(frag_in.pos.xy, 0);
@@ -278,6 +305,25 @@ float3 PSPlanar444_Reverse(FragPos frag_in) : TARGET
 	return rgb;
 }
 
+float4 PSPlanar444A_Reverse(FragPos frag_in) : TARGET
+{
+	int3 xy0 = int3(frag_in.pos.xy, 0);
+	float y = image.Load(xy0).x;
+	float cb = image1.Load(xy0).x;
+	float cr = image2.Load(xy0).x;
+	float alpha = image3.Load(xy0).x;
+	float3 yuv = float3(y, cb, cr);
+	float4 rgba = float4(YUV_to_RGB(yuv), alpha);
+	return rgba;
+}
+
+float4 PSAYUV_Reverse(FragPos frag_in) : TARGET
+{
+	float4 yuva = image.Load(int3(frag_in.pos.xy, 0));
+	float4 rgba = float4(YUV_to_RGB(yuva.xyz), yuva.a);
+	return rgba;
+}
+
 float3 PSNV12_Reverse(VertTexPos frag_in) : TARGET
 {
 	float y = image.Load(int3(frag_in.pos.xy, 0)).x;
@@ -429,6 +475,15 @@ technique I420_Reverse
 	}
 }
 
+technique I40A_Reverse
+{
+	pass
+	{
+		vertex_shader = VSTexPosHalfHalf_Reverse(id);
+		pixel_shader  = PSPlanar420A_Reverse(frag_in);
+	}
+}
+
 technique I422_Reverse
 {
 	pass
@@ -438,6 +493,15 @@ technique I422_Reverse
 	}
 }
 
+technique I42A_Reverse
+{
+	pass
+	{
+		vertex_shader = VSPosWide_Reverse(id);
+		pixel_shader  = PSPlanar422A_Reverse(frag_in);
+	}
+}
+
 technique I444_Reverse
 {
 	pass
@@ -447,6 +511,24 @@ technique I444_Reverse
 	}
 }
 
+technique YUVA_Reverse
+{
+	pass
+	{
+		vertex_shader = VSPos(id);
+		pixel_shader  = PSPlanar444A_Reverse(frag_in);
+	}
+}
+
+technique AYUV_Reverse
+{
+	pass
+	{
+		vertex_shader = VSPos(id);
+		pixel_shader  = PSAYUV_Reverse(frag_in);
+	}
+}
+
 technique NV12_Reverse
 {
 	pass

libobs/media-io/video-frame.c

@@ -85,6 +85,7 @@ void video_frame_init(struct video_frame *frame, enum video_format format,
 	case VIDEO_FORMAT_RGBA:
 	case VIDEO_FORMAT_BGRA:
 	case VIDEO_FORMAT_BGRX:
+	case VIDEO_FORMAT_AYUV:
 		size = width * height * 4;
 		ALIGN_SIZE(size, alignment);
 		frame->data[0] = bmalloc(size);
@@ -125,6 +126,72 @@ void video_frame_init(struct video_frame *frame, enum video_format format,
 		frame->linesize[1] = width / 2;
 		frame->linesize[2] = width / 2;
 		break;
+
+	case VIDEO_FORMAT_I40A:
+		size = width * height;
+		ALIGN_SIZE(size, alignment);
+		offsets[0] = size;
+		size += (width / 2) * (height / 2);
+		ALIGN_SIZE(size, alignment);
+		offsets[1] = size;
+		size += (width / 2) * (height / 2);
+		ALIGN_SIZE(size, alignment);
+		offsets[2] = size;
+		size += width * height;
+		ALIGN_SIZE(size, alignment);
+		frame->data[0] = bmalloc(size);
+		frame->data[1] = (uint8_t *)frame->data[0] + offsets[0];
+		frame->data[2] = (uint8_t *)frame->data[0] + offsets[1];
+		frame->data[3] = (uint8_t *)frame->data[0] + offsets[2];
+		frame->linesize[0] = width;
+		frame->linesize[1] = width / 2;
+		frame->linesize[2] = width / 2;
+		frame->linesize[3] = width;
+		break;
+
+	case VIDEO_FORMAT_I42A:
+		size = width * height;
+		ALIGN_SIZE(size, alignment);
+		offsets[0] = size;
+		size += (width / 2) * height;
+		ALIGN_SIZE(size, alignment);
+		offsets[1] = size;
+		size += (width / 2) * height;
+		ALIGN_SIZE(size, alignment);
+		offsets[2] = size;
+		size += width * height;
+		ALIGN_SIZE(size, alignment);
+		frame->data[0] = bmalloc(size);
+		frame->data[1] = (uint8_t *)frame->data[0] + offsets[0];
+		frame->data[2] = (uint8_t *)frame->data[0] + offsets[1];
+		frame->data[3] = (uint8_t *)frame->data[0] + offsets[2];
+		frame->linesize[0] = width;
+		frame->linesize[1] = width / 2;
+		frame->linesize[2] = width / 2;
+		frame->linesize[3] = width;
+		break;
+
+	case VIDEO_FORMAT_YUVA:
+		size = width * height;
+		ALIGN_SIZE(size, alignment);
+		offsets[0] = size;
+		size += width * height;
+		ALIGN_SIZE(size, alignment);
+		offsets[1] = size;
+		size += width * height;
+		ALIGN_SIZE(size, alignment);
+		offsets[2] = size;
+		size += width * height;
+		ALIGN_SIZE(size, alignment);
+		frame->data[0] = bmalloc(size);
+		frame->data[1] = (uint8_t *)frame->data[0] + offsets[0];
+		frame->data[2] = (uint8_t *)frame->data[0] + offsets[1];
+		frame->data[3] = (uint8_t *)frame->data[0] + offsets[2];
+		frame->linesize[0] = width;
+		frame->linesize[1] = width;
+		frame->linesize[2] = width;
+		frame->linesize[3] = width;
+		break;
 	}
 }
 
@@ -154,6 +221,7 @@ void video_frame_copy(struct video_frame *dst, const struct video_frame *src,
 	case VIDEO_FORMAT_BGRA:
 	case VIDEO_FORMAT_BGRX:
 	case VIDEO_FORMAT_BGR3:
+	case VIDEO_FORMAT_AYUV:
 		memcpy(dst->data[0], src->data[0], src->linesize[0] * cy);
 		break;
 
@@ -163,5 +231,20 @@ void video_frame_copy(struct video_frame *dst, const struct video_frame *src,
 		memcpy(dst->data[1], src->data[1], src->linesize[1] * cy);
 		memcpy(dst->data[2], src->data[2], src->linesize[2] * cy);
 		break;
+
+	case VIDEO_FORMAT_I40A:
+		memcpy(dst->data[0], src->data[0], src->linesize[0] * cy);
+		memcpy(dst->data[1], src->data[1], src->linesize[1] * cy / 2);
+		memcpy(dst->data[2], src->data[2], src->linesize[2] * cy / 2);
+		memcpy(dst->data[3], src->data[3], src->linesize[3] * cy);
+		break;
+
+	case VIDEO_FORMAT_I42A:
+	case VIDEO_FORMAT_YUVA:
+		memcpy(dst->data[0], src->data[0], src->linesize[0] * cy);
+		memcpy(dst->data[1], src->data[1], src->linesize[1] * cy);
+		memcpy(dst->data[2], src->data[2], src->linesize[2] * cy);
+		memcpy(dst->data[3], src->data[3], src->linesize[3] * cy);
+		break;
 	}
 }

libobs/media-io/video-io.h

@@ -56,6 +56,18 @@ enum video_format {
 
 	/* planar 4:2:2 */
 	VIDEO_FORMAT_I422,
+
+	/* planar 4:2:0 with alpha */
+	VIDEO_FORMAT_I40A,
+
+	/* planar 4:2:2 with alpha */
+	VIDEO_FORMAT_I42A,
+
+	/* planar 4:4:4 with alpha */
+	VIDEO_FORMAT_YUVA,
+
+	/* packed 4:4:4 with alpha */
+	VIDEO_FORMAT_AYUV,
 };
 
 enum video_colorspace {
@@ -99,6 +111,10 @@ static inline bool format_is_yuv(enum video_format format)
 	case VIDEO_FORMAT_YUY2:
 	case VIDEO_FORMAT_UYVY:
 	case VIDEO_FORMAT_I444:
+	case VIDEO_FORMAT_I40A:
+	case VIDEO_FORMAT_I42A:
+	case VIDEO_FORMAT_YUVA:
+	case VIDEO_FORMAT_AYUV:
 		return true;
 	case VIDEO_FORMAT_NONE:
 	case VIDEO_FORMAT_RGBA:
@@ -137,6 +153,14 @@ static inline const char *get_video_format_name(enum video_format format)
 		return "Y800";
 	case VIDEO_FORMAT_BGR3:
 		return "BGR3";
+	case VIDEO_FORMAT_I40A:
+		return "I40A";
+	case VIDEO_FORMAT_I42A:
+		return "I42A";
+	case VIDEO_FORMAT_YUVA:
+		return "YUVA";
+	case VIDEO_FORMAT_AYUV:
+		return "AYUV";
 	case VIDEO_FORMAT_NONE:;
 	}
 

libobs/media-io/video-scaler-ffmpeg.c

@@ -55,6 +55,12 @@ get_ffmpeg_video_format(enum video_format format)
 		return AV_PIX_FMT_BGR24;
 	case VIDEO_FORMAT_I422:
 		return AV_PIX_FMT_YUV422P;
+	case VIDEO_FORMAT_I40A:
+		return AV_PIX_FMT_YUVA420P;
+	case VIDEO_FORMAT_I42A:
+		return AV_PIX_FMT_YUVA422P;
+	case VIDEO_FORMAT_YUVA:
+		return AV_PIX_FMT_YUVA444P;
 	}
 
 	return AV_PIX_FMT_NONE;

libobs/obs-internal.h

@@ -772,12 +772,18 @@ static inline bool frame_out_of_bounds(const obs_source_t *source, uint64_t ts)
 static inline enum gs_color_format
 convert_video_format(enum video_format format)
 {
-	if (format == VIDEO_FORMAT_RGBA)
+	switch (format) {
+	case VIDEO_FORMAT_RGBA:
 		return GS_RGBA;
-	else if (format == VIDEO_FORMAT_BGRA)
+	case VIDEO_FORMAT_BGRA:
+	case VIDEO_FORMAT_I40A:
+	case VIDEO_FORMAT_I42A:
+	case VIDEO_FORMAT_YUVA:
+	case VIDEO_FORMAT_AYUV:
 		return GS_BGRA;
-
-	return GS_BGRX;
+	default:
+		return GS_BGRX;
+	}
 }
 
 extern void obs_source_activate(obs_source_t *source, enum view_type type);

libobs/obs-source.c

@@ -1334,10 +1334,14 @@ enum convert_type {
 	CONVERT_NONE,
 	CONVERT_NV12,
 	CONVERT_420,
+	CONVERT_420_A,
 	CONVERT_422,
-	CONVERT_422_U,
+	CONVERT_422_A,
+	CONVERT_422_PACK,
 	CONVERT_422_Y,
 	CONVERT_444,
+	CONVERT_444_A,
+	CONVERT_444_A_PACK,
 	CONVERT_800,
 	CONVERT_RGB_LIMITED,
 	CONVERT_BGR3,
@@ -1358,9 +1362,8 @@ static inline enum convert_type get_convert_type(enum video_format format,
 
 	case VIDEO_FORMAT_YVYU:
 	case VIDEO_FORMAT_YUY2:
-		return CONVERT_422_Y;
 	case VIDEO_FORMAT_UYVY:
-		return CONVERT_422_U;
+		return CONVERT_422_PACK;
 
 	case VIDEO_FORMAT_Y800:
 		return CONVERT_800;
@@ -1373,6 +1376,18 @@ static inline enum convert_type get_convert_type(enum video_format format,
 
 	case VIDEO_FORMAT_BGR3:
 		return CONVERT_BGR3;
+
+	case VIDEO_FORMAT_I40A:
+		return CONVERT_420_A;
+
+	case VIDEO_FORMAT_I42A:
+		return CONVERT_422_A;
+
+	case VIDEO_FORMAT_YUVA:
+		return CONVERT_444_A;
+
+	case VIDEO_FORMAT_AYUV:
+		return CONVERT_444_A_PACK;
 	}
 
 	return CONVERT_NONE;
@@ -1388,6 +1403,17 @@ static inline bool set_packed422_sizes(struct obs_source *source,
 	return true;
 }
 
+static inline bool
+set_packed444_alpha_sizes(struct obs_source *source,
+			  const struct obs_source_frame *frame)
+{
+	source->async_convert_width[0] = frame->width;
+	source->async_convert_height[0] = frame->height;
+	source->async_texture_formats[0] = GS_BGRA;
+	source->async_channel_count = 1;
+	return true;
+}
+
 static inline bool set_planar444_sizes(struct obs_source *source,
 				       const struct obs_source_frame *frame)
 {
@@ -1404,6 +1430,26 @@ static inline bool set_planar444_sizes(struct obs_source *source,
 	return true;
 }
 
+static inline bool
+set_planar444_alpha_sizes(struct obs_source *source,
+			  const struct obs_source_frame *frame)
+{
+	source->async_convert_width[0] = frame->width;
+	source->async_convert_width[1] = frame->width;
+	source->async_convert_width[2] = frame->width;
+	source->async_convert_width[3] = frame->width;
+	source->async_convert_height[0] = frame->height;
+	source->async_convert_height[1] = frame->height;
+	source->async_convert_height[2] = frame->height;
+	source->async_convert_height[3] = frame->height;
+	source->async_texture_formats[0] = GS_R8;
+	source->async_texture_formats[1] = GS_R8;
+	source->async_texture_formats[2] = GS_R8;
+	source->async_texture_formats[3] = GS_R8;
+	source->async_channel_count = 4;
+	return true;
+}
+
 static inline bool set_planar420_sizes(struct obs_source *source,
 				       const struct obs_source_frame *frame)
 {
@@ -1420,6 +1466,26 @@ static inline bool set_planar420_sizes(struct obs_source *source,
 	return true;
 }
 
+static inline bool
+set_planar420_alpha_sizes(struct obs_source *source,
+			  const struct obs_source_frame *frame)
+{
+	source->async_convert_width[0] = frame->width;
+	source->async_convert_width[1] = frame->width / 2;
+	source->async_convert_width[2] = frame->width / 2;
+	source->async_convert_width[3] = frame->width;
+	source->async_convert_height[0] = frame->height;
+	source->async_convert_height[1] = frame->height / 2;
+	source->async_convert_height[2] = frame->height / 2;
+	source->async_convert_height[3] = frame->height;
+	source->async_texture_formats[0] = GS_R8;
+	source->async_texture_formats[1] = GS_R8;
+	source->async_texture_formats[2] = GS_R8;
+	source->async_texture_formats[3] = GS_R8;
+	source->async_channel_count = 4;
+	return true;
+}
+
 static inline bool set_planar422_sizes(struct obs_source *source,
 				       const struct obs_source_frame *frame)
 {
@@ -1436,6 +1502,26 @@ static inline bool set_planar422_sizes(struct obs_source *source,
 	return true;
 }
 
+static inline bool
+set_planar422_alpha_sizes(struct obs_source *source,
+			  const struct obs_source_frame *frame)
+{
+	source->async_convert_width[0] = frame->width;
+	source->async_convert_width[1] = frame->width / 2;
+	source->async_convert_width[2] = frame->width / 2;
+	source->async_convert_width[3] = frame->width;
+	source->async_convert_height[0] = frame->height;
+	source->async_convert_height[1] = frame->height;
+	source->async_convert_height[2] = frame->height;
+	source->async_convert_height[3] = frame->height;
+	source->async_texture_formats[0] = GS_R8;
+	source->async_texture_formats[1] = GS_R8;
+	source->async_texture_formats[2] = GS_R8;
+	source->async_texture_formats[3] = GS_R8;
+	source->async_channel_count = 4;
+	return true;
+}
+
 static inline bool set_nv12_sizes(struct obs_source *source,
 				  const struct obs_source_frame *frame)
 {
@@ -1483,8 +1569,7 @@ static inline bool init_gpu_conversion(struct obs_source *source,
 				       const struct obs_source_frame *frame)
 {
 	switch (get_convert_type(frame->format, frame->full_range)) {
-	case CONVERT_422_Y:
-	case CONVERT_422_U:
+	case CONVERT_422_PACK:
 		return set_packed422_sizes(source, frame);
 
 	case CONVERT_420:
@@ -1508,6 +1593,18 @@ static inline bool init_gpu_conversion(struct obs_source *source,
 	case CONVERT_BGR3:
 		return set_bgr3_sizes(source, frame);
 
+	case CONVERT_420_A:
+		return set_planar420_alpha_sizes(source, frame);
+
+	case CONVERT_422_A:
+		return set_planar422_alpha_sizes(source, frame);
+
+	case CONVERT_444_A:
+		return set_planar444_alpha_sizes(source, frame);
+
+	case CONVERT_444_A_PACK:
+		return set_packed444_alpha_sizes(source, frame);
+
 	case CONVERT_NONE:
 		assert(false && "No conversion requested");
 		break;
@@ -1578,8 +1675,7 @@ static void upload_raw_frame(gs_texture_t *tex[MAX_AV_PLANES],
 			     const struct obs_source_frame *frame)
 {
 	switch (get_convert_type(frame->format, frame->full_range)) {
-	case CONVERT_422_U:
-	case CONVERT_422_Y:
+	case CONVERT_422_PACK:
 	case CONVERT_800:
 	case CONVERT_RGB_LIMITED:
 	case CONVERT_BGR3:
@@ -1587,6 +1683,10 @@ static void upload_raw_frame(gs_texture_t *tex[MAX_AV_PLANES],
 	case CONVERT_422:
 	case CONVERT_NV12:
 	case CONVERT_444:
+	case CONVERT_420_A:
+	case CONVERT_422_A:
+	case CONVERT_444_A:
+	case CONVERT_444_A_PACK:
 		for (size_t c = 0; c < MAX_AV_PLANES; c++) {
 			if (tex[c])
 				gs_texture_set_image(tex[c], frame->data[c],
@@ -1631,6 +1731,18 @@ static const char *select_conversion_technique(enum video_format format,
 	case VIDEO_FORMAT_I422:
 		return "I422_Reverse";
 
+	case VIDEO_FORMAT_I40A:
+		return "I40A_Reverse";
+
+	case VIDEO_FORMAT_I42A:
+		return "I42A_Reverse";
+
+	case VIDEO_FORMAT_YUVA:
+		return "YUVA_Reverse";
+
+	case VIDEO_FORMAT_AYUV:
+		return "AYUV_Reverse";
+
 	case VIDEO_FORMAT_BGRA:
 	case VIDEO_FORMAT_BGRX:
 	case VIDEO_FORMAT_RGBA:
@@ -1695,6 +1807,10 @@ static bool update_async_texrender(struct obs_source *source,
 			gs_effect_set_texture(
 				gs_effect_get_param_by_name(conv, "image2"),
 				tex[2]);
+		if (tex[3])
+			gs_effect_set_texture(
+				gs_effect_get_param_by_name(conv, "image3"),
+				tex[3]);
 		set_eparam(conv, "width", (float)cx);
 		set_eparam(conv, "height", (float)cy);
 		set_eparam(conv, "width_d2", (float)cx * 0.5f);
@@ -2389,8 +2505,24 @@ static void copy_frame_data(struct obs_source_frame *dst,
 	case VIDEO_FORMAT_BGRX:
 	case VIDEO_FORMAT_Y800:
 	case VIDEO_FORMAT_BGR3:
+	case VIDEO_FORMAT_AYUV:
 		copy_frame_data_plane(dst, src, 0, dst->height);
 		break;
+
+	case VIDEO_FORMAT_I40A:
+		copy_frame_data_plane(dst, src, 0, dst->height);
+		copy_frame_data_plane(dst, src, 1, dst->height / 2);
+		copy_frame_data_plane(dst, src, 2, dst->height / 2);
+		copy_frame_data_plane(dst, src, 3, dst->height);
+		break;
+
+	case VIDEO_FORMAT_I42A:
+	case VIDEO_FORMAT_YUVA:
+		copy_frame_data_plane(dst, src, 0, dst->height);
+		copy_frame_data_plane(dst, src, 1, dst->height);
+		copy_frame_data_plane(dst, src, 2, dst->height);
+		copy_frame_data_plane(dst, src, 3, dst->height);
+		break;
 	}
 }