libobs: Fix various alpha issues

There are cases where alpha is multiplied unnecessarily. This change
attempts to use premultiplied alpha blending for composition.

To keep this change simple, The filter chain will continue to use
straight alpha. Otherwise, every source would need to modified to output
premultiplied, and every filter modified for premultiplied input.

"DrawAlphaDivide" shader techniques have been added to convert from
premultiplied alpha to straight alpha for final output. "DrawMatrix"
techniques ignore alpha, so they do not appear to need changing.

One remaining issue is that scale effects are set up here to use the
same shader logic for both scale filters (straight alpha - incorrectly),
and output composition (premultiplied alpha - correctly). A fix could be
made to add additional shaders for straight alpha, but the "real" fix
may be to eliminate the straight alpha path at some point.

For graphics, SrcBlendAlpha and DestBlendAlpha were both ONE, and could
combine together to form alpha values greater than one. This is not as
noticeable of a problem for UNORM targets because the channels are
clamped, but it will likely become a problem in more situations if FLOAT
targets are used.

This change switches DestBlendAlpha to INVSRCALPHA. The blending
behavior of stacked transparents is preserved without overflowing the
alpha channel.

obs-transitions: Use premultiplied alpha blend, and simplify shaders
because both inputs and outputs use premultiplied alpha now.

Fixes https://obsproject.com/mantis/view.php?id=1108

libobs-d3d11/d3d11-subsystem.hpp

@@ -659,7 +659,7 @@ struct BlendState {
 		  srcFactorC   (GS_BLEND_SRCALPHA),
 		  destFactorC  (GS_BLEND_INVSRCALPHA),
 		  srcFactorA   (GS_BLEND_ONE),
-		  destFactorA  (GS_BLEND_ONE),
+		  destFactorA  (GS_BLEND_INVSRCALPHA),
 		  redEnabled   (true),
 		  greenEnabled (true),
 		  blueEnabled  (true),

libobs/data/area.effect

@@ -47,11 +47,11 @@ float4 PSDrawAreaRGBA(VertInOut vert_in) : TARGET
 			float2 targetmax = min(potentialtargetmax, targetposmax);
 			float area = (targetmax.x - targetmin.x) * (targetmax.y - targetmin.y);
 			float4 sample = image.Load(int3(loadindex, 0));
-			totalcolor += area * float4(sample.rgb * sample.a, sample.a);
+			totalcolor += area * sample;
 		}
 	}
 
-	return float4(totalcolor.rgb / totalcolor.a, totalcolor.a);
+	return totalcolor;
 }
 
 technique Draw

libobs/data/bicubic_scale.effect

@@ -130,6 +130,14 @@ float4 PSDrawBicubicRGBA(VertData v_in, bool undistort) : TARGET
 	return DrawBicubic(v_in, undistort);
 }
 
+float4 PSDrawBicubicRGBADivide(VertData v_in) : TARGET
+{
+	float4 rgba = DrawBicubic(v_in, false);
+	float alpha = rgba.a;
+	float multiplier = (alpha > 0.0) ? (1.0 / alpha) : 0.0;
+	return float4(rgba.rgb * multiplier, alpha);
+}
+
 float4 PSDrawBicubicMatrix(VertData v_in) : TARGET
 {
 	float3 rgb = DrawBicubic(v_in, false).rgb;
@@ -146,6 +154,15 @@ technique Draw
 	}
 }
 
+technique DrawAlphaDivide
+{
+	pass
+	{
+		vertex_shader = VSDefault(v_in);
+		pixel_shader  = PSDrawBicubicRGBADivide(v_in);
+	}
+}
+
 technique DrawUndistort
 {
 	pass

libobs/data/bilinear_lowres_scale.effect

@@ -54,6 +54,14 @@ float4 PSDrawLowresBilinearRGBA(VertData v_in) : TARGET
 	return DrawLowresBilinear(v_in);
 }
 
+float4 PSDrawLowresBilinearRGBADivide(VertData v_in) : TARGET
+{
+	float4 rgba = DrawLowresBilinear(v_in);
+	float alpha = rgba.a;
+	float multiplier = (alpha > 0.0) ? (1.0 / alpha) : 0.0;
+	return float4(rgba.rgb * multiplier, alpha);
+}
+
 float4 PSDrawLowresBilinearMatrix(VertData v_in) : TARGET
 {
 	float3 rgb = DrawLowresBilinear(v_in).rgb;
@@ -70,6 +78,15 @@ technique Draw
 	}
 }
 
+technique DrawAlphaDivide
+{
+	pass
+	{
+		vertex_shader = VSDefault(v_in);
+		pixel_shader  = PSDrawLowresBilinearRGBADivide(v_in);
+	}
+}
+
 technique DrawMatrix
 {
 	pass

libobs/data/default.effect

@@ -26,6 +26,14 @@ float4 PSDrawBare(VertInOut vert_in) : TARGET
 	return image.Sample(def_sampler, vert_in.uv);
 }
 
+float4 PSDrawAlphaDivide(VertInOut vert_in) : TARGET
+{
+	float4 rgba = image.Sample(def_sampler, vert_in.uv);
+	float alpha = rgba.a;
+	float multiplier = (alpha > 0.0) ? (1.0 / alpha) : 0.0;
+	return float4(rgba.rgb * multiplier, alpha);
+}
+
 float4 PSDrawMatrix(VertInOut vert_in) : TARGET
 {
 	float3 rgb = image.Sample(def_sampler, vert_in.uv).rgb;
@@ -42,6 +50,15 @@ technique Draw
 	}
 }
 
+technique DrawAlphaDivide
+{
+	pass
+	{
+		vertex_shader = VSDefault(vert_in);
+		pixel_shader  = PSDrawAlphaDivide(vert_in);
+	}
+}
+
 technique DrawMatrix
 {
 	pass

libobs/data/lanczos_scale.effect

@@ -138,6 +138,14 @@ float4 PSDrawLanczosRGBA(FragData v_in, bool undistort) : TARGET
 	return DrawLanczos(v_in, undistort);
 }
 
+float4 PSDrawLanczosRGBADivide(FragData v_in) : TARGET
+{
+	float4 rgba = DrawLanczos(v_in, false);
+	float alpha = rgba.a;
+	float multiplier = (alpha > 0.0) ? (1.0 / alpha) : 0.0;
+	return float4(rgba.rgb * multiplier, alpha);
+}
+
 float4 PSDrawLanczosMatrix(FragData v_in) : TARGET
 {
 	float3 rgb = DrawLanczos(v_in, false).rgb;
@@ -154,6 +162,15 @@ technique Draw
 	}
 }
 
+technique DrawAlphaDivide
+{
+	pass
+	{
+		vertex_shader = VSDefault(v_in);
+		pixel_shader  = PSDrawLanczosRGBADivide(v_in);
+	}
+}
+
 technique DrawUndistort
 {
 	pass

libobs/graphics/graphics.c

@@ -160,12 +160,12 @@ static bool graphics_init(struct graphics_subsystem *graphics)
 
 	graphics->exports.device_blend_function_separate(graphics->device,
 			GS_BLEND_SRCALPHA, GS_BLEND_INVSRCALPHA,
-			GS_BLEND_ONE, GS_BLEND_ONE);
+			GS_BLEND_ONE, GS_BLEND_INVSRCALPHA);
 	graphics->cur_blend_state.enabled = true;
 	graphics->cur_blend_state.src_c   = GS_BLEND_SRCALPHA;
 	graphics->cur_blend_state.dest_c  = GS_BLEND_INVSRCALPHA;
 	graphics->cur_blend_state.src_a   = GS_BLEND_ONE;
-	graphics->cur_blend_state.dest_a  = GS_BLEND_ONE;
+	graphics->cur_blend_state.dest_a  = GS_BLEND_INVSRCALPHA;
 
 	graphics->exports.device_leave_context(graphics->device);
 
@@ -1240,10 +1240,10 @@ void gs_reset_blend_state(void)
 	if (graphics->cur_blend_state.src_c  != GS_BLEND_SRCALPHA ||
 	    graphics->cur_blend_state.dest_c != GS_BLEND_INVSRCALPHA ||
 	    graphics->cur_blend_state.src_a  != GS_BLEND_ONE ||
-	    graphics->cur_blend_state.dest_a != GS_BLEND_ONE)
+	    graphics->cur_blend_state.dest_a != GS_BLEND_INVSRCALPHA)
 		gs_blend_function_separate(
 				GS_BLEND_SRCALPHA, GS_BLEND_INVSRCALPHA,
-				GS_BLEND_ONE, GS_BLEND_ONE);
+				GS_BLEND_ONE, GS_BLEND_INVSRCALPHA);
 }
 
 /* ------------------------------------------------------------------------- */

libobs/obs-scene.c

@@ -507,9 +507,14 @@ static void render_item_texture(struct obs_scene_item *item)
 		}
 	}
 
+	gs_blend_state_push();
+	gs_blend_function(GS_BLEND_ONE, GS_BLEND_INVSRCALPHA);
+
 	while (gs_effect_loop(effect, "Draw"))
 		obs_source_draw(tex, 0, 0, 0, 0, 0);
 
+	gs_blend_state_pop();
+
 	GS_DEBUG_MARKER_END();
 }
 
@@ -545,10 +550,8 @@ static inline void render_item(struct obs_scene_item *item)
 					-(float)item->crop.top,
 					0.0f);
 
-			gs_blend_state_push();
-			gs_blend_function(GS_BLEND_ONE, GS_BLEND_ZERO);
 			obs_source_video_render(item->source);
-			gs_blend_state_pop();
+
 			gs_texrender_end(item->item_render);
 		}
 	}

libobs/obs-source-transition.c

@@ -718,10 +718,16 @@ void obs_transition_video_render(obs_source_t *transition,
 
 		cx = get_cx(transition);
 		cy = get_cy(transition);
-		if (cx && cy)
+		if (cx && cy) {
+			gs_blend_state_push();
+			gs_blend_function(GS_BLEND_ONE, GS_BLEND_INVSRCALPHA);
+
 			callback(transition->context.data, tex[0], tex[1], t,
 					cx, cy);
 
+			gs_blend_state_pop();
+		}
+
 	} else if (state.transitioning_audio) {
 		if (state.s[1]) {
 			gs_matrix_push();

libobs/obs-video.c

@@ -218,7 +218,7 @@ static inline void render_output_texture(struct obs_core_video *video,
 	gs_technique_t *tech;
 
 	if (video->ovi.output_format == VIDEO_FORMAT_RGBA) {
-		tech = gs_effect_get_technique(effect, "Draw");
+		tech = gs_effect_get_technique(effect, "DrawAlphaDivide");
 	} else {
 		tech = gs_effect_get_technique(effect, "DrawMatrix");
 	}

libobs/obs.c

@@ -1651,8 +1651,13 @@ void obs_render_main_texture(void)
 	param = gs_effect_get_param_by_name(effect, "image");
 	gs_effect_set_texture(param, tex);
 
+	gs_blend_state_push();
+	gs_blend_function(GS_BLEND_ONE, GS_BLEND_INVSRCALPHA);
+
 	while (gs_effect_loop(effect, "Draw"))
 		gs_draw_sprite(tex, 0, 0, 0);
+
+	gs_blend_state_pop();
 }
 
 gs_texture_t *obs_get_main_texture(void)

plugins/obs-transitions/data/fade_to_color_transition.effect

@@ -14,8 +14,6 @@ struct VertData {
 	float2 uv  : TEXCOORD0;
 };
 
-#include "premultiplied.inc"
-
 VertData VSDefault(VertData v_in)
 {
 	VertData vert_out;
@@ -26,7 +24,8 @@ VertData VSDefault(VertData v_in)
 
 float4 PSFadeToColor(VertData v_in) : TARGET
 {
-	return lerp(convert_pmalpha(tex.Sample(textureSampler, v_in.uv)), color, swp);
+	float4 premultiplied = float4(color.rgb * color.a, color.a);
+	return lerp(tex.Sample(textureSampler, v_in.uv), premultiplied, swp);
 }
 
 technique FadeToColor

plugins/obs-transitions/data/fade_transition.effect

@@ -14,8 +14,6 @@ struct VertData {
 	float2 uv  : TEXCOORD0;
 };
 
-#include "premultiplied.inc"
-
 VertData VSDefault(VertData v_in)
 {
 	VertData vert_out;
@@ -26,8 +24,8 @@ VertData VSDefault(VertData v_in)
 
 float4 PSFade(VertData v_in) : TARGET
 {
-	float4 a_val = convert_pmalpha(tex_a.Sample(textureSampler, v_in.uv));
-	float4 b_val = convert_pmalpha(tex_b.Sample(textureSampler, v_in.uv));
+	float4 a_val = tex_a.Sample(textureSampler, v_in.uv);
+	float4 b_val = tex_b.Sample(textureSampler, v_in.uv);
 	return lerp(a_val, b_val, fade_val);
 }
 

plugins/obs-transitions/data/luma_wipe_transition.effect

@@ -20,8 +20,6 @@ struct VertData {
 	float2 uv  : TEXCOORD0;
 };
 
-#include "premultiplied.inc"
-
 VertData VSDefault(VertData v_in)
 {
 	VertData vert_out;
@@ -33,8 +31,8 @@ VertData VSDefault(VertData v_in)
 float4 PSLumaWipe(VertData v_in) : TARGET
 {
 	float2 uv = v_in.uv;
-	float4 a_color = convert_pmalpha(a_tex.Sample(textureSampler, uv));
-	float4 b_color = convert_pmalpha(b_tex.Sample(textureSampler, uv));
+	float4 a_color = a_tex.Sample(textureSampler, uv);
+	float4 b_color = b_tex.Sample(textureSampler, uv);
 	float luma     = l_tex.Sample(textureSampler, uv).x;
 
 	if (invert)

plugins/obs-transitions/data/premultiplied.inc

@@ -1,9 +0,0 @@
-float4 convert_pmalpha(float4 color)
-{
-	float4 ret = color;
-	if (color.a >= 0.001)
-		ret.xyz /= color.a;
-	else
-		ret = float4(0.0, 0.0, 0.0, 0.0);
-	return ret;
-}

plugins/obs-transitions/data/slide_transition.effect

@@ -16,8 +16,6 @@ struct VertData {
 	float2 uv  : TEXCOORD0;
 };
 
-#include "premultiplied.inc"
-
 VertData VSDefault(VertData v_in)
 {
 	VertData vert_out;
@@ -37,7 +35,7 @@ float4 PSSlide(VertData v_in) : TARGET
 		   ? tex_b.Sample(textureSampler, tex_b_uv)
 		   : tex_a.Sample(textureSampler, tex_a_uv);
 
-	return convert_pmalpha(outc);
+	return outc;
 }
 
 technique Slide

plugins/obs-transitions/data/swipe_transition.effect

@@ -14,8 +14,6 @@ struct VertData {
 	float2 uv  : TEXCOORD0;
 };
 
-#include "premultiplied.inc"
-
 VertData VSDefault(VertData v_in)
 {
 	VertData vert_out;
@@ -34,7 +32,7 @@ float4 PSSwipe(VertData v_in) : TARGET
 		   ? tex_b.Sample(textureSampler, v_in.uv)
 		   : tex_a.Sample(textureSampler, swipe_uv);
 
-	return convert_pmalpha(outc);
+	return outc;
 }
 
 technique Swipe