Merge pull request #1852 from jpark37/blend-alpha-attempt-two

libobs: libobs-d3d11: Fix alpha blend

libobs-d3d11/d3d11-subsystem.hpp

@@ -659,7 +659,7 @@ struct BlendState {
 		  srcFactorC   (GS_BLEND_SRCALPHA),
 		  destFactorC  (GS_BLEND_INVSRCALPHA),
 		  srcFactorA   (GS_BLEND_ONE),
-		  destFactorA  (GS_BLEND_ONE),
+		  destFactorA  (GS_BLEND_INVSRCALPHA),
 		  redEnabled   (true),
 		  greenEnabled (true),
 		  blueEnabled  (true),

libobs/data/area.effect

@@ -47,11 +47,11 @@ float4 PSDrawAreaRGBA(VertInOut vert_in) : TARGET
 			float2 targetmax = min(potentialtargetmax, targetposmax);
 			float area = (targetmax.x - targetmin.x) * (targetmax.y - targetmin.y);
 			float4 sample = image.Load(int3(loadindex, 0));
-			totalcolor += area * float4(sample.rgb * sample.a, sample.a);
+			totalcolor += area * sample;
 		}
 	}
 
-	return float4(totalcolor.rgb / totalcolor.a, totalcolor.a);
+	return totalcolor;
 }
 
 technique Draw

libobs/data/bicubic_scale.effect

@@ -130,6 +130,14 @@ float4 PSDrawBicubicRGBA(VertData v_in, bool undistort) : TARGET
 	return DrawBicubic(v_in, undistort);
 }
 
+float4 PSDrawBicubicRGBADivide(VertData v_in) : TARGET
+{
+	float4 rgba = DrawBicubic(v_in, false);
+	float alpha = rgba.a;
+	float multiplier = (alpha > 0.0) ? (1.0 / alpha) : 0.0;
+	return float4(rgba.rgb * multiplier, alpha);
+}
+
 float4 PSDrawBicubicMatrix(VertData v_in) : TARGET
 {
 	float3 rgb = DrawBicubic(v_in, false).rgb;
@@ -146,6 +154,15 @@ technique Draw
 	}
 }
 
+technique DrawAlphaDivide
+{
+	pass
+	{
+		vertex_shader = VSDefault(v_in);
+		pixel_shader  = PSDrawBicubicRGBADivide(v_in);
+	}
+}
+
 technique DrawUndistort
 {
 	pass

libobs/data/bilinear_lowres_scale.effect

@@ -54,6 +54,14 @@ float4 PSDrawLowresBilinearRGBA(VertData v_in) : TARGET
 	return DrawLowresBilinear(v_in);
 }
 
+float4 PSDrawLowresBilinearRGBADivide(VertData v_in) : TARGET
+{
+	float4 rgba = DrawLowresBilinear(v_in);
+	float alpha = rgba.a;
+	float multiplier = (alpha > 0.0) ? (1.0 / alpha) : 0.0;
+	return float4(rgba.rgb * multiplier, alpha);
+}
+
 float4 PSDrawLowresBilinearMatrix(VertData v_in) : TARGET
 {
 	float3 rgb = DrawLowresBilinear(v_in).rgb;
@@ -70,6 +78,15 @@ technique Draw
 	}
 }
 
+technique DrawAlphaDivide
+{
+	pass
+	{
+		vertex_shader = VSDefault(v_in);
+		pixel_shader  = PSDrawLowresBilinearRGBADivide(v_in);
+	}
+}
+
 technique DrawMatrix
 {
 	pass

libobs/data/default.effect

@@ -26,6 +26,14 @@ float4 PSDrawBare(VertInOut vert_in) : TARGET
 	return image.Sample(def_sampler, vert_in.uv);
 }
 
+float4 PSDrawAlphaDivide(VertInOut vert_in) : TARGET
+{
+	float4 rgba = image.Sample(def_sampler, vert_in.uv);
+	float alpha = rgba.a;
+	float multiplier = (alpha > 0.0) ? (1.0 / alpha) : 0.0;
+	return float4(rgba.rgb * multiplier, alpha);
+}
+
 float4 PSDrawMatrix(VertInOut vert_in) : TARGET
 {
 	float3 rgb = image.Sample(def_sampler, vert_in.uv).rgb;
@@ -42,6 +50,15 @@ technique Draw
 	}
 }
 
+technique DrawAlphaDivide
+{
+	pass
+	{
+		vertex_shader = VSDefault(vert_in);
+		pixel_shader  = PSDrawAlphaDivide(vert_in);
+	}
+}
+
 technique DrawMatrix
 {
 	pass

libobs/data/lanczos_scale.effect

@@ -138,6 +138,14 @@ float4 PSDrawLanczosRGBA(FragData v_in, bool undistort) : TARGET
 	return DrawLanczos(v_in, undistort);
 }
 
+float4 PSDrawLanczosRGBADivide(FragData v_in) : TARGET
+{
+	float4 rgba = DrawLanczos(v_in, false);
+	float alpha = rgba.a;
+	float multiplier = (alpha > 0.0) ? (1.0 / alpha) : 0.0;
+	return float4(rgba.rgb * multiplier, alpha);
+}
+
 float4 PSDrawLanczosMatrix(FragData v_in) : TARGET
 {
 	float3 rgb = DrawLanczos(v_in, false).rgb;
@@ -154,6 +162,15 @@ technique Draw
 	}
 }
 
+technique DrawAlphaDivide
+{
+	pass
+	{
+		vertex_shader = VSDefault(v_in);
+		pixel_shader  = PSDrawLanczosRGBADivide(v_in);
+	}
+}
+
 technique DrawUndistort
 {
 	pass

libobs/graphics/graphics.c

@@ -160,12 +160,12 @@ static bool graphics_init(struct graphics_subsystem *graphics)
 
 	graphics->exports.device_blend_function_separate(graphics->device,
 			GS_BLEND_SRCALPHA, GS_BLEND_INVSRCALPHA,
-			GS_BLEND_ONE, GS_BLEND_ONE);
+			GS_BLEND_ONE, GS_BLEND_INVSRCALPHA);
 	graphics->cur_blend_state.enabled = true;
 	graphics->cur_blend_state.src_c   = GS_BLEND_SRCALPHA;
 	graphics->cur_blend_state.dest_c  = GS_BLEND_INVSRCALPHA;
 	graphics->cur_blend_state.src_a   = GS_BLEND_ONE;
-	graphics->cur_blend_state.dest_a  = GS_BLEND_ONE;
+	graphics->cur_blend_state.dest_a  = GS_BLEND_INVSRCALPHA;
 
 	graphics->exports.device_leave_context(graphics->device);
 
@@ -1240,10 +1240,10 @@ void gs_reset_blend_state(void)
 	if (graphics->cur_blend_state.src_c  != GS_BLEND_SRCALPHA ||
 	    graphics->cur_blend_state.dest_c != GS_BLEND_INVSRCALPHA ||
 	    graphics->cur_blend_state.src_a  != GS_BLEND_ONE ||
-	    graphics->cur_blend_state.dest_a != GS_BLEND_ONE)
+	    graphics->cur_blend_state.dest_a != GS_BLEND_INVSRCALPHA)
 		gs_blend_function_separate(
 				GS_BLEND_SRCALPHA, GS_BLEND_INVSRCALPHA,
-				GS_BLEND_ONE, GS_BLEND_ONE);
+				GS_BLEND_ONE, GS_BLEND_INVSRCALPHA);
 }
 
 /* ------------------------------------------------------------------------- */

libobs/obs-scene.c

@@ -507,9 +507,14 @@ static void render_item_texture(struct obs_scene_item *item)
 		}
 	}
 
+	gs_blend_state_push();
+	gs_blend_function(GS_BLEND_ONE, GS_BLEND_INVSRCALPHA);
+
 	while (gs_effect_loop(effect, "Draw"))
 		obs_source_draw(tex, 0, 0, 0, 0, 0);
 
+	gs_blend_state_pop();
+
 	GS_DEBUG_MARKER_END();
 }
 
@@ -545,10 +550,8 @@ static inline void render_item(struct obs_scene_item *item)
 					-(float)item->crop.top,
 					0.0f);
 
-			gs_blend_state_push();
-			gs_blend_function(GS_BLEND_ONE, GS_BLEND_ZERO);
 			obs_source_video_render(item->source);
-			gs_blend_state_pop();
+
 			gs_texrender_end(item->item_render);
 		}
 	}

libobs/obs-source-transition.c

@@ -718,10 +718,16 @@ void obs_transition_video_render(obs_source_t *transition,
 
 		cx = get_cx(transition);
 		cy = get_cy(transition);
-		if (cx && cy)
+		if (cx && cy) {
+			gs_blend_state_push();
+			gs_blend_function(GS_BLEND_ONE, GS_BLEND_INVSRCALPHA);
+
 			callback(transition->context.data, tex[0], tex[1], t,
 					cx, cy);
 
+			gs_blend_state_pop();
+		}
+
 	} else if (state.transitioning_audio) {
 		if (state.s[1]) {
 			gs_matrix_push();

libobs/obs-video.c

@@ -218,7 +218,7 @@ static inline void render_output_texture(struct obs_core_video *video,
 	gs_technique_t *tech;
 
 	if (video->ovi.output_format == VIDEO_FORMAT_RGBA) {
-		tech = gs_effect_get_technique(effect, "Draw");
+		tech = gs_effect_get_technique(effect, "DrawAlphaDivide");
 	} else {
 		tech = gs_effect_get_technique(effect, "DrawMatrix");
 	}

libobs/obs.c

@@ -1651,8 +1651,13 @@ void obs_render_main_texture(void)
 	param = gs_effect_get_param_by_name(effect, "image");
 	gs_effect_set_texture(param, tex);
 
+	gs_blend_state_push();
+	gs_blend_function(GS_BLEND_ONE, GS_BLEND_INVSRCALPHA);
+
 	while (gs_effect_loop(effect, "Draw"))
 		gs_draw_sprite(tex, 0, 0, 0);
+
+	gs_blend_state_pop();
 }
 
 gs_texture_t *obs_get_main_texture(void)

plugins/obs-transitions/data/fade_to_color_transition.effect

@@ -14,8 +14,6 @@ struct VertData {
 	float2 uv  : TEXCOORD0;
 };
 
-#include "premultiplied.inc"
-
 VertData VSDefault(VertData v_in)
 {
 	VertData vert_out;
@@ -26,7 +24,8 @@ VertData VSDefault(VertData v_in)
 
 float4 PSFadeToColor(VertData v_in) : TARGET
 {
-	return lerp(convert_pmalpha(tex.Sample(textureSampler, v_in.uv)), color, swp);
+	float4 premultiplied = float4(color.rgb * color.a, color.a);
+	return lerp(tex.Sample(textureSampler, v_in.uv), premultiplied, swp);
 }
 
 technique FadeToColor

plugins/obs-transitions/data/fade_transition.effect

@@ -14,8 +14,6 @@ struct VertData {
 	float2 uv  : TEXCOORD0;
 };
 
-#include "premultiplied.inc"
-
 VertData VSDefault(VertData v_in)
 {
 	VertData vert_out;
@@ -26,8 +24,8 @@ VertData VSDefault(VertData v_in)
 
 float4 PSFade(VertData v_in) : TARGET
 {
-	float4 a_val = convert_pmalpha(tex_a.Sample(textureSampler, v_in.uv));
-	float4 b_val = convert_pmalpha(tex_b.Sample(textureSampler, v_in.uv));
+	float4 a_val = tex_a.Sample(textureSampler, v_in.uv);
+	float4 b_val = tex_b.Sample(textureSampler, v_in.uv);
 	return lerp(a_val, b_val, fade_val);
 }
 

plugins/obs-transitions/data/luma_wipe_transition.effect

@@ -20,8 +20,6 @@ struct VertData {
 	float2 uv  : TEXCOORD0;
 };
 
-#include "premultiplied.inc"
-
 VertData VSDefault(VertData v_in)
 {
 	VertData vert_out;
@@ -33,8 +31,8 @@ VertData VSDefault(VertData v_in)
 float4 PSLumaWipe(VertData v_in) : TARGET
 {
 	float2 uv = v_in.uv;
-	float4 a_color = convert_pmalpha(a_tex.Sample(textureSampler, uv));
-	float4 b_color = convert_pmalpha(b_tex.Sample(textureSampler, uv));
+	float4 a_color = a_tex.Sample(textureSampler, uv);
+	float4 b_color = b_tex.Sample(textureSampler, uv);
 	float luma     = l_tex.Sample(textureSampler, uv).x;
 
 	if (invert)

plugins/obs-transitions/data/premultiplied.inc

@@ -1,9 +0,0 @@
-float4 convert_pmalpha(float4 color)
-{
-	float4 ret = color;
-	if (color.a >= 0.001)
-		ret.xyz /= color.a;
-	else
-		ret = float4(0.0, 0.0, 0.0, 0.0);
-	return ret;
-}

plugins/obs-transitions/data/slide_transition.effect

@@ -16,8 +16,6 @@ struct VertData {
 	float2 uv  : TEXCOORD0;
 };
 
-#include "premultiplied.inc"
-
 VertData VSDefault(VertData v_in)
 {
 	VertData vert_out;
@@ -37,7 +35,7 @@ float4 PSSlide(VertData v_in) : TARGET
 		   ? tex_b.Sample(textureSampler, tex_b_uv)
 		   : tex_a.Sample(textureSampler, tex_a_uv);
 
-	return convert_pmalpha(outc);
+	return outc;
 }
 
 technique Slide

plugins/obs-transitions/data/swipe_transition.effect

@@ -14,8 +14,6 @@ struct VertData {
 	float2 uv  : TEXCOORD0;
 };
 
-#include "premultiplied.inc"
-
 VertData VSDefault(VertData v_in)
 {
 	VertData vert_out;
@@ -34,7 +32,7 @@ float4 PSSwipe(VertData v_in) : TARGET
 		   ? tex_b.Sample(textureSampler, v_in.uv)
 		   : tex_a.Sample(textureSampler, swipe_uv);
 
-	return convert_pmalpha(outc);
+	return outc;
 }
 
 technique Swipe