obs-transitions: Convert premultiplied alpha to straight

In transitions, because the 'to' and 'from' are always rendered to
textures, the end result will always have premultiplied alpha.  This
would cause alpha to have blackish edges during transition, and cause
semi-transparent images to appear darker than they were supposed to.

To replicate, simply set the transparency of a source to 50%, then
transition between that scene and another scene.  The source will appear
to "pop" in and out unnaturally due to the premultiplied alpha effect of
the render targets.

To fix this, the solution is to simply convert premultiplied alpha to
straight alpha in the transition pixel shaders.

plugins/obs-transitions/data/fade_to_color_transition.effect

@@ -14,6 +14,8 @@ struct VertData {
 	float2 uv  : TEXCOORD0;
 };
 
+#include "premultiplied.inc"
+
 VertData VSDefault(VertData v_in)
 {
 	VertData vert_out;
@@ -24,7 +26,7 @@ VertData VSDefault(VertData v_in)
 
 float4 PSFadeToColor(VertData v_in) : TARGET
 {
-	return lerp(tex.Sample(textureSampler, v_in.uv), color, swp);
+	return lerp(convert_pmalpha(tex.Sample(textureSampler, v_in.uv)), color, swp);
 }
 
 technique FadeToColor

plugins/obs-transitions/data/fade_transition.effect

@@ -14,6 +14,8 @@ struct VertData {
 	float2 uv  : TEXCOORD0;
 };
 
+#include "premultiplied.inc"
+
 VertData VSDefault(VertData v_in)
 {
 	VertData vert_out;
@@ -24,8 +26,8 @@ VertData VSDefault(VertData v_in)
 
 float4 PSFade(VertData v_in) : TARGET
 {
-	float4 a_val = tex_a.Sample(textureSampler, v_in.uv);
-	float4 b_val = tex_b.Sample(textureSampler, v_in.uv);
+	float4 a_val = convert_pmalpha(tex_a.Sample(textureSampler, v_in.uv));
+	float4 b_val = convert_pmalpha(tex_b.Sample(textureSampler, v_in.uv));
 	return lerp(a_val, b_val, fade_val);
 }
 

plugins/obs-transitions/data/luma_wipe_transition.effect

@@ -20,6 +20,8 @@ struct VertData {
 	float2 uv  : TEXCOORD0;
 };
 
+#include "premultiplied.inc"
+
 VertData VSDefault(VertData v_in)
 {
 	VertData vert_out;
@@ -31,8 +33,8 @@ VertData VSDefault(VertData v_in)
 float4 PSLumaWipe(VertData v_in) : TARGET
 {
 	float2 uv = v_in.uv;
-	float4 a_color = a_tex.Sample(textureSampler, uv);
-	float4 b_color = b_tex.Sample(textureSampler, uv);
+	float4 a_color = convert_pmalpha(a_tex.Sample(textureSampler, uv));
+	float4 b_color = convert_pmalpha(b_tex.Sample(textureSampler, uv));
 	float luma     = l_tex.Sample(textureSampler, uv).x;
 
 	if (invert)

plugins/obs-transitions/data/premultiplied.inc

@@ -0,0 +1,9 @@
+float4 convert_pmalpha(float4 color)
+{
+	float4 ret = color;
+	if (color.a >= 0.001)
+		ret.xyz /= color.a;
+	else
+		ret = float4(0.0, 0.0, 0.0, 0.0);
+	return ret;
+}

plugins/obs-transitions/data/slide_transition.effect

@@ -16,6 +16,8 @@ struct VertData {
 	float2 uv  : TEXCOORD0;
 };
 
+#include "premultiplied.inc"
+
 VertData VSDefault(VertData v_in)
 {
 	VertData vert_out;
@@ -28,11 +30,14 @@ float4 PSSlide(VertData v_in) : TARGET
 {
 	float2 tex_a_uv = v_in.uv + tex_a_dir;
 	float2 tex_b_uv = v_in.uv - tex_b_dir;
+	float4 outc;
 
-	return (tex_a_uv.x - saturate(tex_a_uv.x) != 0.0) ||
+	outc = (tex_a_uv.x - saturate(tex_a_uv.x) != 0.0) ||
 	       (tex_a_uv.y - saturate(tex_a_uv.y) != 0.0)
 		   ? tex_b.Sample(textureSampler, tex_b_uv)
 		   : tex_a.Sample(textureSampler, tex_a_uv);
+
+	return convert_pmalpha(outc);
 }
 
 technique Slide

plugins/obs-transitions/data/swipe_transition.effect

@@ -14,6 +14,8 @@ struct VertData {
 	float2 uv  : TEXCOORD0;
 };
 
+#include "premultiplied.inc"
+
 VertData VSDefault(VertData v_in)
 {
 	VertData vert_out;
@@ -25,11 +27,14 @@ VertData VSDefault(VertData v_in)
 float4 PSSwipe(VertData v_in) : TARGET
 {
 	float2 swipe_uv = v_in.uv + swipe_val;
+	float4 outc;
 
-	return (swipe_uv.x - saturate(swipe_uv.x) != 0.0) ||
+	outc = (swipe_uv.x - saturate(swipe_uv.x) != 0.0) ||
 	       (swipe_uv.y - saturate(swipe_uv.y) != 0.0)
 		   ? tex_b.Sample(textureSampler, v_in.uv)
 		   : tex_a.Sample(textureSampler, swipe_uv);
+
+	return convert_pmalpha(outc);
 }
 
 technique Swipe