libobs: Fix and simplify Area scale filter

It appears there's a projection flip that is applied in some situations,
like the preview pane in studio mode, and the shader math fails when
it's active causing the output color to be zero. This fixes the math for
GLSL (with a tiny redundancy penalty to HLSL), and cleans up some
unnecessary code along the way.

Use abs() to avoid zero area in case the OpenGL projection flip is
active. Also simplify the math, and remove the unnecessary sampler
state.

libobs/data/area.effect

@@ -5,12 +5,6 @@ uniform float3 color_range_max = {1.0, 1.0, 1.0};
 uniform float2 base_dimension_i;
 uniform texture2d image;
 
-sampler_state def_sampler {
-	Filter   = Linear;
-	AddressU = Clamp;
-	AddressV = Clamp;
-};
-
 struct VertInOut {
 	float4 pos : POSITION;
 	float2 uv  : TEXCOORD0;
@@ -31,6 +25,9 @@ float4 PSDrawAreaRGBA(VertInOut vert_in) : TARGET
 	float2 uv = vert_in.uv;
 	float2 uvdelta = float2(ddx(uv.x), ddy(uv.y));
 
+	// Handle potential OpenGL flip.
+	uvdelta.y = abs(uvdelta.y);
+
 	float2 uvhalfdelta = 0.5 * uvdelta;
 	float2 uvmin = uv - uvhalfdelta;
 	float2 uvmax = uv + uvhalfdelta;
@@ -39,19 +36,20 @@ float4 PSDrawAreaRGBA(VertInOut vert_in) : TARGET
 	int2 loadindexmax = int2(uvmax / base_dimension_i);
 
 	float2 targetpos = uv / uvdelta;
-	float2 targetposleft = targetpos - 0.5;
-	float2 targetposright = targetpos + 0.5;
+	float2 targetposmin = targetpos - 0.5;
+	float2 targetposmax = targetpos + 0.5;
+	float2 scale = base_dimension_i / uvdelta;
 	for (int loadindexy = loadindexmin.y; loadindexy <= loadindexmax.y; ++loadindexy)
 	{
 		for (int loadindexx = loadindexmin.x; loadindexx <= loadindexmax.x; ++loadindexx)
 		{
-			float2 loadindex = float2(loadindexx, loadindexy);
-			float2 potentialtargetmin = loadindex / uvdelta * base_dimension_i;
-			float2 potentialtargetmax = (loadindex + 1.0) / uvdelta * base_dimension_i;
-			float2 targetmin = max(potentialtargetmin, targetposleft);
-			float2 targetmax = min(potentialtargetmax, targetposright);
+			int2 loadindex = int2(loadindexx, loadindexy);
+			float2 potentialtargetmin = float2(loadindex) * scale;
+			float2 potentialtargetmax = potentialtargetmin + scale;
+			float2 targetmin = max(potentialtargetmin, targetposmin);
+			float2 targetmax = min(potentialtargetmax, targetposmax);
 			float area = (targetmax.x - targetmin.x) * (targetmax.y - targetmin.y);
-			float4 sample = image.SampleLevel(def_sampler, (loadindex + 0.5) * base_dimension_i, 0.0);
+			float4 sample = image.Load(int3(loadindex, 0));
 			totalcolor += area * float4(sample.rgb * sample.a, sample.a);
 		}
 	}
@@ -72,6 +70,9 @@ float4 PSDrawAreaMatrix(VertInOut vert_in) : TARGET
 	float2 uv = vert_in.uv;
 	float2 uvdelta = float2(ddx(uv.x), ddy(uv.y));
 
+	// Handle potential OpenGL flip.
+	uvdelta.y = abs(uvdelta.y);
+
 	float2 uvhalfdelta = 0.5 * uvdelta;
 	float2 uvmin = uv - uvhalfdelta;
 	float2 uvmax = uv + uvhalfdelta;
@@ -80,19 +81,20 @@ float4 PSDrawAreaMatrix(VertInOut vert_in) : TARGET
 	int2 loadindexmax = int2(uvmax / base_dimension_i);
 
 	float2 targetpos = uv / uvdelta;
-	float2 targetposleft = targetpos - 0.5;
-	float2 targetposright = targetpos + 0.5;
+	float2 targetposmin = targetpos - 0.5;
+	float2 targetposmax = targetpos + 0.5;
+	float2 scale = base_dimension_i / uvdelta;
 	for (int loadindexy = loadindexmin.y; loadindexy <= loadindexmax.y; ++loadindexy)
 	{
 		for (int loadindexx = loadindexmin.x; loadindexx <= loadindexmax.x; ++loadindexx)
 		{
-			float2 loadindex = float2(loadindexx, loadindexy);
-			float2 potentialtargetmin = loadindex / uvdelta * base_dimension_i;
-			float2 potentialtargetmax = (loadindex + 1.0) / uvdelta * base_dimension_i;
-			float2 targetmin = max(potentialtargetmin, targetposleft);
-			float2 targetmax = min(potentialtargetmax, targetposright);
+			int2 loadindex = int2(loadindexx, loadindexy);
+			float2 potentialtargetmin = float2(loadindex) * scale;
+			float2 potentialtargetmax = potentialtargetmin + scale;
+			float2 targetmin = max(potentialtargetmin, targetposmin);
+			float2 targetmax = min(potentialtargetmax, targetposmax);
 			float area = (targetmax.x - targetmin.x) * (targetmax.y - targetmin.y);
-			float3 yuv = image.SampleLevel(def_sampler, (loadindex + 0.5) * base_dimension_i, 0.0).xyz;
+			float3 yuv = image.Load(int3(loadindex, 0)).xyz;
 			totalcolor += area * ConvertFromYuv(yuv);
 		}
 	}

libobs/obs-scene.c

@@ -488,11 +488,6 @@ static void render_item_texture(struct obs_scene_item *item)
 				effect = obs->video.lanczos_effect;
 			} else if (type == OBS_SCALE_AREA) {
 				effect = obs->video.area_effect;
-
-				gs_eparam_t *image = gs_effect_get_param_by_name(
-					effect, "image");
-				gs_effect_set_next_sampler(image,
-					obs->video.point_sampler);
 			}
 
 			scale_param = gs_effect_get_param_by_name(effect,