libobs-opengl: Fix render targets being flipped

When render targets are used, they output to the render target inverted
due to the way that opengl works.  This fixes that issue by inverting
the projection matrix so that it renders the image upside down and
inverting the front face from counterclockwise to clockwise.

libobs-opengl/gl-subsystem.c

@@ -877,10 +877,25 @@ static inline bool can_render(const gs_device_t *device)
 static void update_viewproj_matrix(struct gs_device *device)
 {
 	struct gs_shader *vs = device->cur_vertex_shader;
+	struct matrix4 cur_proj;
+
 	gs_matrix_get(&device->cur_view);
+	matrix4_copy(&cur_proj, &device->cur_proj);
+
+	if (device->cur_fbo) {
+		cur_proj.x.y = -cur_proj.x.y;
+		cur_proj.y.y = -cur_proj.y.y;
+		cur_proj.z.y = -cur_proj.z.y;
+		cur_proj.t.y = -cur_proj.t.y;
+
+		glFrontFace(GL_CW);
+	} else {
+		glFrontFace(GL_CCW);
+	}
+
+	gl_success("glFrontFace");
 
-	matrix4_mul(&device->cur_viewproj, &device->cur_view,
-			&device->cur_proj);
+	matrix4_mul(&device->cur_viewproj, &device->cur_view, &cur_proj);
 	matrix4_transpose(&device->cur_viewproj, &device->cur_viewproj);
 
 	if (vs->viewproj)

libobs/data/format_conversion.effect

@@ -63,11 +63,7 @@ VertInOut VSDefault(VertInOut vert_in)
 
 float4 PSNV12(VertInOut vert_in) : TARGET
 {
-#ifdef _OPENGL
-	float v_mul = floor((1.0 - vert_in.uv.y) * input_height);
-#else
 	float v_mul = floor(vert_in.uv.y * input_height);
-#endif
 
 	float byte_offset = floor((v_mul + vert_in.uv.x) * width) * 4.0;
 	byte_offset += PRECISION_OFFSET;
@@ -127,11 +123,7 @@ float4 PSNV12(VertInOut vert_in) : TARGET
 
 float4 PSPlanar420(VertInOut vert_in) : TARGET
 {
-#ifdef _OPENGL
-	float v_mul = floor((1.0 - vert_in.uv.y) * input_height);
-#else
 	float v_mul = floor(vert_in.uv.y * input_height);
-#endif
 
 	float byte_offset = floor((v_mul + vert_in.uv.x) * width) * 4.0;
 	byte_offset += PRECISION_OFFSET;
@@ -202,9 +194,6 @@ float4 PSPacked422_Reverse(VertInOut vert_in, int u_pos, int v_pos,
 		int y0_pos, int y1_pos) : TARGET
 {
 	float y = vert_in.uv.y;
-#ifdef _OPENGL
-	y = 1. - y;
-#endif
 	float odd = floor(fmod(width * vert_in.uv.x + PRECISION_OFFSET, 2.0));
 	float x = floor(width_d2 * vert_in.uv.x + PRECISION_OFFSET) *
 			width_d2_i;
@@ -233,9 +222,6 @@ float4 PSPlanar420_Reverse(VertInOut vert_in) : TARGET
 {
 	float x = vert_in.uv.x;
 	float y = vert_in.uv.y;
-#ifdef _OPENGL
-	y = 1. - y;
-#endif
 	float x_offset   = floor(x * width  + PRECISION_OFFSET);
 	float y_offset   = floor(y * height + PRECISION_OFFSET);
 
@@ -258,9 +244,6 @@ float4 PSNV12_Reverse(VertInOut vert_in) : TARGET
 {
 	float x = vert_in.uv.x;
 	float y = vert_in.uv.y;
-#ifdef _OPENGL
-	y = 1. - y;
-#endif
 	float x_offset   = floor(x * width  + PRECISION_OFFSET);
 	float y_offset   = floor(y * height + PRECISION_OFFSET);