OpenGL: Use texture swizzle for BGRA/BGR/A8

On some operating systems, with specific drivers it seems that BGR/BGRA
isn't properly treated as such in certain cases.  This fix will
hopefully force the formats to be treated as BGR/BGRA when actually
rendering, which should get around the implementation-specific issue.

libobs-opengl/gl-subsystem.c

@@ -311,9 +311,28 @@ static void strip_mipmap_filter(GLint *filter)
 	*filter = GL_NEAREST;
 }
 
+static inline void apply_swizzle(struct gs_texture *tex)
+{
+	if (tex->format == GS_A8) {
+		gl_tex_param_i(tex->gl_target, GL_TEXTURE_SWIZZLE_R, GL_ALPHA);
+	} else {
+#ifdef USE_FORMAT_SWIZZLE
+		bool invert_format =
+			(tex->format == GS_BGRA || tex->format == GS_BGRX);
+
+		gl_tex_param_i(tex->gl_target, GL_TEXTURE_SWIZZLE_R,
+				invert_format ? GL_BLUE : GL_RED);
+		gl_tex_param_i(tex->gl_target, GL_TEXTURE_SWIZZLE_B,
+				invert_format ? GL_RED  : GL_BLUE);
+#else
+		gl_tex_param_i(tex->gl_target, GL_TEXTURE_SWIZZLE_R, GL_RED);
+#endif
+	}
+}
+
 static bool load_texture_sampler(texture_t tex, samplerstate_t ss)
 {
-	bool success = true;
+	bool  success = true;
 	GLint min_filter;
 
 	if (tex->cur_sampler == ss)
@@ -347,6 +366,8 @@ static bool load_texture_sampler(texture_t tex, samplerstate_t ss)
 			ss->max_anisotropy))
 		success = false;
 
+	apply_swizzle(tex);
+
 	return success;
 }
 

libobs-opengl/gl-subsystem.h

@@ -27,6 +27,8 @@
 
 #include "gl-helpers.h"
 
+#define USE_FORMAT_SWIZZLE
+
 struct gl_platform;
 struct gl_windowinfo;
 
@@ -42,8 +44,13 @@ static inline GLint convert_gs_format(enum gs_color_format format)
 	case GS_A8:          return GL_RGBA;
 	case GS_R8:          return GL_RED;
 	case GS_RGBA:        return GL_RGBA;
+#ifdef USE_FORMAT_SWIZZLE
+	case GS_BGRX:        return GL_RGB;
+	case GS_BGRA:        return GL_RGBA;
+#else
 	case GS_BGRX:        return GL_BGR;
 	case GS_BGRA:        return GL_BGRA;
+#endif
 	case GS_R10G10B10A2: return GL_RGBA;
 	case GS_RGBA16:      return GL_RGBA;
 	case GS_R16:         return GL_RED;