Fix gs_matrix_* issues

Multiplication of the matricies was being done in the wrong direction.
This caused source transformations to come out looking incorrect, for
example the linux-xshm source's cursor would not be drawn correctly or
in the right position if the source was moved/scaled/rotated.  The
problem just turned out to be that the gs_matrix_* functions were
multiplying in the wrong direction.  Reverse the direction of
multiplication, and the problem is solved.

libobs/graphics/graphics.c

@@ -297,35 +297,35 @@ void gs_matrix_mul(const struct matrix4 *matrix)
 {
 	struct matrix4 *top_mat = top_matrix(thread_graphics);
 	if (top_mat)
-		matrix4_mul(top_mat, top_mat, matrix);
+		matrix4_mul(top_mat, matrix, top_mat);
 }
 
 void gs_matrix_rotquat(const struct quat *rot)
 {
 	struct matrix4 *top_mat = top_matrix(thread_graphics);
 	if (top_mat)
-		matrix4_rotate(top_mat, top_mat, rot);
+		matrix4_rotate_i(top_mat, rot, top_mat);
 }
 
 void gs_matrix_rotaa(const struct axisang *rot)
 {
 	struct matrix4 *top_mat = top_matrix(thread_graphics);
 	if (top_mat)
-		matrix4_rotate_aa(top_mat, top_mat, rot);
+		matrix4_rotate_aa_i(top_mat, rot, top_mat);
 }
 
 void gs_matrix_translate(const struct vec3 *pos)
 {
 	struct matrix4 *top_mat = top_matrix(thread_graphics);
 	if (top_mat)
-		matrix4_translate3v(top_mat, top_mat, pos);
+		matrix4_translate3v_i(top_mat, pos, top_mat);
 }
 
 void gs_matrix_scale(const struct vec3 *scale)
 {
 	struct matrix4 *top_mat = top_matrix(thread_graphics);
 	if (top_mat)
-		matrix4_scale(top_mat, top_mat, scale);
+		matrix4_scale_i(top_mat, scale, top_mat);
 }
 
 void gs_matrix_rotaa4f(float x, float y, float z, float angle)
@@ -335,7 +335,7 @@ void gs_matrix_rotaa4f(float x, float y, float z, float angle)
 
 	if (top_mat) {
 		axisang_set(&aa, x, y, z, angle);
-		matrix4_rotate_aa(top_mat, top_mat, &aa);
+		matrix4_rotate_aa_i(top_mat, &aa, top_mat);
 	}
 }
 
@@ -346,7 +346,7 @@ void gs_matrix_translate3f(float x, float y, float z)
 
 	if (top_mat) {
 		vec3_set(&p, x, y, z);
-		matrix4_translate3v(top_mat, top_mat, &p);
+		matrix4_translate3v_i(top_mat, &p, top_mat);
 	}
 }
 
@@ -357,7 +357,7 @@ void gs_matrix_scale3f(float x, float y, float z)
 
 	if (top_mat) {
 		vec3_set(&p, x, y, z);
-		matrix4_scale(top_mat, top_mat, &p);
+		matrix4_scale_i(top_mat, &p, top_mat);
 	}
 }
 

obs/window-basic-preview.cpp

@@ -682,6 +682,7 @@ static void DrawCircleAtPos(float x, float y, matrix4 &matrix,
 
 	gs_matrix_push();
 	gs_matrix_translate(&pos);
+	gs_matrix_scale3f(HANDLE_RADIUS, HANDLE_RADIUS, 1.0f);
 	gs_draw(GS_LINESTRIP, 0, 0);
 	gs_matrix_pop();
 }
@@ -699,8 +700,6 @@ bool OBSBasicPreview::DrawSelectedItem(obs_scene_t scene, obs_sceneitem_t item,
 	matrix4 boxTransform;
 	obs_sceneitem_get_box_transform(item, &boxTransform);
 
-	gs_matrix_push();
-	gs_matrix_scale3f(HANDLE_RADIUS, HANDLE_RADIUS, 1.0f);
 	DrawCircleAtPos(0.0f, 0.0f, boxTransform, main->previewScale);
 	DrawCircleAtPos(0.0f, 1.0f, boxTransform, main->previewScale);
 	DrawCircleAtPos(1.0f, 0.0f, boxTransform, main->previewScale);
@@ -709,13 +708,12 @@ bool OBSBasicPreview::DrawSelectedItem(obs_scene_t scene, obs_sceneitem_t item,
 	DrawCircleAtPos(0.0f, 0.5f, boxTransform, main->previewScale);
 	DrawCircleAtPos(0.5f, 1.0f, boxTransform, main->previewScale);
 	DrawCircleAtPos(1.0f, 0.5f, boxTransform, main->previewScale);
-	gs_matrix_pop();
 
 	gs_load_vertexbuffer(main->box);
 
 	gs_matrix_push();
-	gs_matrix_set(&boxTransform);
 	gs_matrix_scale3f(main->previewScale, main->previewScale, 1.0f);
+	gs_matrix_mul(&boxTransform);
 	gs_draw(GS_LINESTRIP, 0, 0);
 
 	gs_matrix_pop();

plugins/linux-xshm/xcursor.c

@@ -18,7 +18,6 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #include <stdint.h>
 #include <X11/extensions/Xfixes.h>
 
-#include <graphics/matrix4.h>
 #include <util/bmem.h>
 #include "xcursor.h"
 
@@ -93,18 +92,12 @@ void xcursor_tick(xcursor_t *data) {
 }
 
 void xcursor_render(xcursor_t *data) {
-	struct matrix4 trans;
-
 	gs_effect_t effect  = gs_get_effect();
 	gs_eparam_t image = gs_effect_get_param_by_name(effect, "image");
 	gs_effect_set_texture(image, data->tex);
 
 	gs_matrix_push();
-
-	gs_matrix_get(&trans);
-	gs_matrix_identity();
 	gs_matrix_translate3f(data->pos_x, data->pos_y, 0.0f);
-	gs_matrix_mul(&trans);
 
 	gs_enable_blending(True);
 	gs_blend_function(GS_BLEND_ONE, GS_BLEND_INVSRCALPHA);