UI: Fix scaling in viewport when source flipped

Fixes bug 617 on mantis.  Scaling the source manually in the viewport
wouldn't work properly when the source is flipped horizontally or
vertically.

Closes jp9000/obs-studio#751

UI/window-basic-preview.cpp

@@ -740,18 +740,35 @@ void OBSBasicPreview::ClampAspect(vec3 &tl, vec3 &br, vec2 &size,
 	    stretchHandle == ItemHandle::TopRight   ||
 	    stretchHandle == ItemHandle::BottomLeft ||
 	    stretchHandle == ItemHandle::BottomRight) {
-		if (aspect < baseAspect)
-			size.x = size.y * baseAspect;
-		else
-			size.y = size.x / baseAspect;
+		if (aspect < baseAspect) {
+			if ((size.y >= 0.0f && size.x >= 0.0f) ||
+			    (size.y <= 0.0f && size.x <= 0.0f))
+				size.x = size.y * baseAspect;
+			else
+				size.x = size.y * baseAspect * -1.0f;
+		} else {
+			if ((size.y >= 0.0f && size.x >= 0.0f) ||
+			    (size.y <= 0.0f && size.x <= 0.0f))
+				size.y = size.x / baseAspect;
+			else
+				size.y = size.x / baseAspect * -1.0f;
+		}
 
 	} else if (stretchHandle == ItemHandle::TopCenter ||
 	           stretchHandle == ItemHandle::BottomCenter) {
-		size.x = size.y * baseAspect;
+		if ((size.y >= 0.0f && size.x >= 0.0f) ||
+		    (size.y <= 0.0f && size.x <= 0.0f))
+			size.x = size.y * baseAspect;
+		else
+			size.x = size.y * baseAspect * -1.0f;
 
 	} else if (stretchHandle == ItemHandle::CenterLeft ||
 	           stretchHandle == ItemHandle::CenterRight) {
-		size.y = size.x / baseAspect;
+		if ((size.y >= 0.0f && size.x >= 0.0f) ||
+		    (size.y <= 0.0f && size.x <= 0.0f))
+			size.y = size.x / baseAspect;
+		else
+			size.y = size.x / baseAspect * -1.0f;
 	}
 
 	size.x = std::round(size.x);