Use only one widget for preview

Modify the obs_display API so that it always uses an orthographic
projection that is the size of the display, rather than OBS' base size.
Having it do an orthographic projection to OBS' base size was silly
because it meant that everything would be skewed if you wanted to draw
1:1 in the display.  This deoes mean that the callbacks must handle
resizing the images, but it's worth it to ensure 1:1 draw sizes.

As for the preview widget, instead of making some funky widget within
widget that resizes, it's just going to be a widget within the entire
top layout.  Also changed the preview padding color to gray.

libobs/obs-display.c

@@ -130,7 +130,8 @@ static inline void render_display_begin(struct obs_display *display)
 	}
 
 	gs_beginscene();
-	vec4_set(&clear_color, 0.3f, 0.0f, 0.0f, 1.0f);
+
+	vec4_set(&clear_color, 0.3f, 0.3f, 0.3f, 1.0f);
 	gs_clear(GS_CLEAR_COLOR | GS_CLEAR_DEPTH | GS_CLEAR_STENCIL,
 			&clear_color, 1.0f, 0);
 
@@ -138,8 +139,8 @@ static inline void render_display_begin(struct obs_display *display)
 	/* gs_enable_blending(false); */
 	gs_setcullmode(GS_NEITHER);
 
-	gs_ortho(0.0f, (float)obs->video.base_width,
-			0.0f, (float)obs->video.base_height, -100.0f, 100.0f);
+	gs_ortho(0.0f, (float)display->cx,
+			0.0f, (float)display->cy, -100.0f, 100.0f);
 	gs_setviewport(0, 0, display->cx, display->cy);
 }
 

libobs/obs-video.c

@@ -83,7 +83,7 @@ static inline void render_main_texture(struct obs_core_video *video,
 		int cur_texture)
 {
 	struct vec4 clear_color;
-	vec4_set(&clear_color, 0.3f, 0.0f, 0.0f, 1.0f);
+	vec4_set(&clear_color, 0.0f, 0.0f, 0.0f, 1.0f);
 
 	gs_setrendertarget(video->render_textures[cur_texture], NULL);
 	gs_clear(GS_CLEAR_COLOR, &clear_color, 1.0f, 0);

obs/forms/OBSBasic.ui

@@ -29,7 +29,7 @@
   <widget class="QWidget" name="centralwidget">
    <layout class="QVBoxLayout" name="verticalLayout">
     <item>
-     <widget class="QWidget" name="previewContainer" native="true">
+     <widget class="OBSQTDisplay" name="preview" native="true">
       <property name="sizePolicy">
        <sizepolicy hsizetype="Expanding" vsizetype="Expanding">
         <horstretch>0</horstretch>
@@ -42,28 +42,6 @@
         <height>32</height>
        </size>
       </property>
-      <widget class="OBSQTDisplay" name="preview" native="true">
-       <property name="geometry">
-        <rect>
-         <x>0</x>
-         <y>0</y>
-         <width>32</width>
-         <height>32</height>
-        </rect>
-       </property>
-       <property name="sizePolicy">
-        <sizepolicy hsizetype="Minimum" vsizetype="Minimum">
-         <horstretch>0</horstretch>
-         <verstretch>0</verstretch>
-        </sizepolicy>
-       </property>
-       <property name="minimumSize">
-        <size>
-         <width>32</width>
-         <height>32</height>
-        </size>
-       </property>
-      </widget>
      </widget>
     </item>
     <item alignment="Qt::AlignHCenter|Qt::AlignVCenter">

obs/qt-wrappers.cpp

@@ -36,16 +36,13 @@ void OBSErrorBox(QWidget *parent, const char *msg, ...)
 	va_end(args);
 }
 
-void QTToGSWindow(QWidget *widget, gs_window &gswindow)
+void QTToGSWindow(WId windowId, gs_window &gswindow)
 {
-	if (!widget)
-		return;
-
 #ifdef _WIN32
-	gswindow.hwnd = (HWND)widget->winId();
+	gswindow.hwnd = (HWND)windowId;
 #elif __APPLE__
-	gswindow.view = (id)widget->winId();
+	gswindow.view = (id)windowId;
 #else
-	gswindow.id = widget->winId();
+	gswindow.id = windowId;
 #endif
 }

obs/qt-wrappers.hpp

@@ -17,6 +17,8 @@
 
 #pragma once
 
+#include <QWidget>
+
 #define QT_UTF8(str) QString::fromUtf8(str)
 #define QT_TO_UTF8(str) str.toUtf8().constData()
 
@@ -25,4 +27,4 @@ struct gs_window;
 
 void OBSErrorBox(QWidget *parent, const char *msg, ...);
 
-void QTToGSWindow(QWidget *widget, gs_window &gswindow);
+void QTToGSWindow(WId windowId, gs_window &gswindow);

obs/window-basic-main.cpp

@@ -133,9 +133,6 @@ void OBSBasic::OBSInit()
 	obs_load_module("win-wasapi");
 	obs_load_module("win-capture");
 #endif
-
-	/* HACK: fixes a qt bug with native widgets with native repaint */
-	ui->previewContainer->repaint();
 }
 
 OBSBasic::~OBSBasic()
@@ -341,9 +338,13 @@ void OBSBasic::ChannelChanged(void *data, calldata_t params)
 
 void OBSBasic::RenderMain(void *data, uint32_t cx, uint32_t cy)
 {
+	OBSBasic *window = static_cast<OBSBasic*>(data);
+	gs_matrix_push();
+	gs_matrix_scale3f(window->previewScale, window->previewScale, 1.0f);
+	gs_matrix_translate3f(-window->previewX, -window->previewY, 0.0f);
 	obs_render_main_view();
+	gs_matrix_pop();
 
-	UNUSED_PARAMETER(data);
 	UNUSED_PARAMETER(cx);
 	UNUSED_PARAMETER(cy);
 }
@@ -369,7 +370,7 @@ bool OBSBasic::ResetVideo()
 	ovi.adapter        = 0;
 	ovi.gpu_conversion = true;
 
-	QTToGSWindow(ui->preview, ovi.window);
+	QTToGSWindow(ui->preview->winId(), ovi.window);
 
 	//required to make opengl display stuff on osx(?)
 	ResizePreview(ovi.base_width, ovi.base_height);
@@ -414,25 +415,28 @@ void OBSBasic::ResizePreview(uint32_t cx, uint32_t cy)
 	int x, y;
 
 	/* resize preview panel to fix to the top section of the window */
-	targetSize   = ui->previewContainer->size();
+	targetSize   = ui->preview->size();
 	targetAspect = double(targetSize.width()) / double(targetSize.height());
 	baseAspect   = double(cx) / double(cy);
 
 	if (targetAspect > baseAspect) {
+		previewScale = float(targetSize.height()) / float(cy);
 		cx = targetSize.height() * baseAspect;
 		cy = targetSize.height();
 	} else {
+		previewScale = float(targetSize.width()) / float(cx);
 		cx = targetSize.width();
 		cy = targetSize.width() / baseAspect;
 	}
 
-	x = targetSize.width() /2 - cx/2;
-	y = targetSize.height()/2 - cy/2;
-
-	ui->preview->setGeometry(x, y, cx, cy);
+	previewX = targetSize.width() /2 - cx/2;
+	previewY = targetSize.height()/2 - cy/2;
 
-	if (isVisible())
-		obs_resize(cx, cy);
+	if (isVisible()) {
+		if (resizeTimer)
+			killTimer(resizeTimer);
+		resizeTimer = startTimer(100);
+	}
 }
 
 void OBSBasic::closeEvent(QCloseEvent *event)
@@ -457,6 +461,17 @@ void OBSBasic::resizeEvent(QResizeEvent *event)
 	UNUSED_PARAMETER(event);
 }
 
+void OBSBasic::timerEvent(QTimerEvent *event)
+{
+	if (event->timerId() == resizeTimer) {
+		killTimer(resizeTimer);
+		resizeTimer = 0;
+
+		QSize size = ui->preview->size();
+		obs_resize(size.width(), size.height());
+	}
+}
+
 void OBSBasic::on_action_New_triggered()
 {
 	/* TODO */

obs/window-basic-main.hpp

@@ -36,6 +36,10 @@ private:
 	obs_output_t outputTest;
 	bool         sceneChanging;
 
+	int          previewX,  previewY;
+	float        previewScale;
+	int          resizeTimer;
+
 	ConfigFile   basicConfig;
 
 	void GetFPSCommon(uint32_t &num, uint32_t &den) const;
@@ -86,6 +90,7 @@ protected:
 	virtual void closeEvent(QCloseEvent *event) override;
 	virtual void changeEvent(QEvent *event) override;
 	virtual void resizeEvent(QResizeEvent *event) override;
+	virtual void timerEvent(QTimerEvent *event) override;
 
 private slots:
 	void on_action_New_triggered();