UI: Add scene item canvas overflow to preview

UI/data/locale/en-US.ini

@@ -599,6 +599,10 @@ Basic.Settings.General.SysTray="System Tray"
 Basic.Settings.General.SysTrayWhenStarted="Minimize to system tray when started"
 Basic.Settings.General.SystemTrayHideMinimize="Always minimize to system tray instead of task bar"
 Basic.Settings.General.SaveProjectors="Save projectors on exit"
+Basic.Settings.General.Preview="Preview"
+Basic.Settings.General.OverflowHidden="Hide overflow"
+Basic.Settings.General.OverflowAlwaysVisible="Overflow always visible"
+Basic.Settings.General.OverflowSelectionHidden="Show overflow even when source is invisible"
 Basic.Settings.General.SwitchOnDoubleClick="Transition to scene when double-clicked"
 Basic.Settings.General.StudioPortraitLayout="Enable portrait/vertical layout"
 Basic.Settings.General.Multiview="Multiview"

UI/forms/OBSBasicSettings.ui

@@ -540,6 +540,58 @@
                   </layout>
                  </widget>
                 </item>
+                <item>
+                 <widget class="QGroupBox" name="groupBox_18">
+                  <property name="title">
+                   <string>StudioMode.Preview</string>
+                  </property>
+                  <layout class="QFormLayout" name="formLayout_35">
+                   <property name="fieldGrowthPolicy">
+                    <enum>QFormLayout::AllNonFixedFieldsGrow</enum>
+                   </property>
+                   <property name="labelAlignment">
+                    <set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set>
+                   </property>
+                   <property name="topMargin">
+                    <number>2</number>
+                   </property>
+                   <item row="1" column="1">
+                    <widget class="QCheckBox" name="overflowAlwaysVisible">
+                     <property name="text">
+                      <string>Basic.Settings.General.OverflowAlwaysVisible</string>
+                     </property>
+                    </widget>
+                   </item>
+                   <item row="0" column="0">
+                    <spacer name="horizontalSpacer_25">
+                     <property name="orientation">
+                      <enum>Qt::Horizontal</enum>
+                     </property>
+                     <property name="sizeHint" stdset="0">
+                      <size>
+                       <width>170</width>
+                       <height>5</height>
+                      </size>
+                     </property>
+                    </spacer>
+                   </item>
+                   <item row="2" column="1">
+                    <widget class="QCheckBox" name="overflowSelectionHide">
+                     <property name="text">
+                      <string>Basic.Settings.General.OverflowSelectionHidden</string>
+                     </property>
+                    </widget>
+                   </item>
+                   <item row="0" column="1">
+                    <widget class="QCheckBox" name="overflowHide">
+                     <property name="text">
+                      <string>Basic.Settings.General.OverflowHidden</string>
+                     </property>
+                    </widget>
+                   </item>
+                  </layout>
+                 </widget>
+                </item>
                 <item>
                  <widget class="QGroupBox" name="groupBox_11">
                   <property name="title">

UI/window-basic-main.cpp

@@ -3278,6 +3278,16 @@ void OBSBasic::RenderMain(void *data, uint32_t cx, uint32_t cy)
 	gs_viewport_push();
 	gs_projection_push();
 
+	QSize previewSize = GetPixelSize(window->ui->preview);
+	float right  = float(previewSize.width())  - window->previewX;
+	float bottom = float(previewSize.height()) - window->previewY;
+
+	gs_ortho(-window->previewX, right,
+		-window->previewY, bottom,
+		-100.0f, 100.0f);
+
+	window->ui->preview->DrawOverflow();
+
 	/* --------------------------------------- */
 
 	gs_ortho(0.0f, float(ovi.base_width), 0.0f, float(ovi.base_height),
@@ -3287,6 +3297,7 @@ void OBSBasic::RenderMain(void *data, uint32_t cx, uint32_t cy)
 
 	window->DrawBackdrop(float(ovi.base_width), float(ovi.base_height));
 
+
 	if (window->IsPreviewProgramMode()) {
 		OBSScene scene = window->GetCurrentScene();
 		obs_source_t *source = obs_scene_get_source(scene);
@@ -3299,9 +3310,6 @@ void OBSBasic::RenderMain(void *data, uint32_t cx, uint32_t cy)
 
 	/* --------------------------------------- */
 
-	QSize previewSize = GetPixelSize(window->ui->preview);
-	float right  = float(previewSize.width())  - window->previewX;
-	float bottom = float(previewSize.height()) - window->previewY;
 
 	gs_ortho(-window->previewX, right,
 	         -window->previewY, bottom,

UI/window-basic-preview.cpp

@@ -3,11 +3,13 @@
 
 #include <algorithm>
 #include <cmath>
+#include <string>
 #include <graphics/vec4.h>
 #include <graphics/matrix4.h>
 #include "window-basic-preview.hpp"
 #include "window-basic-main.hpp"
 #include "obs-app.hpp"
+#include "platform.hpp"
 
 #define HANDLE_RADIUS     4.0f
 #define HANDLE_SEL_RADIUS (HANDLE_RADIUS * 1.5f)
@@ -24,6 +26,13 @@ OBSBasicPreview::OBSBasicPreview(QWidget *parent, Qt::WindowFlags flags)
 	setMouseTracking(true);
 }
 
+OBSBasicPreview::~OBSBasicPreview()
+{
+	if (overflow) {
+		gs_texture_destroy(overflow);
+	}
+}
+
 vec2 OBSBasicPreview::GetMouseEventPos(QMouseEvent *event)
 {
 	OBSBasic *main = reinterpret_cast<OBSBasic*>(App()->GetMainWindow());
@@ -1213,6 +1222,92 @@ static inline bool crop_enabled(const obs_sceneitem_crop *crop)
 	       crop->bottom > 0;
 }
 
+bool OBSBasicPreview::DrawSelectedOverflow(obs_scene_t *scene,
+	obs_sceneitem_t *item, void *param)
+{
+	if (obs_sceneitem_locked(item))
+		return true;
+
+	if (!SceneItemHasVideo(item))
+		return true;
+
+	bool select = config_get_bool(GetGlobalConfig(), "BasicWindow",
+		"OverflowSelectionHidden");
+
+	if (!select && !obs_sceneitem_visible(item))
+		return true;
+
+	if (obs_sceneitem_is_group(item)) {
+		matrix4 mat;
+		obs_sceneitem_get_draw_transform(item, &mat);
+
+		gs_matrix_push();
+		gs_matrix_mul(&mat);
+		obs_sceneitem_group_enum_items(item, DrawSelectedOverflow, param);
+		gs_matrix_pop();
+	}
+
+	bool always = config_get_bool(GetGlobalConfig(), "BasicWindow",
+		"OverflowAlwaysVisible");
+
+	if (!always && !obs_sceneitem_selected(item))
+		return true;
+
+	OBSBasicPreview *prev = reinterpret_cast<OBSBasicPreview*>(param);
+
+	matrix4 boxTransform;
+	matrix4 invBoxTransform;
+	obs_sceneitem_get_box_transform(item, &boxTransform);
+	matrix4_inv(&invBoxTransform, &boxTransform);
+
+	vec3 bounds[] = {
+		{{{0.f, 0.f, 0.f}}},
+		{{{1.f, 0.f, 0.f}}},
+		{{{0.f, 1.f, 0.f}}},
+		{{{1.f, 1.f, 0.f}}},
+	};
+
+	bool visible = std::all_of(std::begin(bounds), std::end(bounds),
+		[&](const vec3 &b)
+	{
+		vec3 pos;
+		vec3_transform(&pos, &b, &boxTransform);
+		vec3_transform(&pos, &pos, &invBoxTransform);
+		return CloseFloat(pos.x, b.x) && CloseFloat(pos.y, b.y);
+	});
+
+	if (!visible)
+		return true;
+
+	obs_transform_info info;
+	obs_sceneitem_get_info(item, &info);
+
+	gs_effect_t    *solid = obs_get_base_effect(OBS_EFFECT_REPEAT);
+	gs_eparam_t    *image = gs_effect_get_param_by_name(solid, "image");
+	gs_eparam_t    *scale = gs_effect_get_param_by_name(solid, "scale");
+
+	vec2 s;
+	vec2_set(&s, boxTransform.x.x / 96, boxTransform.y.y / 96);
+
+	gs_effect_set_vec2(scale, &s);
+	gs_effect_set_texture(image, prev->overflow);
+
+	gs_matrix_push();
+	gs_matrix_mul(&boxTransform);
+
+	obs_sceneitem_crop crop;
+	obs_sceneitem_get_crop(item, &crop);
+
+	while (gs_effect_loop(solid, "Draw")) {
+		gs_draw_sprite(prev->overflow, 0, 1, 1);
+	}
+
+	gs_matrix_pop();
+
+	UNUSED_PARAMETER(scene);
+	return true;
+}
+
 bool OBSBasicPreview::DrawSelectedItem(obs_scene_t *scene,
 		obs_sceneitem_t *item, void *param)
 {
@@ -1312,6 +1407,37 @@ bool OBSBasicPreview::DrawSelectedItem(obs_scene_t *scene,
 	return true;
 }
 
+void OBSBasicPreview::DrawOverflow()
+{
+	if (locked)
+		return;
+
+	bool hidden = config_get_bool(GetGlobalConfig(), "BasicWindow",
+		"OverflowHidden");
+
+	if (hidden)
+		return;
+
+	if (!overflow) {
+		std::string path;
+		GetDataFilePath("images/overflow.png", path);
+		overflow = gs_texture_create_from_file(path.c_str());
+	}
+
+	OBSBasic *main = reinterpret_cast<OBSBasic*>(App()->GetMainWindow());
+
+	OBSScene scene = main->GetCurrentScene();
+
+	if (scene) {
+		gs_matrix_push();
+		gs_matrix_scale3f(main->previewScale, main->previewScale, 1.0f);
+		obs_scene_enum_items(scene, DrawSelectedOverflow, this);
+		gs_matrix_pop();
+	}
+
+	gs_load_vertexbuffer(nullptr);
+}
+
 void OBSBasicPreview::DrawSceneEditing()
 {
 	if (locked)

UI/window-basic-preview.hpp

@@ -43,6 +43,8 @@ private:
 	matrix4      itemToScreen;
 	matrix4      invGroupTransform;
 
+	gs_texture_t *overflow = nullptr;
+
 	vec2         startPos;
 	vec2         lastMoveOffset;
 	vec2         scrollingFrom;
@@ -58,6 +60,8 @@ private:
 	float        scalingAmount  = 1.0f;
 
 	static vec2 GetMouseEventPos(QMouseEvent *event);
+	static bool DrawSelectedOverflow(obs_scene_t *scene,
+		obs_sceneitem_t *item, void *param);
 	static bool DrawSelectedItem(obs_scene_t *scene, obs_sceneitem_t *item,
 		void *param);
 
@@ -84,6 +88,7 @@ private:
 
 public:
 	OBSBasicPreview(QWidget *parent, Qt::WindowFlags flags = 0);
+	~OBSBasicPreview();
 
 	virtual void keyPressEvent(QKeyEvent *event) override;
 	virtual void keyReleaseEvent(QKeyEvent *event) override;
@@ -94,6 +99,7 @@ public:
 	virtual void mouseReleaseEvent(QMouseEvent *event) override;
 	virtual void mouseMoveEvent(QMouseEvent *event) override;
 
+	void DrawOverflow();
 	void DrawSceneEditing();
 
 	inline void SetLocked(bool newLockedVal) {locked = newLockedVal;}

UI/window-basic-settings.cpp

@@ -316,6 +316,9 @@ OBSBasicSettings::OBSBasicSettings(QWidget *parent)
 	HookWidget(ui->centerSnapping,       CHECK_CHANGED,  GENERAL_CHANGED);
 	HookWidget(ui->sourceSnapping,       CHECK_CHANGED,  GENERAL_CHANGED);
 	HookWidget(ui->snapDistance,         DSCROLL_CHANGED,GENERAL_CHANGED);
+	HookWidget(ui->overflowHide,         CHECK_CHANGED,  GENERAL_CHANGED);
+	HookWidget(ui->overflowAlwaysVisible,CHECK_CHANGED,  GENERAL_CHANGED);
+	HookWidget(ui->overflowSelectionHide,CHECK_CHANGED,  GENERAL_CHANGED);
 	HookWidget(ui->doubleClickSwitch,    CHECK_CHANGED,  GENERAL_CHANGED);
 	HookWidget(ui->studioPortraitLayout, CHECK_CHANGED,  GENERAL_CHANGED);
 	HookWidget(ui->multiviewMouseSwitch, CHECK_CHANGED,  GENERAL_CHANGED);
@@ -1088,6 +1091,18 @@ void OBSBasicSettings::LoadGeneralSettings()
 			"BasicWindow", "ProjectorAlwaysOnTop");
 	ui->projectorAlwaysOnTop->setChecked(projectorAlwaysOnTop);
 
+	bool overflowHide = config_get_bool(GetGlobalConfig(),
+		"BasicWindow", "OverflowHidden");
+	ui->overflowHide->setChecked(overflowHide);
+
+	bool overflowAlwaysVisible = config_get_bool(GetGlobalConfig(),
+		"BasicWindow", "OverflowAlwaysVisible");
+	ui->overflowAlwaysVisible->setChecked(overflowAlwaysVisible);
+
+	bool overflowSelectionHide = config_get_bool(GetGlobalConfig(),
+		"BasicWindow", "OverflowSelectionHidden");
+	ui->overflowSelectionHide->setChecked(overflowSelectionHide);
+
 	bool doubleClickSwitch = config_get_bool(GetGlobalConfig(),
 			"BasicWindow", "TransitionOnDoubleClick");
 	ui->doubleClickSwitch->setChecked(doubleClickSwitch);
@@ -2666,6 +2681,18 @@ void OBSBasicSettings::SaveGeneralSettings()
 		config_set_double(GetGlobalConfig(), "BasicWindow",
 				"SnapDistance",
 				ui->snapDistance->value());
+	if (WidgetChanged(ui->overflowAlwaysVisible))
+		config_set_bool(GetGlobalConfig(), "BasicWindow",
+			"OverflowAlwaysVisible",
+			ui->overflowAlwaysVisible->isChecked());
+	if (WidgetChanged(ui->overflowHide))
+		config_set_bool(GetGlobalConfig(), "BasicWindow",
+			"OverflowHidden",
+			ui->overflowHide->isChecked());
+	if (WidgetChanged(ui->overflowSelectionHide))
+		config_set_bool(GetGlobalConfig(), "BasicWindow",
+			"OverflowSelectionHidden",
+			ui->overflowSelectionHide->isChecked());
 	if (WidgetChanged(ui->doubleClickSwitch))
 		config_set_bool(GetGlobalConfig(), "BasicWindow",
 				"TransitionOnDoubleClick",

libobs/data/repeat.effect

@@ -0,0 +1,55 @@
+uniform float4x4 ViewProj;
+uniform float4x4 color_matrix;
+uniform float3 color_range_min = {0.0, 0.0, 0.0};
+uniform float3 color_range_max = {1.0, 1.0, 1.0};
+uniform texture2d image;
+uniform float2 scale;
+
+sampler_state def_sampler {
+	Filter   = Linear;
+	AddressU = Repeat;
+	AddressV = Repeat;
+};
+
+struct VertInOut {
+	float4 pos : POSITION;
+	float2 uv  : TEXCOORD0;
+};
+
+VertInOut VSDefault(VertInOut vert_in)
+{
+	VertInOut vert_out;
+	vert_out.pos = mul(float4(vert_in.pos.xyz, 1.0), ViewProj);
+	vert_out.uv  = vert_in.uv * scale;
+	return vert_out;
+}
+
+float4 PSDrawBare(VertInOut vert_in) : TARGET
+{
+	return image.Sample(def_sampler, vert_in.uv);
+}
+
+float4 PSDrawMatrix(VertInOut vert_in) : TARGET
+{
+	float4 yuv = image.Sample(def_sampler, vert_in.uv);
+	yuv.xyz = clamp(yuv.xyz, color_range_min, color_range_max);
+	return saturate(mul(float4(yuv.xyz, 1.0), color_matrix));
+}
+
+technique Draw
+{
+	pass
+	{
+		vertex_shader = VSDefault(vert_in);
+		pixel_shader  = PSDrawBare(vert_in);
+	}
+}
+
+technique DrawMatrix
+{
+	pass
+	{
+		vertex_shader = VSDefault(vert_in);
+		pixel_shader  = PSDrawMatrix(vert_in);
+	}
+}

libobs/obs-internal.h

@@ -255,6 +255,7 @@ struct obs_core_video {
 	gs_effect_t                     *default_rect_effect;
 	gs_effect_t                     *opaque_effect;
 	gs_effect_t                     *solid_effect;
+	gs_effect_t                     *repeat_effect;
 	gs_effect_t                     *conversion_effect;
 	gs_effect_t                     *bicubic_effect;
 	gs_effect_t                     *lanczos_effect;

libobs/obs.c

@@ -309,6 +309,11 @@ static int obs_init_graphics(struct obs_video_info *ovi)
 			NULL);
 	bfree(filename);
 
+	filename = obs_find_data_file("repeat.effect");
+	video->repeat_effect = gs_effect_create_from_file(filename,
+		NULL);
+	bfree(filename);
+
 	filename = obs_find_data_file("format_conversion.effect");
 	video->conversion_effect = gs_effect_create_from_file(filename,
 			NULL);
@@ -1570,6 +1575,8 @@ gs_effect_t *obs_get_base_effect(enum obs_base_effect effect)
 		return obs->video.opaque_effect;
 	case OBS_EFFECT_SOLID:
 		return obs->video.solid_effect;
+	case OBS_EFFECT_REPEAT:
+		return obs->video.repeat_effect;
 	case OBS_EFFECT_BICUBIC:
 		return obs->video.bicubic_effect;
 	case OBS_EFFECT_LANCZOS:

libobs/obs.h

@@ -601,6 +601,7 @@ enum obs_base_effect {
 	OBS_EFFECT_LANCZOS,            /**< Lanczos downscale */
 	OBS_EFFECT_BILINEAR_LOWRES,    /**< Bilinear low resolution downscale */
 	OBS_EFFECT_PREMULTIPLIED_ALPHA,/**< Premultiplied alpha */
+	OBS_EFFECT_REPEAT,             /**< RGB/YUV (repeating) */
 };
 
 /** Returns a commonly used base effect */