virtualcam-module: Fix crash on resolution change

Certain programs can start the virtualcam filter, then they may choose
to call `Stop()` on the filter, call `SetFormat()` to change the
resolution, then call `Run()` again to start the filter again. The
Windows virtual camera filter did not account for this, thus if the
resolution was different, it had potential to cause a crash.

To fix this, store the last filter resolution, then check the resolution
every frame, and if it changes, reset the scaling information.

(Author note: This code is unclean. What we need to do with the virtual
camera filter is make it only create the thread on `Run()`, then join
the thread on `Stop()`. It's currently a bit complicated to make it do
that at the moment, so this code is a kind of an annoying stopgap for
now.)

plugins/win-dshow/virtualcam-module/virtualcam-filter.cpp

@@ -189,6 +189,8 @@ void VCamFilter::Thread()
 	obs_cx = (uint32_t)GetCX();
 	obs_cy = (uint32_t)GetCY();
 	obs_interval = (uint64_t)GetInterval();
+	filter_cx = obs_cx;
+	filter_cy = obs_cy;
 
 	/* ---------------------------------------- */
 	/* load placeholder image                   */
@@ -248,11 +250,14 @@ void VCamFilter::Frame(uint64_t ts)
 		prev_state = state;
 	}
 
+	uint32_t new_filter_cx = (uint32_t)GetCX();
+	uint32_t new_filter_cy = (uint32_t)GetCY();
+
 	if (state != SHARED_QUEUE_STATE_READY) {
 		/* Virtualcam output not yet started, assume it's
 		   the same resolution as the filter output */
-		new_obs_cx = GetCX();
-		new_obs_cy = GetCY();
+		new_obs_cx = new_filter_cx;
+		new_obs_cy = new_filter_cy;
 		new_obs_interval = GetInterval();
 	}
 
@@ -268,12 +273,24 @@ void VCamFilter::Frame(uint64_t ts)
 		}
 
 		/* Re-initialize the main scaler to use the new resolution */
-		nv12_scale_init(&scaler, scaler.format, GetCX(), GetCY(),
-				new_obs_cx, new_obs_cy);
+		nv12_scale_init(&scaler, scaler.format, new_filter_cx,
+				new_filter_cy, new_obs_cx, new_obs_cy);
 
 		obs_cx = new_obs_cx;
 		obs_cy = new_obs_cy;
 		obs_interval = new_obs_interval;
+		filter_cx = new_filter_cx;
+		filter_cy = new_filter_cy;
+
+		UpdatePlaceholder();
+
+	} else if (new_filter_cx != filter_cx || new_filter_cy != filter_cy) {
+		filter_cx = new_filter_cx;
+		filter_cy = new_filter_cy;
+
+		/* Re-initialize the main scaler to use the new resolution */
+		nv12_scale_init(&scaler, scaler.format, new_filter_cx,
+				new_filter_cy, new_obs_cx, new_obs_cy);
 
 		UpdatePlaceholder();
 	}

plugins/win-dshow/virtualcam-module/virtualcam-filter.hpp

@@ -33,6 +33,8 @@ class VCamFilter : public DShow::OutputFilter {
 	uint32_t obs_cx = DEFAULT_CX;
 	uint32_t obs_cy = DEFAULT_CY;
 	uint64_t obs_interval = DEFAULT_INTERVAL;
+	uint32_t filter_cx = DEFAULT_CX;
+	uint32_t filter_cy = DEFAULT_CY;
 	DShow::VideoFormat format;
 	WinHandle thread_start;
 	WinHandle thread_stop;