linux-v4l2: Fix virtual camera start failure

Add function that tries to reset v4l2loopback output for module versions
from 0.12.5 to 0.12.7. If successful, then set flag that STREAMON and
STREAMOFF are necessary each time the device is opened/closed.

(cherry picked from commit 12c6febae21f369da50f09d511b54eadc1dc1342)

plugins/linux-v4l2/v4l2-output.c

@@ -15,6 +15,7 @@ struct virtualcam_data {
 	obs_output_t *output;
 	int device;
 	uint32_t frame_size;
+	bool use_caps_workaround;
 };
 
 static const char *virtualcam_name(void *unused)
@@ -110,11 +111,54 @@ static void *virtualcam_create(obs_data_t *settings, obs_output_t *output)
 	return vcam;
 }
 
+bool try_reset_output_caps(const char *device)
+{
+	struct v4l2_capability capability;
+	struct v4l2_format format;
+	int fd;
+
+	blog(LOG_INFO, "Attempting to reset output capability of '%s'", device);
+
+	fd = open(device, O_RDWR);
+	if (fd < 0)
+		return false;
+
+	format.type = V4L2_BUF_TYPE_VIDEO_OUTPUT;
+	if (ioctl(fd, VIDIOC_G_FMT, &format) < 0)
+		goto fail_close_reset_caps_fd;
+
+	if (ioctl(fd, VIDIOC_S_FMT, &format) < 0)
+		goto fail_close_reset_caps_fd;
+
+	if (ioctl(fd, VIDIOC_STREAMON, &format.type) < 0)
+		goto fail_close_reset_caps_fd;
+
+	if (ioctl(fd, VIDIOC_STREAMOFF, &format.type) < 0)
+		goto fail_close_reset_caps_fd;
+
+	close(fd);
+
+	fd = open(device, O_RDWR);
+	if (fd < 0)
+		return false;
+
+	if (ioctl(fd, VIDIOC_QUERYCAP, &capability) < 0)
+		goto fail_close_reset_caps_fd;
+
+	close(fd);
+	return (capability.device_caps & V4L2_CAP_VIDEO_OUTPUT) != 0;
+
+fail_close_reset_caps_fd:
+	close(fd);
+	return false;
+}
+
 static bool try_connect(void *data, const char *device)
 {
+	static bool use_caps_workaround = false;
 	struct virtualcam_data *vcam = (struct virtualcam_data *)data;
-	struct v4l2_format format;
 	struct v4l2_capability capability;
+	struct v4l2_format format;
 	struct v4l2_streamparm parm;
 
 	uint32_t width = obs_output_get_width(vcam->output);
@@ -130,6 +174,14 @@ static bool try_connect(void *data, const char *device)
 	if (ioctl(vcam->device, VIDIOC_QUERYCAP, &capability) < 0)
 		goto fail_close_device;
 
+	if (!use_caps_workaround && !(capability.device_caps & V4L2_CAP_VIDEO_OUTPUT)) {
+		if (!try_reset_output_caps(device))
+			goto fail_close_device;
+
+		use_caps_workaround = true;
+	}
+	vcam->use_caps_workaround = use_caps_workaround;
+
 	format.type = V4L2_BUF_TYPE_VIDEO_OUTPUT;
 
 	if (ioctl(vcam->device, VIDIOC_G_FMT, &format) < 0)
@@ -165,7 +217,7 @@ static bool try_connect(void *data, const char *device)
 	memset(&parm, 0, sizeof(parm));
 	parm.type = V4L2_BUF_TYPE_VIDEO_OUTPUT;
 
-	if (ioctl(vcam->device, VIDIOC_STREAMON, &parm) < 0) {
+	if (vcam->use_caps_workaround && ioctl(vcam->device, VIDIOC_STREAMON, &format.type) < 0) {
 		blog(LOG_ERROR, "Failed to start streaming on '%s' (%s)", device, strerror(errno));
 		goto fail_close_device;
 	}
@@ -238,10 +290,9 @@ static void virtualcam_stop(void *data, uint64_t ts)
 	struct virtualcam_data *vcam = (struct virtualcam_data *)data;
 	obs_output_end_data_capture(vcam->output);
 
-	struct v4l2_streamparm parm = {0};
-	parm.type = V4L2_BUF_TYPE_VIDEO_OUTPUT;
+	uint32_t buf_type = V4L2_BUF_TYPE_VIDEO_OUTPUT;
 
-	if (ioctl(vcam->device, VIDIOC_STREAMOFF, &parm) < 0) {
+	if (vcam->use_caps_workaround && ioctl(vcam->device, VIDIOC_STREAMOFF, buf_type) < 0) {
 		blog(LOG_WARNING, "Failed to stop streaming on video device %d (%s)", vcam->device, strerror(errno));
 	}