obs-ffmpeg: Prevent invalid NVENC combinations

Don't want to silently generate lossy video.

plugins/obs-ffmpeg/data/locale/en-US.ini

@@ -33,6 +33,8 @@ NVENC.LookAhead.ToolTip="Enables dynamic B-frames.\n\nIf disabled, the encoder w
 NVENC.PsychoVisualTuning="Psycho Visual Tuning"
 NVENC.PsychoVisualTuning.ToolTip="Enables encoder settings that optimize the use of bitrate for increased perceived visual quality,\nespecially in situations with high motion, at the cost of increased GPU utilization."
 NVENC.CQLevel="CQ Level"
+NVENC.8bitUnsupportedHdr="OBS does not support 8-bit output of Rec. 2100."
+NVENC.I010Unsupported="NVENC does not support I010. Use P010 instead."
 NVENC.10bitUnsupported="Cannot perform 10-bit encode on this encoder."
 NVENC.TooManyBFrames="Max B-frames setting (%d) is more than encoder supports (%d)."
 

plugins/obs-ffmpeg/jim-nvenc.c

@@ -982,6 +982,21 @@ static bool init_encoder(struct nvenc_data *enc, bool hevc,
 		return false;
 	}
 
+	video_t *video = obs_encoder_video(enc->encoder);
+	const struct video_output_info *voi = video_output_get_info(video);
+	switch (voi->format) {
+	case VIDEO_FORMAT_I010:
+	case VIDEO_FORMAT_P010:
+		break;
+	default:
+		switch (voi->colorspace) {
+		case VIDEO_CS_2100_PQ:
+		case VIDEO_CS_2100_HLG:
+			NV_FAIL(obs_module_text("NVENC.8bitUnsupportedHdr"));
+			return false;
+		}
+	}
+
 	if (bf > bf_max) {
 		NV_FAIL(obs_module_text("NVENC.TooManyBFrames"), bf, bf_max);
 		return false;

plugins/obs-ffmpeg/obs-ffmpeg-nvenc.c

@@ -313,6 +313,30 @@ fail:
 
 static void *h264_nvenc_create(obs_data_t *settings, obs_encoder_t *encoder)
 {
+	video_t *video = obs_encoder_video(encoder);
+	const struct video_output_info *voi = video_output_get_info(video);
+	switch (voi->format) {
+	case VIDEO_FORMAT_I010:
+	case VIDEO_FORMAT_P010: {
+		const char *const text =
+			obs_module_text("NVENC.10bitUnsupported");
+		obs_encoder_set_last_error(encoder, text);
+		blog(LOG_ERROR, "[NVENC encoder] %s", text);
+		return NULL;
+	}
+	default:
+		switch (voi->colorspace) {
+		case VIDEO_CS_2100_PQ:
+		case VIDEO_CS_2100_HLG: {
+			const char *const text =
+				obs_module_text("NVENC.8bitUnsupportedHdr");
+			obs_encoder_set_last_error(encoder, text);
+			blog(LOG_ERROR, "[NVENC encoder] %s", text);
+			return NULL;
+		}
+		}
+	}
+
 	bool psycho_aq = obs_data_get_bool(settings, "psycho_aq");
 	void *enc = nvenc_create_internal(settings, encoder, psycho_aq, false);
 	if ((enc == NULL) && psycho_aq) {
@@ -328,6 +352,31 @@ static void *h264_nvenc_create(obs_data_t *settings, obs_encoder_t *encoder)
 #ifdef ENABLE_HEVC
 static void *hevc_nvenc_create(obs_data_t *settings, obs_encoder_t *encoder)
 {
+	video_t *video = obs_encoder_video(encoder);
+	const struct video_output_info *voi = video_output_get_info(video);
+	switch (voi->format) {
+	case VIDEO_FORMAT_I010: {
+		const char *const text =
+			obs_module_text("NVENC.I010Unsupported");
+		obs_encoder_set_last_error(encoder, text);
+		blog(LOG_ERROR, "[NVENC encoder] %s", text);
+		return NULL;
+	}
+	case VIDEO_FORMAT_P010:
+		break;
+	default:
+		switch (voi->colorspace) {
+		case VIDEO_CS_2100_PQ:
+		case VIDEO_CS_2100_HLG: {
+			const char *const text =
+				obs_module_text("NVENC.8bitUnsupportedHdr");
+			obs_encoder_set_last_error(encoder, text);
+			blog(LOG_ERROR, "[NVENC encoder] %s", text);
+			return NULL;
+		}
+		}
+	}
+
 	bool psycho_aq = obs_data_get_bool(settings, "psycho_aq");
 	void *enc = nvenc_create_internal(settings, encoder, psycho_aq, true);
 	if ((enc == NULL) && psycho_aq) {