obs-ffmpeg: Improve dialog text for NVENC errors

Add dialog messages for lack of 10-bit support, and exceeding the
hardware B-frame limit. Also move code around to test these just once.

plugins/obs-ffmpeg/jim-nvenc-helpers.c

@@ -10,6 +10,26 @@ NV_CREATE_INSTANCE_FUNC nv_create_instance = NULL;
 
 #define error(format, ...) blog(LOG_ERROR, "[jim-nvenc] " format, ##__VA_ARGS__)
 
+bool nv_fail(obs_encoder_t *encoder, const char *format, ...)
+{
+	struct dstr message = {0};
+	struct dstr error_message = {0};
+
+	va_list args;
+	va_start(args, format);
+	dstr_vprintf(&message, format, args);
+	va_end(args);
+
+	dstr_printf(&error_message, "NVENC Error: %s", message.array);
+	obs_encoder_set_last_error(encoder, error_message.array);
+	error("%s", error_message.array);
+
+	dstr_free(&error_message);
+	dstr_free(&message);
+
+	return true;
+}
+
 bool nv_failed(obs_encoder_t *encoder, NVENCSTATUS err, const char *func,
 	       const char *call)
 {

plugins/obs-ffmpeg/jim-nvenc.c

@@ -89,6 +89,7 @@ struct nv_bitstream {
 	HANDLE event;
 };
 
+#define NV_FAIL(format, ...) nv_fail(enc->encoder, format, __VA_ARGS__)
 #define NV_FAILED(x) nv_failed(enc->encoder, x, __FUNCTION__, #x)
 
 static bool nv_bitstream_init(struct nvenc_data *enc, struct nv_bitstream *bs)
@@ -353,7 +354,7 @@ static bool init_session(struct nvenc_data *enc)
 }
 
 static bool init_encoder_h264(struct nvenc_data *enc, obs_data_t *settings,
-			      bool psycho_aq)
+			      int bf, bool psycho_aq)
 {
 	const char *rc = obs_data_get_string(settings, "rate_control");
 	int bitrate = (int)obs_data_get_int(settings, "bitrate");
@@ -363,17 +364,9 @@ static bool init_encoder_h264(struct nvenc_data *enc, obs_data_t *settings,
 	const char *preset = obs_data_get_string(settings, "preset");
 	const char *profile = obs_data_get_string(settings, "profile");
 	bool lookahead = obs_data_get_bool(settings, "lookahead");
-	int bf = (int)obs_data_get_int(settings, "bf");
 	bool vbr = astrcmpi(rc, "VBR") == 0;
 	NVENCSTATUS err;
 
-	const int bf_max = nv_get_cap_h264(enc, NV_ENC_CAPS_NUM_MAX_BFRAMES);
-	if (bf > bf_max) {
-		error("Max B-frames setting (%d) is more than GPU supports (%d)",
-		      bf, bf_max);
-		return false;
-	}
-
 	video_t *video = obs_encoder_video(enc->encoder);
 	const struct video_output_info *voi = video_output_get_info(video);
 
@@ -633,7 +626,7 @@ static bool init_encoder_h264(struct nvenc_data *enc, obs_data_t *settings,
 
 #ifdef ENABLE_HEVC
 static bool init_encoder_hevc(struct nvenc_data *enc, obs_data_t *settings,
-			      bool psycho_aq)
+			      int bf, bool psycho_aq)
 {
 	const char *rc = obs_data_get_string(settings, "rate_control");
 	int bitrate = (int)obs_data_get_int(settings, "bitrate");
@@ -643,17 +636,9 @@ static bool init_encoder_hevc(struct nvenc_data *enc, obs_data_t *settings,
 	const char *preset = obs_data_get_string(settings, "preset");
 	const char *profile = obs_data_get_string(settings, "profile");
 	bool lookahead = obs_data_get_bool(settings, "lookahead");
-	int bf = (int)obs_data_get_int(settings, "bf");
 	bool vbr = astrcmpi(rc, "VBR") == 0;
 	NVENCSTATUS err;
 
-	const int bf_max = nv_get_cap_hevc(enc, NV_ENC_CAPS_NUM_MAX_BFRAMES);
-	if (bf > bf_max) {
-		error("Max B-frames setting (%d) is more than GPU supports (%d)",
-		      bf, bf_max);
-		return false;
-	}
-
 	video_t *video = obs_encoder_video(enc->encoder);
 	const struct video_output_info *voi = video_output_get_info(video);
 
@@ -954,8 +939,67 @@ static bool init_textures(struct nvenc_data *enc)
 
 static void nvenc_destroy(void *data);
 
+static bool init_specific_encoder(struct nvenc_data *enc, bool hevc,
+				  obs_data_t *settings, obs_encoder_t *encoder,
+				  int bf, bool psycho_aq)
+{
+	bool init = false;
+
+#ifdef ENABLE_HEVC
+	if (hevc)
+		return init_encoder_hevc(enc, settings, bf, psycho_aq);
+#endif
+
+	return init_encoder_h264(enc, settings, bf, psycho_aq);
+}
+
+static bool init_encoder(struct nvenc_data *enc, bool hevc,
+			 obs_data_t *settings, obs_encoder_t *encoder)
+{
+	const int bf = (int)obs_data_get_int(settings, "bf");
+	const bool psycho_aq = obs_data_get_bool(settings, "psycho_aq");
+#ifdef ENABLE_HEVC
+	const bool support_10bit =
+		hevc ? nv_get_cap_hevc(enc, NV_ENC_CAPS_SUPPORT_10BIT_ENCODE)
+		     : nv_get_cap_h264(enc, NV_ENC_CAPS_SUPPORT_10BIT_ENCODE);
+	const int bf_max =
+		hevc ? nv_get_cap_hevc(enc, NV_ENC_CAPS_NUM_MAX_BFRAMES)
+		     : nv_get_cap_h264(enc, NV_ENC_CAPS_NUM_MAX_BFRAMES);
+#else
+	const bool support_10bit =
+		nv_get_cap_h264(enc, NV_ENC_CAPS_SUPPORT_10BIT_ENCODE);
+	const int bf_max = nv_get_cap_h264(enc, NV_ENC_CAPS_NUM_MAX_BFRAMES);
+#endif
+
+	if (obs_p010_tex_active() && !support_10bit) {
+		NV_FAIL("Cannot perform 10-bit encode on this encoder");
+		return false;
+	}
+
+	if (bf > bf_max) {
+		NV_FAIL("Max B-frames setting (%d) is more than encoder supports (%d)",
+			bf, bf_max);
+		return false;
+	}
+
+	if (!init_specific_encoder(enc, hevc, settings, encoder, bf,
+				   psycho_aq)) {
+		if (!psycho_aq)
+			return false;
+
+		blog(LOG_WARNING, "[jim-nvenc] init_specific_encoder failed, "
+				  "trying again without Psycho Visual Tuning");
+		if (!init_specific_encoder(enc, hevc, settings, encoder, bf,
+					   psycho_aq)) {
+			return false;
+		}
+	}
+
+	return true;
+}
+
 static void *nvenc_create_internal(bool hevc, obs_data_t *settings,
-				   obs_encoder_t *encoder, bool psycho_aq)
+				   obs_encoder_t *encoder)
 {
 	NV_ENCODE_API_FUNCTION_LIST init = {NV_ENCODE_API_FUNCTION_LIST_VER};
 	struct nvenc_data *enc = bzalloc(sizeof(*enc));
@@ -974,17 +1018,8 @@ static void *nvenc_create_internal(bool hevc, obs_data_t *settings,
 	if (!init_session(enc)) {
 		goto fail;
 	}
-#ifdef ENABLE_HEVC
-	if (hevc) {
-		if (!init_encoder_hevc(enc, settings, psycho_aq)) {
-			goto fail;
-		}
-	} else
-#endif
-	{
-		if (!init_encoder_h264(enc, settings, psycho_aq)) {
-			goto fail;
-		}
+	if (!init_encoder(enc, hevc, settings, encoder)) {
+		goto fail;
 	}
 	if (!init_bitstreams(enc)) {
 		goto fail;
@@ -1027,14 +1062,7 @@ static void *nvenc_create_h264_hevc(bool hevc, obs_data_t *settings,
 		goto reroute;
 	}
 
-	const bool psycho_aq = obs_data_get_bool(settings, "psycho_aq");
-	struct nvenc_data *enc =
-		nvenc_create_internal(hevc, settings, encoder, psycho_aq);
-	if ((enc == NULL) && psycho_aq) {
-		blog(LOG_WARNING, "[jim-nvenc] nvenc_create_internal failed, "
-				  "trying again without Psycho Visual Tuning");
-		enc = nvenc_create_internal(hevc, settings, encoder, false);
-	}
+	struct nvenc_data *enc = nvenc_create_internal(hevc, settings, encoder);
 
 	if (enc) {
 		return enc;

plugins/obs-ffmpeg/jim-nvenc.h

@@ -13,5 +13,6 @@ extern const char *nv_error_name(NVENCSTATUS err);
 extern NV_ENCODE_API_FUNCTION_LIST nv;
 extern NV_CREATE_INSTANCE_FUNC nv_create_instance;
 extern bool init_nvenc(obs_encoder_t *encoder);
+bool nv_fail(obs_encoder_t *encoder, const char *format, ...);
 bool nv_failed(obs_encoder_t *encoder, NVENCSTATUS err, const char *func,
 	       const char *call);