obs-ffmpeg: NVENC usage fixes

Only use lossless encode if the capability is supported.

Set qpPrimeYZeroTransformBypassFlag to 1 for lossless.

Do not set profileGUID for lossless.

For both NVENC implementations, retry with a heavier reset because both
are unable to recover from failure lightly.

plugins/obs-ffmpeg/jim-nvenc.c

@@ -323,7 +323,8 @@ static bool init_session(struct nvenc_data *enc)
 	return true;
 }
 
-static bool init_encoder(struct nvenc_data *enc, obs_data_t *settings)
+static bool init_encoder(struct nvenc_data *enc, obs_data_t *settings,
+			 bool psycho_aq)
 {
 	const char *rc = obs_data_get_string(settings, "rate_control");
 	int bitrate = (int)obs_data_get_int(settings, "bitrate");
@@ -332,7 +333,6 @@ static bool init_encoder(struct nvenc_data *enc, obs_data_t *settings)
 	int keyint_sec = (int)obs_data_get_int(settings, "keyint_sec");
 	const char *preset = obs_data_get_string(settings, "preset");
 	const char *profile = obs_data_get_string(settings, "profile");
-	bool psycho_aq = obs_data_get_bool(settings, "psycho_aq");
 	bool lookahead = obs_data_get_bool(settings, "lookahead");
 	int bf = (int)obs_data_get_int(settings, "bf");
 	bool vbr = astrcmpi(rc, "VBR") == 0;
@@ -377,9 +377,16 @@ static bool init_encoder(struct nvenc_data *enc, obs_data_t *settings)
 		ll = true;
 	}
 
-	if (astrcmpi(rc, "lossless") == 0) {
-		nv_preset = hp ? NV_ENC_PRESET_LOSSLESS_HP_GUID
-			       : NV_ENC_PRESET_LOSSLESS_DEFAULT_GUID;
+	const bool rc_lossless = astrcmpi(rc, "lossless") == 0;
+	bool lossless = rc_lossless;
+	if (rc_lossless) {
+		lossless = nv_get_cap(enc, NV_ENC_CAPS_SUPPORT_LOSSLESS_ENCODE);
+		if (lossless) {
+			nv_preset = hp ? NV_ENC_PRESET_LOSSLESS_HP_GUID
+				       : NV_ENC_PRESET_LOSSLESS_DEFAULT_GUID;
+		} else {
+			warn("lossless encode is not supported, ignoring");
+		}
 	}
 
 	/* -------------------------- */
@@ -522,9 +529,11 @@ static bool init_encoder(struct nvenc_data *enc, obs_data_t *settings)
 	config->rcParams.rateControlMode = twopass ? NV_ENC_PARAMS_RC_VBR_HQ
 						   : NV_ENC_PARAMS_RC_VBR;
 
-	if (astrcmpi(rc, "cqp") == 0 || astrcmpi(rc, "lossless") == 0) {
-		if (astrcmpi(rc, "lossless") == 0)
+	if (astrcmpi(rc, "cqp") == 0 || rc_lossless) {
+		if (lossless) {
+			h264_config->qpPrimeYZeroTransformBypassFlag = 1;
 			cqp = 0;
+		}
 
 		config->rcParams.rateControlMode = NV_ENC_PARAMS_RC_CONSTQP;
 		config->rcParams.constQP.qpInterP = cqp;
@@ -554,24 +563,14 @@ static bool init_encoder(struct nvenc_data *enc, obs_data_t *settings)
 		config->profileGUID = NV_ENC_H264_PROFILE_MAIN_GUID;
 	} else if (astrcmpi(profile, "baseline") == 0) {
 		config->profileGUID = NV_ENC_H264_PROFILE_BASELINE_GUID;
-	} else {
+	} else if (!lossless) {
 		config->profileGUID = NV_ENC_H264_PROFILE_HIGH_GUID;
 	}
 
 	/* -------------------------- */
 	/* initialize                 */
 
-	err = nv.nvEncInitializeEncoder(enc->session, params);
-	if ((err == NV_ENC_ERR_INVALID_PARAM) && psycho_aq) {
-		warn("nvEncInitializeEncoder failed, "
-		     "trying again without Psycho Visual Tuning");
-		config->rcParams.enableAQ = 0;
-		config->rcParams.enableTemporalAQ = 0;
-		err = nv.nvEncInitializeEncoder(enc->session, params);
-	}
-
-	if (nv_failed(enc->encoder, err, __FUNCTION__,
-		      "nvEncInitializeEncoder")) {
+	if (NV_FAILED(nv.nvEncInitializeEncoder(enc->session, params))) {
 		return false;
 	}
 
@@ -627,29 +626,14 @@ static bool init_textures(struct nvenc_data *enc)
 
 static void nvenc_destroy(void *data);
 
-static void *nvenc_create(obs_data_t *settings, obs_encoder_t *encoder)
+static void *nvenc_create_internal(obs_data_t *settings, obs_encoder_t *encoder,
+				   bool psycho_aq)
 {
 	NV_ENCODE_API_FUNCTION_LIST init = {NV_ENCODE_API_FUNCTION_LIST_VER};
 	struct nvenc_data *enc = bzalloc(sizeof(*enc));
 	enc->encoder = encoder;
 	enc->first_packet = true;
 
-	/* this encoder requires shared textures, this cannot be used on a
-	 * gpu other than the one OBS is currently running on. */
-	int gpu = (int)obs_data_get_int(settings, "gpu");
-	if (gpu != 0) {
-		info("different GPU selected by user, falling back to ffmpeg");
-		goto fail;
-	}
-
-	if (obs_encoder_scaling_enabled(encoder)) {
-		info("scaling enabled, falling back to ffmpeg");
-		goto fail;
-	}
-	if (!obs_nv12_tex_active()) {
-		info("nv12 not active, falling back to ffmpeg");
-		goto fail;
-	}
 	if (!init_nvenc(encoder)) {
 		goto fail;
 	}
@@ -662,7 +646,7 @@ static void *nvenc_create(obs_data_t *settings, obs_encoder_t *encoder)
 	if (!init_session(enc)) {
 		goto fail;
 	}
-	if (!init_encoder(enc, settings)) {
+	if (!init_encoder(enc, settings, psycho_aq)) {
 		goto fail;
 	}
 	if (!init_bitstreams(enc)) {
@@ -676,6 +660,46 @@ static void *nvenc_create(obs_data_t *settings, obs_encoder_t *encoder)
 
 fail:
 	nvenc_destroy(enc);
+	return NULL;
+}
+
+static void *nvenc_create(obs_data_t *settings, obs_encoder_t *encoder)
+{
+	/* this encoder requires shared textures, this cannot be used on a
+	 * gpu other than the one OBS is currently running on. */
+	const int gpu = (int)obs_data_get_int(settings, "gpu");
+	if (gpu != 0) {
+		blog(LOG_INFO,
+		     "[jim-nvenc] different GPU selected by user, falling back to ffmpeg");
+		goto reroute;
+	}
+
+	if (obs_encoder_scaling_enabled(encoder)) {
+		blog(LOG_INFO,
+		     "[jim-nvenc] scaling enabled, falling back to ffmpeg");
+		goto reroute;
+	}
+
+	if (!obs_nv12_tex_active()) {
+		blog(LOG_INFO,
+		     "[jim-nvenc] nv12 not active, falling back to ffmpeg");
+		goto reroute;
+	}
+
+	const bool psycho_aq = obs_data_get_bool(settings, "psycho_aq");
+	struct nvenc_data *enc =
+		nvenc_create_internal(settings, encoder, psycho_aq);
+	if ((enc == NULL) && psycho_aq) {
+		blog(LOG_WARNING, "[jim-nvenc] nvenc_create_internal failed, "
+				  "trying again without Psycho Visual Tuning");
+		enc = nvenc_create_internal(settings, encoder, false);
+	}
+
+	if (enc) {
+		return enc;
+	}
+
+reroute:
 	return obs_encoder_create_rerouted(encoder, "ffmpeg_nvenc");
 }
 

plugins/obs-ffmpeg/obs-ffmpeg-nvenc.c

@@ -102,13 +102,6 @@ static bool nvenc_init_codec(struct nvenc_encoder *enc, bool psycho_aq)
 	}
 
 	ret = avcodec_open2(enc->context, enc->nvenc, NULL);
-	if ((ret < 0) && psycho_aq) {
-		warn("avcodec_open2 failed, "
-		     "trying again without Psycho Visual Tuning");
-		set_psycho_aq(enc, false);
-		ret = avcodec_open2(enc->context, enc->nvenc, NULL);
-	}
-
 	if (ret < 0) {
 		// if we were a fallback from jim-nvenc, there may already be a
 		// more useful error returned from that, so don't overwrite.
@@ -173,10 +166,9 @@ static bool nvenc_init_codec(struct nvenc_encoder *enc, bool psycho_aq)
 
 enum RC_MODE { RC_MODE_CBR, RC_MODE_VBR, RC_MODE_CQP, RC_MODE_LOSSLESS };
 
-static bool nvenc_update(void *data, obs_data_t *settings)
+static bool nvenc_update(struct nvenc_encoder *enc, obs_data_t *settings,
+			 bool psycho_aq)
 {
-	struct nvenc_encoder *enc = data;
-
 	const char *rc = obs_data_get_string(settings, "rate_control");
 	int bitrate = (int)obs_data_get_int(settings, "bitrate");
 	int cqp = (int)obs_data_get_int(settings, "cqp");
@@ -186,7 +178,6 @@ static bool nvenc_update(void *data, obs_data_t *settings)
 	int gpu = (int)obs_data_get_int(settings, "gpu");
 	bool cbr_override = obs_data_get_bool(settings, "cbr");
 	int bf = (int)obs_data_get_int(settings, "bf");
-	bool psycho_aq = obs_data_get_bool(settings, "psycho_aq");
 
 	video_t *video = obs_encoder_video(enc->encoder);
 	const struct video_output_info *voi = video_output_get_info(video);
@@ -357,7 +348,8 @@ static void nvenc_destroy(void *data)
 	bfree(enc);
 }
 
-static void *nvenc_create(obs_data_t *settings, obs_encoder_t *encoder)
+static void *nvenc_create_internal(obs_data_t *settings, obs_encoder_t *encoder,
+				   bool psycho_aq)
 {
 	struct nvenc_encoder *enc;
 
@@ -387,7 +379,7 @@ static void *nvenc_create(obs_data_t *settings, obs_encoder_t *encoder)
 		goto fail;
 	}
 
-	if (!nvenc_update(enc, settings))
+	if (!nvenc_update(enc, settings, psycho_aq))
 		goto fail;
 
 	return enc;
@@ -397,6 +389,18 @@ fail:
 	return NULL;
 }
 
+static void *nvenc_create(obs_data_t *settings, obs_encoder_t *encoder)
+{
+	bool psycho_aq = obs_data_get_bool(settings, "psycho_aq");
+	void *enc = nvenc_create_internal(settings, encoder, psycho_aq);
+	if ((enc == NULL) && psycho_aq) {
+		blog(LOG_WARNING,
+		     "[NVENC encoder] nvenc_create_internal failed, "
+		     "trying again without Psycho Visual Tuning");
+		enc = nvenc_create_internal(settings, encoder, false);
+	}
+}
+
 static inline void copy_data(AVFrame *pic, const struct encoder_frame *frame,
 			     int height, enum AVPixelFormat format)
 {