obs-ffmpeg: Log/fail NVENC for B-frame maximum

Check NV_ENC_CAPS_NUM_MAX_BFRAMES to avoid exceeding the GPU limit.

plugins/obs-ffmpeg/jim-nvenc.c

@@ -364,6 +364,13 @@ static bool init_encoder_h264(struct nvenc_data *enc, obs_data_t *settings,
 	bool vbr = astrcmpi(rc, "VBR") == 0;
 	NVENCSTATUS err;
 
+	const int bf_max = nv_get_cap_h264(enc, NV_ENC_CAPS_NUM_MAX_BFRAMES);
+	if (bf > bf_max) {
+		error("Max B-frames setting (%d) is more than GPU supports (%d)",
+		      bf, bf_max);
+		return false;
+	}
+
 	video_t *video = obs_encoder_video(enc->encoder);
 	const struct video_output_info *voi = video_output_get_info(video);
 
@@ -637,6 +644,13 @@ static bool init_encoder_hevc(struct nvenc_data *enc, obs_data_t *settings,
 	bool vbr = astrcmpi(rc, "VBR") == 0;
 	NVENCSTATUS err;
 
+	const int bf_max = nv_get_cap_hevc(enc, NV_ENC_CAPS_NUM_MAX_BFRAMES);
+	if (bf > bf_max) {
+		error("Max B-frames setting (%d) is more than GPU supports (%d)",
+		      bf, bf_max);
+		return false;
+	}
+
 	video_t *video = obs_encoder_video(enc->encoder);
 	const struct video_output_info *voi = video_output_get_info(video);