obs-ffmpeg: Set AMF codec level properly

The default "level" setting was being used for each codec (AVC, HEVC,
AV1) supported by AMF. For example, all HEVC encoders were using
level 6.2 and this caused some playback devices to reject the
bitstream for decode because the device reported a maximum decode
level lower than 6.2.

Add functionality to determine the best match for the codec level
instead of relying on the defaults.

plugins/obs-ffmpeg/texture-amf.cpp

@@ -72,6 +72,77 @@ static AMFTrace *amf_trace = nullptr;
 static HMODULE amf_module = nullptr;
 static uint64_t amf_version = 0;
 
+/* ================================================================================================================= */
+/* The structure and tables below are used to determine the appropriate minimum encoding level for the codecs. AMF
+ * defaults to the highest level for each codec (AVC, HEVC, AV1), and some client devices will reject playback if the
+ * codec level is higher than its decode abilities.
+ */
+
+struct codec_level_entry {
+	const char *level_str;
+	uint64_t max_luma_sample_rate;
+	uint64_t max_luma_picture_size;
+	amf_int64 amf_level;
+};
+
+// Ensure the table entries are ordered from lowest to highest
+static std::vector<codec_level_entry> avc_levels = {{"1", (uint64_t)1485 * 256, 99 * 256, AMF_H264_LEVEL__1},
+						    {"1.1", (uint64_t)3000 * 256, 396 * 256, AMF_H264_LEVEL__1_1},
+						    {"1.2", (uint64_t)6000 * 256, 396 * 256, AMF_H264_LEVEL__1_2},
+						    {"1.3", (uint64_t)11880 * 256, 396 * 256, AMF_H264_LEVEL__1_3},
+						    {"2", (uint64_t)11880 * 256, 396 * 256, AMF_H264_LEVEL__2},
+						    {"2.1", (uint64_t)19800 * 256, 792 * 256, AMF_H264_LEVEL__2_1},
+						    {"2.2", (uint64_t)20250 * 256, 1620 * 256, AMF_H264_LEVEL__2_2},
+						    {"3", (uint64_t)40500 * 256, 1620 * 256, AMF_H264_LEVEL__3},
+						    {"3.1", (uint64_t)108000 * 256, 3600 * 256, AMF_H264_LEVEL__3_1},
+						    {"3.2", (uint64_t)216000 * 256, 5120 * 256, AMF_H264_LEVEL__3_2},
+						    {"4", (uint64_t)245760 * 256, 8192 * 256, AMF_H264_LEVEL__4},
+						    {"4.1", (uint64_t)245760 * 256, 8192 * 256, AMF_H264_LEVEL__4_1},
+						    {"4.2", (uint64_t)522240 * 256, 8704 * 256, AMF_H264_LEVEL__4_2},
+						    {"5", (uint64_t)589824 * 256, 22080 * 256, AMF_H264_LEVEL__5},
+						    {"5.1", (uint64_t)983040 * 256, 36864 * 256, AMF_H264_LEVEL__5_1},
+						    {"5.2", (uint64_t)2073600 * 256, 36864 * 256, AMF_H264_LEVEL__5_2},
+						    {"6", (uint64_t)4177920 * 256, 139264 * 256, AMF_H264_LEVEL__6},
+						    {"6.1", (uint64_t)8355840 * 256, 139264 * 256, AMF_H264_LEVEL__6_1},
+						    {"6.2", (uint64_t)16711680 * 256, 139264 * 256,
+						     AMF_H264_LEVEL__6_2}};
+
+// Ensure the table entries are ordered from lowest to highest
+static std::vector<codec_level_entry> hevc_levels = {
+	{"1", 552960, 36864, AMF_LEVEL_1},           {"2", 3686400, 122880, AMF_LEVEL_2},
+	{"2.1", 7372800, 245760, AMF_LEVEL_2_1},     {"3", 16588800, 552960, AMF_LEVEL_3},
+	{"3.1", 33177600, 983040, AMF_LEVEL_3_1},    {"4", 66846720, 2228224, AMF_LEVEL_4},
+	{"4.1", 133693440, 2228224, AMF_LEVEL_4_1},  {"5", 267386880, 8912896, AMF_LEVEL_5},
+	{"5.1", 534773760, 8912896, AMF_LEVEL_5_1},  {"5.2", 1069547520, 8912896, AMF_LEVEL_5_2},
+	{"6", 1069547520, 35651584, AMF_LEVEL_6},    {"6.1", 2139095040, 35651584, AMF_LEVEL_6_1},
+	{"6.2", 4278190080, 35651584, AMF_LEVEL_6_2}};
+
+/* Ensure the table entries are ordered from lowest to highest.
+ *
+ * The AV1 specification currently defines 14 levels, even though more are available (reserved) such as 4.3 and 7.0.
+ *
+ * AV1 defines MaxDisplayRate and MaxDecodeRate, which correspond to TotalDisplayLumaSampleRate and
+ * TotalDecodedLumaSampleRate, respectively, defined in the specification. For the table below, MaxDecodeRate is being
+ * used because it corresponds to all frames with show_existing_frame=0.
+ *
+ * Refer to the following for more information: https://github.com/AOMediaCodec/av1-spec/blob/master/annex.a.levels.md
+ */
+static std::vector<codec_level_entry> av1_levels = {
+	{"2.0", (uint64_t)5529600, 147456, AMF_VIDEO_ENCODER_AV1_LEVEL_2_0},
+	{"2.1", (uint64_t)10454400, 278784, AMF_VIDEO_ENCODER_AV1_LEVEL_2_1},
+	{"3.0", (uint64_t)24969600, 665856, AMF_VIDEO_ENCODER_AV1_LEVEL_3_0},
+	{"3.1", (uint64_t)39938400, 1065024, AMF_VIDEO_ENCODER_AV1_LEVEL_3_1},
+	{"4.0", (uint64_t)77856768, 2359296, AMF_VIDEO_ENCODER_AV1_LEVEL_4_0},
+	{"4.1", (uint64_t)155713536, 2359296, AMF_VIDEO_ENCODER_AV1_LEVEL_4_1},
+	{"5.0", (uint64_t)273715200, 8912896, AMF_VIDEO_ENCODER_AV1_LEVEL_5_0},
+	{"5.1", (uint64_t)547430400, 8912896, AMF_VIDEO_ENCODER_AV1_LEVEL_5_1},
+	{"5.2", (uint64_t)1094860800, 8912896, AMF_VIDEO_ENCODER_AV1_LEVEL_5_2},
+	{"5.3", (uint64_t)1176502272, 8912896, AMF_VIDEO_ENCODER_AV1_LEVEL_5_3},
+	{"6.0", (uint64_t)1176502272, 35651584, AMF_VIDEO_ENCODER_AV1_LEVEL_6_0},
+	{"6.1", (uint64_t)2189721600, 35651584, AMF_VIDEO_ENCODER_AV1_LEVEL_6_1},
+	{"6.2", (uint64_t)4379443200, 35651584, AMF_VIDEO_ENCODER_AV1_LEVEL_6_2},
+	{"6.3", (uint64_t)4706009088, 35651584, AMF_VIDEO_ENCODER_AV1_LEVEL_6_3}};
+
 /* ========================================================================= */
 /* Main Implementation                                                       */
 
@@ -1297,6 +1368,57 @@ try {
 	return false;
 }
 
+static void amf_set_codec_level(amf_base *enc)
+{
+	uint64_t luma_pic_size = enc->cx * enc->cy;
+	uint64_t luma_sample_rate = luma_pic_size * (enc->fps_num / enc->fps_den);
+	std::vector<codec_level_entry> *levels;
+
+	if (enc->codec == amf_codec_type::AVC) {
+		levels = &avc_levels;
+	} else if (enc->codec == amf_codec_type::HEVC) {
+		levels = &hevc_levels;
+	} else if (enc->codec == amf_codec_type::AV1) {
+		levels = &av1_levels;
+	} else {
+		blog(LOG_ERROR, "%s: Unknown amf_codec_type", __FUNCTION__);
+		return;
+	}
+
+	std::vector<codec_level_entry>::const_iterator level_it = levels->begin();
+
+	// First check if the requested sample rate and/or picture size is too large for the maximum level.
+	if ((luma_sample_rate > levels->back().max_luma_sample_rate) ||
+	    (luma_pic_size > levels->back().max_luma_picture_size)) {
+		/* If the calculated sample rate is greater than the highest value supported by the codec, clamp to the
+		 * upper limit and log an error.
+		 */
+		level_it = --(levels->end());
+		blog(LOG_ERROR,
+		     "%s: Luma sample rate %u or luma pic size %u is greater than maximum "
+		     "allowed. Setting to level %s.",
+		     __FUNCTION__, luma_sample_rate, luma_pic_size, level_it->level_str);
+	} else {
+		// Walk the table and find the lowest codec level value suitable for the given luma sample rate.
+		while (level_it != levels->end()) {
+			if ((luma_sample_rate <= level_it->max_luma_sample_rate) &&
+			    (luma_pic_size <= level_it->max_luma_picture_size)) {
+				break;
+			}
+			++level_it;
+		}
+	}
+
+	// Set the level for the encoder
+	if (enc->codec == amf_codec_type::AVC) {
+		set_avc_property(enc, PROFILE_LEVEL, level_it->amf_level);
+	} else if (enc->codec == amf_codec_type::HEVC) {
+		set_hevc_property(enc, PROFILE_LEVEL, level_it->amf_level);
+	} else if (enc->codec == amf_codec_type::AV1) {
+		set_av1_property(enc, LEVEL, level_it->amf_level);
+	}
+}
+
 static bool amf_avc_init(void *data, obs_data_t *settings)
 {
 	amf_base *enc = (amf_base *)data;
@@ -1349,6 +1471,9 @@ static bool amf_avc_init(void *data, obs_data_t *settings)
 
 	set_avc_property(enc, DE_BLOCKING_FILTER, true);
 
+	// Determine and set the appropriate AVC level
+	amf_set_codec_level(enc);
+
 	check_preset_compatibility(enc, preset);
 
 	const char *ffmpeg_opts = obs_data_get_string(settings, "ffmpeg_opts");
@@ -1646,6 +1771,9 @@ static bool amf_hevc_init(void *data, obs_data_t *settings)
 
 	set_hevc_property(enc, GOP_SIZE, gop_size);
 
+	// Determine and set the appropriate HEVC level
+	amf_set_codec_level(enc);
+
 	check_preset_compatibility(enc, preset);
 
 	const char *ffmpeg_opts = obs_data_get_string(settings, "ffmpeg_opts");
@@ -1993,6 +2121,9 @@ static bool amf_av1_init(void *data, obs_data_t *settings)
 
 	set_av1_property(enc, ENFORCE_HRD, true);
 
+	// Determine and set the appropriate AV1 level
+	amf_set_codec_level(enc);
+
 	int keyint_sec = (int)obs_data_get_int(settings, "keyint_sec");
 	int gop_size = (keyint_sec) ? keyint_sec * enc->fps_num / enc->fps_den : 250;
 	set_av1_property(enc, GOP_SIZE, gop_size);