libobs: Consolidate H.264 priority scheme

The spec for nal_ref_idc doesn't indicate that it should be used for
priority like x264 seems to be using it for. NVENC seems to pass
different values, so let's not rely on it.

The new scheme might be naive, but we can iterate on it, and apply
evenly to all H.264 encoders.

libobs/obs-avc.c

@@ -51,19 +51,11 @@ const uint8_t *obs_avc_find_startcode(const uint8_t *p, const uint8_t *end)
 	return obs_nal_find_startcode(p, end);
 }
 
-static inline int get_drop_priority(int priority)
-{
-	return priority;
-}
-
 static void serialize_avc_data(struct serializer *s, const uint8_t *data,
 			       size_t size, bool *is_keyframe, int *priority)
 {
-	const uint8_t *nal_start, *nal_end;
-	const uint8_t *end = data + size;
-	int type;
-
-	nal_start = obs_avc_find_startcode(data, end);
+	const uint8_t *const end = data + size;
+	const uint8_t *nal_start = obs_nal_find_startcode(data, end);
 	while (true) {
 		while (nal_start < end && !*(nal_start++))
 			;
@@ -71,18 +63,23 @@ static void serialize_avc_data(struct serializer *s, const uint8_t *data,
 		if (nal_start == end)
 			break;
 
-		type = nal_start[0] & 0x1F;
+		const int type = nal_start[0] & 0x1F;
 
-		if (type == OBS_NAL_SLICE_IDR || type == OBS_NAL_SLICE) {
-			if (is_keyframe)
-				*is_keyframe = (type == OBS_NAL_SLICE_IDR);
-			if (priority)
-				*priority = nal_start[0] >> 5;
+		switch (type) {
+		case OBS_NAL_SLICE:
+			if (*priority < OBS_NAL_PRIORITY_HIGH)
+				*priority = OBS_NAL_PRIORITY_HIGH;
+			break;
+		case OBS_NAL_SLICE_IDR:
+			*is_keyframe = true;
+			*priority = OBS_NAL_PRIORITY_HIGHEST;
 		}
 
-		nal_end = obs_avc_find_startcode(nal_start, end);
-		s_wb32(s, (uint32_t)(nal_end - nal_start));
-		s_write(s, nal_start, nal_end - nal_start);
+		const uint8_t *const nal_end =
+			obs_nal_find_startcode(nal_start, end);
+		const size_t size = nal_end - nal_start;
+		s_wb32(s, (uint32_t)size);
+		s_write(s, nal_start, size);
 		nal_start = nal_end;
 	}
 }
@@ -103,7 +100,7 @@ void obs_parse_avc_packet(struct encoder_packet *avc_packet,
 
 	avc_packet->data = output.bytes.array + sizeof(ref);
 	avc_packet->size = output.bytes.num - sizeof(ref);
-	avc_packet->drop_priority = get_drop_priority(avc_packet->priority);
+	avc_packet->drop_priority = avc_packet->priority;
 }
 
 static inline bool has_start_code(const uint8_t *data)
@@ -121,7 +118,7 @@ static void get_sps_pps(const uint8_t *data, size_t size, const uint8_t **sps,
 	const uint8_t *end = data + size;
 	int type;
 
-	nal_start = obs_avc_find_startcode(data, end);
+	nal_start = obs_nal_find_startcode(data, end);
 	while (true) {
 		while (nal_start < end && !*(nal_start++))
 			;
@@ -129,7 +126,7 @@ static void get_sps_pps(const uint8_t *data, size_t size, const uint8_t **sps,
 		if (nal_start == end)
 			break;
 
-		nal_end = obs_avc_find_startcode(nal_start, end);
+		nal_end = obs_nal_find_startcode(nal_start, end);
 
 		type = nal_start[0] & 0x1F;
 		if (type == OBS_NAL_SPS) {
@@ -195,7 +192,7 @@ void obs_extract_avc_headers(const uint8_t *packet, size_t size,
 	da_init(header);
 	da_init(sei);
 
-	nal_start = obs_avc_find_startcode(packet, end);
+	nal_start = obs_nal_find_startcode(packet, end);
 	nal_end = NULL;
 	while (nal_end != end) {
 		nal_codestart = nal_start;
@@ -208,7 +205,7 @@ void obs_extract_avc_headers(const uint8_t *packet, size_t size,
 
 		const uint8_t type = nal_start[0] & 0x1F;
 
-		nal_end = obs_avc_find_startcode(nal_start, end);
+		nal_end = obs_nal_find_startcode(nal_start, end);
 		if (!nal_end)
 			nal_end = end;
 

libobs/obs-avc.h

@@ -17,7 +17,7 @@
 
 #pragma once
 
-#include "util/c99defs.h"
+#include "obs-nal.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -39,13 +39,6 @@ enum {
 	OBS_NAL_FILLER = 12,
 };
 
-enum {
-	OBS_NAL_PRIORITY_DISPOSABLE = 0,
-	OBS_NAL_PRIORITY_LOW = 1,
-	OBS_NAL_PRIORITY_HIGH = 2,
-	OBS_NAL_PRIORITY_HIGHEST = 3,
-};
-
 /* Helpers for parsing AVC NAL units.  */
 
 EXPORT bool obs_avc_keyframe(const uint8_t *data, size_t size);

libobs/obs-nal.h

@@ -23,6 +23,13 @@
 extern "C" {
 #endif
 
+enum {
+	OBS_NAL_PRIORITY_DISPOSABLE = 0,
+	OBS_NAL_PRIORITY_LOW = 1,
+	OBS_NAL_PRIORITY_HIGH = 2,
+	OBS_NAL_PRIORITY_HIGHEST = 3,
+};
+
 EXPORT const uint8_t *obs_nal_find_startcode(const uint8_t *p,
 					     const uint8_t *end);