libobs: Fix obs_parse_avc_header missing high profile parameters

libobs/obs-avc.c

@@ -20,6 +20,7 @@
 #include "obs.h"
 #include "obs-nal.h"
 #include "util/array-serializer.h"
+#include "util/bitstream.h"
 
 bool obs_avc_keyframe(const uint8_t *data, size_t size)
 {
@@ -170,6 +171,60 @@ static void get_sps_pps(const uint8_t *data, size_t size, const uint8_t **sps,
 	}
 }
 
+static inline uint8_t get_ue_golomb(struct bitstream_reader *gb)
+{
+	int i = 0;
+	while (i < 32 && !bitstream_reader_read_bits(gb, 1))
+		i++;
+
+	return bitstream_reader_read_bits(gb, i) + (1 << i) - 1;
+}
+
+static void get_sps_high_params(const uint8_t *sps, size_t size,
+				uint8_t *chroma_format_idc,
+				uint8_t *bit_depth_luma,
+				uint8_t *bit_depth_chroma)
+{
+	struct bitstream_reader gb;
+
+	/* Extract RBSP */
+	uint8_t *rbsp = bzalloc(size);
+
+	size_t i = 0;
+	size_t rbsp_size = 0;
+
+	while (i + 2 < size) {
+		if (sps[i] == 0 && sps[i + 1] == 0 && sps[i + 2] == 3) {
+			rbsp[rbsp_size++] = sps[i++];
+			rbsp[rbsp_size++] = sps[i++];
+			// skip emulation_prevention_three_byte
+			i++;
+		} else {
+			rbsp[rbsp_size++] = sps[i++];
+		}
+	}
+
+	while (i < size)
+		rbsp[rbsp_size++] = sps[i++];
+
+	/* Read relevant information from SPS */
+	bitstream_reader_init(&gb, rbsp, rbsp_size);
+
+	// skip a whole bunch of stuff we don't care about
+	bitstream_reader_read_bits(&gb, 24); // profile, constraint flags, level
+	get_ue_golomb(&gb);                  // id
+
+	*chroma_format_idc = get_ue_golomb(&gb);
+	// skip separate_colour_plane_flag
+	if (*chroma_format_idc == 3)
+		bitstream_reader_read_bits(&gb, 1);
+
+	*bit_depth_luma = get_ue_golomb(&gb);
+	*bit_depth_chroma = get_ue_golomb(&gb);
+
+	bfree(rbsp);
+}
+
 size_t obs_parse_avc_header(uint8_t **header, const uint8_t *data, size_t size)
 {
 	struct array_output_data output;
@@ -202,6 +257,26 @@ size_t obs_parse_avc_header(uint8_t **header, const uint8_t *data, size_t size)
 	s_wb16(&s, (uint16_t)pps_size);
 	s_write(&s, pps, pps_size);
 
+	uint8_t profile_idc = sps[1];
+
+	/* Additional data required for high, high10, high422, high444 profiles.
+	 * See ISO/IEC 14496-15 Section 5.3.3.1.2. */
+	if (profile_idc == 100 || profile_idc == 110 || profile_idc == 122 ||
+	    profile_idc == 244) {
+		uint8_t chroma_format_idc, bit_depth_luma, bit_depth_chroma;
+		get_sps_high_params(sps + 1, sps_size - 1, &chroma_format_idc,
+				    &bit_depth_luma, &bit_depth_chroma);
+
+		// reserved + chroma_format
+		s_w8(&s, 0xfc | chroma_format_idc);
+		// reserved + bit_depth_luma_minus8
+		s_w8(&s, 0xf8 | bit_depth_luma);
+		// reserved + bit_depth_chroma_minus8
+		s_w8(&s, 0xf8 | bit_depth_chroma);
+		// numOfSequenceParameterSetExt
+		s_w8(&s, 0);
+	}
+
 	*header = output.bytes.array;
 	return output.bytes.num;
 }