libobs: Add HEVC parsing functions

Also create obs_nal_find_startcode from obs_avc_find_startcode to share
with HEVC functions.

libobs/CMakeLists.txt

@@ -31,11 +31,15 @@ target_sources(
           obs-encoder.c
           obs-encoder.h
           obs-ffmpeg-compat.h
+          obs-hevc.c
+          obs-hevc.h
           obs-hotkey.c
           obs-hotkey.h
           obs-hotkeys.h
           obs-missing-files.c
           obs-missing-files.h
+          obs-nal.c
+          obs-nal.h
           obs-hotkey-name-map.c
           obs-interaction.h
           obs-internal.h

libobs/obs-avc.c

@@ -15,17 +15,18 @@
     along with this program.  If not, see <http://www.gnu.org/licenses/>.
 ******************************************************************************/
 
-#include "obs.h"
 #include "obs-avc.h"
+
+#include "obs.h"
+#include "obs-nal.h"
 #include "util/array-serializer.h"
 
 bool obs_avc_keyframe(const uint8_t *data, size_t size)
 {
 	const uint8_t *nal_start, *nal_end;
 	const uint8_t *end = data + size;
-	int type;
 
-	nal_start = obs_avc_find_startcode(data, end);
+	nal_start = obs_nal_find_startcode(data, end);
 	while (true) {
 		while (nal_start < end && !*(nal_start++))
 			;
@@ -33,65 +34,21 @@ bool obs_avc_keyframe(const uint8_t *data, size_t size)
 		if (nal_start == end)
 			break;
 
-		type = nal_start[0] & 0x1F;
+		const uint8_t type = nal_start[0] & 0x1F;
 
 		if (type == OBS_NAL_SLICE_IDR || type == OBS_NAL_SLICE)
-			return (type == OBS_NAL_SLICE_IDR);
+			return type == OBS_NAL_SLICE_IDR;
 
-		nal_end = obs_avc_find_startcode(nal_start, end);
+		nal_end = obs_nal_find_startcode(nal_start, end);
 		nal_start = nal_end;
 	}
 
 	return false;
 }
 
-/* NOTE: I noticed that FFmpeg does some unusual special handling of certain
- * scenarios that I was unaware of, so instead of just searching for {0, 0, 1}
- * we'll just use the code from FFmpeg - http://www.ffmpeg.org/ */
-static const uint8_t *ff_avc_find_startcode_internal(const uint8_t *p,
-						     const uint8_t *end)
-{
-	const uint8_t *a = p + 4 - ((intptr_t)p & 3);
-
-	for (end -= 3; p < a && p < end; p++) {
-		if (p[0] == 0 && p[1] == 0 && p[2] == 1)
-			return p;
-	}
-
-	for (end -= 3; p < end; p += 4) {
-		uint32_t x = *(const uint32_t *)p;
-
-		if ((x - 0x01010101) & (~x) & 0x80808080) {
-			if (p[1] == 0) {
-				if (p[0] == 0 && p[2] == 1)
-					return p;
-				if (p[2] == 0 && p[3] == 1)
-					return p + 1;
-			}
-
-			if (p[3] == 0) {
-				if (p[2] == 0 && p[4] == 1)
-					return p + 2;
-				if (p[4] == 0 && p[5] == 1)
-					return p + 3;
-			}
-		}
-	}
-
-	for (end += 3; p < end; p++) {
-		if (p[0] == 0 && p[1] == 0 && p[2] == 1)
-			return p;
-	}
-
-	return end + 3;
-}
-
 const uint8_t *obs_avc_find_startcode(const uint8_t *p, const uint8_t *end)
 {
-	const uint8_t *out = ff_avc_find_startcode_internal(p, end);
-	if (p < out && out < end && !out[-1])
-		out--;
-	return out;
+	return obs_nal_find_startcode(p, end);
 }
 
 static inline int get_drop_priority(int priority)
@@ -233,7 +190,6 @@ void obs_extract_avc_headers(const uint8_t *packet, size_t size,
 	DARRAY(uint8_t) sei;
 	const uint8_t *nal_start, *nal_end, *nal_codestart;
 	const uint8_t *end = packet + size;
-	int type;
 
 	da_init(new_packet);
 	da_init(header);
@@ -250,7 +206,7 @@ void obs_extract_avc_headers(const uint8_t *packet, size_t size,
 		if (nal_start == end)
 			break;
 
-		type = nal_start[0] & 0x1F;
+		const uint8_t type = nal_start[0] & 0x1F;
 
 		nal_end = obs_avc_find_startcode(nal_start, end);
 		if (!nal_end)

libobs/obs-hevc.c

@@ -0,0 +1,148 @@
+/******************************************************************************
+    Copyright (C) 2022 by Hugh Bailey <[email protected]>
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+******************************************************************************/
+
+#include "obs-hevc.h"
+
+#include "obs-nal.h"
+#include "util/darray.h"
+
+enum {
+	OBS_HEVC_NAL_TRAIL_N = 0,
+	OBS_HEVC_NAL_TRAIL_R = 1,
+	OBS_HEVC_NAL_TSA_N = 2,
+	OBS_HEVC_NAL_TSA_R = 3,
+	OBS_HEVC_NAL_STSA_N = 4,
+	OBS_HEVC_NAL_STSA_R = 5,
+	OBS_HEVC_NAL_RADL_N = 6,
+	OBS_HEVC_NAL_RADL_R = 7,
+	OBS_HEVC_NAL_RASL_N = 8,
+	OBS_HEVC_NAL_RASL_R = 9,
+	OBS_HEVC_NAL_VCL_N10 = 10,
+	OBS_HEVC_NAL_VCL_R11 = 11,
+	OBS_HEVC_NAL_VCL_N12 = 12,
+	OBS_HEVC_NAL_VCL_R13 = 13,
+	OBS_HEVC_NAL_VCL_N14 = 14,
+	OBS_HEVC_NAL_VCL_R15 = 15,
+	OBS_HEVC_NAL_BLA_W_LP = 16,
+	OBS_HEVC_NAL_BLA_W_RADL = 17,
+	OBS_HEVC_NAL_BLA_N_LP = 18,
+	OBS_HEVC_NAL_IDR_W_RADL = 19,
+	OBS_HEVC_NAL_IDR_N_LP = 20,
+	OBS_HEVC_NAL_CRA_NUT = 21,
+	OBS_HEVC_NAL_RSV_IRAP_VCL22 = 22,
+	OBS_HEVC_NAL_RSV_IRAP_VCL23 = 23,
+	OBS_HEVC_NAL_RSV_VCL24 = 24,
+	OBS_HEVC_NAL_RSV_VCL25 = 25,
+	OBS_HEVC_NAL_RSV_VCL26 = 26,
+	OBS_HEVC_NAL_RSV_VCL27 = 27,
+	OBS_HEVC_NAL_RSV_VCL28 = 28,
+	OBS_HEVC_NAL_RSV_VCL29 = 29,
+	OBS_HEVC_NAL_RSV_VCL30 = 30,
+	OBS_HEVC_NAL_RSV_VCL31 = 31,
+	OBS_HEVC_NAL_VPS = 32,
+	OBS_HEVC_NAL_SPS = 33,
+	OBS_HEVC_NAL_PPS = 34,
+	OBS_HEVC_NAL_AUD = 35,
+	OBS_HEVC_NAL_EOS_NUT = 36,
+	OBS_HEVC_NAL_EOB_NUT = 37,
+	OBS_HEVC_NAL_FD_NUT = 38,
+	OBS_HEVC_NAL_SEI_PREFIX = 39,
+	OBS_HEVC_NAL_SEI_SUFFIX = 40,
+};
+
+bool obs_hevc_keyframe(const uint8_t *data, size_t size)
+{
+	const uint8_t *nal_start, *nal_end;
+	const uint8_t *end = data + size;
+
+	nal_start = obs_nal_find_startcode(data, end);
+	while (true) {
+		while (nal_start < end && !*(nal_start++))
+			;
+
+		if (nal_start == end)
+			break;
+
+		const uint8_t type = (nal_start[0] & 0x7F) >> 1;
+
+		if (type <= OBS_HEVC_NAL_RSV_IRAP_VCL23)
+			return type >= OBS_HEVC_NAL_BLA_W_LP;
+
+		nal_end = obs_nal_find_startcode(nal_start, end);
+		nal_start = nal_end;
+	}
+
+	return false;
+}
+
+void obs_extract_hevc_headers(const uint8_t *packet, size_t size,
+			      uint8_t **new_packet_data,
+			      size_t *new_packet_size, uint8_t **header_data,
+			      size_t *header_size, uint8_t **sei_data,
+			      size_t *sei_size)
+{
+	DARRAY(uint8_t) new_packet;
+	DARRAY(uint8_t) header;
+	DARRAY(uint8_t) sei;
+	const uint8_t *nal_start, *nal_end, *nal_codestart;
+	const uint8_t *end = packet + size;
+
+	da_init(new_packet);
+	da_init(header);
+	da_init(sei);
+
+	nal_start = obs_nal_find_startcode(packet, end);
+	nal_end = NULL;
+	while (nal_end != end) {
+		nal_codestart = nal_start;
+
+		while (nal_start < end && !*(nal_start++))
+			;
+
+		if (nal_start == end)
+			break;
+
+		const uint8_t type = (nal_start[0] & 0x7F) >> 1;
+
+		nal_end = obs_nal_find_startcode(nal_start, end);
+		if (!nal_end)
+			nal_end = end;
+
+		if (type == OBS_HEVC_NAL_VPS || type == OBS_HEVC_NAL_SPS ||
+		    type == OBS_HEVC_NAL_PPS) {
+			da_push_back_array(header, nal_codestart,
+					   nal_end - nal_codestart);
+		} else if (type == OBS_HEVC_NAL_SEI_PREFIX ||
+			   type == OBS_HEVC_NAL_SEI_SUFFIX) {
+			da_push_back_array(sei, nal_codestart,
+					   nal_end - nal_codestart);
+
+		} else {
+			da_push_back_array(new_packet, nal_codestart,
+					   nal_end - nal_codestart);
+		}
+
+		nal_start = nal_end;
+	}
+
+	*new_packet_data = new_packet.array;
+	*new_packet_size = new_packet.num;
+	*header_data = header.array;
+	*header_size = header.num;
+	*sei_data = sei.array;
+	*sei_size = sei.num;
+}

libobs/obs-hevc.h

@@ -0,0 +1,35 @@
+/******************************************************************************
+    Copyright (C) 2022 by Hugh Bailey <[email protected]>
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+******************************************************************************/
+
+#pragma once
+
+#include "util/c99defs.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+EXPORT bool obs_hevc_keyframe(const uint8_t *data, size_t size);
+EXPORT void obs_extract_hevc_headers(const uint8_t *packet, size_t size,
+				     uint8_t **new_packet_data,
+				     size_t *new_packet_size,
+				     uint8_t **header_data, size_t *header_size,
+				     uint8_t **sei_data, size_t *sei_size);
+
+#ifdef __cplusplus
+}
+#endif

libobs/obs-nal.c

@@ -0,0 +1,67 @@
+/******************************************************************************
+    Copyright (C) 2022 by Hugh Bailey <[email protected]>
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+******************************************************************************/
+
+#include "obs-nal.h"
+
+/* NOTE: I noticed that FFmpeg does some unusual special handling of certain
+ * scenarios that I was unaware of, so instead of just searching for {0, 0, 1}
+ * we'll just use the code from FFmpeg - http://www.ffmpeg.org/ */
+static const uint8_t *ff_avc_find_startcode_internal(const uint8_t *p,
+						     const uint8_t *end)
+{
+	const uint8_t *a = p + 4 - ((intptr_t)p & 3);
+
+	for (end -= 3; p < a && p < end; p++) {
+		if (p[0] == 0 && p[1] == 0 && p[2] == 1)
+			return p;
+	}
+
+	for (end -= 3; p < end; p += 4) {
+		uint32_t x = *(const uint32_t *)p;
+
+		if ((x - 0x01010101) & (~x) & 0x80808080) {
+			if (p[1] == 0) {
+				if (p[0] == 0 && p[2] == 1)
+					return p;
+				if (p[2] == 0 && p[3] == 1)
+					return p + 1;
+			}
+
+			if (p[3] == 0) {
+				if (p[2] == 0 && p[4] == 1)
+					return p + 2;
+				if (p[4] == 0 && p[5] == 1)
+					return p + 3;
+			}
+		}
+	}
+
+	for (end += 3; p < end; p++) {
+		if (p[0] == 0 && p[1] == 0 && p[2] == 1)
+			return p;
+	}
+
+	return end + 3;
+}
+
+const uint8_t *obs_nal_find_startcode(const uint8_t *p, const uint8_t *end)
+{
+	const uint8_t *out = ff_avc_find_startcode_internal(p, end);
+	if (p < out && out < end && !out[-1])
+		out--;
+	return out;
+}

libobs/obs-nal.h

@@ -0,0 +1,31 @@
+/******************************************************************************
+    Copyright (C) 2022 by Hugh Bailey <[email protected]>
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+******************************************************************************/
+
+#pragma once
+
+#include "util/c99defs.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+EXPORT const uint8_t *obs_nal_find_startcode(const uint8_t *p,
+					     const uint8_t *end);
+
+#ifdef __cplusplus
+}
+#endif