obs-ffmpeg, obs-outputs: Check return of obs_encoder_get_extra_data

A race condition can occur in obs-outputs where the send_thread is in
the process of (re)connecting but the encoder was shut down in the
meantime. This causes the expected header data to be garbage, resulting
in a crash.

plugins/obs-ffmpeg/obs-ffmpeg-mux.c

@@ -638,7 +638,8 @@ static bool send_audio_headers(struct ffmpeg_muxer *stream,
 	struct encoder_packet packet = {
 		.type = OBS_ENCODER_AUDIO, .timebase_den = 1, .track_idx = idx};
 
-	obs_encoder_get_extra_data(aencoder, &packet.data, &packet.size);
+	if (!obs_encoder_get_extra_data(aencoder, &packet.data, &packet.size))
+		return false;
 	return write_packet(stream, &packet);
 }
 
@@ -649,7 +650,8 @@ static bool send_video_headers(struct ffmpeg_muxer *stream)
 	struct encoder_packet packet = {.type = OBS_ENCODER_VIDEO,
 					.timebase_den = 1};
 
-	obs_encoder_get_extra_data(vencoder, &packet.data, &packet.size);
+	if (!obs_encoder_get_extra_data(vencoder, &packet.data, &packet.size))
+		return false;
 	return write_packet(stream, &packet);
 }
 

plugins/obs-outputs/flv-output.c

@@ -119,7 +119,8 @@ static void write_audio_header(struct flv_output *stream)
 	struct encoder_packet packet = {.type = OBS_ENCODER_AUDIO,
 					.timebase_den = 1};
 
-	obs_encoder_get_extra_data(aencoder, &packet.data, &packet.size);
+	if (!obs_encoder_get_extra_data(aencoder, &packet.data, &packet.size))
+		return;
 	write_packet(stream, &packet, true);
 }
 
@@ -133,7 +134,8 @@ static void write_video_header(struct flv_output *stream)
 	struct encoder_packet packet = {
 		.type = OBS_ENCODER_VIDEO, .timebase_den = 1, .keyframe = true};
 
-	obs_encoder_get_extra_data(vencoder, &header, &size);
+	if (!obs_encoder_get_extra_data(vencoder, &header, &size))
+		return;
 	packet.size = obs_parse_avc_header(&packet.data, header, size);
 	write_packet(stream, &packet, true);
 	bfree(packet.data);

plugins/obs-outputs/ftl-stream.c

@@ -553,7 +553,8 @@ static bool send_video_header(struct ftl_stream *stream, int64_t dts_usec)
 					.keyframe = true,
 					.dts_usec = dts_usec};
 
-	obs_encoder_get_extra_data(vencoder, &header, &size);
+	if (!obs_encoder_get_extra_data(vencoder, &header, &size))
+		return false;
 	packet.size = obs_parse_avc_header(&packet.data, header, size);
 	return send_packet(stream, &packet, true) >= 0;
 }

plugins/obs-outputs/rtmp-stream.c

@@ -804,7 +804,8 @@ static bool send_audio_header(struct rtmp_stream *stream, size_t idx,
 		return true;
 	}
 
-	obs_encoder_get_extra_data(aencoder, &header, &packet.size);
+	if (!obs_encoder_get_extra_data(aencoder, &header, &packet.size))
+		return false;
 	packet.data = bmemdup(header, packet.size);
 	return send_packet(stream, &packet, true, idx) >= 0;
 }
@@ -819,7 +820,8 @@ static bool send_video_header(struct rtmp_stream *stream)
 	struct encoder_packet packet = {
 		.type = OBS_ENCODER_VIDEO, .timebase_den = 1, .keyframe = true};
 
-	obs_encoder_get_extra_data(vencoder, &header, &size);
+	if (!obs_encoder_get_extra_data(vencoder, &header, &size))
+		return false;
 	packet.size = obs_parse_avc_header(&packet.data, header, size);
 	return send_packet(stream, &packet, true, 0) >= 0;
 }