libobs: Fix lockup when an encode call fails

(This commit also modifies the UI, obs-ffmpeg, and obs-output modules)

Fixes a long-time regression where the program would lock up if an
encode call fails.  Shuts down all outputs associated with the failing
encoder and displays an error message to the user.

Ideally, it would be best if a more detailed error could be displayed to
the user about the nature of the error, though the primary problem is
the encoder errors are typically not something the user would be able to
understand.  The current message is a bit of a generic error message;
improvement is welcome.

Another suggestion is to try to have the encoder restart seamlessly,
though it would take a significant amount of work to be able to make it
do something like that properly, and it sort of assumes that encoder
failures are sporadic, which may not necessarily be the case with some
hardware encoders on some systems.  It may be better just to use another
encoder in that case.  For now, seamless restart is ruled out.

UI/data/locale/en-US.ini

@@ -279,6 +279,7 @@ Output.StartRecordingFailed="Failed to start recording"
 Output.StartReplayFailed="Failed to start replay buffer"
 Output.StartFailedGeneric="Starting the output failed. Please check the log for details.\n\nNote: If you are using the NVENC or AMD encoders, make sure your video drivers are up to date."
 
+
 # output connect messages
 Output.ConnectFail.Title="Failed to connect"
 Output.ConnectFail.BadPath="Invalid Path or Connection URL. Please check your settings to confirm that they are valid."
@@ -287,6 +288,10 @@ Output.ConnectFail.InvalidStream="Could not access the specified channel or stre
 Output.ConnectFail.Error="An unexpected error occurred when trying to connect to the server. More information in the log file."
 Output.ConnectFail.Disconnected="Disconnected from server."
 
+# output streaming-related messages
+Output.StreamEncodeError.Title="Encoding error"
+Output.StreamEncodeError.Msg="An encoder error occurred while streaming."
+
 # output recording-related messages
 Output.RecordFail.Title="Failed to start recording"
 Output.RecordFail.Unsupported="The output format is either unsupported or does not support more than one audio track. Please check your settings and try again."
@@ -294,6 +299,7 @@ Output.RecordNoSpace.Title="Insufficient disk space"
 Output.RecordNoSpace.Msg="There is not sufficient disk space to continue recording."
 Output.RecordError.Title="Recording error"
 Output.RecordError.Msg="An unspecified error occurred while recording."
+Output.RecordError.EncodeErrorMsg="An encoder error occurred while recording."
 Output.ReplayBuffer.NoHotkey.Title="No hotkey set!"
 Output.ReplayBuffer.NoHotkey.Msg="No save hotkey set for replay buffer. Please set the \"Save\" hotkey to use for saving replay recordings."
 

UI/window-basic-main.cpp

@@ -5189,9 +5189,10 @@ void OBSBasic::StreamStopping()
 
 void OBSBasic::StreamingStop(int code, QString last_error)
 {
-	const char *errorDescription;
+	const char *errorDescription = "";
 	DStr errorMessage;
 	bool use_last_error = false;
+	bool encode_error = false;
 
 	switch (code) {
 	case OBS_OUTPUT_BAD_PATH:
@@ -5207,6 +5208,10 @@ void OBSBasic::StreamingStop(int code, QString last_error)
 		errorDescription = Str("Output.ConnectFail.InvalidStream");
 		break;
 
+	case OBS_OUTPUT_ENCODE_ERROR:
+		encode_error = true;
+		break;
+
 	default:
 	case OBS_OUTPUT_ERROR:
 		use_last_error = true;
@@ -5245,10 +5250,16 @@ void OBSBasic::StreamingStop(int code, QString last_error)
 
 	blog(LOG_INFO, STREAMING_STOP);
 
-	if (code != OBS_OUTPUT_SUCCESS && isVisible()) {
+	if (encode_error) {
+		OBSMessageBox::information(this,
+				QTStr("Output.StreamEncodeError.Title"),
+				QTStr("Output.StreamEncodeError.Msg"));
+
+	} else if (code != OBS_OUTPUT_SUCCESS && isVisible()) {
 		OBSMessageBox::information(this,
 				QTStr("Output.ConnectFail.Title"),
 				QT_UTF8(errorMessage));
+
 	} else if (code != OBS_OUTPUT_SUCCESS && !isVisible()) {
 		SysTrayNotify(QT_UTF8(errorDescription), QSystemTrayIcon::Warning);
 	}
@@ -5373,6 +5384,11 @@ void OBSBasic::RecordingStop(int code, QString last_error)
 				QTStr("Output.RecordFail.Title"),
 				QTStr("Output.RecordFail.Unsupported"));
 
+	} else if (code == OBS_OUTPUT_ENCODE_ERROR && isVisible()) {
+		OBSMessageBox::warning(this,
+				QTStr("Output.RecordError.Title"),
+				QTStr("Output.RecordError.EncodeErrorMsg"));
+
 	} else if (code == OBS_OUTPUT_NO_SPACE && isVisible()) {
 		OBSMessageBox::warning(this,
 				QTStr("Output.RecordNoSpace.Title"),

docs/sphinx/reference-outputs.rst

@@ -137,7 +137,10 @@ Output Definition Structure (obs_output_info)
    Only applies to outputs that are encoded.  Packets will always be
    given in monotonic timestamp order.
 
-   :param packet: The video or audio packet
+   :param packet: The video or audio packet.  If NULL, an encoder error
+                  occurred, and the output should call
+                  :c:func:`obs_output_signal_stop()` with the error code
+                  **OBS_OUTPUT_ENCODE_ERROR**.
 
 .. member:: void (*obs_output_info.update)(void *data, obs_data_t *settings)
 
@@ -252,6 +255,7 @@ Output Signals
                   | OBS_OUTPUT_DISCONNECTED   - Unexpectedly disconnected
                   | OBS_OUTPUT_UNSUPPORTED    - The settings, video/audio format, or codecs are unsupported by this output
                   | OBS_OUTPUT_NO_SPACE       - Ran out of disk space
+                  | OBS_OUTPUT_ENCODE_ERROR   - Encoder error
 
 **starting** (ptr output)
 

libobs/obs-defs.h

@@ -40,6 +40,7 @@
 #define OBS_OUTPUT_DISCONNECTED   -5
 #define OBS_OUTPUT_UNSUPPORTED    -6
 #define OBS_OUTPUT_NO_SPACE       -7
+#define OBS_OUTPUT_ENCODE_ERROR   -8
 
 #define OBS_VIDEO_SUCCESS           0
 #define OBS_VIDEO_FAIL             -1

libobs/obs-encoder.c

@@ -208,7 +208,7 @@ static void add_connection(struct obs_encoder *encoder)
 	set_encoder_active(encoder, true);
 }
 
-static void remove_connection(struct obs_encoder *encoder)
+static void remove_connection(struct obs_encoder *encoder, bool shutdown)
 {
 	if (encoder->info.type == OBS_ENCODER_AUDIO) {
 		audio_output_disconnect(encoder->media, encoder->mixer_idx,
@@ -221,7 +221,13 @@ static void remove_connection(struct obs_encoder *encoder)
 		}
 	}
 
-	obs_encoder_shutdown(encoder);
+	/* obs_encoder_shutdown locks init_mutex, so don't call it on encode
+	 * errors, otherwise you can get a deadlock with outputs when they end
+	 * data capture, which will lock init_mutex and the video callback
+	 * mutex in the reverse order.  instead, call shutdown before starting
+	 * up again */
+	if (shutdown)
+		obs_encoder_shutdown(encoder);
 	set_encoder_active(encoder, false);
 }
 
@@ -575,7 +581,7 @@ static inline bool obs_encoder_stop_internal(obs_encoder_t *encoder,
 	pthread_mutex_unlock(&encoder->callbacks_mutex);
 
 	if (last) {
-		remove_connection(encoder);
+		remove_connection(encoder, true);
 		encoder->initialized = false;
 
 		if (encoder->destroy_on_stop) {
@@ -834,10 +840,23 @@ static inline void send_packet(struct obs_encoder *encoder,
 void full_stop(struct obs_encoder *encoder)
 {
 	if (encoder) {
+		pthread_mutex_lock(&encoder->outputs_mutex);
+		for (size_t i = 0; i < encoder->outputs.num; i++) {
+			struct obs_output *output = encoder->outputs.array[i];
+			obs_output_force_stop(output);
+
+			pthread_mutex_lock(&output->interleaved_mutex);
+			output->info.encoded_packet(output->context.data, NULL);
+			pthread_mutex_unlock(&output->interleaved_mutex);
+		}
+		pthread_mutex_unlock(&encoder->outputs_mutex);
+
 		pthread_mutex_lock(&encoder->callbacks_mutex);
 		da_free(encoder->callbacks);
-		remove_connection(encoder);
 		pthread_mutex_unlock(&encoder->callbacks_mutex);
+
+		remove_connection(encoder, false);
+		encoder->initialized = false;
 	}
 }
 
@@ -845,9 +864,9 @@ void send_off_encoder_packet(obs_encoder_t *encoder, bool success,
 		bool received, struct encoder_packet *pkt)
 {
 	if (!success) {
-		full_stop(encoder);
 		blog(LOG_ERROR, "Error encoding with encoder '%s'",
 				encoder->context.name);
+		full_stop(encoder);
 		return;
 	}
 
@@ -876,7 +895,7 @@ void send_off_encoder_packet(obs_encoder_t *encoder, bool success,
 }
 
 static const char *do_encode_name = "do_encode";
-void do_encode(struct obs_encoder *encoder, struct encoder_frame *frame)
+bool do_encode(struct obs_encoder *encoder, struct encoder_frame *frame)
 {
 	profile_start(do_encode_name);
 	if (!encoder->profile_encoder_encode_name)
@@ -899,6 +918,8 @@ void do_encode(struct obs_encoder *encoder, struct encoder_frame *frame)
 	send_off_encoder_packet(encoder, success, received, &pkt);
 
 	profile_end(do_encode_name);
+
+	return success;
 }
 
 static const char *receive_video_name = "receive_video";
@@ -930,9 +951,8 @@ static void receive_video(void *param, struct video_data *frame)
 	enc_frame.frames = 1;
 	enc_frame.pts    = encoder->cur_pts;
 
-	do_encode(encoder, &enc_frame);
-
-	encoder->cur_pts += encoder->timebase_num;
+	if (do_encode(encoder, &enc_frame))
+		encoder->cur_pts += encoder->timebase_num;
 
 wait_for_audio:
 	profile_end(receive_video_name);
@@ -1040,7 +1060,7 @@ fail:
 	return success;
 }
 
-static void send_audio_data(struct obs_encoder *encoder)
+static bool send_audio_data(struct obs_encoder *encoder)
 {
 	struct encoder_frame  enc_frame;
 
@@ -1058,9 +1078,11 @@ static void send_audio_data(struct obs_encoder *encoder)
 	enc_frame.frames = (uint32_t)encoder->framesize;
 	enc_frame.pts    = encoder->cur_pts;
 
-	do_encode(encoder, &enc_frame);
+	if (!do_encode(encoder, &enc_frame))
+		return false;
 
 	encoder->cur_pts += encoder->framesize;
+	return true;
 }
 
 static const char *receive_audio_name = "receive_audio";
@@ -1079,8 +1101,11 @@ static void receive_audio(void *param, size_t mix_idx, struct audio_data *data)
 	if (!buffer_audio(encoder, data))
 		goto end;
 
-	while (encoder->audio_input_buffer[0].size >= encoder->framesize_bytes)
-		send_audio_data(encoder);
+	while (encoder->audio_input_buffer[0].size >= encoder->framesize_bytes) {
+		if (!send_audio_data(encoder)) {
+			break;
+		}
+	}
 
 	UNUSED_PARAMETER(mix_idx);
 

libobs/obs-internal.h

@@ -1042,7 +1042,7 @@ extern void obs_encoder_remove_output(struct obs_encoder *encoder,
 extern bool start_gpu_encode(obs_encoder_t *encoder);
 extern void stop_gpu_encode(obs_encoder_t *encoder);
 
-extern void do_encode(struct obs_encoder *encoder, struct encoder_frame *frame);
+extern bool do_encode(struct obs_encoder *encoder, struct encoder_frame *frame);
 extern void send_off_encoder_packet(obs_encoder_t *encoder, bool success,
 		bool received, struct encoder_packet *pkt);
 

plugins/obs-ffmpeg/obs-ffmpeg-mux.c

@@ -331,7 +331,7 @@ static bool ffmpeg_mux_start(void *data)
 	return true;
 }
 
-static int deactivate(struct ffmpeg_muxer *stream)
+static int deactivate(struct ffmpeg_muxer *stream, int code)
 {
 	int ret = -1;
 
@@ -345,8 +345,11 @@ static int deactivate(struct ffmpeg_muxer *stream)
 		info("Output of file '%s' stopped", stream->path.array);
 	}
 
-	if (stopping(stream))
+	if (code) {
+		obs_output_signal_stop(stream->output, code);
+	} else if (stopping(stream)) {
 		obs_output_end_data_capture(stream->output);
+	}
 
 	os_atomic_set_bool(&stream->stopping, false);
 	return ret;
@@ -380,7 +383,7 @@ static void signal_failure(struct ffmpeg_muxer *stream)
 		obs_output_set_last_error (stream->output, error);
 	}
 
-	ret = deactivate(stream);
+	ret = deactivate(stream, 0);
 
 	switch (ret) {
 	case FFM_UNSUPPORTED:          code = OBS_OUTPUT_UNSUPPORTED; break;
@@ -479,6 +482,12 @@ static void ffmpeg_mux_data(void *data, struct encoder_packet *packet)
 	if (!active(stream))
 		return;
 
+	/* encoder failure */
+	if (!packet) {
+		deactivate(stream, OBS_OUTPUT_ENCODE_ERROR);
+		return;
+	}
+
 	if (!stream->sent_headers) {
 		if (!send_headers(stream))
 			return;
@@ -488,7 +497,7 @@ static void ffmpeg_mux_data(void *data, struct encoder_packet *packet)
 
 	if (stopping(stream)) {
 		if (packet->sys_dts_usec >= stream->stop_ts) {
-			deactivate(stream);
+			deactivate(stream, 0);
 			return;
 		}
 	}
@@ -803,10 +812,13 @@ static void replay_buffer_save(struct ffmpeg_muxer *stream)
 			replay_buffer_mux_thread, stream) == 0;
 }
 
-static void deactivate_replay_buffer(struct ffmpeg_muxer *stream)
+static void deactivate_replay_buffer(struct ffmpeg_muxer *stream, int code)
 {
-	if (stopping(stream))
+	if (code) {
+		obs_output_signal_stop(stream->output, code);
+	} else if (stopping(stream)) {
 		obs_output_end_data_capture(stream->output);
+	}
 
 	os_atomic_set_bool(&stream->active, false);
 	os_atomic_set_bool(&stream->sent_headers, false);
@@ -822,9 +834,15 @@ static void replay_buffer_data(void *data, struct encoder_packet *packet)
 	if (!active(stream))
 		return;
 
+	/* encoder failure */
+	if (!packet) {
+		deactivate_replay_buffer(stream, OBS_OUTPUT_ENCODE_ERROR);
+		return;
+	}
+
 	if (stopping(stream)) {
 		if (packet->sys_dts_usec >= stream->stop_ts) {
-			deactivate_replay_buffer(stream);
+			deactivate_replay_buffer(stream, 0);
 			return;
 		}
 	}

plugins/obs-outputs/flv-output.c

@@ -193,7 +193,7 @@ static void flv_output_stop(void *data, uint64_t ts)
 	os_atomic_set_bool(&stream->stopping, true);
 }
 
-static void flv_output_actual_stop(struct flv_output *stream)
+static void flv_output_actual_stop(struct flv_output *stream, int code)
 {
 	os_atomic_set_bool(&stream->active, false);
 
@@ -203,7 +203,11 @@ static void flv_output_actual_stop(struct flv_output *stream)
 
 		fclose(stream->file);
 	}
-	obs_output_end_data_capture(stream->output);
+	if (code) {
+		obs_output_signal_stop(stream->output, code);
+	} else {
+		obs_output_end_data_capture(stream->output);
+	}
 
 	info("FLV file output complete");
 }
@@ -218,9 +222,14 @@ static void flv_output_data(void *data, struct encoder_packet *packet)
 	if (!active(stream))
 		goto unlock;
 
+	if (!packet) {
+		flv_output_actual_stop(stream, OBS_OUTPUT_ENCODE_ERROR);
+		goto unlock;
+	}
+
 	if (stopping(stream)) {
 		if (packet->sys_dts_usec >= (int64_t)stream->stop_ts) {
-			flv_output_actual_stop(stream);
+			flv_output_actual_stop(stream, 0);
 			goto unlock;
 		}
 	}

plugins/obs-outputs/ftl-stream.c

@@ -72,6 +72,7 @@ struct ftl_stream {
 
 	volatile bool    active;
 	volatile bool    disconnected;
+	volatile bool    encode_error;
 	pthread_t        send_thread;
 
 	int              max_shutdown_time_sec;
@@ -516,8 +517,12 @@ static void *send_thread(void *data)
 		}
 	}
 
+	bool encode_error = os_atomic_load_bool(&stream->encode_error);
+
 	if (disconnected(stream)) {
 		info("Disconnected from %s", stream->path.array);
+	} else if (encode_error) {
+		info("Encoder error, disconnecting");
 	} else {
 		info("User stopped the stream");
 	}
@@ -525,6 +530,8 @@ static void *send_thread(void *data)
 	if (!stopping(stream)) {
 		pthread_detach(stream->send_thread);
 		obs_output_signal_stop(stream->output, OBS_OUTPUT_DISCONNECTED);
+	} else if (encode_error) {
+		obs_output_signal_stop(stream->output, OBS_OUTPUT_ENCODE_ERROR);
 	} else {
 		obs_output_end_data_capture(stream->output);
 	}
@@ -809,6 +816,13 @@ static void ftl_stream_data(void *data, struct encoder_packet *packet)
 	if (disconnected(stream) || !active(stream))
 		return;
 
+	/* encoder failure */
+	if (!packet) {
+		os_atomic_set_bool(&stream->encode_error, true);
+		os_sem_post(stream->send_sem);
+		return;
+	}
+
 	if (packet->type == OBS_ENCODER_VIDEO)
 		obs_parse_avc_packet(&new_packet, packet);
 	else
@@ -1034,6 +1048,7 @@ static int init_connect(struct ftl_stream *stream)
 	}
 
 	os_atomic_set_bool(&stream->disconnected, false);
+	os_atomic_set_bool(&stream->encode_error, false);
 	stream->total_bytes_sent  = 0;
 	stream->dropped_frames    = 0;
 	stream->min_priority      = 0;

plugins/obs-outputs/rtmp-stream.c

@@ -487,8 +487,12 @@ static void *send_thread(void *data)
 		}
 	}
 
+	bool encode_error = os_atomic_load_bool(&stream->encode_error);
+
 	if (disconnected(stream)) {
 		info("Disconnected from %s", stream->path.array);
+	} else if (encode_error) {
+		info("Encoder error, disconnecting");
 	} else {
 		info("User stopped the stream");
 	}
@@ -507,6 +511,8 @@ static void *send_thread(void *data)
 	if (!stopping(stream)) {
 		pthread_detach(stream->send_thread);
 		obs_output_signal_stop(stream->output, OBS_OUTPUT_DISCONNECTED);
+	} else if (encode_error) {
+		obs_output_signal_stop(stream->output, OBS_OUTPUT_ENCODE_ERROR);
 	} else {
 		obs_output_end_data_capture(stream->output);
 	}
@@ -886,6 +892,7 @@ static bool init_connect(struct rtmp_stream *stream)
 		return false;
 
 	os_atomic_set_bool(&stream->disconnected, false);
+	os_atomic_set_bool(&stream->encode_error, false);
 	stream->total_bytes_sent = 0;
 	stream->dropped_frames   = 0;
 	stream->min_priority     = 0;
@@ -1103,6 +1110,13 @@ static void rtmp_stream_data(void *data, struct encoder_packet *packet)
 	if (disconnected(stream) || !active(stream))
 		return;
 
+	/* encoder fail */
+	if (!packet) {
+		os_atomic_set_bool(&stream->encode_error, true);
+		os_sem_post(stream->send_sem);
+		return;
+	}
+
 	if (packet->type == OBS_ENCODER_VIDEO) {
 		if (!stream->got_first_video) {
 			stream->start_dts_offset =

plugins/obs-outputs/rtmp-stream.h

@@ -59,6 +59,7 @@ struct rtmp_stream {
 
 	volatile bool    active;
 	volatile bool    disconnected;
+	volatile bool    encode_error;
 	pthread_t        send_thread;
 
 	int              max_shutdown_time_sec;