deps-libff: Add clock chaining to packets/frames

This attaches clocks to packets and frames and defers
the start time until that particular frame is presented.
Any packets/frames in the future with the same clock
will reference that start time.

This fixes issues when there are multiple start times
in a large buffer (looped video/images/audio) and different
frames need different reference clocks to present correctly.

deps/libff/libff/ff-audio-decoder.c

@@ -39,6 +39,22 @@ static inline void shrink_packet(struct ff_packet *packet, int packet_length)
 	}
 }
 
+static bool handle_reset_packet(struct ff_decoder *decoder,
+		struct ff_packet *packet)
+{
+	if (decoder->clock != NULL)
+		ff_clock_release(&decoder->clock);
+	decoder->clock = packet->clock;
+	av_free_packet(&packet->base);
+
+	// not a real packet, so try to get another packet
+	if (packet_queue_get(&decoder->packet_queue, packet, 1)
+			== FF_PACKET_FAIL)
+		return false;
+
+	return true;
+}
+
 static int decode_frame(struct ff_decoder *decoder,
 	struct ff_packet *packet, AVFrame *frame, bool *frame_complete)
 {
@@ -86,6 +102,11 @@ static int decode_frame(struct ff_decoder *decoder,
 				return -1;
 			}
 		}
+
+		// Packet has a new clock (reset packet)
+		if (packet->clock != NULL)
+			if (!handle_reset_packet(decoder, packet))
+				return -1;
 	}
 }
 
@@ -113,6 +134,7 @@ static bool queue_frame(struct ff_decoder *decoder, AVFrame *frame,
 		av_frame_free(&queue_frame->frame);
 
 	queue_frame->frame = av_frame_clone(frame);
+	queue_frame->clock = ff_clock_retain(decoder->clock);
 
 	if (call_initialize)
 		ff_callbacks_frame_initialize(queue_frame, decoder->callbacks);
@@ -154,6 +176,9 @@ void *ff_audio_decoder_thread(void *opaque_audio_decoder)
 		av_free_packet(&packet.base);
 	}
 
+	if (decoder->clock != NULL)
+		ff_clock_release(&decoder->clock);
+
 	av_frame_free(&frame);
 	return NULL;
 }

deps/libff/libff/ff-decoder.c

@@ -117,10 +117,13 @@ void ff_decoder_free(struct ff_decoder *decoder)
 
 		ff_callbacks_frame_free(frame, decoder->callbacks);
 
-		if (frame != NULL && frame->frame != NULL)
-			av_frame_unref(frame->frame);
-		if (frame != NULL)
+		if (frame != NULL) {
+			if (frame->frame != NULL)
+				av_frame_unref(frame->frame);
+			if (frame->clock != NULL)
+				ff_clock_release(&frame->clock);
 			av_free(frame);
+		}
 	}
 
 	packet_queue_free(&decoder->packet_queue);
@@ -143,11 +146,11 @@ double ff_decoder_clock(void *opaque)
 	return decoder->current_pts + delta;
 }
 
-static double get_sync_adjusted_pts_diff(struct ff_decoder *decoder,
+static double get_sync_adjusted_pts_diff(struct ff_clock *clock,
 		double pts, double pts_diff)
 {
 	double new_pts_diff = pts_diff;
-	double sync_time = ff_get_sync_clock(decoder->clock);
+	double sync_time = ff_get_sync_clock(clock);
 	double diff = pts - sync_time;
 	double sync_threshold;
 
@@ -191,6 +194,29 @@ void ff_decoder_refresh(void *opaque)
 			frame = ff_circular_queue_peek_read(
 					&decoder->frame_queue);
 
+			// Get frame clock and start it if needed
+			ff_clock_t *clock = ff_clock_move(&frame->clock);
+			if (!ff_clock_start(clock, decoder->natural_sync_clock,
+					&decoder->refresh_timer.abort)) {
+				ff_clock_release(&clock);
+
+				// Our clock was never started and deleted or
+				// aborted
+
+				// Drop this frame? The only way this can happen
+				// is if one stream finishes before another and
+				// the input is looping or canceled.  Until we
+				// get another clock we will unable to continue
+
+				ff_decoder_schedule_refresh(decoder, 100);
+
+				// Drop this frame as we have no way of timing
+				// it
+				ff_circular_queue_advance_read(
+						&decoder->frame_queue);
+				return;
+			}
+
 			decoder->current_pts = frame->pts;
 			decoder->current_pts_time = av_gettime();
 
@@ -208,9 +234,9 @@ void ff_decoder_refresh(void *opaque)
 			decoder->previous_pts = frame->pts;
 
 			// if not synced against natural clock
-			if (decoder->clock->sync_type
+			if (clock->sync_type
 					!= decoder->natural_sync_clock) {
-				pts_diff = get_sync_adjusted_pts_diff(decoder,
+				pts_diff = get_sync_adjusted_pts_diff(clock,
 						frame->pts, pts_diff);
 			}
 
@@ -224,6 +250,10 @@ void ff_decoder_refresh(void *opaque)
 				delay_until_next_wake = 0.010L;
 			}
 
+			if (delay_until_next_wake > pts_diff)
+				delay_until_next_wake = pts_diff;
+
+			ff_clock_release(&clock);
 			ff_callbacks_frame(decoder->callbacks, frame);
 
 			ff_decoder_schedule_refresh(decoder,

deps/libff/libff/ff-demuxer.c

@@ -196,8 +196,6 @@ static bool initialize_decoder(struct ff_demuxer *demuxer,
 		demuxer->audio_decoder->hwaccel_decoder = hwaccel_decoder;
 		demuxer->audio_decoder->natural_sync_clock =
 				AV_SYNC_AUDIO_MASTER;
-		demuxer->audio_decoder->clock = &demuxer->clock;
-
 		demuxer->audio_decoder->callbacks = &demuxer->audio_callbacks;
 
 		if (!ff_callbacks_format(&demuxer->audio_callbacks,
@@ -219,8 +217,6 @@ static bool initialize_decoder(struct ff_demuxer *demuxer,
 		demuxer->video_decoder->hwaccel_decoder = hwaccel_decoder;
 		demuxer->video_decoder->natural_sync_clock =
 				AV_SYNC_VIDEO_MASTER;
-		demuxer->video_decoder->clock = &demuxer->clock;
-
 		demuxer->video_decoder->callbacks = &demuxer->video_callbacks;
 
 		if (!ff_callbacks_format(&demuxer->video_callbacks,
@@ -316,6 +312,27 @@ void ff_demuxer_flush(struct ff_demuxer *demuxer)
 	}
 }
 
+void ff_demuxer_reset(struct ff_demuxer *demuxer)
+{
+	struct ff_packet *packet = av_mallocz(sizeof(struct ff_packet));
+	struct ff_clock *clock = ff_clock_init();
+	clock->sync_type = demuxer->clock.sync_type;
+	clock->sync_clock = demuxer->clock.sync_clock;
+	clock->opaque = demuxer->clock.opaque;
+
+	packet->clock = clock;
+
+	if (demuxer->audio_decoder != NULL) {
+		packet_queue_put(&demuxer->audio_decoder->packet_queue, packet);
+		ff_clock_retain(clock);
+	}
+
+	if (demuxer->video_decoder != NULL) {
+		packet_queue_put(&demuxer->video_decoder->packet_queue, packet);
+		ff_clock_retain(clock);
+	}
+}
+
 static bool open_input(struct ff_demuxer *demuxer,
 		AVFormatContext **format_context)
 {
@@ -446,6 +463,7 @@ static bool handle_seek(struct ff_demuxer *demuxer)
 		} else {
 			if (demuxer->seek_flush)
 				ff_demuxer_flush(demuxer);
+			ff_demuxer_reset(demuxer);
 		}
 
 		demuxer->seek_request = false;
@@ -477,6 +495,8 @@ static void *demux_thread(void *opaque)
 	if (!find_and_initialize_stream_decoders(demuxer))
 		goto fail;
 
+	ff_demuxer_reset(demuxer);
+
 	while (!demuxer->abort) {
 		// failed to seek (looping?)
 		if (!handle_seek(demuxer))

deps/libff/libff/ff-frame.h

@@ -16,11 +16,13 @@
 
 #pragma once
 
+#include "ff-clock.h"
+
 #include <libavcodec/avcodec.h>
 
 struct ff_frame {
 	AVFrame *frame;
-	void *opaque;
+	struct ff_clock *clock;
 	double pts;
 	int64_t duration;
 };

deps/libff/libff/ff-packet-queue.c

@@ -147,6 +147,8 @@ void packet_queue_flush(struct ff_packet_queue *q)
 			packet = q->first_packet) {
 		q->first_packet = packet->next;
 		av_free_packet(&packet->packet.base);
+		if (packet->packet.clock != NULL)
+			ff_clock_release(&packet->packet.clock);
 		av_freep(&packet);
 	}
 

deps/libff/libff/ff-packet-queue.h

@@ -16,6 +16,8 @@
 
 #pragma once
 
+#include "ff-clock.h"
+
 #include <libavformat/avformat.h>
 #include <pthread.h>
 #include <stdbool.h>
@@ -26,6 +28,7 @@
 
 struct ff_packet {
 	AVPacket base;
+	ff_clock_t *clock;
 };
 
 struct ff_packet_list {

deps/libff/libff/ff-video-decoder.c

@@ -53,6 +53,7 @@ static bool queue_frame(struct ff_decoder *decoder, AVFrame *frame,
 		av_frame_free(&queue_frame->frame);
 
 	queue_frame->frame = av_frame_clone(frame);
+	queue_frame->clock = ff_clock_retain(decoder->clock);
 
 	if (call_initialize)
 		ff_callbacks_frame_initialize(queue_frame, decoder->callbacks);
@@ -85,6 +86,15 @@ void *ff_video_decoder_thread(void *opaque_video_decoder)
 			continue;
 		}
 
+		// We received a reset packet with a new clock
+		if (packet.clock != NULL) {
+			if (decoder->clock != NULL)
+				ff_clock_release(&decoder->clock);
+			decoder->clock = ff_clock_move(&packet.clock);
+			av_free_packet(&packet.base);
+			continue;
+		}
+
 		avcodec_decode_video2(decoder->codec, frame,
 				&complete, &packet.base);
 
@@ -107,6 +117,9 @@ void *ff_video_decoder_thread(void *opaque_video_decoder)
 		av_free_packet(&packet.base);
 	}
 
+	if (decoder->clock != NULL)
+		ff_clock_release(&decoder->clock);
+
 	av_frame_free(&frame);
 	return NULL;
 }