Add packet interleaving and improve encoder API

 - Add interleaving of video/audio packets for outputs that are encoded
   and expect both video and audio data, sorting the packets and sending
   them to the output when both video and audio is received.

 - Combine create and initialize callbacks for the encoder API callback
   interface.

libobs/obs-encoder.c

@@ -51,14 +51,6 @@ static bool init_encoder(struct obs_encoder *encoder, const char *name,
 	if (encoder->info.defaults)
 		encoder->info.defaults(encoder->settings);
 
-	encoder->data = encoder->info.create(encoder->settings, encoder);
-
-	if (!encoder->data) {
-		pthread_mutex_destroy(&encoder->callbacks_mutex);
-		obs_data_release(encoder->settings);
-		return false;
-	}
-
 	pthread_mutex_lock(&obs->data.encoders_mutex);
 	da_push_back(obs->data.encoders, &encoder);
 	pthread_mutex_unlock(&obs->data.encoders_mutex);
@@ -188,7 +180,8 @@ static void obs_encoder_actually_destroy(obs_encoder_t encoder)
 		da_free(encoder->outputs);
 		pthread_mutex_unlock(&encoder->outputs_mutex);
 
-		encoder->info.destroy(encoder->data);
+		if (encoder->data)
+			encoder->info.destroy(encoder->data);
 		obs_data_release(encoder->settings);
 		pthread_mutex_destroy(&encoder->callbacks_mutex);
 		pthread_mutex_destroy(&encoder->outputs_mutex);
@@ -265,15 +258,16 @@ void obs_encoder_update(obs_encoder_t encoder, obs_data_t settings)
 	if (!encoder) return;
 
 	obs_data_apply(encoder->settings, settings);
-	if (encoder->info.update)
+	if (encoder->info.update && encoder->data)
 		encoder->info.update(encoder->data, encoder->settings);
 }
 
 bool obs_encoder_get_extra_data(obs_encoder_t encoder, uint8_t **extra_data,
 		size_t *size)
 {
-	if (encoder && encoder->info.extra_data)
-		return encoder->info.extra_data(encoder, extra_data, size);
+	if (encoder && encoder->info.extra_data && encoder->data)
+		return encoder->info.extra_data(encoder->data, extra_data,
+				size);
 
 	return false;
 }
@@ -293,9 +287,11 @@ bool obs_encoder_initialize(obs_encoder_t encoder)
 	if (encoder->active)
 		return true;
 
-	encoder->initialized = encoder->info.initialize(encoder,
-			encoder->settings);
-	return encoder->initialized;
+	if (encoder->data)
+		encoder->info.destroy(encoder->data);
+
+	encoder->data = encoder->info.create(encoder->settings, encoder);
+	return encoder->data != NULL;
 }
 
 static inline size_t get_callback_idx(
@@ -321,7 +317,7 @@ void obs_encoder_start(obs_encoder_t encoder,
 	bool success = true;
 	bool first   = false;
 
-	if (!encoder || !new_packet || !encoder->initialized) return;
+	if (!encoder || !new_packet || !encoder->data) return;
 
 	pthread_mutex_lock(&encoder->callbacks_mutex);
 

libobs/obs-encoder.h

@@ -109,7 +109,8 @@ struct obs_encoder_info {
 	 *
 	 * @param  settings  Settings for the encoder
 	 * @param  encoder   OBS encoder context
-	 * @return           Data associated with this encoder context
+	 * @return           Data associated with this encoder context, or
+	 *                   NULL if initialization failed.
 	 */
 	void *(*create)(obs_data_t settings, obs_encoder_t encoder);
 
@@ -120,16 +121,6 @@ struct obs_encoder_info {
 	 */
 	void (*destroy)(void *data);
 
-	/**
-	 * Initializes the encoder with the specified settings
-	 *
-	 * @param  data      Data associated with this encoder context
-	 * @param  settings  Settings for the encoder
-	 * @return           true if the encoder settings are valid and the
-	 *                   encoder is ready to be used, false otherwise
-	 */
-	bool (*initialize)(void *data, obs_data_t settings);
-
 	/**
 	 * Encodes frame(s), and outputs encoded packets as they become
 	 * available.

libobs/obs-internal.h

@@ -259,6 +259,12 @@ extern void obs_source_video_tick(obs_source_t source, float seconds);
 /* ------------------------------------------------------------------------- */
 /* outputs  */
 
+struct il_packet {
+	int64_t                         input_ts_us;
+	int64_t                         output_ts_us;
+	struct encoder_packet           packet;
+};
+
 struct obs_output {
 	char                            *name;
 	void                            *data;
@@ -268,8 +274,13 @@ struct obs_output {
 	signal_handler_t                signals;
 	proc_handler_t                  procs;
 
+	bool                            received_video;
+	bool                            received_audio;
+	int64_t                         first_video_ts;
+	int64_t                         video_offset;
+	int64_t                         audio_offset;
 	pthread_mutex_t                 interleaved_mutex;
-	DARRAY(struct encoder_packet)   interleaved_packets;
+	DARRAY(struct il_packet)        interleaved_packets;
 
 	bool                            active;
 	video_t                         video;
@@ -304,7 +315,6 @@ struct obs_encoder {
 	struct obs_encoder_info         info;
 	obs_data_t                      settings;
 
-	bool                            initialized;
 	bool                            active;
 
 	uint32_t                        timebase_num;

libobs/obs-module.c

@@ -1,5 +1,5 @@
 /******************************************************************************
-    Copyright (C) 2013 by Hugh Bailey <[email protected]>
+    Copyright (C) 2013-2014 by Hugh Bailey <[email protected]>
 
     This program is free software: you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
@@ -222,7 +222,6 @@ void obs_register_encoder(const struct obs_encoder_info *info)
 	CHECK_REQUIRED_VAL(info, getname,    obs_register_encoder);
 	CHECK_REQUIRED_VAL(info, create,     obs_register_encoder);
 	CHECK_REQUIRED_VAL(info, destroy,    obs_register_encoder);
-	CHECK_REQUIRED_VAL(info, initialize, obs_register_encoder);
 	CHECK_REQUIRED_VAL(info, encode,     obs_register_encoder);
 
 	REGISTER_OBS_DEF(cur_encoder_info_size, obs_encoder_info,

libobs/obs-output.c

@@ -93,10 +93,15 @@ fail:
 	return NULL;
 }
 
+static inline void free_il_packet(struct il_packet *data)
+{
+	obs_free_encoder_packet(&data->packet);
+}
+
 static inline void free_packets(struct obs_output *output)
 {
 	for (size_t i = 0; i < output->interleaved_packets.num; i++)
-		obs_free_encoder_packet(output->interleaved_packets.array+i);
+		free_il_packet(output->interleaved_packets.array+i);
 	da_free(output->interleaved_packets);
 }
 
@@ -342,9 +347,89 @@ static inline struct audio_convert_info *get_audio_conversion(
 	return output->audio_conversion_set ? &output->audio_conversion : NULL;
 }
 
+#define MICROSECOND_DEN 1000000
+
+static inline int64_t convert_packet_dts(struct encoder_packet *packet)
+{
+	return packet->dts * MICROSECOND_DEN / packet->timebase_den;
+}
+
+static bool prepare_interleaved_packet(struct obs_output *output,
+		struct il_packet *out, struct encoder_packet *packet)
+{
+	int64_t offset;
+
+	out->input_ts_us = convert_packet_dts(packet);
+
+	/* audio and video need to start at timestamp 0, and the encoders
+	 * may not currently be at 0 when we get data.  so, we store the
+	 * current dts as offset and subtract that value from the dts/pts
+	 * of the output packet. */
+	if (packet->type == OBS_ENCODER_VIDEO) {
+		if (!output->received_video) {
+			output->first_video_ts = out->input_ts_us;
+			output->video_offset   = packet->dts;
+			output->received_video = true;
+		}
+
+		offset = output->video_offset;
+	} else{
+		/* don't accept audio that's before the first video timestamp */
+		if (!output->received_video ||
+		    out->input_ts_us < output->first_video_ts)
+			return false;
+
+		if (!output->received_audio) {
+			output->audio_offset   = packet->dts;
+			output->received_audio = true;
+		}
+
+		offset = output->audio_offset;
+	}
+
+	obs_duplicate_encoder_packet(&out->packet, packet);
+	out->packet.dts -= offset;
+	out->packet.pts -= offset;
+	out->output_ts_us = convert_packet_dts(&out->packet);
+	return true;
+}
+
+static inline void send_interleaved(struct obs_output *output)
+{
+	struct il_packet out = output->interleaved_packets.array[0];
+	da_erase(output->interleaved_packets, 0);
+
+	output->info.encoded_packet(output->data, &out.packet);
+	free_il_packet(&out);
+}
+
 static void interleave_packets(void *data, struct encoder_packet *packet)
 {
-	
+	struct obs_output *output = data;
+	struct il_packet  out;
+	size_t            idx;
+
+	pthread_mutex_lock(&output->interleaved_mutex);
+
+	if (!prepare_interleaved_packet(output, &out, packet)) {
+
+		for (idx = 0; idx < output->interleaved_packets.num; idx++) {
+			struct il_packet *cur_packet;
+			cur_packet = output->interleaved_packets.array + idx;
+
+			if (out.output_ts_us < cur_packet->output_ts_us)
+				break;
+		}
+
+		da_insert(output->interleaved_packets, idx, &out);
+
+		/* when both video and audio have been received, we're ready
+		 * to start sending out packets (one at a time) */
+		if (output->received_audio && output->received_video)
+			send_interleaved(output);
+	}
+
+	pthread_mutex_unlock(&output->interleaved_mutex);
 }
 
 static void hook_data_capture(struct obs_output *output, bool encoded,
@@ -354,6 +439,9 @@ static void hook_data_capture(struct obs_output *output, bool encoded,
 	void *param;
 
 	if (encoded) {
+		output->received_video = false;
+		output->received_video = false;
+
 		encoded_callback = (has_video && has_audio) ?
 			interleave_packets : output->info.encoded_packet;
 		param = (has_video && has_audio) ? output : output->data;

plugins/obs-ffmpeg/obs-ffmpeg-formats.h

@@ -0,0 +1,37 @@
+#pragma once
+
+static inline enum AVPixelFormat obs_to_ffmpeg_video_format(
+		enum video_format format)
+{
+	switch (format) {
+	case VIDEO_FORMAT_NONE: return AV_PIX_FMT_NONE;
+	case VIDEO_FORMAT_I420: return AV_PIX_FMT_YUV420P;
+	case VIDEO_FORMAT_NV12: return AV_PIX_FMT_NV12;
+	case VIDEO_FORMAT_YVYU: return AV_PIX_FMT_NONE;
+	case VIDEO_FORMAT_YUY2: return AV_PIX_FMT_YUYV422;
+	case VIDEO_FORMAT_UYVY: return AV_PIX_FMT_UYVY422;
+	case VIDEO_FORMAT_RGBA: return AV_PIX_FMT_RGBA;
+	case VIDEO_FORMAT_BGRA: return AV_PIX_FMT_BGRA;
+	case VIDEO_FORMAT_BGRX: return AV_PIX_FMT_BGRA;
+	}
+
+	return AV_PIX_FMT_NONE;
+}
+
+static inline enum audio_format convert_ffmpeg_sample_format(
+		enum AVSampleFormat format)
+{
+	switch ((uint32_t)format) {
+	case AV_SAMPLE_FMT_U8:   return AUDIO_FORMAT_U8BIT;
+	case AV_SAMPLE_FMT_S16:  return AUDIO_FORMAT_16BIT;
+	case AV_SAMPLE_FMT_S32:  return AUDIO_FORMAT_32BIT;
+	case AV_SAMPLE_FMT_FLT:  return AUDIO_FORMAT_FLOAT;
+	case AV_SAMPLE_FMT_U8P:  return AUDIO_FORMAT_U8BIT_PLANAR;
+	case AV_SAMPLE_FMT_S16P: return AUDIO_FORMAT_16BIT_PLANAR;
+	case AV_SAMPLE_FMT_S32P: return AUDIO_FORMAT_32BIT_PLANAR;
+	case AV_SAMPLE_FMT_FLTP: return AUDIO_FORMAT_FLOAT_PLANAR;
+	}
+
+	/* shouldn't get here */
+	return AUDIO_FORMAT_16BIT;
+}

plugins/obs-ffmpeg/obs-ffmpeg-output.c

@@ -26,6 +26,8 @@
 #include <libavformat/avformat.h>
 #include <libswscale/swscale.h>
 
+#include "obs-ffmpeg-formats.h"
+
 //#define OBS_FFMPEG_VIDEO_FORMAT VIDEO_FORMAT_I420
 #define OBS_FFMPEG_VIDEO_FORMAT VIDEO_FORMAT_NV12
 
@@ -82,42 +84,6 @@ struct ffmpeg_output {
 
 /* ------------------------------------------------------------------------- */
 
-static inline enum AVPixelFormat obs_to_ffmpeg_video_format(
-		enum video_format format)
-{
-	switch (format) {
-	case VIDEO_FORMAT_NONE: return AV_PIX_FMT_NONE;
-	case VIDEO_FORMAT_I420: return AV_PIX_FMT_YUV420P;
-	case VIDEO_FORMAT_NV12: return AV_PIX_FMT_NV12;
-	case VIDEO_FORMAT_YVYU: return AV_PIX_FMT_NONE;
-	case VIDEO_FORMAT_YUY2: return AV_PIX_FMT_YUYV422;
-	case VIDEO_FORMAT_UYVY: return AV_PIX_FMT_UYVY422;
-	case VIDEO_FORMAT_RGBA: return AV_PIX_FMT_RGBA;
-	case VIDEO_FORMAT_BGRA: return AV_PIX_FMT_BGRA;
-	case VIDEO_FORMAT_BGRX: return AV_PIX_FMT_BGRA;
-	}
-
-	return AV_PIX_FMT_NONE;
-}
-
-static inline enum audio_format convert_ffmpeg_sample_format(
-		enum AVSampleFormat format)
-{
-	switch ((uint32_t)format) {
-	case AV_SAMPLE_FMT_U8:   return AUDIO_FORMAT_U8BIT;
-	case AV_SAMPLE_FMT_S16:  return AUDIO_FORMAT_16BIT;
-	case AV_SAMPLE_FMT_S32:  return AUDIO_FORMAT_32BIT;
-	case AV_SAMPLE_FMT_FLT:  return AUDIO_FORMAT_FLOAT;
-	case AV_SAMPLE_FMT_U8P:  return AUDIO_FORMAT_U8BIT_PLANAR;
-	case AV_SAMPLE_FMT_S16P: return AUDIO_FORMAT_16BIT_PLANAR;
-	case AV_SAMPLE_FMT_S32P: return AUDIO_FORMAT_32BIT_PLANAR;
-	case AV_SAMPLE_FMT_FLTP: return AUDIO_FORMAT_FLOAT_PLANAR;
-	}
-
-	/* shouldn't get here */
-	return AUDIO_FORMAT_16BIT;
-}
-
 static bool new_stream(struct ffmpeg_data *data, AVStream **stream,
 		AVCodec **codec, enum AVCodecID id)
 {
@@ -180,7 +146,9 @@ static bool open_video_codec(struct ffmpeg_data *data)
 
 static bool init_swscale(struct ffmpeg_data *data, AVCodecContext *context)
 {
-	enum AVPixelFormat format = obs_to_ffmpeg_video_format(OBS_FFMPEG_VIDEO_FORMAT);
+	enum AVPixelFormat format;
+	format = obs_to_ffmpeg_video_format(OBS_FFMPEG_VIDEO_FORMAT);
+
 	data->swscale = sws_getContext(
 			context->width, context->height, format,
 			context->width, context->height, context->pix_fmt,
@@ -198,6 +166,9 @@ static bool create_video_stream(struct ffmpeg_data *data)
 {
 	AVCodecContext *context;
 	struct obs_video_info ovi;
+	enum AVPixelFormat vformat;
+
+	vformat = obs_to_ffmpeg_video_format(OBS_FFMPEG_VIDEO_FORMAT);
 
 	if (!obs_get_video_info(&ovi)) {
 		blog(LOG_WARNING, "No active video");
@@ -218,7 +189,7 @@ static bool create_video_stream(struct ffmpeg_data *data)
 	context->time_base.num  = ovi.fps_den;
 	context->time_base.den  = ovi.fps_num;
 	context->gop_size       = 120;
-	context->pix_fmt        = obs_to_ffmpeg_video_format(OBS_FFMPEG_VIDEO_FORMAT);
+	context->pix_fmt        = vformat;
 
 	if (data->output->oformat->flags & AVFMT_GLOBALHEADER)
 		context->flags |= CODEC_FLAG_GLOBAL_HEADER;
@@ -226,8 +197,7 @@ static bool create_video_stream(struct ffmpeg_data *data)
 	if (!open_video_codec(data))
 		return false;
 
-	enum AVPixelFormat format = obs_to_ffmpeg_video_format(OBS_FFMPEG_VIDEO_FORMAT);
-	if (context->pix_fmt != format)
+	if (context->pix_fmt != vformat)
 		if (!init_swscale(data, context))
 			return false;
 
@@ -523,13 +493,14 @@ static void receive_video(void *param, struct video_data *frame)
 	AVCodecContext *context = data->video->codec;
 	AVPacket packet = {0};
 	int ret = 0, got_packet;
+	enum AVPixelFormat format;
 
 	av_init_packet(&packet);
 
 	if (!data->start_timestamp)
 		data->start_timestamp = frame->timestamp;
 
-	enum AVPixelFormat format = obs_to_ffmpeg_video_format(OBS_FFMPEG_VIDEO_FORMAT);
+	format = obs_to_ffmpeg_video_format(OBS_FFMPEG_VIDEO_FORMAT);
 	if (context->pix_fmt != format)
 		sws_scale(data->swscale, (const uint8_t *const *)frame->data,
 				(const int*)frame->linesize,
@@ -585,7 +556,7 @@ static void receive_video(void *param, struct video_data *frame)
 	data->total_frames++;
 }
 
-static inline void encode_audio(struct ffmpeg_output *output,
+static void encode_audio(struct ffmpeg_output *output,
 		struct AVCodecContext *context, size_t block_size)
 {
 	struct ffmpeg_data *data = &output->ff_data;

plugins/obs-x264/obs-x264.c

@@ -331,11 +331,10 @@ static void load_headers(struct obs_x264 *obsx264)
 	obsx264->sei_size        = sei.num;
 }
 
-static bool obs_x264_initialize(void *data, obs_data_t settings)
+static void *obs_x264_create(obs_data_t settings, obs_encoder_t encoder)
 {
-	struct obs_x264 *obsx264 = data;
-
-	clear_data(data);
+	struct obs_x264 *obsx264 = bzalloc(sizeof(struct obs_x264));
+	obsx264->encoder = encoder;
 
 	if (update_settings(obsx264, settings)) {
 		obsx264->context = x264_encoder_open(&obsx264->params);
@@ -348,16 +347,12 @@ static bool obs_x264_initialize(void *data, obs_data_t settings)
 		blog(LOG_WARNING, "bad settings specified for x264");
 	}
 
-	return obsx264->context != NULL;
-}
-
-static void *obs_x264_create(obs_data_t settings, obs_encoder_t encoder)
-{
-	struct obs_x264 *data = bzalloc(sizeof(struct obs_x264));
-	data->encoder = encoder;
+	if (!obsx264->context) {
+		bfree(obsx264);
+		return NULL;
+	}
 
-	UNUSED_PARAMETER(settings);
-	return data;
+	return obsx264;
 }
 
 static inline int drop_priority(int priority)
@@ -492,7 +487,6 @@ struct obs_encoder_info obs_x264_encoder = {
 	.getname    = obs_x264_getname,
 	.create     = obs_x264_create,
 	.destroy    = obs_x264_destroy,
-	.initialize = obs_x264_initialize,
 	.encode     = obs_x264_encode,
 	.properties = obs_x264_props,
 	.defaults   = obs_x264_defaults,