finish up most of the source audio stuff and rename some variables/structs/enum to be a bit more consistent

libobs/media-io/audio-io.c

@@ -91,7 +91,6 @@ static inline bool valid_audio_params(struct audio_info *info)
 static inline bool ao_add_to_media(audio_t audio)
 {
 	struct media_output_info oi;
-	oi.format  = audio->info.format;
 	oi.obj     = audio;
 	oi.connect = NULL;
 
@@ -117,7 +116,7 @@ int audio_output_open(audio_t *audio, media_t media, struct audio_info *info)
 	pthread_mutex_init_value(&out->line_mutex);
 	out->media = media;
 	out->block_size = get_audio_channels(info->speakers) *
-	                  get_audio_bytes_per_channel(info->type);
+	                  get_audio_bytes_per_channel(info->format);
 
 	if (pthread_mutex_init(&out->line_mutex, NULL) != 0)
 		goto fail;

libobs/media-io/audio-io.h

@@ -64,10 +64,9 @@ struct audio_data {
 
 struct audio_info {
 	const char          *name;
-	const char          *format;
 
 	uint32_t            samples_per_sec;
-	enum audio_format   type;
+	enum audio_format   format;
 	enum speaker_layout speakers;
 };
 

libobs/media-io/audio-resampler-ffmpeg.c

@@ -124,7 +124,8 @@ void audio_resampler_destroy(audio_resampler_t rs)
 
 bool audio_resampler_resample(audio_resampler_t rs,
 		 void **output, uint32_t *out_frames,
-		 void *input, uint32_t in_frames)
+		 const void *input, uint32_t in_frames,
+		 uint64_t *timestamp_offset)
 {
 	struct SwrContext *context = rs->context;
 	int ret;
@@ -134,6 +135,8 @@ bool audio_resampler_resample(audio_resampler_t rs,
 			(int64_t)rs->output_freq, (int64_t)rs->input_freq,
 			AV_ROUND_UP);
 
+	*timestamp_offset = (uint64_t)swr_get_delay(context, 1000000000);
+
 	/* resize the buffer if bigger */
 	if (estimated > rs->output_size) {
 		if (rs->output_buffer)
@@ -146,7 +149,7 @@ bool audio_resampler_resample(audio_resampler_t rs,
 
 	ret = swr_convert(context,
 			&rs->output_buffer, rs->output_size,
-			input, in_frames);
+			(const uint8_t**)&input, in_frames);
 
 	if (ret < 0) {
 		blog(LOG_ERROR, "swr_convert failed: %d", ret);

libobs/media-io/audio-resampler.h

@@ -20,6 +20,10 @@
 #include "../util/c99defs.h"
 #include "audio-io.h"
 
+#ifdef __cplusplus
+extern "C" {
+#endif
+
 struct audio_resampler;
 typedef struct audio_resampler *audio_resampler_t;
 
@@ -35,4 +39,9 @@ EXPORT void audio_resampler_destroy(audio_resampler_t resampler);
 
 EXPORT bool audio_resampler_resample(audio_resampler_t resampler,
 		 void **output, uint32_t *out_frames,
-		 void *input, uint32_t in_frames);
+		 const void *input, uint32_t in_frames,
+		 uint64_t *timestamp_offset);
+
+#ifdef __cplusplus
+}
+#endif

libobs/media-io/video-io.c

@@ -80,13 +80,12 @@ static void *video_thread(void *param)
 static inline bool valid_video_params(struct video_info *info)
 {
 	return info->height != 0 && info->width != 0 && info->fps_den != 0 &&
-	       info->fps_num != 0 && info->format != NULL;
+	       info->fps_num != 0;
 }
 
 static inline bool vo_add_to_media(video_t video)
 {
 	struct media_output_info oi;
-	oi.format  = video->info.format;
 	oi.obj     = video;
 	oi.connect = NULL;
 

libobs/media-io/video-io.h

@@ -32,7 +32,7 @@ extern "C" {
 struct video_output;
 typedef struct video_output *video_t;
 
-enum video_type {
+enum video_format {
 	VIDEO_FORMAT_UNKNOWN,
 
 	/* planar 420 format */
@@ -53,20 +53,19 @@ enum video_type {
 };
 
 struct video_frame {
-	const void      *data;
-	uint32_t        row_size;  /* for RGB/BGR formats and UYVX */
-	uint64_t        timestamp;
+	const void        *data;
+	uint32_t          row_size;  /* for RGB/BGR formats and UYVX */
+	uint64_t          timestamp;
 };
 
 struct video_info {
-	const char      *name;
-	const char      *format;
-
-	enum video_type type;
-	uint32_t        fps_num; /* numerator */
-	uint32_t        fps_den; /* denominator */
-	uint32_t        width;
-	uint32_t        height;
+	const char        *name;
+
+	enum video_format type;
+	uint32_t          fps_num; /* numerator */
+	uint32_t          fps_den; /* denominator */
+	uint32_t          width;
+	uint32_t          height;
 };
 
 #define VIDEO_OUTPUT_SUCCESS       0

libobs/obs-source.c

@@ -171,8 +171,8 @@ void obs_source_destroy(obs_source_t source)
 			pthread_mutex_unlock(&obs->source_list_mutex);
 		}
 
-		for (i = 0; i < source->audio_buffer.num; i++)
-			audiobuf_free(source->audio_buffer.array+i);
+		for (i = 0; i < source->audio_wait_buffer.num; i++)
+			audiobuf_free(source->audio_wait_buffer.array+i);
 		for (i = 0; i < source->video_frames.num; i++)
 			source_frame_destroy(source->video_frames.array[i]);
 
@@ -183,10 +183,12 @@ void obs_source_destroy(obs_source_t source)
 		if (source->data)
 			source->callbacks.destroy(source->data);
 
+		bfree(source->audio_data.data);
 		audio_line_destroy(source->audio_line);
+		audio_resampler_destroy(source->resampler);
 
 		da_free(source->video_frames);
-		da_free(source->audio_buffer);
+		da_free(source->audio_wait_buffer);
 		da_free(source->filters);
 		pthread_mutex_destroy(&source->filter_mutex);
 		pthread_mutex_destroy(&source->audio_mutex);
@@ -259,15 +261,15 @@ static void source_output_audio_line(obs_source_t source,
 	audio_line_output(source->audio_line, &in);
 }
 
-static void obs_source_flush_audio_buffer(obs_source_t source)
+static void obs_source_flush_audio_wait_buffer(obs_source_t source)
 {
 	size_t i;
 
 	pthread_mutex_lock(&source->audio_mutex);
 	source->timing_set = true;
 
-	for (i = 0; i < source->audio_buffer.num; i++) {
-		struct audiobuf *buf = source->audio_buffer.array+i;
+	for (i = 0; i < source->audio_wait_buffer.num; i++) {
+		struct audiobuf *buf = source->audio_wait_buffer.array+i;
 		struct audio_data data;
 
 		data.data      = buf->data;
@@ -277,7 +279,7 @@ static void obs_source_flush_audio_buffer(obs_source_t source)
 		audiobuf_free(buf);
 	}
 
-	da_free(source->audio_buffer);
+	da_free(source->audio_wait_buffer);
 	pthread_mutex_unlock(&source->audio_mutex);
 }
 
@@ -306,9 +308,9 @@ enum convert_type {
 	CONVERT_422_Y,
 };
 
-static inline enum convert_type get_convert_type(enum video_type type)
+static inline enum convert_type get_convert_type(enum video_format format)
 {
-	switch (type) {
+	switch (format) {
 	case VIDEO_FORMAT_I420:
 		return CONVERT_420;
 	case VIDEO_FORMAT_NV12:
@@ -332,9 +334,9 @@ static inline enum convert_type get_convert_type(enum video_type type)
 	return CONVERT_NONE;
 }
 
-static inline bool is_yuv(enum video_type type)
+static inline bool is_yuv(enum video_format format)
 {
-	switch (type) {
+	switch (format) {
 	case VIDEO_FORMAT_I420:
 	case VIDEO_FORMAT_NV12:
 	case VIDEO_FORMAT_YVYU:
@@ -357,7 +359,7 @@ static bool upload_frame(texture_t tex, const struct source_frame *frame)
 {
 	void *ptr;
 	uint32_t row_bytes;
-	enum convert_type type = get_convert_type(frame->type);
+	enum convert_type type = get_convert_type(frame->format);
 
 	if (type == CONVERT_NONE) {
 		texture_setimage(tex, frame->data, frame->row_bytes, false);
@@ -390,7 +392,7 @@ static bool upload_frame(texture_t tex, const struct source_frame *frame)
 static void obs_source_draw_texture(texture_t tex, struct source_frame *frame)
 {
 	effect_t    effect = obs->default_effect;
-	bool        yuv   = is_yuv(frame->type);
+	bool        yuv   = is_yuv(frame->format);
 	const char  *type = yuv ? "DrawYUVToRGB" : "DrawRGB";
 	technique_t tech;
 	eparam_t    param;
@@ -424,8 +426,8 @@ static void obs_source_render_async_video(obs_source_t source)
 		return;
 
 	source->timing_adjust = frame->timestamp - os_gettime_ns();
-	if (!source->timing_set && source->audio_buffer.num)
-		obs_source_flush_audio_buffer(source);
+	if (!source->timing_set && source->audio_wait_buffer.num)
+		obs_source_flush_audio_wait_buffer(source);
 
 	if (set_texture_size(source, frame))
 		obs_source_draw_texture(source->output_texture, frame);
@@ -631,8 +633,8 @@ void obs_source_output_video(obs_source_t source,
 	}
 }
 
-static inline const struct audio_data *filter_async_audio(obs_source_t source,
-		const struct audio_data *in)
+static inline struct filtered_audio *filter_async_audio(obs_source_t source,
+		struct filtered_audio *in)
 {
 	size_t i;
 	for (i = source->filters.num; i > 0; i--) {
@@ -647,42 +649,113 @@ static inline const struct audio_data *filter_async_audio(obs_source_t source,
 	return in;
 }
 
-static struct audio_data *process_audio(obs_source_t source,
+static inline void reset_resampler(obs_source_t source,
 		const struct source_audio *audio)
 {
-	/* TODO: upmix/downmix/resample */
-	return NULL;
+	const struct audio_info *obs_info = audio_output_getinfo(obs->audio);
+	struct resample_info output_info;
+
+	output_info.format           = obs_info->format;
+	output_info.samples_per_sec  = obs_info->samples_per_sec;
+	output_info.speakers         = obs_info->speakers;
+
+	source->sample_info.format          = audio->format;
+	source->sample_info.samples_per_sec = audio->samples_per_sec;
+	source->sample_info.speakers        = audio->speakers;
+
+	if (source->sample_info.samples_per_sec == obs_info->samples_per_sec &&
+	    source->sample_info.format          == obs_info->format          &&
+	    source->sample_info.speakers        == obs_info->speakers) {
+		source->audio_failed = false;
+		return;
+	}
+
+	audio_resampler_destroy(source->resampler);
+	source->resampler = audio_resampler_create(&output_info,
+			&source->sample_info);
+
+	source->audio_failed = source->resampler == NULL;
+	if (source->resampler == NULL)
+		blog(LOG_ERROR, "creation of resampler failed");
+}
+
+static inline void copy_audio_data(obs_source_t source,
+		const void *data, uint32_t frames, uint64_t timestamp)
+{
+	size_t blocksize = audio_output_blocksize(obs->audio);
+	size_t size = (size_t)frames * blocksize;
+
+	/* ensure audio storage capacity */
+	if (source->audio_storage_size < size) {
+		bfree(source->audio_data.data);
+		source->audio_data.data = bmalloc(size);
+		source->audio_storage_size = size;
+	}
+
+	source->audio_data.frames = frames;
+	source->audio_data.timestamp = timestamp;
+	memcpy(source->audio_data.data, data, size);
+}
+
+/* resamples/remixes new audio to the designated main audio output format */
+static void process_audio(obs_source_t source, const struct source_audio *audio)
+{
+	if (source->sample_info.samples_per_sec != audio->samples_per_sec ||
+	    source->sample_info.format          != audio->format          ||
+	    source->sample_info.speakers        != audio->speakers)
+		reset_resampler(source, audio);
+
+	if (source->audio_failed)
+		return;
+
+	if (source->resampler) {
+		void *output;
+		uint32_t frames;
+		uint64_t offset;
+
+		audio_resampler_resample(source->resampler, &output, &frames,
+				audio->data, audio->frames, &offset);
+
+		copy_audio_data(source, output, frames,
+				audio->timestamp - offset);
+	} else {
+		copy_audio_data(source, audio->data, audio->frames,
+				audio->timestamp);
+	}
 }
 
 void obs_source_output_audio(obs_source_t source,
 		const struct source_audio *audio)
 {
 	uint32_t flags = obs_source_get_output_flags(source);
-	struct audio_data *data;
-	const struct audio_data *output;
+	size_t blocksize = audio_output_blocksize(obs->audio);
+	struct filtered_audio *output;
 
-	data = process_audio(source, audio);
+	process_audio(source, audio);
 
 	pthread_mutex_lock(&source->filter_mutex);
-	output = filter_async_audio(source, data);
+	output = filter_async_audio(source, &source->audio_data);
 
 	if (output) {
 		pthread_mutex_lock(&source->audio_mutex);
 
 		if (!source->timing_set && flags & SOURCE_ASYNC_VIDEO) {
 			struct audiobuf newbuf;
-			size_t audio_size = audio_output_blocksize(obs->audio) *
-				output->frames;
+			size_t audio_size = blocksize * output->frames;
 
 			newbuf.data      = bmalloc(audio_size);
 			newbuf.frames    = output->frames;
 			newbuf.timestamp = output->timestamp;
 			memcpy(newbuf.data, output->data, audio_size);
 
-			da_push_back(source->audio_buffer, &newbuf);
+			da_push_back(source->audio_wait_buffer, &newbuf);
 
 		} else {
-			source_output_audio_line(source, output);
+			struct audio_data data;
+			data.data      = output->data;
+			data.frames    = output->frames;
+			data.timestamp = output->timestamp;
+			source_output_audio_line(source, &data);
 		}
 
 		pthread_mutex_unlock(&source->audio_mutex);

libobs/obs-source.h

@@ -22,6 +22,7 @@
 #include "util/dstr.h"
 #include "util/threading.h"
 #include "media-io/media-io.h"
+#include "media-io/audio-resampler.h"
 
 /*
  * ===========================================
@@ -144,8 +145,8 @@
  *       returns: New video frame data (or NULL if pending)
  *
  * ---------------------------------------------------------
- *   const struct audio_data *[name]_filter_audio(void *data,
- *                                     const struct audio_data *audio);
+ *   struct filter_audio [name]_filter_audio(void *data,
+ *                                     struct filter_audio *audio);
  *       Filters video data.  Used with async video data.
  *
  *       audio: Audio data.
@@ -187,8 +188,8 @@ struct source_info {
 
 	struct source_frame *(*filter_video)(void *data,
 			const struct source_frame *frame);
-	const struct audio_data *(*filter_audio)(void *data,
-			const struct audio_data *audio);
+	struct filtered_audio *(*filter_audio)(void *data,
+			struct filtered_audio *audio);
 };
 
 struct audiobuf {
@@ -215,11 +216,16 @@ struct obs_source {
 	uint64_t                     last_sys_timestamp;
 	texture_t                    output_texture;
 
+	bool                         audio_failed;
+	struct resample_info         sample_info;
+	audio_resampler_t            resampler;
 	audio_line_t                 audio_line;
-	DARRAY(struct audiobuf)      audio_buffer;
+	DARRAY(struct audiobuf)      audio_wait_buffer;
 	DARRAY(struct source_frame*) video_frames;
 	pthread_mutex_t              audio_mutex;
 	pthread_mutex_t              video_mutex;
+	struct filtered_audio        audio_data;
+	size_t                       audio_storage_size;
 
 	/* filters */
 	struct obs_source            *filter_parent;

libobs/obs.c

@@ -118,6 +118,8 @@ static inline void obs_free_graphics(void)
 	for (i = 0; i < NUM_TEXTURES; i++)
 		stagesurface_destroy(obs->copy_surfaces[i]);
 
+	effect_destroy(obs->default_effect);
+
 	gs_leavecontext();
 
 	gs_destroy(obs->graphics);

libobs/obs.h

@@ -48,13 +48,19 @@ enum order_movement {
 	ORDER_MOVE_BOTTOM
 };
 
+struct filtered_audio {
+	void                *data;
+	uint32_t            frames;
+	uint64_t            timestamp;
+};
+
 struct source_audio {
 	const void          *data;
 	uint32_t            frames;
 
 	/* audio will be automatically resampled/upmixed/downmixed */
 	enum speaker_layout speakers;
-	enum audio_format   type;
+	enum audio_format   format;
 	uint32_t            samples_per_sec;
 
 	/* can be 0 if 'immediate' */
@@ -68,7 +74,7 @@ struct source_frame {
 	uint32_t            row_bytes;
 	uint64_t            timestamp;
 
-	enum video_type     type;
+	enum video_format   format;
 	float               yuv_matrix[16];
 	bool                flip;
 };

test/win/test.cpp

@@ -3,6 +3,7 @@
 #include <windows.h>
 
 #include "util/base.h"
+#include "media-io/audio-resampler.h"
 #include "obs.h"
 
 #include <intrin.h>
@@ -69,7 +70,6 @@ static void CreateOBS(HWND hwnd)
 
 	struct video_info vi;
 	memset(&vi, 0, sizeof(struct video_info));
-	vi.format  = "RGBA";
 	vi.fps_num = 30000;
 	vi.fps_den = 1001;
 	vi.width   = rc.right;
@@ -111,7 +111,6 @@ static void AddTestItems(obs_scene_t scene, obs_source_t source)
 static HWND CreateTestWindow(HINSTANCE instance)
 {
 	WNDCLASS wc;
-	base_set_log_handler(do_log);
 
 	memset(&wc, 0, sizeof(wc));
 	wc.lpszClassName = L"bla";
@@ -134,6 +133,7 @@ int WINAPI WinMain(HINSTANCE instance, HINSTANCE prevInstance, LPSTR cmdLine,
 		int numCmd)
 {
 	HWND hwnd = NULL;
+	base_set_log_handler(do_log);
 
 	try {
 		hwnd = CreateTestWindow(instance);