fill out the rest of the source video frame functions, added nv12 decompression function, and cleaned up the design of the source video frame stuff

libobs/media-io/format-conversion.c

@@ -214,6 +214,42 @@ void decompress_420(const void *input_v, uint32_t width, uint32_t height,
 	}
 }
 
+void decompress_nv12(const void *input_v, uint32_t width, uint32_t height,
+		uint32_t row_bytes, uint32_t start_y, uint32_t end_y,
+		void *output_v)
+{
+	uint8_t       *output = output_v;
+	const uint8_t *input  = input_v;
+	const uint8_t *input2 = input + width * height;
+
+	uint32_t start_y_d2 = start_y/2;
+	uint32_t width_d2   = width/2;
+	uint32_t height_d2  = end_y/2;
+	uint32_t y;
+
+	for (y = start_y_d2; y < height_d2; y++) {
+		const uint16_t *chroma = (uint16_t*)(input2 + y * width);
+		register const uint8_t *lum0, *lum1;
+		register uint32_t *output0, *output1;
+		uint32_t x;
+
+		lum0 = input + y * 2*width;
+		lum1 = lum0 + width;
+		output0 = (uint32_t*)(output + y * 2*row_bytes);
+		output1 = (uint32_t*)((uint8_t*)output0 + row_bytes);
+
+		for (x = 0; x < width_d2; x++) {
+			uint32_t out = *(chroma++) << 8;
+
+			*(output0++) = *(lum0++) | out;
+			*(output0++) = *(lum0++) | out;
+
+			*(output1++) = *(lum1++) | out;
+			*(output1++) = *(lum1++) | out;
+		}
+	}
+}
+
 void decompress_422(const void *input_v, uint32_t width, uint32_t height,
 		uint32_t row_bytes, uint32_t start_y, uint32_t end_y,
 		void *output_v, bool leading_lum)
@@ -221,8 +257,8 @@ void decompress_422(const void *input_v, uint32_t width, uint32_t height,
 	const uint8_t *input  = input_v;
 	uint8_t       *output = output_v;
 
-	uint32_t width_d2  = width>>1;
-	uint32_t line_size = width*2;
+	uint32_t width_d2  = width >> 1;
+	uint32_t line_size = width * 2;
 	uint32_t y;
 
 	register const uint32_t *input32;
@@ -230,7 +266,7 @@ void decompress_422(const void *input_v, uint32_t width, uint32_t height,
 	register uint32_t       *output32;
 
 	if (leading_lum) {
-		for (y = 0; y < height; y++) {
+		for (y = start_y; y < end_y; y++) {
 			input32     = (uint32_t*)(input + y*line_size);
 			input32_end = input32 + width_d2;
 			output32    = (uint32_t*)(output + y*row_bytes);
@@ -248,7 +284,7 @@ void decompress_422(const void *input_v, uint32_t width, uint32_t height,
 			}
 		}
 	} else {
-		for (y = 0; y < height; y++) {
+		for (y = start_y; y < end_y; y++) {
 			input32     = (uint32_t*)(input + y*line_size);
 			input32_end = input32 + width_d2;
 			output32    = (uint32_t*)(output + y*row_bytes);

libobs/media-io/format-conversion.h

@@ -31,6 +31,10 @@ EXPORT void compress_uyvx_to_nv12(const void *input,
 		uint32_t width, uint32_t height, uint32_t row_bytes,
 		uint32_t start_y, uint32_t end_y, void **output);
 
+EXPORT void decompress_nv12(const void *input,
+		uint32_t width, uint32_t height, uint32_t row_bytes,
+		uint32_t start_y, uint32_t end_y, void *output);
+
 EXPORT void decompress_420(const void *input,
 		uint32_t width, uint32_t height, uint32_t row_bytes,
 		uint32_t start_y, uint32_t end_y, void *output);

libobs/media-io/video-io.h

@@ -36,8 +36,8 @@ enum video_type {
 	VIDEO_FORMAT_UNKNOWN,
 
 	/* planar 420 format */
-	VIDEO_FORMAT_I420, /* planar 4:2:0 */
-	VIDEO_FORMAT_NV12, /* two-plane lum and packed chroma */
+	VIDEO_FORMAT_I420, /* three-plane */
+	VIDEO_FORMAT_NV12, /* two-plane, lum and packed chroma */
 
 	/* packed 422 formats */
 	VIDEO_FORMAT_YVYU,
@@ -45,7 +45,8 @@ enum video_type {
 	VIDEO_FORMAT_UYVY,
 
 	/* packed uncompressed formats */
-	VIDEO_FORMAT_UYVX, /* packed UYV */
+	VIDEO_FORMAT_YUVX,
+	VIDEO_FORMAT_UYVX,
 	VIDEO_FORMAT_RGBA,
 	VIDEO_FORMAT_BGRA,
 	VIDEO_FORMAT_BGRX,

libobs/obs-source.c

@@ -15,6 +15,7 @@
     along with this program.  If not, see <http://www.gnu.org/licenses/>.
 ******************************************************************************/
 
+#include "media-io\format-conversion.h"
 #include "util/platform.h"
 
 #include "obs.h"
@@ -297,20 +298,111 @@ static bool set_texture_size(obs_source_t source, struct source_frame *frame)
 	return source->output_texture != NULL;
 }
 
+enum convert_type {
+	CONVERT_NONE,
+	CONVERT_NV12,
+	CONVERT_420,
+	CONVERT_422_U,
+	CONVERT_422_Y,
+};
+
+static inline enum convert_type get_convert_type(enum video_type type)
+{
+	switch (type) {
+	case VIDEO_FORMAT_I420:
+		return CONVERT_420;
+	case VIDEO_FORMAT_NV12:
+		return CONVERT_NV12;
+
+	case VIDEO_FORMAT_YVYU:
+	case VIDEO_FORMAT_YUY2:
+		return CONVERT_422_Y;
+	case VIDEO_FORMAT_UYVY:
+		return CONVERT_422_U;
+
+	case VIDEO_FORMAT_UNKNOWN:
+	case VIDEO_FORMAT_YUVX:
+	case VIDEO_FORMAT_UYVX:
+	case VIDEO_FORMAT_RGBA:
+	case VIDEO_FORMAT_BGRA:
+	case VIDEO_FORMAT_BGRX:
+		return CONVERT_NONE;
+	}
+
+	return CONVERT_NONE;
+}
+
+static inline bool is_yuv(enum video_type type)
+{
+	switch (type) {
+	case VIDEO_FORMAT_I420:
+	case VIDEO_FORMAT_NV12:
+	case VIDEO_FORMAT_YVYU:
+	case VIDEO_FORMAT_YUY2:
+	case VIDEO_FORMAT_UYVY:
+	case VIDEO_FORMAT_YUVX:
+	case VIDEO_FORMAT_UYVX:
+		return true;
+	case VIDEO_FORMAT_UNKNOWN:
+	case VIDEO_FORMAT_RGBA:
+	case VIDEO_FORMAT_BGRA:
+	case VIDEO_FORMAT_BGRX:
+		return false;
+	}
+
+	return false;
+}
+
+static bool upload_frame(texture_t tex, const struct source_frame *frame)
+{
+	void *ptr;
+	uint32_t row_bytes;
+	enum convert_type type = get_convert_type(frame->type);
+
+	if (type == CONVERT_NONE) {
+		texture_setimage(tex, frame->data, frame->row_bytes, false);
+		return true;
+	}
+
+	if (!texture_map(tex, &ptr, &row_bytes))
+		return false;
+
+	if (type == CONVERT_420)
+		decompress_420(frame->data, frame->width, frame->height,
+				frame->row_bytes, 0, frame->height, ptr);
+
+	else if (type == CONVERT_NV12)
+		decompress_nv12(frame->data, frame->width, frame->height,
+				frame->row_bytes, 0, frame->height, ptr);
+
+	else if (type == CONVERT_422_Y)
+		decompress_422(frame->data, frame->width, frame->height,
+				frame->row_bytes, 0, frame->height, ptr, true);
+
+	else if (type == CONVERT_422_U)
+		decompress_422(frame->data, frame->width, frame->height,
+				frame->row_bytes, 0, frame->height, ptr, false);
+
+	texture_unmap(tex);
+	return true;
+}
+
 static void obs_source_draw_texture(texture_t tex, struct source_frame *frame)
 {
 	effect_t    effect = obs->default_effect;
-	const char  *type = frame->yuv ? "DrawYUVToRGB" : "DrawRGB";
+	bool        yuv   = is_yuv(frame->type);
+	const char  *type = yuv ? "DrawYUVToRGB" : "DrawRGB";
 	technique_t tech;
-	eparam_t param;
+	eparam_t    param;
 
-	texture_setimage(tex, frame->data, frame->row_bytes, frame->flip);
+	if (!upload_frame(tex, frame))
+		return;
 
 	tech = effect_gettechnique(effect, type);
 	technique_begin(tech);
 	technique_beginpass(tech, 0);
 
-	if (frame->yuv) {
+	if (yuv) {
 		param = effect_getparambyname(effect, "yuv_matrix");
 		effect_setval(effect, param, frame->yuv_matrix,
 				sizeof(float) * 16);
@@ -335,8 +427,10 @@ static void obs_source_render_async_video(obs_source_t source)
 	if (!source->timing_set && source->audio_buffer.num)
 		obs_source_flush_audio_buffer(source);
 
-	if (set_texture_size(source, frame))
+	if (set_texture_size(source, frame)) {
+		source->flip = frame->flip;
 		obs_source_draw_texture(source->output_texture, frame);
+	}
 
 	obs_source_releaseframe(source, frame);
 }
@@ -496,8 +590,8 @@ void obs_source_save_settings(obs_source_t source, const char *settings)
 	dstr_copy(&source->settings, settings);
 }
 
-static inline struct filter_frame *filter_async_video(obs_source_t source,
-		struct filter_frame *in)
+static inline struct source_frame *filter_async_video(obs_source_t source,
+		struct source_frame *in)
 {
 	size_t i;
 	for (i = source->filters.num; i > 0; i--) {
@@ -512,27 +606,31 @@ static inline struct filter_frame *filter_async_video(obs_source_t source,
 	return in;
 }
 
-static struct filter_frame *process_video(obs_source_t source,
-		const struct source_video *frame)
+static inline struct source_frame *cache_video(obs_source_t source,
+		const struct source_frame *frame)
 {
-	/* TODO: convert to UYV444 or RGB */
-	return NULL;
+	/* TODO: use an actual cache */
+	struct source_frame *new_frame = bmalloc(sizeof(struct source_frame));
+	memcpy(new_frame, frame, sizeof(struct source_frame));
+	new_frame->data = bmalloc(frame->row_bytes * frame->height);
+
+	return new_frame;
 }
 
 void obs_source_output_video(obs_source_t source,
-		const struct source_video *frame)
+		const struct source_frame *frame)
 {
-	struct filter_frame *output;
-
-	output = process_video(source, frame);
+	struct source_frame *output = cache_video(source, frame);
 
 	pthread_mutex_lock(&source->filter_mutex);
 	output = filter_async_video(source, output);
 	pthread_mutex_unlock(&source->filter_mutex);
 
-	pthread_mutex_lock(&source->video_mutex);
-	da_push_back(source->video_frames, &output);
-	pthread_mutex_unlock(&source->video_mutex);
+	if (output) {
+		pthread_mutex_lock(&source->video_mutex);
+		da_push_back(source->video_frames, &output);
+		pthread_mutex_unlock(&source->video_mutex);
+	}
 }
 
 static inline const struct audio_data *filter_async_audio(obs_source_t source,
@@ -595,13 +693,63 @@ void obs_source_output_audio(obs_source_t source,
 	pthread_mutex_unlock(&source->filter_mutex);
 }
 
+/* 
+ * Ensures that cached frames are displayed on time.  If multiple frames
+ * were cached between renders, then releases the unnecessary frames and uses
+ * the frame with the closest timing.
+ */
 struct source_frame *obs_source_getframe(obs_source_t source)
 {
-	/* TODO */
-	return NULL;
+	uint64_t last_frame_time = source->last_frame_timestamp;
+	struct   source_frame *frame = NULL;
+	struct   source_frame *next_frame;
+	uint64_t sys_time, frame_time;
+
+	pthread_mutex_lock(&source->video_mutex);
+
+	if (!source->video_frames.num)
+		goto unlock;
+
+	next_frame = source->video_frames.array[0];
+	sys_time   = os_gettime_ns();
+	frame_time = next_frame->timestamp;
+
+	if (!source->last_frame_timestamp) {
+		frame = next_frame;
+		da_erase(source->video_frames, 0);
+
+		source->last_frame_timestamp = frame_time;
+	} else {
+		uint64_t sys_offset, frame_offset;
+		sys_offset   = sys_time   - source->last_sys_timestamp;
+		frame_offset = frame_time - last_frame_time;
+
+		source->last_frame_timestamp += sys_offset;
+
+		while (frame_offset <= sys_offset) {
+			if (frame)
+				source_frame_destroy(frame);
+
+			frame = next_frame;
+			da_erase(source->video_frames, 0);
+
+			if (!source->video_frames.num)
+				break;
+
+			next_frame   = source->video_frames.array[0];
+			frame_time   = next_frame->timestamp;
+			frame_offset = frame_time - last_frame_time;
+		}
+	}
+
+	source->last_sys_timestamp = sys_time;
+
+unlock:
+	pthread_mutex_unlock(&source->video_mutex);
+	return frame;
 }
 
 void obs_source_releaseframe(obs_source_t source, struct source_frame *frame)
 {
-	/* TODO */
+	source_frame_destroy(frame);
 }

libobs/obs-source.h

@@ -136,8 +136,8 @@
  *       Return value: true if sources remaining, otherwise false.
  *
  * ---------------------------------------------------------
- *   struct filter_frame *[name]_filter_video(void *data,
- *                                     struct filter_frame *frame);
+ *   struct source_frame *[name]_filter_video(void *data,
+ *                                     const struct source_frame *frame);
  *       Filters audio data.  Used with audio filters.
  *
  *       frame: Video frame data.
@@ -185,8 +185,8 @@ struct source_info {
 
 	bool (*enum_children)(void *data, size_t idx, obs_source_t *child);
 
-	struct filter_frame *(*filter_video)(void *data,
-			struct filter_frame *frame);
+	struct source_frame *(*filter_video)(void *data,
+			const struct source_frame *frame);
 	const struct audio_data *(*filter_audio)(void *data,
 			const struct audio_data *audio);
 };
@@ -211,7 +211,10 @@ struct obs_source {
 	/* async video and audio */
 	bool                         timing_set;
 	uint64_t                     timing_adjust;
+	uint64_t                     last_frame_timestamp;
+	uint64_t                     last_sys_timestamp;
 	texture_t                    output_texture;
+	bool                         flip;
 
 	audio_line_t                 audio_line;
 	DARRAY(struct audiobuf)      audio_buffer;

libobs/obs.h

@@ -61,19 +61,6 @@ struct source_audio {
 	uint64_t           timestamp;
 };
 
-struct source_video {
-	const void         *data;
-	uint32_t           width;
-	uint32_t           height;
-	uint32_t           row_bytes;
-	uint64_t           timestamp;
-
-	enum video_type    type;
-	float              yuv_matrix[16];
-	bool               flip;
-};
-
-/* differs from source_video in that it's YUV444 or RGB only */
 struct source_frame {
 	void               *data;
 	uint32_t           width;
@@ -81,7 +68,7 @@ struct source_frame {
 	uint32_t           row_bytes;
 	uint64_t           timestamp;
 
-	bool               yuv;
+	enum video_type    type;
 	float              yuv_matrix[16];
 	bool               flip;
 };
@@ -291,7 +278,7 @@ EXPORT void obs_source_save_settings(obs_source_t source, const char *settings);
 
 /** Outputs asynchronous video data */
 EXPORT void obs_source_obs_async_video(obs_source_t source,
-		const struct video_frame *frame);
+		const struct source_frame *frame);
 
 /** Outputs audio data (always asynchronous) */
 EXPORT void obs_source_obs_async_audio(obs_source_t source,