win-dshow, obs-ffmpeg: Add hardware decoding support

Fixes hardware decoding support and updates it to FFmpeg 4.0.

deps/media-playback/media-playback/decode.c

@@ -17,29 +17,58 @@
 #include "decode.h"
 #include "media.h"
 
-static AVCodec *find_hardware_decoder(enum AVCodecID id)
+#if LIBAVCODEC_VERSION_INT > AV_VERSION_INT(58, 4, 100)
+#define USE_NEW_HARDWARE_CODEC_METHOD
+#endif
+
+#ifdef USE_NEW_HARDWARE_CODEC_METHOD
+enum AVHWDeviceType hw_priority[] = {
+	AV_HWDEVICE_TYPE_D3D11VA, AV_HWDEVICE_TYPE_DXVA2,
+	AV_HWDEVICE_TYPE_VAAPI,   AV_HWDEVICE_TYPE_VDPAU,
+	AV_HWDEVICE_TYPE_QSV,     AV_HWDEVICE_TYPE_CUDA,
+	AV_HWDEVICE_TYPE_NONE,
+};
+
+static bool has_hw_type(AVCodec *c, enum AVHWDeviceType type)
 {
-	AVHWAccel *hwa = av_hwaccel_next(NULL);
-	AVCodec *c = NULL;
-
-	while (hwa) {
-		if (hwa->id == id) {
-			if (hwa->pix_fmt == AV_PIX_FMT_VDTOOL ||
-			    hwa->pix_fmt == AV_PIX_FMT_DXVA2_VLD ||
-			    hwa->pix_fmt == AV_PIX_FMT_VAAPI_VLD) {
-				c = avcodec_find_decoder_by_name(hwa->name);
-				if (c)
-					break;
-			}
+	for (int i = 0;; i++) {
+		const AVCodecHWConfig *config = avcodec_get_hw_config(c, i);
+		if (!config) {
+			break;
+		}
+
+		if (config->methods & AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX &&
+		    config->device_type == type)
+			return true;
+	}
+
+	return false;
+}
+
+static void init_hw_decoder(struct mp_decode *d, AVCodecContext *c)
+{
+	enum AVHWDeviceType *priority = hw_priority;
+	AVBufferRef *hw_ctx = NULL;
+
+	while (*priority != AV_HWDEVICE_TYPE_NONE) {
+		if (has_hw_type(d->codec, *priority)) {
+			int ret = av_hwdevice_ctx_create(&hw_ctx, *priority,
+							 NULL, NULL, 0);
+			if (ret == 0)
+				break;
 		}
 
-		hwa = av_hwaccel_next(hwa);
+		priority++;
 	}
 
-	return c;
+	if (hw_ctx) {
+		c->hw_device_ctx = av_buffer_ref(hw_ctx);
+		d->hw = true;
+	}
 }
+#endif
 
-static int mp_open_codec(struct mp_decode *d)
+static int mp_open_codec(struct mp_decode *d, bool hw)
 {
 	AVCodecContext *c;
 	int ret;
@@ -58,6 +87,13 @@ static int mp_open_codec(struct mp_decode *d)
 	c = d->stream->codec;
 #endif
 
+	d->hw = false;
+
+#ifdef USE_NEW_HARDWARE_CODEC_METHOD
+	if (hw)
+		init_hw_decoder(d, c);
+#endif
+
 	if (c->thread_count == 1 && c->codec_id != AV_CODEC_ID_PNG &&
 	    c->codec_id != AV_CODEC_ID_TIFF &&
 	    c->codec_id != AV_CODEC_ID_JPEG2000 &&
@@ -101,35 +137,25 @@ bool mp_decode_init(mp_media_t *m, enum AVMediaType type, bool hw)
 	id = stream->codec->codec_id;
 #endif
 
-	if (hw) {
-		d->codec = find_hardware_decoder(id);
-		if (d->codec) {
-			ret = mp_open_codec(d);
-			if (ret < 0)
-				d->codec = NULL;
-		}
-	}
+	if (id == AV_CODEC_ID_VP8)
+		d->codec = avcodec_find_decoder_by_name("libvpx");
+	else if (id == AV_CODEC_ID_VP9)
+		d->codec = avcodec_find_decoder_by_name("libvpx-vp9");
+
+	if (!d->codec)
+		d->codec = avcodec_find_decoder(id);
 
 	if (!d->codec) {
-		if (id == AV_CODEC_ID_VP8)
-			d->codec = avcodec_find_decoder_by_name("libvpx");
-		else if (id == AV_CODEC_ID_VP9)
-			d->codec = avcodec_find_decoder_by_name("libvpx-vp9");
-
-		if (!d->codec)
-			d->codec = avcodec_find_decoder(id);
-		if (!d->codec) {
-			blog(LOG_WARNING, "MP: Failed to find %s codec",
-			     av_get_media_type_string(type));
-			return false;
-		}
+		blog(LOG_WARNING, "MP: Failed to find %s codec",
+		     av_get_media_type_string(type));
+		return false;
+	}
 
-		ret = mp_open_codec(d);
-		if (ret < 0) {
-			blog(LOG_WARNING, "MP: Failed to open %s decoder: %s",
-			     av_get_media_type_string(type), av_err2str(ret));
-			return false;
-		}
+	ret = mp_open_codec(d, hw);
+	if (ret < 0) {
+		blog(LOG_WARNING, "MP: Failed to open %s decoder: %s",
+		     av_get_media_type_string(type), av_err2str(ret));
+		return false;
 	}
 
 	d->frame = av_frame_alloc();
@@ -139,6 +165,19 @@ bool mp_decode_init(mp_media_t *m, enum AVMediaType type, bool hw)
 		return false;
 	}
 
+	if (d->hw) {
+		d->hw_frame = av_frame_alloc();
+		if (!d->hw_frame) {
+			blog(LOG_WARNING, "MP: Failed to allocate %s hw frame",
+			     av_get_media_type_string(type));
+			return false;
+		}
+
+		d->in_frame = d->hw_frame;
+	} else {
+		d->in_frame = d->frame;
+	}
+
 	if (d->codec->capabilities & CODEC_CAP_TRUNC)
 		d->decoder->flags |= CODEC_FLAG_TRUNC;
 	return true;
@@ -163,6 +202,10 @@ void mp_decode_free(struct mp_decode *d)
 	mp_decode_clear_packets(d);
 	circlebuf_free(&d->packets);
 
+	if (d->hw_frame) {
+		av_frame_unref(d->hw_frame);
+		av_free(d->hw_frame);
+	}
 	if (d->decoder) {
 #if LIBAVFORMAT_VERSION_INT >= AV_VERSION_INT(57, 40, 101)
 		avcodec_free_context(&d->decoder);
@@ -190,8 +233,8 @@ static inline int64_t get_estimated_duration(struct mp_decode *d,
 		return d->frame_pts - last_pts;
 
 	if (d->audio) {
-		return av_rescale_q(d->frame->nb_samples,
-				    (AVRational){1, d->frame->sample_rate},
+		return av_rescale_q(d->in_frame->nb_samples,
+				    (AVRational){1, d->in_frame->sample_rate},
 				    (AVRational){1, 1000000000});
 	} else {
 		if (d->last_duration)
@@ -209,7 +252,7 @@ static int decode_packet(struct mp_decode *d, int *got_frame)
 	*got_frame = 0;
 
 #ifdef USE_NEW_FFMPEG_DECODE_API
-	ret = avcodec_receive_frame(d->decoder, d->frame);
+	ret = avcodec_receive_frame(d->decoder, d->in_frame);
 	if (ret != 0 && ret != AVERROR(EAGAIN)) {
 		if (ret == AVERROR_EOF)
 			ret = 0;
@@ -224,7 +267,7 @@ static int decode_packet(struct mp_decode *d, int *got_frame)
 			return ret;
 		}
 
-		ret = avcodec_receive_frame(d->decoder, d->frame);
+		ret = avcodec_receive_frame(d->decoder, d->in_frame);
 		if (ret != 0 && ret != AVERROR(EAGAIN)) {
 			if (ret == AVERROR_EOF)
 				ret = 0;
@@ -240,13 +283,23 @@ static int decode_packet(struct mp_decode *d, int *got_frame)
 
 #else
 	if (d->audio) {
-		ret = avcodec_decode_audio4(d->decoder, d->frame, got_frame,
+		ret = avcodec_decode_audio4(d->decoder, d->in_frame, got_frame,
 					    &d->pkt);
 	} else {
-		ret = avcodec_decode_video2(d->decoder, d->frame, got_frame,
+		ret = avcodec_decode_video2(d->decoder, d->in_frame, got_frame,
 					    &d->pkt);
 	}
 #endif
+
+#ifdef USE_NEW_HARDWARE_CODEC_METHOD
+	if (*got_frame && ret && d->hw) {
+		int err = av_hwframe_transfer_data(d->frame, d->hw_frame, 0);
+		if (err != 0) {
+			ret = 0;
+			*got_frame = false;
+		}
+	}
+#endif
 	return ret;
 }
 
@@ -319,15 +372,15 @@ bool mp_decode_next(struct mp_decode *d)
 	if (d->frame_ready) {
 		int64_t last_pts = d->frame_pts;
 
-		if (d->frame->best_effort_timestamp == AV_NOPTS_VALUE)
+		if (d->in_frame->best_effort_timestamp == AV_NOPTS_VALUE)
 			d->frame_pts = d->next_pts;
 		else
 			d->frame_pts =
-				av_rescale_q(d->frame->best_effort_timestamp,
+				av_rescale_q(d->in_frame->best_effort_timestamp,
 					     d->stream->time_base,
 					     (AVRational){1, 1000000000});
 
-		int64_t duration = d->frame->pkt_duration;
+		int64_t duration = d->in_frame->pkt_duration;
 		if (!duration)
 			duration = get_estimated_duration(d, last_pts);
 		else

deps/media-playback/media-playback/decode.h

@@ -63,10 +63,13 @@ struct mp_decode {
 	int64_t last_duration;
 	int64_t frame_pts;
 	int64_t next_pts;
+	AVFrame *in_frame;
+	AVFrame *hw_frame;
 	AVFrame *frame;
 	bool got_first_keyframe;
 	bool frame_ready;
 	bool eof;
+	bool hw;
 
 	AVPacket orig_pkt;
 	AVPacket pkt;

plugins/win-dshow/ffmpeg-decode.c

@@ -19,7 +19,57 @@
 #include "obs-ffmpeg-compat.h"
 #include <obs-avc.h>
 
-int ffmpeg_decode_init(struct ffmpeg_decode *decode, enum AVCodecID id)
+#if LIBAVCODEC_VERSION_INT > AV_VERSION_INT(58, 4, 100)
+#define USE_NEW_HARDWARE_CODEC_METHOD
+#endif
+
+#ifdef USE_NEW_HARDWARE_CODEC_METHOD
+enum AVHWDeviceType hw_priority[] = {
+	AV_HWDEVICE_TYPE_D3D11VA, AV_HWDEVICE_TYPE_DXVA2, AV_HWDEVICE_TYPE_QSV,
+	AV_HWDEVICE_TYPE_CUDA,    AV_HWDEVICE_TYPE_NONE,
+};
+
+static bool has_hw_type(AVCodec *c, enum AVHWDeviceType type)
+{
+	for (int i = 0;; i++) {
+		const AVCodecHWConfig *config = avcodec_get_hw_config(c, i);
+		if (!config) {
+			break;
+		}
+
+		if (config->methods & AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX &&
+		    config->device_type == type)
+			return true;
+	}
+
+	return false;
+}
+
+static void init_hw_decoder(struct ffmpeg_decode *d)
+{
+	enum AVHWDeviceType *priority = hw_priority;
+	AVBufferRef *hw_ctx = NULL;
+
+	while (*priority != AV_HWDEVICE_TYPE_NONE) {
+		if (has_hw_type(d->codec, *priority)) {
+			int ret = av_hwdevice_ctx_create(&hw_ctx, *priority,
+							 NULL, NULL, 0);
+			if (ret == 0)
+				break;
+		}
+
+		priority++;
+	}
+
+	if (hw_ctx) {
+		d->decoder->hw_device_ctx = av_buffer_ref(hw_ctx);
+		d->hw = true;
+	}
+}
+#endif
+
+int ffmpeg_decode_init(struct ffmpeg_decode *decode, enum AVCodecID id,
+		       bool use_hw)
 {
 	int ret;
 
@@ -32,6 +82,15 @@ int ffmpeg_decode_init(struct ffmpeg_decode *decode, enum AVCodecID id)
 
 	decode->decoder = avcodec_alloc_context3(decode->codec);
 
+	decode->decoder->thread_count = 0;
+
+#ifdef USE_NEW_HARDWARE_CODEC_METHOD
+	if (use_hw)
+		init_hw_decoder(decode);
+#else
+	(void)use_hw;
+#endif
+
 	ret = avcodec_open2(decode->decoder, decode->codec, NULL);
 	if (ret < 0) {
 		ffmpeg_decode_free(decode);
@@ -46,6 +105,9 @@ int ffmpeg_decode_init(struct ffmpeg_decode *decode, enum AVCodecID id)
 
 void ffmpeg_decode_free(struct ffmpeg_decode *decode)
 {
+	if (decode->hw_frame)
+		av_free(decode->hw_frame);
+
 	if (decode->decoder) {
 		avcodec_close(decode->decoder);
 		av_free(decode->decoder);
@@ -214,6 +276,7 @@ bool ffmpeg_decode_video(struct ffmpeg_decode *decode, uint8_t *data,
 	AVPacket packet = {0};
 	int got_frame = false;
 	enum video_format new_format;
+	AVFrame *out_frame;
 	int ret;
 
 	*got_output = false;
@@ -233,11 +296,20 @@ bool ffmpeg_decode_video(struct ffmpeg_decode *decode, uint8_t *data,
 		decode->frame = av_frame_alloc();
 		if (!decode->frame)
 			return false;
+
+		if (decode->hw && !decode->hw_frame) {
+			decode->hw_frame = av_frame_alloc();
+			if (!decode->hw_frame)
+				return false;
+		}
 	}
 
+	out_frame = decode->hw ? decode->hw_frame : decode->frame;
+
 	ret = avcodec_send_packet(decode->decoder, &packet);
-	if (ret == 0)
-		ret = avcodec_receive_frame(decode->decoder, decode->frame);
+	if (ret == 0) {
+		ret = avcodec_receive_frame(decode->decoder, out_frame);
+	}
 
 	got_frame = (ret == 0);
 
@@ -249,6 +321,15 @@ bool ffmpeg_decode_video(struct ffmpeg_decode *decode, uint8_t *data,
 	else if (!got_frame)
 		return true;
 
+#ifdef USE_NEW_HARDWARE_CODEC_METHOD
+	if (got_frame && decode->hw) {
+		ret = av_hwframe_transfer_data(decode->frame, out_frame, 0);
+		if (ret < 0) {
+			return false;
+		}
+	}
+#endif
+
 	for (size_t i = 0; i < MAX_AV_PLANES; i++) {
 		frame->data[i] = decode->frame->data[i];
 		frame->linesize[i] = decode->frame->linesize[i];

plugins/win-dshow/ffmpeg-decode.h

@@ -40,13 +40,16 @@ struct ffmpeg_decode {
 	AVCodecContext *decoder;
 	AVCodec *codec;
 
+	AVFrame *hw_frame;
 	AVFrame *frame;
+	bool hw;
 
 	uint8_t *packet_buffer;
 	size_t packet_size;
 };
 
-extern int ffmpeg_decode_init(struct ffmpeg_decode *decode, enum AVCodecID id);
+extern int ffmpeg_decode_init(struct ffmpeg_decode *decode, enum AVCodecID id,
+			      bool use_hw);
 extern void ffmpeg_decode_free(struct ffmpeg_decode *decode);
 
 extern bool ffmpeg_decode_audio(struct ffmpeg_decode *decode, uint8_t *data,

plugins/win-dshow/win-dshow.cpp

@@ -462,11 +462,18 @@ static inline enum speaker_layout convert_speaker_layout(uint8_t channels)
 //#define LOG_ENCODED_VIDEO_TS 1
 //#define LOG_ENCODED_AUDIO_TS 1
 
+#define MAX_SW_RES_INT (1920 * 1080)
+
 void DShowInput::OnEncodedVideoData(enum AVCodecID id, unsigned char *data,
 				    size_t size, long long ts)
 {
 	if (!ffmpeg_decode_valid(video_decoder)) {
-		if (ffmpeg_decode_init(video_decoder, id) < 0) {
+		/* Only use MJPEG hardware decoding on resolutions higher
+		 * than 1920x1080.  The reason why is because we want to strike
+		 * a reasonable balance between hardware and CPU usage. */
+		bool useHW = videoConfig.format != VideoFormat::MJPEG ||
+			     (videoConfig.cx * videoConfig.cy) > MAX_SW_RES_INT;
+		if (ffmpeg_decode_init(video_decoder, id, useHW) < 0) {
 			blog(LOG_WARNING, "Could not initialize video decoder");
 			return;
 		}
@@ -571,7 +578,7 @@ void DShowInput::OnEncodedAudioData(enum AVCodecID id, unsigned char *data,
 				    size_t size, long long ts)
 {
 	if (!ffmpeg_decode_valid(audio_decoder)) {
-		if (ffmpeg_decode_init(audio_decoder, id) < 0) {
+		if (ffmpeg_decode_init(audio_decoder, id, false) < 0) {
 			blog(LOG_WARNING, "Could not initialize audio decoder");
 			return;
 		}