libobs: Fix FFmpeg constants

(This commit also modifies the deps/media-playback, obs-ffmpeg, and
win-dshow modules)

More fixes due to ffmpeg renaming some constants and deprecating
AVFMT_RAWPICTURE and AV_PIX_FMT_VDA_VLD.
Latter replaced by AV_PIX_FMT_VIDEOTOOLBOX per ffmpeg dev advice.

Closes jp9000/obs-studio#1061

deps/media-playback/media-playback/decode.c

@@ -24,8 +24,8 @@ static AVCodec *find_hardware_decoder(enum AVCodecID id)
 
 	while (hwa) {
 		if (hwa->id == id) {
-			if (hwa->pix_fmt == AV_PIX_FMT_VDA_VLD ||
-			    hwa->pix_fmt == AV_PIX_FMT_DXVA2_VLD ||
+			if (hwa->pix_fmt == AV_PIX_FMT_VIDEOTOOLBOX ||
+				hwa->pix_fmt == AV_PIX_FMT_DXVA2_VLD ||
 			    hwa->pix_fmt == AV_PIX_FMT_VAAPI_VLD) {
 				c = avcodec_find_decoder_by_name(hwa->name);
 				if (c)
@@ -142,8 +142,8 @@ bool mp_decode_init(mp_media_t *m, enum AVMediaType type, bool hw)
 		return false;
 	}
 
-	if (d->codec->capabilities & AV_CODEC_CAP_TRUNCATED)
-		d->decoder->flags |= AV_CODEC_FLAG_TRUNCATED;
+	if (d->codec->capabilities & CODEC_CAP_TRUNC)
+		d->decoder->flags |= CODEC_FLAG_TRUNC;
 	return true;
 }
 

deps/media-playback/media-playback/decode.h

@@ -36,6 +36,14 @@ extern "C" {
 #pragma warning(pop)
 #endif
 
+#if LIBAVCODEC_VERSION_MAJOR >= 58
+#define CODEC_CAP_TRUNC AV_CODEC_CAP_TRUNCATED
+#define CODEC_FLAG_TRUNC AV_CODEC_FLAG_TRUNCATED
+#else
+#define CODEC_CAP_TRUNC CODEC_CAP_TRUNCATED
+#define CODEC_FLAG_TRUNC CODEC_FLAG_TRUNCATED
+#endif
+
 struct mp_media;
 
 struct mp_decode {

libobs/media-io/media-remux.c

@@ -26,6 +26,12 @@
 #include <sys/types.h>
 #include <sys/stat.h>
 
+#if LIBAVCODEC_VERSION_MAJOR >= 58
+#define CODEC_FLAG_GLOBAL_H AV_CODEC_FLAG_GLOBAL_HEADER
+#else
+#define CODEC_FLAG_GLOBAL_H CODEC_FLAG_GLOBAL_HEADER
+#endif
+
 struct media_remux_job {
 	int64_t in_size;
 	AVFormatContext *ifmt_ctx, *ofmt_ctx;
@@ -95,7 +101,7 @@ static inline bool init_output(media_remux_job_t job, const char *out_filename)
 
 		out_stream->codec->codec_tag = 0;
 		if (job->ofmt_ctx->oformat->flags & AVFMT_GLOBALHEADER)
-			out_stream->codec->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;
+			out_stream->codec->flags |= CODEC_FLAG_GLOBAL_H;
 	}
 
 #ifndef _NDEBUG

libobs/obs-ffmpeg-compat.h

@@ -20,3 +20,12 @@
 # define av_frame_free avcodec_free_frame
 #endif
 
+#if LIBAVCODEC_VERSION_MAJOR >= 58
+#define CODEC_CAP_TRUNC AV_CODEC_CAP_TRUNCATED
+#define CODEC_FLAG_TRUNC AV_CODEC_FLAG_TRUNCATED
+#define INPUT_BUFFER_PADDING_SIZE AV_INPUT_BUFFER_PADDING_SIZE
+#else
+#define CODEC_CAP_TRUNC CODEC_CAP_TRUNCATED
+#define CODEC_FLAG_TRUNC CODEC_FLAG_TRUNCATED
+#define INPUT_BUFFER_PADDING_SIZE FF_INPUT_BUFFER_PADDING_SIZE
+#endif

plugins/obs-ffmpeg/ffmpeg-mux/ffmpeg-mux.c

@@ -29,6 +29,12 @@
 
 #include <libavformat/avformat.h>
 
+#if LIBAVCODEC_VERSION_MAJOR >= 58
+#define CODEC_FLAG_GLOBAL_H AV_CODEC_FLAG_GLOBAL_HEADER
+#else
+#define CODEC_FLAG_GLOBAL_H CODEC_FLAG_GLOBAL_HEADER
+#endif
+
 /* ------------------------------------------------------------------------- */
 
 struct resize_buf {
@@ -312,7 +318,7 @@ static void create_video_stream(struct ffmpeg_mux *ffm)
 	ffm->video_stream->time_base = context->time_base;
 
 	if (ffm->output->oformat->flags & AVFMT_GLOBALHEADER)
-		context->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;
+		context->flags |= CODEC_FLAG_GLOBAL_H;
 }
 
 static void create_audio_stream(struct ffmpeg_mux *ffm, int idx)
@@ -348,7 +354,7 @@ static void create_audio_stream(struct ffmpeg_mux *ffm, int idx)
 			av_get_default_channel_layout(context->channels);
 
 	if (ffm->output->oformat->flags & AVFMT_GLOBALHEADER)
-		context->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;
+		context->flags |= CODEC_FLAG_GLOBAL_H;
 
 	ffm->num_audio_streams++;
 }

plugins/obs-ffmpeg/obs-ffmpeg-audio-encoders.c

@@ -214,7 +214,7 @@ static void *enc_create(obs_data_t *settings, obs_encoder_t *encoder,
 	/* enable experimental FFmpeg encoder if the only one available */
 	enc->context->strict_std_compliance = -2;
 
-	enc->context->flags = AV_CODEC_FLAG_GLOBAL_HEADER;
+	enc->context->flags = CODEC_FLAG_GLOBAL_H;
 
 	if (initialize_codec(enc))
 		return enc;

plugins/obs-ffmpeg/obs-ffmpeg-compat.h

@@ -23,3 +23,13 @@
 #if LIBAVCODEC_VERSION_MAJOR >= 57
 #define av_free_packet av_packet_unref
 #endif
+
+#if LIBAVCODEC_VERSION_MAJOR >= 58
+#define CODEC_CAP_TRUNC AV_CODEC_CAP_TRUNCATED
+#define CODEC_FLAG_TRUNC AV_CODEC_FLAG_TRUNCATED
+#define CODEC_FLAG_GLOBAL_H AV_CODEC_FLAG_GLOBAL_HEADER
+#else
+#define CODEC_CAP_TRUNC CODEC_CAP_TRUNCATED
+#define CODEC_FLAG_TRUNC CODEC_FLAG_TRUNCATED
+#define CODEC_FLAG_GLOBAL_H CODEC_FLAG_GLOBAL_HEADER
+#endif

plugins/obs-ffmpeg/obs-ffmpeg-output.c

@@ -260,7 +260,7 @@ static bool create_video_stream(struct ffmpeg_data *data)
 	data->video->time_base = context->time_base;
 
 	if (data->output->oformat->flags & AVFMT_GLOBALHEADER)
-		context->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;
+		context->flags |= CODEC_FLAG_GLOBAL_H;
 
 	if (!open_video_codec(data))
 		return false;
@@ -348,7 +348,7 @@ static bool create_audio_stream(struct ffmpeg_data *data)
 	data->audio_size = get_audio_size(data->audio_format, aoi.speakers, 1);
 
 	if (data->output->oformat->flags & AVFMT_GLOBALHEADER)
-		context->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;
+		context->flags |= CODEC_FLAG_GLOBAL_H;
 
 	return open_audio_codec(data);
 }
@@ -686,7 +686,7 @@ static void receive_video(void *param, struct video_data *frame)
 	else
 		copy_data(&data->dst_picture, frame, context->height, context->pix_fmt);
 
-	if (data->output->flags & AVFMT_RAWPICTURE) {
+	if (data->output->flags) {
 		packet.flags        |= AV_PKT_FLAG_KEY;
 		packet.stream_index  = data->video->index;
 		packet.data          = data->dst_picture.data[0];

plugins/win-dshow/ffmpeg-decode.c

@@ -16,6 +16,7 @@
 ******************************************************************************/
 
 #include "ffmpeg-decode.h"
+#include "obs-ffmpeg-compat.h"
 #include <obs-avc.h>
 
 int ffmpeg_decode_init(struct ffmpeg_decode *decode, enum AVCodecID id)
@@ -37,8 +38,8 @@ int ffmpeg_decode_init(struct ffmpeg_decode *decode, enum AVCodecID id)
 		return ret;
 	}
 
-	if (decode->codec->capabilities & CODEC_CAP_TRUNCATED)
-		decode->decoder->flags |= CODEC_FLAG_TRUNCATED;
+	if (decode->codec->capabilities & CODEC_CAP_TRUNC)
+		decode->decoder->flags |= CODEC_FLAG_TRUNC;
 
 	return 0;
 }
@@ -96,7 +97,7 @@ static inline enum audio_format convert_sample_format(int f)
 static inline void copy_data(struct ffmpeg_decode *decode, uint8_t *data,
 		size_t size)
 {
-	size_t new_size = size + FF_INPUT_BUFFER_PADDING_SIZE;
+	size_t new_size = size + INPUT_BUFFER_PADDING_SIZE;
 
 	if (decode->packet_size < new_size) {
 		decode->packet_buffer = brealloc(decode->packet_buffer,
@@ -104,7 +105,7 @@ static inline void copy_data(struct ffmpeg_decode *decode, uint8_t *data,
 		decode->packet_size   = new_size;
 	}
 
-	memset(decode->packet_buffer + size, 0, FF_INPUT_BUFFER_PADDING_SIZE);
+	memset(decode->packet_buffer + size, 0, INPUT_BUFFER_PADDING_SIZE);
 	memcpy(decode->packet_buffer, data, size);
 }