obs-ffmpeg: Add HEVC support to NVENC

plugins/obs-ffmpeg/jim-nvenc-helpers.c

@@ -52,13 +52,10 @@ bool nv_failed(obs_encoder_t *encoder, NVENCSTATUS err, const char *func,
 
 bool load_nvenc_lib(void)
 {
-	if (sizeof(void *) == 8) {
-		nvenc_lib = os_dlopen("nvEncodeAPI64.dll");
-	} else {
-		nvenc_lib = os_dlopen("nvEncodeAPI.dll");
-	}
-
-	return !!nvenc_lib;
+	const char *const file = (sizeof(void *) == 8) ? "nvEncodeAPI64.dll"
+						       : "nvEncodeAPI.dll";
+	nvenc_lib = os_dlopen(file);
+	return nvenc_lib != NULL;
 }
 
 static void *load_nv_func(const char *func)
@@ -174,12 +171,20 @@ bool init_nvenc(obs_encoder_t *encoder)
 	return success;
 }
 
-extern struct obs_encoder_info nvenc_info;
+extern struct obs_encoder_info h264_nvenc_info;
+#ifdef ENABLE_HEVC
+extern struct obs_encoder_info hevc_nvenc_info;
+#endif
 
-void jim_nvenc_load(void)
+void jim_nvenc_load(bool h264, bool hevc)
 {
 	pthread_mutex_init(&init_mutex, NULL);
-	obs_register_encoder(&nvenc_info);
+	if (h264)
+		obs_register_encoder(&h264_nvenc_info);
+#ifdef ENABLE_HEVC
+	if (hevc)
+		obs_register_encoder(&hevc_nvenc_info);
+#endif
 }
 
 void jim_nvenc_unload(void)

plugins/obs-ffmpeg/jim-nvenc.c

@@ -3,6 +3,7 @@
 #include <util/darray.h>
 #include <util/dstr.h>
 #include <obs-avc.h>
+#include <obs-hevc.h>
 #include <libavutil/rational.h>
 #define INITGUID
 #include <dxgi.h>
@@ -38,6 +39,9 @@ struct handle_tex {
 
 struct nvenc_data {
 	obs_encoder_t *encoder;
+#ifdef ENABLE_HEVC
+	bool hevc;
+#endif
 
 	void *session;
 	NV_ENC_INITIALIZE_PARAMS params;
@@ -145,20 +149,20 @@ struct nv_texture {
 
 static bool nv_texture_init(struct nvenc_data *enc, struct nv_texture *nvtex)
 {
-	ID3D11Device *device = enc->device;
-	ID3D11Texture2D *tex;
-	HRESULT hr;
+	const bool p010 = obs_p010_tex_active();
 
 	D3D11_TEXTURE2D_DESC desc = {0};
 	desc.Width = enc->cx;
 	desc.Height = enc->cy;
 	desc.MipLevels = 1;
 	desc.ArraySize = 1;
-	desc.Format = DXGI_FORMAT_NV12;
+	desc.Format = p010 ? DXGI_FORMAT_P010 : DXGI_FORMAT_NV12;
 	desc.SampleDesc.Count = 1;
 	desc.BindFlags = D3D11_BIND_RENDER_TARGET;
 
-	hr = device->lpVtbl->CreateTexture2D(device, &desc, NULL, &tex);
+	ID3D11Device *const device = enc->device;
+	ID3D11Texture2D *tex;
+	HRESULT hr = device->lpVtbl->CreateTexture2D(device, &desc, NULL, &tex);
 	if (FAILED(hr)) {
 		error_hr("Failed to create texture");
 		return false;
@@ -171,7 +175,8 @@ static bool nv_texture_init(struct nvenc_data *enc, struct nv_texture *nvtex)
 	res.resourceToRegister = tex;
 	res.width = enc->cx;
 	res.height = enc->cy;
-	res.bufferFormat = NV_ENC_BUFFER_FORMAT_NV12;
+	res.bufferFormat = p010 ? NV_ENC_BUFFER_FORMAT_YUV420_10BIT
+				: NV_ENC_BUFFER_FORMAT_NV12;
 
 	if (NV_FAILED(nv.nvEncRegisterResource(enc->session, &res))) {
 		tex->lpVtbl->Release(tex);
@@ -198,13 +203,22 @@ static void nv_texture_free(struct nvenc_data *enc, struct nv_texture *nvtex)
 /* ------------------------------------------------------------------------- */
 /* Implementation                                                            */
 
-static const char *nvenc_get_name(void *type_data)
+static const char *h264_nvenc_get_name(void *type_data)
 {
 	UNUSED_PARAMETER(type_data);
 	return "NVIDIA NVENC H.264 (new)";
 }
 
-static inline int nv_get_cap(struct nvenc_data *enc, NV_ENC_CAPS cap)
+#ifdef ENABLE_HEVC
+static const char *hevc_nvenc_get_name(void *type_data)
+{
+	UNUSED_PARAMETER(type_data);
+	return "NVIDIA NVENC HEVC (new)";
+}
+#endif
+
+static inline int nv_get_cap_internal(struct nvenc_data *enc, NV_ENC_CAPS cap,
+				      GUID encodeGUID)
 {
 	if (!enc->session)
 		return 0;
@@ -213,10 +227,22 @@ static inline int nv_get_cap(struct nvenc_data *enc, NV_ENC_CAPS cap)
 	int v;
 
 	param.capsToQuery = cap;
-	nv.nvEncGetEncodeCaps(enc->session, NV_ENC_CODEC_H264_GUID, &param, &v);
+	nv.nvEncGetEncodeCaps(enc->session, encodeGUID, &param, &v);
 	return v;
 }
 
+static inline int nv_get_cap_h264(struct nvenc_data *enc, NV_ENC_CAPS cap)
+{
+	return nv_get_cap_internal(enc, cap, NV_ENC_CODEC_H264_GUID);
+}
+
+#ifdef ENABLE_HEVC
+static inline int nv_get_cap_hevc(struct nvenc_data *enc, NV_ENC_CAPS cap)
+{
+	return nv_get_cap_internal(enc, cap, NV_ENC_CODEC_HEVC_GUID);
+}
+#endif
+
 static bool nvenc_update(void *data, obs_data_t *settings)
 {
 	struct nvenc_data *enc = data;
@@ -323,8 +349,8 @@ static bool init_session(struct nvenc_data *enc)
 	return true;
 }
 
-static bool init_encoder(struct nvenc_data *enc, obs_data_t *settings,
-			 bool psycho_aq)
+static bool init_encoder_h264(struct nvenc_data *enc, obs_data_t *settings,
+			      bool psycho_aq)
 {
 	const char *rc = obs_data_get_string(settings, "rate_control");
 	int bitrate = (int)obs_data_get_int(settings, "bitrate");
@@ -380,7 +406,8 @@ static bool init_encoder(struct nvenc_data *enc, obs_data_t *settings,
 	const bool rc_lossless = astrcmpi(rc, "lossless") == 0;
 	bool lossless = rc_lossless;
 	if (rc_lossless) {
-		lossless = nv_get_cap(enc, NV_ENC_CAPS_SUPPORT_LOSSLESS_ENCODE);
+		lossless = nv_get_cap_h264(enc,
+					   NV_ENC_CAPS_SUPPORT_LOSSLESS_ENCODE);
 		if (lossless) {
 			nv_preset = hp ? NV_ENC_PRESET_LOSSLESS_HP_GUID
 				       : NV_ENC_PRESET_LOSSLESS_DEFAULT_GUID;
@@ -476,7 +503,7 @@ static bool init_encoder(struct nvenc_data *enc, obs_data_t *settings,
 
 	/* lookahead */
 	const bool use_profile_lookahead = config->rcParams.enableLookahead;
-	lookahead = nv_get_cap(enc, NV_ENC_CAPS_SUPPORT_LOOKAHEAD) &&
+	lookahead = nv_get_cap_h264(enc, NV_ENC_CAPS_SUPPORT_LOOKAHEAD) &&
 		    (lookahead || use_profile_lookahead);
 	if (lookahead) {
 		enc->rc_lookahead = use_profile_lookahead
@@ -511,7 +538,7 @@ static bool init_encoder(struct nvenc_data *enc, obs_data_t *settings,
 	}
 
 	/* psycho aq */
-	if (nv_get_cap(enc, NV_ENC_CAPS_SUPPORT_TEMPORAL_AQ)) {
+	if (nv_get_cap_h264(enc, NV_ENC_CAPS_SUPPORT_TEMPORAL_AQ)) {
 		config->rcParams.enableAQ = psycho_aq;
 		config->rcParams.aqStrength = 8;
 		config->rcParams.enableTemporalAQ = psycho_aq;
@@ -523,7 +550,7 @@ static bool init_encoder(struct nvenc_data *enc, obs_data_t *settings,
 	/* rate control               */
 
 	enc->can_change_bitrate =
-		nv_get_cap(enc, NV_ENC_CAPS_SUPPORT_DYN_BITRATE_CHANGE) &&
+		nv_get_cap_h264(enc, NV_ENC_CAPS_SUPPORT_DYN_BITRATE_CHANGE) &&
 		!lookahead;
 
 	config->rcParams.rateControlMode = twopass ? NV_ENC_PARAMS_RC_VBR_HQ
@@ -594,6 +621,290 @@ static bool init_encoder(struct nvenc_data *enc, obs_data_t *settings,
 	return true;
 }
 
+#ifdef ENABLE_HEVC
+static bool init_encoder_hevc(struct nvenc_data *enc, obs_data_t *settings,
+			      bool psycho_aq)
+{
+	const char *rc = obs_data_get_string(settings, "rate_control");
+	int bitrate = (int)obs_data_get_int(settings, "bitrate");
+	int max_bitrate = (int)obs_data_get_int(settings, "max_bitrate");
+	int cqp = (int)obs_data_get_int(settings, "cqp");
+	int keyint_sec = (int)obs_data_get_int(settings, "keyint_sec");
+	const char *preset = obs_data_get_string(settings, "preset");
+	const char *profile = obs_data_get_string(settings, "profile");
+	bool lookahead = obs_data_get_bool(settings, "lookahead");
+	int bf = (int)obs_data_get_int(settings, "bf");
+	bool vbr = astrcmpi(rc, "VBR") == 0;
+	NVENCSTATUS err;
+
+	video_t *video = obs_encoder_video(enc->encoder);
+	const struct video_output_info *voi = video_output_get_info(video);
+
+	enc->cx = voi->width;
+	enc->cy = voi->height;
+
+	/* -------------------------- */
+	/* get preset                 */
+
+	GUID nv_preset = NV_ENC_PRESET_DEFAULT_GUID;
+	bool twopass = false;
+	bool hp = false;
+	bool ll = false;
+
+	if (astrcmpi(preset, "hq") == 0) {
+		nv_preset = NV_ENC_PRESET_HQ_GUID;
+
+	} else if (astrcmpi(preset, "mq") == 0) {
+		nv_preset = NV_ENC_PRESET_HQ_GUID;
+		twopass = true;
+
+	} else if (astrcmpi(preset, "hp") == 0) {
+		nv_preset = NV_ENC_PRESET_HP_GUID;
+		hp = true;
+
+	} else if (astrcmpi(preset, "ll") == 0) {
+		nv_preset = NV_ENC_PRESET_LOW_LATENCY_DEFAULT_GUID;
+		ll = true;
+
+	} else if (astrcmpi(preset, "llhq") == 0) {
+		nv_preset = NV_ENC_PRESET_LOW_LATENCY_HQ_GUID;
+		ll = true;
+
+	} else if (astrcmpi(preset, "llhp") == 0) {
+		nv_preset = NV_ENC_PRESET_LOW_LATENCY_HP_GUID;
+		hp = true;
+		ll = true;
+	}
+
+	const bool rc_lossless = astrcmpi(rc, "lossless") == 0;
+	bool lossless = rc_lossless;
+	if (rc_lossless) {
+		lossless = nv_get_cap_hevc(enc,
+					   NV_ENC_CAPS_SUPPORT_LOSSLESS_ENCODE);
+		if (lossless) {
+			nv_preset = hp ? NV_ENC_PRESET_LOSSLESS_HP_GUID
+				       : NV_ENC_PRESET_LOSSLESS_DEFAULT_GUID;
+		} else {
+			warn("lossless encode is not supported, ignoring");
+		}
+	}
+
+	/* -------------------------- */
+	/* get preset default config  */
+
+	NV_ENC_PRESET_CONFIG preset_config = {NV_ENC_PRESET_CONFIG_VER,
+					      {NV_ENC_CONFIG_VER}};
+
+	err = nv.nvEncGetEncodePresetConfig(enc->session,
+					    NV_ENC_CODEC_HEVC_GUID, nv_preset,
+					    &preset_config);
+	if (nv_failed(enc->encoder, err, __FUNCTION__,
+		      "nvEncGetEncodePresetConfig")) {
+		return false;
+	}
+
+	/* -------------------------- */
+	/* main configuration         */
+
+	enc->config = preset_config.presetCfg;
+
+	uint32_t gop_size =
+		(keyint_sec) ? keyint_sec * voi->fps_num / voi->fps_den : 250;
+
+	NV_ENC_INITIALIZE_PARAMS *params = &enc->params;
+	NV_ENC_CONFIG *config = &enc->config;
+	NV_ENC_CONFIG_HEVC *hevc_config = &config->encodeCodecConfig.hevcConfig;
+	NV_ENC_CONFIG_H264_VUI_PARAMETERS *vui_params =
+		&hevc_config->hevcVUIParameters;
+
+	int darWidth, darHeight;
+	av_reduce(&darWidth, &darHeight, voi->width, voi->height, 1024 * 1024);
+
+	memset(params, 0, sizeof(*params));
+	params->version = NV_ENC_INITIALIZE_PARAMS_VER;
+	params->encodeGUID = NV_ENC_CODEC_HEVC_GUID;
+	params->presetGUID = nv_preset;
+	params->encodeWidth = voi->width;
+	params->encodeHeight = voi->height;
+	params->darWidth = darWidth;
+	params->darHeight = darHeight;
+	params->frameRateNum = voi->fps_num;
+	params->frameRateDen = voi->fps_den;
+	params->enableEncodeAsync = 1;
+	params->enablePTD = 1;
+	params->encodeConfig = &enc->config;
+	config->gopLength = gop_size;
+	config->frameIntervalP = 1 + bf;
+	hevc_config->idrPeriod = gop_size;
+
+	bool repeat_headers = obs_data_get_bool(settings, "repeat_headers");
+	if (repeat_headers) {
+		hevc_config->repeatSPSPPS = 1;
+		hevc_config->disableSPSPPS = 0;
+		hevc_config->outputAUD = 1;
+	}
+
+	hevc_config->sliceMode = 3;
+	hevc_config->sliceModeData = 1;
+
+	hevc_config->useBFramesAsRef = NV_ENC_BFRAME_REF_MODE_DISABLED;
+
+	vui_params->videoSignalTypePresentFlag = 1;
+	vui_params->videoFullRangeFlag = (voi->range == VIDEO_RANGE_FULL);
+	vui_params->colourDescriptionPresentFlag = 1;
+
+	hevc_config->pixelBitDepthMinus8 = obs_p010_tex_active() ? 2 : 0;
+
+	switch (voi->colorspace) {
+	case VIDEO_CS_601:
+		vui_params->colourPrimaries = 6;
+		vui_params->transferCharacteristics = 6;
+		vui_params->colourMatrix = 6;
+		break;
+	case VIDEO_CS_DEFAULT:
+	case VIDEO_CS_709:
+		vui_params->colourPrimaries = 1;
+		vui_params->transferCharacteristics = 1;
+		vui_params->colourMatrix = 1;
+		break;
+	case VIDEO_CS_SRGB:
+		vui_params->colourPrimaries = 1;
+		vui_params->transferCharacteristics = 13;
+		vui_params->colourMatrix = 1;
+		break;
+	case VIDEO_CS_2100_PQ:
+		vui_params->colourPrimaries = 9;
+		vui_params->transferCharacteristics = 16;
+		vui_params->colourMatrix = 9;
+		break;
+	case VIDEO_CS_2100_HLG:
+		vui_params->colourPrimaries = 9;
+		vui_params->transferCharacteristics = 18;
+		vui_params->colourMatrix = 9;
+	}
+
+	enc->bframes = bf;
+
+	/* lookahead */
+	const bool use_profile_lookahead = config->rcParams.enableLookahead;
+	lookahead = nv_get_cap_hevc(enc, NV_ENC_CAPS_SUPPORT_LOOKAHEAD) &&
+		    (lookahead || use_profile_lookahead);
+	if (lookahead) {
+		enc->rc_lookahead = use_profile_lookahead
+					    ? config->rcParams.lookaheadDepth
+					    : 8;
+	}
+
+	int buf_count = max(4, config->frameIntervalP * 2 * 2);
+	if (lookahead) {
+		buf_count = max(buf_count, config->frameIntervalP +
+						   enc->rc_lookahead +
+						   EXTRA_BUFFERS);
+	}
+
+	buf_count = min(64, buf_count);
+	enc->buf_count = buf_count;
+
+	const int output_delay = buf_count - 1;
+	enc->output_delay = output_delay;
+
+	if (lookahead) {
+		const int lkd_bound = output_delay - config->frameIntervalP - 4;
+		if (lkd_bound >= 0) {
+			config->rcParams.enableLookahead = 1;
+			config->rcParams.lookaheadDepth =
+				max(enc->rc_lookahead, lkd_bound);
+			config->rcParams.disableIadapt = 0;
+			config->rcParams.disableBadapt = 0;
+		} else {
+			lookahead = false;
+		}
+	}
+
+	/* psycho aq */
+	if (nv_get_cap_hevc(enc, NV_ENC_CAPS_SUPPORT_TEMPORAL_AQ)) {
+		config->rcParams.enableAQ = psycho_aq;
+		config->rcParams.aqStrength = 8;
+		config->rcParams.enableTemporalAQ = psycho_aq;
+	} else if (psycho_aq) {
+		warn("Ignoring Psycho Visual Tuning request since GPU is not capable");
+	}
+
+	/* -------------------------- */
+	/* rate control               */
+
+	enc->can_change_bitrate =
+		nv_get_cap_hevc(enc, NV_ENC_CAPS_SUPPORT_DYN_BITRATE_CHANGE) &&
+		!lookahead;
+
+	config->rcParams.rateControlMode = twopass ? NV_ENC_PARAMS_RC_VBR_HQ
+						   : NV_ENC_PARAMS_RC_VBR;
+
+	if (astrcmpi(rc, "cqp") == 0 || rc_lossless) {
+		if (lossless)
+			cqp = 0;
+
+		config->rcParams.rateControlMode = NV_ENC_PARAMS_RC_CONSTQP;
+		config->rcParams.constQP.qpInterP = cqp;
+		config->rcParams.constQP.qpInterB = cqp;
+		config->rcParams.constQP.qpIntra = cqp;
+		enc->can_change_bitrate = false;
+
+		bitrate = 0;
+		max_bitrate = 0;
+
+	} else if (astrcmpi(rc, "vbr") != 0) { /* CBR by default */
+		hevc_config->outputBufferingPeriodSEI = 1;
+		config->rcParams.rateControlMode =
+			twopass ? NV_ENC_PARAMS_RC_2_PASS_QUALITY
+				: NV_ENC_PARAMS_RC_CBR;
+	}
+
+	hevc_config->outputPictureTimingSEI = 1;
+	config->rcParams.averageBitRate = bitrate * 1000;
+	config->rcParams.maxBitRate = vbr ? max_bitrate * 1000 : bitrate * 1000;
+	config->rcParams.vbvBufferSize = bitrate * 1000;
+
+	/* -------------------------- */
+	/* profile                    */
+
+	if (astrcmpi(profile, "main10") == 0) {
+		config->profileGUID = NV_ENC_HEVC_PROFILE_MAIN10_GUID;
+	} else if (obs_p010_tex_active()) {
+		blog(LOG_WARNING, "[jim-nvenc] Forcing main10 for P010");
+		config->profileGUID = NV_ENC_HEVC_PROFILE_MAIN10_GUID;
+	} else {
+		config->profileGUID = NV_ENC_HEVC_PROFILE_MAIN_GUID;
+	}
+
+	/* -------------------------- */
+	/* initialize                 */
+
+	if (NV_FAILED(nv.nvEncInitializeEncoder(enc->session, params))) {
+		return false;
+	}
+
+	info("settings:\n"
+	     "\trate_control: %s\n"
+	     "\tbitrate:      %d\n"
+	     "\tcqp:          %d\n"
+	     "\tkeyint:       %d\n"
+	     "\tpreset:       %s\n"
+	     "\tprofile:      %s\n"
+	     "\twidth:        %d\n"
+	     "\theight:       %d\n"
+	     "\t2-pass:       %s\n"
+	     "\tb-frames:     %d\n"
+	     "\tlookahead:    %s\n"
+	     "\tpsycho_aq:    %s\n",
+	     rc, bitrate, cqp, gop_size, preset, profile, enc->cx, enc->cy,
+	     twopass ? "true" : "false", bf, lookahead ? "true" : "false",
+	     psycho_aq ? "true" : "false");
+
+	return true;
+}
+#endif
+
 static bool init_bitstreams(struct nvenc_data *enc)
 {
 	da_reserve(enc->bitstreams, enc->buf_count);
@@ -626,8 +937,8 @@ static bool init_textures(struct nvenc_data *enc)
 
 static void nvenc_destroy(void *data);
 
-static void *nvenc_create_internal(obs_data_t *settings, obs_encoder_t *encoder,
-				   bool psycho_aq)
+static void *nvenc_create_internal(bool hevc, obs_data_t *settings,
+				   obs_encoder_t *encoder, bool psycho_aq)
 {
 	NV_ENCODE_API_FUNCTION_LIST init = {NV_ENCODE_API_FUNCTION_LIST_VER};
 	struct nvenc_data *enc = bzalloc(sizeof(*enc));
@@ -646,8 +957,17 @@ static void *nvenc_create_internal(obs_data_t *settings, obs_encoder_t *encoder,
 	if (!init_session(enc)) {
 		goto fail;
 	}
-	if (!init_encoder(enc, settings, psycho_aq)) {
-		goto fail;
+#ifdef ENABLE_HEVC
+	if (hevc) {
+		if (!init_encoder_hevc(enc, settings, psycho_aq)) {
+			goto fail;
+		}
+	} else
+#endif
+	{
+		if (!init_encoder_h264(enc, settings, psycho_aq)) {
+			goto fail;
+		}
 	}
 	if (!init_bitstreams(enc)) {
 		goto fail;
@@ -656,6 +976,9 @@ static void *nvenc_create_internal(obs_data_t *settings, obs_encoder_t *encoder,
 		goto fail;
 	}
 
+#ifdef ENABLE_HEVC
+	enc->hevc = hevc;
+#endif
 	return enc;
 
 fail:
@@ -663,7 +986,8 @@ fail:
 	return NULL;
 }
 
-static void *nvenc_create(obs_data_t *settings, obs_encoder_t *encoder)
+static void *nvenc_create_h264_hevc(bool hevc, obs_data_t *settings,
+				    obs_encoder_t *encoder)
 {
 	/* this encoder requires shared textures, this cannot be used on a
 	 * gpu other than the one OBS is currently running on. */
@@ -680,19 +1004,19 @@ static void *nvenc_create(obs_data_t *settings, obs_encoder_t *encoder)
 		goto reroute;
 	}
 
-	if (!obs_nv12_tex_active()) {
+	if (!obs_p010_tex_active() && !obs_nv12_tex_active()) {
 		blog(LOG_INFO,
-		     "[jim-nvenc] nv12 not active, falling back to ffmpeg");
+		     "[jim-nvenc] nv12/p010 not active, falling back to ffmpeg");
 		goto reroute;
 	}
 
 	const bool psycho_aq = obs_data_get_bool(settings, "psycho_aq");
 	struct nvenc_data *enc =
-		nvenc_create_internal(settings, encoder, psycho_aq);
+		nvenc_create_internal(hevc, settings, encoder, psycho_aq);
 	if ((enc == NULL) && psycho_aq) {
 		blog(LOG_WARNING, "[jim-nvenc] nvenc_create_internal failed, "
 				  "trying again without Psycho Visual Tuning");
-		enc = nvenc_create_internal(settings, encoder, false);
+		enc = nvenc_create_internal(hevc, settings, encoder, false);
 	}
 
 	if (enc) {
@@ -703,6 +1027,18 @@ reroute:
 	return obs_encoder_create_rerouted(encoder, "ffmpeg_nvenc");
 }
 
+static void *h264_nvenc_create(obs_data_t *settings, obs_encoder_t *encoder)
+{
+	return nvenc_create_h264_hevc(false, settings, encoder);
+}
+
+#ifdef ENABLE_HEVC
+static void *hevc_nvenc_create(obs_data_t *settings, obs_encoder_t *encoder)
+{
+	return nvenc_create_h264_hevc(true, settings, encoder);
+}
+#endif
+
 static bool get_encoded_packet(struct nvenc_data *enc, bool finalize);
 
 static void nvenc_destroy(void *data)
@@ -828,11 +1164,22 @@ static bool get_encoded_packet(struct nvenc_data *enc, bool finalize)
 			size_t size;
 
 			enc->first_packet = false;
-			obs_extract_avc_headers(lock.bitstreamBufferPtr,
-						lock.bitstreamSizeInBytes,
-						&new_packet, &size,
-						&enc->header, &enc->header_size,
-						&enc->sei, &enc->sei_size);
+#ifdef ENABLE_HEVC
+			if (enc->hevc) {
+				obs_extract_hevc_headers(
+					lock.bitstreamBufferPtr,
+					lock.bitstreamSizeInBytes, &new_packet,
+					&size, &enc->header, &enc->header_size,
+					&enc->sei, &enc->sei_size);
+			} else
+#endif
+			{
+				obs_extract_avc_headers(
+					lock.bitstreamBufferPtr,
+					lock.bitstreamSizeInBytes, &new_packet,
+					&size, &enc->header, &enc->header_size,
+					&enc->sei, &enc->sei_size);
+			}
 
 			da_copy_array(enc->packet_data, new_packet, size);
 			bfree(new_packet);
@@ -938,10 +1285,13 @@ static bool nvenc_encode_tex(void *data, uint32_t handle, int64_t pts,
 	params.version = NV_ENC_PIC_PARAMS_VER;
 	params.pictureStruct = NV_ENC_PIC_STRUCT_FRAME;
 	params.inputBuffer = nvtex->mapped_res;
-	params.bufferFmt = NV_ENC_BUFFER_FORMAT_NV12;
+	params.bufferFmt = obs_p010_tex_active()
+				   ? NV_ENC_BUFFER_FORMAT_YUV420_10BIT
+				   : NV_ENC_BUFFER_FORMAT_NV12;
 	params.inputTimeStamp = (uint64_t)pts;
 	params.inputWidth = enc->cx;
 	params.inputHeight = enc->cy;
+	params.inputPitch = enc->cx;
 	params.outputBitstream = bs->ptr;
 	params.completionEvent = bs->event;
 
@@ -990,8 +1340,12 @@ static bool nvenc_encode_tex(void *data, uint32_t handle, int64_t pts,
 	return true;
 }
 
-extern void nvenc_defaults(obs_data_t *settings);
-extern obs_properties_t *nvenc_properties(void *unused);
+extern void h264_nvenc_defaults(obs_data_t *settings);
+extern obs_properties_t *h264_nvenc_properties(void *unused);
+#ifdef ENABLE_HEVC
+extern void hevc_nvenc_defaults(obs_data_t *settings);
+extern obs_properties_t *hevc_nvenc_properties(void *unused);
+#endif
 
 static bool nvenc_extra_data(void *data, uint8_t **header, size_t *size)
 {
@@ -1019,18 +1373,36 @@ static bool nvenc_sei_data(void *data, uint8_t **sei, size_t *size)
 	return true;
 }
 
-struct obs_encoder_info nvenc_info = {
+struct obs_encoder_info h264_nvenc_info = {
 	.id = "jim_nvenc",
 	.codec = "h264",
 	.type = OBS_ENCODER_VIDEO,
 	.caps = OBS_ENCODER_CAP_PASS_TEXTURE | OBS_ENCODER_CAP_DYN_BITRATE,
-	.get_name = nvenc_get_name,
-	.create = nvenc_create,
+	.get_name = h264_nvenc_get_name,
+	.create = h264_nvenc_create,
+	.destroy = nvenc_destroy,
+	.update = nvenc_update,
+	.encode_texture = nvenc_encode_tex,
+	.get_defaults = h264_nvenc_defaults,
+	.get_properties = h264_nvenc_properties,
+	.get_extra_data = nvenc_extra_data,
+	.get_sei_data = nvenc_sei_data,
+};
+
+#ifdef ENABLE_HEVC
+struct obs_encoder_info hevc_nvenc_info = {
+	.id = "jim_hevc_nvenc",
+	.codec = "hevc",
+	.type = OBS_ENCODER_VIDEO,
+	.caps = OBS_ENCODER_CAP_PASS_TEXTURE | OBS_ENCODER_CAP_DYN_BITRATE,
+	.get_name = hevc_nvenc_get_name,
+	.create = hevc_nvenc_create,
 	.destroy = nvenc_destroy,
 	.update = nvenc_update,
 	.encode_texture = nvenc_encode_tex,
-	.get_defaults = nvenc_defaults,
-	.get_properties = nvenc_properties,
+	.get_defaults = hevc_nvenc_defaults,
+	.get_properties = hevc_nvenc_properties,
 	.get_extra_data = nvenc_extra_data,
 	.get_sei_data = nvenc_sei_data,
 };
+#endif

plugins/obs-ffmpeg/obs-ffmpeg-config.h.in

@@ -15,4 +15,3 @@
 #endif
 
 #define ENABLE_FFMPEG_LOGGING @ENABLE_FFMPEG_LOGGING@
-#define ENABLE_HEVC @ENABLE_HEVC@

plugins/obs-ffmpeg/obs-ffmpeg-nvenc.c

@@ -16,6 +16,9 @@
 ******************************************************************************/
 
 #include <obs-avc.h>
+#ifdef ENABLE_HEVC
+#include <obs-hevc.h>
+#endif
 
 #include "obs-ffmpeg-video-encoders.h"
 
@@ -29,17 +32,28 @@
 
 struct nvenc_encoder {
 	struct ffmpeg_video_encoder ffve;
+#ifdef ENABLE_HEVC
+	bool hevc;
+#endif
 	DARRAY(uint8_t) header;
 	DARRAY(uint8_t) sei;
 };
 
-#define ENCODER_NAME "NVIDIA NVENC H.264"
+#define ENCODER_NAME_H264 "NVIDIA NVENC H.264"
+static const char *h264_nvenc_getname(void *unused)
+{
+	UNUSED_PARAMETER(unused);
+	return ENCODER_NAME_H264;
+}
 
-static const char *nvenc_getname(void *unused)
+#ifdef ENABLE_HEVC
+#define ENCODER_NAME_HEVC "NVIDIA NVENC HEVC"
+static const char *hevc_nvenc_getname(void *unused)
 {
 	UNUSED_PARAMETER(unused);
-	return ENCODER_NAME;
+	return ENCODER_NAME_HEVC;
 }
+#endif
 
 static inline bool valid_format(enum video_format format)
 {
@@ -232,21 +246,46 @@ static void on_first_packet(void *data, AVPacket *pkt, struct darray *da)
 	struct nvenc_encoder *enc = data;
 
 	darray_free(da);
-	obs_extract_avc_headers(pkt->data, pkt->size, (uint8_t **)&da->array,
-				&da->num, &enc->header.array, &enc->header.num,
-				&enc->sei.array, &enc->sei.num);
+#ifdef ENABLE_HEVC
+	if (enc->hevc) {
+		obs_extract_hevc_headers(pkt->data, pkt->size,
+					 (uint8_t **)&da->array, &da->num,
+					 &enc->header.array, &enc->header.num,
+					 &enc->sei.array, &enc->sei.num);
+	} else
+#endif
+	{
+		obs_extract_avc_headers(pkt->data, pkt->size,
+					(uint8_t **)&da->array, &da->num,
+					&enc->header.array, &enc->header.num,
+					&enc->sei.array, &enc->sei.num);
+	}
 	da->capacity = da->num;
 }
 
 static void *nvenc_create_internal(obs_data_t *settings, obs_encoder_t *encoder,
-				   bool psycho_aq)
+				   bool psycho_aq, bool hevc)
 {
 	struct nvenc_encoder *enc = bzalloc(sizeof(*enc));
 
-	if (!ffmpeg_video_encoder_init(&enc->ffve, enc, settings, encoder,
-				       "h264_nvenc", "nvenc_h264", ENCODER_NAME,
-				       on_init_error, on_first_packet))
-		goto fail;
+#ifdef ENABLE_HEVC
+	enc->hevc = hevc;
+	if (hevc) {
+		if (!ffmpeg_video_encoder_init(&enc->ffve, enc, settings,
+					       encoder, "hevc_nvenc",
+					       "nvenc_hevc", ENCODER_NAME_HEVC,
+					       on_init_error, on_first_packet))
+			goto fail;
+	} else
+#endif
+	{
+		if (!ffmpeg_video_encoder_init(&enc->ffve, enc, settings,
+					       encoder, "h264_nvenc",
+					       "nvenc_h264", ENCODER_NAME_H264,
+					       on_init_error, on_first_packet))
+			goto fail;
+	}
+
 	if (!nvenc_update(enc, settings, psycho_aq))
 		goto fail;
 
@@ -257,20 +296,36 @@ fail:
 	return NULL;
 }
 
-static void *nvenc_create(obs_data_t *settings, obs_encoder_t *encoder)
+static void *h264_nvenc_create(obs_data_t *settings, obs_encoder_t *encoder)
 {
 	bool psycho_aq = obs_data_get_bool(settings, "psycho_aq");
-	void *enc = nvenc_create_internal(settings, encoder, psycho_aq);
+	void *enc = nvenc_create_internal(settings, encoder, psycho_aq, false);
 	if ((enc == NULL) && psycho_aq) {
 		blog(LOG_WARNING,
 		     "[NVENC encoder] nvenc_create_internal failed, "
 		     "trying again without Psycho Visual Tuning");
-		enc = nvenc_create_internal(settings, encoder, false);
+		enc = nvenc_create_internal(settings, encoder, false, false);
 	}
 
 	return enc;
 }
 
+#ifdef ENABLE_HEVC
+static void *hevc_nvenc_create(obs_data_t *settings, obs_encoder_t *encoder)
+{
+	bool psycho_aq = obs_data_get_bool(settings, "psycho_aq");
+	void *enc = nvenc_create_internal(settings, encoder, psycho_aq, true);
+	if ((enc == NULL) && psycho_aq) {
+		blog(LOG_WARNING,
+		     "[NVENC encoder] nvenc_create_internal failed, "
+		     "trying again without Psycho Visual Tuning");
+		enc = nvenc_create_internal(settings, encoder, false, true);
+	}
+
+	return enc;
+}
+#endif
+
 static bool nvenc_encode(void *data, struct encoder_frame *frame,
 			 struct encoder_packet *packet, bool *received_packet)
 {
@@ -278,7 +333,7 @@ static bool nvenc_encode(void *data, struct encoder_frame *frame,
 	return ffmpeg_video_encode(&enc->ffve, frame, packet, received_packet);
 }
 
-void nvenc_defaults(obs_data_t *settings)
+void h264_nvenc_defaults(obs_data_t *settings)
 {
 	obs_data_set_default_int(settings, "bitrate", 2500);
 	obs_data_set_default_int(settings, "max_bitrate", 5000);
@@ -293,6 +348,23 @@ void nvenc_defaults(obs_data_t *settings)
 	obs_data_set_default_bool(settings, "repeat_headers", false);
 }
 
+#ifdef ENABLE_HEVC
+void hevc_nvenc_defaults(obs_data_t *settings)
+{
+	obs_data_set_default_int(settings, "bitrate", 2500);
+	obs_data_set_default_int(settings, "max_bitrate", 5000);
+	obs_data_set_default_int(settings, "keyint_sec", 0);
+	obs_data_set_default_int(settings, "cqp", 20);
+	obs_data_set_default_string(settings, "rate_control", "CBR");
+	obs_data_set_default_string(settings, "preset", "hq");
+	obs_data_set_default_string(settings, "profile", "main");
+	obs_data_set_default_bool(settings, "psycho_aq", true);
+	obs_data_set_default_int(settings, "gpu", 0);
+	obs_data_set_default_int(settings, "bf", 2);
+	obs_data_set_default_bool(settings, "repeat_headers", false);
+}
+#endif
+
 static bool rate_control_modified(obs_properties_t *ppts, obs_property_t *p,
 				  obs_data_t *settings)
 {
@@ -320,7 +392,7 @@ static bool rate_control_modified(obs_properties_t *ppts, obs_property_t *p,
 	return true;
 }
 
-obs_properties_t *nvenc_properties_internal(bool ffmpeg)
+obs_properties_t *nvenc_properties_internal(bool hevc, bool ffmpeg)
 {
 	obs_properties_t *props = obs_properties_create();
 	obs_property_t *p;
@@ -374,9 +446,14 @@ obs_properties_t *nvenc_properties_internal(bool ffmpeg)
 				    OBS_COMBO_FORMAT_STRING);
 
 #define add_profile(val) obs_property_list_add_string(p, val, val)
-	add_profile("high");
-	add_profile("main");
-	add_profile("baseline");
+	if (hevc) {
+		add_profile("main10");
+		add_profile("main");
+	} else {
+		add_profile("high");
+		add_profile("main");
+		add_profile("baseline");
+	}
 #undef add_profile
 
 	if (!ffmpeg) {
@@ -402,17 +479,33 @@ obs_properties_t *nvenc_properties_internal(bool ffmpeg)
 	return props;
 }
 
-obs_properties_t *nvenc_properties(void *unused)
+obs_properties_t *h264_nvenc_properties(void *unused)
+{
+	UNUSED_PARAMETER(unused);
+	return nvenc_properties_internal(false, false);
+}
+
+#ifdef ENABLE_HEVC
+obs_properties_t *hevc_nvenc_properties(void *unused)
+{
+	UNUSED_PARAMETER(unused);
+	return nvenc_properties_internal(true, false);
+}
+#endif
+
+obs_properties_t *h264_nvenc_properties_ffmpeg(void *unused)
 {
 	UNUSED_PARAMETER(unused);
-	return nvenc_properties_internal(false);
+	return nvenc_properties_internal(false, true);
 }
 
-obs_properties_t *nvenc_properties_ffmpeg(void *unused)
+#ifdef ENABLE_HEVC
+obs_properties_t *hevc_nvenc_properties_ffmpeg(void *unused)
 {
 	UNUSED_PARAMETER(unused);
-	return nvenc_properties_internal(true);
+	return nvenc_properties_internal(true, true);
 }
+#endif
 
 static bool nvenc_extra_data(void *data, uint8_t **extra_data, size_t *size)
 {
@@ -432,17 +525,39 @@ static bool nvenc_sei_data(void *data, uint8_t **extra_data, size_t *size)
 	return true;
 }
 
-struct obs_encoder_info nvenc_encoder_info = {
+struct obs_encoder_info h264_nvenc_encoder_info = {
 	.id = "ffmpeg_nvenc",
 	.type = OBS_ENCODER_VIDEO,
 	.codec = "h264",
-	.get_name = nvenc_getname,
-	.create = nvenc_create,
+	.get_name = h264_nvenc_getname,
+	.create = h264_nvenc_create,
+	.destroy = nvenc_destroy,
+	.encode = nvenc_encode,
+	.update = nvenc_reconfigure,
+	.get_defaults = h264_nvenc_defaults,
+	.get_properties = h264_nvenc_properties_ffmpeg,
+	.get_extra_data = nvenc_extra_data,
+	.get_sei_data = nvenc_sei_data,
+	.get_video_info = nvenc_video_info,
+#ifdef _WIN32
+	.caps = OBS_ENCODER_CAP_DYN_BITRATE | OBS_ENCODER_CAP_INTERNAL,
+#else
+	.caps = OBS_ENCODER_CAP_DYN_BITRATE,
+#endif
+};
+
+#ifdef ENABLE_HEVC
+struct obs_encoder_info hevc_nvenc_encoder_info = {
+	.id = "ffmpeg_hevc_nvenc",
+	.type = OBS_ENCODER_VIDEO,
+	.codec = "hevc",
+	.get_name = hevc_nvenc_getname,
+	.create = hevc_nvenc_create,
 	.destroy = nvenc_destroy,
 	.encode = nvenc_encode,
 	.update = nvenc_reconfigure,
-	.get_defaults = nvenc_defaults,
-	.get_properties = nvenc_properties_ffmpeg,
+	.get_defaults = hevc_nvenc_defaults,
+	.get_properties = hevc_nvenc_properties_ffmpeg,
 	.get_extra_data = nvenc_extra_data,
 	.get_sei_data = nvenc_sei_data,
 	.get_video_info = nvenc_video_info,
@@ -452,3 +567,4 @@ struct obs_encoder_info nvenc_encoder_info = {
 	.caps = OBS_ENCODER_CAP_DYN_BITRATE,
 #endif
 };
+#endif

plugins/obs-ffmpeg/obs-ffmpeg.c

@@ -27,7 +27,10 @@ extern struct obs_output_info replay_buffer;
 extern struct obs_output_info ffmpeg_hls_muxer;
 extern struct obs_encoder_info aac_encoder_info;
 extern struct obs_encoder_info opus_encoder_info;
-extern struct obs_encoder_info nvenc_encoder_info;
+extern struct obs_encoder_info h264_nvenc_encoder_info;
+#ifdef ENABLE_HEVC
+extern struct obs_encoder_info hevc_nvenc_encoder_info;
+#endif
 extern struct obs_encoder_info svt_av1_encoder_info;
 extern struct obs_encoder_info aom_av1_encoder_info;
 
@@ -223,51 +226,55 @@ static bool nvenc_device_available(void)
 extern bool load_nvenc_lib(void);
 #endif
 
-static bool nvenc_supported(void)
+static bool nvenc_codec_exists(const char *name, const char *fallback)
 {
-#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(58, 9, 100)
-	av_register_all();
-#endif
+	AVCodec *nvenc = avcodec_find_encoder_by_name(name);
+	if (!nvenc)
+		nvenc = avcodec_find_encoder_by_name(fallback);
 
+	return nvenc != NULL;
+}
+
+static bool nvenc_supported(bool *out_h264, bool *out_hevc)
+{
 	profile_start(nvenc_check_name);
 
-	AVCodec *nvenc = avcodec_find_encoder_by_name("nvenc_h264");
-	void *lib = NULL;
-	bool success = false;
+#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(58, 9, 100)
+	av_register_all();
+#endif
 
-	if (!nvenc) {
-		nvenc = avcodec_find_encoder_by_name("h264_nvenc");
-		if (!nvenc)
-			goto cleanup;
-	}
+	const bool h264 = nvenc_codec_exists("h264_nvenc", "nvenc_h264");
+#ifdef ENABLE_HEVC
+	const bool hevc = nvenc_codec_exists("hevc_nvenc", "nvenc_hevc");
+#else
+	const bool hevc = false;
+#endif
 
+	bool success = h264 || hevc;
+	if (success) {
 #if defined(_WIN32)
-	if (!nvenc_device_available()) {
-		goto cleanup;
-	}
-	if (load_nvenc_lib()) {
-		success = true;
-		goto finish;
-	}
+		success = nvenc_device_available() && load_nvenc_lib();
 #elif defined(__linux__)
-	if (!nvenc_device_available()) {
-		goto cleanup;
-	}
-	lib = os_dlopen("libnvidia-encode.so.1");
+		success = nvenc_device_available();
+		if (success) {
+			void *const lib = os_dlopen("libnvidia-encode.so.1");
+			success = lib != NULL;
+			if (success)
+				os_dlclose(lib);
+		}
 #else
-	lib = os_dlopen("libnvidia-encode.so.1");
+		void *const lib = os_dlopen("libnvidia-encode.so.1");
+		success = lib != NULL;
+		if (success)
+			os_dlclose(lib);
 #endif
 
-	/* ------------------------------------------- */
-
-	success = !!lib;
+		if (success) {
+			*out_h264 = h264;
+			*out_hevc = hevc;
+		}
+	}
 
-cleanup:
-	if (lib)
-		os_dlclose(lib);
-#if defined(_WIN32)
-finish:
-#endif
 	profile_end(nvenc_check_name);
 	return success;
 }
@@ -283,7 +290,7 @@ static bool vaapi_supported(void)
 #endif
 
 #ifdef _WIN32
-extern void jim_nvenc_load(void);
+extern void jim_nvenc_load(bool h264, bool hevc);
 extern void jim_nvenc_unload(void);
 #endif
 
@@ -314,19 +321,35 @@ bool obs_module_load(void)
 	register_encoder_if_available(&aom_av1_encoder_info, "libaom-av1");
 	obs_register_encoder(&opus_encoder_info);
 #ifndef __APPLE__
-	if (nvenc_supported()) {
+	bool h264 = false;
+	bool hevc = false;
+	if (nvenc_supported(&h264, &hevc)) {
 		blog(LOG_INFO, "NVENC supported");
 #ifdef _WIN32
 		if (get_win_ver_int() > 0x0601) {
-			jim_nvenc_load();
+			jim_nvenc_load(h264, hevc);
 		} else {
 			// if on Win 7, new nvenc isn't available so there's
 			// no nvenc encoder for the user to select, expose
 			// the old encoder directly
-			nvenc_encoder_info.caps &= ~OBS_ENCODER_CAP_INTERNAL;
+			if (h264) {
+				h264_nvenc_encoder_info.caps &=
+					~OBS_ENCODER_CAP_INTERNAL;
+			}
+#ifdef ENABLE_HEVC
+			if (hevc) {
+				hevc_nvenc_encoder_info.caps &=
+					~OBS_ENCODER_CAP_INTERNAL;
+			}
+#endif
 		}
 #endif
-		obs_register_encoder(&nvenc_encoder_info);
+		if (h264)
+			obs_register_encoder(&h264_nvenc_encoder_info);
+#ifdef ENABLE_HEVC
+		if (hevc)
+			obs_register_encoder(&hevc_nvenc_encoder_info);
+#endif
 	}
 #if !defined(_WIN32) && defined(LIBAVUTIL_VAAPI_AVAILABLE)
 	if (vaapi_supported()) {