obs-qsv11: Redo session data releasing for Linux

This reverts the changes to Windows where Release() was called every
time, since we need share a single DX context across multiple encoders.

Instead introduce a ReleaseSessionData() function and some platform
specific session data that will be passed to it. We use this to track
the VA-API display and fd to release them at the right time. Leaking
displays will also lead to cache pollution in the intel-media-driver
crashing users so we cannot do that.

fixes #9611

plugins/obs-qsv11/QSV_Encoder_Internal.cpp

@@ -81,7 +81,8 @@ QSV_Encoder_Internal::QSV_Encoder_Internal(mfxVersion &version, bool isDGPU)
 	  m_nTaskIdx(0),
 	  m_nFirstSyncTask(0),
 	  m_outBitstream(),
-	  m_isDGPU(isDGPU)
+	  m_isDGPU(isDGPU),
+	  m_sessionData(NULL)
 {
 	mfxVariant tempImpl;
 	mfxStatus sts;
@@ -180,16 +181,19 @@ mfxStatus QSV_Encoder_Internal::Open(qsv_param_t *pParams, enum qsv_codec codec)
 	if (m_bUseD3D11)
 		// Use D3D11 surface
 		sts = Initialize(m_ver, &m_session, &m_mfxAllocator,
-				 &g_DX_Handle, false, false, codec);
+				 &g_DX_Handle, false, false, codec,
+				 &m_sessionData);
 	else if (m_bD3D9HACK)
 		// Use hack
 		sts = Initialize(m_ver, &m_session, &m_mfxAllocator,
-				 &g_DX_Handle, false, true, codec);
+				 &g_DX_Handle, false, true, codec,
+				 &m_sessionData);
 	else
 		sts = Initialize(m_ver, &m_session, NULL, NULL, NULL, NULL,
-				 codec);
+				 codec, &m_sessionData);
 #else
-	sts = Initialize(m_ver, &m_session, NULL, NULL, false, false, codec);
+	sts = Initialize(m_ver, &m_session, NULL, NULL, false, false, codec,
+			 &m_sessionData);
 #endif
 
 	MSDK_CHECK_RESULT(sts, MFX_ERR_NONE, sts);
@@ -958,10 +962,12 @@ mfxStatus QSV_Encoder_Internal::ClearData()
 	}
 
 	if ((m_bUseTexAlloc) && (g_numEncodersOpen <= 0)) {
+		Release();
 		g_DX_Handle = NULL;
 	}
-	Release();
 	MFXVideoENCODE_Close(m_session);
+	ReleaseSessionData(m_sessionData);
+	m_sessionData = NULL;
 	return sts;
 }
 

plugins/obs-qsv11/QSV_Encoder_Internal.h

@@ -101,6 +101,7 @@ protected:
 private:
 	mfxVersion m_ver;
 	mfxSession m_session;
+	void *m_sessionData;
 	mfxFrameAllocator m_mfxAllocator;
 	mfxVideoParam m_mfxEncParams;
 	mfxFrameAllocResponse m_mfxResponse;

plugins/obs-qsv11/common_utils.h

@@ -162,10 +162,12 @@ int GetFreeTaskIndex(Task *pTaskPool, mfxU16 nPoolSize);
 mfxStatus Initialize(mfxVersion ver, mfxSession *pSession,
 		     mfxFrameAllocator *pmfxAllocator, mfxHDL *deviceHandle,
 		     bool bCreateSharedHandles, bool dx9hack,
-		     enum qsv_codec codec); //vpl change
+		     enum qsv_codec codec, void **data); //vpl change
 
-// Release resources (device/display)
+// Release global shared resources (device/display)
 void Release();
+// Release per session resources
+void ReleaseSessionData(void *data);
 
 // Convert frame type to string
 char mfxFrameTypeString(mfxU16 FrameType);

plugins/obs-qsv11/common_utils_linux.cpp

@@ -21,7 +21,11 @@
 static const char *default_h264_device = nullptr;
 static const char *default_hevc_device = nullptr;
 static const char *default_av1_device = nullptr;
-static int default_fd = -1;
+
+struct linux_data {
+	int fd;
+	VADisplay vaDisplay;
+};
 
 mfxStatus simple_alloc(mfxHDL pthis, mfxFrameAllocRequest *request,
 		       mfxFrameAllocResponse *response)
@@ -83,7 +87,7 @@ void ClearRGBSurfaceVMem(mfxMemId memId);
 mfxStatus Initialize(mfxVersion ver, mfxSession *pSession,
 		     mfxFrameAllocator *pmfxAllocator, mfxHDL *deviceHandle,
 		     bool bCreateSharedHandles, bool dx9hack,
-		     enum qsv_codec codec)
+		     enum qsv_codec codec, void **data)
 {
 	UNUSED_PARAMETER(ver);
 	UNUSED_PARAMETER(pmfxAllocator);
@@ -113,19 +117,20 @@ mfxStatus Initialize(mfxVersion ver, mfxSession *pSession,
 		cfg, (const mfxU8 *)"mfxImplDescription.AccelerationMode",
 		impl);
 
+	int fd = -1;
 	if (codec == QSV_CODEC_AVC && default_h264_device)
-		default_fd = open(default_h264_device, O_RDWR);
+		fd = open(default_h264_device, O_RDWR);
 	if (codec == QSV_CODEC_HEVC && default_hevc_device)
-		default_fd = open(default_hevc_device, O_RDWR);
+		fd = open(default_hevc_device, O_RDWR);
 	if (codec == QSV_CODEC_AV1 && default_av1_device)
-		default_fd = open(default_av1_device, O_RDWR);
-	if (default_fd < 0) {
+		fd = open(default_av1_device, O_RDWR);
+	if (fd < 0) {
 		blog(LOG_ERROR, "Failed to open device '%s'",
 		     default_h264_device);
 		return MFX_ERR_DEVICE_FAILED;
 	}
 
-	mfxHDL vaDisplay = vaGetDisplayDRM(default_fd);
+	mfxHDL vaDisplay = vaGetDisplayDRM(fd);
 	if (!vaDisplay) {
 		return MFX_ERR_DEVICE_FAILED;
 	}
@@ -134,8 +139,7 @@ mfxStatus Initialize(mfxVersion ver, mfxSession *pSession,
 	if (MFX_ERR_NONE > sts) {
 		blog(LOG_ERROR, "Failed to initialize MFX");
 		MSDK_PRINT_RET_MSG(sts);
-		close(default_fd);
-		default_fd = -1;
+		close(fd);
 		return sts;
 	}
 
@@ -145,24 +149,34 @@ mfxStatus Initialize(mfxVersion ver, mfxSession *pSession,
 	if (vaInitialize(vaDisplay, &major, &minor) != VA_STATUS_SUCCESS) {
 		blog(LOG_ERROR, "Failed to initialize VA-API");
 		vaTerminate(vaDisplay);
-		close(default_fd);
-		default_fd = -1;
+		close(fd);
 		return MFX_ERR_DEVICE_FAILED;
 	}
+
 	sts = MFXVideoCORE_SetHandle(*pSession, MFX_HANDLE_VA_DISPLAY,
 				     vaDisplay);
 	MSDK_CHECK_RESULT(sts, MFX_ERR_NONE, sts);
 
+	struct linux_data *d =
+		(struct linux_data *)bmalloc(sizeof(struct linux_data));
+	d->fd = fd;
+	d->vaDisplay = (VADisplay)vaDisplay;
+	*data = d;
+
 	return sts;
 }
 
-// Release resources (device/display)
-void Release()
-{
-	if (default_fd > 0)
-		close(default_fd);
+void Release() {}
 
-	default_fd = -1;
+// Release per session resources.
+void ReleaseSessionData(void *data)
+{
+	struct linux_data *d = (struct linux_data *)data;
+	if (d) {
+		vaTerminate(d->vaDisplay);
+		close(d->fd);
+		bfree(d);
+	}
 }
 
 void mfxGetTime(mfxTime *timestamp)

plugins/obs-qsv11/common_utils_windows.cpp

@@ -25,9 +25,10 @@
 mfxStatus Initialize(mfxVersion ver, mfxSession *pSession,
 		     mfxFrameAllocator *pmfxAllocator, mfxHDL *deviceHandle,
 		     bool bCreateSharedHandles, bool dx9hack,
-		     enum qsv_codec codec)
+		     enum qsv_codec codec, void **data)
 {
 	UNUSED_PARAMETER(codec);
+	UNUSED_PARAMETER(data);
 
 	mfxStatus sts = MFX_ERR_NONE;
 	mfxVariant impl;
@@ -175,6 +176,8 @@ void Release()
 #endif
 }
 
+void ReleaseSessionData(void *) {}
+
 void mfxGetTime(mfxTime *timestamp)
 {
 	QueryPerformanceCounter(timestamp);