obs-studio/plugins/obs-qsv11/common_directx9.cpp

#include "common_directx9.h"
#include "device_directx9.h"

#include <objbase.h>
#include <initguid.h>
#include <d3d9.h>
#include <map>
#include <atlbase.h>


#define D3DFMT_NV12 (D3DFORMAT)MAKEFOURCC('N','V','1','2')
#define D3DFMT_YV12 (D3DFORMAT)MAKEFOURCC('Y','V','1','2')
#define D3DFMT_P010 (D3DFORMAT)MAKEFOURCC('P','0','1','0')
#define MSDK_SAFE_FREE(X) {if (X) { free(X); X = NULL; }}

std::map<mfxMemId*, mfxHDL>             dx9_allocResponses;
std::map<mfxHDL, mfxFrameAllocResponse> dx9_allocDecodeResponses;
std::map<mfxHDL, int>                   dx9_allocDecodeRefCount;

CComPtr<IDirect3DDeviceManager9> m_manager;
CComPtr<IDirectXVideoDecoderService> m_decoderService;
CComPtr<IDirectXVideoProcessorService> m_processorService;
HANDLE m_hDecoder;
HANDLE m_hProcessor;
DWORD m_surfaceUsage;

CD3D9Device* g_hwdevice;

const struct {
	mfxIMPL impl;       // actual implementation
	mfxU32  adapterID;  // device adapter number
} implTypes[] = {
	{ MFX_IMPL_HARDWARE, 0 },
	{ MFX_IMPL_HARDWARE2, 1 },
	{ MFX_IMPL_HARDWARE3, 2 },
	{ MFX_IMPL_HARDWARE4, 3 }
};

struct mfxAllocatorParams
{
    virtual ~mfxAllocatorParams(){};
};

struct D3DAllocatorParams : mfxAllocatorParams
{
    IDirect3DDeviceManager9 *pManager;
    DWORD surfaceUsage;

    D3DAllocatorParams()
        : pManager()
        , surfaceUsage()
    {
    }
};

mfxStatus DX9_Alloc_Init(D3DAllocatorParams *pParams)
{
    D3DAllocatorParams *pd3dParams = 0;
    pd3dParams = dynamic_cast<D3DAllocatorParams *>(pParams);
    if (!pd3dParams)
        return MFX_ERR_NOT_INITIALIZED;

    m_manager = pd3dParams->pManager;
    m_surfaceUsage = pd3dParams->surfaceUsage;

    return MFX_ERR_NONE;
}

mfxStatus DX9_CreateHWDevice(mfxSession session, mfxHDL* deviceHandle, HWND, bool)
{
	mfxStatus result;

	g_hwdevice = new CD3D9Device;
	mfxU32  adapterNum = 0;
	mfxIMPL impl;

	MFXQueryIMPL(session, &impl);

	mfxIMPL baseImpl = MFX_IMPL_BASETYPE(impl); // Extract Media SDK base implementation type

						    // get corresponding adapter number
	for (mfxU8 i = 0; i < sizeof(implTypes) / sizeof(implTypes[0]); i++) {
		if (implTypes[i].impl == baseImpl) {
			adapterNum = implTypes[i].adapterID;
			break;
		}
	}

	POINT point = { 0, 0 };
	HWND window = WindowFromPoint(point);

	result = g_hwdevice->Init(window, 0, adapterNum);
	if (result != MFX_ERR_NONE) {
		return result;
	}

	g_hwdevice->GetHandle(MFX_HANDLE_D3D9_DEVICE_MANAGER, deviceHandle);


	D3DAllocatorParams dx9_allocParam;
	dx9_allocParam.pManager = reinterpret_cast<IDirect3DDeviceManager9 *>(*deviceHandle);
	DX9_Alloc_Init(&dx9_allocParam);
	return MFX_ERR_NONE;
}

void DX9_CleanupHWDevice()
{
	if (g_hwdevice) {
		// g_hwdevice->Close();
		delete g_hwdevice;
		g_hwdevice = NULL;
	}
	if (m_manager && m_hDecoder) {
		m_manager->CloseDeviceHandle(m_hDecoder);
		m_manager = NULL;
		m_hDecoder = NULL;
	}

	if (m_manager && m_hProcessor) {
		m_manager->CloseDeviceHandle(m_hProcessor);
		m_manager = NULL;
		m_hProcessor = NULL;
	}

	if (m_decoderService) {
		// delete m_decoderService;
		m_decoderService = NULL;
	}

	if (m_processorService) {
		// delete m_processorService;
		m_processorService = NULL;
	}
}

D3DFORMAT ConvertMfxFourccToD3dFormat(mfxU32 fourcc)
{
	switch (fourcc)
	{
	case MFX_FOURCC_NV12:
		return D3DFMT_NV12;
	case MFX_FOURCC_YV12:
		return D3DFMT_YV12;
	case MFX_FOURCC_YUY2:
		return D3DFMT_YUY2;
	case MFX_FOURCC_RGB3:
		return D3DFMT_R8G8B8;
	case MFX_FOURCC_RGB4:
		return D3DFMT_A8R8G8B8;
	case MFX_FOURCC_P8:
		return D3DFMT_P8;
	case MFX_FOURCC_P010:
		return D3DFMT_P010;
	case MFX_FOURCC_A2RGB10:
		return D3DFMT_A2R10G10B10;
	default:
		return D3DFMT_UNKNOWN;
	}
}

mfxStatus dx9_simple_lock(mfxHDL pthis, mfxMemId mid, mfxFrameData* ptr)
{
	pthis; // To suppress warning for this unused parameter

	if (!ptr || !mid)
		return MFX_ERR_NULL_PTR;

	mfxHDLPair *dxmid = (mfxHDLPair*)mid;
	IDirect3DSurface9 *pSurface = static_cast<IDirect3DSurface9*>(dxmid->first);
	if (pSurface == 0)
		return MFX_ERR_INVALID_HANDLE;

	D3DSURFACE_DESC desc;
	HRESULT hr = pSurface->GetDesc(&desc);
	if (FAILED(hr))
		return MFX_ERR_LOCK_MEMORY;

	if (desc.Format != D3DFMT_NV12 &&
		desc.Format != D3DFMT_YV12 &&
		desc.Format != D3DFMT_YUY2 &&
		desc.Format != D3DFMT_R8G8B8 &&
		desc.Format != D3DFMT_A8R8G8B8 &&
		desc.Format != D3DFMT_P8 &&
		desc.Format != D3DFMT_P010 &&
		desc.Format != D3DFMT_A2R10G10B10)
		return MFX_ERR_LOCK_MEMORY;

	D3DLOCKED_RECT locked;

	hr = pSurface->LockRect(&locked, 0, D3DLOCK_NOSYSLOCK);
	if (FAILED(hr))
		return MFX_ERR_LOCK_MEMORY;

	switch ((DWORD)desc.Format)
	{
	case D3DFMT_NV12:
		ptr->Pitch = (mfxU16)locked.Pitch;
		ptr->Y = (mfxU8 *)locked.pBits;
		ptr->U = (mfxU8 *)locked.pBits + desc.Height * locked.Pitch;
		ptr->V = ptr->U + 1;
		break;
	case D3DFMT_YV12:
		ptr->Pitch = (mfxU16)locked.Pitch;
		ptr->Y = (mfxU8 *)locked.pBits;
		ptr->V = ptr->Y + desc.Height * locked.Pitch;
		ptr->U = ptr->V + (desc.Height * locked.Pitch) / 4;
		break;
	case D3DFMT_YUY2:
		ptr->Pitch = (mfxU16)locked.Pitch;
		ptr->Y = (mfxU8 *)locked.pBits;
		ptr->U = ptr->Y + 1;
		ptr->V = ptr->Y + 3;
		break;
	case D3DFMT_R8G8B8:
		ptr->Pitch = (mfxU16)locked.Pitch;
		ptr->B = (mfxU8 *)locked.pBits;
		ptr->G = ptr->B + 1;
		ptr->R = ptr->B + 2;
		break;
	case D3DFMT_A8R8G8B8:
	case D3DFMT_A2R10G10B10:
		ptr->Pitch = (mfxU16)locked.Pitch;
		ptr->B = (mfxU8 *)locked.pBits;
		ptr->G = ptr->B + 1;
		ptr->R = ptr->B + 2;
		ptr->A = ptr->B + 3;
		break;
	case D3DFMT_P8:
		ptr->Pitch = (mfxU16)locked.Pitch;
		ptr->Y = (mfxU8 *)locked.pBits;
		ptr->U = 0;
		ptr->V = 0;
		break;
	case D3DFMT_P010:
		ptr->PitchHigh = (mfxU16)(locked.Pitch / (1 << 16));
		ptr->PitchLow = (mfxU16)(locked.Pitch % (1 << 16));
		ptr->Y = (mfxU8 *)locked.pBits;
		ptr->U = (mfxU8 *)locked.pBits + desc.Height * locked.Pitch;
		ptr->V = ptr->U + 1;
		break;
	}

	return MFX_ERR_NONE;
}

mfxStatus dx9_simple_unlock(mfxHDL, mfxMemId mid, mfxFrameData* ptr)
{
	if (!mid)
		return MFX_ERR_NULL_PTR;

	mfxHDLPair *dxmid = (mfxHDLPair*)mid;
	IDirect3DSurface9 *pSurface = static_cast<IDirect3DSurface9*>(dxmid->first);
	if (pSurface == 0)
		return MFX_ERR_INVALID_HANDLE;

	pSurface->UnlockRect();

	if (NULL != ptr)
	{
		ptr->Pitch = 0;
		ptr->Y = 0;
		ptr->U = 0;
		ptr->V = 0;
	}

	return MFX_ERR_NONE;
}

mfxStatus dx9_simple_gethdl(mfxHDL, mfxMemId mid, mfxHDL* handle)
{
	if (!mid || !handle)
		return MFX_ERR_NULL_PTR;

	mfxHDLPair *dxMid = (mfxHDLPair*)mid;
	*handle = dxMid->first;
	return MFX_ERR_NONE;
}

mfxStatus _dx9_simple_free(mfxFrameAllocResponse* response)
{
	if (!response)
		return MFX_ERR_NULL_PTR;

	mfxStatus sts = MFX_ERR_NONE;

	if (response->mids) {
		for (mfxU32 i = 0; i < response->NumFrameActual; i++) {
			if (response->mids[i]) {
				mfxHDLPair *dxMids = (mfxHDLPair*)response->mids[i];
				static_cast<IDirect3DSurface9*>(dxMids->first)->Release();
			}
		}
		MSDK_SAFE_FREE(response->mids[0]);
	}
	MSDK_SAFE_FREE(response->mids);

	return sts;
}

mfxStatus dx9_simple_free(mfxHDL pthis, mfxFrameAllocResponse* response)
{
    if (NULL == response)
        return MFX_ERR_NULL_PTR;

    if (dx9_allocResponses.find(response->mids) == dx9_allocResponses.end()) {
        // Decode free response handling
        if (--dx9_allocDecodeRefCount[pthis] == 0) {
            _dx9_simple_free(response);
            dx9_allocDecodeResponses.erase(pthis);
            dx9_allocDecodeRefCount.erase(pthis);
        }
    } else {
        // Encode and VPP free response handling
        dx9_allocResponses.erase(response->mids);
        _dx9_simple_free(response);
    }

    return MFX_ERR_NONE;
}

mfxStatus _dx9_simple_alloc(mfxFrameAllocRequest* request, mfxFrameAllocResponse* response)
{
	HRESULT hr;

	MSDK_CHECK_POINTER(request, MFX_ERR_NULL_PTR);
	if (request->NumFrameSuggested == 0)
		return MFX_ERR_UNKNOWN;

	D3DFORMAT format = ConvertMfxFourccToD3dFormat(request->Info.FourCC);

	if (format == D3DFMT_UNKNOWN)
		return MFX_ERR_UNSUPPORTED;

	DWORD   target;

	if (MFX_MEMTYPE_DXVA2_DECODER_TARGET & request->Type)
	{
		target = DXVA2_VideoDecoderRenderTarget;
	}
	else if (MFX_MEMTYPE_DXVA2_PROCESSOR_TARGET & request->Type)
	{
		target = DXVA2_VideoProcessorRenderTarget;
	}
	else
		return MFX_ERR_UNSUPPORTED;

	IDirectXVideoAccelerationService* videoService = NULL;

	if (target == DXVA2_VideoProcessorRenderTarget) {
		if (!m_hProcessor) {
			hr = m_manager->OpenDeviceHandle(&m_hProcessor);
			if (FAILED(hr))
				return MFX_ERR_MEMORY_ALLOC;

			hr = m_manager->GetVideoService(m_hProcessor, IID_IDirectXVideoProcessorService, (void**)&m_processorService);
			if (FAILED(hr))
				return MFX_ERR_MEMORY_ALLOC;
		}
		videoService = m_processorService;
	}
	else {
		if (!m_hDecoder)
		{
			hr = m_manager->OpenDeviceHandle(&m_hDecoder);
			if (FAILED(hr))
				return MFX_ERR_MEMORY_ALLOC;

			hr = m_manager->GetVideoService(m_hDecoder, IID_IDirectXVideoDecoderService, (void**)&m_decoderService);
			if (FAILED(hr))
				return MFX_ERR_MEMORY_ALLOC;
		}
		videoService = m_decoderService;
	}

	mfxHDLPair *dxMids = NULL, **dxMidPtrs = NULL;
	dxMids = (mfxHDLPair*)calloc(request->NumFrameSuggested, sizeof(mfxHDLPair));
	dxMidPtrs = (mfxHDLPair**)calloc(request->NumFrameSuggested, sizeof(mfxHDLPair*));

	if (!dxMids || !dxMidPtrs) {
		MSDK_SAFE_FREE(dxMids);
		MSDK_SAFE_FREE(dxMidPtrs);
		return MFX_ERR_MEMORY_ALLOC;
	}

	response->mids = (mfxMemId*)dxMidPtrs;
	response->NumFrameActual = request->NumFrameSuggested;

	if (request->Type & MFX_MEMTYPE_EXTERNAL_FRAME) {
		for (int i = 0; i < request->NumFrameSuggested; i++) {
			hr = videoService->CreateSurface(request->Info.Width, request->Info.Height, 0, format,
				D3DPOOL_DEFAULT, m_surfaceUsage, target, (IDirect3DSurface9**)&dxMids[i].first, &dxMids[i].second);
			if (FAILED(hr)) {
				_dx9_simple_free(response);
				MSDK_SAFE_FREE(dxMids);
				return MFX_ERR_MEMORY_ALLOC;
			}
			dxMidPtrs[i] = &dxMids[i];
		}
	}
	else {
		safe_array<IDirect3DSurface9*> dxSrf(new IDirect3DSurface9*[request->NumFrameSuggested]);
		if (!dxSrf.get())
		{
			MSDK_SAFE_FREE(dxMids);
			return MFX_ERR_MEMORY_ALLOC;
		}
		hr = videoService->CreateSurface(request->Info.Width, request->Info.Height, request->NumFrameSuggested - 1, format,
			D3DPOOL_DEFAULT, m_surfaceUsage, target, dxSrf.get(), NULL);
		if (FAILED(hr))
		{
			MSDK_SAFE_FREE(dxMids);
			return MFX_ERR_MEMORY_ALLOC;
		}


		for (int i = 0; i < request->NumFrameSuggested; i++) {
			dxMids[i].first = dxSrf.get()[i];
			dxMidPtrs[i] = &dxMids[i];
		}
	}
	return MFX_ERR_NONE;
}

mfxStatus dx9_simple_alloc(mfxHDL pthis, mfxFrameAllocRequest* request, mfxFrameAllocResponse* response)
{
    mfxStatus sts = MFX_ERR_NONE;

    if (request->Type & MFX_MEMTYPE_SYSTEM_MEMORY)
        return MFX_ERR_UNSUPPORTED;

    if (dx9_allocDecodeResponses.find(pthis) != dx9_allocDecodeResponses.end() &&
        MFX_MEMTYPE_EXTERNAL_FRAME & request->Type &&
        MFX_MEMTYPE_FROM_DECODE & request->Type) {
        // Memory for this request was already allocated during manual allocation stage. Return saved response
        //   When decode acceleration device (DXVA) is created it requires a list of d3d surfaces to be passed.
        //   Therefore Media SDK will ask for the surface info/mids again at Init() stage, thus requiring us to return the saved response
        //   (No such restriction applies to Encode or VPP)
        *response = dx9_allocDecodeResponses[pthis];
        dx9_allocDecodeRefCount[pthis]++;
    } else {
        sts = _dx9_simple_alloc(request, response);

        if (MFX_ERR_NONE == sts) {
            if ( MFX_MEMTYPE_EXTERNAL_FRAME & request->Type &&
                 MFX_MEMTYPE_FROM_DECODE & request->Type) {
                // Decode alloc response handling
                dx9_allocDecodeResponses[pthis] = *response;
                dx9_allocDecodeRefCount[pthis]++;
            } else {
                // Encode and VPP alloc response handling
                dx9_allocResponses[response->mids] = pthis;
            }
        }
    }

    return sts;
}