decklink: Schedule video frames for playback

DisplayVideoFrameSync is unusable at 4K, over 20 milliseconds per call.
ScheduleVideoFrame is probably what we should have been using all along.

plugins/decklink/DecklinkOutput.cpp

@@ -82,9 +82,9 @@ obs_output_t *DeckLinkOutput::GetOutput(void) const
 	return output;
 }
 
-void DeckLinkOutput::DisplayVideoFrame(video_data *frame)
+void DeckLinkOutput::UpdateVideoFrame(video_data *frame)
 {
-	instance->DisplayVideoFrame(frame);
+	instance->UpdateVideoFrame(frame);
 }
 
 void DeckLinkOutput::WriteAudio(audio_data *frames)

plugins/decklink/DecklinkOutput.hpp

@@ -28,7 +28,7 @@ public:
 	obs_output_t *GetOutput(void) const;
 	bool Activate(DeckLinkDevice *device, long long modeId) override;
 	void Deactivate() override;
-	void DisplayVideoFrame(video_data *pData);
+	void UpdateVideoFrame(video_data *pData);
 	void WriteAudio(audio_data *frames);
 	void SetSize(int width, int height);
 	int GetWidth();

plugins/decklink/decklink-device-instance.cpp

@@ -17,6 +17,50 @@
 #include <caption/caption.h>
 #include <util/bitstream.h>
 
+template<typename T> RenderDelegate<T>::RenderDelegate(T *pOwner)
+{
+	m_pOwner = pOwner;
+}
+
+template<typename T> RenderDelegate<T>::~RenderDelegate() {}
+
+template<typename T>
+HRESULT RenderDelegate<T>::QueryInterface(REFIID, LPVOID *ppv)
+{
+	*ppv = NULL;
+	return E_NOINTERFACE;
+}
+
+template<typename T> ULONG RenderDelegate<T>::AddRef()
+{
+	return ++m_refCount;
+}
+
+template<typename T> ULONG RenderDelegate<T>::Release()
+{
+	const ULONG newRefValue = --m_refCount;
+	if (newRefValue == 0) {
+		delete this;
+		return 0;
+	}
+
+	return newRefValue;
+}
+
+template<typename T>
+HRESULT
+RenderDelegate<T>::ScheduledFrameCompleted(IDeckLinkVideoFrame *completedFrame,
+					   BMDOutputFrameCompletionResult)
+{
+	m_pOwner->ScheduleVideoFrame(completedFrame);
+	return S_OK;
+}
+
+template<typename T> HRESULT RenderDelegate<T>::ScheduledPlaybackHasStopped()
+{
+	return S_OK;
+}
+
 static inline enum video_format ConvertPixelFormat(BMDPixelFormat format)
 {
 	switch (format) {
@@ -528,19 +572,24 @@ bool DeckLinkDeviceInstance::StartOutput(DeckLinkDeviceMode *mode_)
 	if (mode_ == nullptr)
 		return false;
 
+	auto decklinkOutput = dynamic_cast<DeckLinkOutput *>(decklink);
+	if (decklinkOutput == nullptr)
+		return false;
+
 	LOG(LOG_INFO, "Starting output...");
 
-	if (!device->GetOutput(&output))
+	ComPtr<IDeckLinkOutput> output_;
+	if (!device->GetOutput(&output_))
 		return false;
 
-	const HRESULT videoResult = output->EnableVideoOutput(
+	const HRESULT videoResult = output_->EnableVideoOutput(
 		mode_->GetDisplayMode(), bmdVideoOutputFlagDefault);
 	if (videoResult != S_OK) {
 		LOG(LOG_ERROR, "Failed to enable video output");
 		return false;
 	}
 
-	const HRESULT audioResult = output->EnableAudioOutput(
+	const HRESULT audioResult = output_->EnableAudioOutput(
 		bmdAudioSampleRate48kHz, bmdAudioSampleType16bitInteger, 2,
 		bmdAudioOutputStreamTimestamped);
 	if (audioResult != S_OK) {
@@ -548,7 +597,10 @@ bool DeckLinkDeviceInstance::StartOutput(DeckLinkDeviceMode *mode_)
 		return false;
 	}
 
-	mode = mode_;
+	if (!mode_->GetFrameRate(&frameDuration, &frameTimescale)) {
+		LOG(LOG_ERROR, "Failed to get frame rate");
+		return false;
+	}
 
 	ComPtr<IDeckLinkKeyer> deckLinkKeyer;
 	if (device->GetKeyer(&deckLinkKeyer)) {
@@ -561,19 +613,37 @@ bool DeckLinkDeviceInstance::StartOutput(DeckLinkDeviceMode *mode_)
 		}
 	}
 
-	auto decklinkOutput = dynamic_cast<DeckLinkOutput *>(decklink);
-	if (decklinkOutput == nullptr)
-		return false;
+	frameData.clear();
+	size_t i = 0;
+	for (; i < 3; ++i) {
+		ComPtr<IDeckLinkMutableVideoFrame> decklinkOutputFrame;
+		HRESULT result = output_->CreateVideoFrame(
+			decklinkOutput->GetWidth(), decklinkOutput->GetHeight(),
+			decklinkOutput->GetWidth() * 4, bmdFormat8BitBGRA,
+			bmdFrameFlagDefault, &decklinkOutputFrame);
+		if (result != S_OK) {
+			blog(LOG_ERROR, "failed to create video frame 0x%X",
+			     result);
+			return false;
+		}
 
-	HRESULT result;
-	result = output->CreateVideoFrame(
-		decklinkOutput->GetWidth(), decklinkOutput->GetHeight(),
-		decklinkOutput->GetWidth() * 4, bmdFormat8BitBGRA,
-		bmdFrameFlagDefault, &decklinkOutputFrame);
-	if (result != S_OK) {
-		blog(LOG_ERROR, "failed to make frame 0x%X", result);
-		return false;
+		const long size = decklinkOutputFrame->GetRowBytes() *
+				  decklinkOutputFrame->GetHeight();
+		frameData.resize(size);
+		output_->ScheduleVideoFrame(decklinkOutputFrame,
+					    (i * frameDuration), frameDuration,
+					    frameTimescale);
 	}
+	totalFramesScheduled = i;
+
+	*renderDelegate.Assign() =
+		new RenderDelegate<DeckLinkDeviceInstance>(this);
+	output_->SetScheduledFrameCompletionCallback(renderDelegate);
+
+	output_->StartScheduledPlayback(0, 100, 1.0);
+
+	mode = mode_;
+	output = std::move(output_);
 
 	return true;
 }
@@ -586,31 +656,43 @@ bool DeckLinkDeviceInstance::StopOutput()
 	LOG(LOG_INFO, "Stopping output of '%s'...",
 	    GetDevice()->GetDisplayName().c_str());
 
+	output->SetScheduledFrameCompletionCallback(NULL);
 	output->DisableVideoOutput();
 	output->DisableAudioOutput();
-
-	decklinkOutputFrame.Clear();
+	output.Clear();
+	renderDelegate.Clear();
 
 	return true;
 }
 
-void DeckLinkDeviceInstance::DisplayVideoFrame(video_data *frame)
+void DeckLinkDeviceInstance::UpdateVideoFrame(video_data *frame)
 {
 	auto decklinkOutput = dynamic_cast<DeckLinkOutput *>(decklink);
 	if (decklinkOutput == nullptr)
 		return;
 
-	uint8_t *destData;
-	decklinkOutputFrame->GetBytes((void **)&destData);
-
-	uint8_t *outData = frame->data[0];
+	std::lock_guard lock(frameDataMutex);
+	const uint8_t *const outData = frame->data[0];
+	frameData.assign(outData,
+			 outData + decklinkOutput->GetWidth() *
+					   decklinkOutput->GetHeight() * 4);
+}
 
-	std::copy(outData,
-		  outData + (decklinkOutput->GetWidth() *
-			     decklinkOutput->GetHeight() * 4),
-		  destData);
+void DeckLinkDeviceInstance::ScheduleVideoFrame(IDeckLinkVideoFrame *frame)
+{
+	void *bytes;
+	if (SUCCEEDED(frame->GetBytes(&bytes))) {
+		{
+			std::lock_guard lock(frameDataMutex);
+			memcpy(bytes, frameData.data(),
+			       frame->GetRowBytes() * frame->GetHeight());
+		}
 
-	output->DisplayVideoFrameSync(decklinkOutputFrame);
+		output->ScheduleVideoFrame(
+			frame, (totalFramesScheduled * frameDuration),
+			frameDuration, frameTimescale);
+		++totalFramesScheduled;
+	}
 }
 
 void DeckLinkDeviceInstance::WriteAudio(audio_data *frames)

plugins/decklink/decklink-device-instance.hpp

@@ -7,10 +7,37 @@
 #include <media-io/video-scaler.h>
 #include "decklink-device.hpp"
 #include "OBSVideoFrame.h"
+#include <atomic>
+#include <mutex>
+#include <vector>
 
 class AudioRepacker;
 class DecklinkBase;
 
+template<typename T>
+class RenderDelegate : public IDeckLinkVideoOutputCallback {
+private:
+	std::atomic<unsigned long> m_refCount = 1;
+	T *m_pOwner;
+
+	~RenderDelegate();
+
+public:
+	RenderDelegate(T *pOwner);
+
+	// IUnknown
+	virtual HRESULT STDMETHODCALLTYPE QueryInterface(REFIID iid,
+							 LPVOID *ppv);
+	virtual ULONG STDMETHODCALLTYPE AddRef();
+	virtual ULONG STDMETHODCALLTYPE Release();
+
+	// IDeckLinkVideoOutputCallback
+	virtual HRESULT STDMETHODCALLTYPE
+	ScheduledFrameCompleted(IDeckLinkVideoFrame *completedFrame,
+				BMDOutputFrameCompletionResult result);
+	virtual HRESULT STDMETHODCALLTYPE ScheduledPlaybackHasStopped();
+};
+
 class DeckLinkDeviceInstance : public IDeckLinkInputCallback {
 protected:
 	ComPtr<IDeckLinkConfiguration> deckLinkConfiguration;
@@ -39,7 +66,12 @@ protected:
 	bool allow10Bit;
 
 	OBSVideoFrame *convertFrame = nullptr;
-	ComPtr<IDeckLinkMutableVideoFrame> decklinkOutputFrame;
+	std::mutex frameDataMutex;
+	std::vector<uint8_t> frameData;
+	BMDTimeValue frameDuration;
+	BMDTimeScale frameTimescale;
+	size_t totalFramesScheduled;
+	ComPtr<RenderDelegate<DeckLinkDeviceInstance>> renderDelegate;
 
 	void FinalizeStream();
 	void SetupVideoFormat(DeckLinkDeviceMode *mode_);
@@ -107,7 +139,8 @@ public:
 	HRESULT STDMETHODCALLTYPE QueryInterface(REFIID iid, LPVOID *ppv);
 	ULONG STDMETHODCALLTYPE Release(void);
 
-	void DisplayVideoFrame(video_data *frame);
+	void UpdateVideoFrame(video_data *frame);
+	void ScheduleVideoFrame(IDeckLinkVideoFrame *frame);
 	void WriteAudio(audio_data *frames);
 	void HandleCaptionPacket(IDeckLinkAncillaryPacket *packet,
 				 const uint64_t timestamp);

plugins/decklink/decklink-device-mode.cpp

@@ -42,6 +42,15 @@ int DeckLinkDeviceMode::GetHeight()
 	return 0;
 }
 
+bool DeckLinkDeviceMode::GetFrameRate(BMDTimeValue *frameDuration,
+				      BMDTimeScale *timeScale)
+{
+	if (mode != nullptr)
+		return SUCCEEDED(mode->GetFrameRate(frameDuration, timeScale));
+
+	return false;
+}
+
 BMDDisplayModeFlags DeckLinkDeviceMode::GetDisplayModeFlags(void) const
 {
 	if (mode != nullptr)

plugins/decklink/decklink-device-mode.hpp

@@ -27,4 +27,5 @@ public:
 
 	int GetWidth();
 	int GetHeight();
+	bool GetFrameRate(BMDTimeValue *frameDuration, BMDTimeScale *timeScale);
 };

plugins/decklink/decklink-output.cpp

@@ -130,7 +130,7 @@ static void decklink_output_raw_video(void *data, struct video_data *frame)
 	if (!decklink->start_timestamp)
 		decklink->start_timestamp = frame->timestamp;
 
-	decklink->DisplayVideoFrame(frame);
+	decklink->UpdateVideoFrame(frame);
 }
 
 static bool prepare_audio(DeckLinkOutput *decklink,