From 6d3d51553b2b66fff58134e8d29a917cedc60d10 Mon Sep 17 00:00:00 2001
From: Cameron Gutman <aicommander@gmail.com>
Date: Thu, 7 Apr 2022 21:46:48 -0500
Subject: [PATCH] Wait on our D3D11 swapchain before rendering to reduce
 latency

---
 .../video/ffmpeg-renderers/d3d11va.cpp        | 36 +++++++++----------
 .../video/ffmpeg-renderers/d3d11va.h          |  1 +
 .../video/ffmpeg-renderers/pacer/pacer.cpp    |  3 ++
 .../video/ffmpeg-renderers/renderer.h         |  7 ++++
 4 files changed, 27 insertions(+), 20 deletions(-)

diff --git a/app/streaming/video/ffmpeg-renderers/d3d11va.cpp b/app/streaming/video/ffmpeg-renderers/d3d11va.cpp
index 8df61477..84e39da3 100644
--- a/app/streaming/video/ffmpeg-renderers/d3d11va.cpp
+++ b/app/streaming/video/ffmpeg-renderers/d3d11va.cpp
@@ -464,10 +464,6 @@ bool D3D11VARenderer::initialize(PDECODER_PARAMETERS params)
 
         m_FrameWaitableObject = m_SwapChain->GetFrameLatencyWaitableObject();
         SDL_assert(m_FrameWaitableObject != nullptr);
-
-        // Wait for the swap chain to be ready. This is required because we don't
-        // we're waiting after presenting in the general case, not before.
-        WaitForSingleObjectEx(m_FrameWaitableObject, 1000, FALSE);
     }
     else {
         IDXGIDevice1* dxgiDevice;
@@ -582,6 +578,22 @@ void D3D11VARenderer::setHdrMode(bool enabled)
     unlockContext(this);
 }
 
+void D3D11VARenderer::waitToRender()
+{
+    if (m_FrameWaitableObject != nullptr) {
+        SDL_assert(m_Windowed);
+        SDL_assert(m_DecoderParams.enableVsync);
+
+        // Wait for the pipeline to be ready for the next frame in V-Sync mode.
+        //
+        // This callback happens before selecting the next frame to render, so
+        // we can wait for the previous frame to finish prior to picking the
+        // next one to display. This reduces the effective display latency
+        // by ensuring we always render the most recent frame immediately.
+        WaitForSingleObjectEx(m_FrameWaitableObject, 500, FALSE);
+    }
+}
+
 void D3D11VARenderer::renderFrame(AVFrame* frame)
 {
     // Acquire the context lock for rendering to prevent concurrent
@@ -669,22 +681,6 @@ void D3D11VARenderer::renderFrame(AVFrame* frame)
         SDL_PushEvent(&event);
         return;
     }
-
-    if (m_FrameWaitableObject != nullptr) {
-        SDL_assert(m_Windowed);
-        SDL_assert(m_DecoderParams.enableVsync);
-
-        // Wait for the pipeline to be ready for the next frame in V-Sync mode.
-        //
-        // MSDN advises us to wait *before* doing any rendering operations,
-        // however that assumes the a typical game which will latch inputs,
-        // run the engine, draw, etc. after WaitForSingleObjectEx(). In our case,
-        // we actually want wait *after* our rendering operations, because our AVFrame
-        // is already set in stone by the time we enter this function. Waiting after
-        // presenting allows a more recent frame to be received before renderFrame()
-        // is called again.
-        WaitForSingleObjectEx(m_FrameWaitableObject, 1000, FALSE);
-    }
 }
 
 void D3D11VARenderer::renderOverlay(Overlay::OverlayType type)
diff --git a/app/streaming/video/ffmpeg-renderers/d3d11va.h b/app/streaming/video/ffmpeg-renderers/d3d11va.h
index 0c746672..bfc5b569 100644
--- a/app/streaming/video/ffmpeg-renderers/d3d11va.h
+++ b/app/streaming/video/ffmpeg-renderers/d3d11va.h
@@ -19,6 +19,7 @@ public:
     virtual bool prepareDecoderContext(AVCodecContext* context, AVDictionary**) override;
     virtual bool prepareDecoderContextInGetFormat(AVCodecContext* context, AVPixelFormat pixelFormat) override;
     virtual void renderFrame(AVFrame* frame) override;
+    virtual void waitToRender() override;
     virtual void notifyOverlayUpdated(Overlay::OverlayType) override;
     virtual void setHdrMode(bool enabled) override;
     virtual int getRendererAttributes() override;
diff --git a/app/streaming/video/ffmpeg-renderers/pacer/pacer.cpp b/app/streaming/video/ffmpeg-renderers/pacer/pacer.cpp
index d216bb6f..52c8ce87 100644
--- a/app/streaming/video/ffmpeg-renderers/pacer/pacer.cpp
+++ b/app/streaming/video/ffmpeg-renderers/pacer/pacer.cpp
@@ -91,6 +91,9 @@ int Pacer::renderThread(void* context)
     }
 
     while (!me->m_Stopping) {
+        // Wait for the renderer to be ready for the next frame
+        me->m_VsyncRenderer->waitToRender();
+
         // Acquire the frame queue lock to protect the queue and
         // the not empty condition
         me->m_FrameQueueLock.lock();
diff --git a/app/streaming/video/ffmpeg-renderers/renderer.h b/app/streaming/video/ffmpeg-renderers/renderer.h
index 3ec93b7e..7cf9cc04 100644
--- a/app/streaming/video/ffmpeg-renderers/renderer.h
+++ b/app/streaming/video/ffmpeg-renderers/renderer.h
@@ -102,6 +102,13 @@ public:
     virtual bool prepareDecoderContext(AVCodecContext* context, AVDictionary** options) = 0;
     virtual void renderFrame(AVFrame* frame) = 0;
 
+    // Called for threaded renderers to allow them to wait prior to us latching
+    // the next frame for rendering (as opposed to waiting on buffer swap with
+    // an older frame already queued for display).
+    virtual void waitToRender() {
+        // Don't wait by default
+    }
+
     // Called on the same thread as renderFrame() during destruction of the renderer
     virtual void cleanupRenderContext() {
         // Nothing