From b8b633a6ccfbf8771b034f526636caf12fb168b3 Mon Sep 17 00:00:00 2001
From: Cameron Gutman <aicommander@gmail.com>
Date: Fri, 12 Apr 2019 22:54:21 -0700
Subject: [PATCH] Use SDL to render on Wayland for VAAPI

---
 .../video/ffmpeg-renderers/renderer.h         |  5 +++
 .../video/ffmpeg-renderers/vaapi.cpp          | 35 +++++++------------
 app/streaming/video/ffmpeg-renderers/vaapi.h  |  6 +---
 app/streaming/video/ffmpeg.cpp                | 13 ++++++-
 4 files changed, 30 insertions(+), 29 deletions(-)

diff --git a/app/streaming/video/ffmpeg-renderers/renderer.h b/app/streaming/video/ffmpeg-renderers/renderer.h
index 9441fffd..297eb357 100644
--- a/app/streaming/video/ffmpeg-renderers/renderer.h
+++ b/app/streaming/video/ffmpeg-renderers/renderer.h
@@ -41,6 +41,11 @@ public:
         return true;
     }
 
+    virtual bool isDirectRenderingSupported() {
+        // The renderer can render directly to the display
+        return true;
+    }
+
     // IOverlayRenderer
     virtual void notifyOverlayUpdated(Overlay::OverlayType) override {
         // Nothing
diff --git a/app/streaming/video/ffmpeg-renderers/vaapi.cpp b/app/streaming/video/ffmpeg-renderers/vaapi.cpp
index 4c21373c..c0efbe6d 100644
--- a/app/streaming/video/ffmpeg-renderers/vaapi.cpp
+++ b/app/streaming/video/ffmpeg-renderers/vaapi.cpp
@@ -76,8 +76,6 @@ VAAPIRenderer::initialize(PDECODER_PARAMETERS params)
     }
     else if (info.subsystem == SDL_SYSWM_WAYLAND) {
 #ifdef HAVE_LIBVA_WAYLAND
-        m_WaylandSurface = info.info.wl.surface;
-        m_WaylandDisplay = info.info.wl.display;
         vaDeviceContext->display = vaGetDisplayWl(info.info.wl.display);
         if (!vaDeviceContext->display) {
             SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
@@ -171,6 +169,14 @@ VAAPIRenderer::needsTestFrame()
     return true;
 }
 
+bool
+VAAPIRenderer::isDirectRenderingSupported()
+{
+    // Many Wayland renderers don't support YUV surfaces, so use
+    // another frontend renderer to draw our frames.
+    return m_WindowSystem == SDL_SYSWM_X11;
+}
+
 void
 VAAPIRenderer::renderFrame(AVFrame* frame)
 {
@@ -201,27 +207,10 @@ VAAPIRenderer::renderFrame(AVFrame* frame)
 #endif
     }
     else if (m_WindowSystem == SDL_SYSWM_WAYLAND) {
-#ifdef HAVE_LIBVA_WAYLAND
-        struct wl_buffer* buffer;
-        VAStatus status;
-
-        status = vaGetSurfaceBufferWl(vaDeviceContext->display,
-                                      surface,
-                                      VA_FRAME_PICTURE,
-                                      &buffer);
-        if (status == VA_STATUS_SUCCESS) {
-            wl_surface_attach(m_WaylandSurface, buffer, 0, 0);
-            wl_surface_damage(m_WaylandSurface, dst.x, dst.y, dst.w, dst.h);
-
-            wl_display_flush(m_WaylandDisplay);
-            wl_surface_commit(m_WaylandSurface);
-        }
-        else {
-            SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
-                         "vaGetSurfaceBufferWl failed(): %d",
-                         status);
-        }
-#endif
+        // We don't support direct rendering on Wayland, so we should
+        // never get called there. Many common Wayland compositors don't
+        // support YUV surfaces, so direct rendering would fail.
+        SDL_assert(false);
     }
     else {
         // We don't accept anything else in initialize().
diff --git a/app/streaming/video/ffmpeg-renderers/vaapi.h b/app/streaming/video/ffmpeg-renderers/vaapi.h
index e46325ab..58a8260b 100644
--- a/app/streaming/video/ffmpeg-renderers/vaapi.h
+++ b/app/streaming/video/ffmpeg-renderers/vaapi.h
@@ -34,6 +34,7 @@ public:
     virtual bool prepareDecoderContext(AVCodecContext* context) override;
     virtual void renderFrame(AVFrame* frame) override;
     virtual bool needsTestFrame() override;
+    virtual bool isDirectRenderingSupported() override;
 
 private:
     int m_WindowSystem;
@@ -43,11 +44,6 @@ private:
     Window m_XWindow;
 #endif
 
-#ifdef HAVE_LIBVA_WAYLAND
-    struct wl_surface* m_WaylandSurface;
-    struct wl_display* m_WaylandDisplay;
-#endif
-
     int m_VideoWidth;
     int m_VideoHeight;
     int m_DisplayWidth;
diff --git a/app/streaming/video/ffmpeg.cpp b/app/streaming/video/ffmpeg.cpp
index ac95e332..576eb0fc 100644
--- a/app/streaming/video/ffmpeg.cpp
+++ b/app/streaming/video/ffmpeg.cpp
@@ -137,7 +137,18 @@ void FFmpegVideoDecoder::reset()
 
 bool FFmpegVideoDecoder::createFrontendRenderer(PDECODER_PARAMETERS params)
 {
-    m_FrontendRenderer = m_BackendRenderer;
+    if (m_BackendRenderer->isDirectRenderingSupported()) {
+        // The backend renderer can render to the display
+        m_FrontendRenderer = m_BackendRenderer;
+    }
+    else {
+        // The backend renderer cannot directly render to the display, so
+        // we will create an SDL renderer to draw the frames.
+        m_FrontendRenderer = new SdlRenderer();
+        if (!m_FrontendRenderer->initialize(params)) {
+            return false;
+        }
+    }
 
     // Determine whether the frontend renderer prefers frame pacing
     auto vsyncConstraint = m_FrontendRenderer->getFramePacingConstraint();