winrt: Introduce DirectX pipeline bypass.

Qt Windows Runtime camera uses DirectVideo to convert NV12 format
texture to BGRA format texture. As the EGL Node can draw using NV12
already, allow video render control to choose which path to take. By
default use DirectVideo. Bypass can be used as fallback when
DirectVideo cannot be used or is not working.

Task-Id: QTBUG-48331
Change-Id: I0cb87a7c4523bfb60610e6b41ab3fb05aff092a1
Reviewed-by: Andrew Knight <andrew.knight@intopalo.com>

5 files changed, 154 insertions, 49 deletions

diff --git a/src/plugins/winrt/qwinrtabstractvideorenderercontrol.cpp b/src/plugins/winrt/qwinrtabstractvideorenderercontrol.cpp
index 4a225e9ce..cf62843f7 100644
--- a/src/plugins/winrt/qwinrtabstractvideorenderercontrol.cpp
+++ b/src/plugins/winrt/qwinrtabstractvideorenderercontrol.cpp
@@ -182,6 +182,8 @@ public:
 
     QThread renderThread;
     bool active;
+    QWinRTAbstractVideoRendererControl::BlitMode blitMode;
+    CRITICAL_SECTION mutex;
 };
 
 ID3D11Device *QWinRTAbstractVideoRendererControl::d3dDevice()
@@ -212,6 +214,8 @@ QWinRTAbstractVideoRendererControl::QWinRTAbstractVideoRendererControl(const QSi
     d->eglConfig = 0;
     d->eglSurface = EGL_NO_SURFACE;
     d->active = false;
+    d->blitMode = DirectVideo;
+    InitializeCriticalSectionEx(&d->mutex, 0, 0);
 
     connect(&d->renderThread, &QThread::started,
             this, &QWinRTAbstractVideoRendererControl::syncAndRender,
@@ -220,7 +224,11 @@ QWinRTAbstractVideoRendererControl::QWinRTAbstractVideoRendererControl(const QSi
 
 QWinRTAbstractVideoRendererControl::~QWinRTAbstractVideoRendererControl()
 {
+    Q_D(QWinRTAbstractVideoRendererControl);
+    CriticalSectionLocker locker(&d->mutex);
     shutdown();
+    DeleteCriticalSection(&d->mutex);
+    eglDestroySurface(d->eglDisplay, d->eglSurface);
 }
 
 QAbstractVideoSurface *QWinRTAbstractVideoRendererControl::surface() const
@@ -244,31 +252,45 @@ void QWinRTAbstractVideoRendererControl::syncAndRender()
     forever {
         if (currentThread->isInterruptionRequested())
             break;
+        {
+            CriticalSectionLocker lock(&d->mutex);
+            HRESULT hr;
+            if (d->dirtyState == TextureDirty) {
+                CD3D11_TEXTURE2D_DESC desc(DXGI_FORMAT_B8G8R8A8_UNORM, d->format.frameWidth(), d->format.frameHeight(), 1, 1);
+                desc.BindFlags |= D3D11_BIND_RENDER_TARGET;
+                desc.MiscFlags = D3D11_RESOURCE_MISC_SHARED;
+                hr = g->device->CreateTexture2D(&desc, NULL, d->texture.ReleaseAndGetAddressOf());
+                BREAK_IF_FAILED("Failed to get create video texture");
+                ComPtr<IDXGIResource> resource;
+                hr = d->texture.As(&resource);
+                BREAK_IF_FAILED("Failed to cast texture to resource");
+                hr = resource->GetSharedHandle(&d->shareHandle);
+                BREAK_IF_FAILED("Failed to get texture share handle");
+                d->dirtyState = SurfaceDirty;
+            }
 
-        HRESULT hr;
-        if (d->dirtyState == TextureDirty) {
-            CD3D11_TEXTURE2D_DESC desc(DXGI_FORMAT_B8G8R8A8_UNORM, d->format.frameWidth(), d->format.frameHeight(), 1, 1);
-            desc.BindFlags |= D3D11_BIND_RENDER_TARGET;
-            desc.MiscFlags = D3D11_RESOURCE_MISC_SHARED;
-            hr = g->device->CreateTexture2D(&desc, NULL, d->texture.ReleaseAndGetAddressOf());
-            BREAK_IF_FAILED("Failed to get create video texture");
-            ComPtr<IDXGIResource> resource;
-            hr = d->texture.As(&resource);
-            BREAK_IF_FAILED("Failed to cast texture to resource");
-            hr = resource->GetSharedHandle(&d->shareHandle);
-            BREAK_IF_FAILED("Failed to get texture share handle");
-            d->dirtyState = SurfaceDirty;
-        }
-
-        hr = g->output->WaitForVBlank();
-        CONTINUE_IF_FAILED("Failed to wait for vertical blank");
+            hr = g->output->WaitForVBlank();
+            CONTINUE_IF_FAILED("Failed to wait for vertical blank");
+
+            bool success = false;
+            switch (d->blitMode) {
+            case DirectVideo:
+                success = render(d->texture.Get());
+                break;
+            case MediaFoundation:
+                success = dequeueFrame(&d->presentFrame);
+                break;
+            default:
+                success = false;
+            }
 
-        if (!render(d->texture.Get()))
-            continue;
+            if (!success)
+                continue;
 
-        // Queue to the control's thread for presentation
-        present.invoke(this, Qt::QueuedConnection);
-        currentThread->eventDispatcher()->processEvents(QEventLoop::AllEvents);
+            // Queue to the control's thread for presentation
+            present.invoke(this, Qt::QueuedConnection);
+            currentThread->eventDispatcher()->processEvents(QEventLoop::AllEvents);
+        }
     }
 
     // All done, exit render loop
@@ -326,7 +348,44 @@ void QWinRTAbstractVideoRendererControl::setActive(bool active)
         d->surface->stop();
 }
 
-void QWinRTAbstractVideoRendererControl::present()
+QWinRTAbstractVideoRendererControl::BlitMode QWinRTAbstractVideoRendererControl::blitMode() const
+{
+    Q_D(const QWinRTAbstractVideoRendererControl);
+    return d->blitMode;
+}
+
+void QWinRTAbstractVideoRendererControl::setBlitMode(QWinRTAbstractVideoRendererControl::BlitMode mode)
+{
+    Q_D(QWinRTAbstractVideoRendererControl);
+    CriticalSectionLocker lock(&d->mutex);
+
+    if (d->blitMode == mode)
+        return;
+
+    d->blitMode = mode;
+    d->dirtyState = d->blitMode == MediaFoundation ? NotDirty : TextureDirty;
+
+    if (d->blitMode == DirectVideo)
+        return;
+
+    if (d->texture) {
+        d->texture.Reset();
+        d->shareHandle = 0;
+    }
+
+    if (d->eglSurface) {
+        eglDestroySurface(d->eglDisplay, d->eglSurface);
+        d->eglSurface = EGL_NO_SURFACE;
+    }
+}
+
+bool QWinRTAbstractVideoRendererControl::dequeueFrame(QVideoFrame *frame)
+{
+    Q_UNUSED(frame)
+    return false;
+}
+
+void QWinRTAbstractVideoRendererControl::textureToFrame()
 {
     Q_D(QWinRTAbstractVideoRendererControl);
 
@@ -387,6 +446,13 @@ void QWinRTAbstractVideoRendererControl::present()
 
         d->dirtyState = NotDirty;
     }
+}
+
+void QWinRTAbstractVideoRendererControl::present()
+{
+    Q_D(QWinRTAbstractVideoRendererControl);
+    if (d->blitMode == DirectVideo)
+        textureToFrame();
 
     // Present the frame
     d->surface->present(d->presentFrame);
diff --git a/src/plugins/winrt/qwinrtabstractvideorenderercontrol.h b/src/plugins/winrt/qwinrtabstractvideorenderercontrol.h
index ed8b76a6e..70227c53c 100644
--- a/src/plugins/winrt/qwinrtabstractvideorenderercontrol.h
+++ b/src/plugins/winrt/qwinrtabstractvideorenderercontrol.h
@@ -40,6 +40,8 @@
 #include <QtMultimedia/QVideoRendererControl>
 #include <QtMultimedia/QVideoSurfaceFormat>
 
+#include <qt_windows.h>
+
 struct ID3D11Device;
 struct ID3D11Texture2D;
 
@@ -53,6 +55,11 @@ public:
     explicit QWinRTAbstractVideoRendererControl(const QSize &size, QObject *parent = 0);
     ~QWinRTAbstractVideoRendererControl();
 
+    enum BlitMode {
+        DirectVideo,
+        MediaFoundation
+    };
+
     QAbstractVideoSurface *surface() const Q_DECL_OVERRIDE;
     void setSurface(QAbstractVideoSurface *surface) Q_DECL_OVERRIDE;
 
@@ -63,7 +70,11 @@ public:
 
     void setActive(bool active);
 
+    BlitMode blitMode() const;
+    void setBlitMode(BlitMode mode);
+
     virtual bool render(ID3D11Texture2D *texture) = 0;
+    virtual bool dequeueFrame(QVideoFrame *frame);
 
     static ID3D11Device *d3dDevice();
 
@@ -74,12 +85,29 @@ private slots:
     void syncAndRender();
 
 private:
+    void textureToFrame();
     Q_INVOKABLE void present();
 
     QScopedPointer<QWinRTAbstractVideoRendererControlPrivate> d_ptr;
     Q_DECLARE_PRIVATE(QWinRTAbstractVideoRendererControl)
 };
 
+class CriticalSectionLocker
+{
+public:
+    CriticalSectionLocker(CRITICAL_SECTION *section)
+        : m_section(section)
+    {
+        EnterCriticalSection(m_section);
+    }
+    ~CriticalSectionLocker()
+    {
+        LeaveCriticalSection(m_section);
+    }
+private:
+    CRITICAL_SECTION *m_section;
+};
+
 QT_END_NAMESPACE
 
 #endif // QWINRTABSTRACTVIDEORENDERERCONTROL_H
diff --git a/src/plugins/winrt/qwinrtcameracontrol.cpp b/src/plugins/winrt/qwinrtcameracontrol.cpp
index 6abf3a052..b503007c3 100644
--- a/src/plugins/winrt/qwinrtcameracontrol.cpp
+++ b/src/plugins/winrt/qwinrtcameracontrol.cpp
@@ -198,22 +198,6 @@ private:
     ComPtr<IRegionOfInterest> regionOfInterest;
 };
 
-class CriticalSectionLocker
-{
-public:
-    CriticalSectionLocker(CRITICAL_SECTION *section)
-        : m_section(section)
-    {
-        EnterCriticalSection(m_section);
-    }
-    ~CriticalSectionLocker()
-    {
-        LeaveCriticalSection(m_section);
-    }
-private:
-    CRITICAL_SECTION *m_section;
-};
-
 class MediaStream : public RuntimeClass<RuntimeClassFlags<WinRtClassicComMix>, IMFStreamSink, IMFMediaEventGenerator, IMFMediaTypeHandler>
 {
     enum Flags { NoFlag = 0, BufferLockRequired = 1 };
diff --git a/src/plugins/winrt/qwinrtcameravideorenderercontrol.cpp b/src/plugins/winrt/qwinrtcameravideorenderercontrol.cpp
index 6c5575a17..d42cc5779 100644
--- a/src/plugins/winrt/qwinrtcameravideorenderercontrol.cpp
+++ b/src/plugins/winrt/qwinrtcameravideorenderercontrol.cpp
@@ -201,12 +201,15 @@ public:
     QVideoFrame::PixelFormat cameraSampleformat;
     int cameraSampleSize;
     uint videoProbesCounter;
-    bool getCameraSampleInfo(const ComPtr<IMF2DBuffer> &buffer);
+    bool getCameraSampleInfo(const ComPtr<IMF2DBuffer> &buffer,
+                             QWinRTAbstractVideoRendererControl::BlitMode *mode);
     ComPtr<IMF2DBuffer> dequeueBuffer();
 };
 
-bool QWinRTCameraVideoRendererControlPrivate::getCameraSampleInfo(const ComPtr<IMF2DBuffer> &buffer)
+bool QWinRTCameraVideoRendererControlPrivate::getCameraSampleInfo(const ComPtr<IMF2DBuffer> &buffer,
+                                                                  QWinRTAbstractVideoRendererControl::BlitMode *mode)
 {
+    Q_ASSERT(mode);
     ComPtr<ID3D11Texture2D> sourceTexture;
     ComPtr<IMFDXGIBuffer> dxgiBuffer;
     HRESULT hr = buffer.As(&dxgiBuffer);
@@ -219,6 +222,10 @@ bool QWinRTCameraVideoRendererControlPrivate::getCameraSampleInfo(const ComPtr<I
     }
     D3D11_TEXTURE2D_DESC desc;
     sourceTexture->GetDesc(&desc);
+
+    if (!(desc.MiscFlags & D3D11_RESOURCE_MISC_SHARED))
+        *mode = QWinRTAbstractVideoRendererControl::MediaFoundation;
+
     switch (desc.Format) {
     case DXGI_FORMAT_R8G8B8A8_TYPELESS:
         cameraSampleformat = QVideoFrame::Format_ARGB32;
@@ -281,20 +288,39 @@ bool QWinRTCameraVideoRendererControl::render(ID3D11Texture2D *target)
     return true;
 }
 
+bool QWinRTCameraVideoRendererControl::dequeueFrame(QVideoFrame *frame)
+{
+    Q_ASSERT(frame);
+    Q_D(QWinRTCameraVideoRendererControl);
+
+    ComPtr<IMF2DBuffer> buffer = d->dequeueBuffer();
+    if (!buffer || d->cameraSampleformat == QVideoFrame::Format_Invalid) {
+        emit bufferRequested();
+        return false;
+    }
+
+    QWinRTCameraVideoBuffer *videoBuffer = new QWinRTCameraVideoBuffer(buffer.Get(), d->cameraSampleSize);
+    *frame = QVideoFrame(videoBuffer, size(), d->cameraSampleformat);
+
+    emit bufferRequested();
+    return true;
+}
+
 void QWinRTCameraVideoRendererControl::queueBuffer(IMF2DBuffer *buffer)
 {
     Q_D(QWinRTCameraVideoRendererControl);
     Q_ASSERT(buffer);
 
-    if (d->videoProbesCounter > 0) {
-        if (d->cameraSampleformat == QVideoFrame::Format_User)
-            d->getCameraSampleInfo(buffer);
+    if (d->cameraSampleformat == QVideoFrame::Format_User) {
+        BlitMode mode = blitMode();
+        d->getCameraSampleInfo(buffer, &mode);
+        setBlitMode(mode);
+    }
 
-        if (d->cameraSampleformat != QVideoFrame::Format_Invalid) {
-            QWinRTCameraVideoBuffer *videoBuffer = new QWinRTCameraVideoBuffer(buffer, d->cameraSampleSize);
-            QVideoFrame frame(videoBuffer, size(), d->cameraSampleformat);
-            emit videoFrameProbed(frame);
-        }
+    if (d->videoProbesCounter > 0 && d->cameraSampleformat != QVideoFrame::Format_Invalid) {
+        QWinRTCameraVideoBuffer *videoBuffer = new QWinRTCameraVideoBuffer(buffer, d->cameraSampleSize);
+        QVideoFrame frame(videoBuffer, size(), d->cameraSampleformat);
+        emit videoFrameProbed(frame);
     }
 
     const quint16 writeIndex = (d->writeIndex + 1) % CAMERA_SAMPLE_QUEUE_SIZE;
diff --git a/src/plugins/winrt/qwinrtcameravideorenderercontrol.h b/src/plugins/winrt/qwinrtcameravideorenderercontrol.h
index 122418de3..76bff5e0b 100644
--- a/src/plugins/winrt/qwinrtcameravideorenderercontrol.h
+++ b/src/plugins/winrt/qwinrtcameravideorenderercontrol.h
@@ -56,6 +56,7 @@ public:
     ~QWinRTCameraVideoRendererControl();
 
     bool render(ID3D11Texture2D *texture) Q_DECL_OVERRIDE;
+    bool dequeueFrame(QVideoFrame *frame) Q_DECL_OVERRIDE;
     void queueBuffer(IMF2DBuffer *buffer);
     void discardBuffers();
     void incrementProbe();