Fix setting pixel format to AVFVideoSink

- When lazy loading the video, the output settings dictionary was invalid.
Properly initialize and release m_outputSettings to fix.

- Unify and refactor code to set the output settings and pixel format

- Set the pixel format only once rhi is set on the sink so we know the
rhi backend. This is because the openGL backend accepts only RGBA pixel
format.

- Fix the incorrect assigning of a QVideoFrameFormat::PixelFormat to
avPixelFormat when the rhi was openGL.

Fixes: QTBUG-110868
Fixes: QTBUG-110812
Change-Id: I98de219bcc94dad04c4db9ea22aeb852dc2a193b
Reviewed-by: Artem Dyomin <artem.dyomin@qt.io>
Reviewed-by: Lars Knoll <lars@knoll.priv.no>
(cherry picked from commit 2e43d29e1d5d50b44b8f6d4f000968e3933c279a)
Reviewed-by: Qt Cherry-pick Bot <cherrypick_bot@qt-project.org>

11 files changed, 175 insertions, 45 deletions

diff --git a/src/plugins/multimedia/darwin/avfvideosink.mm b/src/plugins/multimedia/darwin/avfvideosink.mm
index 0f321c3d3..1fa316b98 100644
--- a/src/plugins/multimedia/darwin/avfvideosink.mm
+++ b/src/plugins/multimedia/darwin/avfvideosink.mm
@@ -55,18 +55,12 @@ void AVFVideoSink::setVideoSinkInterface(AVFVideoSinkInterface *interface)
         m_interface->setRhi(m_rhi);
 }
 
-// The OpengGL texture cache can apparently only handle single plane formats, so lets simply restrict to BGRA
-static NSDictionary* const AVF_OUTPUT_SETTINGS_OPENGL = @{
-        (NSString *)kCVPixelBufferPixelFormatTypeKey: @(kCVPixelFormatType_32BGRA)
-#ifndef Q_OS_IOS // On iOS this key generates a warning about unsupported key.
-        , (NSString *)kCVPixelBufferOpenGLCompatibilityKey: @true
-#endif // Q_OS_IOS
-};
-
 AVFVideoSinkInterface::~AVFVideoSinkInterface()
 {
     if (m_layer)
         [m_layer release];
+    if (m_outputSettings)
+        [m_outputSettings release];
     freeTextureCaches();
 }
 
@@ -124,8 +118,6 @@ void AVFVideoSinkInterface::setRhi(QRhi *rhi)
         }
     } else if (rhi->backend() == QRhi::OpenGLES2) {
 #if QT_CONFIG(opengl)
-        setOutputSettings(AVF_OUTPUT_SETTINGS_OPENGL);
-
 #ifdef Q_OS_MACOS
         const auto *gl = static_cast<const QRhiGles2NativeHandles *>(rhi->nativeHandles());
 
@@ -160,6 +152,7 @@ void AVFVideoSinkInterface::setRhi(QRhi *rhi)
         m_rhi = nullptr;
 #endif // QT_CONFIG(opengl)
     }
+    setOutputSettings();
 }
 
 void AVFVideoSinkInterface::setLayer(CALayer *layer)
@@ -177,9 +170,48 @@ void AVFVideoSinkInterface::setLayer(CALayer *layer)
     reconfigure();
 }
 
-void AVFVideoSinkInterface::setOutputSettings(NSDictionary *settings)
+void AVFVideoSinkInterface::setOutputSettings()
 {
-    m_outputSettings = settings;
+    if (!m_rhi)
+        return;
+
+    if (m_outputSettings)
+        [m_outputSettings release];
+    m_outputSettings = nil;
+
+    // Set pixel format
+    NSDictionary *dictionary = nil;
+    if (m_rhi->backend() == QRhi::Metal) {
+        dictionary = @{(NSString *)kCVPixelBufferPixelFormatTypeKey:
+        @[
+            @(kCVPixelFormatType_32BGRA),
+            @(kCVPixelFormatType_32RGBA),
+            @(kCVPixelFormatType_422YpCbCr8),
+            @(kCVPixelFormatType_422YpCbCr8_yuvs),
+            @(kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange),
+            @(kCVPixelFormatType_420YpCbCr8BiPlanarFullRange),
+            @(kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange),
+            @(kCVPixelFormatType_420YpCbCr10BiPlanarFullRange),
+            @(kCVPixelFormatType_OneComponent8),
+            @(kCVPixelFormatType_OneComponent16),
+            @(kCVPixelFormatType_420YpCbCr8Planar),
+            @(kCVPixelFormatType_420YpCbCr8PlanarFullRange)
+        ]
+#ifndef Q_OS_IOS // This key is not supported and generates a warning.
+        , (NSString *)kCVPixelBufferMetalCompatibilityKey: @true
+#endif // Q_OS_IOS
+        };
+    } else if (m_rhi->backend() == QRhi::OpenGLES2) {
+#if QT_CONFIG(opengl)
+        dictionary = @{(NSString *)kCVPixelBufferPixelFormatTypeKey:
+            @(kCVPixelFormatType_32BGRA)
+#ifndef Q_OS_IOS // On iOS this key generates a warning about unsupported key.
+            , (NSString *)kCVPixelBufferOpenGLCompatibilityKey: @true
+#endif // Q_OS_IOS
+        };
+#endif
+    }
+    m_outputSettings = [[NSDictionary alloc] initWithDictionary:dictionary];
 }
 
 void AVFVideoSinkInterface::updateLayerBounds()
diff --git a/src/plugins/multimedia/darwin/avfvideosink_p.h b/src/plugins/multimedia/darwin/avfvideosink_p.h
index bc91c2265..9b66e79f2 100644
--- a/src/plugins/multimedia/darwin/avfvideosink_p.h
+++ b/src/plugins/multimedia/darwin/avfvideosink_p.h
@@ -65,7 +65,7 @@ public:
     virtual void reconfigure() = 0;
     virtual void setRhi(QRhi *);
     virtual void setLayer(CALayer *layer);
-    virtual void setOutputSettings(NSDictionary *settings);
+    virtual void setOutputSettings();
 
     QMutex *textureCacheMutex() { return &m_textureCacheMutex; }
 
diff --git a/src/plugins/multimedia/darwin/camera/avfcamerarenderer.mm b/src/plugins/multimedia/darwin/camera/avfcamerarenderer.mm
index 35b339ab9..3a047f573 100644
--- a/src/plugins/multimedia/darwin/camera/avfcamerarenderer.mm
+++ b/src/plugins/multimedia/darwin/camera/avfcamerarenderer.mm
@@ -108,13 +108,25 @@ void AVFCameraRenderer::reconfigure()
     deviceOrientationChanged();
 }
 
-void AVFCameraRenderer::setOutputSettings(NSDictionary *settings)
+void AVFCameraRenderer::setOutputSettings()
 {
     if (!m_videoDataOutput)
         return;
 
-    m_videoDataOutput.videoSettings = settings;
-    AVFVideoSinkInterface::setOutputSettings(settings);
+    const auto cameraPixelFormat = m_cameraSession ? m_cameraSession->cameraFormat().pixelFormat()
+                                                   : QVideoFrameFormat::Format_Invalid;
+    if (cameraPixelFormat != QVideoFrameFormat::Format_Invalid)
+        setPixelFormat(cameraPixelFormat);
+
+    // If no output settings set from above,
+    // it's most likely because the rhi is OpenGL
+    // and the pixel format is not BGRA.
+    // We force this in the base class implementation
+    if (!m_outputSettings)
+        AVFVideoSinkInterface::setOutputSettings();
+
+    if (m_outputSettings)
+        m_videoDataOutput.videoSettings = m_outputSettings;
 }
 
 void AVFCameraRenderer::configureAVCaptureSession(AVFCameraSession *cameraSession)
@@ -267,7 +279,7 @@ void AVFCameraRenderer::setPixelFormat(const QVideoFrameFormat::PixelFormat pixe
             , (NSString *)kCVPixelBufferMetalCompatibilityKey: @true
 #endif // Q_OS_IOS
         };
-        setOutputSettings(outputSettings);
+        m_outputSettings = outputSettings;
     } else {
         qWarning() << "QCamera::setCameraFormat: requested pixel format not supported. Did you use a camera format from another camera?";
     }
diff --git a/src/plugins/multimedia/darwin/camera/avfcamerarenderer_p.h b/src/plugins/multimedia/darwin/camera/avfcamerarenderer_p.h
index 0a43b78bb..6e42f52ec 100644
--- a/src/plugins/multimedia/darwin/camera/avfcamerarenderer_p.h
+++ b/src/plugins/multimedia/darwin/camera/avfcamerarenderer_p.h
@@ -49,7 +49,7 @@ public:
     ~AVFCameraRenderer();
 
     void reconfigure() override;
-    void setOutputSettings(NSDictionary *settings) override;
+    void setOutputSettings() override;
 
     void configureAVCaptureSession(AVFCameraSession *cameraSession);
     void syncHandleViewfinderFrame(const QVideoFrame &frame);
diff --git a/src/plugins/multimedia/darwin/camera/avfcamerasession.mm b/src/plugins/multimedia/darwin/camera/avfcamerasession.mm
index 936a69ae5..0939c77d4 100644
--- a/src/plugins/multimedia/darwin/camera/avfcamerasession.mm
+++ b/src/plugins/multimedia/darwin/camera/avfcamerasession.mm
@@ -158,6 +158,11 @@ void AVFCameraSession::setCameraFormat(const QCameraFormat &format)
     updateCameraFormat(format);
 }
 
+QCameraFormat AVFCameraSession::cameraFormat() const
+{
+    return m_cameraFormat;
+}
+
 void AVFCameraSession::updateCameraFormat(const QCameraFormat &format)
 {
     m_cameraFormat = format;
@@ -167,11 +172,8 @@ void AVFCameraSession::updateCameraFormat(const QCameraFormat &format)
         return;
 
     AVCaptureDeviceFormat *newFormat = qt_convert_to_capture_device_format(captureDevice, format);
-    if (newFormat) {
+    if (newFormat)
         qt_set_active_format(captureDevice, newFormat, false);
-        if (m_videoOutput)
-            m_videoOutput->setPixelFormat(format.pixelFormat());
-    }
 }
 
 void AVFCameraSession::setVideoOutput(AVFCameraRenderer *output)
diff --git a/src/plugins/multimedia/darwin/camera/avfcamerasession_p.h b/src/plugins/multimedia/darwin/camera/avfcamerasession_p.h
index cfc048211..931943153 100644
--- a/src/plugins/multimedia/darwin/camera/avfcamerasession_p.h
+++ b/src/plugins/multimedia/darwin/camera/avfcamerasession_p.h
@@ -44,6 +44,7 @@ public:
     void setActiveCamera(const QCameraDevice &info);
 
     void setCameraFormat(const QCameraFormat &format);
+    QCameraFormat cameraFormat() const;
 
     AVFCameraRenderer *videoOutput() const { return m_videoOutput; }
     AVCaptureAudioDataOutput *audioOutput() const { return m_audioOutput; }
diff --git a/src/plugins/multimedia/darwin/mediaplayer/avfvideorenderercontrol.mm b/src/plugins/multimedia/darwin/mediaplayer/avfvideorenderercontrol.mm
index 3bc115a17..0ebffa30c 100644
--- a/src/plugins/multimedia/darwin/mediaplayer/avfvideorenderercontrol.mm
+++ b/src/plugins/multimedia/darwin/mediaplayer/avfvideorenderercontrol.mm
@@ -158,26 +158,6 @@ void AVFVideoRendererControl::updateVideoFrame(const CVTimeStamp &ts)
     m_sink->setVideoFrame(frame);
 }
 
-static NSDictionary* const AVF_OUTPUT_SETTINGS = @{
-        (NSString *)kCVPixelBufferPixelFormatTypeKey: @[
-            @(kCVPixelFormatType_32BGRA),
-            @(kCVPixelFormatType_32RGBA),
-            @(kCVPixelFormatType_422YpCbCr8),
-            @(kCVPixelFormatType_422YpCbCr8_yuvs),
-            @(kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange),
-            @(kCVPixelFormatType_420YpCbCr8BiPlanarFullRange),
-            @(kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange),
-            @(kCVPixelFormatType_420YpCbCr10BiPlanarFullRange),
-            @(kCVPixelFormatType_OneComponent8),
-            @(q_kCVPixelFormatType_OneComponent16),
-            @(kCVPixelFormatType_420YpCbCr8Planar),
-            @(kCVPixelFormatType_420YpCbCr8PlanarFullRange)
-        ]
-#ifndef Q_OS_IOS // This key is not supported and generates a warning.
-        , (NSString *)kCVPixelBufferMetalCompatibilityKey: @true
-#endif // Q_OS_IOS
-};
-
 CVPixelBufferRef AVFVideoRendererControl::copyPixelBufferFromLayer(size_t& width, size_t& height)
 {
     AVPlayerLayer *layer = playerLayer();
@@ -193,7 +173,7 @@ CVPixelBufferRef AVFVideoRendererControl::copyPixelBufferFromLayer(size_t& width
 
     if (!m_videoOutput) {
         if (!m_outputSettings)
-            m_outputSettings = AVF_OUTPUT_SETTINGS;
+            setOutputSettings();
         m_videoOutput = [[AVPlayerItemVideoOutput alloc] initWithPixelBufferAttributes:m_outputSettings];
         [m_videoOutput setDelegate:nil queue:nil];
     }
diff --git a/src/plugins/multimedia/darwin/qavfhelpers.mm b/src/plugins/multimedia/darwin/qavfhelpers.mm
index bbfcc2d85..c8195a81e 100644
--- a/src/plugins/multimedia/darwin/qavfhelpers.mm
+++ b/src/plugins/multimedia/darwin/qavfhelpers.mm
@@ -80,13 +80,13 @@ bool QAVFHelpers::toCVPixelFormat(QVideoFrameFormat::PixelFormat qtFormat, unsig
 QVideoFrameFormat QAVFHelpers::videoFormatForImageBuffer(CVImageBufferRef buffer, bool openGL)
 {
     auto avPixelFormat = CVPixelBufferGetPixelFormatType(buffer);
+    auto pixelFormat = fromCVPixelFormat(avPixelFormat);
     if (openGL) {
         if (avPixelFormat == kCVPixelFormatType_32BGRA)
-            avPixelFormat = QVideoFrameFormat::Format_SamplerRect;
+            pixelFormat = QVideoFrameFormat::Format_SamplerRect;
         else
             qWarning() << "Accelerated macOS OpenGL video supports BGRA only, got CV pixel format" << avPixelFormat;
     }
-    auto pixelFormat = fromCVPixelFormat(avPixelFormat);
 
     size_t width = CVPixelBufferGetWidth(buffer);
     size_t height = CVPixelBufferGetHeight(buffer);
diff --git a/tests/auto/integration/qmediaplayerbackend/CMakeLists.txt b/tests/auto/integration/qmediaplayerbackend/CMakeLists.txt
index 48ad8bea7..8418b7527 100644
--- a/tests/auto/integration/qmediaplayerbackend/CMakeLists.txt
+++ b/tests/auto/integration/qmediaplayerbackend/CMakeLists.txt
@@ -20,6 +20,11 @@ qt_internal_add_test(tst_qmediaplayerbackend
     LIBRARIES
         Qt::Gui
         Qt::MultimediaPrivate
+        Qt::MultimediaPrivate
+        Qt::MultimediaQuickPrivate
+        Qt::Qml
+        Qt::Quick
+        Qt::QuickPrivate
     TESTDATA ${testdata_resource_files}
 )
 
@@ -28,6 +33,7 @@ qt_internal_add_resource(tst_qmediaplayerbackend "testdata"
         "/"
     FILES
         ${testdata_resource_files}
+        "LazyLoad.qml"
 )
 
 ## Scopes:
diff --git a/tests/auto/integration/qmediaplayerbackend/LazyLoad.qml b/tests/auto/integration/qmediaplayerbackend/LazyLoad.qml
new file mode 100644
index 000000000..3074c897b
--- /dev/null
+++ b/tests/auto/integration/qmediaplayerbackend/LazyLoad.qml
@@ -0,0 +1,52 @@
+// Copyright (C) 2023 The Qt Company Ltd.
+// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0
+
+import QtQuick
+import QtMultimedia
+
+Rectangle {
+    id: root
+    width: 600
+    height: 800
+    color: "black"
+
+    Component {
+        id: videoOutputComponent
+
+        Item {
+            objectName: "videoPlayer"
+            property alias mediaPlayer: mediaPlayer
+            property alias videoOutput: videoOutput
+            property alias videoSink: videoOutput.videoSink
+
+            property alias playbackState: mediaPlayer.playbackState
+            property alias error: mediaPlayer.error
+
+
+            MediaPlayer {
+                id: mediaPlayer
+                objectName: "mediaPlayer"
+                source: "qrc:/testdata/colors.mp4"
+            }
+            VideoOutput {
+                id: videoOutput
+                objectName: "videoOutput"
+                anchors.fill: parent
+            }
+        }
+    }
+
+    Loader {
+        id: loader
+        objectName: "loader"
+        sourceComponent: videoOutputComponent
+        anchors.fill: parent
+        active: false
+        onActiveChanged: {
+            if (active) {
+                loader.item.mediaPlayer.videoOutput = loader.item.videoOutput
+                loader.item.mediaPlayer.play()
+            }
+        }
+    }
+}
diff --git a/tests/auto/integration/qmediaplayerbackend/tst_qmediaplayerbackend.cpp b/tests/auto/integration/qmediaplayerbackend/tst_qmediaplayerbackend.cpp
index 6e1365808..fa50e2cf7 100644
--- a/tests/auto/integration/qmediaplayerbackend/tst_qmediaplayerbackend.cpp
+++ b/tests/auto/integration/qmediaplayerbackend/tst_qmediaplayerbackend.cpp
@@ -10,10 +10,18 @@
 #include <qvideoframe.h>
 #include <qaudiooutput.h>
 
+#include <QtQml/qqmlengine.h>
+#include <QtQml/qqmlcomponent.h>
+#include <QtQml/qqmlproperty.h>
+#include <QtQuick/qquickitem.h>
+#include <QtQuick/qquickview.h>
+#include <QtQuick/private/qquickloader_p.h>
+
 #include "../shared/mediafileselector.h"
 //TESTED_COMPONENT=src/multimedia
 
 #include <QtMultimedia/private/qtmultimedia-config_p.h>
+#include "private/qquickvideooutput_p.h"
 
 #include <array>
 
@@ -67,6 +75,7 @@ private slots:
     void durationDetectionIssues();
     void finiteLoops();
     void infiteLoops();
+    void lazyLoadVideo();
 
 private:
     QUrl selectVideoFile(const QStringList& mediaCandidates);
@@ -1745,6 +1754,42 @@ void tst_QMediaPlayerBackend::infiteLoops()
     QCOMPARE(player.playbackState(), QMediaPlayer::StoppedState);
 }
 
+void tst_QMediaPlayerBackend::lazyLoadVideo()
+{
+    QQmlEngine engine;
+    QQmlComponent component(&engine);
+    component.loadUrl(QUrl("qrc:/LazyLoad.qml"));
+    QScopedPointer<QObject> root(component.create());
+    QQuickItem *rootItem = qobject_cast<QQuickItem *>(root.get());
+    QVERIFY(rootItem);
+
+    QQuickView view;
+    rootItem->setParentItem(view.contentItem());
+    view.resize(600, 800);
+    view.show();
+
+    QQuickLoader *loader = qobject_cast<QQuickLoader *>(rootItem->findChild<QQuickItem *>("loader"));
+    QVERIFY(loader);
+    QCOMPARE(QQmlProperty::read(loader, "active").toBool(), false);
+    loader->setProperty("active", true);
+    QCOMPARE(QQmlProperty::read(loader, "active").toBool(), true);
+
+    QQuickItem *videoPlayer = qobject_cast<QQuickItem *>(loader->findChild<QQuickItem *>("videoPlayer"));
+    QVERIFY(videoPlayer);
+
+    QCOMPARE(QQmlProperty::read(videoPlayer, "playbackState").value<QMediaPlayer::PlaybackState>(), QMediaPlayer::PlayingState);
+    QCOMPARE(QQmlProperty::read(videoPlayer, "error").value<QMediaPlayer::Error>(), QMediaPlayer::NoError);
+
+    QVideoSink *videoSink = QQmlProperty::read(videoPlayer, "videoSink").value<QVideoSink *>();
+    QVERIFY(videoSink);
+
+    QSignalSpy spy(videoSink, &QVideoSink::videoFrameChanged);
+    QVERIFY(spy.wait());
+
+    QVideoFrame frame = spy.at(0).at(0).value<QVideoFrame>();
+    QVERIFY(frame.isValid());
+}
+
 QTEST_MAIN(tst_QMediaPlayerBackend)
 #include "tst_qmediaplayerbackend.moc"