Fix crash with ffmpeg on specific video files

FFmpeg doc says that the following case is possible:
AVFrame.linesize[i] < 0 and AVFrame.data[i] points to the end of data,
it's a weird case of flipped pixels direction, only one video with
such an encoding has been found for now.

Function sws_scale solves the problem so we should call it
in the cases of flipped videos.

I'm going to add a test after finding a way of getting such files.

Task-number: QTBUG-108403
Change-Id: I72f45781e2c3983d6a1991db2b833b78184b90dd
Reviewed-by: Lars Knoll <lars@knoll.priv.no>
(cherry picked from commit 1e8653062d6d23e4dc026661c485e74750c9d4c2)

1 files changed, 12 insertions, 3 deletions

diff --git a/src/plugins/multimedia/ffmpeg/qffmpegvideobuffer.cpp b/src/plugins/multimedia/ffmpeg/qffmpegvideobuffer.cpp
index 637ad5d06..a5a7471ec 100644
--- a/src/plugins/multimedia/ffmpeg/qffmpegvideobuffer.cpp
+++ b/src/plugins/multimedia/ffmpeg/qffmpegvideobuffer.cpp
@@ -11,6 +11,15 @@ extern "C" {
 #include <libavutil/mastering_display_metadata.h>
 }
 
+static bool isFrameFlipped(const AVFrame& frame) {
+    for (int i = 0; i < AV_NUM_DATA_POINTERS && frame.data[i]; ++i) {
+        if (frame.linesize[i] < 0)
+            return true;
+    }
+
+    return false;
+}
+
 QT_BEGIN_NAMESPACE
 
 QFFmpegVideoBuffer::QFFmpegVideoBuffer(AVFrameUPtr frame)
@@ -35,9 +44,7 @@ void QFFmpegVideoBuffer::convertSWFrame()
     Q_ASSERT(swFrame);
     bool needsConversion = false;
     auto pixelFormat = toQtPixelFormat(AVPixelFormat(swFrame->format), &needsConversion);
-//    qDebug() << "SW frame format:" << pixelFormat << swFrame->format << needsConversion;
-
-    if (pixelFormat != m_pixelFormat) {
+    if (pixelFormat != m_pixelFormat || isFrameFlipped(*swFrame)) {
         AVPixelFormat newFormat = toAVPixelFormat(m_pixelFormat);
         // convert the format into something we can handle
         SwsContext *c = sws_getContext(swFrame->width, swFrame->height, AVPixelFormat(swFrame->format),
@@ -177,6 +184,8 @@ QAbstractVideoBuffer::MapData QFFmpegVideoBuffer::map(QVideoFrame::MapMode mode)
     auto *desc = QVideoTextureHelper::textureDescription(pixelFormat());
     mapData.nPlanes = desc->nplanes;
     for (int i = 0; i < mapData.nPlanes; ++i) {
+        Q_ASSERT(swFrame->linesize[i] >= 0);
+
         mapData.data[i] = swFrame->data[i];
         mapData.bytesPerLine[i] = swFrame->linesize[i];
         mapData.size[i] = mapData.bytesPerLine[i]*desc->heightForPlane(swFrame->height, i);