Implement custom X11 screen capturing

The implementation works on linux with run X11. It's disabled with wayland.

Achivements

- Fixed problem with performance.
  The standard implementation affected CPU too much.
  On my local release build the speedup
  in capturing of a single frame is 4-5 times.
  The explanation of such a huge difference is that opening of capturing
  contexts on X11 is an expensive action, whereas implemented single
  context allocation and transferring images through
  shared memory (Xshm) is pretty fast.
  Also, QScreen::grabWindow permorms additional pixels processings
  under the hood what is not needed for videoframes.
- Improved errors handling
- The patch allows to run screen capture backend UTs on linux CI
  with X11, namely, ubuntu, opensuse. Test results:
  https://testresults.qt.io/coin/integration/qt/qtmultimedia/tasks/web_qt_qtmultimedia_1675977558004
- Probably, fixed problem with windows overlaping, the matching
  tests became pass, it should be checked on different platforms.
  Also, the approach cannot capture windows
  out of the screen borders, we throw an error signal.
  Both windows issues are to be investigated deeper in 6.6.

With the next commits I might move some common threading code for
SC to a separate class.

Task-number: QTBUG-103226
Change-Id: Ibd3600fdbc9714f14dc80c2a330aea5e0d064791
Reviewed-by: Lars Knoll <lars@knoll.priv.no>
(cherry picked from commit fc079734d34747d64fa26645bda36699b26c047e)
Reviewed-by: Qt Cherry-pick Bot <cherrypick_bot@qt-project.org>

6 files changed, 461 insertions, 3 deletions

diff --git a/src/multimedia/platform/qplatformscreencapture.cpp b/src/multimedia/platform/qplatformscreencapture.cpp
index ed2003b25..57ff87029 100644
--- a/src/multimedia/platform/qplatformscreencapture.cpp
+++ b/src/multimedia/platform/qplatformscreencapture.cpp
@@ -59,10 +59,11 @@ void QPlatformScreenCapture::updateError(QScreenCapture::Error error, const QStr
     m_error = error;
     m_errorString = errorString;
     if (changed) {
-        qWarning() << "Screen capture fail:" << error << "," << errorString;
-
         if (m_error != QScreenCapture::NoError)
             emit m_screenCapture->errorOccurred(error, errorString);
+        else
+            qWarning() << "Screen capture fail:" << error << "," << errorString;
+
         emit m_screenCapture->errorChanged();
     }
 }
diff --git a/src/plugins/multimedia/ffmpeg/CMakeLists.txt b/src/plugins/multimedia/ffmpeg/CMakeLists.txt
index 849edbb77..102eb4ab1 100644
--- a/src/plugins/multimedia/ffmpeg/CMakeLists.txt
+++ b/src/plugins/multimedia/ffmpeg/CMakeLists.txt
@@ -113,6 +113,15 @@ qt_internal_extend_target(QFFmpegMediaPlugin CONDITION QT_FEATURE_cpp_winrt
         windowsapp
 )
 
+qt_internal_extend_target(QFFmpegMediaPlugin CONDITION QT_FEATURE_xlib
+    SOURCES
+        qx11screencapture.cpp qx11screencapture_p.h
+    LIBRARIES
+        X11
+        Xrandr
+        Xext
+)
+
 qt_internal_extend_target(QFFmpegMediaPlugin CONDITION QT_FEATURE_linux_v4l
     SOURCES
         qv4l2camera.cpp qv4l2camera_p.h
diff --git a/src/plugins/multimedia/ffmpeg/qffmpegmediaintegration.cpp b/src/plugins/multimedia/ffmpeg/qffmpegmediaintegration.cpp
index 71cad4f10..e731a31a6 100644
--- a/src/plugins/multimedia/ffmpeg/qffmpegmediaintegration.cpp
+++ b/src/plugins/multimedia/ffmpeg/qffmpegmediaintegration.cpp
@@ -44,6 +44,10 @@ extern "C" {
 #include "qffmpegscreencapture_uwp_p.h"
 #endif
 
+#if QT_CONFIG(xlib)
+#include "qx11screencapture_p.h"
+#endif
+
 QT_BEGIN_NAMESPACE
 
 class QFFmpegMediaPlugin : public QPlatformMediaPlugin
@@ -137,6 +141,12 @@ QPlatformScreenCapture *QFFmpegMediaIntegration::createScreenCapture(QScreenCapt
     if (QFFmpegScreenCaptureUwp::isSupported())
         return new QFFmpegScreenCaptureUwp(screenCapture);
 #endif
+
+#if QT_CONFIG(xlib)
+    if (QX11ScreenCapture::isSupported())
+        return new QX11ScreenCapture(screenCapture);
+#endif
+
 #if defined(Q_OS_WINDOWS)
     return new QFFmpegScreenCaptureDxgi(screenCapture);
 #elif defined(Q_OS_MACOS) // TODO: probably use it for iOS as well
diff --git a/src/plugins/multimedia/ffmpeg/qx11screencapture.cpp b/src/plugins/multimedia/ffmpeg/qx11screencapture.cpp
new file mode 100644
index 000000000..3e67e6654
--- /dev/null
+++ b/src/plugins/multimedia/ffmpeg/qx11screencapture.cpp
@@ -0,0 +1,392 @@
+// Copyright (C) 2016 The Qt Company Ltd.
+// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
+
+#include "qx11screencapture_p.h"
+
+#include <qvideoframe.h>
+#include <qthread.h>
+#include <qscreen.h>
+#include <qwindow.h>
+#include <qtimer.h>
+#include <qdebug.h>
+#include <QGuiApplication>
+#include <qloggingcategory.h>
+
+#include "private/qabstractvideobuffer_p.h"
+
+#include <X11/Xlib.h>
+#include <sys/shm.h>
+#include <X11/extensions/XShm.h>
+#include <X11/Xutil.h>
+#include <X11/extensions/Xrandr.h>
+
+#include <optional>
+
+QT_BEGIN_NAMESPACE
+
+static Q_LOGGING_CATEGORY(qLcX11ScreenCapture, "qt.multimedia.ffmpeg.x11screencapture")
+
+namespace {
+
+const qreal MaxFrameRate = 60.;
+
+void destroyXImage(XImage* image) {
+    XDestroyImage(image); // macro
+}
+
+template <typename T, typename D>
+std::unique_ptr<T, D> makeXUptr(T* ptr, D deleter) {
+   return std::unique_ptr<T, D>(ptr, deleter);
+}
+
+int screenNumberByName(Display *display, const QString &name)
+{
+    int size = 0;
+    auto monitors = makeXUptr(
+            XRRGetMonitors(display, XDefaultRootWindow(display), true, &size),
+                &XRRFreeMonitors);
+    const auto end = monitors.get() + size;
+    auto found = std::find_if(monitors.get(), end, [&](const XRRMonitorInfo &info) {
+        auto atomName = makeXUptr(XGetAtomName(display, info.name), &XFree);
+        return atomName && name == QString::fromUtf8(atomName.get());
+    });
+
+    return found == end ? -1 : std::distance(monitors.get(), found);
+}
+
+QVideoFrameFormat::PixelFormat xImagePixelFormat(const XImage &image)
+{
+    if (image.bits_per_pixel != 32) return QVideoFrameFormat::Format_Invalid;
+
+    if (image.red_mask == 0xff0000 &&
+        image.green_mask == 0xff00 &&
+        image.blue_mask == 0xff)
+        return QVideoFrameFormat::Format_BGRX8888;
+
+    if (image.red_mask == 0xff00 &&
+        image.green_mask == 0xff0000 &&
+        image.blue_mask == 0xff000000)
+        return QVideoFrameFormat::Format_XBGR8888;
+
+    if (image.blue_mask == 0xff0000 &&
+        image.green_mask == 0xff00 &&
+        image.red_mask == 0xff)
+        return QVideoFrameFormat::Format_RGBX8888;
+
+    if (image.red_mask == 0xff00 &&
+        image.green_mask == 0xff0000 &&
+        image.blue_mask == 0xff000000)
+        return QVideoFrameFormat::Format_XRGB8888;
+
+    return QVideoFrameFormat::Format_Invalid;
+}
+
+class DataVideoBuffer : public QAbstractVideoBuffer
+{
+public:
+    DataVideoBuffer(const char *data, int bytesPerLine, int size)
+        : QAbstractVideoBuffer(QVideoFrame::NoHandle),
+          m_data(data, size),
+          m_size(size),
+          m_bytesPerLine(bytesPerLine)
+    {
+    }
+
+    QVideoFrame::MapMode mapMode() const override { return m_mapMode; }
+
+    MapData map(QVideoFrame::MapMode mode) override
+    {
+        MapData mapData;
+        if (m_mapMode == QVideoFrame::NotMapped) {
+            m_mapMode = mode;
+
+            mapData.nPlanes = 1;
+            mapData.bytesPerLine[0] = m_bytesPerLine;
+            mapData.data[0] = (uchar *)m_data.data();
+            mapData.size[0] = m_size;
+        }
+
+        return mapData;
+    }
+
+    void unmap() override { m_mapMode = QVideoFrame::NotMapped; }
+
+private:
+    QByteArray m_data;
+    QVideoFrame::MapMode m_mapMode = QVideoFrame::NotMapped;
+    int m_size;
+    int m_bytesPerLine;
+};
+
+} // namespace
+
+class QX11ScreenCapture::Grabber : private QThread
+{
+public:
+    static std::unique_ptr<Grabber> create(QX11ScreenCapture &capture, QScreen *screen)
+    {
+        std::unique_ptr<Grabber> result(new Grabber(capture));
+        return result->init(screen) ? std::move(result) : nullptr;
+    }
+
+    static std::unique_ptr<Grabber> create(QX11ScreenCapture &capture, WId wid)
+    {
+        std::unique_ptr<Grabber> result(new Grabber(capture));
+        return result->init(wid) ? std::move(result) : nullptr;
+    }
+
+    ~Grabber() override
+    {
+        quit();
+        wait();
+
+        detachShm();
+    }
+
+    const QVideoFrameFormat &format() const { return m_format; }
+
+private:
+    Grabber(QX11ScreenCapture &capture) : m_capture(capture) { }
+
+    bool createDisplay()
+    {
+        if (!m_display)
+            m_display.reset(XOpenDisplay(nullptr));
+
+        return m_display != nullptr;
+    }
+
+    bool init(WId wid) {
+        if (auto screen = QGuiApplication::primaryScreen())
+            m_frameRate = std::min(screen->refreshRate(), MaxFrameRate);
+
+        return createDisplay() && initWithXID(static_cast<XID>(wid));
+    }
+
+    bool init(QScreen *screen)
+    {
+        if (!createDisplay())
+            return false;
+
+        auto screenNumber = screenNumberByName(m_display.get(), screen->name());
+
+        if (screenNumber < 0)
+            return false;
+
+        m_frameRate = std::min(screen->refreshRate(), MaxFrameRate);
+
+        return initWithXID(RootWindow(m_display.get(), screenNumber));
+    }
+
+    bool initWithXID(XID xid)
+    {
+        m_xid = xid;
+
+        if (update()) {
+            start();
+            return true;
+        }
+
+        return false;
+    }
+
+    void detachShm()
+    {
+        if (std::exchange(m_attached, false)) {
+            XShmDetach(m_display.get(), &m_shmInfo);
+            shmdt(m_shmInfo.shmaddr);
+            shmctl(m_shmInfo.shmid, IPC_RMID, 0);
+        }
+    }
+
+    void attachShm()
+    {
+        Q_ASSERT(!m_attached);
+
+        m_shmInfo.shmid =
+                shmget(IPC_PRIVATE, m_xImage->bytes_per_line * m_xImage->height, IPC_CREAT | 0777);
+
+        if (m_shmInfo.shmid == -1)
+            return;
+
+        m_shmInfo.readOnly = false;
+        m_shmInfo.shmaddr = m_xImage->data = (char *)shmat(m_shmInfo.shmid, 0, 0);
+
+        m_attached = XShmAttach(m_display.get(), &m_shmInfo);
+    }
+
+    bool update()
+    {
+        XWindowAttributes wndattr = {};
+        if (XGetWindowAttributes(m_display.get(), m_xid, &wndattr) == 0) {
+            updateError(QScreenCapture::CaptureFailed, QLatin1String("Cannot get window attributes"));
+            return false;
+        }
+
+        // TODO: if capture windows, we should adjust offsets and size if
+        // the window is out of the screen borders
+        // m_xOffset = ...
+        // m_yOffset = ...
+
+        // check window params for the root window as well since
+        // it potentially can be changed (e.g. on VM with resizing)
+        if (!m_xImage || wndattr.width != m_xImage->width || wndattr.height != m_xImage->height
+            || wndattr.depth != m_xImage->depth || wndattr.visual->visualid != m_visualID) {
+
+            qCDebug(qLcX11ScreenCapture) << "recreate ximage: " << wndattr.width << wndattr.height << wndattr.depth
+                     << wndattr.visual->visualid;
+
+            detachShm();
+            m_xImage.reset();
+
+            m_visualID = wndattr.visual->visualid;
+            m_xImage.reset(XShmCreateImage(m_display.get(), wndattr.visual, wndattr.depth, ZPixmap,
+                                           nullptr, &m_shmInfo, wndattr.width, wndattr.height));
+
+            if (!m_xImage) {
+                updateError(QScreenCapture::CaptureFailed, QLatin1String("Cannot create image"));
+                return false;
+            }
+
+            const auto pixelFormat = xImagePixelFormat(*m_xImage);
+
+            // TODO: probably, add a converter instead
+            if (pixelFormat == QVideoFrameFormat::Format_Invalid) {
+                updateError(QScreenCapture::CaptureFailed,
+                            QLatin1String("Not handled pixel format, bpp=")
+                                    + QString::number(m_xImage->bits_per_pixel));
+                return false;
+            }
+
+            attachShm();
+
+            if (!m_attached) {
+                updateError(QScreenCapture::CaptureFailed,
+                            QLatin1String("Cannot attach shared memory"));
+                return false;
+            }
+
+            m_format = QVideoFrameFormat(QSize(m_xImage->width, m_xImage->height),
+                                         pixelFormat);
+            m_format.setFrameRate(m_frameRate);
+        }
+
+        return m_attached;
+    }
+
+private:
+    void updateError(QScreenCapture::Error error, const QString &description)
+    {
+        if (error != QScreenCapture::NoError) {
+            if (m_timer)
+                m_timer->setInterval(1000);
+        }
+
+        const auto prevError = std::exchange(m_prevGrabberError, error);
+
+        if (error != QScreenCapture::NoError || prevError != QScreenCapture::NoError) {
+            QMetaObject::invokeMethod(&m_capture,
+                                      std::bind(&QPlatformScreenCapture::updateError, &m_capture,
+                                                error, description));
+        }
+    }
+
+    void run() override
+    {
+        m_timer = std::make_unique<QTimer>();
+        auto deleter = qScopeGuard([&]() { m_timer.reset(); });
+        m_timer->setTimerType(Qt::PreciseTimer);
+        QElapsedTimer elapsedTimer;
+        qint64 lastFrameTime = 0;
+
+        m_timer->callOnTimeout([&]() { grab(elapsedTimer, lastFrameTime); });
+        m_timer->start();
+
+        exec();
+    }
+
+    void grab(const QElapsedTimer &elapsedTimer, qint64 &lastFrameTime)
+    {
+        m_timer->setInterval(1000 / m_frameRate);
+
+        if (!update())
+            return;
+
+        if (!XShmGetImage(m_display.get(), m_xid, m_xImage.get(), m_xOffset, m_yOffset,
+                          AllPlanes)) {
+            updateError(QScreenCapture::CaptureFailed,
+                        QLatin1String(
+                                "Cannot get ximage; the window may be out of the screen borders"));
+            return;
+        }
+
+        auto buffer = new DataVideoBuffer(m_xImage->data, m_xImage->bytes_per_line,
+                                          m_xImage->bytes_per_line * m_xImage->height);
+
+        QVideoFrame frame(buffer, m_format);
+
+        const auto endTime = elapsedTimer.nsecsElapsed() / 1000;
+        frame.setStartTime(lastFrameTime);
+        frame.setEndTime(endTime);
+        emit m_capture.newVideoFrame(frame);
+
+        updateError(QScreenCapture::NoError, {});
+
+        lastFrameTime = endTime;
+    }
+
+private:
+    QX11ScreenCapture &m_capture;
+    std::optional<QScreenCapture::Error> m_prevGrabberError;
+    XID m_xid = None;
+    int m_xOffset = 0;
+    int m_yOffset = 0;
+    std::unique_ptr<Display, decltype(&XCloseDisplay)> m_display{nullptr, &XCloseDisplay};
+    std::unique_ptr<XImage, decltype(&destroyXImage)> m_xImage{nullptr, &destroyXImage};
+    XShmSegmentInfo m_shmInfo;
+    bool m_attached = false;
+    qreal m_frameRate = MaxFrameRate;
+    VisualID m_visualID = None;
+    std::unique_ptr<QTimer> m_timer;
+    QVideoFrameFormat m_format;
+};
+
+QX11ScreenCapture::QX11ScreenCapture(QScreenCapture *screenCapture)
+    : QFFmpegScreenCaptureBase(screenCapture)
+{
+    // For debug
+    //  XSetErrorHandler([](Display *, XErrorEvent * e) {
+    //      qDebug() << "error handler" << e->error_code;
+    //      return 0;
+    //  });
+}
+
+QX11ScreenCapture::~QX11ScreenCapture() = default;
+
+QVideoFrameFormat QX11ScreenCapture::format() const
+{
+    return m_grabber ? m_grabber->format() : QVideoFrameFormat{};
+}
+
+bool QX11ScreenCapture::setActiveInternal(bool active)
+{
+    qCDebug(qLcX11ScreenCapture) << "set active" << active;
+
+    if (active) {
+        if (auto wid = window() ? window()->winId() : windowId())
+            m_grabber = Grabber::create(*this, wid);
+        else if (auto scrn = screen() ? screen() : QGuiApplication::primaryScreen())
+            m_grabber = Grabber::create(*this, scrn);
+    } else {
+        m_grabber.reset();
+    }
+
+    return static_cast<bool>(m_grabber) == active;
+}
+
+bool QX11ScreenCapture::isSupported()
+{
+    return qgetenv("XDG_SESSION_TYPE").compare(QLatin1String("x11"), Qt::CaseInsensitive) == 0;
+}
+
+QT_END_NAMESPACE
diff --git a/src/plugins/multimedia/ffmpeg/qx11screencapture_p.h b/src/plugins/multimedia/ffmpeg/qx11screencapture_p.h
new file mode 100644
index 000000000..78e302c9a
--- /dev/null
+++ b/src/plugins/multimedia/ffmpeg/qx11screencapture_p.h
@@ -0,0 +1,44 @@
+// Copyright (C) 2016 The Qt Company Ltd.
+// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
+
+#ifndef X11SCREENCAPTURE_P_H
+#define X11SCREENCAPTURE_P_H
+
+//
+//  W A R N I N G
+//  -------------
+//
+// This file is not part of the Qt API.  It exists purely as an
+// implementation detail.  This header file may change from version to
+// version without notice, or even be removed.
+//
+// We mean it.
+//
+
+#include "qffmpegscreencapturebase_p.h"
+
+QT_BEGIN_NAMESPACE
+
+class QX11ScreenCapture : public QFFmpegScreenCaptureBase
+{
+    class Grabber;
+
+public:
+    explicit QX11ScreenCapture(QScreenCapture *screenCapture);
+    ~QX11ScreenCapture() override;
+
+    QVideoFrameFormat format() const override;
+
+    static bool isSupported();
+
+protected:
+    bool setActiveInternal(bool active) override;
+
+private:
+    std::optional<QVideoFrameFormat> m_format;
+    std::unique_ptr<Grabber> m_grabber;
+};
+
+QT_END_NAMESPACE
+
+#endif // X11SCREENCAPTURE_P_H
diff --git a/tests/auto/integration/qscreencapture_integration/BLACKLIST b/tests/auto/integration/qscreencapture_integration/BLACKLIST
index 550ecdd6f..97f518059 100644
--- a/tests/auto/integration/qscreencapture_integration/BLACKLIST
+++ b/tests/auto/integration/qscreencapture_integration/BLACKLIST
@@ -1 +1,3 @@
-ci
+macos ci
+windows ci
+android ci