summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorArtem Dyomin <artem.dyomin@qt.io>2023-02-07 16:10:59 +0100
committerQt Cherry-pick Bot <cherrypick_bot@qt-project.org>2023-02-07 18:09:52 +0000
commitb7b034176ff5ef04158445bb3dacf24ebf1e1ea5 (patch)
treefde07f99176e906d16190ef4d046f52cb2b31a12
parente9e5720108e28c2cbe378af4a1eef4d313775735 (diff)
downloadqtmultimedia-b7b034176ff5ef04158445bb3dacf24ebf1e1ea5.tar.gz
Rewrite screen capturing on macos
The previous generic implementation uses QScreen::grabWindow that is not stable for both cases, screens and windows capturing. In order to fix it, I've reimplemented it via native macos functionality. Some part of the functionality is shared with avfcamera. As a result, we got the following: - fix not capturing of some windows on the screen - enable hw acceleration for screens capturing (shared with camera functionality) - enable foreign app windows capturing (by CGWindowID as WId). We might need some additional api for getting the list of ids. Probably, the code is reusable for ios, to be investigated. Screen capturing tests work locally if permissions are granted; on CI some problems with giving the permissions. Task-number: QTBUG-103226 Change-Id: Iada34467e4b9074eb43a1fbbdc4409e663ea668a Reviewed-by: Lars Knoll <lars@knoll.priv.no> (cherry picked from commit a128d5ecf40d8d670d4d3aa8b9ddc860baa3cb5e) Reviewed-by: Qt Cherry-pick Bot <cherrypick_bot@qt-project.org>
-rw-r--r--src/multimedia/platform/qplatformscreencapture.cpp3
-rw-r--r--src/plugins/multimedia/ffmpeg/CMakeLists.txt3
-rw-r--r--src/plugins/multimedia/ffmpeg/qavfcamera.mm134
-rw-r--r--src/plugins/multimedia/ffmpeg/qavfsamplebufferdelegate.mm127
-rw-r--r--src/plugins/multimedia/ffmpeg/qavfsamplebufferdelegate_p.h48
-rw-r--r--src/plugins/multimedia/ffmpeg/qavfscreencapture.mm390
-rw-r--r--src/plugins/multimedia/ffmpeg/qavfscreencapture_p.h62
-rw-r--r--src/plugins/multimedia/ffmpeg/qffmpegmediaintegration.cpp3
-rw-r--r--tests/auto/integration/qscreencapture_integration/tst_qscreencapture_integration.cpp16
9 files changed, 650 insertions, 136 deletions
diff --git a/src/multimedia/platform/qplatformscreencapture.cpp b/src/multimedia/platform/qplatformscreencapture.cpp
index 58de67f24..ed2003b25 100644
--- a/src/multimedia/platform/qplatformscreencapture.cpp
+++ b/src/multimedia/platform/qplatformscreencapture.cpp
@@ -3,6 +3,7 @@
#include "platform/qplatformscreencapture_p.h"
#include "qvideoframe.h"
+#include "qdebug.h"
QT_BEGIN_NAMESPACE
@@ -58,6 +59,8 @@ void QPlatformScreenCapture::updateError(QScreenCapture::Error error, const QStr
m_error = error;
m_errorString = errorString;
if (changed) {
+ qWarning() << "Screen capture fail:" << error << "," << errorString;
+
if (m_error != QScreenCapture::NoError)
emit m_screenCapture->errorOccurred(error, errorString);
emit m_screenCapture->errorChanged();
diff --git a/src/plugins/multimedia/ffmpeg/CMakeLists.txt b/src/plugins/multimedia/ffmpeg/CMakeLists.txt
index 267b90786..849edbb77 100644
--- a/src/plugins/multimedia/ffmpeg/CMakeLists.txt
+++ b/src/plugins/multimedia/ffmpeg/CMakeLists.txt
@@ -65,6 +65,7 @@ qt_internal_extend_target(QFFmpegMediaPlugin CONDITION QT_FEATURE_ffmpeg AND QT_
EGL::EGL
)
+
qt_internal_extend_target(QFFmpegMediaPlugin CONDITION APPLE
SOURCES
../darwin/qavfhelpers.mm ../darwin/qavfhelpers_p.h
@@ -72,6 +73,8 @@ qt_internal_extend_target(QFFmpegMediaPlugin CONDITION APPLE
../darwin/camera/avfcamerautility_p.h ../darwin/camera/avfcamerautility.mm
qffmpeghwaccel_videotoolbox.mm qffmpeghwaccel_videotoolbox_p.h
qavfcamera.mm qavfcamera_p.h
+ qavfsamplebufferdelegate.mm qavfsamplebufferdelegate_p.h
+ qavfscreencapture.mm qavfscreencapture_p.h
INCLUDE_DIRECTORIES
../darwin
../darwin/camera
diff --git a/src/plugins/multimedia/ffmpeg/qavfcamera.mm b/src/plugins/multimedia/ffmpeg/qavfcamera.mm
index 9cdb84dc7..1a1f19561 100644
--- a/src/plugins/multimedia/ffmpeg/qavfcamera.mm
+++ b/src/plugins/multimedia/ffmpeg/qavfcamera.mm
@@ -6,6 +6,7 @@
#include <private/qplatformmediacapture_p.h>
#include "avfcamerautility_p.h"
#include "qavfhelpers_p.h"
+#include "qavfsamplebufferdelegate_p.h"
#include <qvideosink.h>
#include <private/qrhi_p.h>
#define AVMediaType XAVMediaType
@@ -17,145 +18,16 @@ extern "C" {
}
#undef AVMediaType
-
-
-#import <AVFoundation/AVFoundation.h>
-#include <CoreVideo/CoreVideo.h>
-
-static void releaseHwFrame(void */*opaque*/, uint8_t *data)
-{
- CVPixelBufferRelease(CVPixelBufferRef(data));
-}
-
-// Make sure this is compatible with the layout used in ffmpeg's hwcontext_videotoolbox
-static QFFmpeg::AVFrameUPtr allocHWFrame(AVBufferRef *hwContext, const CVPixelBufferRef &pixbuf)
-{
- AVHWFramesContext *ctx = (AVHWFramesContext*)hwContext->data;
- auto frame = QFFmpeg::makeAVFrame();
- frame->hw_frames_ctx = av_buffer_ref(hwContext);
- frame->extended_data = frame->data;
-
- frame->buf[0] = av_buffer_create((uint8_t *)pixbuf, 1, releaseHwFrame, NULL, 0);
- frame->data[3] = (uint8_t *)pixbuf;
- CVPixelBufferRetain(pixbuf);
- frame->width = ctx->width;
- frame->height = ctx->height;
- frame->format = AV_PIX_FMT_VIDEOTOOLBOX;
- if (frame->width != (int)CVPixelBufferGetWidth(pixbuf) ||
- frame->height != (int)CVPixelBufferGetHeight(pixbuf)) {
- // This can happen while changing camera format
- return nullptr;
- }
- return frame;
-}
-
static AVAuthorizationStatus m_cameraAuthorizationStatus = AVAuthorizationStatusNotDetermined;
-@interface QAVFSampleBufferDelegate : NSObject <AVCaptureVideoDataOutputSampleBufferDelegate>
-
-- (QAVFSampleBufferDelegate *) initWithCamera:(QAVFCamera *)renderer;
-
-- (void) captureOutput:(AVCaptureOutput *)captureOutput
- didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer
- fromConnection:(AVCaptureConnection *)connection;
-
-- (void) setHWAccel:(std::unique_ptr<QFFmpeg::HWAccel> &&)accel;
-
-@end
-
-@implementation QAVFSampleBufferDelegate
-{
-@private
- QAVFCamera *m_camera;
- AVBufferRef *hwFramesContext;
- std::unique_ptr<QFFmpeg::HWAccel> m_accel;
- qint64 startTime;
- qint64 baseTime;
-}
-
-- (QAVFSampleBufferDelegate *) initWithCamera:(QAVFCamera *)renderer
-{
- if (!(self = [super init]))
- return nil;
-
- m_camera = renderer;
- hwFramesContext = nullptr;
- startTime = 0;
- baseTime = 0;
- return self;
-}
-
-- (void)captureOutput:(AVCaptureOutput *)captureOutput
- didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer
- fromConnection:(AVCaptureConnection *)connection
-{
- Q_UNUSED(connection);
- Q_UNUSED(captureOutput);
-
- // NB: on iOS captureOutput/connection can be nil (when recording a video -
- // avfmediaassetwriter).
-
- CVImageBufferRef imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer);
-
- CMTime time = CMSampleBufferGetPresentationTimeStamp(sampleBuffer);
- qint64 frameTime = time.timescale ? time.value*1000/time.timescale : 0;
- if (baseTime == 0) {
- // drop the first frame to get a valid frame start time
- baseTime = frameTime;
- startTime = 0;
- return;
- }
-
- if (!m_accel)
- return;
-
- auto avFrame = allocHWFrame(m_accel->hwFramesContextAsBuffer(), imageBuffer);
- if (!avFrame)
- return;
-
-#ifdef USE_SW_FRAMES
- {
- auto swFrame = QFFmpeg::makeAVFrame();
- /* retrieve data from GPU to CPU */
- const int ret = av_hwframe_transfer_data(swFrame.get(), avFrame.get(), 0);
- if (ret < 0) {
- qWarning() << "Error transferring the data to system memory:" << ret;
- } else {
- avFrame = std::move(swFrame);
- }
- }
-#endif
-
- QVideoFrameFormat format = QAVFHelpers::videoFormatForImageBuffer(imageBuffer);
- if (!format.isValid()) {
- return;
- }
-
- avFrame->pts = startTime;
-
- QFFmpegVideoBuffer *buffer = new QFFmpegVideoBuffer(std::move(avFrame));
- QVideoFrame frame(buffer, format);
- frame.setStartTime(startTime);
- frame.setEndTime(frameTime);
- startTime = frameTime;
-
- m_camera->syncHandleFrame(frame);
-}
-
-- (void) setHWAccel:(std::unique_ptr<QFFmpeg::HWAccel> &&)accel
-{
- m_accel = std::move(accel);
-}
-
-@end
-
QT_BEGIN_NAMESPACE
QAVFCamera::QAVFCamera(QCamera *parent)
: QAVFCameraBase(parent)
{
m_captureSession = [[AVCaptureSession alloc] init];
- m_sampleBufferDelegate = [[QAVFSampleBufferDelegate alloc] initWithCamera:this];
+ m_sampleBufferDelegate = [[QAVFSampleBufferDelegate alloc]
+ initWithFrameHandler:[this](const QVideoFrame &frame) { syncHandleFrame(frame); }];
}
QAVFCamera::~QAVFCamera()
diff --git a/src/plugins/multimedia/ffmpeg/qavfsamplebufferdelegate.mm b/src/plugins/multimedia/ffmpeg/qavfsamplebufferdelegate.mm
new file mode 100644
index 000000000..4bc5dd1a3
--- /dev/null
+++ b/src/plugins/multimedia/ffmpeg/qavfsamplebufferdelegate.mm
@@ -0,0 +1,127 @@
+// Copyright (C) 2022 The Qt Company Ltd.
+// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
+
+#include "qavfsamplebufferdelegate_p.h"
+
+#define AVMediaType XAVMediaType
+
+#include "qffmpeghwaccel_p.h"
+#include "qavfhelpers_p.h"
+#include "qffmpegvideobuffer_p.h"
+
+#undef AVMediaType
+
+QT_USE_NAMESPACE
+
+static void releaseHwFrame(void * /*opaque*/, uint8_t *data)
+{
+ CVPixelBufferRelease(CVPixelBufferRef(data));
+}
+
+// Make sure this is compatible with the layout used in ffmpeg's hwcontext_videotoolbox
+static QFFmpeg::AVFrameUPtr allocHWFrame(AVBufferRef *hwContext, const CVPixelBufferRef &pixbuf)
+{
+ AVHWFramesContext *ctx = (AVHWFramesContext *)hwContext->data;
+ auto frame = QFFmpeg::makeAVFrame();
+ frame->hw_frames_ctx = av_buffer_ref(hwContext);
+ frame->extended_data = frame->data;
+
+ frame->buf[0] = av_buffer_create((uint8_t *)pixbuf, 1, releaseHwFrame, NULL, 0);
+ frame->data[3] = (uint8_t *)pixbuf;
+ CVPixelBufferRetain(pixbuf);
+ frame->width = ctx->width;
+ frame->height = ctx->height;
+ frame->format = AV_PIX_FMT_VIDEOTOOLBOX;
+ if (frame->width != (int)CVPixelBufferGetWidth(pixbuf)
+ || frame->height != (int)CVPixelBufferGetHeight(pixbuf)) {
+
+ // This can happen while changing camera format
+ return nullptr;
+ }
+ return frame;
+}
+
+@implementation QAVFSampleBufferDelegate {
+@private
+ std::function<void(const QVideoFrame &)> frameHandler;
+ AVBufferRef *hwFramesContext;
+ std::unique_ptr<QFFmpeg::HWAccel> m_accel;
+ qint64 startTime;
+ qint64 baseTime;
+}
+
+- (instancetype)initWithFrameHandler:(std::function<void(const QVideoFrame &)>)handler
+{
+ if (!(self = [super init]))
+ return nil;
+
+ frameHandler = std::move(handler);
+ hwFramesContext = nullptr;
+ startTime = 0;
+ baseTime = 0;
+ return self;
+}
+
+- (void)captureOutput:(AVCaptureOutput *)captureOutput
+ didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer
+ fromConnection:(AVCaptureConnection *)connection
+{
+ Q_UNUSED(connection);
+ Q_UNUSED(captureOutput);
+
+ // NB: on iOS captureOutput/connection can be nil (when recording a video -
+ // avfmediaassetwriter).
+
+ CVImageBufferRef imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer);
+
+ CMTime time = CMSampleBufferGetPresentationTimeStamp(sampleBuffer);
+ qint64 frameTime = time.timescale ? time.value * 1000 / time.timescale : 0;
+ if (baseTime == 0) {
+ // drop the first frame to get a valid frame start time
+ baseTime = frameTime;
+ startTime = 0;
+ return;
+ }
+
+ if (!m_accel)
+ return;
+
+ auto avFrame = allocHWFrame(m_accel->hwFramesContextAsBuffer(), imageBuffer);
+ if (!avFrame)
+ return;
+
+#ifdef USE_SW_FRAMES
+ {
+ auto swFrame = QFFmpeg::makeAVFrame();
+ /* retrieve data from GPU to CPU */
+ const int ret = av_hwframe_transfer_data(swFrame.get(), avFrame.get(), 0);
+ if (ret < 0) {
+ qWarning() << "Error transferring the data to system memory:" << ret;
+ } else {
+ avFrame = std::move(swFrame);
+ }
+ }
+#endif
+
+ QVideoFrameFormat format = QAVFHelpers::videoFormatForImageBuffer(imageBuffer);
+ if (!format.isValid()) {
+ return;
+ }
+
+ avFrame->pts = startTime;
+
+ QFFmpegVideoBuffer *buffer = new QFFmpegVideoBuffer(std::move(avFrame));
+ QVideoFrame frame(buffer, format);
+ frame.setStartTime(startTime);
+ frame.setEndTime(frameTime);
+ startTime = frameTime;
+
+ frameHandler(frame);
+}
+
+- (void)setHWAccel:(std::unique_ptr<QFFmpeg::HWAccel> &&)accel
+{
+ m_accel = std::move(accel);
+}
+
+@end
diff --git a/src/plugins/multimedia/ffmpeg/qavfsamplebufferdelegate_p.h b/src/plugins/multimedia/ffmpeg/qavfsamplebufferdelegate_p.h
new file mode 100644
index 000000000..a366c8c0a
--- /dev/null
+++ b/src/plugins/multimedia/ffmpeg/qavfsamplebufferdelegate_p.h
@@ -0,0 +1,48 @@
+// Copyright (C) 2022 The Qt Company Ltd.
+// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
+
+#ifndef QAVFSAMPLEBUFFERDELEGATE_P_H
+#define QAVFSAMPLEBUFFERDELEGATE_P_H
+
+//
+// W A R N I N G
+// -------------
+//
+// This file is not part of the Qt API. It exists purely as an
+// implementation detail. This header file may change from version to
+// version without notice, or even be removed.
+//
+// We mean it.
+//
+
+#import <AVFoundation/AVFoundation.h>
+#import <CoreVideo/CoreVideo.h>
+
+#include <qtconfigmacros.h>
+
+#include <memory>
+#include <functional>
+
+QT_BEGIN_NAMESPACE
+
+class QAVSampleBufferDelegateFrameHandler;
+class QVideoFrame;
+namespace QFFmpeg {
+class HWAccel;
+}
+
+QT_END_NAMESPACE
+
+@interface QAVFSampleBufferDelegate : NSObject <AVCaptureVideoDataOutputSampleBufferDelegate>
+
+- (instancetype)initWithFrameHandler:(std::function<void(const QVideoFrame &)>)handler;
+
+- (void)captureOutput:(AVCaptureOutput *)captureOutput
+ didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer
+ fromConnection:(AVCaptureConnection *)connection;
+
+- (void)setHWAccel:(std::unique_ptr<QT_PREPEND_NAMESPACE(QFFmpeg::HWAccel)> &&)accel;
+
+@end
+
+#endif
diff --git a/src/plugins/multimedia/ffmpeg/qavfscreencapture.mm b/src/plugins/multimedia/ffmpeg/qavfscreencapture.mm
new file mode 100644
index 000000000..984704fe3
--- /dev/null
+++ b/src/plugins/multimedia/ffmpeg/qavfscreencapture.mm
@@ -0,0 +1,390 @@
+// Copyright (C) 2022 The Qt Company Ltd.
+// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
+
+#include "qavfscreencapture_p.h"
+#include <qpointer.h>
+#include <qscreencapture.h>
+#include <qscreen.h>
+#include <qthread.h>
+#include <qtimer.h>
+#include <QGuiApplication>
+#include <private/qrhi_p.h>
+#include "qavfsamplebufferdelegate_p.h"
+
+#define AVMediaType XAVMediaType
+#include "qffmpeghwaccel_p.h"
+
+extern "C" {
+#include <libavutil/hwcontext_videotoolbox.h>
+#include <libavutil/hwcontext.h>
+}
+#undef AVMediaType
+
+#include <ApplicationServices/ApplicationServices.h>
+#include <IOKit/graphics/IOGraphicsLib.h>
+#include <AppKit/NSScreen.h>
+#include <AppKit/NSApplication.h>
+#include <AppKit/NSWindow.h>
+
+#include <dispatch/dispatch.h>
+
+namespace {
+
+const auto DefaultCVPixelFormat = kCVPixelFormatType_32BGRA;
+
+// Mac screens often support 120 frames per sec; it looks, this is not
+// needed for the capturing now since it just affects CPI without valuable
+// advantages. In the future, the frame rate should be customized by
+// user's API.
+const qreal MaxFrameRate = 60.;
+
+CGDirectDisplayID findDisplayByName(const QString &name)
+{
+ for (NSScreen *screen in NSScreen.screens) {
+ if (name == QString::fromNSString(screen.localizedName))
+ return [screen.deviceDescription[@"NSScreenNumber"] unsignedIntValue];
+ }
+ return kCGNullDirectDisplay;
+}
+
+CGWindowID findNativeWindowID(WId wid)
+{
+ // qtbase functionality sets QWidget::winId to the pointer
+ // value of the matching NSView. This is kind of mess,
+ // so we're trying resolving it via this lookup.
+ for (NSWindow *window in NSApp.windows) {
+ if (window.initialFirstResponder == (NSView *)wid)
+ return static_cast<CGWindowID>(window.windowNumber);
+ }
+
+ return static_cast<CGWindowID>(wid);
+}
+
+qreal frameRateForWindow(CGWindowID /*wid*/)
+{
+ // TODO: detect the frame rate
+ // if (window && window.screen) {
+ // CGDirectDisplayID displayID = [window.screen.deviceDescription[@"NSScreenNumber"]
+ // unsignedIntValue]; const auto displayRefreshRate =
+ // CGDisplayModeGetRefreshRate(CGDisplayCopyDisplayMode(displayID)); if (displayRefreshRate > 0 &&
+ // displayRefreshRate < frameRate) frameRate = displayRefreshRate;
+ // }
+
+ return MaxFrameRate;
+}
+
+}
+
+QT_BEGIN_NAMESPACE
+
+QAVFScreenCapture::QAVFScreenCapture(QScreenCapture *screenCapture)
+ : QFFmpegScreenCaptureBase(screenCapture)
+{
+ CGRequestScreenCaptureAccess();
+}
+
+QAVFScreenCapture::~QAVFScreenCapture()
+{
+ resetCapture();
+}
+
+bool QAVFScreenCapture::setActiveInternal(bool active)
+{
+ if (active) {
+ if (!CGPreflightScreenCaptureAccess()) {
+ updateError(QScreenCapture::CaptureFailed, QLatin1String("Permissions denied"));
+ return false;
+ }
+
+ if (auto winId = window() ? window()->winId() : windowId())
+ return initWindowCapture(winId);
+ else if (auto scrn = screen() ? screen() : QGuiApplication::primaryScreen())
+ return initScreenCapture(scrn);
+ else {
+ updateError(QScreenCapture::NotFound, QLatin1String("Primary screen not found"));
+ return false;
+ }
+ } else {
+ resetCapture();
+ }
+
+ return true;
+}
+
+void QAVFScreenCapture::onNewFrame(const QVideoFrame &frame)
+{
+ // Since writing of the format is supposed to be only from one thread,
+ // the read-only comparison without a mutex is thread-safe
+ if (!m_format || m_format != frame.surfaceFormat()) {
+ QMutexLocker<QMutex> locker(&m_formatMutex);
+
+ m_format = frame.surfaceFormat();
+
+ locker.unlock();
+
+ m_waitForFormat.notify_one();
+ }
+
+ emit newVideoFrame(frame);
+}
+
+QVideoFrameFormat QAVFScreenCapture::format() const
+{
+ if (!m_grabber)
+ return {};
+
+ QMutexLocker<QMutex> locker(&m_formatMutex);
+ while (!m_format)
+ m_waitForFormat.wait(&m_formatMutex);
+ return *m_format;
+}
+
+class QCGImageVideoBuffer : public QAbstractVideoBuffer
+{
+public:
+ QCGImageVideoBuffer(CGImageRef image) : QAbstractVideoBuffer(QVideoFrame::NoHandle)
+ {
+ auto provider = CGImageGetDataProvider(image);
+ m_data = CGDataProviderCopyData(provider);
+ m_bytesPerLine = CGImageGetBytesPerRow(image);
+ }
+
+ ~QCGImageVideoBuffer() override { CFRelease(m_data); }
+
+ QVideoFrame::MapMode mapMode() const override { return m_mapMode; }
+
+ MapData map(QVideoFrame::MapMode mode) override
+ {
+ MapData mapData;
+ if (m_mapMode == QVideoFrame::NotMapped) {
+ m_mapMode = mode;
+
+ mapData.nPlanes = 1;
+ mapData.bytesPerLine[0] = static_cast<int>(m_bytesPerLine);
+ mapData.data[0] = (uchar *)CFDataGetBytePtr(m_data);
+ mapData.size[0] = static_cast<int>(CFDataGetLength(m_data));
+ }
+
+ return mapData;
+ }
+
+ void unmap() override { m_mapMode = QVideoFrame::NotMapped; }
+
+private:
+ QVideoFrame::MapMode m_mapMode = QVideoFrame::NotMapped;
+ CFDataRef m_data;
+ size_t m_bytesPerLine = 0;
+};
+
+class QAVFScreenCapture::Grabber
+{
+public:
+ virtual ~Grabber() = default;
+};
+
+class QAVFScreenCapture::WindowGrabber : public QThread, public QAVFScreenCapture::Grabber
+{
+public:
+ WindowGrabber(QAVFScreenCapture &capture, CGWindowID wid) : m_capture(capture), m_wid(wid)
+ {
+ start();
+ }
+
+ ~WindowGrabber() override
+ {
+ quit();
+ wait();
+ }
+
+private:
+ void run() override
+ {
+ m_timer = std::make_unique<QTimer>();
+ // should be deleted in this thread
+ auto deleter = qScopeGuard([this]() { m_timer.reset(); });
+ m_timer->setTimerType(Qt::PreciseTimer);
+
+ QElapsedTimer elapsedTimer;
+ qint64 lastFrameTime = 0;
+
+ m_timer->callOnTimeout([&]() { grabWindow(elapsedTimer, lastFrameTime); });
+ m_timer->start();
+
+ exec();
+ }
+
+ void grabWindow(const QElapsedTimer &elapsedTimer, qint64 &lastFrameTime)
+ {
+ const auto frameRate = frameRateForWindow(m_wid);
+
+ m_timer->setInterval(1000 / frameRate);
+
+ auto imageRef = CGWindowListCreateImage(CGRectNull, kCGWindowListOptionIncludingWindow,
+ m_wid, kCGWindowImageBoundsIgnoreFraming);
+ if (!imageRef) {
+ updateError(QScreenCapture::CaptureFailed,
+ QLatin1String("Cannot create image by window"));
+ return;
+ }
+
+ auto imageDeleter = qScopeGuard([imageRef]() { CGImageRelease(imageRef); });
+
+ if (CGImageGetBitsPerPixel(imageRef) != 32
+ || CGImageGetPixelFormatInfo(imageRef) != kCGImagePixelFormatPacked
+ || CGImageGetByteOrderInfo(imageRef) != kCGImageByteOrder32Little) {
+ qWarning() << "Unexpected image format. PixelFormatInfo:"
+ << CGImageGetPixelFormatInfo(imageRef)
+ << "BitsPerPixel:" << CGImageGetBitsPerPixel(imageRef) << "AlphaInfo"
+ << CGImageGetAlphaInfo(imageRef)
+ << "ByteOrderInfo:" << CGImageGetByteOrderInfo(imageRef);
+
+ updateError(QScreenCapture::WindowCapturingNotSupported,
+ QLatin1String("Not supported pixel format"));
+ return;
+ }
+
+ QVideoFrameFormat format(QSize(CGImageGetWidth(imageRef), CGImageGetHeight(imageRef)),
+ QVideoFrameFormat::Format_BGRA8888);
+ format.setFrameRate(frameRate);
+
+ QVideoFrame frame(new QCGImageVideoBuffer(imageRef), format);
+
+ const auto endTime = elapsedTimer.nsecsElapsed() / 1000;
+
+ frame.setStartTime(lastFrameTime);
+ frame.setEndTime(endTime);
+
+ m_capture.onNewFrame(frame);
+
+ lastFrameTime = endTime;
+ }
+
+private:
+ void updateError(QScreenCapture::Error error, const QString &description)
+ {
+ m_timer->setInterval(1000); // set some big interval in order not to overdo with the updates
+
+ QMetaObject::invokeMethod(&m_capture, [&capture = m_capture, error, description]() {
+ capture.updateError(error, description);
+ });
+ }
+
+private:
+ QAVFScreenCapture &m_capture;
+ CGWindowID m_wid;
+ std::unique_ptr<QTimer> m_timer;
+};
+
+class QAVFScreenCapture::ScreenGrabber : public QAVFScreenCapture::Grabber
+{
+public:
+ ScreenGrabber(QAVFScreenCapture &capture, QScreen *screen, CGDirectDisplayID screenID,
+ std::unique_ptr<QFFmpeg::HWAccel> hwAccel)
+ {
+ m_captureSession = [[AVCaptureSession alloc] init];
+
+ m_sampleBufferDelegate = [[QAVFSampleBufferDelegate alloc]
+ initWithFrameHandler:[&capture](const QVideoFrame &frame) {
+ capture.onNewFrame(frame);
+ }];
+
+ m_videoDataOutput = [[AVCaptureVideoDataOutput alloc] init];
+
+ NSDictionary *videoSettings = [NSDictionary
+ dictionaryWithObjectsAndKeys:[NSNumber numberWithUnsignedInt:DefaultCVPixelFormat],
+ kCVPixelBufferPixelFormatTypeKey, nil];
+
+ [m_videoDataOutput setVideoSettings:videoSettings];
+ [m_videoDataOutput setAlwaysDiscardsLateVideoFrames:true];
+
+ // Configure video output
+ m_dispatchQueue = dispatch_queue_create("vf_queue", nullptr);
+ [m_videoDataOutput setSampleBufferDelegate:m_sampleBufferDelegate queue:m_dispatchQueue];
+
+ [m_captureSession addOutput:m_videoDataOutput];
+
+ [m_sampleBufferDelegate setHWAccel:std::move(hwAccel)];
+
+ m_screenInput = [[AVCaptureScreenInput alloc] initWithDisplayID:screenID];
+ [m_screenInput
+ setMinFrameDuration:CMTimeMake(1, std::min(screen->refreshRate(), MaxFrameRate))];
+ [m_captureSession addInput:m_screenInput];
+
+ [m_captureSession startRunning];
+ }
+
+ ~ScreenGrabber() override
+ {
+ if (m_captureSession)
+ [m_captureSession stopRunning];
+
+ if (m_dispatchQueue)
+ dispatch_release(m_dispatchQueue);
+
+ [m_sampleBufferDelegate release];
+ [m_screenInput release];
+ [m_videoDataOutput release];
+ [m_captureSession release];
+ }
+
+private:
+ AVCaptureSession *m_captureSession = nullptr;
+ AVCaptureScreenInput *m_screenInput = nullptr;
+ AVCaptureVideoDataOutput *m_videoDataOutput = nullptr;
+ QAVFSampleBufferDelegate *m_sampleBufferDelegate = nullptr;
+ dispatch_queue_t m_dispatchQueue = nullptr;
+};
+
+bool QAVFScreenCapture::initScreenCapture(QScreen *screen)
+{
+ const auto screenID = findDisplayByName(screen->name());
+
+ if (screenID == kCGNullDirectDisplay) {
+ updateError(QScreenCapture::InternalError,
+ QLatin1String("Screen exists but couldn't been found by name"));
+ return false;
+ }
+
+ auto hwAccel = QFFmpeg::HWAccel::create(AV_HWDEVICE_TYPE_VIDEOTOOLBOX);
+
+ if (!hwAccel) {
+ updateError(QScreenCapture::CaptureFailed,
+ QLatin1String("Couldn't create videotoolbox hw acceleration"));
+ return false;
+ }
+
+ hwAccel->createFramesContext(av_map_videotoolbox_format_to_pixfmt(DefaultCVPixelFormat),
+ screen->size() * screen->devicePixelRatio());
+
+ if (!hwAccel->hwFramesContextAsBuffer()) {
+ updateError(QScreenCapture::CaptureFailed,
+ QLatin1String("Couldn't create hw frames context"));
+ return false;
+ }
+
+ m_grabber = std::make_unique<ScreenGrabber>(*this, screen, screenID, std::move(hwAccel));
+ return true;
+}
+
+bool QAVFScreenCapture::initWindowCapture(WId wid)
+{
+ const auto nativeWindowID = findNativeWindowID(wid);
+
+ if (!nativeWindowID) {
+ updateError(QScreenCapture::NotFound, QLatin1String("No native windows found"));
+ return false;
+ }
+
+ m_grabber = std::make_unique<WindowGrabber>(*this, nativeWindowID);
+ return true;
+}
+
+void QAVFScreenCapture::resetCapture()
+{
+ m_grabber.reset();
+ m_format = {};
+}
+
+QT_END_NAMESPACE
+
+#include "moc_qavfscreencapture_p.cpp"
diff --git a/src/plugins/multimedia/ffmpeg/qavfscreencapture_p.h b/src/plugins/multimedia/ffmpeg/qavfscreencapture_p.h
new file mode 100644
index 000000000..bdfcce462
--- /dev/null
+++ b/src/plugins/multimedia/ffmpeg/qavfscreencapture_p.h
@@ -0,0 +1,62 @@
+// Copyright (C) 2022 The Qt Company Ltd.
+// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
+
+#ifndef QAVFSCREENCAPTURE_H
+#define QAVFSCREENCAPTURE_H
+
+//
+// W A R N I N G
+// -------------
+//
+// This file is not part of the Qt API. It exists purely as an
+// implementation detail. This header file may change from version to
+// version without notice, or even be removed.
+//
+// We mean it.
+//
+
+#include "qffmpegscreencapturebase_p.h"
+#include <qmutex.h>
+#include <qwaitcondition.h>
+
+QT_BEGIN_NAMESPACE
+
+class QFFmpegVideoSink;
+
+class QAVFScreenCapture : public QFFmpegScreenCaptureBase
+{
+ Q_OBJECT
+
+ class Grabber;
+ class WindowGrabber;
+ class ScreenGrabber;
+
+public:
+ explicit QAVFScreenCapture(QScreenCapture *screenCapture);
+ ~QAVFScreenCapture() override;
+
+ QVideoFrameFormat format() const override;
+
+protected:
+ bool setActiveInternal(bool active) override;
+
+private:
+ void onNewFrame(const QVideoFrame &frame);
+
+ bool initScreenCapture(QScreen *screen);
+
+ bool initWindowCapture(WId wid);
+
+ void resetCapture();
+
+private:
+ std::optional<QVideoFrameFormat> m_format;
+ mutable QMutex m_formatMutex;
+ mutable QWaitCondition m_waitForFormat;
+
+ std::unique_ptr<Grabber> m_grabber;
+};
+
+QT_END_NAMESPACE
+
+#endif // QAVFSCREENCAPTURE_H
diff --git a/src/plugins/multimedia/ffmpeg/qffmpegmediaintegration.cpp b/src/plugins/multimedia/ffmpeg/qffmpegmediaintegration.cpp
index 953e777c4..71cad4f10 100644
--- a/src/plugins/multimedia/ffmpeg/qffmpegmediaintegration.cpp
+++ b/src/plugins/multimedia/ffmpeg/qffmpegmediaintegration.cpp
@@ -20,6 +20,7 @@
#ifdef Q_OS_DARWIN
#include "qavfcamera_p.h"
+#include "qavfscreencapture_p.h"
#elif defined(Q_OS_WINDOWS)
#include "qwindowscamera_p.h"
#include "qwindowsvideodevices_p.h"
@@ -138,6 +139,8 @@ QPlatformScreenCapture *QFFmpegMediaIntegration::createScreenCapture(QScreenCapt
#endif
#if defined(Q_OS_WINDOWS)
return new QFFmpegScreenCaptureDxgi(screenCapture);
+#elif defined(Q_OS_MACOS) // TODO: probably use it for iOS as well
+ return new QAVFScreenCapture(screenCapture);
#else
return new QFFmpegScreenCapture(screenCapture);
#endif
diff --git a/tests/auto/integration/qscreencapture_integration/tst_qscreencapture_integration.cpp b/tests/auto/integration/qscreencapture_integration/tst_qscreencapture_integration.cpp
index c5baab359..617eacf97 100644
--- a/tests/auto/integration/qscreencapture_integration/tst_qscreencapture_integration.cpp
+++ b/tests/auto/integration/qscreencapture_integration/tst_qscreencapture_integration.cpp
@@ -208,7 +208,7 @@ void tst_QScreenCaptureIntegration::capture(QTestWidget &widget, const QPoint &d
QTest::qWait(delay);
const auto expectedFramesCount =
- delay / static_cast<int>(1000 / widget.screen()->refreshRate());
+ delay / static_cast<int>(1000 / std::min(widget.screen()->refreshRate(), 60.));
const int framesCount = static_cast<int>(sink.images().size());
@@ -284,7 +284,8 @@ void tst_QScreenCaptureIntegration::initTestCase()
void tst_QScreenCaptureIntegration::captureWindowById()
{
- auto widget = QTestWidget::createAndShow(Qt::Window, QRect{ 200, 100, 430, 351 });
+ auto widget = QTestWidget::createAndShow(Qt::Window | Qt::FramelessWindowHint,
+ QRect{ 200, 100, 430, 351 });
QVERIFY(QTest::qWaitForWindowExposed(widget.get()));
capture(*widget, { 0, 0 }, { 430, 351 },
@@ -293,7 +294,8 @@ void tst_QScreenCaptureIntegration::captureWindowById()
void tst_QScreenCaptureIntegration::captureWindow()
{
- auto widget = QTestWidget::createAndShow(Qt::Window, QRect{ 200, 100, 430, 351 });
+ auto widget = QTestWidget::createAndShow(Qt::Window | Qt::FramelessWindowHint,
+ QRect{ 200, 100, 430, 351 });
QVERIFY(QTest::qWaitForWindowExposed(widget.get()));
capture(*widget, { 0, 0 }, { 430, 351 },
@@ -302,10 +304,12 @@ void tst_QScreenCaptureIntegration::captureWindow()
void tst_QScreenCaptureIntegration::captureOverlappedWindow()
{
- auto overlappedWidget = QTestWidget::createAndShow(Qt::Window, QRect{ 200, 100, 430, 351 });
+ auto overlappedWidget = QTestWidget::createAndShow(Qt::Window | Qt::FramelessWindowHint,
+ QRect{ 200, 100, 430, 351 });
QVERIFY(QTest::qWaitForWindowExposed(overlappedWidget.get()));
- auto overlappingWidget = QTestWidget::createAndShow(Qt::Window, QRect{ 210, 110, 430, 351 });
+ auto overlappingWidget = QTestWidget::createAndShow(Qt::Window | Qt::FramelessWindowHint,
+ QRect{ 210, 110, 430, 351 });
QVERIFY(QTest::qWaitForWindowExposed(overlappingWidget.get()));
capture(*overlappedWidget, { 0, 0 }, { 430, 351 }, [&overlappedWidget](QScreenCapture &sc) {
@@ -366,6 +370,8 @@ void tst_QScreenCaptureIntegration::removeWindowWhileCapture()
void tst_QScreenCaptureIntegration::removeScreenWhileCapture()
{
+ QSKIP("TODO: find a reliable way to emulate it");
+
removeWhileCapture([](QScreenCapture &sc) { sc.setScreen(QApplication::primaryScreen()); },
[]() {
// It's something that doesn't look safe but it performs required flow