diff options
author | Artem Dyomin <artem.dyomin@qt.io> | 2023-02-15 15:39:39 +0100 |
---|---|---|
committer | Artem Dyomin <artem.dyomin@qt.io> | 2023-02-17 11:28:35 +0000 |
commit | 7088aee0d279cb1370b967ef68abf606f88be762 (patch) | |
tree | cd9d847befc9117db7114bc1633eb94bafcdc707 | |
parent | 34a32676f53c296b5c02cbb349bf3a7725de3051 (diff) | |
download | qtmultimedia-7088aee0d279cb1370b967ef68abf606f88be762.tar.gz |
Fix ffmpeg encoding issues
Fixed a bunch of problems:
- wrong frame intervals in macos camera and screen capturing
- missing frame rate in screen capturing format
- wrong scaling in encoder
Also, some recording tests have been added
Task-number: QTBUG-103226
Change-Id: Id775f31e01d75d5c9f3c4ec20c33074acab1ab20
Reviewed-by: Lars Knoll <lars@knoll.priv.no>
(cherry picked from commit 68430756ac8f477bf91f9691d39f1605829f9500)
14 files changed, 180 insertions, 43 deletions
diff --git a/.gitignore b/.gitignore index 581e85566..e95f14f7b 100644 --- a/.gitignore +++ b/.gitignore @@ -52,9 +52,12 @@ tests/auto/cmake/build # Generated static plugin import sources *_plugin_import.cpp -# Build artifacts +# IDE artifacts CMakeLists.txt.user CMakeLists.txt.user.* .vscode +.xdp_CMakeLists.txt.* + +# Build artifacts build-*-Debug build-*-Release diff --git a/src/multimedia/platform/qplatformscreencapture.cpp b/src/multimedia/platform/qplatformscreencapture.cpp index 57ff87029..7a9f568d9 100644 --- a/src/multimedia/platform/qplatformscreencapture.cpp +++ b/src/multimedia/platform/qplatformscreencapture.cpp @@ -53,6 +53,11 @@ QScreenCapture *QPlatformScreenCapture::screenCapture() const return m_screenCapture; } +std::optional<int> QPlatformScreenCapture::ffmpegHWPixelFormat() const +{ + return {}; +} + void QPlatformScreenCapture::updateError(QScreenCapture::Error error, const QString &errorString) { bool changed = error != m_error || errorString != m_errorString; diff --git a/src/multimedia/platform/qplatformscreencapture_p.h b/src/multimedia/platform/qplatformscreencapture_p.h index c98437268..8da927d8d 100644 --- a/src/multimedia/platform/qplatformscreencapture_p.h +++ b/src/multimedia/platform/qplatformscreencapture_p.h @@ -24,6 +24,8 @@ #include "qscreencapture.h" #include "qvideoframeformat.h" +#include <optional> + QT_BEGIN_NAMESPACE class QVideoFrame; @@ -54,6 +56,8 @@ public: QScreenCapture *screenCapture() const; + virtual std::optional<int> ffmpegHWPixelFormat() const; + public Q_SLOTS: void updateError(QScreenCapture::Error error, const QString &errorString); diff --git a/src/plugins/multimedia/ffmpeg/qavfcamera.mm b/src/plugins/multimedia/ffmpeg/qavfcamera.mm index 1a1f19561..c7556ce3b 100644 --- a/src/plugins/multimedia/ffmpeg/qavfcamera.mm +++ b/src/plugins/multimedia/ffmpeg/qavfcamera.mm @@ -261,6 +261,7 @@ void QAVFCamera::updateCameraFormat() hwPixelFormat = AV_PIX_FMT_NONE; } [m_sampleBufferDelegate setHWAccel:std::move(hwAccel)]; + [m_sampleBufferDelegate setVideoFormatFrameRate:m_cameraFormat.maxFrameRate()]; } std::uint32_t QAVFCamera::setPixelFormat(const QVideoFrameFormat::PixelFormat pixelFormat) diff --git a/src/plugins/multimedia/ffmpeg/qavfsamplebufferdelegate.mm b/src/plugins/multimedia/ffmpeg/qavfsamplebufferdelegate.mm index 4bc5dd1a3..6770e1cce 100644 --- a/src/plugins/multimedia/ffmpeg/qavfsamplebufferdelegate.mm +++ b/src/plugins/multimedia/ffmpeg/qavfsamplebufferdelegate.mm @@ -9,6 +9,8 @@ #include "qavfhelpers_p.h" #include "qffmpegvideobuffer_p.h" +#include <optional> + #undef AVMediaType QT_USE_NAMESPACE @@ -47,7 +49,8 @@ static QFFmpeg::AVFrameUPtr allocHWFrame(AVBufferRef *hwContext, const CVPixelBu AVBufferRef *hwFramesContext; std::unique_ptr<QFFmpeg::HWAccel> m_accel; qint64 startTime; - qint64 baseTime; + std::optional<qint64> baseTime; + qreal frameRate; } - (instancetype)initWithFrameHandler:(std::function<void(const QVideoFrame &)>)handler @@ -55,10 +58,12 @@ static QFFmpeg::AVFrameUPtr allocHWFrame(AVBufferRef *hwContext, const CVPixelBu if (!(self = [super init])) return nil; + Q_ASSERT(handler); + frameHandler = std::move(handler); hwFramesContext = nullptr; startTime = 0; - baseTime = 0; + frameRate = 0.; return self; } @@ -74,12 +79,12 @@ static QFFmpeg::AVFrameUPtr allocHWFrame(AVBufferRef *hwContext, const CVPixelBu CVImageBufferRef imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer); - CMTime time = CMSampleBufferGetPresentationTimeStamp(sampleBuffer); - qint64 frameTime = time.timescale ? time.value * 1000 / time.timescale : 0; - if (baseTime == 0) { + const CMTime time = CMSampleBufferGetPresentationTimeStamp(sampleBuffer); + const qint64 frameTime = time.timescale ? time.value * 1000000 / time.timescale : 0; + if (!baseTime) { // drop the first frame to get a valid frame start time baseTime = frameTime; - startTime = 0; + startTime = frameTime; return; } @@ -108,12 +113,14 @@ static QFFmpeg::AVFrameUPtr allocHWFrame(AVBufferRef *hwContext, const CVPixelBu return; } - avFrame->pts = startTime; + format.setFrameRate(frameRate); + + avFrame->pts = startTime - *baseTime; QFFmpegVideoBuffer *buffer = new QFFmpegVideoBuffer(std::move(avFrame)); QVideoFrame frame(buffer, format); - frame.setStartTime(startTime); - frame.setEndTime(frameTime); + frame.setStartTime(startTime - *baseTime); + frame.setEndTime(frameTime - *baseTime); startTime = frameTime; frameHandler(frame); @@ -124,4 +131,9 @@ static QFFmpeg::AVFrameUPtr allocHWFrame(AVBufferRef *hwContext, const CVPixelBu m_accel = std::move(accel); } +- (void)setVideoFormatFrameRate:(qreal)rate +{ + frameRate = rate; +} + @end diff --git a/src/plugins/multimedia/ffmpeg/qavfsamplebufferdelegate_p.h b/src/plugins/multimedia/ffmpeg/qavfsamplebufferdelegate_p.h index a366c8c0a..5a7e16c71 100644 --- a/src/plugins/multimedia/ffmpeg/qavfsamplebufferdelegate_p.h +++ b/src/plugins/multimedia/ffmpeg/qavfsamplebufferdelegate_p.h @@ -19,6 +19,7 @@ #import <CoreVideo/CoreVideo.h> #include <qtconfigmacros.h> +#include <qtypes.h> #include <memory> #include <functional> @@ -43,6 +44,8 @@ QT_END_NAMESPACE - (void)setHWAccel:(std::unique_ptr<QT_PREPEND_NAMESPACE(QFFmpeg::HWAccel)> &&)accel; +- (void)setVideoFormatFrameRate:(qreal)frameRate; + @end #endif diff --git a/src/plugins/multimedia/ffmpeg/qavfscreencapture.mm b/src/plugins/multimedia/ffmpeg/qavfscreencapture.mm index 984704fe3..04777f146 100644 --- a/src/plugins/multimedia/ffmpeg/qavfscreencapture.mm +++ b/src/plugins/multimedia/ffmpeg/qavfscreencapture.mm @@ -139,6 +139,11 @@ QVideoFrameFormat QAVFScreenCapture::format() const return *m_format; } +std::optional<int> QAVFScreenCapture::ffmpegHWPixelFormat() const +{ + return AV_PIX_FMT_VIDEOTOOLBOX; +} + class QCGImageVideoBuffer : public QAbstractVideoBuffer { public: @@ -305,9 +310,11 @@ public: [m_sampleBufferDelegate setHWAccel:std::move(hwAccel)]; + const auto frameRate = std::min(screen->refreshRate(), MaxFrameRate); + [m_sampleBufferDelegate setVideoFormatFrameRate:frameRate]; + m_screenInput = [[AVCaptureScreenInput alloc] initWithDisplayID:screenID]; - [m_screenInput - setMinFrameDuration:CMTimeMake(1, std::min(screen->refreshRate(), MaxFrameRate))]; + [m_screenInput setMinFrameDuration:CMTimeMake(1, static_cast<int32_t>(frameRate))]; [m_captureSession addInput:m_screenInput]; [m_captureSession startRunning]; diff --git a/src/plugins/multimedia/ffmpeg/qavfscreencapture_p.h b/src/plugins/multimedia/ffmpeg/qavfscreencapture_p.h index bdfcce462..4204f1195 100644 --- a/src/plugins/multimedia/ffmpeg/qavfscreencapture_p.h +++ b/src/plugins/multimedia/ffmpeg/qavfscreencapture_p.h @@ -37,6 +37,8 @@ public: QVideoFrameFormat format() const override; + std::optional<int> ffmpegHWPixelFormat() const override; + protected: bool setActiveInternal(bool active) override; diff --git a/src/plugins/multimedia/ffmpeg/qffmpegencoder.cpp b/src/plugins/multimedia/ffmpeg/qffmpegencoder.cpp index e590959f4..ecf674e04 100644 --- a/src/plugins/multimedia/ffmpeg/qffmpegencoder.cpp +++ b/src/plugins/multimedia/ffmpeg/qffmpegencoder.cpp @@ -41,8 +41,8 @@ Encoder::Encoder(const QMediaEncoderSettings &settings, const QUrl &url) formatContext->url = (char *)av_malloc(encoded.size() + 1); memcpy(formatContext->url, encoded.constData(), encoded.size() + 1); formatContext->pb = nullptr; - avio_open2(&formatContext->pb, formatContext->url, AVIO_FLAG_WRITE, nullptr, nullptr); - qCDebug(qLcFFmpegEncoder) << "opened" << formatContext->url; + auto result = avio_open2(&formatContext->pb, formatContext->url, AVIO_FLAG_WRITE, nullptr, nullptr); + qCDebug(qLcFFmpegEncoder) << "opened" << result << formatContext->url; muxer = new Muxer(this); } @@ -76,7 +76,10 @@ void Encoder::addCamera(QPlatformCamera *camera) void Encoder::addScreenCapture(QPlatformScreenCapture *screenCapture) { - auto *ve = new VideoEncoder(this, settings, screenCapture->format(), {}); + std::optional<AVPixelFormat> hwPixelFormat = screenCapture->ffmpegHWPixelFormat() + ? AVPixelFormat(*screenCapture->ffmpegHWPixelFormat()) + : std::optional<AVPixelFormat>{}; + auto *ve = new VideoEncoder(this, settings, screenCapture->format(), hwPixelFormat); auto conn = connect(screenCapture, &QPlatformScreenCapture::newVideoFrame, [=](const QVideoFrame &frame){ ve->addFrame(frame); }); videoEncoders.append(ve); @@ -423,11 +426,19 @@ VideoEncoder::VideoEncoder(Encoder *encoder, const QMediaEncoderSettings &settin this->encoder = encoder; setObjectName(QLatin1String("VideoEncoder")); - qCDebug(qLcFFmpegEncoder) << "VideoEncoder" << settings.videoCodec(); AVPixelFormat swFormat = QFFmpegVideoBuffer::toAVPixelFormat(format.pixelFormat()); AVPixelFormat ffmpegPixelFormat = hwFormat ? *hwFormat : swFormat; - frameEncoder = new VideoFrameEncoder(settings, format.frameSize(), float(format.frameRate()), ffmpegPixelFormat, swFormat); + auto frameRate = format.frameRate(); + if (frameRate <= 0.) { + qWarning() << "Invalid frameRate" << frameRate << "; Using the default instead"; + + // set some default frame rate since ffmpeg has UB if it's 0. + frameRate = 30.; + } + + frameEncoder = new VideoFrameEncoder(settings, format.frameSize(), frameRate, ffmpegPixelFormat, + swFormat); frameEncoder->initWithFormatContext(encoder->formatContext); } diff --git a/src/plugins/multimedia/ffmpeg/qffmpegvideoframeencoder.cpp b/src/plugins/multimedia/ffmpeg/qffmpegvideoframeencoder.cpp index 8792945a6..6985ffea4 100644 --- a/src/plugins/multimedia/ffmpeg/qffmpegvideoframeencoder.cpp +++ b/src/plugins/multimedia/ffmpeg/qffmpegvideoframeencoder.cpp @@ -149,7 +149,7 @@ VideoFrameEncoder::VideoFrameEncoder(const QMediaEncoderSettings &encoderSetting if (desc->comp[0].depth == sourceDepth) s += 100; else if (desc->comp[0].depth < sourceDepth) - s -= 100; + s -= 100 + (sourceDepth - desc->comp[0].depth); if (desc->log2_chroma_h == 1) s += 1; if (desc->log2_chroma_w == 1) @@ -184,7 +184,7 @@ VideoFrameEncoder::VideoFrameEncoder(const QMediaEncoderSettings &encoderSetting av_hwframe_constraints_free(&constraints); // need to create a frames context to convert the input data - d->accel->createFramesContext(d->targetSWFormat, sourceSize); + d->accel->createFramesContext(d->targetSWFormat, d->settings.videoResolution()); } } else { d->targetSWFormat = d->targetFormat; @@ -193,10 +193,19 @@ VideoFrameEncoder::VideoFrameEncoder(const QMediaEncoderSettings &encoderSetting if (d->sourceSWFormat != d->targetSWFormat || needToScale) { auto resolution = d->settings.videoResolution(); qCDebug(qLcVideoFrameEncoder) << "camera and encoder use different formats:" << d->sourceSWFormat << d->targetSWFormat; + d->converter = sws_getContext(d->sourceSize.width(), d->sourceSize.height(), d->sourceSWFormat, resolution.width(), resolution.height(), d->targetSWFormat, SWS_FAST_BILINEAR, nullptr, nullptr, nullptr); } + + qCDebug(qLcVideoFrameEncoder) << "VideoFrameEncoder conversions initialized:" + << "sourceFormat:" << d->sourceFormat + << (d->sourceFormatIsHWFormat ? "(hw)" : "(sw)") + << "targetFormat:" << d->targetFormat + << (d->targetFormatIsHWFormat ? "(hw)" : "(sw)") + << "sourceSWFormat:" << d->sourceSWFormat + << "targetSWFormat:" << d->targetSWFormat; } VideoFrameEncoder::~VideoFrameEncoder() @@ -303,7 +312,6 @@ int VideoFrameEncoder::sendFrame(AVFrameUPtr frame) if (d->downloadFromHW) { auto f = makeAVFrame(); - f->format = d->sourceSWFormat; int err = av_hwframe_transfer_data(f.get(), frame.get(), 0); if (err < 0) { qCDebug(qLcVideoFrameEncoder) << "Error transferring frame data to surface." << err2str(err); @@ -319,8 +327,14 @@ int VideoFrameEncoder::sendFrame(AVFrameUPtr frame) f->format = d->targetSWFormat; f->width = d->settings.videoResolution().width(); f->height = d->settings.videoResolution().height(); + av_frame_get_buffer(f.get(), 0); - sws_scale(d->converter, frame->data, frame->linesize, 0, f->height, f->data, f->linesize); + const auto scaledHeight = sws_scale(d->converter, frame->data, frame->linesize, 0, + frame->height, f->data, f->linesize); + + if (scaledHeight != f->height) + qCWarning(qLcVideoFrameEncoder) << "Scaled height" << scaledHeight << "!=" << f->height; + frame = std::move(f); } diff --git a/src/plugins/multimedia/ffmpeg/qx11screencapture_p.h b/src/plugins/multimedia/ffmpeg/qx11screencapture_p.h index 78e302c9a..0930dde87 100644 --- a/src/plugins/multimedia/ffmpeg/qx11screencapture_p.h +++ b/src/plugins/multimedia/ffmpeg/qx11screencapture_p.h @@ -35,7 +35,6 @@ protected: bool setActiveInternal(bool active) override; private: - std::optional<QVideoFrameFormat> m_format; std::unique_ptr<Grabber> m_grabber; }; diff --git a/tests/auto/integration/qcamerabackend/tst_qcamerabackend.cpp b/tests/auto/integration/qcamerabackend/tst_qcamerabackend.cpp index 05dd1def1..0f9ebab6f 100644 --- a/tests/auto/integration/qcamerabackend/tst_qcamerabackend.cpp +++ b/tests/auto/integration/qcamerabackend/tst_qcamerabackend.cpp @@ -9,6 +9,7 @@ #include <QtCore/qlocale.h> #include <QDebug> #include <QVideoSink> +#include <QMediaPlayer> #include <private/qplatformcamera_p.h> #include <private/qplatformimagecapture_p.h> @@ -571,32 +572,39 @@ void tst_QCameraBackend::testVideoRecording() QTRY_VERIFY(camera->isActive()); QTRY_VERIFY(camera->isActive()); - for (int recordings = 0; recordings < 2; ++recordings) { - //record 200ms clip - recorder.record(); - durationChanged.clear(); - if (!recorderErrorSignal.empty() || recorderErrorSignal.wait(100)) { - QCOMPARE(recorderErrorSignal.last().first().toInt(), QMediaRecorder::FormatError); - break; - } - QTRY_VERIFY(durationChanged.size()); + recorder.record(); + durationChanged.clear(); + if (!recorderErrorSignal.empty() || recorderErrorSignal.wait(550)) { + QCOMPARE(recorderErrorSignal.last().first().toInt(), QMediaRecorder::FormatError); + return; + } - QCOMPARE(recorder.metaData(), metaData); + QTRY_VERIFY(durationChanged.size()); - recorderStateChanged.clear(); - recorder.stop(); - QTRY_VERIFY(recorderStateChanged.size() > 0); - QVERIFY(recorder.recorderState() == QMediaRecorder::StoppedState); + QCOMPARE(recorder.metaData(), metaData); - QVERIFY(errorSignal.isEmpty()); - QVERIFY(recorderErrorSignal.isEmpty()); + recorderStateChanged.clear(); + recorder.stop(); + QTRY_VERIFY(recorderStateChanged.size() > 0); + QVERIFY(recorder.recorderState() == QMediaRecorder::StoppedState); - QString fileName = recorder.actualLocation().toLocalFile(); - QVERIFY(!fileName.isEmpty()); - QVERIFY(QFileInfo(fileName).size() > 0); - QFile(fileName).remove(); - } + QVERIFY(errorSignal.isEmpty()); + QVERIFY(recorderErrorSignal.isEmpty()); + + QString fileName = recorder.actualLocation().toLocalFile(); + QVERIFY(!fileName.isEmpty()); + QVERIFY(QFileInfo(fileName).size() > 0); + + QMediaPlayer player; + player.setSource(fileName); + QCOMPARE_EQ(player.metaData().value(QMediaMetaData::Resolution).toSize(), QSize(320, 240)); + QCOMPARE_GT(player.duration(), 350); + QCOMPARE_LT(player.duration(), 550); + + // TODO: integrate with a virtual camera and check mediaplayer output + + QFile(fileName).remove(); } void tst_QCameraBackend::testNativeMetadata() diff --git a/tests/auto/integration/qscreencapture_integration/BLACKLIST b/tests/auto/integration/qscreencapture_integration/BLACKLIST index 97f518059..36abeb884 100644 --- a/tests/auto/integration/qscreencapture_integration/BLACKLIST +++ b/tests/auto/integration/qscreencapture_integration/BLACKLIST @@ -1,3 +1,7 @@ macos ci windows ci android ci + +#QTBUG-111190, v4l2m2m issues +[recordToFile] +linux ci diff --git a/tests/auto/integration/qscreencapture_integration/tst_qscreencapture_integration.cpp b/tests/auto/integration/qscreencapture_integration/tst_qscreencapture_integration.cpp index 4e7b7ea1f..42a45c1f1 100644 --- a/tests/auto/integration/qscreencapture_integration/tst_qscreencapture_integration.cpp +++ b/tests/auto/integration/qscreencapture_integration/tst_qscreencapture_integration.cpp @@ -11,6 +11,8 @@ #include <qpainter.h> #include <qscreencapture.h> #include <qsignalspy.h> +#include <qmediarecorder.h> +#include <qmediaplayer.h> #include <vector> @@ -111,6 +113,7 @@ private slots: void captureScreen(); void captureScreenByDefault(); void captureSecondaryScreen(); + void recordToFile(); void removeScreenWhileCapture(); // Keep the test last defined. TODO: find a way to restore // application screens. @@ -270,6 +273,12 @@ void tst_QScreenCaptureIntegration::removeWhileCapture( void tst_QScreenCaptureIntegration::initTestCase() { +#if defined(Q_OS_LINUX) + if (qEnvironmentVariable("QTEST_ENVIRONMENT").toLower() == "ci" && + qEnvironmentVariable("XDG_SESSION_TYPE").toLower() != "x11") + QSKIP("Skip on wayland; to be fixed"); +#endif + if (!QApplication::primaryScreen()) QSKIP("No screens found"); @@ -320,6 +329,61 @@ void tst_QScreenCaptureIntegration::captureSecondaryScreen() [&screens](QScreenCapture &sc) { sc.setScreen(screens.back()); }); } +void tst_QScreenCaptureIntegration::recordToFile() +{ + QScreenCapture sc; + QSignalSpy errorsSpy(&sc, &QScreenCapture::errorOccurred); + // sc.setScreen(screen); + QMediaCaptureSession session; + QMediaRecorder recorder; + session.setScreenCapture(&sc); + session.setRecorder(&recorder); + recorder.setVideoResolution(1280, 960); + + // Insert metadata + QMediaMetaData metaData; + metaData.insert(QMediaMetaData::Author, QString::fromUtf8("Author")); + metaData.insert(QMediaMetaData::Date, QDateTime::currentDateTime()); + recorder.setMetaData(metaData); + sc.setActive(true); + + QTest::qWait(200); // wait a bit for SC threading activating + + { + QSignalSpy recorderStateChanged(&recorder, &QMediaRecorder::recorderStateChanged); + + recorder.record(); + + QTRY_VERIFY(!recorderStateChanged.empty()); + QCOMPARE(recorder.recorderState(), QMediaRecorder::RecordingState); + } + + QTest::qWait(600); + + { + QSignalSpy recorderStateChanged(&recorder, &QMediaRecorder::recorderStateChanged); + + recorder.stop(); + + QTRY_VERIFY(!recorderStateChanged.empty()); + QCOMPARE(recorder.recorderState(), QMediaRecorder::StoppedState); + } + + QString fileName = recorder.actualLocation().toLocalFile(); + QVERIFY(!fileName.isEmpty()); + QVERIFY(QFileInfo(fileName).size() > 0); + + QMediaPlayer player; + player.setSource(fileName); + QCOMPARE_EQ(player.metaData().value(QMediaMetaData::Resolution).toSize(), QSize(1280, 960)); + QCOMPARE_GT(player.duration(), 350); + QCOMPARE_LT(player.duration(), 650); + + // TODO: check frames changes with QMediaPlayer + + QFile(fileName).remove(); +} + void tst_QScreenCaptureIntegration::removeScreenWhileCapture() { QSKIP("TODO: find a reliable way to emulate it"); |