diff options
author | Artem Dyomin <artem.dyomin@qt.io> | 2023-03-30 18:54:44 +0200 |
---|---|---|
committer | Qt Cherry-pick Bot <cherrypick_bot@qt-project.org> | 2023-04-12 11:59:25 +0000 |
commit | 962a4f93ace92deb02ab5590be8854532609d255 (patch) | |
tree | c2fdb4bf08d74253dadfd291f8b4527927c7d676 | |
parent | 8108d06690087f610863e09863a9ef2525a65598 (diff) | |
download | qtmultimedia-962a4f93ace92deb02ab5590be8854532609d255.tar.gz |
Implement seamless ffmpeg playback looping
Users need seamless looping, in other words, looping without
little delays on jumping from the media end to the start.
The only way to make it seamles and smooth is to intrude into the
playback engine. As result, we have just a regular delay between frames
on shifting.
Also, a bunch of adjuscent small improvements have been introduced:
- simplify criteria of the demuxer's buffer size, it was more complex
and a bit not relevant before. Actually, we need to check
only max buffering time.
- Improve handling of media with streams of different length.
- Fix setting of the playback rate before opening sources.
- Add new auto tests for looping.
- Add debug logs.
Task-number: QTBUG-112305
Change-Id: Ic9073d77535f5aae19f9ea48d4e745c9f2fa9ea3
Reviewed-by: Lars Knoll <lars@knoll.priv.no>
(cherry picked from commit 57ebebae16d9b4d984709541a1b37e711f48b14d)
Reviewed-by: Qt Cherry-pick Bot <cherrypick_bot@qt-project.org>
16 files changed, 447 insertions, 119 deletions
diff --git a/src/multimedia/platform/qplatformmediaplayer_p.h b/src/multimedia/platform/qplatformmediaplayer_p.h index a04565668..b2cd5e571 100644 --- a/src/multimedia/platform/qplatformmediaplayer_p.h +++ b/src/multimedia/platform/qplatformmediaplayer_p.h @@ -116,7 +116,8 @@ public: return isSeekable() && (m_loops < 0 || ++m_currentLoop < m_loops); } int loops() { return m_loops; } - void setLoops(int loops) { + virtual void setLoops(int loops) + { if (m_loops == loops) return; m_loops = loops; diff --git a/src/plugins/multimedia/ffmpeg/CMakeLists.txt b/src/plugins/multimedia/ffmpeg/CMakeLists.txt index 7dee4f557..49775aa42 100644 --- a/src/plugins/multimedia/ffmpeg/CMakeLists.txt +++ b/src/plugins/multimedia/ffmpeg/CMakeLists.txt @@ -52,6 +52,7 @@ qt_internal_add_plugin(QFFmpegMediaPlugin playbackengine/qffmpegcodec.cpp playbackengine/qffmpegcodec_p.h playbackengine/qffmpegpacket_p.h playbackengine/qffmpegframe_p.h + playbackengine/qffmpegpositionwithoffset_p.h DEFINES QT_COMPILING_FFMPEG LIBRARIES diff --git a/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegdemuxer.cpp b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegdemuxer.cpp index b80cd47c4..fdf987b60 100644 --- a/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegdemuxer.cpp +++ b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegdemuxer.cpp @@ -2,23 +2,46 @@ // SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "playbackengine/qffmpegdemuxer_p.h" +#include <qloggingcategory.h> QT_BEGIN_NAMESPACE -// queue up max 16M of encoded data, that should always be enough -// (it's around 2 secs of 4K HDR video, longer for almost all other formats) -static constexpr quint64 MaxQueueSize = 16 * 1024 * 1024; +// 4 sec for buffering. TODO: maybe move to env var customization +static constexpr qint64 MaxBufferingTimeUs = 4'000'000; + +// Currently, consider only time. TODO: maybe move to env var customization +static constexpr qint64 MaxBufferingSize = std::numeric_limits<qint64>::max(); namespace QFFmpeg { -Demuxer::Demuxer(AVFormatContext *context, qint64 seekPos, const StreamIndexes &streamIndexes) - : m_context(context), m_seekPos(seekPos) +static Q_LOGGING_CATEGORY(qLcDemuxer, "qt.multimedia.ffmpeg.demuxer"); + +static qint64 streamTimeToUs(const AVStream *stream, qint64 time) +{ + Q_ASSERT(stream); + + const auto res = mul(time * 1000000, stream->time_base); + return res ? *res : time; +} + +Demuxer::Demuxer(AVFormatContext *context, const PositionWithOffset &posWithOffset, + const StreamIndexes &streamIndexes, int loops) + : m_context(context), m_posWithOffset(posWithOffset) { + qCDebug(qLcDemuxer) << "Create demuxer." + << "pos:" << posWithOffset.pos << "loop offset:" << posWithOffset.offset.pos + << "loop index:" << posWithOffset.offset.index << "loops:" << loops; + m_loops = loops; + Q_ASSERT(m_context); + Q_ASSERT(m_loops < 0 || m_posWithOffset.offset.index < m_loops); for (auto i = 0; i < QPlatformMediaPlayer::NTrackTypes; ++i) { - if (streamIndexes[i] >= 0) - m_streams[streamIndexes[i]] = { static_cast<QPlatformMediaPlayer::TrackType>(i) }; + if (streamIndexes[i] >= 0) { + const auto trackType = static_cast<QPlatformMediaPlayer::TrackType>(i); + qCDebug(qLcDemuxer) << "Activate demuxing stream" << i << ", trackType:" << trackType; + m_streams[streamIndexes[i]] = { trackType }; + } } } @@ -26,33 +49,43 @@ void Demuxer::doNextStep() { ensureSeeked(); - Packet packet(AVPacketUPtr{ av_packet_alloc() }); + Packet packet(m_posWithOffset.offset, AVPacketUPtr{ av_packet_alloc() }); if (av_read_frame(m_context, packet.avPacket()) < 0) { - setAtEnd(true); + ++m_posWithOffset.offset.index; + + if (m_loops >= 0 && m_posWithOffset.offset.index >= m_loops) { + qCDebug(qLcDemuxer) << "finish demuxing"; + setAtEnd(true); + } else { + m_seeked = false; + m_posWithOffset.pos = 0; + m_posWithOffset.offset.pos = m_endPts; + m_endPts = 0; + + ensureSeeked(); + + qCDebug(qLcDemuxer) << "Demuxer loops changed. Index:" << m_posWithOffset.offset.index + << "Offset:" << m_posWithOffset.offset.pos; + } + return; } const auto streamIndex = packet.avPacket()->stream_index; + const auto stream = m_context->streams[streamIndex]; auto it = m_streams.find(streamIndex); if (it != m_streams.end()) { - it->second.dataSize += packet.avPacket()->size; - it->second.duration += packet.avPacket()->duration; - - switch (it->second.trackType) { - case QPlatformMediaPlayer::TrackType::VideoStream: - emit requestProcessVideoPacket(packet); - break; - case QPlatformMediaPlayer::TrackType::AudioStream: - emit requestProcessAudioPacket(packet); - break; - case QPlatformMediaPlayer::TrackType::SubtitleStream: - emit requestProcessSubtitlePacket(packet); - break; - default: - Q_ASSERT(!"Unknown track type"); - } + const auto packetEndPos = + streamTimeToUs(stream, packet.avPacket()->pts + packet.avPacket()->duration); + m_endPts = std::max(m_endPts, m_posWithOffset.offset.pos + packetEndPos); + + it->second.bufferingTime += streamTimeToUs(stream, packet.avPacket()->duration); + it->second.bufferingSize += packet.avPacket()->size; + + auto signal = signalByTrackType(it->second.trackType); + emit (this->*signal)(packet); } scheduleNextStep(false); @@ -65,11 +98,12 @@ void Demuxer::onPacketProcessed(Packet packet) auto it = m_streams.find(streamIndex); if (it != m_streams.end()) { - it->second.dataSize -= packet.avPacket()->size; - it->second.duration -= packet.avPacket()->duration; + it->second.bufferingTime -= + streamTimeToUs(m_context->streams[streamIndex], packet.avPacket()->duration); + it->second.bufferingSize -= packet.avPacket()->size; - Q_ASSERT(it->second.dataSize >= 0); - Q_ASSERT(it->second.duration >= 0); + Q_ASSERT(it->second.bufferingTime >= 0); + Q_ASSERT(it->second.bufferingSize >= 0); } } @@ -81,21 +115,12 @@ bool Demuxer::canDoNextStep() const if (!PlaybackEngineObject::canDoNextStep() || isAtEnd() || m_streams.empty()) return false; - const bool hasSmallDuration = - std::any_of(m_streams.begin(), m_streams.end(), - [](const auto &s) { return s.second.duration < 200; }); - - if (hasSmallDuration) - return true; - - const auto dataSize = - std::accumulate(m_streams.begin(), m_streams.end(), quint64(0), - [](quint64 value, const auto &s) { return value + s.second.dataSize; }); + auto checkBufferingTime = [](const auto &streamIndexToData) { + return streamIndexToData.second.bufferingTime < MaxBufferingTimeUs && + streamIndexToData.second.bufferingSize < MaxBufferingSize; + }; - if (dataSize > MaxQueueSize) - return false; - - return true; + return std::all_of(m_streams.begin(), m_streams.end(), checkBufferingTime); } void Demuxer::ensureSeeked() @@ -103,7 +128,7 @@ void Demuxer::ensureSeeked() if (std::exchange(m_seeked, true)) return; - const qint64 seekPos = m_seekPos * AV_TIME_BASE / 1000000; + const qint64 seekPos = m_posWithOffset.pos * AV_TIME_BASE / 1000000; auto err = av_seek_frame(m_context, -1, seekPos, AVSEEK_FLAG_BACKWARD); if (err < 0) { @@ -132,6 +157,12 @@ Demuxer::RequestingSignal Demuxer::signalByTrackType(QPlatformMediaPlayer::Track return nullptr; } +void Demuxer::setLoops(int loopsCount) +{ + qCDebug(qLcDemuxer) << "setLoops to demuxer" << loopsCount; + m_loops = loopsCount; +} + } // namespace QFFmpeg QT_END_NAMESPACE diff --git a/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegdemuxer_p.h b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegdemuxer_p.h index 6bfd4431f..290626af2 100644 --- a/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegdemuxer_p.h +++ b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegdemuxer_p.h @@ -17,6 +17,7 @@ #include "playbackengine/qffmpegplaybackengineobject_p.h" #include "private/qplatformmediaplayer_p.h" #include "playbackengine/qffmpegpacket_p.h" +#include "playbackengine/qffmpegpositionwithoffset_p.h" #include <unordered_map> @@ -28,11 +29,14 @@ class Demuxer : public PlaybackEngineObject { Q_OBJECT public: - Demuxer(AVFormatContext *context, qint64 seekPos, const StreamIndexes &streamIndexes); + Demuxer(AVFormatContext *context, const PositionWithOffset &posWithOffset, + const StreamIndexes &streamIndexes, int loops); using RequestingSignal = void (Demuxer::*)(Packet); static RequestingSignal signalByTrackType(QPlatformMediaPlayer::TrackType trackType); + void setLoops(int loopsCount); + public slots: void onPacketProcessed(Packet); @@ -52,14 +56,16 @@ private: struct StreamData { QPlatformMediaPlayer::TrackType trackType = QPlatformMediaPlayer::TrackType::NTrackTypes; - qint64 duration = 0; - qint64 dataSize = 0; + qint64 bufferingTime = 0; + qint64 bufferingSize = 0; }; AVFormatContext *m_context = nullptr; bool m_seeked = false; std::unordered_map<int, StreamData> m_streams; - const qint64 m_seekPos = 0; + PositionWithOffset m_posWithOffset; + qint64 m_endPts = 0; + std::atomic<int> m_loops = QMediaPlayer::Once; }; } // namespace QFFmpeg diff --git a/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegframe_p.h b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegframe_p.h index bccb13ebe..459ceb9b4 100644 --- a/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegframe_p.h +++ b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegframe_p.h @@ -17,6 +17,7 @@ #include "qffmpeg_p.h" #include "playbackengine/qffmpegcodec_p.h" +#include "playbackengine/qffmpegpositionwithoffset_p.h" #include "QtCore/qsharedpointer.h" #include "qpointer.h" #include "qobject.h" @@ -31,8 +32,9 @@ struct Frame { struct Data { - Data(AVFrameUPtr f, const Codec &codec, qint64, const QObject *source) - : codec(codec), frame(std::move(f)), source(source) + Data(const LoopOffset &offset, AVFrameUPtr f, const Codec &codec, qint64, + const QObject *source) + : loopOffset(offset), codec(codec), frame(std::move(f)), source(source) { Q_ASSERT(frame); if (frame->pts != AV_NOPTS_VALUE) @@ -44,12 +46,14 @@ struct Frame ? (1000000 * avgFrameRate.den + avgFrameRate.num / 2) / avgFrameRate.num : 0; } - Data(const QString &text, qint64 pts, qint64 duration, const QObject *source) - : text(text), pts(pts), duration(duration), source(source) + Data(const LoopOffset &offset, const QString &text, qint64 pts, qint64 duration, + const QObject *source) + : loopOffset(offset), text(text), pts(pts), duration(duration), source(source) { } QAtomicInt ref; + LoopOffset loopOffset; std::optional<Codec> codec; AVFrameUPtr frame; QString text; @@ -59,12 +63,14 @@ struct Frame }; Frame() = default; - Frame(AVFrameUPtr f, const Codec &codec, qint64 pts, const QObject *source = nullptr) - : d(new Data(std::move(f), codec, pts, source)) + Frame(const LoopOffset &offset, AVFrameUPtr f, const Codec &codec, qint64 pts, + const QObject *source = nullptr) + : d(new Data(offset, std::move(f), codec, pts, source)) { } - Frame(const QString &text, qint64 pts, qint64 duration, const QObject *source = nullptr) - : d(new Data(text, pts, duration, source)) + Frame(const LoopOffset &offset, const QString &text, qint64 pts, qint64 duration, + const QObject *source = nullptr) + : d(new Data(offset, text, pts, duration, source)) { } bool isValid() const { return !!d; } @@ -77,6 +83,9 @@ struct Frame qint64 end() const { return data().pts + data().duration; } QString text() const { return data().text; } const QObject *source() const { return data().source; }; + const LoopOffset &loopOffset() const { return data().loopOffset; }; + qint64 absolutePts() const { return pts() + loopOffset().pos; } + qint64 absoluteEnd() const { return end() + loopOffset().pos; } private: Data &data() const diff --git a/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegpacket_p.h b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegpacket_p.h index 5e489b3cd..3d037a72b 100644 --- a/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegpacket_p.h +++ b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegpacket_p.h @@ -17,6 +17,7 @@ #include "qffmpeg_p.h" #include "QtCore/qsharedpointer.h" +#include "playbackengine/qffmpegpositionwithoffset_p.h" QT_BEGIN_NAMESPACE @@ -26,16 +27,19 @@ struct Packet { struct Data { - Data(AVPacketUPtr p) : packet(std::move(p)) { } + Data(const LoopOffset &offset, AVPacketUPtr p) + : loopOffset(offset), packet(std::move(p)) { } QAtomicInt ref; + LoopOffset loopOffset; AVPacketUPtr packet; }; Packet() = default; - Packet(AVPacketUPtr p) : d(new Data(std::move(p))) { } + Packet(const LoopOffset &offset, AVPacketUPtr p) : d(new Data(offset, std::move(p))) { } bool isValid() const { return !!d; } AVPacket *avPacket() const { return d->packet.get(); } + const LoopOffset &loopOffset() const { return d->loopOffset; } private: QExplicitlySharedDataPointer<Data> d; diff --git a/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegpositionwithoffset_p.h b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegpositionwithoffset_p.h new file mode 100644 index 000000000..a30fdc119 --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegpositionwithoffset_p.h @@ -0,0 +1,40 @@ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#ifndef qffmpegpositionwithoffset_p_H +#define qffmpegpositionwithoffset_p_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#include <qtypes.h> + +QT_BEGIN_NAMESPACE + +namespace QFFmpeg { + +struct LoopOffset +{ + qint64 pos = 0; + int index = 0; +}; + +struct PositionWithOffset +{ + qint64 pos = 0; + LoopOffset offset; +}; + +} // namespace QFFmpeg + +QT_END_NAMESPACE + +#endif // qffmpegpositionwithoffset_p_H diff --git a/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegrenderer.cpp b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegrenderer.cpp index d12f38e84..9258f9677 100644 --- a/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegrenderer.cpp +++ b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegrenderer.cpp @@ -2,11 +2,14 @@ // SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "playbackengine/qffmpegrenderer_p.h" +#include <qloggingcategory.h> QT_BEGIN_NAMESPACE namespace QFFmpeg { +static Q_LOGGING_CATEGORY(qLcRenderer, "qt.multimedia.ffmpeg.renderer"); + Renderer::Renderer(const TimeController &tc, const std::chrono::microseconds &seekPosTimeOffset) : m_timeController(tc), m_lastPosition(tc.currentPosition()), @@ -68,9 +71,11 @@ void Renderer::render(Frame frame) { using namespace std::chrono; - const auto isFrameOutdated = frame.isValid() && frame.end() < m_seekPos; + const auto isFrameOutdated = frame.isValid() && frame.absoluteEnd() < m_seekPos; if (isFrameOutdated) { + qCDebug(qLcRenderer) << "frame outdated! absEnd:" << frame.absoluteEnd() << "absPts" + << frame.absolutePts() << "seekPos:" << m_seekPos; emit frameProcessed(frame); return; } @@ -101,7 +106,8 @@ int Renderer::timerInterval() const { if (auto frame = m_frames.front(); frame.isValid() && !m_isStepForced) { using namespace std::chrono; - const auto delay = m_timeController.timeFromPosition(frame.pts()) - steady_clock::now(); + const auto delay = + m_timeController.timeFromPosition(frame.absolutePts()) - steady_clock::now(); return std::max(0, static_cast<int>(duration_cast<milliseconds>(delay).count())); } @@ -133,16 +139,23 @@ void Renderer::doNextStep() if (result.timeLeft.count() && frame.isValid()) { const auto now = std::chrono::steady_clock::now(); - m_timeController.sync(now + result.timeLeft, frame.pts()); - emit synchronized(now + result.timeLeft, frame.pts()); + m_timeController.sync(now + result.timeLeft, frame.absolutePts()); + emit synchronized(now + result.timeLeft, frame.absolutePts()); } if (done) { m_frames.dequeue(); if (frame.isValid()) { - m_lastPosition = std::max(frame.pts(), m_lastPosition.load()); - m_seekPos = frame.end(); + m_lastPosition = std::max(frame.absolutePts(), m_lastPosition.load()); + m_seekPos = frame.absoluteEnd(); + + const auto loopIndex = frame.loopOffset().index; + if (m_loopIndex < loopIndex) { + m_loopIndex = loopIndex; + emit loopChanged(frame.loopOffset().pos, m_loopIndex); + } + emit frameProcessed(frame); } } diff --git a/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegrenderer_p.h b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegrenderer_p.h index 23ccab883..8949213a4 100644 --- a/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegrenderer_p.h +++ b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegrenderer_p.h @@ -55,6 +55,8 @@ signals: void forceStepDone(); + void loopChanged(qint64 offset, int index); + protected: bool setForceStepDone(); @@ -82,6 +84,7 @@ private: TimeController m_timeController; std::atomic<qint64> m_lastPosition = 0; std::atomic<qint64> m_seekPos = 0; + int m_loopIndex = 0; QQueue<Frame> m_frames; std::atomic_bool m_isStepForced = false; diff --git a/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegstreamdecoder.cpp b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegstreamdecoder.cpp index 01dfb1f84..67151618b 100644 --- a/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegstreamdecoder.cpp +++ b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegstreamdecoder.cpp @@ -3,16 +3,21 @@ #include "playbackengine/qffmpegstreamdecoder_p.h" #include "playbackengine/qffmpegmediadataholder_p.h" +#include <qloggingcategory.h> QT_BEGIN_NAMESPACE +static Q_LOGGING_CATEGORY(qLcStreamDecoder, "qt.multimedia.ffmpeg.streamdecoder"); + namespace QFFmpeg { -StreamDecoder::StreamDecoder(const Codec &codec, qint64 seekPos) +StreamDecoder::StreamDecoder(const Codec &codec, qint64 absSeekPos) : m_codec(codec), - m_seekPos(seekPos), + m_absSeekPos(absSeekPos), m_trackType(MediaDataHolder::trackTypeFromMediaType(codec.context()->codec_type)) { + qCDebug(qLcStreamDecoder) << "Create stream decoder, trackType" << m_trackType + << "absSeekPos:" << absSeekPos; Q_ASSERT(m_trackType != QPlatformMediaPlayer::NTrackTypes); } @@ -36,10 +41,24 @@ void StreamDecoder::decode(Packet packet) void StreamDecoder::doNextStep() { auto packet = m_packets.dequeue(); - if (trackType() == QPlatformMediaPlayer::SubtitleStream) - decodeSubtitle(packet); - else - decodeMedia(packet); + + auto decodePacket = [this](Packet packet) { + if (trackType() == QPlatformMediaPlayer::SubtitleStream) + decodeSubtitle(packet); + else + decodeMedia(packet); + }; + + if (packet.isValid() && packet.loopOffset().index != m_offset.index) { + decodePacket({}); + + qCDebug(qLcStreamDecoder) << "flush buffers due to new loop:" << packet.loopOffset().index; + + avcodec_flush_buffers(m_codec.context()); + m_offset = packet.loopOffset(); + } + + decodePacket(packet); setAtEnd(!packet.isValid()); @@ -82,7 +101,7 @@ bool StreamDecoder::canDoNextStep() const void StreamDecoder::onFrameFound(Frame frame) { - if (frame.isValid() && frame.end() < m_seekPos) + if (frame.isValid() && frame.absoluteEnd() < m_absSeekPos) return; Q_ASSERT(m_pendingFramesCount >= 0); @@ -131,7 +150,7 @@ void StreamDecoder::receiveAVFrames() break; } - onFrameFound({ std::move(avFrame), m_codec, 0, this }); + onFrameFound({ m_offset, std::move(avFrame), m_codec, 0, this }); } } @@ -196,10 +215,10 @@ void StreamDecoder::decodeSubtitle(Packet packet) if (text.endsWith(QLatin1Char('\n'))) text.chop(1); - onFrameFound({ text, start, end - start, this }); + onFrameFound({ m_offset, text, start, end - start, this }); // TODO: maybe optimize - onFrameFound({ QString(), end, 0, this }); + onFrameFound({ m_offset, QString(), end, 0, this }); } } // namespace QFFmpeg diff --git a/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegstreamdecoder_p.h b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegstreamdecoder_p.h index cc12b1e9a..001ec6b90 100644 --- a/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegstreamdecoder_p.h +++ b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegstreamdecoder_p.h @@ -16,6 +16,7 @@ #include "playbackengine/qffmpegplaybackengineobject_p.h" #include "playbackengine/qffmpegframe_p.h" #include "playbackengine/qffmpegpacket_p.h" +#include "playbackengine/qffmpegpositionwithoffset_p.h" #include "private/qplatformmediaplayer_p.h" #include <optional> @@ -28,7 +29,7 @@ class StreamDecoder : public PlaybackEngineObject { Q_OBJECT public: - StreamDecoder(const Codec &codec, qint64 seekPos); + StreamDecoder(const Codec &codec, qint64 absSeekPos); ~StreamDecoder(); @@ -64,11 +65,13 @@ private: private: Codec m_codec; - qint64 m_seekPos = 0; + const qint64 m_absSeekPos = 0; const QPlatformMediaPlayer::TrackType m_trackType; qint32 m_pendingFramesCount = 0; + LoopOffset m_offset; + QQueue<Packet> m_packets; }; diff --git a/src/plugins/multimedia/ffmpeg/qffmpegmediaplayer.cpp b/src/plugins/multimedia/ffmpeg/qffmpegmediaplayer.cpp index 507550ab5..171f082d5 100644 --- a/src/plugins/multimedia/ffmpeg/qffmpegmediaplayer.cpp +++ b/src/plugins/multimedia/ffmpeg/qffmpegmediaplayer.cpp @@ -53,15 +53,21 @@ void QFFmpegMediaPlayer::endOfStream() m_positionUpdateTimer.stop(); positionChanged(duration()); - if (doLoop()) { - m_playbackEngine->seek(0); - positionChanged(0); + stateChanged(QMediaPlayer::StoppedState); + mediaStatusChanged(QMediaPlayer::EndOfMedia); +} - runPlayback(); - } else { - stateChanged(QMediaPlayer::StoppedState); - mediaStatusChanged(QMediaPlayer::EndOfMedia); - } +void QFFmpegMediaPlayer::onLoopChanged() +{ + // report about finish and start + // reporting both signals is a bit contraversial + // but it eshures the idea of notifications about + // imporatant position points. + // Also, it ensures more predictable flow for testing. + positionChanged(duration()); + positionChanged(0); + m_positionUpdateTimer.stop(); + m_positionUpdateTimer.start(); } float QFFmpegMediaPlayer::bufferProgress() const @@ -121,10 +127,13 @@ void QFFmpegMediaPlayer::setMedia(const QUrl &media, QIODevice *stream) mediaStatusChanged(QMediaPlayer::LoadingMedia); m_playbackEngine = std::make_unique<PlaybackEngine>(); + connect(m_playbackEngine.get(), &PlaybackEngine::endOfStream, this, &QFFmpegMediaPlayer::endOfStream); connect(m_playbackEngine.get(), &PlaybackEngine::errorOccured, this, &QFFmpegMediaPlayer::error); + connect(m_playbackEngine.get(), &PlaybackEngine::loopChanged, this, + &QFFmpegMediaPlayer::onLoopChanged); if (!m_playbackEngine->setMedia(media, stream)) { m_playbackEngine.reset(); @@ -134,6 +143,8 @@ void QFFmpegMediaPlayer::setMedia(const QUrl &media, QIODevice *stream) m_playbackEngine->setAudioSink(m_audioOutput); m_playbackEngine->setVideoSink(m_videoSink); + m_playbackEngine->setLoops(loops()); + m_playbackEngine->setPlaybackRate(m_playbackRate); durationChanged(duration()); tracksChanged(); @@ -145,6 +156,7 @@ void QFFmpegMediaPlayer::setMedia(const QUrl &media, QIODevice *stream) videoAvailableChanged( !m_playbackEngine->streamInfo(QPlatformMediaPlayer::VideoStream).isEmpty()); + // TODO: get rid of the delayed update QMetaObject::invokeMethod(this, "delayedLoadedStatus", Qt::QueuedConnection); } @@ -156,7 +168,6 @@ void QFFmpegMediaPlayer::play() if (mediaStatus() == QMediaPlayer::EndOfMedia && state() == QMediaPlayer::StoppedState) { m_playbackEngine->seek(0); positionChanged(0); - resetCurrentLoop(); } runPlayback(); @@ -247,6 +258,16 @@ void QFFmpegMediaPlayer::setActiveTrack(TrackType type, int streamNumber) { if (m_playbackEngine) m_playbackEngine->setActiveTrack(type, streamNumber); + else + qWarning() << "Cannot set active track without open source"; +} + +void QFFmpegMediaPlayer::setLoops(int loops) +{ + if (m_playbackEngine) + m_playbackEngine->setLoops(loops); + + QPlatformMediaPlayer::setLoops(loops); } QT_END_NAMESPACE diff --git a/src/plugins/multimedia/ffmpeg/qffmpegmediaplayer_p.h b/src/plugins/multimedia/ffmpeg/qffmpegmediaplayer_p.h index 7e9126a22..b330948f6 100644 --- a/src/plugins/multimedia/ffmpeg/qffmpegmediaplayer_p.h +++ b/src/plugins/multimedia/ffmpeg/qffmpegmediaplayer_p.h @@ -65,8 +65,12 @@ public: QMediaMetaData trackMetaData(TrackType type, int streamNumber) override; int activeTrack(TrackType) override; void setActiveTrack(TrackType, int streamNumber) override; + void setLoops(int loops) override; - Q_INVOKABLE void delayedLoadedStatus() { mediaStatusChanged(QMediaPlayer::LoadedMedia); } + Q_INVOKABLE void delayedLoadedStatus() { + if (mediaStatus() == QMediaPlayer::LoadingMedia) + mediaStatusChanged(QMediaPlayer::LoadedMedia); + } private: void runPlayback(); @@ -78,6 +82,7 @@ private slots: { QPlatformMediaPlayer::error(error, errorString); } + void onLoopChanged(); private: QTimer m_positionUpdateTimer; diff --git a/src/plugins/multimedia/ffmpeg/qffmpegplaybackengine.cpp b/src/plugins/multimedia/ffmpeg/qffmpegplaybackengine.cpp index 19105499d..c9c4f20f2 100644 --- a/src/plugins/multimedia/ffmpeg/qffmpegplaybackengine.cpp +++ b/src/plugins/multimedia/ffmpeg/qffmpegplaybackengine.cpp @@ -18,6 +18,8 @@ QT_BEGIN_NAMESPACE namespace QFFmpeg { +static Q_LOGGING_CATEGORY(qLcPlaybackEngine, "qt.multimedia.ffmpeg.playbackengine"); + // The helper is needed since on some compilers std::unique_ptr // doesn't have a default constructor in the case of sizeof(CustomDeleter) > 0 template<typename Array> @@ -41,11 +43,13 @@ PlaybackEngine::PlaybackEngine() m_streams(defaultObjectsArray<decltype(m_streams)>()), m_renderers(defaultObjectsArray<decltype(m_renderers)>()) { + qCDebug(qLcPlaybackEngine) << "Create PlaybackEngine"; qRegisterMetaType<QFFmpeg::Packet>(); qRegisterMetaType<QFFmpeg::Frame>(); } PlaybackEngine::~PlaybackEngine() { + qCDebug(qLcPlaybackEngine) << "Delete PlaybackEngine"; forEachExistingObject([](auto &object) { object.reset(); }); deleteFreeThreads(); } @@ -62,22 +66,33 @@ void PlaybackEngine::onRendererFinished() if (!isAtEnd(QPlatformMediaPlayer::AudioStream)) return; - if (!isAtEnd(QPlatformMediaPlayer::SubtitleStream) - && !m_renderers[QPlatformMediaPlayer::AudioStream] - && !m_renderers[QPlatformMediaPlayer::VideoStream]) + if (!isAtEnd(QPlatformMediaPlayer::SubtitleStream) && !hasMediaStream()) return; if (std::exchange(m_state, QMediaPlayer::StoppedState) == QMediaPlayer::StoppedState) return; - m_timeController.setPaused(true); - m_timeController.sync(m_duration); + finilizeTime(duration()); forceUpdate(); + qCDebug(qLcPlaybackEngine) << "Playback engine end of stream"; + emit endOfStream(); } +void PlaybackEngine::onRendererLoopChanged(qint64 offset, int loopIndex) +{ + if (loopIndex > m_currentLoopOffset.index) { + m_currentLoopOffset = { offset, loopIndex }; + emit loopChanged(); + } else if (loopIndex == m_currentLoopOffset.index && offset != m_currentLoopOffset.pos) { + qWarning() << "Unexpected offset for loop" << loopIndex << ":" << offset << "vs" + << m_currentLoopOffset.pos; + m_currentLoopOffset.pos = offset; + } +} + void PlaybackEngine::onRendererSynchronized(std::chrono::steady_clock::time_point tp, qint64 pos) { Q_ASSERT(QObject::sender() == m_renderers[QPlatformMediaPlayer::AudioStream].get()); @@ -94,15 +109,13 @@ void PlaybackEngine::setState(QMediaPlayer::PlaybackState state) { if (!m_context) return; - if ( state == m_state ) + if (state == m_state) return; const auto prevState = std::exchange(m_state, state); - if (m_state == QMediaPlayer::StoppedState) { - m_timeController.setPaused(true); - m_timeController.sync(); - } + if (m_state == QMediaPlayer::StoppedState) + finilizeTime(0); if (prevState == QMediaPlayer::StoppedState || m_state == QMediaPlayer::StoppedState) recreateObjects(); @@ -208,14 +221,31 @@ void PlaybackEngine::forEachExistingObject(Action &&action) void PlaybackEngine::seek(qint64 pos) { - pos = qBound(0, pos, m_duration); + pos = qBound(0, pos, duration()); m_timeController.setPaused(true); - m_timeController.sync(pos); + m_timeController.sync(m_currentLoopOffset.pos + pos); forceUpdate(); } +void PlaybackEngine::setLoops(int loops) +{ + if (!isSeekable()) { + qWarning() << "Cannot set loops for non-seekable source"; + return; + } + + if (std::exchange(m_loops, loops) == loops) + return; + + qCDebug(qLcPlaybackEngine) << "set playback engine loops:" << loops << "prev loops:" << m_loops + << "index:" << m_currentLoopOffset.index; + + if (m_demuxer) + m_demuxer->setLoops(loops); +} + void PlaybackEngine::triggerStepIfNeeded() { if (m_state != QMediaPlayer::PausedState) @@ -295,6 +325,9 @@ void PlaybackEngine::createStreamAndRenderer(QPlatformMediaPlayer::TrackType tra connect(renderer.get(), &Renderer::synchronized, this, &PlaybackEngine::onRendererSynchronized); + connect(renderer.get(), &Renderer::loopChanged, this, + &PlaybackEngine::onRendererLoopChanged); + if constexpr (shouldPauseStreams) connect(renderer.get(), &Renderer::forceStepDone, this, &PlaybackEngine::updateObjectsPausedState); @@ -337,6 +370,8 @@ std::optional<Codec> PlaybackEngine::codecForTrack(QPlatformMediaPlayer::TrackTy auto &result = m_codecs[trackType]; if (!result) { + qCDebug(qLcPlaybackEngine) + << "Create codec for stream:" << streamIndex << "trackType:" << trackType; auto maybeCodec = Codec::create(m_context->streams[streamIndex]); if (!maybeCodec) { @@ -351,6 +386,12 @@ std::optional<Codec> PlaybackEngine::codecForTrack(QPlatformMediaPlayer::TrackTy return result; } +bool PlaybackEngine::hasMediaStream() const +{ + return m_renderers[QPlatformMediaPlayer::AudioStream] + || m_renderers[QPlatformMediaPlayer::VideoStream]; +} + void PlaybackEngine::createDemuxer() { decltype(m_currentAVStreamIndex) streamIndexes = { -1, -1, -1 }; @@ -365,8 +406,10 @@ void PlaybackEngine::createDemuxer() if (!hasStreams) return; - m_demuxer = - createPlaybackEngineObject<Demuxer>(m_context.get(), currentPosition(), streamIndexes); + const PositionWithOffset positionWithOffset{ currentPosition(false), m_currentLoopOffset }; + + m_demuxer = createPlaybackEngineObject<Demuxer>(m_context.get(), positionWithOffset, + streamIndexes, m_loops); forEachExistingObject<StreamDecoder>([&](auto &stream) { connect(m_demuxer.get(), Demuxer::signalByTrackType(stream->trackType()), stream.get(), @@ -425,14 +468,28 @@ void PlaybackEngine::setAudioSink(QAudioOutput *output) forceUpdate(); } -qint64 PlaybackEngine::currentPosition() const { - auto pos = std::numeric_limits<qint64>::max(); +qint64 PlaybackEngine::currentPosition(bool topPos) const { + std::optional<qint64> pos; + + for (size_t i = 0; i < m_renderers.size(); ++i) { + const auto &renderer = m_renderers[i]; + if (!renderer) + continue; + + // skip subtitle stream for finding lower rendering position + if (!topPos && i == QPlatformMediaPlayer::SubtitleStream && hasMediaStream()) + continue; + + const auto rendererPos = renderer->lastPosition(); + pos = !pos ? rendererPos + : topPos ? std::max(*pos, rendererPos) + : std::min(*pos, rendererPos); + } - for (auto trackType : { QPlatformMediaPlayer::VideoStream, QPlatformMediaPlayer::AudioStream }) - if (auto &renderer = m_renderers[trackType]) - pos = std::min(pos, renderer->lastPosition()); + if (!pos) + pos = m_timeController.currentPosition(); - return pos == std::numeric_limits<qint64>::max() ? m_timeController.currentPosition() : pos; + return qBound(0, *pos - m_currentLoopOffset.pos, duration()); } void PlaybackEngine::setActiveTrack(QPlatformMediaPlayer::TrackType trackType, int streamNumber) @@ -450,6 +507,14 @@ void PlaybackEngine::setActiveTrack(QPlatformMediaPlayer::TrackType trackType, i updateObjectsPausedState(); } +void PlaybackEngine::finilizeTime(qint64 pos) +{ + Q_ASSERT(pos >= 0 && pos <= duration()); + + m_timeController.setPaused(true); + m_timeController.sync(pos); + m_currentLoopOffset = {}; +} } QT_END_NAMESPACE diff --git a/src/plugins/multimedia/ffmpeg/qffmpegplaybackengine_p.h b/src/plugins/multimedia/ffmpeg/qffmpegplaybackengine_p.h index 682d9093b..4a61efa2f 100644 --- a/src/plugins/multimedia/ffmpeg/qffmpegplaybackengine_p.h +++ b/src/plugins/multimedia/ffmpeg/qffmpegplaybackengine_p.h @@ -49,6 +49,7 @@ #include "playbackengine/qffmpegtimecontroller_p.h" #include "playbackengine/qffmpegmediadataholder_p.h" #include "playbackengine/qffmpegcodec_p.h" +#include "playbackengine/qffmpegpositionwithoffset_p.h" #include <unordered_map> @@ -92,6 +93,8 @@ public: void seek(qint64 pos); + void setLoops(int loopsCount); + void setPlaybackRate(float rate); float playbackRate() const; @@ -100,11 +103,12 @@ public: using MediaDataHolder::activeTrack; - qint64 currentPosition() const; + qint64 currentPosition(bool topPos = true) const; signals: void endOfStream(); void errorOccured(int, const QString &); + void loopChanged(); protected: // objects managing struct ObjectDeleter @@ -152,12 +156,18 @@ private: void onRendererFinished(); + void onRendererLoopChanged(qint64 offset, int loopIndex); + void triggerStepIfNeeded(); static QString objectThreadName(const PlaybackEngineObject &object); std::optional<Codec> codecForTrack(QPlatformMediaPlayer::TrackType trackType); + bool hasMediaStream() const; + + void finilizeTime(qint64 pos); + private: TimeController m_timeController; @@ -174,6 +184,8 @@ private: std::array<RendererPtr, QPlatformMediaPlayer::NTrackTypes> m_renderers; std::array<std::optional<Codec>, QPlatformMediaPlayer::NTrackTypes> m_codecs; + int m_loops = QMediaPlayer::Once; + LoopOffset m_currentLoopOffset; }; template<typename T, typename... Args> diff --git a/tests/auto/integration/qmediaplayerbackend/tst_qmediaplayerbackend.cpp b/tests/auto/integration/qmediaplayerbackend/tst_qmediaplayerbackend.cpp index 8f01d30f6..170f5d680 100644 --- a/tests/auto/integration/qmediaplayerbackend/tst_qmediaplayerbackend.cpp +++ b/tests/auto/integration/qmediaplayerbackend/tst_qmediaplayerbackend.cpp @@ -74,7 +74,9 @@ private slots: void playbackRateChanging(); void durationDetectionIssues(); void finiteLoops(); - void infiteLoops(); + void infiniteLoops(); + void seekOnLoops(); + void changeLoopsOnTheFly(); void lazyLoadVideo(); private: @@ -1322,15 +1324,16 @@ void tst_QMediaPlayerBackend::playbackRateChanging() player.setVideoOutput(&surface); player.setSource(localVideoFile3ColorsWithSound); - QImage frameImage; - connect(&surface, &QVideoSink::videoFrameChanged, [&frameImage](const QVideoFrame& frame) { - frameImage = frame.toImage(); + std::optional<QRgb> color; + connect(&surface, &QVideoSink::videoFrameChanged, [&](const QVideoFrame& frame) { + auto image = frame.toImage(); + color = image.isNull() ? std::optional<QRgb>{} : image.pixel(1, 1); }); auto checkColorAndPosition = [&](int colorIndex, QString errorTag) { - QVERIFY(!frameImage.isNull()); + QVERIFY(color); const auto expectedColor = video3Colors[colorIndex]; - const auto actualColor = frameImage.pixel(1, 1); + const auto actualColor = *color; auto errorPrintingGuard = qScopeGuard([&]() { qDebug() << "Error Tag:" << errorTag; @@ -1689,10 +1692,8 @@ void tst_QMediaPlayerBackend::finiteLoops() QCOMPARE(intervals.size(), 3u); QCOMPARE_GT(intervals[0].first, 0); QCOMPARE(intervals[0].second, player.duration()); - QCOMPARE(intervals[1].first, 0); - QCOMPARE(intervals[1].second, player.duration()); - QCOMPARE(intervals[2].first, 0); - QCOMPARE(intervals[2].second, player.duration()); + QCOMPARE(intervals[1], std::make_pair(qint64(0), player.duration())); + QCOMPARE(intervals[2], std::make_pair(qint64(0), player.duration())); QCOMPARE(player.mediaStatus(), QMediaPlayer::EndOfMedia); @@ -1709,7 +1710,7 @@ void tst_QMediaPlayerBackend::finiteLoops() } } -void tst_QMediaPlayerBackend::infiteLoops() +void tst_QMediaPlayerBackend::infiniteLoops() { if (localVideoFile2.isEmpty()) QSKIP("Video format is not supported"); @@ -1754,6 +1755,100 @@ void tst_QMediaPlayerBackend::infiteLoops() QCOMPARE(player.playbackState(), QMediaPlayer::StoppedState); } +void tst_QMediaPlayerBackend::seekOnLoops() +{ + if (localVideoFile3ColorsWithSound.isEmpty()) + QSKIP("Video format is not supported"); + +#ifdef Q_OS_MACOS + if (qEnvironmentVariable("QTEST_ENVIRONMENT").toLower() == "ci") + QSKIP("The test accidently gets crashed on macOS CI, not reproduced locally. To be " + "investigated: QTBUG-111744"); +#endif + + TestVideoSink surface(false); + QMediaPlayer player; + + QSignalSpy positionSpy(&player, &QMediaPlayer::positionChanged); + + player.setVideoOutput(&surface); + player.setLoops(3); + player.setPlaybackRate(2); + + player.setSource(localVideoFile3ColorsWithSound); + + player.play(); + surface.waitForFrame(); + + // seek in the 1st loop + player.setPosition(player.duration() * 4 / 5); + + // wait for the 2nd loop and seek + surface.waitForFrame(); + QTRY_VERIFY(player.position() < player.duration() / 2); + player.setPosition(player.duration() * 8 / 9); + + // wait for the 3rd loop and seek + surface.waitForFrame(); + QTRY_VERIFY(player.position() < player.duration() / 2); + player.setPosition(player.duration() * 4 / 5); + + QTRY_COMPARE(player.playbackState(), QMediaPlayer::StoppedState); + + auto intervals = positionChangingIntervals(positionSpy); + + QCOMPARE(intervals.size(), 3); + QCOMPARE_GT(intervals[0].first, 0); + QCOMPARE(intervals[0].second, player.duration()); + QCOMPARE(intervals[1], std::make_pair(qint64(0), player.duration())); + QCOMPARE(intervals[2], std::make_pair(qint64(0), player.duration())); + + QCOMPARE(player.mediaStatus(), QMediaPlayer::EndOfMedia); +} + +void tst_QMediaPlayerBackend::changeLoopsOnTheFly() +{ + if (localVideoFile3ColorsWithSound.isEmpty()) + QSKIP("Video format is not supported"); + +#ifdef Q_OS_MACOS + if (qEnvironmentVariable("QTEST_ENVIRONMENT").toLower() == "ci") + QSKIP("The test accidently gets crashed on macOS CI, not reproduced locally. To be " + "investigated: QTBUG-111744"); +#endif + + TestVideoSink surface(false); + QMediaPlayer player; + + QSignalSpy positionSpy(&player, &QMediaPlayer::positionChanged); + + player.setVideoOutput(&surface); + player.setLoops(4); + player.setPlaybackRate(5); + + player.setSource(localVideoFile3ColorsWithSound); + + player.play(); + surface.waitForFrame(); + + player.setPosition(player.duration() * 4 / 5); + + // wait for the 2nd loop + surface.waitForFrame(); + QTRY_VERIFY(player.position() < player.duration() / 2); + player.setPosition(player.duration() * 8 / 9); + + player.setLoops(1); + + QTRY_COMPARE(player.playbackState(), QMediaPlayer::StoppedState); + QCOMPARE(player.mediaStatus(), QMediaPlayer::EndOfMedia); + + auto intervals = positionChangingIntervals(positionSpy); + QCOMPARE(intervals.size(), 2); + + QCOMPARE(intervals[1], std::make_pair(qint64(0), player.duration())); +} + void tst_QMediaPlayerBackend::lazyLoadVideo() { QQmlEngine engine; |