diff options
16 files changed, 447 insertions, 119 deletions
diff --git a/src/multimedia/platform/qplatformmediaplayer_p.h b/src/multimedia/platform/qplatformmediaplayer_p.h index a04565668..b2cd5e571 100644 --- a/src/multimedia/platform/qplatformmediaplayer_p.h +++ b/src/multimedia/platform/qplatformmediaplayer_p.h @@ -116,7 +116,8 @@ public: return isSeekable() && (m_loops < 0 || ++m_currentLoop < m_loops); } int loops() { return m_loops; } - void setLoops(int loops) { + virtual void setLoops(int loops) + { if (m_loops == loops) return; m_loops = loops; diff --git a/src/plugins/multimedia/ffmpeg/CMakeLists.txt b/src/plugins/multimedia/ffmpeg/CMakeLists.txt index 7dee4f557..49775aa42 100644 --- a/src/plugins/multimedia/ffmpeg/CMakeLists.txt +++ b/src/plugins/multimedia/ffmpeg/CMakeLists.txt @@ -52,6 +52,7 @@ qt_internal_add_plugin(QFFmpegMediaPlugin playbackengine/qffmpegcodec.cpp playbackengine/qffmpegcodec_p.h playbackengine/qffmpegpacket_p.h playbackengine/qffmpegframe_p.h + playbackengine/qffmpegpositionwithoffset_p.h DEFINES QT_COMPILING_FFMPEG LIBRARIES diff --git a/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegdemuxer.cpp b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegdemuxer.cpp index b80cd47c4..fdf987b60 100644 --- a/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegdemuxer.cpp +++ b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegdemuxer.cpp @@ -2,23 +2,46 @@ // SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "playbackengine/qffmpegdemuxer_p.h" +#include <qloggingcategory.h> QT_BEGIN_NAMESPACE -// queue up max 16M of encoded data, that should always be enough -// (it's around 2 secs of 4K HDR video, longer for almost all other formats) -static constexpr quint64 MaxQueueSize = 16 * 1024 * 1024; +// 4 sec for buffering. TODO: maybe move to env var customization +static constexpr qint64 MaxBufferingTimeUs = 4'000'000; + +// Currently, consider only time. TODO: maybe move to env var customization +static constexpr qint64 MaxBufferingSize = std::numeric_limits<qint64>::max(); namespace QFFmpeg { -Demuxer::Demuxer(AVFormatContext *context, qint64 seekPos, const StreamIndexes &streamIndexes) - : m_context(context), m_seekPos(seekPos) +static Q_LOGGING_CATEGORY(qLcDemuxer, "qt.multimedia.ffmpeg.demuxer"); + +static qint64 streamTimeToUs(const AVStream *stream, qint64 time) +{ + Q_ASSERT(stream); + + const auto res = mul(time * 1000000, stream->time_base); + return res ? *res : time; +} + +Demuxer::Demuxer(AVFormatContext *context, const PositionWithOffset &posWithOffset, + const StreamIndexes &streamIndexes, int loops) + : m_context(context), m_posWithOffset(posWithOffset) { + qCDebug(qLcDemuxer) << "Create demuxer." + << "pos:" << posWithOffset.pos << "loop offset:" << posWithOffset.offset.pos + << "loop index:" << posWithOffset.offset.index << "loops:" << loops; + m_loops = loops; + Q_ASSERT(m_context); + Q_ASSERT(m_loops < 0 || m_posWithOffset.offset.index < m_loops); for (auto i = 0; i < QPlatformMediaPlayer::NTrackTypes; ++i) { - if (streamIndexes[i] >= 0) - m_streams[streamIndexes[i]] = { static_cast<QPlatformMediaPlayer::TrackType>(i) }; + if (streamIndexes[i] >= 0) { + const auto trackType = static_cast<QPlatformMediaPlayer::TrackType>(i); + qCDebug(qLcDemuxer) << "Activate demuxing stream" << i << ", trackType:" << trackType; + m_streams[streamIndexes[i]] = { trackType }; + } } } @@ -26,33 +49,43 @@ void Demuxer::doNextStep() { ensureSeeked(); - Packet packet(AVPacketUPtr{ av_packet_alloc() }); + Packet packet(m_posWithOffset.offset, AVPacketUPtr{ av_packet_alloc() }); if (av_read_frame(m_context, packet.avPacket()) < 0) { - setAtEnd(true); + ++m_posWithOffset.offset.index; + + if (m_loops >= 0 && m_posWithOffset.offset.index >= m_loops) { + qCDebug(qLcDemuxer) << "finish demuxing"; + setAtEnd(true); + } else { + m_seeked = false; + m_posWithOffset.pos = 0; + m_posWithOffset.offset.pos = m_endPts; + m_endPts = 0; + + ensureSeeked(); + + qCDebug(qLcDemuxer) << "Demuxer loops changed. Index:" << m_posWithOffset.offset.index + << "Offset:" << m_posWithOffset.offset.pos; + } + return; } const auto streamIndex = packet.avPacket()->stream_index; + const auto stream = m_context->streams[streamIndex]; auto it = m_streams.find(streamIndex); if (it != m_streams.end()) { - it->second.dataSize += packet.avPacket()->size; - it->second.duration += packet.avPacket()->duration; - - switch (it->second.trackType) { - case QPlatformMediaPlayer::TrackType::VideoStream: - emit requestProcessVideoPacket(packet); - break; - case QPlatformMediaPlayer::TrackType::AudioStream: - emit requestProcessAudioPacket(packet); - break; - case QPlatformMediaPlayer::TrackType::SubtitleStream: - emit requestProcessSubtitlePacket(packet); - break; - default: - Q_ASSERT(!"Unknown track type"); - } + const auto packetEndPos = + streamTimeToUs(stream, packet.avPacket()->pts + packet.avPacket()->duration); + m_endPts = std::max(m_endPts, m_posWithOffset.offset.pos + packetEndPos); + + it->second.bufferingTime += streamTimeToUs(stream, packet.avPacket()->duration); + it->second.bufferingSize += packet.avPacket()->size; + + auto signal = signalByTrackType(it->second.trackType); + emit (this->*signal)(packet); } scheduleNextStep(false); @@ -65,11 +98,12 @@ void Demuxer::onPacketProcessed(Packet packet) auto it = m_streams.find(streamIndex); if (it != m_streams.end()) { - it->second.dataSize -= packet.avPacket()->size; - it->second.duration -= packet.avPacket()->duration; + it->second.bufferingTime -= + streamTimeToUs(m_context->streams[streamIndex], packet.avPacket()->duration); + it->second.bufferingSize -= packet.avPacket()->size; - Q_ASSERT(it->second.dataSize >= 0); - Q_ASSERT(it->second.duration >= 0); + Q_ASSERT(it->second.bufferingTime >= 0); + Q_ASSERT(it->second.bufferingSize >= 0); } } @@ -81,21 +115,12 @@ bool Demuxer::canDoNextStep() const if (!PlaybackEngineObject::canDoNextStep() || isAtEnd() || m_streams.empty()) return false; - const bool hasSmallDuration = - std::any_of(m_streams.begin(), m_streams.end(), - [](const auto &s) { return s.second.duration < 200; }); - - if (hasSmallDuration) - return true; - - const auto dataSize = - std::accumulate(m_streams.begin(), m_streams.end(), quint64(0), - [](quint64 value, const auto &s) { return value + s.second.dataSize; }); + auto checkBufferingTime = [](const auto &streamIndexToData) { + return streamIndexToData.second.bufferingTime < MaxBufferingTimeUs && + streamIndexToData.second.bufferingSize < MaxBufferingSize; + }; - if (dataSize > MaxQueueSize) - return false; - - return true; + return std::all_of(m_streams.begin(), m_streams.end(), checkBufferingTime); } void Demuxer::ensureSeeked() @@ -103,7 +128,7 @@ void Demuxer::ensureSeeked() if (std::exchange(m_seeked, true)) return; - const qint64 seekPos = m_seekPos * AV_TIME_BASE / 1000000; + const qint64 seekPos = m_posWithOffset.pos * AV_TIME_BASE / 1000000; auto err = av_seek_frame(m_context, -1, seekPos, AVSEEK_FLAG_BACKWARD); if (err < 0) { @@ -132,6 +157,12 @@ Demuxer::RequestingSignal Demuxer::signalByTrackType(QPlatformMediaPlayer::Track return nullptr; } +void Demuxer::setLoops(int loopsCount) +{ + qCDebug(qLcDemuxer) << "setLoops to demuxer" << loopsCount; + m_loops = loopsCount; +} + } // namespace QFFmpeg QT_END_NAMESPACE diff --git a/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegdemuxer_p.h b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegdemuxer_p.h index 6bfd4431f..290626af2 100644 --- a/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegdemuxer_p.h +++ b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegdemuxer_p.h @@ -17,6 +17,7 @@ #include "playbackengine/qffmpegplaybackengineobject_p.h" #include "private/qplatformmediaplayer_p.h" #include "playbackengine/qffmpegpacket_p.h" +#include "playbackengine/qffmpegpositionwithoffset_p.h" #include <unordered_map> @@ -28,11 +29,14 @@ class Demuxer : public PlaybackEngineObject { Q_OBJECT public: - Demuxer(AVFormatContext *context, qint64 seekPos, const StreamIndexes &streamIndexes); + Demuxer(AVFormatContext *context, const PositionWithOffset &posWithOffset, + const StreamIndexes &streamIndexes, int loops); using RequestingSignal = void (Demuxer::*)(Packet); static RequestingSignal signalByTrackType(QPlatformMediaPlayer::TrackType trackType); + void setLoops(int loopsCount); + public slots: void onPacketProcessed(Packet); @@ -52,14 +56,16 @@ private: struct StreamData { QPlatformMediaPlayer::TrackType trackType = QPlatformMediaPlayer::TrackType::NTrackTypes; - qint64 duration = 0; - qint64 dataSize = 0; + qint64 bufferingTime = 0; + qint64 bufferingSize = 0; }; AVFormatContext *m_context = nullptr; bool m_seeked = false; std::unordered_map<int, StreamData> m_streams; - const qint64 m_seekPos = 0; + PositionWithOffset m_posWithOffset; + qint64 m_endPts = 0; + std::atomic<int> m_loops = QMediaPlayer::Once; }; } // namespace QFFmpeg diff --git a/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegframe_p.h b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegframe_p.h index bccb13ebe..459ceb9b4 100644 --- a/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegframe_p.h +++ b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegframe_p.h @@ -17,6 +17,7 @@ #include "qffmpeg_p.h" #include "playbackengine/qffmpegcodec_p.h" +#include "playbackengine/qffmpegpositionwithoffset_p.h" #include "QtCore/qsharedpointer.h" #include "qpointer.h" #include "qobject.h" @@ -31,8 +32,9 @@ struct Frame { struct Data { - Data(AVFrameUPtr f, const Codec &codec, qint64, const QObject *source) - : codec(codec), frame(std::move(f)), source(source) + Data(const LoopOffset &offset, AVFrameUPtr f, const Codec &codec, qint64, + const QObject *source) + : loopOffset(offset), codec(codec), frame(std::move(f)), source(source) { Q_ASSERT(frame); if (frame->pts != AV_NOPTS_VALUE) @@ -44,12 +46,14 @@ struct Frame ? (1000000 * avgFrameRate.den + avgFrameRate.num / 2) / avgFrameRate.num : 0; } - Data(const QString &text, qint64 pts, qint64 duration, const QObject *source) - : text(text), pts(pts), duration(duration), source(source) + Data(const LoopOffset &offset, const QString &text, qint64 pts, qint64 duration, + const QObject *source) + : loopOffset(offset), text(text), pts(pts), duration(duration), source(source) { } QAtomicInt ref; + LoopOffset loopOffset; std::optional<Codec> codec; AVFrameUPtr frame; QString text; @@ -59,12 +63,14 @@ struct Frame }; Frame() = default; - Frame(AVFrameUPtr f, const Codec &codec, qint64 pts, const QObject *source = nullptr) - : d(new Data(std::move(f), codec, pts, source)) + Frame(const LoopOffset &offset, AVFrameUPtr f, const Codec &codec, qint64 pts, + const QObject *source = nullptr) + : d(new Data(offset, std::move(f), codec, pts, source)) { } - Frame(const QString &text, qint64 pts, qint64 duration, const QObject *source = nullptr) - : d(new Data(text, pts, duration, source)) + Frame(const LoopOffset &offset, const QString &text, qint64 pts, qint64 duration, + const QObject *source = nullptr) + : d(new Data(offset, text, pts, duration, source)) { } bool isValid() const { return !!d; } @@ -77,6 +83,9 @@ struct Frame qint64 end() const { return data().pts + data().duration; } QString text() const { return data().text; } const QObject *source() const { return data().source; }; + const LoopOffset &loopOffset() const { return data().loopOffset; }; + qint64 absolutePts() const { return pts() + loopOffset().pos; } + qint64 absoluteEnd() const { return end() + loopOffset().pos; } private: Data &data() const diff --git a/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegpacket_p.h b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegpacket_p.h index 5e489b3cd..3d037a72b 100644 --- a/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegpacket_p.h +++ b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegpacket_p.h @@ -17,6 +17,7 @@ #include "qffmpeg_p.h" #include "QtCore/qsharedpointer.h" +#include "playbackengine/qffmpegpositionwithoffset_p.h" QT_BEGIN_NAMESPACE @@ -26,16 +27,19 @@ struct Packet { struct Data { - Data(AVPacketUPtr p) : packet(std::move(p)) { } + Data(const LoopOffset &offset, AVPacketUPtr p) + : loopOffset(offset), packet(std::move(p)) { } QAtomicInt ref; + LoopOffset loopOffset; AVPacketUPtr packet; }; Packet() = default; - Packet(AVPacketUPtr p) : d(new Data(std::move(p))) { } + Packet(const LoopOffset &offset, AVPacketUPtr p) : d(new Data(offset, std::move(p))) { } bool isValid() const { return !!d; } AVPacket *avPacket() const { return d->packet.get(); } + const LoopOffset &loopOffset() const { return d->loopOffset; } private: QExplicitlySharedDataPointer<Data> d; diff --git a/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegpositionwithoffset_p.h b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegpositionwithoffset_p.h new file mode 100644 index 000000000..a30fdc119 --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegpositionwithoffset_p.h @@ -0,0 +1,40 @@ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#ifndef qffmpegpositionwithoffset_p_H +#define qffmpegpositionwithoffset_p_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#include <qtypes.h> + +QT_BEGIN_NAMESPACE + +namespace QFFmpeg { + +struct LoopOffset +{ + qint64 pos = 0; + int index = 0; +}; + +struct PositionWithOffset +{ + qint64 pos = 0; + LoopOffset offset; +}; + +} // namespace QFFmpeg + +QT_END_NAMESPACE + +#endif // qffmpegpositionwithoffset_p_H diff --git a/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegrenderer.cpp b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegrenderer.cpp index d12f38e84..9258f9677 100644 --- a/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegrenderer.cpp +++ b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegrenderer.cpp @@ -2,11 +2,14 @@ // SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "playbackengine/qffmpegrenderer_p.h" +#include <qloggingcategory.h> QT_BEGIN_NAMESPACE namespace QFFmpeg { +static Q_LOGGING_CATEGORY(qLcRenderer, "qt.multimedia.ffmpeg.renderer"); + Renderer::Renderer(const TimeController &tc, const std::chrono::microseconds &seekPosTimeOffset) : m_timeController(tc), m_lastPosition(tc.currentPosition()), @@ -68,9 +71,11 @@ void Renderer::render(Frame frame) { using namespace std::chrono; - const auto isFrameOutdated = frame.isValid() && frame.end() < m_seekPos; + const auto isFrameOutdated = frame.isValid() && frame.absoluteEnd() < m_seekPos; if (isFrameOutdated) { + qCDebug(qLcRenderer) << "frame outdated! absEnd:" << frame.absoluteEnd() << "absPts" + << frame.absolutePts() << "seekPos:" << m_seekPos; emit frameProcessed(frame); return; } @@ -101,7 +106,8 @@ int Renderer::timerInterval() const { if (auto frame = m_frames.front(); frame.isValid() && !m_isStepForced) { using namespace std::chrono; - const auto delay = m_timeController.timeFromPosition(frame.pts()) - steady_clock::now(); + const auto delay = + m_timeController.timeFromPosition(frame.absolutePts()) - steady_clock::now(); return std::max(0, static_cast<int>(duration_cast<milliseconds>(delay).count())); } @@ -133,16 +139,23 @@ void Renderer::doNextStep() if (result.timeLeft.count() && frame.isValid()) { const auto now = std::chrono::steady_clock::now(); - m_timeController.sync(now + result.timeLeft, frame.pts()); - emit synchronized(now + result.timeLeft, frame.pts()); + m_timeController.sync(now + result.timeLeft, frame.absolutePts()); + emit synchronized(now + result.timeLeft, frame.absolutePts()); } if (done) { m_frames.dequeue(); if (frame.isValid()) { - m_lastPosition = std::max(frame.pts(), m_lastPosition.load()); - m_seekPos = frame.end(); + m_lastPosition = std::max(frame.absolutePts(), m_lastPosition.load()); + m_seekPos = frame.absoluteEnd(); + + const auto loopIndex = frame.loopOffset().index; + if (m_loopIndex < loopIndex) { + m_loopIndex = loopIndex; + emit loopChanged(frame.loopOffset().pos, m_loopIndex); + } + emit frameProcessed(frame); } } diff --git a/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegrenderer_p.h b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegrenderer_p.h index 23ccab883..8949213a4 100644 --- a/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegrenderer_p.h +++ b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegrenderer_p.h @@ -55,6 +55,8 @@ signals: void forceStepDone(); + void loopChanged(qint64 offset, int index); + protected: bool setForceStepDone(); @@ -82,6 +84,7 @@ private: TimeController m_timeController; std::atomic<qint64> m_lastPosition = 0; std::atomic<qint64> m_seekPos = 0; + int m_loopIndex = 0; QQueue<Frame> m_frames; std::atomic_bool m_isStepForced = false; diff --git a/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegstreamdecoder.cpp b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegstreamdecoder.cpp index 01dfb1f84..67151618b 100644 --- a/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegstreamdecoder.cpp +++ b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegstreamdecoder.cpp @@ -3,16 +3,21 @@ #include "playbackengine/qffmpegstreamdecoder_p.h" #include "playbackengine/qffmpegmediadataholder_p.h" +#include <qloggingcategory.h> QT_BEGIN_NAMESPACE +static Q_LOGGING_CATEGORY(qLcStreamDecoder, "qt.multimedia.ffmpeg.streamdecoder"); + namespace QFFmpeg { -StreamDecoder::StreamDecoder(const Codec &codec, qint64 seekPos) +StreamDecoder::StreamDecoder(const Codec &codec, qint64 absSeekPos) : m_codec(codec), - m_seekPos(seekPos), + m_absSeekPos(absSeekPos), m_trackType(MediaDataHolder::trackTypeFromMediaType(codec.context()->codec_type)) { + qCDebug(qLcStreamDecoder) << "Create stream decoder, trackType" << m_trackType + << "absSeekPos:" << absSeekPos; Q_ASSERT(m_trackType != QPlatformMediaPlayer::NTrackTypes); } @@ -36,10 +41,24 @@ void StreamDecoder::decode(Packet packet) void StreamDecoder::doNextStep() { auto packet = m_packets.dequeue(); - if (trackType() == QPlatformMediaPlayer::SubtitleStream) - decodeSubtitle(packet); - else - decodeMedia(packet); + + auto decodePacket = [this](Packet packet) { + if (trackType() == QPlatformMediaPlayer::SubtitleStream) + decodeSubtitle(packet); + else + decodeMedia(packet); + }; + + if (packet.isValid() && packet.loopOffset().index != m_offset.index) { + decodePacket({}); + + qCDebug(qLcStreamDecoder) << "flush buffers due to new loop:" << packet.loopOffset().index; + + avcodec_flush_buffers(m_codec.context()); + m_offset = packet.loopOffset(); + } + + decodePacket(packet); setAtEnd(!packet.isValid()); @@ -82,7 +101,7 @@ bool StreamDecoder::canDoNextStep() const void StreamDecoder::onFrameFound(Frame frame) { - if (frame.isValid() && frame.end() < m_seekPos) + if (frame.isValid() && frame.absoluteEnd() < m_absSeekPos) return; Q_ASSERT(m_pendingFramesCount >= 0); @@ -131,7 +150,7 @@ void StreamDecoder::receiveAVFrames() break; } - onFrameFound({ std::move(avFrame), m_codec, 0, this }); + onFrameFound({ m_offset, std::move(avFrame), m_codec, 0, this }); } } @@ -196,10 +215,10 @@ void StreamDecoder::decodeSubtitle(Packet packet) if (text.endsWith(QLatin1Char('\n'))) text.chop(1); - onFrameFound({ text, start, end - start, this }); + onFrameFound({ m_offset, text, start, end - start, this }); // TODO: maybe optimize - onFrameFound({ QString(), end, 0, this }); + onFrameFound({ m_offset, QString(), end, 0, this }); } } // namespace QFFmpeg diff --git a/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegstreamdecoder_p.h b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegstreamdecoder_p.h index cc12b1e9a..001ec6b90 100644 --- a/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegstreamdecoder_p.h +++ b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegstreamdecoder_p.h @@ -16,6 +16,7 @@ #include "playbackengine/qffmpegplaybackengineobject_p.h" #include "playbackengine/qffmpegframe_p.h" #include "playbackengine/qffmpegpacket_p.h" +#include "playbackengine/qffmpegpositionwithoffset_p.h" #include "private/qplatformmediaplayer_p.h" #include <optional> @@ -28,7 +29,7 @@ class StreamDecoder : public PlaybackEngineObject { Q_OBJECT public: - StreamDecoder(const Codec &codec, qint64 seekPos); + StreamDecoder(const Codec &codec, qint64 absSeekPos); ~StreamDecoder(); @@ -64,11 +65,13 @@ private: private: Codec m_codec; - qint64 m_seekPos = 0; + const qint64 m_absSeekPos = 0; const QPlatformMediaPlayer::TrackType m_trackType; qint32 m_pendingFramesCount = 0; + LoopOffset m_offset; + QQueue<Packet> m_packets; }; diff --git a/src/plugins/multimedia/ffmpeg/qffmpegmediaplayer.cpp b/src/plugins/multimedia/ffmpeg/qffmpegmediaplayer.cpp index 507550ab5..171f082d5 100644 --- a/src/plugins/multimedia/ffmpeg/qffmpegmediaplayer.cpp +++ b/src/plugins/multimedia/ffmpeg/qffmpegmediaplayer.cpp @@ -53,15 +53,21 @@ void QFFmpegMediaPlayer::endOfStream() m_positionUpdateTimer.stop(); positionChanged(duration()); - if (doLoop()) { - m_playbackEngine->seek(0); - positionChanged(0); + stateChanged(QMediaPlayer::StoppedState); + mediaStatusChanged(QMediaPlayer::EndOfMedia); +} - runPlayback(); - } else { - stateChanged(QMediaPlayer::StoppedState); - mediaStatusChanged(QMediaPlayer::EndOfMedia); - } +void QFFmpegMediaPlayer::onLoopChanged() +{ + // report about finish and start + // reporting both signals is a bit contraversial + // but it eshures the idea of notifications about + // imporatant position points. + // Also, it ensures more predictable flow for testing. + positionChanged(duration()); + positionChanged(0); + m_positionUpdateTimer.stop(); + m_positionUpdateTimer.start(); } float QFFmpegMediaPlayer::bufferProgress() const @@ -121,10 +127,13 @@ void QFFmpegMediaPlayer::setMedia(const QUrl &media, QIODevice *stream) mediaStatusChanged(QMediaPlayer::LoadingMedia); m_playbackEngine = std::make_unique<PlaybackEngine>(); + connect(m_playbackEngine.get(), &PlaybackEngine::endOfStream, this, &QFFmpegMediaPlayer::endOfStream); connect(m_playbackEngine.get(), &PlaybackEngine::errorOccured, this, &QFFmpegMediaPlayer::error); + connect(m_playbackEngine.get(), &PlaybackEngine::loopChanged, this, + &QFFmpegMediaPlayer::onLoopChanged); if (!m_playbackEngine->setMedia(media, stream)) { m_playbackEngine.reset(); @@ -134,6 +143,8 @@ void QFFmpegMediaPlayer::setMedia(const QUrl &media, QIODevice *stream) m_playbackEngine->setAudioSink(m_audioOutput); m_playbackEngine->setVideoSink(m_videoSink); + m_playbackEngine->setLoops(loops()); + m_playbackEngine->setPlaybackRate(m_playbackRate); durationChanged(duration()); tracksChanged(); @@ -145,6 +156,7 @@ void QFFmpegMediaPlayer::setMedia(const QUrl &media, QIODevice *stream) videoAvailableChanged( !m_playbackEngine->streamInfo(QPlatformMediaPlayer::VideoStream).isEmpty()); + // TODO: get rid of the delayed update QMetaObject::invokeMethod(this, "delayedLoadedStatus", Qt::QueuedConnection); } @@ -156,7 +168,6 @@ void QFFmpegMediaPlayer::play() if (mediaStatus() == QMediaPlayer::EndOfMedia && state() == QMediaPlayer::StoppedState) { m_playbackEngine->seek(0); positionChanged(0); - resetCurrentLoop(); } runPlayback(); @@ -247,6 +258,16 @@ void QFFmpegMediaPlayer::setActiveTrack(TrackType type, int streamNumber) { if (m_playbackEngine) m_playbackEngine->setActiveTrack(type, streamNumber); + else + qWarning() << "Cannot set active track without open source"; +} + +void QFFmpegMediaPlayer::setLoops(int loops) +{ + if (m_playbackEngine) + m_playbackEngine->setLoops(loops); + + QPlatformMediaPlayer::setLoops(loops); } QT_END_NAMESPACE diff --git a/src/plugins/multimedia/ffmpeg/qffmpegmediaplayer_p.h b/src/plugins/multimedia/ffmpeg/qffmpegmediaplayer_p.h index 7e9126a22..b330948f6 100644 --- a/src/plugins/multimedia/ffmpeg/qffmpegmediaplayer_p.h +++ b/src/plugins/multimedia/ffmpeg/qffmpegmediaplayer_p.h @@ -65,8 +65,12 @@ public: QMediaMetaData trackMetaData(TrackType type, int streamNumber) override; int activeTrack(TrackType) override; void setActiveTrack(TrackType, int streamNumber) override; + void setLoops(int loops) override; - Q_INVOKABLE void delayedLoadedStatus() { mediaStatusChanged(QMediaPlayer::LoadedMedia); } + Q_INVOKABLE void delayedLoadedStatus() { + if (mediaStatus() == QMediaPlayer::LoadingMedia) + mediaStatusChanged(QMediaPlayer::LoadedMedia); + } private: void runPlayback(); @@ -78,6 +82,7 @@ private slots: { QPlatformMediaPlayer::error(error, errorString); } + void onLoopChanged(); private: QTimer m_positionUpdateTimer; diff --git a/src/plugins/multimedia/ffmpeg/qffmpegplaybackengine.cpp b/src/plugins/multimedia/ffmpeg/qffmpegplaybackengine.cpp index 19105499d..c9c4f20f2 100644 --- a/src/plugins/multimedia/ffmpeg/qffmpegplaybackengine.cpp +++ b/src/plugins/multimedia/ffmpeg/qffmpegplaybackengine.cpp @@ -18,6 +18,8 @@ QT_BEGIN_NAMESPACE namespace QFFmpeg { +static Q_LOGGING_CATEGORY(qLcPlaybackEngine, "qt.multimedia.ffmpeg.playbackengine"); + // The helper is needed since on some compilers std::unique_ptr // doesn't have a default constructor in the case of sizeof(CustomDeleter) > 0 template<typename Array> @@ -41,11 +43,13 @@ PlaybackEngine::PlaybackEngine() m_streams(defaultObjectsArray<decltype(m_streams)>()), m_renderers(defaultObjectsArray<decltype(m_renderers)>()) { + qCDebug(qLcPlaybackEngine) << "Create PlaybackEngine"; qRegisterMetaType<QFFmpeg::Packet>(); qRegisterMetaType<QFFmpeg::Frame>(); } PlaybackEngine::~PlaybackEngine() { + qCDebug(qLcPlaybackEngine) << "Delete PlaybackEngine"; forEachExistingObject([](auto &object) { object.reset(); }); deleteFreeThreads(); } @@ -62,22 +66,33 @@ void PlaybackEngine::onRendererFinished() if (!isAtEnd(QPlatformMediaPlayer::AudioStream)) return; - if (!isAtEnd(QPlatformMediaPlayer::SubtitleStream) - && !m_renderers[QPlatformMediaPlayer::AudioStream] - && !m_renderers[QPlatformMediaPlayer::VideoStream]) + if (!isAtEnd(QPlatformMediaPlayer::SubtitleStream) && !hasMediaStream()) return; if (std::exchange(m_state, QMediaPlayer::StoppedState) == QMediaPlayer::StoppedState) return; - m_timeController.setPaused(true); - m_timeController.sync(m_duration); + finilizeTime(duration()); forceUpdate(); + qCDebug(qLcPlaybackEngine) << "Playback engine end of stream"; + emit endOfStream(); } +void PlaybackEngine::onRendererLoopChanged(qint64 offset, int loopIndex) +{ + if (loopIndex > m_currentLoopOffset.index) { + m_currentLoopOffset = { offset, loopIndex }; + emit loopChanged(); + } else if (loopIndex == m_currentLoopOffset.index && offset != m_currentLoopOffset.pos) { + qWarning() << "Unexpected offset for loop" << loopIndex << ":" << offset << "vs" + << m_currentLoopOffset.pos; + m_currentLoopOffset.pos = offset; + } +} + void PlaybackEngine::onRendererSynchronized(std::chrono::steady_clock::time_point tp, qint64 pos) { Q_ASSERT(QObject::sender() == m_renderers[QPlatformMediaPlayer::AudioStream].get()); @@ -94,15 +109,13 @@ void PlaybackEngine::setState(QMediaPlayer::PlaybackState state) { if (!m_context) return; - if ( state == m_state ) + if (state == m_state) return; const auto prevState = std::exchange(m_state, state); - if (m_state == QMediaPlayer::StoppedState) { - m_timeController.setPaused(true); - m_timeController.sync(); - } + if (m_state == QMediaPlayer::StoppedState) + finilizeTime(0); if (prevState == QMediaPlayer::StoppedState || m_state == QMediaPlayer::StoppedState) recreateObjects(); @@ -208,14 +221,31 @@ void PlaybackEngine::forEachExistingObject(Action &&action) void PlaybackEngine::seek(qint64 pos) { - pos = qBound(0, pos, m_duration); + pos = qBound(0, pos, duration()); m_timeController.setPaused(true); - m_timeController.sync(pos); + m_timeController.sync(m_currentLoopOffset.pos + pos); forceUpdate(); } +void PlaybackEngine::setLoops(int loops) +{ + if (!isSeekable()) { + qWarning() << "Cannot set loops for non-seekable source"; + return; + } + + if (std::exchange(m_loops, loops) == loops) + return; + + qCDebug(qLcPlaybackEngine) << "set playback engine loops:" << loops << "prev loops:" << m_loops + << "index:" << m_currentLoopOffset.index; + + if (m_demuxer) + m_demuxer->setLoops(loops); +} + void PlaybackEngine::triggerStepIfNeeded() { if (m_state != QMediaPlayer::PausedState) @@ -295,6 +325,9 @@ void PlaybackEngine::createStreamAndRenderer(QPlatformMediaPlayer::TrackType tra connect(renderer.get(), &Renderer::synchronized, this, &PlaybackEngine::onRendererSynchronized); + connect(renderer.get(), &Renderer::loopChanged, this, + &PlaybackEngine::onRendererLoopChanged); + if constexpr (shouldPauseStreams) connect(renderer.get(), &Renderer::forceStepDone, this, &PlaybackEngine::updateObjectsPausedState); @@ -337,6 +370,8 @@ std::optional<Codec> PlaybackEngine::codecForTrack(QPlatformMediaPlayer::TrackTy auto &result = m_codecs[trackType]; if (!result) { + qCDebug(qLcPlaybackEngine) + << "Create codec for stream:" << streamIndex << "trackType:" << trackType; auto maybeCodec = Codec::create(m_context->streams[streamIndex]); if (!maybeCodec) { @@ -351,6 +386,12 @@ std::optional<Codec> PlaybackEngine::codecForTrack(QPlatformMediaPlayer::TrackTy return result; } +bool PlaybackEngine::hasMediaStream() const +{ + return m_renderers[QPlatformMediaPlayer::AudioStream] + || m_renderers[QPlatformMediaPlayer::VideoStream]; +} + void PlaybackEngine::createDemuxer() { decltype(m_currentAVStreamIndex) streamIndexes = { -1, -1, -1 }; @@ -365,8 +406,10 @@ void PlaybackEngine::createDemuxer() if (!hasStreams) return; - m_demuxer = - createPlaybackEngineObject<Demuxer>(m_context.get(), currentPosition(), streamIndexes); + const PositionWithOffset positionWithOffset{ currentPosition(false), m_currentLoopOffset }; + + m_demuxer = createPlaybackEngineObject<Demuxer>(m_context.get(), positionWithOffset, + streamIndexes, m_loops); forEachExistingObject<StreamDecoder>([&](auto &stream) { connect(m_demuxer.get(), Demuxer::signalByTrackType(stream->trackType()), stream.get(), @@ -425,14 +468,28 @@ void PlaybackEngine::setAudioSink(QAudioOutput *output) forceUpdate(); } -qint64 PlaybackEngine::currentPosition() const { - auto pos = std::numeric_limits<qint64>::max(); +qint64 PlaybackEngine::currentPosition(bool topPos) const { + std::optional<qint64> pos; + + for (size_t i = 0; i < m_renderers.size(); ++i) { + const auto &renderer = m_renderers[i]; + if (!renderer) + continue; + + // skip subtitle stream for finding lower rendering position + if (!topPos && i == QPlatformMediaPlayer::SubtitleStream && hasMediaStream()) + continue; + + const auto rendererPos = renderer->lastPosition(); + pos = !pos ? rendererPos + : topPos ? std::max(*pos, rendererPos) + : std::min(*pos, rendererPos); + } - for (auto trackType : { QPlatformMediaPlayer::VideoStream, QPlatformMediaPlayer::AudioStream }) - if (auto &renderer = m_renderers[trackType]) - pos = std::min(pos, renderer->lastPosition()); + if (!pos) + pos = m_timeController.currentPosition(); - return pos == std::numeric_limits<qint64>::max() ? m_timeController.currentPosition() : pos; + return qBound(0, *pos - m_currentLoopOffset.pos, duration()); } void PlaybackEngine::setActiveTrack(QPlatformMediaPlayer::TrackType trackType, int streamNumber) @@ -450,6 +507,14 @@ void PlaybackEngine::setActiveTrack(QPlatformMediaPlayer::TrackType trackType, i updateObjectsPausedState(); } +void PlaybackEngine::finilizeTime(qint64 pos) +{ + Q_ASSERT(pos >= 0 && pos <= duration()); + + m_timeController.setPaused(true); + m_timeController.sync(pos); + m_currentLoopOffset = {}; +} } QT_END_NAMESPACE diff --git a/src/plugins/multimedia/ffmpeg/qffmpegplaybackengine_p.h b/src/plugins/multimedia/ffmpeg/qffmpegplaybackengine_p.h index 682d9093b..4a61efa2f 100644 --- a/src/plugins/multimedia/ffmpeg/qffmpegplaybackengine_p.h +++ b/src/plugins/multimedia/ffmpeg/qffmpegplaybackengine_p.h @@ -49,6 +49,7 @@ #include "playbackengine/qffmpegtimecontroller_p.h" #include "playbackengine/qffmpegmediadataholder_p.h" #include "playbackengine/qffmpegcodec_p.h" +#include "playbackengine/qffmpegpositionwithoffset_p.h" #include <unordered_map> @@ -92,6 +93,8 @@ public: void seek(qint64 pos); + void setLoops(int loopsCount); + void setPlaybackRate(float rate); float playbackRate() const; @@ -100,11 +103,12 @@ public: using MediaDataHolder::activeTrack; - qint64 currentPosition() const; + qint64 currentPosition(bool topPos = true) const; signals: void endOfStream(); void errorOccured(int, const QString &); + void loopChanged(); protected: // objects managing struct ObjectDeleter @@ -152,12 +156,18 @@ private: void onRendererFinished(); + void onRendererLoopChanged(qint64 offset, int loopIndex); + void triggerStepIfNeeded(); static QString objectThreadName(const PlaybackEngineObject &object); std::optional<Codec> codecForTrack(QPlatformMediaPlayer::TrackType trackType); + bool hasMediaStream() const; + + void finilizeTime(qint64 pos); + private: TimeController m_timeController; @@ -174,6 +184,8 @@ private: std::array<RendererPtr, QPlatformMediaPlayer::NTrackTypes> m_renderers; std::array<std::optional<Codec>, QPlatformMediaPlayer::NTrackTypes> m_codecs; + int m_loops = QMediaPlayer::Once; + LoopOffset m_currentLoopOffset; }; template<typename T, typename... Args> diff --git a/tests/auto/integration/qmediaplayerbackend/tst_qmediaplayerbackend.cpp b/tests/auto/integration/qmediaplayerbackend/tst_qmediaplayerbackend.cpp index 8f01d30f6..170f5d680 100644 --- a/tests/auto/integration/qmediaplayerbackend/tst_qmediaplayerbackend.cpp +++ b/tests/auto/integration/qmediaplayerbackend/tst_qmediaplayerbackend.cpp @@ -74,7 +74,9 @@ private slots: void playbackRateChanging(); void durationDetectionIssues(); void finiteLoops(); - void infiteLoops(); + void infiniteLoops(); + void seekOnLoops(); + void changeLoopsOnTheFly(); void lazyLoadVideo(); private: @@ -1322,15 +1324,16 @@ void tst_QMediaPlayerBackend::playbackRateChanging() player.setVideoOutput(&surface); player.setSource(localVideoFile3ColorsWithSound); - QImage frameImage; - connect(&surface, &QVideoSink::videoFrameChanged, [&frameImage](const QVideoFrame& frame) { - frameImage = frame.toImage(); + std::optional<QRgb> color; + connect(&surface, &QVideoSink::videoFrameChanged, [&](const QVideoFrame& frame) { + auto image = frame.toImage(); + color = image.isNull() ? std::optional<QRgb>{} : image.pixel(1, 1); }); auto checkColorAndPosition = [&](int colorIndex, QString errorTag) { - QVERIFY(!frameImage.isNull()); + QVERIFY(color); const auto expectedColor = video3Colors[colorIndex]; - const auto actualColor = frameImage.pixel(1, 1); + const auto actualColor = *color; auto errorPrintingGuard = qScopeGuard([&]() { qDebug() << "Error Tag:" << errorTag; @@ -1689,10 +1692,8 @@ void tst_QMediaPlayerBackend::finiteLoops() QCOMPARE(intervals.size(), 3u); QCOMPARE_GT(intervals[0].first, 0); QCOMPARE(intervals[0].second, player.duration()); - QCOMPARE(intervals[1].first, 0); - QCOMPARE(intervals[1].second, player.duration()); - QCOMPARE(intervals[2].first, 0); - QCOMPARE(intervals[2].second, player.duration()); + QCOMPARE(intervals[1], std::make_pair(qint64(0), player.duration())); + QCOMPARE(intervals[2], std::make_pair(qint64(0), player.duration())); QCOMPARE(player.mediaStatus(), QMediaPlayer::EndOfMedia); @@ -1709,7 +1710,7 @@ void tst_QMediaPlayerBackend::finiteLoops() } } -void tst_QMediaPlayerBackend::infiteLoops() +void tst_QMediaPlayerBackend::infiniteLoops() { if (localVideoFile2.isEmpty()) QSKIP("Video format is not supported"); @@ -1754,6 +1755,100 @@ void tst_QMediaPlayerBackend::infiteLoops() QCOMPARE(player.playbackState(), QMediaPlayer::StoppedState); } +void tst_QMediaPlayerBackend::seekOnLoops() +{ + if (localVideoFile3ColorsWithSound.isEmpty()) + QSKIP("Video format is not supported"); + +#ifdef Q_OS_MACOS + if (qEnvironmentVariable("QTEST_ENVIRONMENT").toLower() == "ci") + QSKIP("The test accidently gets crashed on macOS CI, not reproduced locally. To be " + "investigated: QTBUG-111744"); +#endif + + TestVideoSink surface(false); + QMediaPlayer player; + + QSignalSpy positionSpy(&player, &QMediaPlayer::positionChanged); + + player.setVideoOutput(&surface); + player.setLoops(3); + player.setPlaybackRate(2); + + player.setSource(localVideoFile3ColorsWithSound); + + player.play(); + surface.waitForFrame(); + + // seek in the 1st loop + player.setPosition(player.duration() * 4 / 5); + + // wait for the 2nd loop and seek + surface.waitForFrame(); + QTRY_VERIFY(player.position() < player.duration() / 2); + player.setPosition(player.duration() * 8 / 9); + + // wait for the 3rd loop and seek + surface.waitForFrame(); + QTRY_VERIFY(player.position() < player.duration() / 2); + player.setPosition(player.duration() * 4 / 5); + + QTRY_COMPARE(player.playbackState(), QMediaPlayer::StoppedState); + + auto intervals = positionChangingIntervals(positionSpy); + + QCOMPARE(intervals.size(), 3); + QCOMPARE_GT(intervals[0].first, 0); + QCOMPARE(intervals[0].second, player.duration()); + QCOMPARE(intervals[1], std::make_pair(qint64(0), player.duration())); + QCOMPARE(intervals[2], std::make_pair(qint64(0), player.duration())); + + QCOMPARE(player.mediaStatus(), QMediaPlayer::EndOfMedia); +} + +void tst_QMediaPlayerBackend::changeLoopsOnTheFly() +{ + if (localVideoFile3ColorsWithSound.isEmpty()) + QSKIP("Video format is not supported"); + +#ifdef Q_OS_MACOS + if (qEnvironmentVariable("QTEST_ENVIRONMENT").toLower() == "ci") + QSKIP("The test accidently gets crashed on macOS CI, not reproduced locally. To be " + "investigated: QTBUG-111744"); +#endif + + TestVideoSink surface(false); + QMediaPlayer player; + + QSignalSpy positionSpy(&player, &QMediaPlayer::positionChanged); + + player.setVideoOutput(&surface); + player.setLoops(4); + player.setPlaybackRate(5); + + player.setSource(localVideoFile3ColorsWithSound); + + player.play(); + surface.waitForFrame(); + + player.setPosition(player.duration() * 4 / 5); + + // wait for the 2nd loop + surface.waitForFrame(); + QTRY_VERIFY(player.position() < player.duration() / 2); + player.setPosition(player.duration() * 8 / 9); + + player.setLoops(1); + + QTRY_COMPARE(player.playbackState(), QMediaPlayer::StoppedState); + QCOMPARE(player.mediaStatus(), QMediaPlayer::EndOfMedia); + + auto intervals = positionChangingIntervals(positionSpy); + QCOMPARE(intervals.size(), 2); + + QCOMPARE(intervals[1], std::make_pair(qint64(0), player.duration())); +} + void tst_QMediaPlayerBackend::lazyLoadVideo() { QQmlEngine engine; |