8 files changed, 456 insertions, 7 deletions

diff --git a/src/multimedia/qtmultimediaquicktools_headers/qdeclarativevideooutput_backend_p.h b/src/multimedia/qtmultimediaquicktools_headers/qdeclarativevideooutput_backend_p.h
index 5996af773..1d267df82 100644
--- a/src/multimedia/qtmultimediaquicktools_headers/qdeclarativevideooutput_backend_p.h
+++ b/src/multimedia/qtmultimediaquicktools_headers/qdeclarativevideooutput_backend_p.h
@@ -46,6 +46,7 @@ QT_BEGIN_NAMESPACE
 class QAbstractVideoSurface;
 class QDeclarativeVideoOutput;
 class QMediaService;
+class QAbstractVideoFilter;
 
 class Q_MULTIMEDIAQUICK_EXPORT QDeclarativeVideoBackend
 {
@@ -70,6 +71,12 @@ public:
     // The viewport, adjusted for the pixel aspect ratio
     virtual QRectF adjustedViewport() const = 0;
 
+    virtual void appendFilter(QAbstractVideoFilter *filter) { Q_UNUSED(filter); }
+    virtual void clearFilters() { }
+
+    virtual void releaseResources() { }
+    virtual void invalidateSceneGraph() { }
+
 protected:
     QDeclarativeVideoOutput *q;
     QPointer<QMediaService> m_service;
diff --git a/src/multimedia/qtmultimediaquicktools_headers/qdeclarativevideooutput_p.h b/src/multimedia/qtmultimediaquicktools_headers/qdeclarativevideooutput_p.h
index e1dbf5646..f022c0e3e 100644
--- a/src/multimedia/qtmultimediaquicktools_headers/qdeclarativevideooutput_p.h
+++ b/src/multimedia/qtmultimediaquicktools_headers/qdeclarativevideooutput_p.h
@@ -40,6 +40,7 @@
 #include <QtQuick/qquickitem.h>
 #include <QtCore/qpointer.h>
 #include <QtMultimedia/qcamerainfo.h>
+#include <QtMultimedia/qabstractvideofilter.h>
 
 #include <private/qtmultimediaquickdefs_p.h>
 
@@ -60,6 +61,7 @@ class Q_MULTIMEDIAQUICK_EXPORT QDeclarativeVideoOutput : public QQuickItem
     Q_PROPERTY(bool autoOrientation READ autoOrientation WRITE setAutoOrientation NOTIFY autoOrientationChanged REVISION 2)
     Q_PROPERTY(QRectF sourceRect READ sourceRect NOTIFY sourceRectChanged)
     Q_PROPERTY(QRectF contentRect READ contentRect NOTIFY contentRectChanged)
+    Q_PROPERTY(QQmlListProperty<QAbstractVideoFilter> filters READ filters);
     Q_ENUMS(FillMode)
 
 public:
@@ -104,6 +106,8 @@ public:
     };
     SourceType sourceType() const;
 
+    QQmlListProperty<QAbstractVideoFilter> filters();
+
 Q_SIGNALS:
     void sourceChanged();
     void fillModeChanged(QDeclarativeVideoOutput::FillMode);
@@ -116,6 +120,7 @@ protected:
     QSGNode *updatePaintNode(QSGNode *, UpdatePaintNodeData *);
     void itemChange(ItemChange change, const ItemChangeData &changeData);
     void geometryChanged(const QRectF &newGeometry, const QRectF &oldGeometry);
+    void releaseResources();
 
 private Q_SLOTS:
     void _q_updateMediaObject();
@@ -123,10 +128,16 @@ private Q_SLOTS:
     void _q_updateNativeSize();
     void _q_updateGeometry();
     void _q_screenOrientationChanged(int);
+    void _q_invalidateSceneGraph();
 
 private:
     bool createBackend(QMediaService *service);
 
+    static void filter_append(QQmlListProperty<QAbstractVideoFilter> *property, QAbstractVideoFilter *value);
+    static int filter_count(QQmlListProperty<QAbstractVideoFilter> *property);
+    static QAbstractVideoFilter *filter_at(QQmlListProperty<QAbstractVideoFilter> *property, int index);
+    static void filter_clear(QQmlListProperty<QAbstractVideoFilter> *property);
+
     SourceType m_sourceType;
 
     QPointer<QObject> m_source;
@@ -145,6 +156,8 @@ private:
     QVideoOutputOrientationHandler *m_screenOrientationHandler;
 
     QScopedPointer<QDeclarativeVideoBackend> m_backend;
+
+    QList<QAbstractVideoFilter *> m_filters;
 };
 
 QT_END_NAMESPACE
diff --git a/src/multimedia/qtmultimediaquicktools_headers/qsgvideonode_p.h b/src/multimedia/qtmultimediaquicktools_headers/qsgvideonode_p.h
index 8be77ff07..9a15f4afa 100644
--- a/src/multimedia/qtmultimediaquicktools_headers/qsgvideonode_p.h
+++ b/src/multimedia/qtmultimediaquicktools_headers/qsgvideonode_p.h
@@ -46,10 +46,16 @@ QT_BEGIN_NAMESPACE
 class Q_MULTIMEDIAQUICK_EXPORT QSGVideoNode : public QSGGeometryNode
 {
 public:
+    enum FrameFlag {
+        FrameFiltered = 0x01
+    };
+    Q_DECLARE_FLAGS(FrameFlags, FrameFlag)
+
     QSGVideoNode();
 
-    virtual void setCurrentFrame(const QVideoFrame &frame) = 0;
+    virtual void setCurrentFrame(const QVideoFrame &frame, FrameFlags flags) = 0;
     virtual QVideoFrame::PixelFormat pixelFormat() const = 0;
+    virtual QAbstractVideoBuffer::HandleType handleType() const = 0;
 
     void setTexturedRectGeometry(const QRectF &boundingRect, const QRectF &textureRect, int orientation);
 
@@ -59,6 +65,8 @@ private:
     int m_orientation;
 };
 
+Q_DECLARE_OPERATORS_FOR_FLAGS(QSGVideoNode::FrameFlags)
+
 class Q_MULTIMEDIAQUICK_EXPORT QSGVideoNodeFactoryInterface
 {
 public:
diff --git a/src/multimedia/video/qabstractvideofilter.cpp b/src/multimedia/video/qabstractvideofilter.cpp
new file mode 100644
index 000000000..e04cbf1df
--- /dev/null
+++ b/src/multimedia/video/qabstractvideofilter.cpp
@@ -0,0 +1,319 @@
+/****************************************************************************
+**
+** Copyright (C) 2015 Digia Plc and/or its subsidiary(-ies).
+** Contact: http://www.qt-project.org/legal
+**
+** This file is part of the Qt Toolkit.
+**
+** $QT_BEGIN_LICENSE:LGPL21$
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and Digia. For licensing terms and
+** conditions see http://qt.digia.com/licensing. For further information
+** use the contact form at http://qt.digia.com/contact-us.
+**
+** GNU Lesser General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU Lesser
+** General Public License version 2.1 or version 3 as published by the Free
+** Software Foundation and appearing in the file LICENSE.LGPLv21 and
+** LICENSE.LGPLv3 included in the packaging of this file. Please review the
+** following information to ensure the GNU Lesser General Public License
+** requirements will be met: https://www.gnu.org/licenses/lgpl.html and
+** http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
+**
+** In addition, as a special exception, Digia gives you certain additional
+** rights. These rights are described in the Digia Qt LGPL Exception
+** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+
+#include "qabstractvideofilter.h"
+
+QT_BEGIN_NAMESPACE
+
+/*!
+  \class QAbstractVideoFilter
+  \since 5.5
+  \brief The QAbstractVideoFilter class represents a filter that is applied to the video frames
+  received by a VideoOutput type.
+  \inmodule QtMultimedia
+
+  \ingroup multimedia
+  \ingroup multimedia_video
+
+  QAbstractVideoFilter provides a convenient way for applications to run image
+  processing, computer vision algorithms or any generic transformation or
+  calculation on the output of a VideoOutput type, regardless of the source
+  (video or camera). By providing a simple interface it allows applications and
+  third parties to easily develop QML types that provide image processing
+  algorithms using popular frameworks like \l{http://opencv.org}{OpenCV}. Due to
+  the close integration with the final stages of the Qt Multimedia video
+  pipeline, accelerated and possibly zero-copy solutions are feasible too: for
+  instance, a plugin providing OpenCL-based algorithms can use OpenCL's OpenGL
+  interop to use the OpenGL textures created by a hardware accelerated video
+  decoder, without additional readbacks and copies.
+
+  \note QAbstractVideoFilter is not always the best choice. To apply effects or
+  transformations using OpenGL shaders to the image shown on screen, the
+  standard Qt Quick approach of using ShaderEffect items in combination with
+  VideoOutput should be used. VideoFilter is not a replacement for this. It is
+  rather targeted for performing computations (that do not necessarily change
+  the image shown on screen) and computer vision algorithms provided by
+  external frameworks.
+
+  QAbstractVideoFilter is meant to be subclassed. The subclasses are then registered to
+  the QML engine, so they can be used as a QML type. The list of filters are
+  assigned to a VideoOutput type via its \l{QtMultimedia::VideoOutput::filters}{filters}
+  property.
+
+  A single filter represents one transformation or processing step on
+  a video frame. The output is a modified video frame, some arbitrary data or
+  both. For example, image transformations will result in a different image,
+  whereas an algorithm for detecting objects on an image will likely provide
+  a list of rectangles.
+
+  Arbitrary data can be represented as properties on the QAbstractVideoFilter subclass
+  and on the QObject or QJSValue instances passed to its signals. What exactly
+  these properties and signals are, is up to the individual video
+  filters. Completion of the operations can be indicated by
+  signals. Computations that do not result in a modified image will pass the
+  input image through so that subsequent filters can be placed after them.
+
+  Properties set on QAbstractVideoFilter serve as input to the computation, similarly
+  to how uniform values are specified in ShaderEffect types. The changed
+  property values are taken into use when the next video frame is processed.
+
+  The typical usage is to subclass QAbstractVideoFilter and QVideoFilterRunnable:
+
+  \badcode
+    class MyFilterRunnable : public QVideoFilterRunnable {
+    public:
+        QVideoFrame run(QVideoFrame *input, const QVideoSurfaceFormat &surfaceFormat, RunFlags flags) { ... }
+    };
+
+    class MyFilter : public QAbstractVideoFilter {
+    public:
+        QVideoFilterRunnable *createFilterRunnable() { return new MyFilterRunnable; }
+    signals:
+        void finished(QObject *result);
+    };
+
+    int main(int argc, char **argv) {
+        ...
+        qmlRegisterType<MyFilter>("my.uri", 1, 0, "MyFilter");
+        ...
+    }
+  \endcode
+
+  MyFilter is thus accessible from QML:
+
+  \badcode
+    import my.uri 1.0
+
+    Camera {
+        id: camera
+    }
+    MyFilter {
+        id: filter
+        // set properties, they can also be animated
+        onFinished: console.log("results of the computation: " + result)
+    }
+    VideoOutput {
+        source: camera
+        filters: [ filter ]
+        anchors.fill: parent
+    }
+  \endcode
+
+  This also allows providing filters in QML plugins, separately from the application.
+
+  \sa VideoOutput, Camera, MediaPlayer, QVideoFilterRunnable
+*/
+
+/*!
+  \class QVideoFilterRunnable
+  \since 5.5
+  \brief The QVideoFilterRunnable class represents the implementation of a filter
+  that owns all graphics and computational resources, and performs the actual filtering
+  or calculations.
+  \inmodule QtMultimedia
+
+  \ingroup multimedia
+  \ingroup multimedia_video
+
+  Video filters are split into QAbstractVideoFilter and corresponding QVideoFilterRunnable
+  instances, similar to QQuickItem and QSGNode. This is necessary to support
+  threaded rendering scenarios. When using the threaded render loop of the Qt
+  Quick scene graph, all rendering happens on a dedicated thread.
+  QVideoFilterRunnable instances always live on this thread and all its functions,
+  run(), the constructor, and the destructor, are guaranteed to be invoked on
+  that thread with the OpenGL context bound. QAbstractVideoFilter instances live on
+  the main (GUI) thread, like any other QObject and QQuickItem instances
+  created from QML.
+
+  Once created, QVideoFilterRunnable instances are managed by Qt Multimedia and
+  will be automatically destroyed and recreated when necessary, for example
+  when the scene graph is invalidated or the QQuickWindow changes or is closed.
+  Creation happens via the QAbstractVideoFilter::createFilterRunnable() factory function.
+
+  \sa QAbstractVideoFilter
+ */
+
+/*!
+  \fn QVideoFrame QVideoFilterRunnable::run(QVideoFrame *input, const QVideoSurfaceFormat &surfaceFormat, RunFlags flags)
+
+  Reimplement this function to perform filtering or computation on the \a
+  input video frame. Like the constructor and destructor, this function is
+  always called on the render thread with the OpenGL context bound.
+
+  Implementations that do not modify the video frame can simply return \a input.
+
+  It is safe to access properties of the associated QAbstractVideoFilter instance from
+  this function.
+
+  \a input will not be mapped, it is up to this function to call QVideoFrame::map()
+  and QVideoFrame::unmap() as necessary.
+
+  \a surfaceFormat provides additional information, for example it can be used
+  to determine which way is up in the input image as that is important for
+  filters to operate on multiple platforms with multiple cameras.
+
+  \a flags contains additional information about the filter's invocation. For
+  example the LastInChain flag indicates that the filter is the last in a
+  VideoOutput's associated filter list. This can be very useful in cases where
+  multiple filters are chained together and the work is performed on image data
+  in some custom format (for example a format specific to some computer vision
+  framework). To avoid conversion on every filter in the chain, all
+  intermediate filters can return a QVideoFrame hosting data in the custom
+  format. Only the last, where the flag is set, returns a QVideoFrame in a
+  format compatible with Qt.
+
+  Filters that want to expose the results of their computation to Javascript
+  code in QML can declare their own custom signals in the QAbstractVideoFilter
+  subclass to indicate the completion of the operation. For filters that only
+  calculate some results and do not modify the video frame, it is also possible
+  to operate asynchronously. They can queue the necessary operations using the
+  compute API and return from this function without emitting any signals. The
+  signal indicating the completion is then emitted only when the compute API
+  indicates that the operations were done and the results are available. Note
+  that it is strongly recommended to represent the filter's output data as a
+  separate instance of QJSValue or a QObject-derived class which is passed as a
+  parameter to the signal and becomes exposed to the Javascript engine. In case
+  of QObject the ownership of this object is controlled by the standard QML
+  rules: if it has no parent, ownership is transferred to the Javascript engine,
+  otherwise it stays with the emitter. Note that the signal connection may be
+  queued,for example when using the threaded render loop of Qt Quick, and so the
+  object must stay valid for a longer time, destroying it right after calling
+  this function is not safe. Using a dedicated results object is guaranteed to
+  be safe even when using threaded rendering. The same is not necessarily true
+  for properties on the QAbstractVideoFilter instance itself: properties can
+  safely be read in run() since the gui thread is blocked during that time but
+  writing may become problematic.
+
+  \note Avoid time consuming operations in this function as they block the
+  entire rendering of the application.
+
+  \note The handleType() and pixelFormat() of \a input is completely up to the
+  video decoding backend on the platform in use. On some platforms different
+  forms of input are used depending on the graphics stack. For example, when
+  playing back videos on Windows with the WMF backend, QVideoFrame contains
+  OpenGL-wrapped Direct3D textures in case of using ANGLE, but regular pixel
+  data when using desktop OpenGL (opengl32.dll). Similarly, the video file
+  format will often decide if the data is RGB or YUV, but this may also depend
+  on the decoder and the configuration in use. The returned video frame does
+  not have to be in the same format as the input, for example a filter with an
+  input of a QVideoFrame backed by system memory can output a QVideoFrame with
+  an OpenGL texture handle.
+
+  \sa QVideoFrame, QVideoSurfaceFormat
+ */
+
+/*!
+  \enum QVideoFilterRunnable::RunFlag
+
+  \value LastInChain Indicates that the filter runnable's associated QAbstractVideoFilter
+  is the last in the corresponding VideoOutput type's filters list, meaning
+  that the returned frame is the one that is going to be presented to the scene
+  graph without invoking any further filters.
+ */
+
+class QAbstractVideoFilterPrivate
+{
+public:
+    QAbstractVideoFilterPrivate() :
+        active(true)
+    { }
+
+    bool active;
+};
+
+/*!
+  \internal
+ */
+QVideoFilterRunnable::~QVideoFilterRunnable()
+{
+}
+
+/*!
+  Constructs a new QAbstractVideoFilter instance.
+ */
+QAbstractVideoFilter::QAbstractVideoFilter(QObject *parent) :
+    QObject(parent),
+    d_ptr(new QAbstractVideoFilterPrivate)
+{
+}
+
+/*!
+  \internal
+ */
+QAbstractVideoFilter::~QAbstractVideoFilter()
+{
+    delete d_ptr;
+}
+
+/*!
+  \return \c true if the filter is active.
+
+  By default filters are active. When set to \c false, the filter will be
+  ignored by the VideoOutput type.
+ */
+bool QAbstractVideoFilter::isActive() const
+{
+    Q_D(const QAbstractVideoFilter);
+    return d->active;
+}
+
+/*!
+  \internal
+ */
+void QAbstractVideoFilter::setActive(bool v)
+{
+    Q_D(QAbstractVideoFilter);
+    if (d->active != v) {
+        d->active = v;
+        emit activeChanged();
+    }
+}
+
+/*!
+  \fn QVideoFilterRunnable *QAbstractVideoFilter::createFilterRunnable()
+
+  Factory function to create a new instance of a QVideoFilterRunnable subclass
+  corresponding to this filter.
+
+  This function is called on the thread on which the Qt Quick scene graph
+  performs rendering, with the OpenGL context bound. Ownership of the returned
+  instance is transferred: the returned instance will live on the render thread
+  and will be destroyed automatically when necessary.
+
+  Typically, implementations of the function will simply construct a new
+  QVideoFilterRunnable instance, passing \c this to the constructor as the
+  filter runnables must know their associated QAbstractVideoFilter instance to
+  access dynamic properties and optionally emit signals.
+ */
+
+QT_END_NAMESPACE
diff --git a/src/multimedia/video/qabstractvideofilter.h b/src/multimedia/video/qabstractvideofilter.h
new file mode 100644
index 000000000..01e39fb8a
--- /dev/null
+++ b/src/multimedia/video/qabstractvideofilter.h
@@ -0,0 +1,85 @@
+/****************************************************************************
+**
+** Copyright (C) 2015 Digia Plc and/or its subsidiary(-ies).
+** Contact: http://www.qt-project.org/legal
+**
+** This file is part of the Qt Toolkit.
+**
+** $QT_BEGIN_LICENSE:LGPL21$
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and Digia. For licensing terms and
+** conditions see http://qt.digia.com/licensing. For further information
+** use the contact form at http://qt.digia.com/contact-us.
+**
+** GNU Lesser General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU Lesser
+** General Public License version 2.1 or version 3 as published by the Free
+** Software Foundation and appearing in the file LICENSE.LGPLv21 and
+** LICENSE.LGPLv3 included in the packaging of this file. Please review the
+** following information to ensure the GNU Lesser General Public License
+** requirements will be met: https://www.gnu.org/licenses/lgpl.html and
+** http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
+**
+** In addition, as a special exception, Digia gives you certain additional
+** rights. These rights are described in the Digia Qt LGPL Exception
+** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+
+#ifndef QABSTRACTVIDEOFILTER_H
+#define QABSTRACTVIDEOFILTER_H
+
+#include <QtCore/qobject.h>
+#include <QtMultimedia/qvideoframe.h>
+#include <QtMultimedia/qvideosurfaceformat.h>
+
+QT_BEGIN_NAMESPACE
+
+class QAbstractVideoFilterPrivate;
+
+class Q_MULTIMEDIA_EXPORT QVideoFilterRunnable
+{
+public:
+    enum RunFlag {
+        LastInChain = 0x01
+    };
+    Q_DECLARE_FLAGS(RunFlags, RunFlag)
+
+    virtual ~QVideoFilterRunnable();
+    virtual QVideoFrame run(QVideoFrame *input, const QVideoSurfaceFormat &surfaceFormat, RunFlags flags) = 0;
+};
+
+Q_DECLARE_OPERATORS_FOR_FLAGS(QVideoFilterRunnable::RunFlags)
+
+class Q_MULTIMEDIA_EXPORT QAbstractVideoFilter : public QObject
+{
+    Q_OBJECT
+    Q_PROPERTY(bool active READ isActive WRITE setActive NOTIFY activeChanged)
+
+public:
+    QAbstractVideoFilter(QObject *parent = 0);
+    ~QAbstractVideoFilter();
+
+    bool isActive() const;
+    void setActive(bool v);
+
+    virtual QVideoFilterRunnable *createFilterRunnable() = 0;
+
+Q_SIGNALS:
+    void activeChanged();
+
+private:
+    Q_DECLARE_PRIVATE(QAbstractVideoFilter)
+    Q_DISABLE_COPY(QAbstractVideoFilter)
+
+    QAbstractVideoFilterPrivate *d_ptr;
+};
+
+QT_END_NAMESPACE
+
+#endif // QABSTRACTVIDEOFILTER_H
diff --git a/src/multimedia/video/qvideoframe.cpp b/src/multimedia/video/qvideoframe.cpp
index 8dd23d2f5..869972a5c 100644
--- a/src/multimedia/video/qvideoframe.cpp
+++ b/src/multimedia/video/qvideoframe.cpp
@@ -369,6 +369,23 @@ QVideoFrame &QVideoFrame::operator =(const QVideoFrame &other)
 }
 
 /*!
+  \return \c true if this QVideoFrame and \a other reflect the same frame.
+ */
+bool QVideoFrame::operator==(const QVideoFrame &other) const
+{
+    // Due to explicit sharing we just compare the QSharedData which in turn compares the pointers.
+    return d == other.d;
+}
+
+/*!
+  \return \c true if this QVideoFrame and \a other do not reflect the same frame.
+ */
+bool QVideoFrame::operator!=(const QVideoFrame &other) const
+{
+    return d != other.d;
+}
+
+/*!
     Destroys a video frame.
 */
 QVideoFrame::~QVideoFrame()
diff --git a/src/multimedia/video/qvideoframe.h b/src/multimedia/video/qvideoframe.h
index d44219d36..89cf2fe32 100644
--- a/src/multimedia/video/qvideoframe.h
+++ b/src/multimedia/video/qvideoframe.h
@@ -107,6 +107,8 @@ public:
     ~QVideoFrame();
 
     QVideoFrame &operator =(const QVideoFrame &other);
+    bool operator==(const QVideoFrame &other) const;
+    bool operator!=(const QVideoFrame &other) const;
 
     bool isValid() const;
 
diff --git a/src/multimedia/video/video.pri b/src/multimedia/video/video.pri
index 161163783..3e39d9d18 100644
--- a/src/multimedia/video/video.pri
+++ b/src/multimedia/video/video.pri
@@ -6,7 +6,8 @@ PUBLIC_HEADERS += \
     video/qabstractvideosurface.h \
     video/qvideoframe.h \
     video/qvideosurfaceformat.h \
-    video/qvideoprobe.h
+    video/qvideoprobe.h \
+    video/qabstractvideofilter.h
 
 PRIVATE_HEADERS += \
     video/qabstractvideobuffer_p.h \
@@ -24,8 +25,5 @@ SOURCES += \
     video/qvideooutputorientationhandler.cpp \
     video/qvideosurfaceformat.cpp \
     video/qvideosurfaceoutput.cpp \
-    video/qvideoprobe.cpp
-
-
-
-
+    video/qvideoprobe.cpp \
+    video/qabstractvideofilter.cpp