Fix multi-view picking

When using Scene3DView, qt3d still has a single scene graph and and a
single framegraph. It automatically creates layers and layer filters
to make sure the right objects are only rendered in the right view.

This affects picking though as it was not aware of layer filters.
This patch collects the filtered layers for each view and makes sure
only entities matching those layers are tested for picking (this uses
code that existed for ray casting).

This changes the behavior of Qt3D though as non rendered objects (ie,
excluded by layer filtering) are no longer pickable. Only way now would
be to provide a shader that just discarded everything.

Note: Scene3DView is not available in Qt6 so on this branch this only
really affects custom scenes with multiple views each showing different
content.

[ChangeLog] Non rendered entities (due to layer filtering) are no
longer pickable

Ticket-number: QTBUG-94214
Change-Id: I8515368e43342b33ac219dff533e92efa72fbe7d
Reviewed-by: Paul Lemire <paul.lemire@kdab.com>
(cherry picked from commit d79376732105dea749e71cdb114251084c7138a9)
Reviewed-by: Mike Krus <mike.krus@kdab.com>

7 files changed, 271 insertions, 4 deletions

diff --git a/src/render/jobs/pickboundingvolumejob.cpp b/src/render/jobs/pickboundingvolumejob.cpp
index c38d3b2e9..e157e2265 100644
--- a/src/render/jobs/pickboundingvolumejob.cpp
+++ b/src/render/jobs/pickboundingvolumejob.cpp
@@ -330,6 +330,7 @@ bool PickBoundingVolumeJob::runHelper()
             }
 
             PickingUtils::HierarchicalEntityPicker entityPicker(ray);
+            entityPicker.setFilterLayers(vca.layers, vca.layerFilterMode);
             if (entityPicker.collectHits(m_manager, m_node)) {
                 if (trianglePickingRequested) {
                     PickingUtils::TriangleCollisionGathererFunctor gathererFunctor;
@@ -472,6 +473,7 @@ void PickBoundingVolumeJob::dispatchPickEvents(const QMouseEvent &event,
                 case QEvent::MouseButtonPress: {
                     // Store pressed object handle
                     m_currentPicker = objectPickerHandle;
+                    m_currentViewport = viewportNodeId;
                     // Send pressed event to m_currentPicker
                     d->dispatches.push_back({objectPicker->peerId(), event.type(), pickEvent, viewportNodeId});
                     objectPicker->setPressed(true);
@@ -489,6 +491,7 @@ void PickBoundingVolumeJob::dispatchPickEvents(const QMouseEvent &event,
                                                  PickBoundingVolumeJobPrivate::MouseButtonClick,
                                                  pickEvent, viewportNodeId});
                         m_currentPicker = HObjectPicker();
+                        m_currentViewport = {};
                     }
                     break;
                 }
@@ -533,8 +536,9 @@ void PickBoundingVolumeJob::dispatchPickEvents(const QMouseEvent &event,
         switch (event.type()) {
         case QEvent::MouseButtonRelease: {
             // Send release event to m_currentPicker
-            if (lastCurrentPicker != nullptr) {
+            if (lastCurrentPicker != nullptr && m_currentViewport == viewportNodeId) {
                 m_currentPicker = HObjectPicker();
+                m_currentViewport = {};
                 QPickEventPtr pickEvent(new QPickEvent);
                 lastCurrentPicker->setPressed(false);
                 d->dispatches.push_back({lastCurrentPicker->peerId(), event.type(), pickEvent, viewportNodeId});
diff --git a/src/render/jobs/pickboundingvolumejob_p.h b/src/render/jobs/pickboundingvolumejob_p.h
index 2e24b59fb..ee8a3b74b 100644
--- a/src/render/jobs/pickboundingvolumejob_p.h
+++ b/src/render/jobs/pickboundingvolumejob_p.h
@@ -112,6 +112,7 @@ private:
     bool m_pickersDirty;
     bool m_oneHoverAtLeast;
     HObjectPicker m_currentPicker;
+    Qt3DCore::QNodeId m_currentViewport;
     QVector<HObjectPicker> m_hoveredPickers;
     QVector<HObjectPicker> m_hoveredPickersToClear;
 };
diff --git a/src/render/jobs/pickboundingvolumeutils.cpp b/src/render/jobs/pickboundingvolumeutils.cpp
index 9866b9bee..d8572301d 100644
--- a/src/render/jobs/pickboundingvolumeutils.cpp
+++ b/src/render/jobs/pickboundingvolumeutils.cpp
@@ -52,6 +52,8 @@
 #include <Qt3DRender/private/segmentsvisitor_p.h>
 #include <Qt3DRender/private/pointsvisitor_p.h>
 #include <Qt3DRender/private/layer_p.h>
+#include <Qt3DRender/private/layerfilternode_p.h>
+#include <Qt3DRender/private/rendersettings_p.h>
 
 #include <vector>
 #include <algorithm>
@@ -105,6 +107,19 @@ ViewportCameraAreaDetails ViewportCameraAreaGatherer::gatherUpViewportCameraArea
                 // prevent picking in the presence of a NoPicking node
                 return {};
             }
+            case FrameGraphNode::LayerFilter: {
+                auto fnode = static_cast<const LayerFilterNode *>(node);
+                const auto &layers = fnode->layerIds();
+                for (const auto &id: layers)
+                    vca.layers.append(id);
+                switch (fnode->filterMode()) {
+                case Qt3DRender::QLayerFilter::AcceptAllMatchingLayers: vca.layerFilterMode = Qt3DRender::QAbstractRayCaster::AcceptAllMatchingLayers; break;
+                case Qt3DRender::QLayerFilter::AcceptAnyMatchingLayers: vca.layerFilterMode = Qt3DRender::QAbstractRayCaster::AcceptAnyMatchingLayers; break;
+                case Qt3DRender::QLayerFilter::DiscardAllMatchingLayers: vca.layerFilterMode = Qt3DRender::QAbstractRayCaster::DiscardAllMatchingLayers; break;
+                case Qt3DRender::QLayerFilter::DiscardAnyMatchingLayers: vca.layerFilterMode = Qt3DRender::QAbstractRayCaster::DiscardAnyMatchingLayers; break;
+                }
+                break;
+            }
             default:
                 break;
             }
diff --git a/src/render/jobs/pickboundingvolumeutils_p.h b/src/render/jobs/pickboundingvolumeutils_p.h
index 8af7aae35..d48944f1f 100644
--- a/src/render/jobs/pickboundingvolumeutils_p.h
+++ b/src/render/jobs/pickboundingvolumeutils_p.h
@@ -83,8 +83,10 @@ struct Q_AUTOTEST_EXPORT ViewportCameraAreaDetails
     QRectF viewport;
     QSize area;
     QSurface *surface = nullptr;
+    Qt3DCore::QNodeIdVector layers;
+    QAbstractRayCaster::FilterMode layerFilterMode = QAbstractRayCaster::AcceptAnyMatchingLayers;
 };
-QT3D_DECLARE_TYPEINFO_3(Qt3DRender, Render, PickingUtils, ViewportCameraAreaDetails, Q_PRIMITIVE_TYPE)
+QT3D_DECLARE_TYPEINFO_3(Qt3DRender, Render, PickingUtils, ViewportCameraAreaDetails, Q_COMPLEX_TYPE)
 
 class Q_AUTOTEST_EXPORT ViewportCameraAreaGatherer
 {
diff --git a/tests/auto/render/pickboundingvolumejob/pickboundingvolumejob.qrc b/tests/auto/render/pickboundingvolumejob/pickboundingvolumejob.qrc
index 76150da31..2b35ed718 100644
--- a/tests/auto/render/pickboundingvolumejob/pickboundingvolumejob.qrc
+++ b/tests/auto/render/pickboundingvolumejob/pickboundingvolumejob.qrc
@@ -12,5 +12,6 @@
         <file>testscene_cameraposition.qml</file>
         <file>testscene_priorityoverlapping.qml</file>
         <file>testscene_nopicking.qml</file>
+        <file>testscene_layerfilter.qml</file>
     </qresource>
 </RCC>
diff --git a/tests/auto/render/pickboundingvolumejob/testscene_layerfilter.qml b/tests/auto/render/pickboundingvolumejob/testscene_layerfilter.qml
new file mode 100644
index 000000000..392623a6a
--- /dev/null
+++ b/tests/auto/render/pickboundingvolumejob/testscene_layerfilter.qml
@@ -0,0 +1,162 @@
+/****************************************************************************
+**
+** Copyright (C) 2021 Klaralvdalens Datakonsult AB (KDAB).
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of the Qt3D module of the Qt Toolkit.
+**
+** $QT_BEGIN_LICENSE:BSD$
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** BSD License Usage
+** Alternatively, you may use this file under the terms of the BSD license
+** as follows:
+**
+** "Redistribution and use in source and binary forms, with or without
+** modification, are permitted provided that the following conditions are
+** met:
+**   * Redistributions of source code must retain the above copyright
+**     notice, this list of conditions and the following disclaimer.
+**   * Redistributions in binary form must reproduce the above copyright
+**     notice, this list of conditions and the following disclaimer in
+**     the documentation and/or other materials provided with the
+**     distribution.
+**   * Neither the name of The Qt Company Ltd nor the names of its
+**     contributors may be used to endorse or promote products derived
+**     from this software without specific prior written permission.
+**
+**
+** THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+** "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+** LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+** A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+** OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+** SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+** LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+** DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+** THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+** (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+** OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE."
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+
+import Qt3D.Core 2.0
+import Qt3D.Render 2.0
+import Qt3D.Extras 2.0
+import QtQuick.Window 2.0
+
+Entity {
+    id: sceneRoot
+
+    Window {
+        id: _view
+        width: 600
+        height: 600
+        visible: true
+    }
+
+    Camera {
+        id: camera
+        projectionType: CameraLens.PerspectiveProjection
+        fieldOfView: 45
+        aspectRatio: _view.width / 2 / _view.height
+        nearPlane : 0.1
+        farPlane : 1000.0
+        position: Qt.vector3d( 0.0, 0.0, -10.0 )
+        upVector: Qt.vector3d( 0.0, 1.0, 0.0 )
+        viewCenter: Qt.vector3d( 0.0, 0.0, 0.0 )
+    }
+
+    Camera {
+        id: camera2
+        projectionType: CameraLens.PerspectiveProjection
+        fieldOfView: 45
+        aspectRatio: _view.width / 2 / _view.height
+        nearPlane : 0.1
+        farPlane : 1000.0
+        position: Qt.vector3d( 0.0, 0.0, -10.0 )
+        upVector: Qt.vector3d( 0.0, 1.0, 0.0 )
+        viewCenter: Qt.vector3d( 0.0, 0.0, 0.0 )
+    }
+
+    FirstPersonCameraController {
+        camera: camera
+    }
+
+    DirectionalLight {
+        worldDirection: camera.viewVector.times(-1)
+    }
+
+    // Draw 2 viewports
+    components: [
+        RenderSettings {
+            RenderSurfaceSelector {
+                surface: _view
+
+                Viewport {
+                    normalizedRect: Qt.rect(0.0, 0.0, 0.5, 1.0)
+                    ClearBuffers {
+                        buffers : ClearBuffers.ColorDepthBuffer
+                        clearColor: "white"
+                        CameraSelector {
+                            camera: camera
+
+                            LayerFilter {
+                                layers: [ layer1 ]
+                            }
+                        }
+                    }
+                }
+
+                Viewport {
+                    normalizedRect: Qt.rect(0.5, 0.0, 0.5, 1.0)
+                    CameraSelector {
+                        camera: camera2
+
+                        LayerFilter {
+                            layers: [ layer2 ]
+                        }
+                    }
+                }
+            }
+        }
+    ]
+
+    CuboidMesh { id: cubeMesh }
+
+    Entity {
+        readonly property ObjectPicker objectPicker: ObjectPicker {
+            objectName: "Picker1"
+            onClicked: console.log("o1")
+        }
+        readonly property Transform transform: Transform {
+            scale: 4
+        }
+        readonly property PhongMaterial material: PhongMaterial { diffuse: "red" }
+        readonly property Layer layer: Layer { id: layer1 }
+
+        components: [cubeMesh, transform, material, objectPicker, layer ]
+    }
+
+    Entity {
+        readonly property ObjectPicker objectPicker: ObjectPicker {
+            objectName: "Picker2"
+            onClicked: console.log("o2")
+        }
+        readonly property Transform transform: Transform {
+            scale: 3
+        }
+        readonly property PhongMaterial material: PhongMaterial { diffuse: "green" }
+        readonly property Layer layer: Layer { id: layer2 }
+
+        components: [cubeMesh, transform, material, objectPicker, layer ]
+    }
+}
diff --git a/tests/auto/render/pickboundingvolumejob/tst_pickboundingvolumejob.cpp b/tests/auto/render/pickboundingvolumejob/tst_pickboundingvolumejob.cpp
index 0c1ee0183..c090e0d6a 100644
--- a/tests/auto/render/pickboundingvolumejob/tst_pickboundingvolumejob.cpp
+++ b/tests/auto/render/pickboundingvolumejob/tst_pickboundingvolumejob.cpp
@@ -287,7 +287,6 @@ private:
     }
 
 private Q_SLOTS:
-
     void viewportCameraAreaGather()
     {
         // GIVEN
@@ -1547,6 +1546,90 @@ private Q_SLOTS:
         QCOMPARE(mouseExited.count(), 0);
     }
 
+    void checkPickerAndLayerFilters()
+    {
+        // GIVEN
+        QmlSceneReader sceneReader(QUrl("qrc:/testscene_layerfilter.qml"));
+        QScopedPointer<Qt3DCore::QNode> root(qobject_cast<Qt3DCore::QNode *>(sceneReader.root()));
+        QVERIFY(root);
+
+        QScopedPointer<Qt3DRender::TestAspect> test(new Qt3DRender::TestAspect(root.data()));
+        TestArbiter arbiter;
+
+        // Runs Required jobs
+        runRequiredJobs(test.data());
+
+        // THEN
+        // object partially obscured by another viewport, make sure only visible portion is pickable
+        QList<Qt3DRender::QObjectPicker *> pickers = root->findChildren<Qt3DRender::QObjectPicker *>();
+        QCOMPARE(pickers.size(), 2);
+
+        Qt3DRender::QObjectPicker *picker1 = pickers.front();
+        QCOMPARE(picker1->objectName(), QLatin1String("Picker1"));
+
+        Qt3DRender::Render::ObjectPicker *backendPicker1 = test->nodeManagers()->objectPickerManager()->lookupResource(picker1->id());
+        QVERIFY(backendPicker1);
+
+        QSignalSpy mouseButtonPressedSpy1(picker1, &Qt3DRender::QObjectPicker::pressed);
+
+        QVERIFY(mouseButtonPressedSpy1.isValid());
+
+        Qt3DRender::QObjectPicker *picker2 = pickers.last();
+        QCOMPARE(picker2->objectName(), QLatin1String("Picker2"));
+
+        Qt3DRender::Render::ObjectPicker *backendPicker2 = test->nodeManagers()->objectPickerManager()->lookupResource(picker2->id());
+        QVERIFY(backendPicker2);
+
+        QSignalSpy mouseButtonPressedSpy2(picker2, &Qt3DRender::QObjectPicker::pressed);
+
+        QVERIFY(mouseButtonPressedSpy2.isValid());
+
+        // WHEN -> Pressed on object in vp1
+        Qt3DRender::Render::PickBoundingVolumeJob pickBVJob;
+        initializePickBoundingVolumeJob(&pickBVJob, test.data());
+
+        {
+            QList<QPair<QObject *, QMouseEvent>> events;
+            events.push_back({nullptr, QMouseEvent(QMouseEvent::MouseButtonPress, QPointF(150., 300.),
+                              Qt::LeftButton, Qt::LeftButton, Qt::NoModifier) });
+            pickBVJob.setMouseEvents(events);
+            bool earlyReturn = !pickBVJob.runHelper();
+            Qt3DCore::QAspectJobPrivate::get(&pickBVJob)->postFrame(test->aspectManager());
+
+            // THEN -> Pressed
+            QVERIFY(!earlyReturn);
+            QVERIFY(backendPicker1->isPressed());
+            QVERIFY(picker1->isPressed());
+            QVERIFY(!backendPicker2->isPressed());
+            QVERIFY(!picker2->isPressed());
+            QCOMPARE(mouseButtonPressedSpy1.count(), 1);
+            QCOMPARE(mouseButtonPressedSpy2.count(), 0);
+
+            events.clear();
+
+            events.push_back({nullptr, QMouseEvent(QMouseEvent::MouseButtonRelease, QPointF(150., 300.),
+                                     Qt::LeftButton, Qt::LeftButton, Qt::NoModifier)});
+            pickBVJob.setMouseEvents(events);
+            pickBVJob.runHelper();
+        }
+
+        {
+            QList<QPair<QObject *, QMouseEvent>> events;
+            events.push_back({nullptr, QMouseEvent(QMouseEvent::MouseButtonPress, QPointF(450., 300.),
+                              Qt::LeftButton, Qt::LeftButton, Qt::NoModifier)});
+            pickBVJob.setMouseEvents(events);
+            bool earlyReturn = !pickBVJob.runHelper();
+            Qt3DCore::QAspectJobPrivate::get(&pickBVJob)->postFrame(test->aspectManager());
+
+            // THEN -> Nothing happened
+            QVERIFY(!earlyReturn);
+            QVERIFY(backendPicker2->isPressed());
+            QVERIFY(picker2->isPressed());
+            QCOMPARE(mouseButtonPressedSpy1.count(), 1);
+            QCOMPARE(mouseButtonPressedSpy2.count(), 1);
+        }
+    }
+
     void checkMultipleRayDirections_data()
     {
         QTest::addColumn<QVector3D>("cameraOrigin");
@@ -1879,7 +1962,6 @@ private Q_SLOTS:
         QCOMPARE(vca.cameraId, camera->id());
         QCOMPARE(vca.viewport, QRectF(0., 0., 1., 1.));
     }
-
 };
 
 QTEST_MAIN(tst_PickBoundingVolumeJob)