QmlDesigner: Fix 3D edit gizmos orientation and rotation order

When selected node has non-default rotation order or orientation,
gizmos need to account for that.

Change-Id: Ie3817fd057b43f708ac1feea3e98e1e44f56d66a
Fixes: QDS-1290
Reviewed-by: Mahmoud Badri <mahmoud.badri@qt.io>
Reviewed-by: Thomas Hartmann <thomas.hartmann@qt.io>

10 files changed, 93 insertions, 32 deletions

diff --git a/share/qtcreator/qml/qmlpuppet/mockfiles/AutoScaleHelper.qml b/share/qtcreator/qml/qmlpuppet/mockfiles/AutoScaleHelper.qml
index dc4c38d100..9c1bf043dc 100644
--- a/share/qtcreator/qml/qmlpuppet/mockfiles/AutoScaleHelper.qml
+++ b/share/qtcreator/qml/qmlpuppet/mockfiles/AutoScaleHelper.qml
@@ -33,15 +33,18 @@ Node {
     property View3D view3D
     property Node target: parent
     property bool autoScale: true
+    property Camera camera: view3D.camera
 
     // Read-only
     property real relativeScale: 1
 
     onSceneTransformChanged: updateScale()
     onAutoScaleChanged: updateScale()
+    // Trigger delayed update on camera change to ensure camera values are correct
+    onCameraChanged: _generalHelper.requestOverlayUpdate();
 
     Connections {
-        target: view3D.camera
+        target: camera
         onSceneTransformChanged: updateScale()
     }
 
@@ -66,16 +69,22 @@ Node {
             // "anchor" distance. Map the two positions back to the target node, and measure the
             // distance between them now, in the 3D scene. The difference of the two distances,
             // view and scene, will tell us what the distance independent scale should be.
-            var posInView1 = view3D.mapFrom3DScene(scenePosition);
+
+            // Need to recreate vector as we need to adjust it and we can't do that on reference of
+            // scenePosition, which is read-only property
+            var scenePos = Qt.vector3d(scenePosition.x, scenePosition.y, scenePosition.z);
+            if (orientation === Node.RightHanded)
+                scenePos.z = -scenePos.z;
+
+            var posInView1 = view3D.mapFrom3DScene(scenePos);
             var posInView2 = Qt.vector3d(posInView1.x + 100, posInView1.y, posInView1.z);
 
             var rayPos1 = view3D.mapTo3DScene(Qt.vector3d(posInView2.x, posInView2.y, 0));
             var rayPos2 = view3D.mapTo3DScene(Qt.vector3d(posInView2.x, posInView2.y, 10));
 
-            var planeNormal = view3D.camera.forward;
-            var rayHitPos = helper.rayIntersectsPlane(rayPos1, rayPos2, scenePosition,
-                                                      planeNormal);
-            relativeScale = scenePosition.minus(rayHitPos).length() / 100;
+            var planeNormal = camera.forward;
+            var rayHitPos = helper.rayIntersectsPlane(rayPos1, rayPos2, scenePos, planeNormal);
+            relativeScale = scenePos.minus(rayHitPos).length() / 100;
         }
     }
 
diff --git a/share/qtcreator/qml/qmlpuppet/mockfiles/DirectionalDraggable.qml b/share/qtcreator/qml/qmlpuppet/mockfiles/DirectionalDraggable.qml
index 7ccfaa9816..081936d007 100644
--- a/share/qtcreator/qml/qmlpuppet/mockfiles/DirectionalDraggable.qml
+++ b/share/qtcreator/qml/qmlpuppet/mockfiles/DirectionalDraggable.qml
@@ -29,7 +29,6 @@ import MouseArea3D 1.0
 
 Model {
     id: rootModel
-    rotationOrder: Node.XYZr
 
     property View3D view3D
     property alias color: material.emissiveColor
@@ -59,6 +58,8 @@ Model {
 
         var maskedPosition = Qt.vector3d(scenePos.x, 0, 0);
         _pointerPosPressed = mouseArea.mapPositionToScene(maskedPosition);
+        if (targetNode.orientation === Node.RightHanded)
+            _pointerPosPressed.z = -_pointerPosPressed.z;
         var sp = targetNode.scenePosition;
         _targetStartPos = Qt.vector3d(sp.x, sp.y, sp.z);
         pressed(mouseArea);
@@ -68,9 +69,9 @@ Model {
     {
         var maskedPosition = Qt.vector3d(scenePos.x, 0, 0);
         var scenePointerPos = mouseArea.mapPositionToScene(maskedPosition);
-        return Qt.vector3d(scenePointerPos.x - _pointerPosPressed.x,
-                           scenePointerPos.y - _pointerPosPressed.y,
-                           scenePointerPos.z - _pointerPosPressed.z);
+        if (targetNode.orientation === Node.RightHanded)
+            scenePointerPos.z = -scenePointerPos.z;
+        return scenePointerPos.minus(_pointerPosPressed);
     }
 
     function handleDragged(mouseArea, scenePos)
diff --git a/share/qtcreator/qml/qmlpuppet/mockfiles/EditView3D.qml b/share/qtcreator/qml/qmlpuppet/mockfiles/EditView3D.qml
index d64005d1f6..f4ef3301de 100644
--- a/share/qtcreator/qml/qmlpuppet/mockfiles/EditView3D.qml
+++ b/share/qtcreator/qml/qmlpuppet/mockfiles/EditView3D.qml
@@ -188,8 +188,6 @@ Window {
             scale: autoScale.getScale(Qt.vector3d(5, 5, 5))
             highlightOnHover: true
             targetNode: viewWindow.selectedNode
-            position: viewWindow.selectedNode ? viewWindow.selectedNode.scenePosition
-                                              : Qt.vector3d(0, 0, 0)
             globalOrientation: btnLocalGlobal.toggled
             visible: selectedNode && btnMove.selected
             view3D: overlayView
@@ -203,8 +201,6 @@ Window {
             scale: autoScale.getScale(Qt.vector3d(5, 5, 5))
             highlightOnHover: true
             targetNode: viewWindow.selectedNode
-            position: viewWindow.selectedNode ? viewWindow.selectedNode.scenePosition
-                                              : Qt.vector3d(0, 0, 0)
             globalOrientation: false
             visible: selectedNode && btnScale.selected
             view3D: overlayView
@@ -218,8 +214,6 @@ Window {
             scale: autoScale.getScale(Qt.vector3d(7, 7, 7))
             highlightOnHover: true
             targetNode: viewWindow.selectedNode
-            position: viewWindow.selectedNode ? viewWindow.selectedNode.scenePosition
-                                              : Qt.vector3d(0, 0, 0)
             globalOrientation: btnLocalGlobal.toggled
             visible: selectedNode && btnRotate.selected
             view3D: overlayView
@@ -232,6 +226,7 @@ Window {
             id: autoScale
             view3D: overlayView
             position: moveGizmo.scenePosition
+            orientation: moveGizmo.orientation
         }
     }
 
diff --git a/share/qtcreator/qml/qmlpuppet/mockfiles/MoveGizmo.qml b/share/qtcreator/qml/qmlpuppet/mockfiles/MoveGizmo.qml
index b8e6cdafb9..37e7a17a2c 100644
--- a/share/qtcreator/qml/qmlpuppet/mockfiles/MoveGizmo.qml
+++ b/share/qtcreator/qml/qmlpuppet/mockfiles/MoveGizmo.qml
@@ -37,12 +37,17 @@ Node {
     readonly property bool dragging: arrowX.dragging || arrowY.dragging || arrowZ.dragging
                                      || planeX.dragging || planeY.dragging || planeZ.dragging
                                      || centerBall.dragging
+    position: targetNode ? targetNode.scenePosition : Qt.vector3d(0, 0, 0)
+    orientation: targetNode ? targetNode.orientation : Node.LeftHanded
 
     signal positionCommit()
     signal positionMove()
 
     Node {
-        rotation: globalOrientation || !targetNode ? Qt.vector3d(0, 0, 0) : targetNode.sceneRotation
+        rotation: globalOrientation || !moveGizmo.targetNode ? Qt.vector3d(0, 0, 0)
+                                                             : moveGizmo.targetNode.sceneRotation
+        rotationOrder: moveGizmo.targetNode ? moveGizmo.targetNode.rotationOrder : Node.YXZ
+        orientation: moveGizmo.orientation
 
         Arrow {
             id: arrowX
diff --git a/share/qtcreator/qml/qmlpuppet/mockfiles/Overlay2D.qml b/share/qtcreator/qml/qmlpuppet/mockfiles/Overlay2D.qml
index f4a85226b7..87528eddd3 100644
--- a/share/qtcreator/qml/qmlpuppet/mockfiles/Overlay2D.qml
+++ b/share/qtcreator/qml/qmlpuppet/mockfiles/Overlay2D.qml
@@ -55,9 +55,13 @@ Item {
     function updateOverlay()
     {
         var scenePos = targetNode ? targetNode.scenePosition : Qt.vector3d(0, 0, 0);
+        // Need separate variable as scenePos is reference to read-only property
+        var scenePosZ = scenePos.z
+        if (targetNode && targetNode.orientation === Node.RightHanded)
+            scenePosZ = -scenePosZ;
         var scenePosWithOffset = Qt.vector3d(scenePos.x + offset.x,
                                              scenePos.y + offset.y,
-                                             scenePos.z + offset.z);
+                                             scenePosZ + offset.z);
         var viewPos = targetView ? targetView.mapFrom3DScene(scenePosWithOffset)
                                  : Qt.vector3d(0, 0, 0);
         root.x = viewPos.x;
diff --git a/share/qtcreator/qml/qmlpuppet/mockfiles/PlanarDraggable.qml b/share/qtcreator/qml/qmlpuppet/mockfiles/PlanarDraggable.qml
index bd4fcd388b..935175054e 100644
--- a/share/qtcreator/qml/qmlpuppet/mockfiles/PlanarDraggable.qml
+++ b/share/qtcreator/qml/qmlpuppet/mockfiles/PlanarDraggable.qml
@@ -63,6 +63,8 @@ Model {
             return;
 
         _pointerPosPressed = mouseArea.mapPositionToScene(scenePos);
+        if (targetNode.orientation === Node.RightHanded)
+            _pointerPosPressed.z = -_pointerPosPressed.z;
         var sp = targetNode.scenePosition;
         _targetStartPos = Qt.vector3d(sp.x, sp.y, sp.z);
         pressed(mouseArea);
@@ -71,9 +73,9 @@ Model {
     function calcRelativeDistance(mouseArea, scenePos)
     {
         var scenePointerPos = mouseArea.mapPositionToScene(scenePos);
-        return Qt.vector3d(scenePointerPos.x - _pointerPosPressed.x,
-                           scenePointerPos.y - _pointerPosPressed.y,
-                           scenePointerPos.z - _pointerPosPressed.z);
+        if (targetNode.orientation === Node.RightHanded)
+            scenePointerPos.z = -scenePointerPos.z;
+        return scenePointerPos.minus(_pointerPosPressed);
     }
 
     function handleDragged(mouseArea, scenePos)
diff --git a/share/qtcreator/qml/qmlpuppet/mockfiles/RotateGizmo.qml b/share/qtcreator/qml/qmlpuppet/mockfiles/RotateGizmo.qml
index b730e86910..f481f7f435 100644
--- a/share/qtcreator/qml/qmlpuppet/mockfiles/RotateGizmo.qml
+++ b/share/qtcreator/qml/qmlpuppet/mockfiles/RotateGizmo.qml
@@ -39,6 +39,9 @@ Node {
     property real currentAngle
     property point currentMousePos
 
+    position: targetNode ? targetNode.scenePosition : Qt.vector3d(0, 0, 0)
+    orientation: targetNode ? targetNode.orientation : Node.LeftHanded
+
     signal rotateCommit()
     signal rotateChange()
 
@@ -69,7 +72,11 @@ Node {
     }
 
     Node {
-        rotation: globalOrientation || !targetNode ? Qt.vector3d(0, 0, 0) : targetNode.sceneRotation
+        id: rotNode
+        rotation: globalOrientation || !rotateGizmo.targetNode ? Qt.vector3d(0, 0, 0)
+                                                               : rotateGizmo.targetNode.sceneRotation
+        rotationOrder: rotateGizmo.targetNode ? rotateGizmo.targetNode.rotationOrder : Node.YXZ
+        orientation: rotateGizmo.orientation
 
         RotateRing {
             id: rotRingX
@@ -166,7 +173,14 @@ Node {
             if (!rotateGizmo.targetNode)
                 return;
 
-            _targetPosOnScreen = view3D.mapFrom3DScene(rotateGizmo.targetNode.scenePosition);
+            // Need to recreate vector as we need to adjust it and we can't do that on reference of
+            // scenePosition, which is read-only property
+            var scenePos = Qt.vector3d(rotateGizmo.targetNode.scenePosition.x,
+                                       rotateGizmo.targetNode.scenePosition.y,
+                                       rotateGizmo.targetNode.scenePosition.z);
+            if (rotateGizmo.targetNode && rotateGizmo.targetNode.orientation === Node.RightHanded)
+                scenePos.z = -scenePos.z
+            _targetPosOnScreen = view3D.mapFrom3DScene(scenePos);
             _targetPosOnScreen.z = 0;
             _pointerPosPressed = Qt.vector3d(screenPos.x, screenPos.y, 0);
 
diff --git a/share/qtcreator/qml/qmlpuppet/mockfiles/RotateRing.qml b/share/qtcreator/qml/qmlpuppet/mockfiles/RotateRing.qml
index 3d21f8fd6a..a026b49ba4 100644
--- a/share/qtcreator/qml/qmlpuppet/mockfiles/RotateRing.qml
+++ b/share/qtcreator/qml/qmlpuppet/mockfiles/RotateRing.qml
@@ -77,7 +77,15 @@ Model {
         if (!targetNode)
             return;
 
-        _targetPosOnScreen = view3D.mapFrom3DScene(targetNode.scenePosition);
+        // Need to recreate vector as we need to adjust it and we can't do that on reference of
+        // scenePosition, which is read-only property
+        var scenePos = Qt.vector3d(targetNode.scenePosition.x,
+                                   targetNode.scenePosition.y,
+                                   targetNode.scenePosition.z);
+        if (targetNode && targetNode.orientation === Node.RightHanded)
+            scenePos.z = -scenePos.z
+
+        _targetPosOnScreen = view3D.mapFrom3DScene(scenePos);
         _targetPosOnScreen.z = 0;
         _pointerPosPressed = Qt.vector3d(screenPos.x, screenPos.y, 0);
         _trackBall = angle < 0.1;
diff --git a/share/qtcreator/qml/qmlpuppet/mockfiles/ScaleGizmo.qml b/share/qtcreator/qml/qmlpuppet/mockfiles/ScaleGizmo.qml
index 8a55812e71..5f4a6d1641 100644
--- a/share/qtcreator/qml/qmlpuppet/mockfiles/ScaleGizmo.qml
+++ b/share/qtcreator/qml/qmlpuppet/mockfiles/ScaleGizmo.qml
@@ -38,11 +38,16 @@ Node {
                                      || planeX.dragging || planeY.dragging || planeZ.dragging
                                      || centerMouseArea.dragging
 
+    position: targetNode ? targetNode.scenePosition : Qt.vector3d(0, 0, 0)
+    orientation: targetNode ? targetNode.orientation : Node.LeftHanded
+
     signal scaleCommit()
     signal scaleChange()
 
     Node {
         rotation: globalOrientation || !targetNode ? Qt.vector3d(0, 0, 0) : targetNode.sceneRotation
+        rotationOrder: scaleGizmo.targetNode ? scaleGizmo.targetNode.rotationOrder : Node.YXZ
+        orientation: scaleGizmo.orientation
 
         ScaleRod {
             id: scaleRodX
diff --git a/share/qtcreator/qml/qmlpuppet/qml2puppet/editor3d/mousearea3d.cpp b/share/qtcreator/qml/qmlpuppet/qml2puppet/editor3d/mousearea3d.cpp
index d571b24dcb..3dd360e1ea 100644
--- a/share/qtcreator/qml/qmlpuppet/qml2puppet/editor3d/mousearea3d.cpp
+++ b/share/qtcreator/qml/qmlpuppet/qml2puppet/editor3d/mousearea3d.cpp
@@ -344,11 +344,17 @@ qreal QmlDesigner::Internal::MouseArea3D::getNewRotationAngle(
         QQuick3DNode *node, const QVector3D &pressPos, const QVector3D &currentPos,
         const QVector3D &nodePos, qreal prevAngle, bool trackBall)
 {
-    const QVector3D cameraToNodeDir = getCameraToNodeDir(node);
+    // Get camera to node direction in node orientation
+    QVector3D cameraToNodeDir = getCameraToNodeDir(node);
     if (trackBall) {
         // Only the distance in plane direction is relevant in trackball drag
         QVector3D dragDir = QVector3D::crossProduct(getNormal(), cameraToNodeDir).normalized();
-        QVector3D screenDragDir = m_view3D->mapFrom3DScene(node->scenePosition() + dragDir);
+        QVector3D scenePos = node->scenePosition();
+        if (node->orientation() == QQuick3DNode::RightHanded) {
+            scenePos.setZ(-scenePos.z());
+            dragDir = -dragDir;
+        }
+        QVector3D screenDragDir = m_view3D->mapFrom3DScene(scenePos + dragDir);
         screenDragDir.setZ(0);
         dragDir = (screenDragDir - nodePos).normalized();
         const QVector3D pressToCurrent = (currentPos - pressPos);
@@ -361,7 +367,9 @@ qreal QmlDesigner::Internal::MouseArea3D::getNewRotationAngle(
         qreal angle = qAcos(qreal(QVector3D::dotProduct(nodeToPress, nodeToCurrent)));
 
         // Determine drag direction left/right
-        const QVector3D dragNormal = QVector3D::crossProduct(nodeToPress, nodeToCurrent).normalized();
+        QVector3D dragNormal = QVector3D::crossProduct(nodeToPress, nodeToCurrent).normalized();
+        if (node->orientation() == QQuick3DNode::RightHanded)
+            dragNormal = -dragNormal;
         angle *= QVector3D::dotProduct(QVector3D(0.f, 0.f, 1.f), dragNormal) < 0 ? -1.0 : 1.0;
 
         // Determine drag ring orientation relative to camera
@@ -392,7 +400,10 @@ void QmlDesigner::Internal::MouseArea3D::applyRotationAngleToNode(
 {
     if (!qFuzzyIsNull(angle)) {
         node->setRotation(startRotation);
-        node->rotate(qRadiansToDegrees(angle), getNormal(), QQuick3DNode::SceneSpace);
+        QVector3D normal = getNormal();
+        if (orientation() != node->orientation())
+            normal.setZ(-normal.z());
+        node->rotate(qRadiansToDegrees(angle), normal, QQuick3DNode::SceneSpace);
     }
 }
 
@@ -407,6 +418,10 @@ void MouseArea3D::applyFreeRotation(QQuick3DNode *node, const QVector3D &startRo
     const float *dataPtr(sceneTransform().data());
     QVector3D xAxis = QVector3D(-dataPtr[0], -dataPtr[1], -dataPtr[2]).normalized();
     QVector3D yAxis = QVector3D(-dataPtr[4], -dataPtr[5], -dataPtr[6]).normalized();
+    if (node->orientation() == QQuick3DNode::RightHanded) {
+        xAxis = QVector3D(-xAxis.x(), -xAxis.y(), xAxis.z());
+        yAxis = QVector3D(-yAxis.x(), -yAxis.y(), yAxis.z());
+    }
 
     QVector3D finalAxis = (dragVector.x() * yAxis + dragVector.y() * xAxis);
 
@@ -596,11 +611,14 @@ QVector3D MouseArea3D::getCameraToNodeDir(QQuick3DNode *node) const
 {
     QVector3D dir;
     if (qobject_cast<QQuick3DOrthographicCamera *>(m_view3D->camera())) {
-        dir = m_view3D->camera()->cameraNode()->getDirection();
-        // Camera direction has x and y flipped
-        dir = QVector3D(-dir.x(), -dir.y(), dir.z());
+        dir = -m_view3D->camera()->cameraNode()->getDirection();
+        dir.setZ(-dir.z());
     } else {
-        dir = (node->scenePosition() - m_view3D->camera()->scenePosition()).normalized();
+        QVector3D camPos = m_view3D->camera()->scenePosition();
+        QVector3D nodePos = node->scenePosition();
+        if (node->orientation() == QQuick3DNode::RightHanded)
+            nodePos.setZ(-nodePos.z());
+        dir = (node->scenePosition() - camPos).normalized();
     }
     return dir;
 }