clutter/pick-stack: Use graphene_box_t for axis-aligned rectangles

The most common case for Clutter is 2D axis-aligned actors, which
maintain these properties even after projecting to eye coordinates.
In those cases, we can use a simpler hit test by checking against
boxes.

Not only this is simpler, but this maintains an important aspect
of picking that is a requirement for Clutter: watertightness. Even
though the triangles checks do work on x86_64, they do not guarantee
watertightness. This breaks tests on ARM.

Use graphene_box_t to hit-test axis-aligned 2D actors.

Part-of: <https://gitlab.gnome.org/GNOME/mutter/-/merge_requests/1599>

1 files changed, 33 insertions, 0 deletions

diff --git a/clutter/clutter/clutter-pick-stack.c b/clutter/clutter/clutter-pick-stack.c
index 6a0b13e58..71d679833 100644
--- a/clutter/clutter/clutter-pick-stack.c
+++ b/clutter/clutter/clutter-pick-stack.c
@@ -91,6 +91,30 @@ maybe_project_record (Record *rec)
     }
 }
 
+static inline gboolean
+is_axis_aligned_2d_rectangle (const graphene_point3d_t vertices[4])
+{
+  int i;
+
+  for (i = 0; i < 4; i++)
+    {
+      if (!G_APPROX_VALUE (vertices[i].z,
+                           vertices[(i + 1) % 4].z,
+                           FLT_EPSILON))
+        return FALSE;
+
+      if (!G_APPROX_VALUE (vertices[i].x,
+                           vertices[(i + 1) % 4].x,
+                           FLT_EPSILON) &&
+          !G_APPROX_VALUE (vertices[i].y,
+                           vertices[(i + 1) % 4].y,
+                           FLT_EPSILON))
+        return FALSE;
+    }
+
+  return TRUE;
+}
+
 static gboolean
 ray_intersects_input_region (Record                   *rec,
                              const graphene_ray_t     *ray,
@@ -100,6 +124,15 @@ ray_intersects_input_region (Record                   *rec,
 
   maybe_project_record (rec);
 
+  if (G_LIKELY (is_axis_aligned_2d_rectangle (rec->vertices)))
+    {
+      graphene_box_t box;
+
+      graphene_box_init_from_points (&box, 4, rec->vertices);
+      return graphene_box_contains_point (&box, point) ||
+             graphene_ray_intersects_box (ray, &box);
+    }
+
   /*
    * Degrade the projected quad into the following triangles:
    *