shaped-texture: Fix damage propagation for rotated transforms with viewport

When a viewport source rect or destination size is set, `stex->dst_width`
gives us the the cropped and/or scaled size. At this step, we need the
uncropped/unscaled size however.

Note: this is only ever relevant if buffer transform and viewport are
used together - otherwise the values are the same.

Part-of: <https://gitlab.gnome.org/GNOME/mutter/-/merge_requests/1836>

1 files changed, 16 insertions, 5 deletions

diff --git a/src/compositor/meta-shaped-texture.c b/src/compositor/meta-shaped-texture.c
index 6f6d85345..6a8af828f 100644
--- a/src/compositor/meta-shaped-texture.c
+++ b/src/compositor/meta-shaped-texture.c
@@ -989,6 +989,8 @@ meta_shaped_texture_update_area (MetaShapedTexture     *stex,
                                  cairo_rectangle_int_t *clip)
 {
   MetaMonitorTransform inverted_transform;
+  int scaled_and_transformed_width;
+  int scaled_and_transformed_height;
 
   if (stex->texture == NULL)
     return FALSE;
@@ -1005,12 +1007,21 @@ meta_shaped_texture_update_area (MetaShapedTexture     *stex,
                                META_ROUNDING_STRATEGY_GROW,
                                clip);
 
+  if (meta_monitor_transform_is_rotated (stex->transform))
+    {
+      scaled_and_transformed_width = stex->tex_height / stex->buffer_scale;
+      scaled_and_transformed_height = stex->tex_width / stex->buffer_scale;
+    }
+  else
+    {
+      scaled_and_transformed_width = stex->tex_width / stex->buffer_scale;
+      scaled_and_transformed_height = stex->tex_height / stex->buffer_scale;
+    }
   inverted_transform = meta_monitor_transform_invert (stex->transform);
-  meta_shaped_texture_ensure_size_valid (stex);
   meta_rectangle_transform (clip,
                             inverted_transform,
-                            stex->dst_width,
-                            stex->dst_height,
+                            scaled_and_transformed_width,
+                            scaled_and_transformed_height,
                             clip);
 
   if (stex->has_viewport_src_rect || stex->has_viewport_dst_size)
@@ -1031,8 +1042,8 @@ meta_shaped_texture_update_area (MetaShapedTexture     *stex,
           viewport = (graphene_rect_t) {
             .origin.x = 0,
             .origin.y = 0,
-            .size.width = stex->tex_width / stex->buffer_scale,
-            .size.height = stex->tex_height / stex->buffer_scale,
+            .size.width = scaled_and_transformed_width,
+            .size.height = scaled_and_transformed_height,
           };
         }