vulkan: Don't crash with overly large nodeswip/otte/vulkan

... when these nodes are used as children of a 3D transform node and we
lose the clip.

4 files changed, 28 insertions, 1 deletions

diff --git a/gsk/vulkan/gskvulkanrenderpass.c b/gsk/vulkan/gskvulkanrenderpass.c
index 63e9beeea9..ae15fa478a 100644
--- a/gsk/vulkan/gskvulkanrenderpass.c
+++ b/gsk/vulkan/gskvulkanrenderpass.c
@@ -621,7 +621,7 @@ gsk_vulkan_render_pass_add_transform_node (GskVulkanRenderPass       *self,
         graphene_vec3_t matrix_scale;
         graphene_vec3_t shear;
         GskTransform *clip_transform;
-        float scale_x, scale_y;
+        float scale_x, scale_y, old_pixels, new_pixels;
 
         clip_transform = gsk_transform_transform (gsk_transform_translate (NULL, &state->offset), transform);
 
@@ -648,6 +648,15 @@ gsk_vulkan_render_pass_add_transform_node (GskVulkanRenderPass       *self,
 
         scale_x = fabs (graphene_vec3_get_x (&matrix_scale));
         scale_y = fabs (graphene_vec3_get_y (&matrix_scale));
+        old_pixels = graphene_vec2_get_x (&state->scale) * graphene_vec2_get_y (&state->scale) *
+                     state->clip.rect.bounds.size.width * state->clip.rect.bounds.size.height;
+        new_pixels = scale_x * scale_y * new_state.clip.rect.bounds.size.width * new_state.clip.rect.bounds.size.height;
+        if (new_pixels > 2 * old_pixels)
+          {
+            float forced_downscale = 2 * old_pixels / new_pixels;
+            scale_x *= forced_downscale;
+            scale_y *= forced_downscale;
+          }
         new_state.modelview = gsk_transform_scale (new_state.modelview, 1 / scale_x, 1 / scale_y);
         graphene_vec2_init (&new_state.scale, scale_x, scale_y);
         new_state.offset = *graphene_point_zero ();
diff --git a/testsuite/gsk/compare/transform-huge-child-3d.node b/testsuite/gsk/compare/transform-huge-child-3d.node
new file mode 100644
index 0000000000..832a99c5a7
--- /dev/null
+++ b/testsuite/gsk/compare/transform-huge-child-3d.node
@@ -0,0 +1,17 @@
+/* a normal clip */
+clip {
+  clip: 0 0 50 50;
+  /* a 3D transform */
+  child: transform {
+    transform: rotateX(1);
+    /* a node that forces an offscreen */
+    child: rounded-clip {
+      clip: -30000 -30000 60000 60000 / 5;
+      child: color {
+        /* way too big, but it's clipped, so no problem */
+        bounds: -30000 -30000 60000 60000;
+        color: red;
+      }
+    }
+  }
+}
diff --git a/testsuite/gsk/compare/transform-huge-child-3d.png b/testsuite/gsk/compare/transform-huge-child-3d.png
new file mode 100644
index 0000000000..9a5764a9b2
--- /dev/null
+++ b/testsuite/gsk/compare/transform-huge-child-3d.png
diff --git a/testsuite/gsk/meson.build b/testsuite/gsk/meson.build
index 7e785fdc38..ec138bd4b5 100644
--- a/testsuite/gsk/meson.build
+++ b/testsuite/gsk/meson.build
@@ -89,6 +89,7 @@ compare_render_tests = [
   'texture-scale-magnify-rotate',
   'texture-scale-stripes',
   'texture-url',
+  'transform-huge-child-3d',
   'transform-in-transform',
   'transform-in-transform-in-transform',
   'z-transform-clipping-bounds-3d',