gsk: Normalize rects in GLSL

Our coverage computation only works for well-behaved
rects and rounded rects. But our modelview transform
might flip x or y around, causing things to fail.

Add functions to normalize rects and rounded rects,
and use it whenever we transform a rounded rect in GLSL.

6 files changed, 78 insertions, 8 deletions

diff --git a/gsk/gl/resources/border.glsl b/gsk/gl/resources/border.glsl
index b8653bad49..0fa3203887 100644
--- a/gsk/gl/resources/border.glsl
+++ b/gsk/gl/resources/border.glsl
@@ -19,6 +19,9 @@ void main() {
   gsk_rounded_rect_transform(outside, u_modelview);
   gsk_rounded_rect_transform(inside, u_modelview);
 
+  gsk_rounded_rect_normalize(outside);
+  gsk_rounded_rect_normalize(inside);
+
   gsk_rounded_rect_encode(outside, transformed_outside_outline);
   gsk_rounded_rect_encode(inside, transformed_inside_outline);
 }
@@ -34,10 +37,9 @@ _IN_ _GSK_ROUNDED_RECT_UNIFORM_ transformed_inside_outline;
 
 void main() {
   vec2 frag = gsk_get_frag_coord();
-
-  float alpha = clamp(gsk_rounded_rect_coverage(gsk_decode_rect(transformed_outside_outline), frag) -
-                      gsk_rounded_rect_coverage(gsk_decode_rect(transformed_inside_outline), frag),
-                      0.0, 1.0);
+  float outer_coverage = gsk_rounded_rect_coverage(gsk_decode_rect(transformed_outside_outline), frag);
+  float inner_coverage = gsk_rounded_rect_coverage(gsk_decode_rect(transformed_inside_outline), frag);
+  float alpha = clamp(outer_coverage - inner_coverage, 0.0, 1.0);
 
   gskSetScaledOutputColor(final_color, alpha);
 }
diff --git a/gsk/gl/resources/filled_border.glsl b/gsk/gl/resources/filled_border.glsl
index d503dd0500..23af3aa89c 100644
--- a/gsk/gl/resources/filled_border.glsl
+++ b/gsk/gl/resources/filled_border.glsl
@@ -21,6 +21,9 @@ void main() {
   gsk_rounded_rect_transform(outside, u_modelview);
   gsk_rounded_rect_transform(inside, u_modelview);
 
+  gsk_rounded_rect_normalize(outside);
+  gsk_rounded_rect_normalize(inside);
+
   gsk_rounded_rect_encode(outside, transformed_outside_outline);
   gsk_rounded_rect_encode(inside, transformed_inside_outline);
 }
diff --git a/gsk/gl/resources/inset_shadow.glsl b/gsk/gl/resources/inset_shadow.glsl
index f052a08580..83d4287f13 100644
--- a/gsk/gl/resources/inset_shadow.glsl
+++ b/gsk/gl/resources/inset_shadow.glsl
@@ -22,6 +22,9 @@ void main() {
   gsk_rounded_rect_transform(outside, u_modelview);
   gsk_rounded_rect_transform(inside, u_modelview);
 
+  gsk_rounded_rect_normalize(outside);
+  gsk_rounded_rect_normalize(inside);
+
   gsk_rounded_rect_encode(outside, transformed_outside_outline);
   gsk_rounded_rect_encode(inside, transformed_inside_outline);
 }
diff --git a/gsk/gl/resources/outset_shadow.glsl b/gsk/gl/resources/outset_shadow.glsl
index 44b05aa152..fac86c0093 100644
--- a/gsk/gl/resources/outset_shadow.glsl
+++ b/gsk/gl/resources/outset_shadow.glsl
@@ -15,6 +15,7 @@ void main() {
 
   GskRoundedRect outline = gsk_create_rect(u_outline_rect);
   gsk_rounded_rect_transform(outline, u_modelview);
+  gsk_rounded_rect_normalize(outline);
   gsk_rounded_rect_encode(outline, transformed_outline);
 }
 
diff --git a/gsk/gl/resources/preamble.fs.glsl b/gsk/gl/resources/preamble.fs.glsl
index c1c5a954ee..0c366bd847 100644
--- a/gsk/gl/resources/preamble.fs.glsl
+++ b/gsk/gl/resources/preamble.fs.glsl
@@ -16,11 +16,15 @@ _IN_ vec2 vUv;
 
 GskRoundedRect gsk_decode_rect(_GSK_ROUNDED_RECT_UNIFORM_ r)
 {
+  GskRoundedRect rect;
 #if defined(GSK_GLES) || defined(GSK_LEGACY)
-  return GskRoundedRect(r[0], r[1], r[2]);
+  rect = GskRoundedRect(r[0], r[1], r[2]);
 #else
-  return r;
+  rect = r;
 #endif
+  gsk_rounded_rect_normalize (rect);
+
+  return rect;
 }
 
 float
diff --git a/gsk/gl/resources/preamble.glsl b/gsk/gl/resources/preamble.glsl
index 5f5d2b780a..46bebc14e0 100644
--- a/gsk/gl/resources/preamble.glsl
+++ b/gsk/gl/resources/preamble.glsl
@@ -22,6 +22,55 @@ struct GskRoundedRect
   vec4 corner_points2; // xy = bottom right, zw = bottom left
 };
 
+void gsk_rounded_rect_normalize(inout GskRoundedRect r)
+{
+  if (r.bounds.x > r.bounds.z)
+    {
+      float t = r.bounds.x;
+      r.bounds.x = r.bounds.z;
+      r.bounds.z = t;
+
+      vec2 c = r.corner_points1.xy;
+      r.corner_points1.xy = r.corner_points1.zw;
+      r.corner_points1.zw = c;
+
+      c = r.corner_points2.xy;
+      r.corner_points2.xy = r.corner_points2.zw;
+      r.corner_points2.zw = c;
+    }
+
+  if (r.bounds.y > r.bounds.w)
+    {
+      float t = r.bounds.y;
+      r.bounds.y = r.bounds.w;
+      r.bounds.w = t;
+
+      vec2 c = r.corner_points1.xy;
+      r.corner_points1.xy = r.corner_points2.xy;
+      r.corner_points2.xy = c;
+
+      c = r.corner_points1.zw;
+      r.corner_points1.zw = r.corner_points2.zw;
+      r.corner_points2.zw = c;
+    }
+}
+
+void gsk_bounds_normalize (inout vec4 bounds)
+{
+  if (bounds.x > bounds.z)
+    {
+      float t = bounds.x;
+      bounds.x = bounds.z;
+      bounds.z = t;
+    }
+  if (bounds.y > bounds.w)
+    {
+      float t = bounds.y;
+      bounds.y = bounds.w;
+      bounds.w = t;
+    }
+}
+
 // Transform from a C GskRoundedRect to what we need.
 GskRoundedRect
 gsk_create_rect(vec4[3] data)
@@ -33,13 +82,21 @@ gsk_create_rect(vec4[3] data)
   vec4 corner_points2 = vec4(bounds.zw + (data[2].xy * vec2(-1, -1)),
                              bounds.xw + vec2(data[2].zw * vec2(1, -1)));
 
-  return GskRoundedRect(bounds, corner_points1, corner_points2);
+  GskRoundedRect rect = GskRoundedRect(bounds, corner_points1, corner_points2);
+
+  gsk_rounded_rect_normalize (rect);
+
+  return rect;
 }
 
 vec4
 gsk_get_bounds(vec4[3] data)
 {
-  return vec4(data[0].xy, data[0].xy + data[0].zw);
+  vec4 bounds = vec4(data[0].xy, data[0].xy + data[0].zw);
+
+  gsk_bounds_normalize (bounds);
+
+  return bounds;
 }
 
 vec4 gsk_premultiply(vec4 c) {