[core] Better align fill-extrusion vertex layout

6 files changed, 25 insertions, 29 deletions

diff --git a/src/mbgl/programs/attributes.hpp b/src/mbgl/programs/attributes.hpp
index 437ae2195c..b0582b0bc2 100644
--- a/src/mbgl/programs/attributes.hpp
+++ b/src/mbgl/programs/attributes.hpp
@@ -28,8 +28,7 @@ MBGL_DEFINE_ATTRIBUTE(float, 3, a_projected_pos);
 MBGL_DEFINE_ATTRIBUTE(int16_t, 2, a_label_pos);
 MBGL_DEFINE_ATTRIBUTE(int16_t, 2, a_anchor_pos);
 MBGL_DEFINE_ATTRIBUTE(uint16_t, 2, a_texture_pos);
-MBGL_DEFINE_ATTRIBUTE(int16_t,  3, a_normal);
-MBGL_DEFINE_ATTRIBUTE(uint16_t, 1, a_edgedistance);
+MBGL_DEFINE_ATTRIBUTE(int16_t,  4, a_normal_ed);
 MBGL_DEFINE_ATTRIBUTE(uint8_t, 1, a_fade_opacity);
 MBGL_DEFINE_ATTRIBUTE(uint8_t, 2, a_placed);                                        
 
diff --git a/src/mbgl/programs/fill_extrusion_program.hpp b/src/mbgl/programs/fill_extrusion_program.hpp
index 820670068e..c499e9ef2d 100644
--- a/src/mbgl/programs/fill_extrusion_program.hpp
+++ b/src/mbgl/programs/fill_extrusion_program.hpp
@@ -30,8 +30,7 @@ MBGL_DEFINE_UNIFORM_SCALAR(float,    u_height_factor);
 
 struct FillExtrusionLayoutAttributes : gl::Attributes<
     attributes::a_pos,
-    attributes::a_normal,
-    attributes::a_edgedistance>
+    attributes::a_normal_ed>
 {};
 
 struct FillExtrusionUniforms : gl::Uniforms<
@@ -100,12 +99,9 @@ public:
                 // We pack a bool (`t`) into the x component indicating whether it is an upper or lower vertex
                 static_cast<int16_t>(floor(nx * factor) * 2 + t),
                 static_cast<int16_t>(ny * factor * 2),
-                static_cast<int16_t>(nz * factor * 2)
-                
-            }},
-            {{
+                static_cast<int16_t>(nz * factor * 2),
                 // The edgedistance attribute is used for wrapping fill_extrusion patterns
-                e
+                static_cast<int16_t>(e)
             }}
         };
     }
diff --git a/src/mbgl/renderer/buckets/fill_extrusion_bucket.cpp b/src/mbgl/renderer/buckets/fill_extrusion_bucket.cpp
index 7f53326fe1..5e2c937091 100644
--- a/src/mbgl/renderer/buckets/fill_extrusion_bucket.cpp
+++ b/src/mbgl/renderer/buckets/fill_extrusion_bucket.cpp
@@ -101,13 +101,17 @@ void FillExtrusionBucket::addFeature(const GeometryTileFeature& feature,
                     const auto d2 = convertPoint<double>(p2);
 
                     const Point<double> perp = util::unit(util::perp(d1 - d2));
+                    const auto dist = util::dist<int16_t>(d1, d2);
+                    if (dist > std::numeric_limits<int16_t>::max()) {
+                        edgeDistance = 0;
+                    }
 
                     vertices.emplace_back(
                         FillExtrusionProgram::layoutVertex(p1, perp.x, perp.y, 0, 0, edgeDistance));
                     vertices.emplace_back(
                         FillExtrusionProgram::layoutVertex(p1, perp.x, perp.y, 0, 1, edgeDistance));
 
-                    edgeDistance += util::dist<int16_t>(d1, d2);
+                    edgeDistance += dist;
 
                     vertices.emplace_back(
                         FillExtrusionProgram::layoutVertex(p2, perp.x, perp.y, 0, 0, edgeDistance));
diff --git a/src/mbgl/shaders/debug.cpp b/src/mbgl/shaders/debug.cpp
index d18f3be5d1..9012cfa755 100644
--- a/src/mbgl/shaders/debug.cpp
+++ b/src/mbgl/shaders/debug.cpp
@@ -12,12 +12,7 @@ attribute vec2 a_pos;
 uniform mat4 u_matrix;
 
 void main() {
-    // We are using Int16 for texture position coordinates to give us enough precision for
-    // fractional coordinates. We use 8192 to scale the texture coordinates in the buffer
-    // as an arbitrarily high number to preserve adequate precision when rendering.
-    // This is also the same value as the EXTENT we are using for our tile buffer pos coordinates,
-    // so math for modifying either is consistent.
-    gl_Position = u_matrix * vec4(a_pos, step(8192.0, a_pos.x), 1);
+    gl_Position = u_matrix * vec4(a_pos, 0, 1);
 }
 
 )MBGL_SHADER";
diff --git a/src/mbgl/shaders/fill_extrusion.cpp b/src/mbgl/shaders/fill_extrusion.cpp
index 817f73391c..5bb2b9cd07 100644
--- a/src/mbgl/shaders/fill_extrusion.cpp
+++ b/src/mbgl/shaders/fill_extrusion.cpp
@@ -13,8 +13,7 @@ uniform lowp vec3 u_lightpos;
 uniform lowp float u_lightintensity;
 
 attribute vec2 a_pos;
-attribute vec3 a_normal;
-attribute float a_edgedistance;
+attribute vec4 a_normal_ed;
 
 varying vec4 v_color;
 
@@ -70,11 +69,12 @@ void main() {
 #endif
 
 
+    vec3 normal = a_normal_ed.xyz;
+
     base = max(0.0, base);
     height = max(0.0, height);
 
-    float ed = a_edgedistance; // use each attrib in order to not trip a VAO assert
-    float t = mod(a_normal.x, 2.0);
+    float t = mod(normal.x, 2.0);
 
     gl_Position = u_matrix * vec4(a_pos, t > 0.0 ? height : base, 1);
 
@@ -88,7 +88,7 @@ void main() {
     color += ambientlight;
 
     // Calculate cos(theta), where theta is the angle between surface normal and diffuse light ray
-    float directional = clamp(dot(a_normal / 16384.0, u_lightpos), 0.0, 1.0);
+    float directional = clamp(dot(normal / 16384.0, u_lightpos), 0.0, 1.0);
 
     // Adjust directional so that
     // the range of values for highlight/shading is narrower
@@ -97,7 +97,7 @@ void main() {
     directional = mix((1.0 - u_lightintensity), max((1.0 - colorvalue + u_lightintensity), 1.0), directional);
 
     // Add gradient along z axis of side surfaces
-    if (a_normal.y != 0.0) {
+    if (normal.y != 0.0) {
         directional *= clamp((t + base) * pow(height / 150.0, 0.5), mix(0.7, 0.98, 1.0 - u_lightintensity), 1.0);
     }
 
diff --git a/src/mbgl/shaders/fill_extrusion_pattern.cpp b/src/mbgl/shaders/fill_extrusion_pattern.cpp
index d3e5eef1bf..466d0e04fe 100644
--- a/src/mbgl/shaders/fill_extrusion_pattern.cpp
+++ b/src/mbgl/shaders/fill_extrusion_pattern.cpp
@@ -22,8 +22,7 @@ uniform lowp vec3 u_lightpos;
 uniform lowp float u_lightintensity;
 
 attribute vec2 a_pos;
-attribute vec3 a_normal;
-attribute float a_edgedistance;
+attribute vec4 a_normal_ed;
 
 varying vec2 v_pos_a;
 varying vec2 v_pos_b;
@@ -65,26 +64,29 @@ void main() {
 #endif
 
 
+    vec3 normal = a_normal_ed.xyz;
+    float edgedistance = a_normal_ed.w;
+
     base = max(0.0, base);
     height = max(0.0, height);
 
-    float t = mod(a_normal.x, 2.0);
+    float t = mod(normal.x, 2.0);
     float z = t > 0.0 ? height : base;
 
     gl_Position = u_matrix * vec4(a_pos, z, 1);
 
-    vec2 pos = a_normal.x == 1.0 && a_normal.y == 0.0 && a_normal.z == 16384.0
+    vec2 pos = normal.x == 1.0 && normal.y == 0.0 && normal.z == 16384.0
         ? a_pos // extrusion top
-        : vec2(a_edgedistance, z * u_height_factor); // extrusion side
+        : vec2(edgedistance, z * u_height_factor); // extrusion side
 
     v_pos_a = get_pattern_pos(u_pixel_coord_upper, u_pixel_coord_lower, u_scale_a * u_pattern_size_a, u_tile_units_to_pixels, pos);
     v_pos_b = get_pattern_pos(u_pixel_coord_upper, u_pixel_coord_lower, u_scale_b * u_pattern_size_b, u_tile_units_to_pixels, pos);
 
     v_lighting = vec4(0.0, 0.0, 0.0, 1.0);
-    float directional = clamp(dot(a_normal / 16383.0, u_lightpos), 0.0, 1.0);
+    float directional = clamp(dot(normal / 16383.0, u_lightpos), 0.0, 1.0);
     directional = mix((1.0 - u_lightintensity), max((0.5 + u_lightintensity), 1.0), directional);
 
-    if (a_normal.y != 0.0) {
+    if (normal.y != 0.0) {
         directional *= clamp((t + base) * pow(height / 150.0, 0.5), mix(0.7, 0.98, 1.0 - u_lightintensity), 1.0);
     }