[core] Limit pitch based on edge insets. Fix max Z calculation in getProjMatrix.

Patch partly fixes #15163 in a way that it doesn't allow loading tens of thousands of tiles and attempt to show area above horizon:

Limit pitch based on edge insets. It is not too bad - current limit of 60 degrees stays active until center of perspective is moved towards the bottom, to 84% of screen height. The plan is to split removal of 60 degrees limit to follow up patch.

Fix max Z calculation in getProjMatrix. TransformState::getProjMatrix calculation of farZ was complex with possibility to lead to negative z values. Replacing it with simpler, precise calculation:
furthestDistance = cameraToCenterDistance / (1 - tanFovAboveCenter * std::tan(getPitch()));

TransformState::getProjMatrix calculation of farZ was an aproximation. Replacing it with simpler, but precise calculation.

Related to: #15163

6 files changed, 72 insertions, 17 deletions

diff --git a/include/mbgl/util/projection.hpp b/include/mbgl/util/projection.hpp
index 9619a380b4..f0b9115eb0 100644
--- a/include/mbgl/util/projection.hpp
+++ b/include/mbgl/util/projection.hpp
@@ -98,7 +98,7 @@ private:
         return Point<double> {
             util::LONGITUDE_MAX + latLng.longitude(),
             util::LONGITUDE_MAX - util::RAD2DEG * std::log(std::tan(M_PI / 4 + latitude * M_PI / util::DEGREES_MAX))
-        } * worldSize / util::DEGREES_MAX;
+        } * (worldSize / util::DEGREES_MAX);
     }
 };
 
diff --git a/src/mbgl/map/transform.cpp b/src/mbgl/map/transform.cpp
index a51d4dd4ff..0651cfd284 100644
--- a/src/mbgl/map/transform.cpp
+++ b/src/mbgl/map/transform.cpp
@@ -138,9 +138,6 @@ void Transform::easeTo(const CameraOptions& camera, const AnimationOptions& anim
         if (bearing != startBearing) {
             state.bearing = util::wrap(util::interpolate(startBearing, bearing, t), -M_PI, M_PI);
         }
-        if (pitch != startPitch) {
-            state.pitch = util::interpolate(startPitch, pitch, t);
-        }
         if (padding != startEdgeInsets) {
             // Interpolate edge insets
             state.edgeInsets = {
@@ -150,6 +147,10 @@ void Transform::easeTo(const CameraOptions& camera, const AnimationOptions& anim
                 util::interpolate(startEdgeInsets.right(), padding.right(), t)
             };
         }
+        auto maxPitch = getMaxPitchForEdgeInsets(state.edgeInsets);
+        if (pitch != startPitch || maxPitch < startPitch) {
+            state.pitch = std::min(maxPitch, util::interpolate(startPitch, pitch, t));
+        }
     }, duration);
 }
 
@@ -302,9 +303,6 @@ void Transform::flyTo(const CameraOptions &camera, const AnimationOptions &anima
         if (bearing != startBearing) {
             state.bearing = util::wrap(util::interpolate(startBearing, bearing, k), -M_PI, M_PI);
         }
-        if (pitch != startPitch) {
-            state.pitch = util::interpolate(startPitch, pitch, k);
-        }
         if (padding != startEdgeInsets) {
             // Interpolate edge insets
             state.edgeInsets = {
@@ -314,6 +312,10 @@ void Transform::flyTo(const CameraOptions &camera, const AnimationOptions &anima
                 util::interpolate(startEdgeInsets.right(), padding.right(), us)
             };
         }
+        auto maxPitch = getMaxPitchForEdgeInsets(state.edgeInsets);
+        if (pitch != startPitch || maxPitch < startPitch) {
+            state.pitch = std::min(maxPitch, util::interpolate(startPitch, pitch, k));
+        }
     }, duration);
 }
 
@@ -576,4 +578,16 @@ LatLng Transform::screenCoordinateToLatLng(const ScreenCoordinate& point, LatLng
     return state.screenCoordinateToLatLng(flippedPoint, wrapMode);
 }
 
+double Transform::getMaxPitchForEdgeInsets(const EdgeInsets &insets) const
+{
+    double centerOffsetY = 0.5 * (insets.top() - insets.bottom()); // See TransformState::getCenterOffset.
+
+    const auto height = state.size.height;
+    assert(height);
+    // See TransformState::fov description: fov = 2 * arctan((height / 2) / (height * 1.5)).
+    const double fovAboveCenter = std::atan((0.666666 + 0.02) * (0.5 + centerOffsetY / height));
+    return M_PI * 0.5 - fovAboveCenter;   // 0.02 added ^^^^ to prevent parallel ground to viewport clipping plane.
+    // e.g. Maximum pitch of 60 degrees is when perspective center's offset from the top is 84% of screen height.
+}
+
 } // namespace mbgl
diff --git a/src/mbgl/map/transform.hpp b/src/mbgl/map/transform.hpp
index 0c018a6e48..75dfeff645 100644
--- a/src/mbgl/map/transform.hpp
+++ b/src/mbgl/map/transform.hpp
@@ -115,6 +115,9 @@ private:
                          std::function<void(double)>,
                          const Duration&);
 
+    // We don't want to show horizon: limit max pitch based on edge insets.
+    double getMaxPitchForEdgeInsets(const EdgeInsets &insets) const;
+
     TimePoint transitionStart;
     Duration transitionDuration;
     std::function<bool(const TimePoint)> transitionFrameFn;
diff --git a/src/mbgl/map/transform_state.cpp b/src/mbgl/map/transform_state.cpp
index 92c02d0bc7..77309a2a55 100644
--- a/src/mbgl/map/transform_state.cpp
+++ b/src/mbgl/map/transform_state.cpp
@@ -36,18 +36,18 @@ void TransformState::getProjMatrix(mat4& projMatrix, uint16_t nearZ, bool aligne
     const double cameraToCenterDistance = getCameraToCenterDistance();
     auto offset = getCenterOffset();
 
-    // Find the distance from the viewport center point
-    // [width/2 + offset.x, height/2 + offset.y] to the top edge, to point
-    // [width/2 + offset.x, 0] in Z units, using the law of sines.
+    // Find the Z distance from the viewport center point
+    // [width/2 + offset.x, height/2 + offset.y] to the top edge; to point
+    // [width/2 + offset.x, 0] in Z units.
     // 1 Z unit is equivalent to 1 horizontal px at the center of the map
     // (the distance between[width/2, height/2] and [width/2 + 1, height/2])
-    const double fovAboveCenter = getFieldOfView() * (0.5 + offset.y / size.height);
-    const double groundAngle = M_PI / 2.0 + getPitch();
-    const double aboveCenterSurfaceDistance = std::sin(fovAboveCenter) * cameraToCenterDistance / std::sin(M_PI - groundAngle - fovAboveCenter);
-
-
+    // See https://github.com/mapbox/mapbox-gl-native/pull/15195 for details.
+    // See TransformState::fov description: fov = 2 * arctan((height / 2) / (height * 1.5)).
+    const double tanFovAboveCenter = (size.height * 0.5 + offset.y) / (size.height * 1.5);
+    const double tanMultiple = tanFovAboveCenter * std::tan(getPitch());
+    assert(tanMultiple < 1);
     // Calculate z distance of the farthest fragment that should be rendered.
-    const double furthestDistance = std::cos(M_PI / 2 - getPitch()) * aboveCenterSurfaceDistance + cameraToCenterDistance;
+    const double furthestDistance = cameraToCenterDistance / (1 - tanMultiple);
     // Add a bit extra to avoid precision problems when a fragment's distance is exactly `furthestDistance`
     const double farZ = furthestDistance * 1.01;
 
@@ -64,7 +64,7 @@ void TransformState::getProjMatrix(mat4& projMatrix, uint16_t nearZ, bool aligne
     const bool flippedY = viewportMode == ViewportMode::FlippedY;
     matrix::scale(projMatrix, projMatrix, 1.0, flippedY ? 1 : -1, 1);
 
-    matrix::translate(projMatrix, projMatrix, 0, 0, -getCameraToCenterDistance());
+    matrix::translate(projMatrix, projMatrix, 0, 0, -cameraToCenterDistance);
 
     using NO = NorthOrientation;
     switch (getNorthOrientation()) {
diff --git a/test/map/transform.test.cpp b/test/map/transform.test.cpp
index 3d37312b17..84fdb06b21 100644
--- a/test/map/transform.test.cpp
+++ b/test/map/transform.test.cpp
@@ -9,6 +9,7 @@ using namespace mbgl;
 
 TEST(Transform, InvalidZoom) {
     Transform transform;
+    transform.resize({1, 1});
 
     ASSERT_DOUBLE_EQ(0, transform.getLatLng().latitude());
     ASSERT_DOUBLE_EQ(0, transform.getLatLng().longitude());
@@ -56,6 +57,7 @@ TEST(Transform, InvalidZoom) {
 
 TEST(Transform, InvalidBearing) {
     Transform transform;
+    transform.resize({1, 1});
 
     ASSERT_DOUBLE_EQ(0, transform.getLatLng().latitude());
     ASSERT_DOUBLE_EQ(0, transform.getLatLng().longitude());
@@ -78,6 +80,7 @@ TEST(Transform, InvalidBearing) {
 
 TEST(Transform, IntegerZoom) {
     Transform transform;
+    transform.resize({1, 1});
 
     auto checkIntegerZoom = [&transform](uint8_t zoomInt, double zoom) {
         transform.jumpTo(CameraOptions().withZoom(zoom));
diff --git a/test/util/tile_cover.test.cpp b/test/util/tile_cover.test.cpp
index 3fc7681520..e35e6e2e99 100644
--- a/test/util/tile_cover.test.cpp
+++ b/test/util/tile_cover.test.cpp
@@ -43,6 +43,41 @@ TEST(TileCover, Pitch) {
               util::tileCover(transform.getState(), 2));
 }
 
+TEST(TileCover, PitchOverAllowedByContentInsets) {
+    Transform transform;
+    transform.resize({ 512, 512 });
+
+    transform.jumpTo(CameraOptions().withCenter(LatLng { 0.1, -0.1 }).withPadding(EdgeInsets { 376, 0, 0, 0 })
+                                    .withZoom(8.0).withBearing(45.0).withPitch(60.0));
+    // Top padding of 376 leads to capped pitch. See Transform::getMaxPitchForEdgeInsets.
+    EXPECT_LE(transform.getPitch() + 0.001, util::DEG2RAD * 60);
+
+    EXPECT_EQ((std::vector<UnwrappedTileID>{
+        { 3, 4, 3 }, { 3, 3, 3 }, { 3, 4, 4 }, { 3, 3, 4 }, { 3, 4, 2 }, { 3, 5, 3 }, { 3, 5, 2 }
+    }),
+              util::tileCover(transform.getState(), 3));
+}
+
+TEST(TileCover, PitchWithLargerResultSet) {
+    Transform transform;
+    transform.resize({ 1024, 768 });
+
+    // The values used here triggered the regression with left and right edge
+    // selection in tile_cover.cpp scanSpans.
+    transform.jumpTo(CameraOptions().withCenter(LatLng { 0.1, -0.1 }).withPadding(EdgeInsets { 400, 0, 0, 0 })
+                                    .withZoom(5).withBearing(-142.2630000003529176).withPitch(60.0));
+
+    auto cover = util::tileCover(transform.getState(), 5);
+    // Returned vector has above 100 elements, we check first 16 as there is a
+    // plan to return lower LOD for distant tiles.
+    EXPECT_EQ((std::vector<UnwrappedTileID> {
+        { 5, 15, 16 }, { 5, 15, 17 }, { 5, 14, 16 }, { 5, 14, 17 },
+        { 5, 16, 16 }, { 5, 16, 17 }, { 5, 15, 15 }, { 5, 14, 15 },
+        { 5, 15, 18 }, { 5, 14, 18 }, { 5, 16, 15 }, { 5, 13, 16 },
+        { 5, 13, 17 }, { 5, 16, 18 }, { 5, 13, 18 }, { 5, 15, 19 }
+    }), (std::vector<UnwrappedTileID> { cover.begin(), cover.begin() + 16}) );
+}
+
 TEST(TileCover, WorldZ1) {
     EXPECT_EQ((std::vector<UnwrappedTileID>{
         { 1, 0, 0 }, { 1, 0, 1 }, { 1, 1, 0 }, { 1, 1, 1 },