[build] Fix undefined behavour sanitizer (#16375)

* [build] Fix integer overflow runtime error for core part

Temporarily remove circle ci UBSAN build precondition

* [build] Enable all of the ubsans

[build] Check runtime error

[build] Update UBSAN_OPTION

* [build] Add UBSAN blacklist

[build] Ignore system libraries

[build] Ignore vendor library

* [build] Fix implicit conversion runtime error in core

* [build] Fix division by zero runtime error

* [build] Add unfixed error to ubsan blacklist

* [build] Make UBSAN halt on error

Revert "Temporary remove build precondition"

* [build] Fix division by zero error

* [build] Make UBSAN officially work without FIXME prefix

* [build] Fix implicit conversion from int64_t to uint64_t

* [build] Rename style test json file name

* Address review findings

12 files changed, 53 insertions, 32 deletions

diff --git a/src/mbgl/geometry/line_atlas.cpp b/src/mbgl/geometry/line_atlas.cpp
index 6ddf16af0b..2042195b6c 100644
--- a/src/mbgl/geometry/line_atlas.cpp
+++ b/src/mbgl/geometry/line_atlas.cpp
@@ -54,13 +54,17 @@ void addRoundDash(
     if (ranges.empty()) return;
 
     for (int y = -n; y <= n; y++) {
-        int row = yOffset + n + y;
+        int row = static_cast<int32_t>(yOffset) + n + y;
         const uint32_t index = image.size.width * row;
         uint32_t currIndex = 0;
         DashRange range = ranges[currIndex];
 
         for (uint32_t x = 0; x < image.size.width; ++x) {
-            if (x / range.right > 1.0f && ++currIndex < ranges.size()) {
+            if (range.right == 0) {
+                if (x != 0 && ++currIndex < ranges.size()) {
+                    range = ranges[currIndex];
+                }
+            } else if (x / range.right > 1.0f && ++currIndex < ranges.size()) {
                 range = ranges[currIndex];
             }
 
diff --git a/src/mbgl/layout/symbol_layout.cpp b/src/mbgl/layout/symbol_layout.cpp
index 0a1c3c4fbd..d996041ebe 100644
--- a/src/mbgl/layout/symbol_layout.cpp
+++ b/src/mbgl/layout/symbol_layout.cpp
@@ -738,10 +738,12 @@ std::vector<float> SymbolLayout::calculateTileDistances(const GeometryCoordinate
                 sumForwardLength += util::dist<float>(line[i + 1], line[i]);
             }
         }
-        for (std::size_t i = ++segment; i-- > 0;) {
+        for (std::size_t i = segment;; --i) {
             tileDistances[i] = sumBackwardLength;
             if (i != 0u) {
                 sumBackwardLength += util::dist<float>(line[i - 1], line[i]);
+            } else {
+                break; // Add break to avoid unsigned integer overflow when i==0
             }
         }
     }
diff --git a/src/mbgl/map/transform_state.cpp b/src/mbgl/map/transform_state.cpp
index 1894c59e2b..0333f4860c 100644
--- a/src/mbgl/map/transform_state.cpp
+++ b/src/mbgl/map/transform_state.cpp
@@ -70,8 +70,11 @@ void TransformState::matrixFor(mat4& matrix, const UnwrappedTileID& tileID) cons
     const double s = Projection::worldSize(scale) / tileScale;
 
     matrix::identity(matrix);
-    matrix::translate(
-        matrix, matrix, int64_t(tileID.canonical.x + tileID.wrap * tileScale) * s, int64_t(tileID.canonical.y) * s, 0);
+    matrix::translate(matrix,
+                      matrix,
+                      int64_t(tileID.canonical.x + tileID.wrap * static_cast<int64_t>(tileScale)) * s,
+                      int64_t(tileID.canonical.y) * s,
+                      0);
     matrix::scale(matrix, matrix, s / util::EXTENT, s / util::EXTENT, 1);
 }
 
diff --git a/src/mbgl/renderer/layers/render_line_layer.cpp b/src/mbgl/renderer/layers/render_line_layer.cpp
index af5c288704..b03ac8027b 100644
--- a/src/mbgl/renderer/layers/render_line_layer.cpp
+++ b/src/mbgl/renderer/layers/render_line_layer.cpp
@@ -242,7 +242,7 @@ GeometryCollection offsetLine(const GeometryCollection& rings, double offset) {
             Point<double> extrude = util::unit(aToB + bToC);
 
             const double cosHalfAngle = extrude.x * bToC.x + extrude.y * bToC.y;
-            extrude *= (1.0 / cosHalfAngle);
+            extrude *= (cosHalfAngle != 0) ? (1.0 / cosHalfAngle) : 0;
 
             newRing.emplace_back(convertPoint<int16_t>(extrude * offset) + p);
         }
diff --git a/src/mbgl/renderer/renderer_impl.cpp b/src/mbgl/renderer/renderer_impl.cpp
index cdb85385ed..aeba4b5eec 100644
--- a/src/mbgl/renderer/renderer_impl.cpp
+++ b/src/mbgl/renderer/renderer_impl.cpp
@@ -108,8 +108,8 @@ void Renderer::Impl::render(const RenderTree& renderTree) {
         }
         parameters.staticData.depthRenderbuffer->setShouldClear(true);
 
-        uint32_t i = static_cast<uint32_t>(layerRenderItems.size()) - 1;
-        for (auto it = layerRenderItems.begin(); it != layerRenderItems.end(); ++it, --i) {
+        int32_t i = static_cast<int32_t>(layerRenderItems.size()) - 1;
+        for (auto it = layerRenderItems.begin(); it != layerRenderItems.end() && i >= 0; ++it, --i) {
             parameters.currentLayer = i;
             const RenderItem& renderItem = it->get();
             if (renderItem.hasRenderPass(parameters.pass)) {
@@ -159,8 +159,8 @@ void Renderer::Impl::render(const RenderTree& renderTree) {
         parameters.pass = RenderPass::Translucent;
         const auto debugGroup(parameters.renderPass->createDebugGroup("translucent"));
 
-        uint32_t i = static_cast<uint32_t>(layerRenderItems.size()) - 1;
-        for (auto it = layerRenderItems.begin(); it != layerRenderItems.end(); ++it, --i) {
+        int32_t i = static_cast<int32_t>(layerRenderItems.size()) - 1;
+        for (auto it = layerRenderItems.begin(); it != layerRenderItems.end() && i >= 0; ++it, --i) {
             parameters.currentLayer = i;
             const RenderItem& renderItem = it->get();
             if (renderItem.hasRenderPass(parameters.pass)) {
diff --git a/src/mbgl/sprite/sprite_parser.cpp b/src/mbgl/sprite/sprite_parser.cpp
index ac0b7f91f1..997e363e50 100644
--- a/src/mbgl/sprite/sprite_parser.cpp
+++ b/src/mbgl/sprite/sprite_parser.cpp
@@ -17,21 +17,22 @@ namespace mbgl {
 
 std::unique_ptr<style::Image> createStyleImage(const std::string& id,
                                                const PremultipliedImage& image,
-                                               const uint32_t srcX,
-                                               const uint32_t srcY,
-                                               const uint32_t width,
-                                               const uint32_t height,
+                                               const int32_t srcX,
+                                               const int32_t srcY,
+                                               const int32_t width,
+                                               const int32_t height,
                                                const double ratio,
                                                const bool sdf,
                                                style::ImageStretches&& stretchX,
                                                style::ImageStretches&& stretchY,
                                                const optional<style::ImageContent>& content) {
     // Disallow invalid parameter configurations.
-    if (width <= 0 || height <= 0 || width > 1024 || height > 1024 || ratio <= 0 || ratio > 10 ||
-        srcX >= image.size.width || srcY >= image.size.height || srcX + width > image.size.width ||
-        srcY + height > image.size.height) {
+    if (width <= 0 || height <= 0 || width > 1024 || height > 1024 || ratio <= 0 || ratio > 10 || srcX < 0 ||
+        srcY < 0 || srcX >= static_cast<int32_t>(image.size.width) || srcY >= static_cast<int32_t>(image.size.height) ||
+        srcX + width > static_cast<int32_t>(image.size.width) ||
+        srcY + height > static_cast<int32_t>(image.size.height)) {
         Log::Error(Event::Sprite,
-                   "Can't create image with invalid metrics: %ux%u@%u,%u in %ux%u@%sx sprite",
+                   "Can't create image with invalid metrics: %dx%d@%d,%d in %ux%u@%sx sprite",
                    width,
                    height,
                    srcX,
@@ -42,10 +43,11 @@ std::unique_ptr<style::Image> createStyleImage(const std::string& id,
         return nullptr;
     }
 
-    PremultipliedImage dstImage({width, height});
+    const Size size(static_cast<uint32_t>(width), static_cast<uint32_t>(height));
+    PremultipliedImage dstImage(size);
 
     // Copy from the source image into our individual sprite image
-    PremultipliedImage::copy(image, dstImage, {srcX, srcY}, {0, 0}, {width, height});
+    PremultipliedImage::copy(image, dstImage, {static_cast<uint32_t>(srcX), static_cast<uint32_t>(srcY)}, {0, 0}, size);
 
     try {
         return std::make_unique<style::Image>(
diff --git a/src/mbgl/sprite/sprite_parser.hpp b/src/mbgl/sprite/sprite_parser.hpp
index 7d545a6d98..baba427f43 100644
--- a/src/mbgl/sprite/sprite_parser.hpp
+++ b/src/mbgl/sprite/sprite_parser.hpp
@@ -9,10 +9,10 @@ namespace mbgl {
 // Extracts an individual image from a spritesheet from the given location.
 std::unique_ptr<style::Image> createStyleImage(const std::string& id,
                                                const PremultipliedImage&,
-                                               uint32_t srcX,
-                                               uint32_t srcY,
-                                               uint32_t srcWidth,
-                                               uint32_t srcHeight,
+                                               int32_t srcX,
+                                               int32_t srcY,
+                                               int32_t srcWidth,
+                                               int32_t srcHeight,
                                                double ratio,
                                                bool sdf,
                                                style::ImageStretches&& stretchX = {},
diff --git a/src/mbgl/style/expression/compound_expression.cpp b/src/mbgl/style/expression/compound_expression.cpp
index 2477fe7917..bae0258aa0 100644
--- a/src/mbgl/style/expression/compound_expression.cpp
+++ b/src/mbgl/style/expression/compound_expression.cpp
@@ -15,6 +15,7 @@
 #include <mapbox/eternal.hpp>
 
 #include <cmath>
+#include <limits>
 
 namespace mbgl {
 namespace style {
@@ -498,7 +499,15 @@ const auto& multiplyCompoundExpression() {
 }
 
 const auto& divideCompoundExpression() {
-    static auto signature = detail::makeSignature("/", [](double a, double b) -> Result<double> { return a / b; });
+    static auto signature = detail::makeSignature("/", [](double a, double b) -> Result<double> {
+        if (b == 0) {
+            if (a == 0) return std::numeric_limits<double>::quiet_NaN();
+            double inf = std::numeric_limits<double>::infinity();
+            if (a > 0) return inf;
+            if (a < 0) return -inf;
+        }
+        return a / b;
+    });
     return signature;
 }
 
diff --git a/src/mbgl/text/tagged_string.cpp b/src/mbgl/text/tagged_string.cpp
index b18ad4d148..9c374ae274 100644
--- a/src/mbgl/text/tagged_string.cpp
+++ b/src/mbgl/text/tagged_string.cpp
@@ -29,7 +29,7 @@ void TaggedString::addImageSection(const std::string& imageID) {
 
     styledText.first += *nextImageSectionCharCode;
     sections.emplace_back(imageID);
-    styledText.second.resize(styledText.first.size(), sections.size() - 1);
+    styledText.second.resize(styledText.first.size(), static_cast<uint8_t>(sections.size() - 1));
 }
 
 optional<char16_t> TaggedString::getNextImageSectionCharCode() {
diff --git a/src/mbgl/tile/raster_dem_tile.cpp b/src/mbgl/tile/raster_dem_tile.cpp
index e3c8eed829..b70e581ea1 100644
--- a/src/mbgl/tile/raster_dem_tile.cpp
+++ b/src/mbgl/tile/raster_dem_tile.cpp
@@ -84,8 +84,9 @@ HillshadeBucket* RasterDEMTile::getBucket() const {
 }
 
 void RasterDEMTile::backfillBorder(const RasterDEMTile& borderTile, const DEMTileNeighbors mask) {
-    int32_t dx = borderTile.id.canonical.x - id.canonical.x;
-    const int8_t dy = borderTile.id.canonical.y - id.canonical.y;
+    int32_t dx = static_cast<int32_t>(borderTile.id.canonical.x) - static_cast<int32_t>(id.canonical.x);
+    const auto dy =
+        static_cast<int8_t>(static_cast<int32_t>(borderTile.id.canonical.y) - static_cast<int32_t>(id.canonical.y));
     const uint32_t dim = pow(2, id.canonical.z);
     if (dx == 0 && dy == 0) return;
     if (std::abs(dy) > 1) return;
diff --git a/src/mbgl/util/premultiply.cpp b/src/mbgl/util/premultiply.cpp
index d9fb2480de..02c045fbb4 100644
--- a/src/mbgl/util/premultiply.cpp
+++ b/src/mbgl/util/premultiply.cpp
@@ -40,9 +40,9 @@ UnassociatedImage unpremultiply(PremultipliedImage&& src) {
         uint8_t& b = data[i + 2];
         uint8_t& a = data[i + 3];
         if (a) {
-            r = (255 * r + (a / 2)) / a;
-            g = (255 * g + (a / 2)) / a;
-            b = (255 * b + (a / 2)) / a;
+            r = static_cast<uint8_t>((255 * r + (a / 2)) / a);
+            g = static_cast<uint8_t>((255 * g + (a / 2)) / a);
+            b = static_cast<uint8_t>((255 * b + (a / 2)) / a);
         }
     }
 
diff --git a/src/mbgl/util/tiny_sdf.cpp b/src/mbgl/util/tiny_sdf.cpp
index 6edcd83bc2..a71d04386f 100644
--- a/src/mbgl/util/tiny_sdf.cpp
+++ b/src/mbgl/util/tiny_sdf.cpp
@@ -35,7 +35,7 @@ void edt1d(std::vector<double>& f,
 
     for (uint32_t q = 0, k = 0; q < n; q++) {
         while (z[k + 1] < q) k++;
-        d[q] = (q - v[k]) * (q - v[k]) + f[v[k]];
+        d[q] = (static_cast<double>(q) - v[k]) * (static_cast<double>(q) - v[k]) + f[v[k]];
     }
 }