[test] Add pixelmatch; use in annotation tests

20 files changed, 116 insertions, 21 deletions

diff --git a/.gitignore b/.gitignore
index 21a7079faa..d5fb118e90 100644
--- a/.gitignore
+++ b/.gitignore
@@ -25,6 +25,8 @@
 /test/fixtures/api/1.png
 /test/fixtures/api/2.png
 /test/fixtures/database/*.db
+/test/fixtures/*/*/actual.png
+/test/fixtures/*/*/diff.png
 /test/output
 /include/mbgl/shader/shaders.hpp
 /src/shader/shaders_gl.cpp
diff --git a/configure b/configure
index da993492a1..8c74b1db8e 100755
--- a/configure
+++ b/configure
@@ -104,6 +104,7 @@ print_flags geojsonvt static_libs cflags ldflags
 print_flags variant static_libs cflags ldflags
 print_flags rapidjson static_libs cflags ldflags
 print_flags gtest static_libs cflags ldflags
+print_flags pixelmatch static_libs cflags ldflags
 
 CONFIG+="  }
 }
diff --git a/scripts/linux/after_script.sh b/scripts/linux/after_script.sh
new file mode 100755
index 0000000000..f8e4bc3486
--- /dev/null
+++ b/scripts/linux/after_script.sh
@@ -0,0 +1,15 @@
+#!/bin/bash
+
+set -e
+set -o pipefail
+
+if [ ! -z "${AWS_ACCESS_KEY_ID}" ] && [ ! -z "${AWS_SECRET_ACCESS_KEY}" ] ; then
+    # Install and add awscli to PATH for uploading the results
+    pip install --user awscli
+    export PATH="`python -m site --user-base`/bin:${PATH}"
+
+    REPO_NAME=$(basename $TRAVIS_REPO_SLUG)
+
+    aws s3 cp --recursive --acl public-read test/fixtures/annotations \
+        s3://mapbox/$REPO_NAME/render-tests/$TRAVIS_JOB_NUMBER/annotations
+fi
diff --git a/scripts/linux/configure.sh b/scripts/linux/configure.sh
index f270c903ee..6a88c13c01 100644
--- a/scripts/linux/configure.sh
+++ b/scripts/linux/configure.sh
@@ -15,6 +15,7 @@ GEOJSONVT_VERSION=2.1.6.3
 VARIANT_VERSION=1.0
 RAPIDJSON_VERSION=1.0.2
 GTEST_VERSION=1.7.0
+PIXELMATCH_VERSION=0.9.0
 
 function print_opengl_flags {
     CONFIG+="    'opengl_cflags%': $(quote_flags $(pkg-config gl x11 --cflags)),"$LN
diff --git a/scripts/osx/configure.sh b/scripts/osx/configure.sh
index 34024a40f4..823e3b6260 100644
--- a/scripts/osx/configure.sh
+++ b/scripts/osx/configure.sh
@@ -15,3 +15,4 @@ GEOJSONVT_VERSION=2.1.6.3
 VARIANT_VERSION=1.0
 RAPIDJSON_VERSION=1.0.2
 GTEST_VERSION=1.7.0
+PIXELMATCH_VERSION=0.9.0
diff --git a/test/api/annotations.cpp b/test/api/annotations.cpp
index b16ec71ef6..c4fea02079 100644
--- a/test/api/annotations.cpp
+++ b/test/api/annotations.cpp
@@ -7,7 +7,6 @@
 #include <mbgl/platform/default/headless_display.hpp>
 #include <mbgl/platform/default/headless_view.hpp>
 #include <mbgl/storage/default_file_source.hpp>
-#include <mbgl/util/image.hpp>
 #include <mbgl/util/io.hpp>
 
 #include <future>
@@ -15,12 +14,18 @@
 
 using namespace mbgl;
 
-std::string renderPNG(Map& map) {
+UnassociatedImage render(Map& map) {
     std::promise<UnassociatedImage> promise;
     map.renderStill([&](std::exception_ptr, UnassociatedImage&& image) {
         promise.set_value(std::move(image));
     });
-    return encodePNG(promise.get_future().get());
+    return std::move(promise.get_future().get());
+}
+
+void checkRendering(Map& map, const char * name) {
+    UnassociatedImage actual = render(map);
+    test::checkImage(std::string("test/fixtures/annotations/") + name + "/",
+                     actual, 0.0002, 0.1);
 }
 
 TEST(Annotations, PointAnnotation) {
@@ -30,9 +35,9 @@ TEST(Annotations, PointAnnotation) {
 
     Map map(view, fileSource, MapMode::Still);
     map.setStyleJSON(util::read_file("test/fixtures/api/empty.json"), "");
-    map.addPointAnnotation(PointAnnotation({ 0, 0 }, "default_marker"));
+    map.addPointAnnotation(PointAnnotation({ 0, -20 }, "default_marker"));
 
-    util::write_file("test/output/point_annotation.png", renderPNG(map));
+    checkRendering(map, "point_annotation");
 }
 
 TEST(Annotations, LineAnnotation) {
@@ -51,7 +56,7 @@ TEST(Annotations, LineAnnotation) {
 
     map.addShapeAnnotation(ShapeAnnotation(segments, properties));
 
-    util::write_file("test/output/line_annotation.png", renderPNG(map));
+    checkRendering(map, "line_annotation");
 }
 
 TEST(Annotations, FillAnnotation) {
@@ -69,7 +74,7 @@ TEST(Annotations, FillAnnotation) {
 
     map.addShapeAnnotation(ShapeAnnotation(segments, properties));
 
-    util::write_file("test/output/fill_annotation.png", renderPNG(map));
+    checkRendering(map, "fill_annotation");
 }
 
 TEST(Annotations, StyleSourcedShapeAnnotation) {
@@ -84,7 +89,7 @@ TEST(Annotations, StyleSourcedShapeAnnotation) {
 
     map.addShapeAnnotation(ShapeAnnotation(segments, "annotation"));
 
-    util::write_file("test/output/style_sourced_shape_annotation.png", renderPNG(map));
+    checkRendering(map, "style_sourced_shape_annotation");
 }
 
 TEST(Annotations, AddMultiple) {
@@ -96,11 +101,11 @@ TEST(Annotations, AddMultiple) {
     map.setStyleJSON(util::read_file("test/fixtures/api/empty.json"), "");
     map.addPointAnnotation(PointAnnotation({ 0, -20 }, "default_marker"));
 
-    renderPNG(map);
+    render(map);
 
     map.addPointAnnotation(PointAnnotation({ 0, 20 }, "default_marker"));
 
-    util::write_file("test/output/add_multiple.png", renderPNG(map));
+    checkRendering(map, "add_multiple");
 }
 
 TEST(Annotations, NonImmediateAdd) {
@@ -111,7 +116,7 @@ TEST(Annotations, NonImmediateAdd) {
     Map map(view, fileSource, MapMode::Still);
     map.setStyleJSON(util::read_file("test/fixtures/api/empty.json"), "");
 
-    renderPNG(map);
+    render(map);
 
     AnnotationSegments segments = {{ {{ { 0, 0 }, { 0, 45 }, { 45, 45 }, { 45, 0 } }} }};
 
@@ -120,7 +125,7 @@ TEST(Annotations, NonImmediateAdd) {
 
     map.addShapeAnnotation(ShapeAnnotation(segments, properties));
 
-    util::write_file("test/output/non_immediate_add.png", renderPNG(map));
+    checkRendering(map, "non_immediate_add");
 }
 
 TEST(Annotations, RemovePoint) {
@@ -132,11 +137,11 @@ TEST(Annotations, RemovePoint) {
     map.setStyleJSON(util::read_file("test/fixtures/api/empty.json"), "");
     uint32_t point = map.addPointAnnotation(PointAnnotation({ 0, 0 }, "default_marker"));
 
-    renderPNG(map);
+    render(map);
 
     map.removeAnnotation(point);
 
-    util::write_file("test/output/remove_point.png", renderPNG(map));
+    checkRendering(map, "remove_point");
 }
 
 TEST(Annotations, RemoveShape) {
@@ -154,11 +159,11 @@ TEST(Annotations, RemoveShape) {
     map.setStyleJSON(util::read_file("test/fixtures/api/empty.json"), "");
     uint32_t shape = map.addShapeAnnotation(ShapeAnnotation(segments, properties));
 
-    renderPNG(map);
+    render(map);
 
     map.removeAnnotation(shape);
 
-    util::write_file("test/output/remove_shape.png", renderPNG(map));
+    checkRendering(map, "remove_shape");
 }
 
 TEST(Annotations, ImmediateRemoveShape) {
@@ -170,7 +175,7 @@ TEST(Annotations, ImmediateRemoveShape) {
     map.removeAnnotation(map.addShapeAnnotation(ShapeAnnotation({}, {})));
     map.setStyleJSON(util::read_file("test/fixtures/api/empty.json"), "");
 
-    renderPNG(map);
+    render(map);
 }
 
 TEST(Annotations, SwitchStyle) {
@@ -180,13 +185,13 @@ TEST(Annotations, SwitchStyle) {
 
     Map map(view, fileSource, MapMode::Still);
     map.setStyleJSON(util::read_file("test/fixtures/api/empty.json"), "");
-    map.addPointAnnotation(PointAnnotation({ 0, 0 }, "default_marker"));
+    map.addPointAnnotation(PointAnnotation({ 0, -20 }, "default_marker"));
 
-    renderPNG(map);
+    render(map);
 
     map.setStyleJSON(util::read_file("test/fixtures/api/empty.json"), "");
 
-    util::write_file("test/output/switch_style.png", renderPNG(map));
+    checkRendering(map, "switch_style");
 }
 
 TEST(Annotations, CustomIcon) {
@@ -199,5 +204,5 @@ TEST(Annotations, CustomIcon) {
     map.setSprite("cafe", std::make_shared<SpriteImage>(12, 12, 1, std::string(12 * 12 * 4, '\xFF')));
     map.addPointAnnotation(PointAnnotation({ 0, 0 }, "cafe"));
 
-    util::write_file("test/output/custom_icon.png", renderPNG(map));
+    checkRendering(map, "custom_icon");
 }
diff --git a/test/fixtures/annotations/add_multiple/expected.png b/test/fixtures/annotations/add_multiple/expected.png
new file mode 100644
index 0000000000..deedf84330
--- /dev/null
+++ b/test/fixtures/annotations/add_multiple/expected.png
diff --git a/test/fixtures/annotations/custom_icon/expected.png b/test/fixtures/annotations/custom_icon/expected.png
new file mode 100644
index 0000000000..924e4df403
--- /dev/null
+++ b/test/fixtures/annotations/custom_icon/expected.png
diff --git a/test/fixtures/annotations/fill_annotation/expected.png b/test/fixtures/annotations/fill_annotation/expected.png
new file mode 100644
index 0000000000..30f6f1eb59
--- /dev/null
+++ b/test/fixtures/annotations/fill_annotation/expected.png
diff --git a/test/fixtures/annotations/line_annotation/expected.png b/test/fixtures/annotations/line_annotation/expected.png
new file mode 100644
index 0000000000..067027f1d1
--- /dev/null
+++ b/test/fixtures/annotations/line_annotation/expected.png
diff --git a/test/fixtures/annotations/non_immediate_add/expected.png b/test/fixtures/annotations/non_immediate_add/expected.png
new file mode 100644
index 0000000000..30f6f1eb59
--- /dev/null
+++ b/test/fixtures/annotations/non_immediate_add/expected.png
diff --git a/test/fixtures/annotations/point_annotation/expected.png b/test/fixtures/annotations/point_annotation/expected.png
new file mode 100644
index 0000000000..33299a2d6a
--- /dev/null
+++ b/test/fixtures/annotations/point_annotation/expected.png
diff --git a/test/fixtures/annotations/remove_point/expected.png b/test/fixtures/annotations/remove_point/expected.png
new file mode 100644
index 0000000000..04f8682f88
--- /dev/null
+++ b/test/fixtures/annotations/remove_point/expected.png
diff --git a/test/fixtures/annotations/remove_shape/expected.png b/test/fixtures/annotations/remove_shape/expected.png
new file mode 100644
index 0000000000..04f8682f88
--- /dev/null
+++ b/test/fixtures/annotations/remove_shape/expected.png
diff --git a/test/fixtures/annotations/style_sourced_shape_annotation/expected.png b/test/fixtures/annotations/style_sourced_shape_annotation/expected.png
new file mode 100644
index 0000000000..09f48081a8
--- /dev/null
+++ b/test/fixtures/annotations/style_sourced_shape_annotation/expected.png
diff --git a/test/fixtures/annotations/switch_style/expected.png b/test/fixtures/annotations/switch_style/expected.png
new file mode 100644
index 0000000000..33299a2d6a
--- /dev/null
+++ b/test/fixtures/annotations/switch_style/expected.png
diff --git a/test/fixtures/util.cpp b/test/fixtures/util.cpp
index aad349d668..aa6371a4c7 100644
--- a/test/fixtures/util.cpp
+++ b/test/fixtures/util.cpp
@@ -1,6 +1,10 @@
 #include "util.hpp"
 
 #include <mbgl/platform/log.hpp>
+#include <mbgl/util/image.hpp>
+#include <mbgl/util/io.hpp>
+
+#include <mapbox/pixelmatch.hpp>
 
 #pragma GCC diagnostic push
 #pragma GCC diagnostic ignored "-Wshadow"
@@ -81,5 +85,61 @@ uint64_t crc64(const std::string& str) {
     return crc64(str.data(), str.size());
 }
 
+UnassociatedImage unpremultiply(const PremultipliedImage& src) {
+    UnassociatedImage dst { src.width, src.height };
+    std::copy(src.data.get(), src.data.get() + src.size(), dst.data.get());
+
+    uint8_t* data = dst.data.get();
+    for (size_t i = 0; i < dst.size(); i += 4) {
+        uint8_t& r = data[i + 0];
+        uint8_t& g = data[i + 1];
+        uint8_t& b = data[i + 2];
+        uint8_t& a = data[i + 3];
+        if (a) {
+            r = (255 * r + (a / 2)) / a;
+            g = (255 * g + (a / 2)) / a;
+            b = (255 * b + (a / 2)) / a;
+        }
+    }
+
+    return std::move(dst);
+}
+
+void checkImage(const std::string& base,
+                const UnassociatedImage& actual,
+                double imageThreshold,
+                double pixelThreshold) {
+    // TODO: the pixels produced by Map::renderStill are probably actually premultiplied,
+    // but Map::renderStill produces an UnassociatedImage. This probably should be fixed;
+    // here we just hack around it by copying the pixels to a PremultipliedImage (and
+    // un-premultiplying them when updating expected.png, since encodePNG wants
+    // un-premultiplied pixels).
+    PremultipliedImage actualActual { actual.width, actual.height };
+    std::copy(actual.data.get(), actual.data.get() + actual.size(), actualActual.data.get());
+
+    if (getenv("UPDATE")) {
+        util::write_file(base + "/expected.png", encodePNG(unpremultiply(actualActual)));
+        return;
+    }
+
+    PremultipliedImage expected = decodeImage(util::read_file(base + "/expected.png"));
+    UnassociatedImage diff { expected.width, expected.height };
+
+    ASSERT_EQ(expected.width, actual.width);
+    ASSERT_EQ(expected.height, actual.height);
+
+    double pixels = mapbox::pixelmatch(actual.data.get(),
+                                       expected.data.get(),
+                                       expected.width,
+                                       expected.height,
+                                       diff.data.get(),
+                                       pixelThreshold);
+
+    EXPECT_LE(pixels / (expected.width * expected.height), imageThreshold);
+
+    util::write_file(base + "/actual.png", encodePNG(actual));
+    util::write_file(base + "/diff.png", encodePNG(diff));
+}
+
 }
 }
diff --git a/test/fixtures/util.hpp b/test/fixtures/util.hpp
index fbee03c2d1..b369979e9c 100644
--- a/test/fixtures/util.hpp
+++ b/test/fixtures/util.hpp
@@ -1,6 +1,10 @@
 #ifndef MBGL_TEST_UTIL
 #define MBGL_TEST_UTIL
 
+#include <mbgl/util/image.hpp>
+
+#include <cstdint>
+
 #include <gtest/gtest.h>
 
 #define SCOPED_TEST(name) \
@@ -20,6 +24,11 @@ void stopServer(pid_t pid);
 uint64_t crc64(const char*, size_t);
 uint64_t crc64(const std::string&);
 
+void checkImage(const std::string& base,
+                const UnassociatedImage& actual,
+                double imageThreshold = 0,
+                double pixelThreshold = 0);
+
 }
 }
 
diff --git a/test/output/.gitkeep b/test/output/.gitkeep
deleted file mode 100644
index e69de29bb2..0000000000
--- a/test/output/.gitkeep
+++ /dev/null
diff --git a/test/test.gypi b/test/test.gypi
index 9530d5d498..c96b4f686c 100644
--- a/test/test.gypi
+++ b/test/test.gypi
@@ -110,6 +110,7 @@
           '<@(geojsonvt_cflags)',
           '<@(variant_cflags)',
           '<@(rapidjson_cflags)',
+          '<@(pixelmatch_cflags)',
         ],
         'ldflags': [
           '<@(gtest_ldflags)',