diff options
20 files changed, 116 insertions, 21 deletions
diff --git a/.gitignore b/.gitignore index 21a7079faa..d5fb118e90 100644 --- a/.gitignore +++ b/.gitignore @@ -25,6 +25,8 @@ /test/fixtures/api/1.png /test/fixtures/api/2.png /test/fixtures/database/*.db +/test/fixtures/*/*/actual.png +/test/fixtures/*/*/diff.png /test/output /include/mbgl/shader/shaders.hpp /src/shader/shaders_gl.cpp @@ -104,6 +104,7 @@ print_flags geojsonvt static_libs cflags ldflags print_flags variant static_libs cflags ldflags print_flags rapidjson static_libs cflags ldflags print_flags gtest static_libs cflags ldflags +print_flags pixelmatch static_libs cflags ldflags CONFIG+=" } } diff --git a/scripts/linux/after_script.sh b/scripts/linux/after_script.sh new file mode 100755 index 0000000000..f8e4bc3486 --- /dev/null +++ b/scripts/linux/after_script.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +set -e +set -o pipefail + +if [ ! -z "${AWS_ACCESS_KEY_ID}" ] && [ ! -z "${AWS_SECRET_ACCESS_KEY}" ] ; then + # Install and add awscli to PATH for uploading the results + pip install --user awscli + export PATH="`python -m site --user-base`/bin:${PATH}" + + REPO_NAME=$(basename $TRAVIS_REPO_SLUG) + + aws s3 cp --recursive --acl public-read test/fixtures/annotations \ + s3://mapbox/$REPO_NAME/render-tests/$TRAVIS_JOB_NUMBER/annotations +fi diff --git a/scripts/linux/configure.sh b/scripts/linux/configure.sh index f270c903ee..6a88c13c01 100644 --- a/scripts/linux/configure.sh +++ b/scripts/linux/configure.sh @@ -15,6 +15,7 @@ GEOJSONVT_VERSION=2.1.6.3 VARIANT_VERSION=1.0 RAPIDJSON_VERSION=1.0.2 GTEST_VERSION=1.7.0 +PIXELMATCH_VERSION=0.9.0 function print_opengl_flags { CONFIG+=" 'opengl_cflags%': $(quote_flags $(pkg-config gl x11 --cflags)),"$LN diff --git a/scripts/osx/configure.sh b/scripts/osx/configure.sh index 34024a40f4..823e3b6260 100644 --- a/scripts/osx/configure.sh +++ b/scripts/osx/configure.sh @@ -15,3 +15,4 @@ GEOJSONVT_VERSION=2.1.6.3 VARIANT_VERSION=1.0 RAPIDJSON_VERSION=1.0.2 GTEST_VERSION=1.7.0 +PIXELMATCH_VERSION=0.9.0 diff --git a/test/api/annotations.cpp b/test/api/annotations.cpp index b16ec71ef6..c4fea02079 100644 --- a/test/api/annotations.cpp +++ b/test/api/annotations.cpp @@ -7,7 +7,6 @@ #include <mbgl/platform/default/headless_display.hpp> #include <mbgl/platform/default/headless_view.hpp> #include <mbgl/storage/default_file_source.hpp> -#include <mbgl/util/image.hpp> #include <mbgl/util/io.hpp> #include <future> @@ -15,12 +14,18 @@ using namespace mbgl; -std::string renderPNG(Map& map) { +UnassociatedImage render(Map& map) { std::promise<UnassociatedImage> promise; map.renderStill([&](std::exception_ptr, UnassociatedImage&& image) { promise.set_value(std::move(image)); }); - return encodePNG(promise.get_future().get()); + return std::move(promise.get_future().get()); +} + +void checkRendering(Map& map, const char * name) { + UnassociatedImage actual = render(map); + test::checkImage(std::string("test/fixtures/annotations/") + name + "/", + actual, 0.0002, 0.1); } TEST(Annotations, PointAnnotation) { @@ -30,9 +35,9 @@ TEST(Annotations, PointAnnotation) { Map map(view, fileSource, MapMode::Still); map.setStyleJSON(util::read_file("test/fixtures/api/empty.json"), ""); - map.addPointAnnotation(PointAnnotation({ 0, 0 }, "default_marker")); + map.addPointAnnotation(PointAnnotation({ 0, -20 }, "default_marker")); - util::write_file("test/output/point_annotation.png", renderPNG(map)); + checkRendering(map, "point_annotation"); } TEST(Annotations, LineAnnotation) { @@ -51,7 +56,7 @@ TEST(Annotations, LineAnnotation) { map.addShapeAnnotation(ShapeAnnotation(segments, properties)); - util::write_file("test/output/line_annotation.png", renderPNG(map)); + checkRendering(map, "line_annotation"); } TEST(Annotations, FillAnnotation) { @@ -69,7 +74,7 @@ TEST(Annotations, FillAnnotation) { map.addShapeAnnotation(ShapeAnnotation(segments, properties)); - util::write_file("test/output/fill_annotation.png", renderPNG(map)); + checkRendering(map, "fill_annotation"); } TEST(Annotations, StyleSourcedShapeAnnotation) { @@ -84,7 +89,7 @@ TEST(Annotations, StyleSourcedShapeAnnotation) { map.addShapeAnnotation(ShapeAnnotation(segments, "annotation")); - util::write_file("test/output/style_sourced_shape_annotation.png", renderPNG(map)); + checkRendering(map, "style_sourced_shape_annotation"); } TEST(Annotations, AddMultiple) { @@ -96,11 +101,11 @@ TEST(Annotations, AddMultiple) { map.setStyleJSON(util::read_file("test/fixtures/api/empty.json"), ""); map.addPointAnnotation(PointAnnotation({ 0, -20 }, "default_marker")); - renderPNG(map); + render(map); map.addPointAnnotation(PointAnnotation({ 0, 20 }, "default_marker")); - util::write_file("test/output/add_multiple.png", renderPNG(map)); + checkRendering(map, "add_multiple"); } TEST(Annotations, NonImmediateAdd) { @@ -111,7 +116,7 @@ TEST(Annotations, NonImmediateAdd) { Map map(view, fileSource, MapMode::Still); map.setStyleJSON(util::read_file("test/fixtures/api/empty.json"), ""); - renderPNG(map); + render(map); AnnotationSegments segments = {{ {{ { 0, 0 }, { 0, 45 }, { 45, 45 }, { 45, 0 } }} }}; @@ -120,7 +125,7 @@ TEST(Annotations, NonImmediateAdd) { map.addShapeAnnotation(ShapeAnnotation(segments, properties)); - util::write_file("test/output/non_immediate_add.png", renderPNG(map)); + checkRendering(map, "non_immediate_add"); } TEST(Annotations, RemovePoint) { @@ -132,11 +137,11 @@ TEST(Annotations, RemovePoint) { map.setStyleJSON(util::read_file("test/fixtures/api/empty.json"), ""); uint32_t point = map.addPointAnnotation(PointAnnotation({ 0, 0 }, "default_marker")); - renderPNG(map); + render(map); map.removeAnnotation(point); - util::write_file("test/output/remove_point.png", renderPNG(map)); + checkRendering(map, "remove_point"); } TEST(Annotations, RemoveShape) { @@ -154,11 +159,11 @@ TEST(Annotations, RemoveShape) { map.setStyleJSON(util::read_file("test/fixtures/api/empty.json"), ""); uint32_t shape = map.addShapeAnnotation(ShapeAnnotation(segments, properties)); - renderPNG(map); + render(map); map.removeAnnotation(shape); - util::write_file("test/output/remove_shape.png", renderPNG(map)); + checkRendering(map, "remove_shape"); } TEST(Annotations, ImmediateRemoveShape) { @@ -170,7 +175,7 @@ TEST(Annotations, ImmediateRemoveShape) { map.removeAnnotation(map.addShapeAnnotation(ShapeAnnotation({}, {}))); map.setStyleJSON(util::read_file("test/fixtures/api/empty.json"), ""); - renderPNG(map); + render(map); } TEST(Annotations, SwitchStyle) { @@ -180,13 +185,13 @@ TEST(Annotations, SwitchStyle) { Map map(view, fileSource, MapMode::Still); map.setStyleJSON(util::read_file("test/fixtures/api/empty.json"), ""); - map.addPointAnnotation(PointAnnotation({ 0, 0 }, "default_marker")); + map.addPointAnnotation(PointAnnotation({ 0, -20 }, "default_marker")); - renderPNG(map); + render(map); map.setStyleJSON(util::read_file("test/fixtures/api/empty.json"), ""); - util::write_file("test/output/switch_style.png", renderPNG(map)); + checkRendering(map, "switch_style"); } TEST(Annotations, CustomIcon) { @@ -199,5 +204,5 @@ TEST(Annotations, CustomIcon) { map.setSprite("cafe", std::make_shared<SpriteImage>(12, 12, 1, std::string(12 * 12 * 4, '\xFF'))); map.addPointAnnotation(PointAnnotation({ 0, 0 }, "cafe")); - util::write_file("test/output/custom_icon.png", renderPNG(map)); + checkRendering(map, "custom_icon"); } diff --git a/test/fixtures/annotations/add_multiple/expected.png b/test/fixtures/annotations/add_multiple/expected.png Binary files differnew file mode 100644 index 0000000000..deedf84330 --- /dev/null +++ b/test/fixtures/annotations/add_multiple/expected.png diff --git a/test/fixtures/annotations/custom_icon/expected.png b/test/fixtures/annotations/custom_icon/expected.png Binary files differnew file mode 100644 index 0000000000..924e4df403 --- /dev/null +++ b/test/fixtures/annotations/custom_icon/expected.png diff --git a/test/fixtures/annotations/fill_annotation/expected.png b/test/fixtures/annotations/fill_annotation/expected.png Binary files differnew file mode 100644 index 0000000000..30f6f1eb59 --- /dev/null +++ b/test/fixtures/annotations/fill_annotation/expected.png diff --git a/test/fixtures/annotations/line_annotation/expected.png b/test/fixtures/annotations/line_annotation/expected.png Binary files differnew file mode 100644 index 0000000000..067027f1d1 --- /dev/null +++ b/test/fixtures/annotations/line_annotation/expected.png diff --git a/test/fixtures/annotations/non_immediate_add/expected.png b/test/fixtures/annotations/non_immediate_add/expected.png Binary files differnew file mode 100644 index 0000000000..30f6f1eb59 --- /dev/null +++ b/test/fixtures/annotations/non_immediate_add/expected.png diff --git a/test/fixtures/annotations/point_annotation/expected.png b/test/fixtures/annotations/point_annotation/expected.png Binary files differnew file mode 100644 index 0000000000..33299a2d6a --- /dev/null +++ b/test/fixtures/annotations/point_annotation/expected.png diff --git a/test/fixtures/annotations/remove_point/expected.png b/test/fixtures/annotations/remove_point/expected.png Binary files differnew file mode 100644 index 0000000000..04f8682f88 --- /dev/null +++ b/test/fixtures/annotations/remove_point/expected.png diff --git a/test/fixtures/annotations/remove_shape/expected.png b/test/fixtures/annotations/remove_shape/expected.png Binary files differnew file mode 100644 index 0000000000..04f8682f88 --- /dev/null +++ b/test/fixtures/annotations/remove_shape/expected.png diff --git a/test/fixtures/annotations/style_sourced_shape_annotation/expected.png b/test/fixtures/annotations/style_sourced_shape_annotation/expected.png Binary files differnew file mode 100644 index 0000000000..09f48081a8 --- /dev/null +++ b/test/fixtures/annotations/style_sourced_shape_annotation/expected.png diff --git a/test/fixtures/annotations/switch_style/expected.png b/test/fixtures/annotations/switch_style/expected.png Binary files differnew file mode 100644 index 0000000000..33299a2d6a --- /dev/null +++ b/test/fixtures/annotations/switch_style/expected.png diff --git a/test/fixtures/util.cpp b/test/fixtures/util.cpp index aad349d668..aa6371a4c7 100644 --- a/test/fixtures/util.cpp +++ b/test/fixtures/util.cpp @@ -1,6 +1,10 @@ #include "util.hpp" #include <mbgl/platform/log.hpp> +#include <mbgl/util/image.hpp> +#include <mbgl/util/io.hpp> + +#include <mapbox/pixelmatch.hpp> #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wshadow" @@ -81,5 +85,61 @@ uint64_t crc64(const std::string& str) { return crc64(str.data(), str.size()); } +UnassociatedImage unpremultiply(const PremultipliedImage& src) { + UnassociatedImage dst { src.width, src.height }; + std::copy(src.data.get(), src.data.get() + src.size(), dst.data.get()); + + uint8_t* data = dst.data.get(); + for (size_t i = 0; i < dst.size(); i += 4) { + uint8_t& r = data[i + 0]; + uint8_t& g = data[i + 1]; + uint8_t& b = data[i + 2]; + uint8_t& a = data[i + 3]; + if (a) { + r = (255 * r + (a / 2)) / a; + g = (255 * g + (a / 2)) / a; + b = (255 * b + (a / 2)) / a; + } + } + + return std::move(dst); +} + +void checkImage(const std::string& base, + const UnassociatedImage& actual, + double imageThreshold, + double pixelThreshold) { + // TODO: the pixels produced by Map::renderStill are probably actually premultiplied, + // but Map::renderStill produces an UnassociatedImage. This probably should be fixed; + // here we just hack around it by copying the pixels to a PremultipliedImage (and + // un-premultiplying them when updating expected.png, since encodePNG wants + // un-premultiplied pixels). + PremultipliedImage actualActual { actual.width, actual.height }; + std::copy(actual.data.get(), actual.data.get() + actual.size(), actualActual.data.get()); + + if (getenv("UPDATE")) { + util::write_file(base + "/expected.png", encodePNG(unpremultiply(actualActual))); + return; + } + + PremultipliedImage expected = decodeImage(util::read_file(base + "/expected.png")); + UnassociatedImage diff { expected.width, expected.height }; + + ASSERT_EQ(expected.width, actual.width); + ASSERT_EQ(expected.height, actual.height); + + double pixels = mapbox::pixelmatch(actual.data.get(), + expected.data.get(), + expected.width, + expected.height, + diff.data.get(), + pixelThreshold); + + EXPECT_LE(pixels / (expected.width * expected.height), imageThreshold); + + util::write_file(base + "/actual.png", encodePNG(actual)); + util::write_file(base + "/diff.png", encodePNG(diff)); +} + } } diff --git a/test/fixtures/util.hpp b/test/fixtures/util.hpp index fbee03c2d1..b369979e9c 100644 --- a/test/fixtures/util.hpp +++ b/test/fixtures/util.hpp @@ -1,6 +1,10 @@ #ifndef MBGL_TEST_UTIL #define MBGL_TEST_UTIL +#include <mbgl/util/image.hpp> + +#include <cstdint> + #include <gtest/gtest.h> #define SCOPED_TEST(name) \ @@ -20,6 +24,11 @@ void stopServer(pid_t pid); uint64_t crc64(const char*, size_t); uint64_t crc64(const std::string&); +void checkImage(const std::string& base, + const UnassociatedImage& actual, + double imageThreshold = 0, + double pixelThreshold = 0); + } } diff --git a/test/output/.gitkeep b/test/output/.gitkeep deleted file mode 100644 index e69de29bb2..0000000000 --- a/test/output/.gitkeep +++ /dev/null diff --git a/test/test.gypi b/test/test.gypi index 9530d5d498..c96b4f686c 100644 --- a/test/test.gypi +++ b/test/test.gypi @@ -110,6 +110,7 @@ '<@(geojsonvt_cflags)', '<@(variant_cflags)', '<@(rapidjson_cflags)', + '<@(pixelmatch_cflags)', ], 'ldflags': [ '<@(gtest_ldflags)', |
